[ceph.git] / ceph / src / spdk / scripts / setup.sh

#!/usr/bin/env bash

set -e

rootdir=$(readlink -f $(dirname $0))/..
source "$rootdir/scripts/common.sh"

function usage()
{
	if [ `uname` = Linux ]; then
		options="[config|reset|status|cleanup|help]"
	else
		options="[config|reset|help]"
	fi

	[[ ! -z $2 ]] && ( echo "$2"; echo ""; )
	echo "Helper script for allocating hugepages and binding NVMe, I/OAT and Virtio devices to"
	echo "a generic VFIO kernel driver. If VFIO is not available on the system, this script will"
	echo "fall back to UIO. NVMe and Virtio devices with active mountpoints will be ignored."
	echo "All hugepage operations use default hugepage size on the system (hugepagesz)."
	echo "Usage: $(basename $1) $options"
	echo
	echo "$options - as following:"
	echo "config            Default mode. Allocate hugepages and bind PCI devices."
	if [ `uname` = Linux ]; then
		echo "cleanup            Remove any orphaned files that can be left in the system after SPDK application exit"
	fi
	echo "reset             Rebind PCI devices back to their original drivers."
	echo "                  Also cleanup any leftover spdk files/resources."
	echo "                  Hugepage memory size will remain unchanged."
	if [ `uname` = Linux ]; then
		echo "status            Print status of all SPDK-compatible devices on the system."
	fi
	echo "help              Print this help message."
	echo
	echo "The following environment variables can be specified."
	echo "HUGEMEM           Size of hugepage memory to allocate (in MB). 2048 by default."
	echo "                  For NUMA systems, the hugepages will be evenly distributed"
	echo "                  between CPU nodes"
	echo "NRHUGE            Number of hugepages to allocate. This variable overwrites HUGEMEM."
	echo "HUGENODE          Specific NUMA node to allocate hugepages on. To allocate"
	echo "                  hugepages on multiple nodes run this script multiple times -"
	echo "                  once for each node."
	echo "PCI_WHITELIST     Whitespace separated list of PCI devices (NVMe, I/OAT, Virtio) to bind."
	echo "                  Each device must be specified as a full PCI address."
	echo "                  E.g. PCI_WHITELIST=\"0000:01:00.0 0000:02:00.0\""
	echo "                  To blacklist all PCI devices use a non-valid address."
	echo "                  E.g. PCI_WHITELIST=\"none\""
	echo "                  If empty or unset, all PCI devices will be bound."
	echo "TARGET_USER       User that will own hugepage mountpoint directory and vfio groups."
	echo "                  By default the current user will be used."
	exit 0
}

# In monolithic kernels the lsmod won't work. So
# back that with a /sys/modules check. Return a different code for
# built-in vs module just in case we want that down the road.
function check_for_driver {
	$(lsmod | grep $1 > /dev/null)
	if [ $? -eq 0 ]; then
		return 1
	else
		if [[ -d /sys/module/$1 ]]; then
			return 2
		else
			return 0
		fi
	fi
	return 0
}

function pci_can_bind() {
	if [[ ${#PCI_WHITELIST[@]} == 0 ]]; then
		#no whitelist specified, bind all devices
		return 1
	fi

	for i in ${PCI_WHITELIST[@]}
	do
		if [ "$i" == "$1" ] ; then
			 return 1
		fi
	done
	return 0
}

function linux_bind_driver() {
	bdf="$1"
	driver_name="$2"
	old_driver_name="no driver"
	ven_dev_id=$(lspci -n -s $bdf | cut -d' ' -f3 | sed 's/:/ /')

	if [ -e "/sys/bus/pci/devices/$bdf/driver" ]; then
		old_driver_name=$(basename $(readlink /sys/bus/pci/devices/$bdf/driver))

		if [ "$driver_name" = "$old_driver_name" ]; then
			return 0
		fi

		echo "$ven_dev_id" > "/sys/bus/pci/devices/$bdf/driver/remove_id" 2> /dev/null || true
		echo "$bdf" > "/sys/bus/pci/devices/$bdf/driver/unbind"
	fi

	echo "$bdf ($ven_dev_id): $old_driver_name -> $driver_name"

	echo "$ven_dev_id" > "/sys/bus/pci/drivers/$driver_name/new_id" 2> /dev/null || true
	echo "$bdf" > "/sys/bus/pci/drivers/$driver_name/bind" 2> /dev/null || true

	iommu_group=$(basename $(readlink -f /sys/bus/pci/devices/$bdf/iommu_group))
	if [ -e "/dev/vfio/$iommu_group" ]; then
		if [ -n "$TARGET_USER" ]; then
			chown "$TARGET_USER" "/dev/vfio/$iommu_group"
		fi
	fi
}

function linux_unbind_driver() {
	bdf="$1"
	ven_dev_id=$(lspci -n -s $bdf | cut -d' ' -f3 | sed 's/:/ /')

	if ! [ -e "/sys/bus/pci/devices/$bdf/driver" ]; then
		return 0
	fi

	old_driver_name=$(basename $(readlink /sys/bus/pci/devices/$bdf/driver))

	echo "$ven_dev_id" > "/sys/bus/pci/devices/$bdf/driver/remove_id" 2> /dev/null || true
	echo "$bdf" > "/sys/bus/pci/devices/$bdf/driver/unbind"
	echo "$bdf ($ven_dev_id): $old_driver_name -> no driver"
}

function linux_hugetlbfs_mounts() {
	mount | grep ' type hugetlbfs ' | awk '{ print $3 }'
}

function get_nvme_name_from_bdf {
	set +e
	nvme_devs=`lsblk -d --output NAME | grep "^nvme"`
	set -e
	for dev in $nvme_devs; do
		link_name=$(readlink /sys/block/$dev/device/device) || true
		if [ -z "$link_name" ]; then
			link_name=$(readlink /sys/block/$dev/device)
		fi
		link_bdf=$(basename "$link_name")
		if [ "$link_bdf" = "$1" ]; then
			eval "$2=$dev"
			return
		fi
	done
}

function get_virtio_names_from_bdf {
	blk_devs=`lsblk --nodeps --output NAME`
	virtio_names=''

	for dev in $blk_devs; do
		if readlink "/sys/block/$dev" | grep -q "$1"; then
			virtio_names="$virtio_names $dev"
		fi
	done

	eval "$2='$virtio_names'"
}

function configure_linux_pci {
	driver_name=vfio-pci
	if [ -z "$(ls /sys/kernel/iommu_groups)" ]; then
		# No IOMMU. Use uio.
		driver_name=uio_pci_generic
	fi

	# NVMe
	modprobe $driver_name || true
	for bdf in $(iter_pci_class_code 01 08 02); do
		blkname=''
		get_nvme_name_from_bdf "$bdf" blkname
		if pci_can_bind $bdf == "0" ; then
			echo "Skipping un-whitelisted NVMe controller $blkname ($bdf)"
			continue
		fi
		if [ "$blkname" != "" ]; then
			mountpoints=$(lsblk /dev/$blkname --output MOUNTPOINT -n | wc -w)
		else
			mountpoints="0"
		fi
		if [ "$mountpoints" = "0" ]; then
			linux_bind_driver "$bdf" "$driver_name"
		else
			echo Active mountpoints on /dev/$blkname, so not binding PCI dev $bdf
		fi
	done

	# IOAT
	TMP=`mktemp`
	#collect all the device_id info of ioat devices.
	grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/include/spdk/pci_ids.h \
	| awk -F"x" '{print $2}' > $TMP

	for dev_id in `cat $TMP`; do
		for bdf in $(iter_pci_dev_id 8086 $dev_id); do
			if pci_can_bind $bdf == "0" ; then
				echo "Skipping un-whitelisted I/OAT device at $bdf"
				continue
			fi
			linux_bind_driver "$bdf" "$driver_name"
		done
	done
	rm $TMP

	# virtio
	TMP=`mktemp`
	#collect all the device_id info of virtio devices.
	grep "PCI_DEVICE_ID_VIRTIO" $rootdir/include/spdk/pci_ids.h \
	| awk -F"x" '{print $2}' > $TMP

	for dev_id in `cat $TMP`; do
		for bdf in $(iter_pci_dev_id 1af4 $dev_id); do
			if pci_can_bind $bdf == "0" ; then
				echo "Skipping un-whitelisted Virtio device at $bdf"
				continue
			fi
			blknames=''
			get_virtio_names_from_bdf "$bdf" blknames
			for blkname in $blknames; do
				if mount | grep -q "/dev/$blkname"; then
					echo Active mountpoints on /dev/$blkname, so not binding PCI dev $bdf
					continue 2
				fi
			done

			linux_bind_driver "$bdf" "$driver_name"
		done
	done
	rm $TMP

	echo "1" > "/sys/bus/pci/rescan"
}

function cleanup_linux {
	shopt -s extglob nullglob
	dirs_to_clean=""
	dirs_to_clean="$(echo {/var/run,/tmp}/dpdk/spdk{,_pid}+([0-9])) "
	if [[ -d $XDG_RUNTIME_DIR && $XDG_RUNTIME_DIR != *" "* ]]; then
		dirs_to_clean+="$(readlink -e assert_not_empty $XDG_RUNTIME_DIR/dpdk/spdk{,_pid}+([0-9]) || true) "
	fi

	files_to_clean=""
	for dir in $dirs_to_clean; do
		files_to_clean+="$(echo $dir/*) "
	done
	shopt -u extglob nullglob

	files_to_clean+="$(echo /dev/shm/* | egrep '(spdk_tgt|iscsi|vhost|nvmf|rocksdb|bdevtest|bdevperf)_trace|spdk_iscsi_conns' || true) "
	files_to_clean="$(readlink -e assert_not_empty $files_to_clean || true)"
	if [[ -z "$files_to_clean" ]]; then
		echo "Clean"
		return 0;
	fi

	shopt -s extglob
	for fd_dir in $(echo /proc/+([0-9])); do
		opened_files+="$(readlink -e assert_not_empty $fd_dir/fd/* || true)"
	done
	shopt -u extglob

	if [[ -z "$opened_files" ]]; then
		echo "Can't get list of opened files!"
		exit 1
	fi

	echo 'Cleaning'
	for f in $files_to_clean; do
		if ! echo "$opened_files" | egrep -q "^$f\$"; then
			echo "Removing:    $f"
			rm $f
		else
			echo "Still open: $f"
		fi
	done

	for dir in $dirs_to_clean; do
	if ! echo "$opened_files" | egrep -q "^$dir\$"; then
		echo "Removing:    $dir"
		rmdir $dir
	else
		echo "Still open: $dir"
	fi
	done
	echo "Clean"

	unset dirs_to_clean files_to_clean opened_files
}

function configure_linux {
	configure_linux_pci
	hugetlbfs_mounts=$(linux_hugetlbfs_mounts)

	if [ -z "$hugetlbfs_mounts" ]; then
		hugetlbfs_mounts=/mnt/huge
		echo "Mounting hugetlbfs at $hugetlbfs_mounts"
		mkdir -p "$hugetlbfs_mounts"
		mount -t hugetlbfs nodev "$hugetlbfs_mounts"
	fi

	if [ -z "$HUGENODE" ]; then
		hugepages_target="/proc/sys/vm/nr_hugepages"
	else
		hugepages_target="/sys/devices/system/node/node${HUGENODE}/hugepages/hugepages-${HUGEPGSZ}kB/nr_hugepages"
	fi

	echo "$NRHUGE" > "$hugepages_target"
	allocated_hugepages=`cat $hugepages_target`
	if [ "$allocated_hugepages" -lt "$NRHUGE" ]; then
		echo ""
		echo "## ERROR: requested $NRHUGE hugepages but only $allocated_hugepages could be allocated."
		echo "## Memory might be heavily fragmented. Please try flushing the system cache, or reboot the machine."
		exit 1
	fi

	if [ "$driver_name" = "vfio-pci" ]; then
		if [ -n "$TARGET_USER" ]; then
			for mount in $hugetlbfs_mounts; do
				chown "$TARGET_USER" "$mount"
				chmod g+w "$mount"
			done
		fi

		MEMLOCK_AMNT=`ulimit -l`
		if [ "$MEMLOCK_AMNT" != "unlimited" ] ; then
			MEMLOCK_MB=$(( $MEMLOCK_AMNT / 1024 ))
			echo ""
			echo "Current user memlock limit: ${MEMLOCK_MB} MB"
			echo ""
			echo "This is the maximum amount of memory you will be"
			echo "able to use with DPDK and VFIO if run as current user."
			echo -n "To change this, please adjust limits.conf memlock "
			echo "limit for current user."

			if [ $MEMLOCK_AMNT -lt 65536 ] ; then
				echo ""
				echo "## WARNING: memlock limit is less than 64MB"
				echo -n "## DPDK with VFIO may not be able to initialize "
				echo "if run as current user."
			fi
		fi
	fi
}

function reset_linux_pci {
	# NVMe
	set +e
	check_for_driver nvme
	driver_loaded=$?
	set -e
	for bdf in $(iter_pci_class_code 01 08 02); do
		if pci_can_bind $bdf == "0" ; then
			echo "Skipping un-whitelisted NVMe controller $blkname ($bdf)"
			continue
		fi
		if [ $driver_loaded -ne 0 ]; then
			linux_bind_driver "$bdf" nvme
		else
			linux_unbind_driver "$bdf"
		fi
	done

	# IOAT
	TMP=`mktemp`
	#collect all the device_id info of ioat devices.
	grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/include/spdk/pci_ids.h \
	| awk -F"x" '{print $2}' > $TMP

	set +e
	check_for_driver ioatdma
	driver_loaded=$?
	set -e
	for dev_id in `cat $TMP`; do
		for bdf in $(iter_pci_dev_id 8086 $dev_id); do
			if pci_can_bind $bdf == "0" ; then
				echo "Skipping un-whitelisted I/OAT device at $bdf"
				continue
			fi
			if [ $driver_loaded -ne 0 ]; then
				linux_bind_driver "$bdf" ioatdma
			else
				linux_unbind_driver "$bdf"
			fi
		done
	done
	rm $TMP

	# virtio
	TMP=`mktemp`
	#collect all the device_id info of virtio devices.
	grep "PCI_DEVICE_ID_VIRTIO" $rootdir/include/spdk/pci_ids.h \
	| awk -F"x" '{print $2}' > $TMP

	# TODO: check if virtio-pci is loaded first and just unbind if it is not loaded
	# Requires some more investigation - for example, some kernels do not seem to have
	#  virtio-pci but just virtio_scsi instead.  Also need to make sure we get the
	#  underscore vs. dash right in the virtio_scsi name.
	modprobe virtio-pci || true
	for dev_id in `cat $TMP`; do
		for bdf in $(iter_pci_dev_id 1af4 $dev_id); do
			if pci_can_bind $bdf == "0" ; then
				echo "Skipping un-whitelisted Virtio device at $bdf"
				continue
			fi
			linux_bind_driver "$bdf" virtio-pci
		done
	done
	rm $TMP

	echo "1" > "/sys/bus/pci/rescan"
}

function reset_linux {
	reset_linux_pci
	for mount in $(linux_hugetlbfs_mounts); do
		rm -f "$mount"/spdk*map_*
	done
	rm -f /run/.spdk*
}

function status_linux {
	echo "Hugepages"
	printf "%-6s %10s %8s / %6s\n" "node" "hugesize"  "free" "total"

	numa_nodes=0
	shopt -s nullglob
	for path in /sys/devices/system/node/node?/hugepages/hugepages-*/; do
		numa_nodes=$((numa_nodes + 1))
		free_pages=`cat $path/free_hugepages`
		all_pages=`cat $path/nr_hugepages`

		[[ $path =~ (node[0-9]+)/hugepages/hugepages-([0-9]+kB) ]]

		node=${BASH_REMATCH[1]}
		huge_size=${BASH_REMATCH[2]}

		printf "%-6s %10s %8s / %6s\n" $node $huge_size $free_pages $all_pages
	done
	shopt -u nullglob

	# fall back to system-wide hugepages
	if [ "$numa_nodes" = "0" ]; then
		free_pages=`grep HugePages_Free /proc/meminfo | awk '{ print $2 }'`
		all_pages=`grep HugePages_Total /proc/meminfo | awk '{ print $2 }'`
		node="-"
		huge_size="$HUGEPGSZ"

		printf "%-6s %10s %8s / %6s\n" $node $huge_size $free_pages $all_pages
	fi

	echo "NVMe devices"

	echo -e "BDF\t\tNuma Node\tDriver name\t\tDevice name"
	for bdf in $(iter_pci_class_code 01 08 02); do
		driver=`grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}'`
		node=`cat /sys/bus/pci/devices/$bdf/numa_node`;
		if [ "$driver" = "nvme" -a -d /sys/bus/pci/devices/$bdf/nvme ]; then
			name="\t"`ls /sys/bus/pci/devices/$bdf/nvme`;
		else
			name="-";
		fi
		echo -e "$bdf\t$node\t\t$driver\t\t$name";
	done

	echo "I/OAT DMA"

	#collect all the device_id info of ioat devices.
	TMP=`grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/include/spdk/pci_ids.h \
	| awk -F"x" '{print $2}'`
	echo -e "BDF\t\tNuma Node\tDriver Name"
	for dev_id in $TMP; do
		for bdf in $(iter_pci_dev_id 8086 $dev_id); do
			driver=`grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}'`
			node=`cat /sys/bus/pci/devices/$bdf/numa_node`;
			echo -e "$bdf\t$node\t\t$driver"
		done
	done

	echo "virtio"

	#collect all the device_id info of virtio devices.
	TMP=`grep "PCI_DEVICE_ID_VIRTIO" $rootdir/include/spdk/pci_ids.h \
	| awk -F"x" '{print $2}'`
	echo -e "BDF\t\tNuma Node\tDriver Name\t\tDevice Name"
	for dev_id in $TMP; do
		for bdf in $(iter_pci_dev_id 1af4 $dev_id); do
			driver=`grep DRIVER /sys/bus/pci/devices/$bdf/uevent |awk -F"=" '{print $2}'`
			node=`cat /sys/bus/pci/devices/$bdf/numa_node`;
			blknames=''
			get_virtio_names_from_bdf "$bdf" blknames
			echo -e "$bdf\t$node\t\t$driver\t\t$blknames"
		done
	done
}

function configure_freebsd_pci {
	TMP=`mktemp`

	# NVMe
	GREP_STR="class=0x010802"

	# IOAT
	grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/include/spdk/pci_ids.h \
	| awk -F"x" '{print $2}' > $TMP
	for dev_id in `cat $TMP`; do
		GREP_STR="${GREP_STR}\|chip=0x${dev_id}8086"
	done

	AWK_PROG="{if (count > 0) printf \",\"; printf \"%s:%s:%s\",\$2,\$3,\$4; count++}"
	echo $AWK_PROG > $TMP

	BDFS=`pciconf -l | grep "${GREP_STR}" | awk -F: -f $TMP`

	kldunload nic_uio.ko || true
	kenv hw.nic_uio.bdfs=$BDFS
	kldload nic_uio.ko
	rm $TMP
}

function configure_freebsd {
	configure_freebsd_pci
	# If contigmem is already loaded but the HUGEMEM specified doesn't match the
	#  previous value, unload contigmem so that we can reload with the new value.
	if kldstat -q -m contigmem; then
		if [ `kenv hw.contigmem.num_buffers` -ne "$((HUGEMEM / 256))" ]; then
			kldunload contigmem.ko
		fi
	fi
	if ! kldstat -q -m contigmem; then
		kenv hw.contigmem.num_buffers=$((HUGEMEM / 256))
		kenv hw.contigmem.buffer_size=$((256 * 1024 * 1024))
		kldload contigmem.ko
	fi
}

function reset_freebsd {
	kldunload contigmem.ko || true
	kldunload nic_uio.ko || true
}

mode=$1

if [ -z "$mode" ]; then
	mode="config"
fi

: ${HUGEMEM:=2048}
: ${PCI_WHITELIST:=""}

if [ -n "$NVME_WHITELIST" ]; then
	PCI_WHITELIST="$PCI_WHITELIST $NVME_WHITELIST"
fi

if [ -n "$SKIP_PCI" ]; then
	PCI_WHITELIST="none"
fi

declare -a PCI_WHITELIST=(${PCI_WHITELIST})

if [ -z "$TARGET_USER" ]; then
	TARGET_USER="$SUDO_USER"
	if [ -z "$TARGET_USER" ]; then
		TARGET_USER=`logname 2>/dev/null` || true
	fi
fi

if [ `uname` = Linux ]; then
	HUGEPGSZ=$(( `grep Hugepagesize /proc/meminfo | cut -d : -f 2 | tr -dc '0-9'` ))
	HUGEPGSZ_MB=$(( $HUGEPGSZ / 1024 ))
	: ${NRHUGE=$(( (HUGEMEM + HUGEPGSZ_MB - 1) / HUGEPGSZ_MB ))}

	if [ "$mode" == "config" ]; then
		configure_linux
	elif [ "$mode" == "cleanup" ]; then
		cleanup_linux
	elif [ "$mode" == "reset" ]; then
		reset_linux
	elif [ "$mode" == "status" ]; then
		status_linux
	elif [ "$mode" == "help" ]; then
		usage $0
	else
		usage $0 "Invalid argument '$mode'"
	fi
else
	if [ "$mode" == "config" ]; then
		configure_freebsd
	elif [ "$mode" == "reset" ]; then
		reset_freebsd
	elif [ "$mode" == "cleanup" ]; then
		echo "setup.sh cleanup function not yet supported on $(uname)"
	elif [ "$mode" == "status" ]; then
		echo "setup.sh status function not yet supported on $(uname)"
	elif [ "$mode" == "help" ]; then
		usage $0
	else
		usage $0 "Invalid argument '$mode'"
	fi
fi
Commit	Line	Data
7c673cae FG	1	#!/usr/bin/env bash
	2
	3	set -e
	4
	5	rootdir=$(readlink -f $(dirname $0))/..
11fdf7f2 TL	6	source "$rootdir/scripts/common.sh"
	7
	8	function usage()
	9	{
	10	if [ `uname` = Linux ]; then
	11	options="[config\|reset\|status\|cleanup\|help]"
	12	else
	13	options="[config\|reset\|help]"
	14	fi
	15
	16	[[ ! -z $2 ]] && ( echo "$2"; echo ""; )
	17	echo "Helper script for allocating hugepages and binding NVMe, I/OAT and Virtio devices to"
	18	echo "a generic VFIO kernel driver. If VFIO is not available on the system, this script will"
	19	echo "fall back to UIO. NVMe and Virtio devices with active mountpoints will be ignored."
	20	echo "All hugepage operations use default hugepage size on the system (hugepagesz)."
	21	echo "Usage: $(basename $1) $options"
	22	echo
	23	echo "$options - as following:"
	24	echo "config Default mode. Allocate hugepages and bind PCI devices."
	25	if [ `uname` = Linux ]; then
	26	echo "cleanup Remove any orphaned files that can be left in the system after SPDK application exit"
	27	fi
	28	echo "reset Rebind PCI devices back to their original drivers."
	29	echo " Also cleanup any leftover spdk files/resources."
	30	echo " Hugepage memory size will remain unchanged."
	31	if [ `uname` = Linux ]; then
	32	echo "status Print status of all SPDK-compatible devices on the system."
	33	fi
	34	echo "help Print this help message."
	35	echo
	36	echo "The following environment variables can be specified."
	37	echo "HUGEMEM Size of hugepage memory to allocate (in MB). 2048 by default."
	38	echo " For NUMA systems, the hugepages will be evenly distributed"
	39	echo " between CPU nodes"
	40	echo "NRHUGE Number of hugepages to allocate. This variable overwrites HUGEMEM."
	41	echo "HUGENODE Specific NUMA node to allocate hugepages on. To allocate"
	42	echo " hugepages on multiple nodes run this script multiple times -"
	43	echo " once for each node."
	44	echo "PCI_WHITELIST Whitespace separated list of PCI devices (NVMe, I/OAT, Virtio) to bind."
	45	echo " Each device must be specified as a full PCI address."
	46	echo " E.g. PCI_WHITELIST=\"0000:01:00.0 0000:02:00.0\""
	47	echo " To blacklist all PCI devices use a non-valid address."
	48	echo " E.g. PCI_WHITELIST=\"none\""
	49	echo " If empty or unset, all PCI devices will be bound."
	50	echo "TARGET_USER User that will own hugepage mountpoint directory and vfio groups."
	51	echo " By default the current user will be used."
	52	exit 0
	53	}
	54
	55	# In monolithic kernels the lsmod won't work. So
	56	# back that with a /sys/modules check. Return a different code for
	57	# built-in vs module just in case we want that down the road.
	58	function check_for_driver {
	59	$(lsmod \| grep $1 > /dev/null)
	60	if [ $? -eq 0 ]; then
	61	return 1
	62	else
	63	if [[ -d /sys/module/$1 ]]; then
	64	return 2
	65	else
	66	return 0
	67	fi
	68	fi
	69	return 0
70	}
71
72	function pci_can_bind() {
73	if [[ ${#PCI_WHITELIST[@]} == 0 ]]; then
74	#no whitelist specified, bind all devices
75	return 1
76	fi
7c673cae	77
11fdf7f2 TL	78	for i in ${PCI_WHITELIST[@]}
	79	do
	80	if [ "$i" == "$1" ] ; then
	81	return 1
	82	fi
	83	done
	84	return 0
7c673cae FG	85	}
	86
	87	function linux_bind_driver() {
	88	bdf="$1"
	89	driver_name="$2"
	90	old_driver_name="no driver"
	91	ven_dev_id=$(lspci -n -s $bdf \| cut -d' ' -f3 \| sed 's/:/ /')
	92
	93	if [ -e "/sys/bus/pci/devices/$bdf/driver" ]; then
	94	old_driver_name=$(basename $(readlink /sys/bus/pci/devices/$bdf/driver))
	95
	96	if [ "$driver_name" = "$old_driver_name" ]; then
	97	return 0
	98	fi
	99
	100	echo "$ven_dev_id" > "/sys/bus/pci/devices/$bdf/driver/remove_id" 2> /dev/null \|\| true
	101	echo "$bdf" > "/sys/bus/pci/devices/$bdf/driver/unbind"
	102	fi
	103
	104	echo "$bdf ($ven_dev_id): $old_driver_name -> $driver_name"
	105
	106	echo "$ven_dev_id" > "/sys/bus/pci/drivers/$driver_name/new_id" 2> /dev/null \|\| true
	107	echo "$bdf" > "/sys/bus/pci/drivers/$driver_name/bind" 2> /dev/null \|\| true
	108
	109	iommu_group=$(basename $(readlink -f /sys/bus/pci/devices/$bdf/iommu_group))
	110	if [ -e "/dev/vfio/$iommu_group" ]; then
11fdf7f2 TL	111	if [ -n "$TARGET_USER" ]; then
11fdf7f2 TL	112	chown "$TARGET_USER" "/dev/vfio/$iommu_group"
7c673cae FG	113	fi
	114	fi
	115	}
	116
11fdf7f2 TL	117	function linux_unbind_driver() {
	118	bdf="$1"
	119	ven_dev_id=$(lspci -n -s $bdf \| cut -d' ' -f3 \| sed 's/:/ /')
	120
	121	if ! [ -e "/sys/bus/pci/devices/$bdf/driver" ]; then
	122	return 0
	123	fi
	124
	125	old_driver_name=$(basename $(readlink /sys/bus/pci/devices/$bdf/driver))
	126
	127	echo "$ven_dev_id" > "/sys/bus/pci/devices/$bdf/driver/remove_id" 2> /dev/null \|\| true
	128	echo "$bdf" > "/sys/bus/pci/devices/$bdf/driver/unbind"
	129	echo "$bdf ($ven_dev_id): $old_driver_name -> no driver"
	130	}
	131
	132	function linux_hugetlbfs_mounts() {
	133	mount \| grep ' type hugetlbfs ' \| awk '{ print $3 }'
7c673cae FG	134	}
7c673cae FG	135
11fdf7f2 TL	136	function get_nvme_name_from_bdf {
	137	set +e
	138	nvme_devs=`lsblk -d --output NAME \| grep "^nvme"`
	139	set -e
	140	for dev in $nvme_devs; do
	141	link_name=$(readlink /sys/block/$dev/device/device) \|\| true
	142	if [ -z "$link_name" ]; then
	143	link_name=$(readlink /sys/block/$dev/device)
	144	fi
	145	link_bdf=$(basename "$link_name")
	146	if [ "$link_bdf" = "$1" ]; then
	147	eval "$2=$dev"
	148	return
	149	fi
	150	done
	151	}
	152
	153	function get_virtio_names_from_bdf {
	154	blk_devs=`lsblk --nodeps --output NAME`
	155	virtio_names=''
	156
	157	for dev in $blk_devs; do
	158	if readlink "/sys/block/$dev" \| grep -q "$1"; then
	159	virtio_names="$virtio_names $dev"
	160	fi
	161	done
	162
	163	eval "$2='$virtio_names'"
	164	}
	165
	166	function configure_linux_pci {
7c673cae FG	167	driver_name=vfio-pci
	168	if [ -z "$(ls /sys/kernel/iommu_groups)" ]; then
	169	# No IOMMU. Use uio.
	170	driver_name=uio_pci_generic
	171	fi
	172
	173	# NVMe
	174	modprobe $driver_name \|\| true
11fdf7f2 TL	175	for bdf in $(iter_pci_class_code 01 08 02); do
	176	blkname=''
	177	get_nvme_name_from_bdf "$bdf" blkname
	178	if pci_can_bind $bdf == "0" ; then
	179	echo "Skipping un-whitelisted NVMe controller $blkname ($bdf)"
	180	continue
	181	fi
	182	if [ "$blkname" != "" ]; then
	183	mountpoints=$(lsblk /dev/$blkname --output MOUNTPOINT -n \| wc -w)
	184	else
	185	mountpoints="0"
	186	fi
	187	if [ "$mountpoints" = "0" ]; then
	188	linux_bind_driver "$bdf" "$driver_name"
	189	else
	190	echo Active mountpoints on /dev/$blkname, so not binding PCI dev $bdf
	191	fi
7c673cae FG	192	done
7c673cae FG	193
7c673cae FG	194	# IOAT
	195	TMP=`mktemp`
	196	#collect all the device_id info of ioat devices.
	197	grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/include/spdk/pci_ids.h \
	198	\| awk -F"x" '{print $2}' > $TMP
	199
	200	for dev_id in `cat $TMP`; do
11fdf7f2 TL	201	for bdf in $(iter_pci_dev_id 8086 $dev_id); do
	202	if pci_can_bind $bdf == "0" ; then
	203	echo "Skipping un-whitelisted I/OAT device at $bdf"
	204	continue
	205	fi
	206	linux_bind_driver "$bdf" "$driver_name"
	207	done
	208	done
	209	rm $TMP
	210
	211	# virtio
	212	TMP=`mktemp`
	213	#collect all the device_id info of virtio devices.
	214	grep "PCI_DEVICE_ID_VIRTIO" $rootdir/include/spdk/pci_ids.h \
	215	\| awk -F"x" '{print $2}' > $TMP
	216
	217	for dev_id in `cat $TMP`; do
	218	for bdf in $(iter_pci_dev_id 1af4 $dev_id); do
	219	if pci_can_bind $bdf == "0" ; then
	220	echo "Skipping un-whitelisted Virtio device at $bdf"
	221	continue
	222	fi
	223	blknames=''
	224	get_virtio_names_from_bdf "$bdf" blknames
	225	for blkname in $blknames; do
	226	if mount \| grep -q "/dev/$blkname"; then
	227	echo Active mountpoints on /dev/$blkname, so not binding PCI dev $bdf
	228	continue 2
	229	fi
	230	done
	231
7c673cae FG	232	linux_bind_driver "$bdf" "$driver_name"
	233	done
	234	done
	235	rm $TMP
	236
	237	echo "1" > "/sys/bus/pci/rescan"
11fdf7f2 TL	238	}
	239
	240	function cleanup_linux {
	241	shopt -s extglob nullglob
	242	dirs_to_clean=""
	243	dirs_to_clean="$(echo {/var/run,/tmp}/dpdk/spdk{,_pid}+([0-9])) "
	244	if [[ -d $XDG_RUNTIME_DIR && $XDG_RUNTIME_DIR != " " ]]; then
	245	dirs_to_clean+="$(readlink -e assert_not_empty $XDG_RUNTIME_DIR/dpdk/spdk{,_pid}+([0-9]) \|\| true) "
	246	fi
	247
	248	files_to_clean=""
	249	for dir in $dirs_to_clean; do
	250	files_to_clean+="$(echo $dir/*) "
	251	done
	252	shopt -u extglob nullglob
	253
	254	files_to_clean+="$(echo /dev/shm/* \| egrep '(spdk_tgt\|iscsi\|vhost\|nvmf\|rocksdb\|bdevtest\|bdevperf)_trace\|spdk_iscsi_conns' \|\| true) "
	255	files_to_clean="$(readlink -e assert_not_empty $files_to_clean \|\| true)"
	256	if [[ -z "$files_to_clean" ]]; then
	257	echo "Clean"
	258	return 0;
	259	fi
	260
	261	shopt -s extglob
	262	for fd_dir in $(echo /proc/+([0-9])); do
	263	opened_files+="$(readlink -e assert_not_empty $fd_dir/fd/* \|\| true)"
	264	done
	265	shopt -u extglob
	266
	267	if [[ -z "$opened_files" ]]; then
	268	echo "Can't get list of opened files!"
	269	exit 1
	270	fi
	271
	272	echo 'Cleaning'
	273	for f in $files_to_clean; do
	274	if ! echo "$opened_files" \| egrep -q "^$f\$"; then
	275	echo "Removing: $f"
	276	rm $f
	277	else
	278	echo "Still open: $f"
	279	fi
	280	done
	281
	282	for dir in $dirs_to_clean; do
	283	if ! echo "$opened_files" \| egrep -q "^$dir\$"; then
	284	echo "Removing: $dir"
	285	rmdir $dir
	286	else
	287	echo "Still open: $dir"
	288	fi
	289	done
	290	echo "Clean"
7c673cae	291
11fdf7f2 TL	292	unset dirs_to_clean files_to_clean opened_files
	293	}
	294
	295	function configure_linux {
	296	configure_linux_pci
	297	hugetlbfs_mounts=$(linux_hugetlbfs_mounts)
	298
	299	if [ -z "$hugetlbfs_mounts" ]; then
	300	hugetlbfs_mounts=/mnt/huge
	301	echo "Mounting hugetlbfs at $hugetlbfs_mounts"
	302	mkdir -p "$hugetlbfs_mounts"
	303	mount -t hugetlbfs nodev "$hugetlbfs_mounts"
	304	fi
7c673cae	305
11fdf7f2 TL	306	if [ -z "$HUGENODE" ]; then
	307	hugepages_target="/proc/sys/vm/nr_hugepages"
	308	else
	309	hugepages_target="/sys/devices/system/node/node${HUGENODE}/hugepages/hugepages-${HUGEPGSZ}kB/nr_hugepages"
	310	fi
	311
	312	echo "$NRHUGE" > "$hugepages_target"
	313	allocated_hugepages=`cat $hugepages_target`
	314	if [ "$allocated_hugepages" -lt "$NRHUGE" ]; then
	315	echo ""
	316	echo "## ERROR: requested $NRHUGE hugepages but only $allocated_hugepages could be allocated."
	317	echo "## Memory might be heavily fragmented. Please try flushing the system cache, or reboot the machine."
	318	exit 1
7c673cae	319	fi
7c673cae FG	320
7c673cae FG	321	if [ "$driver_name" = "vfio-pci" ]; then
11fdf7f2 TL	322	if [ -n "$TARGET_USER" ]; then
	323	for mount in $hugetlbfs_mounts; do
	324	chown "$TARGET_USER" "$mount"
	325	chmod g+w "$mount"
	326	done
7c673cae FG	327	fi
	328
	329	MEMLOCK_AMNT=`ulimit -l`
	330	if [ "$MEMLOCK_AMNT" != "unlimited" ] ; then
	331	MEMLOCK_MB=$(( $MEMLOCK_AMNT / 1024 ))
	332	echo ""
	333	echo "Current user memlock limit: ${MEMLOCK_MB} MB"
	334	echo ""
	335	echo "This is the maximum amount of memory you will be"
	336	echo "able to use with DPDK and VFIO if run as current user."
	337	echo -n "To change this, please adjust limits.conf memlock "
	338	echo "limit for current user."
	339
	340	if [ $MEMLOCK_AMNT -lt 65536 ] ; then
	341	echo ""
	342	echo "## WARNING: memlock limit is less than 64MB"
	343	echo -n "## DPDK with VFIO may not be able to initialize "
	344	echo "if run as current user."
	345	fi
	346	fi
	347	fi
	348	}
	349
11fdf7f2	350	function reset_linux_pci {
7c673cae	351	# NVMe
11fdf7f2 TL	352	set +e
	353	check_for_driver nvme
	354	driver_loaded=$?
	355	set -e
	356	for bdf in $(iter_pci_class_code 01 08 02); do
	357	if pci_can_bind $bdf == "0" ; then
	358	echo "Skipping un-whitelisted NVMe controller $blkname ($bdf)"
	359	continue
	360	fi
	361	if [ $driver_loaded -ne 0 ]; then
	362	linux_bind_driver "$bdf" nvme
	363	else
	364	linux_unbind_driver "$bdf"
	365	fi
7c673cae FG	366	done
7c673cae FG	367
7c673cae FG	368	# IOAT
	369	TMP=`mktemp`
	370	#collect all the device_id info of ioat devices.
	371	grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/include/spdk/pci_ids.h \
	372	\| awk -F"x" '{print $2}' > $TMP
	373
11fdf7f2 TL	374	set +e
	375	check_for_driver ioatdma
	376	driver_loaded=$?
	377	set -e
	378	for dev_id in `cat $TMP`; do
	379	for bdf in $(iter_pci_dev_id 8086 $dev_id); do
	380	if pci_can_bind $bdf == "0" ; then
	381	echo "Skipping un-whitelisted I/OAT device at $bdf"
	382	continue
	383	fi
	384	if [ $driver_loaded -ne 0 ]; then
	385	linux_bind_driver "$bdf" ioatdma
	386	else
	387	linux_unbind_driver "$bdf"
	388	fi
	389	done
	390	done
	391	rm $TMP
	392
	393	# virtio
	394	TMP=`mktemp`
	395	#collect all the device_id info of virtio devices.
	396	grep "PCI_DEVICE_ID_VIRTIO" $rootdir/include/spdk/pci_ids.h \
	397	\| awk -F"x" '{print $2}' > $TMP
	398
	399	# TODO: check if virtio-pci is loaded first and just unbind if it is not loaded
	400	# Requires some more investigation - for example, some kernels do not seem to have
	401	# virtio-pci but just virtio_scsi instead. Also need to make sure we get the
	402	# underscore vs. dash right in the virtio_scsi name.
	403	modprobe virtio-pci \|\| true
7c673cae	404	for dev_id in `cat $TMP`; do
11fdf7f2 TL	405	for bdf in $(iter_pci_dev_id 1af4 $dev_id); do
	406	if pci_can_bind $bdf == "0" ; then
	407	echo "Skipping un-whitelisted Virtio device at $bdf"
	408	continue
	409	fi
	410	linux_bind_driver "$bdf" virtio-pci
7c673cae FG	411	done
	412	done
	413	rm $TMP
	414
	415	echo "1" > "/sys/bus/pci/rescan"
11fdf7f2	416	}
7c673cae	417
11fdf7f2 TL	418	function reset_linux {
	419	reset_linux_pci
	420	for mount in $(linux_hugetlbfs_mounts); do
	421	rm -f "$mount"/spdkmap_
	422	done
	423	rm -f /run/.spdk*
7c673cae FG	424	}
	425
	426	function status_linux {
11fdf7f2 TL	427	echo "Hugepages"
	428	printf "%-6s %10s %8s / %6s\n" "node" "hugesize" "free" "total"
	429
	430	numa_nodes=0
	431	shopt -s nullglob
	432	for path in /sys/devices/system/node/node?/hugepages/hugepages-*/; do
	433	numa_nodes=$((numa_nodes + 1))
	434	free_pages=`cat $path/free_hugepages`
	435	all_pages=`cat $path/nr_hugepages`
	436
	437	[[ $path =~ (node[0-9]+)/hugepages/hugepages-([0-9]+kB) ]]
	438
	439	node=${BASH_REMATCH[1]}
	440	huge_size=${BASH_REMATCH[2]}
	441
	442	printf "%-6s %10s %8s / %6s\n" $node $huge_size $free_pages $all_pages
	443	done
	444	shopt -u nullglob
	445
	446	# fall back to system-wide hugepages
	447	if [ "$numa_nodes" = "0" ]; then
	448	free_pages=`grep HugePages_Free /proc/meminfo \| awk '{ print $2 }'`
	449	all_pages=`grep HugePages_Total /proc/meminfo \| awk '{ print $2 }'`
	450	node="-"
	451	huge_size="$HUGEPGSZ"
	452
	453	printf "%-6s %10s %8s / %6s\n" $node $huge_size $free_pages $all_pages
	454	fi
	455
7c673cae FG	456	echo "NVMe devices"
	457
	458	echo -e "BDF\t\tNuma Node\tDriver name\t\tDevice name"
11fdf7f2	459	for bdf in $(iter_pci_class_code 01 08 02); do
7c673cae FG	460	driver=`grep DRIVER /sys/bus/pci/devices/$bdf/uevent \|awk -F"=" '{print $2}'`
7c673cae FG	461	node=`cat /sys/bus/pci/devices/$bdf/numa_node`;
11fdf7f2	462	if [ "$driver" = "nvme" -a -d /sys/bus/pci/devices/$bdf/nvme ]; then
7c673cae FG	463	name="\t"`ls /sys/bus/pci/devices/$bdf/nvme`;
	464	else
	465	name="-";
	466	fi
	467	echo -e "$bdf\t$node\t\t$driver\t\t$name";
	468	done
	469
	470	echo "I/OAT DMA"
	471
	472	#collect all the device_id info of ioat devices.
	473	TMP=`grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/include/spdk/pci_ids.h \
	474	\| awk -F"x" '{print $2}'`
	475	echo -e "BDF\t\tNuma Node\tDriver Name"
	476	for dev_id in $TMP; do
11fdf7f2	477	for bdf in $(iter_pci_dev_id 8086 $dev_id); do
7c673cae FG	478	driver=`grep DRIVER /sys/bus/pci/devices/$bdf/uevent \|awk -F"=" '{print $2}'`
	479	node=`cat /sys/bus/pci/devices/$bdf/numa_node`;
	480	echo -e "$bdf\t$node\t\t$driver"
	481	done
	482	done
11fdf7f2 TL	483
	484	echo "virtio"
	485
	486	#collect all the device_id info of virtio devices.
	487	TMP=`grep "PCI_DEVICE_ID_VIRTIO" $rootdir/include/spdk/pci_ids.h \
	488	\| awk -F"x" '{print $2}'`
	489	echo -e "BDF\t\tNuma Node\tDriver Name\t\tDevice Name"
	490	for dev_id in $TMP; do
	491	for bdf in $(iter_pci_dev_id 1af4 $dev_id); do
	492	driver=`grep DRIVER /sys/bus/pci/devices/$bdf/uevent \|awk -F"=" '{print $2}'`
	493	node=`cat /sys/bus/pci/devices/$bdf/numa_node`;
	494	blknames=''
	495	get_virtio_names_from_bdf "$bdf" blknames
	496	echo -e "$bdf\t$node\t\t$driver\t\t$blknames"
	497	done
	498	done
7c673cae FG	499	}
7c673cae FG	500
11fdf7f2	501	function configure_freebsd_pci {
7c673cae FG	502	TMP=`mktemp`
	503
	504	# NVMe
	505	GREP_STR="class=0x010802"
	506
	507	# IOAT
	508	grep "PCI_DEVICE_ID_INTEL_IOAT" $rootdir/include/spdk/pci_ids.h \
	509	\| awk -F"x" '{print $2}' > $TMP
	510	for dev_id in `cat $TMP`; do
	511	GREP_STR="${GREP_STR}\\|chip=0x${dev_id}8086"
	512	done
	513
	514	AWK_PROG="{if (count > 0) printf \",\"; printf \"%s:%s:%s\",\$2,\$3,\$4; count++}"
	515	echo $AWK_PROG > $TMP
	516
	517	BDFS=`pciconf -l \| grep "${GREP_STR}" \| awk -F: -f $TMP`
	518
	519	kldunload nic_uio.ko \|\| true
	520	kenv hw.nic_uio.bdfs=$BDFS
	521	kldload nic_uio.ko
	522	rm $TMP
11fdf7f2	523	}
7c673cae	524
11fdf7f2 TL	525	function configure_freebsd {
	526	configure_freebsd_pci
	527	# If contigmem is already loaded but the HUGEMEM specified doesn't match the
	528	# previous value, unload contigmem so that we can reload with the new value.
	529	if kldstat -q -m contigmem; then
	530	if [ `kenv hw.contigmem.num_buffers` -ne "$((HUGEMEM / 256))" ]; then
	531	kldunload contigmem.ko
	532	fi
	533	fi
	534	if ! kldstat -q -m contigmem; then
	535	kenv hw.contigmem.num_buffers=$((HUGEMEM / 256))
	536	kenv hw.contigmem.buffer_size=$((256 * 1024 * 1024))
	537	kldload contigmem.ko
	538	fi
7c673cae FG	539	}
	540
	541	function reset_freebsd {
	542	kldunload contigmem.ko \|\| true
	543	kldunload nic_uio.ko \|\| true
	544	}
	545
11fdf7f2 TL	546	mode=$1
	547
	548	if [ -z "$mode" ]; then
	549	mode="config"
	550	fi
7c673cae	551
11fdf7f2 TL	552	: ${HUGEMEM:=2048}
11fdf7f2 TL	553	: ${PCI_WHITELIST:=""}
7c673cae	554
11fdf7f2 TL	555	if [ -n "$NVME_WHITELIST" ]; then
11fdf7f2 TL	556	PCI_WHITELIST="$PCI_WHITELIST $NVME_WHITELIST"
7c673cae FG	557	fi
7c673cae FG	558
11fdf7f2 TL	559	if [ -n "$SKIP_PCI" ]; then
11fdf7f2 TL	560	PCI_WHITELIST="none"
7c673cae FG	561	fi
7c673cae FG	562
11fdf7f2 TL	563	declare -a PCI_WHITELIST=(${PCI_WHITELIST})
	564
	565	if [ -z "$TARGET_USER" ]; then
	566	TARGET_USER="$SUDO_USER"
	567	if [ -z "$TARGET_USER" ]; then
	568	TARGET_USER=`logname 2>/dev/null` \|\| true
7c673cae FG	569	fi
	570	fi
	571
	572	if [ `uname` = Linux ]; then
11fdf7f2 TL	573	HUGEPGSZ=$(( `grep Hugepagesize /proc/meminfo \| cut -d : -f 2 \| tr -dc '0-9'` ))
	574	HUGEPGSZ_MB=$(( $HUGEPGSZ / 1024 ))
	575	: ${NRHUGE=$(( (HUGEMEM + HUGEPGSZ_MB - 1) / HUGEPGSZ_MB ))}
	576
7c673cae FG	577	if [ "$mode" == "config" ]; then
7c673cae FG	578	configure_linux
11fdf7f2 TL	579	elif [ "$mode" == "cleanup" ]; then
11fdf7f2 TL	580	cleanup_linux
7c673cae FG	581	elif [ "$mode" == "reset" ]; then
	582	reset_linux
	583	elif [ "$mode" == "status" ]; then
	584	status_linux
11fdf7f2 TL	585	elif [ "$mode" == "help" ]; then
	586	usage $0
	587	else
	588	usage $0 "Invalid argument '$mode'"
7c673cae FG	589	fi
	590	else
	591	if [ "$mode" == "config" ]; then
	592	configure_freebsd
	593	elif [ "$mode" == "reset" ]; then
	594	reset_freebsd
11fdf7f2 TL	595	elif [ "$mode" == "cleanup" ]; then
	596	echo "setup.sh cleanup function not yet supported on $(uname)"
	597	elif [ "$mode" == "status" ]; then
	598	echo "setup.sh status function not yet supported on $(uname)"
	599	elif [ "$mode" == "help" ]; then
	600	usage $0
	601	else
	602	usage $0 "Invalid argument '$mode'"
7c673cae FG	603	fi
7c673cae FG	604	fi