]>
git.proxmox.com Git - ceph.git/blob - ceph/src/dpdk/tools/dpdk-setup.sh
5 # Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
8 # Redistribution and use in source and binary forms, with or without
9 # modification, are permitted provided that the following conditions
12 # * Redistributions of source code must retain the above copyright
13 # notice, this list of conditions and the following disclaimer.
14 # * Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in
16 # the documentation and/or other materials provided with the
18 # * Neither the name of Intel Corporation nor the names of its
19 # contributors may be used to endorse or promote products derived
20 # from this software without specific prior written permission.
22 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 # Run with "source /path/to/dpdk-setup.sh"
39 # Change to DPDK directory ( <this-script's-dir>/.. ), and export it as RTE_SDK
41 cd $
(dirname ${BASH_SOURCE[0]})/..
43 echo "------------------------------------------------------------------------------"
44 echo " RTE_SDK exported as $RTE_SDK"
45 echo "------------------------------------------------------------------------------"
47 HUGEPGSZ
=`cat /proc/meminfo | grep Hugepagesize | cut -d : -f 2 | tr -d ' '`
50 # Application EAL parameters for setting memory options (amount/channels/ranks).
55 # Sets QUIT variable so script will finish.
69 # Sets up environmental variables for ICC.
73 DEFAULT_PATH
=/opt
/intel
/bin
/iccvars.sh
75 shpath
=`which iccvars.sh 2> /dev/null`
76 if [ $?
-eq 0 ] ; then
77 echo "Loading iccvars.sh from $shpath for $param"
79 elif [ -f $DEFAULT_PATH ] ; then
80 echo "Loading iccvars.sh from $DEFAULT_PATH for $param"
81 source $DEFAULT_PATH $param
83 echo "## ERROR: cannot find 'iccvars.sh' script to set up ICC."
84 echo "## To fix, please add the directory that contains"
85 echo "## iccvars.sh to your 'PATH' environment variable."
91 # Sets RTE_TARGET and does a "make install".
96 export RTE_TARGET
=${TARGETS[option]}
98 compiler
=${RTE_TARGET##*-}
99 if [ "$compiler" == "icc" ] ; then
100 platform
=${RTE_TARGET%%-*}
101 if [ "$platform" == "x86_64" ] ; then
107 if [ "$QUIT" == "0" ] ; then
108 make install T
=${RTE_TARGET}
110 echo "------------------------------------------------------------------------------"
111 echo " RTE_TARGET exported as $RTE_TARGET"
112 echo "------------------------------------------------------------------------------"
116 # Creates hugepage filesystem.
120 echo "Creating /mnt/huge and mounting as hugetlbfs"
121 sudo mkdir
-p /mnt
/huge
123 grep -s '/mnt/huge' /proc
/mounts
> /dev
/null
124 if [ $?
-ne 0 ] ; then
125 sudo mount
-t hugetlbfs nodev
/mnt
/huge
130 # Removes hugepage filesystem.
134 echo "Unmounting /mnt/huge and removing directory"
135 grep -s '/mnt/huge' /proc
/mounts
> /dev
/null
136 if [ $?
-eq 0 ] ; then
137 sudo umount
/mnt
/huge
140 if [ -d /mnt
/huge
] ; then
146 # Unloads igb_uio.ko.
148 remove_igb_uio_module
()
150 echo "Unloading any existing DPDK UIO module"
151 /sbin
/lsmod |
grep -s igb_uio
> /dev
/null
152 if [ $?
-eq 0 ] ; then
153 sudo
/sbin
/rmmod igb_uio
158 # Loads new igb_uio.ko (and uio module if needed).
160 load_igb_uio_module
()
162 if [ ! -f $RTE_SDK/$RTE_TARGET/kmod
/igb_uio.ko
];then
163 echo "## ERROR: Target does not have the DPDK UIO Kernel Module."
164 echo " To fix, please try to rebuild target."
168 remove_igb_uio_module
170 /sbin
/lsmod |
grep -s uio
> /dev
/null
171 if [ $?
-ne 0 ] ; then
172 modinfo uio
> /dev
/null
173 if [ $?
-eq 0 ]; then
174 echo "Loading uio module"
175 sudo
/sbin
/modprobe uio
179 # UIO may be compiled into kernel, so it may not be an error if it can't
182 echo "Loading DPDK UIO module"
183 sudo
/sbin
/insmod
$RTE_SDK/$RTE_TARGET/kmod
/igb_uio.ko
184 if [ $?
-ne 0 ] ; then
185 echo "## ERROR: Could not load kmod/igb_uio.ko."
191 # Unloads VFIO modules.
195 echo "Unloading any existing VFIO module"
196 /sbin
/lsmod |
grep -s vfio
> /dev
/null
197 if [ $?
-eq 0 ] ; then
198 sudo
/sbin
/rmmod vfio-pci
199 sudo
/sbin
/rmmod vfio_iommu_type1
200 sudo
/sbin
/rmmod vfio
205 # Loads new vfio-pci (and vfio module if needed).
211 VFIO_PATH
="kernel/drivers/vfio/pci/vfio-pci.ko"
213 echo "Loading VFIO module"
214 /sbin
/lsmod |
grep -s vfio_pci
> /dev
/null
215 if [ $?
-ne 0 ] ; then
216 if [ -f /lib
/modules
/$
(uname
-r)/$VFIO_PATH ] ; then
217 sudo
/sbin
/modprobe vfio-pci
221 # make sure regular users can read /dev/vfio
222 echo "chmod /dev/vfio"
223 sudo
chmod a
+x
/dev
/vfio
224 if [ $?
-ne 0 ] ; then
230 # check if /dev/vfio/vfio exists - that way we
231 # know we either loaded the module, or it was
232 # compiled into the kernel
233 if [ ! -e /dev
/vfio
/vfio
] ; then
234 echo "## ERROR: VFIO not found!"
239 # Unloads the rte_kni.ko module.
243 echo "Unloading any existing DPDK KNI module"
244 /sbin
/lsmod |
grep -s rte_kni
> /dev
/null
245 if [ $?
-eq 0 ] ; then
246 sudo
/sbin
/rmmod rte_kni
251 # Loads the rte_kni.ko module.
255 # Check that the KNI module is already built.
256 if [ ! -f $RTE_SDK/$RTE_TARGET/kmod
/rte_kni.ko
];then
257 echo "## ERROR: Target does not have the DPDK KNI Module."
258 echo " To fix, please try to rebuild target."
262 # Unload existing version if present.
265 # Now try load the KNI module.
266 echo "Loading DPDK KNI module"
267 sudo
/sbin
/insmod
$RTE_SDK/$RTE_TARGET/kmod
/rte_kni.ko
268 if [ $?
-ne 0 ] ; then
269 echo "## ERROR: Could not load kmod/rte_kni.ko."
275 # Sets appropriate permissions on /dev/vfio/* files
277 set_vfio_permissions
()
279 # make sure regular users can read /dev/vfio
280 echo "chmod /dev/vfio"
281 sudo
chmod a
+x
/dev
/vfio
282 if [ $?
-ne 0 ] ; then
288 # make sure regular user can access everything inside /dev/vfio
289 echo "chmod /dev/vfio/*"
290 sudo
chmod 0666 /dev
/vfio
/*
291 if [ $?
-ne 0 ] ; then
297 # since permissions are only to be set when running as
298 # regular user, we only check ulimit here
300 # warn if regular user is only allowed
301 # to memlock <64M of memory
302 MEMLOCK_AMNT
=`ulimit -l`
304 if [ "$MEMLOCK_AMNT" != "unlimited" ] ; then
305 MEMLOCK_MB
=`expr $MEMLOCK_AMNT / 1024`
307 echo "Current user memlock limit: ${MEMLOCK_MB} MB"
309 echo "This is the maximum amount of memory you will be"
310 echo "able to use with DPDK and VFIO if run as current user."
311 echo -n "To change this, please adjust limits.conf memlock "
312 echo "limit for current user."
314 if [ $MEMLOCK_AMNT -lt 65536 ] ; then
316 echo "## WARNING: memlock limit is less than 64MB"
317 echo -n "## DPDK with VFIO may not be able to initialize "
318 echo "if run as current user."
324 # Removes all reserved hugepages.
329 for d
in /sys
/devices
/system
/node
/node?
; do
330 echo "echo 0 > $d/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" >> .echo_tmp
332 echo "Removing currently reserved hugepages"
347 echo " Input the number of ${HUGEPGSZ} hugepages"
348 echo " Example: to have 128MB of hugepages available in a 2MB huge page system,"
349 echo " enter '64' to reserve 64 * 2MB pages"
350 echo -n "Number of pages: "
353 echo "echo $Pages > /sys/kernel/mm/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" > .echo_tmp
355 echo "Reserving hugepages"
363 # Creates hugepages on specific NUMA nodes.
370 echo " Input the number of ${HUGEPGSZ} hugepages for each node"
371 echo " Example: to have 128MB of hugepages available per node in a 2MB huge page system,"
372 echo " enter '64' to reserve 64 * 2MB pages on each node"
375 for d
in /sys
/devices
/system
/node
/node?
; do
377 echo -n "Number of pages for $node: "
379 echo "echo $Pages > $d/hugepages/hugepages-${HUGEPGSZ}/nr_hugepages" >> .echo_tmp
381 echo "Reserving hugepages"
389 # Run unit test application.
394 echo " Enter hex bitmask of cores to execute test app on"
395 echo " Example: to execute app on cores 0 to 7, enter 0xff"
399 sudo
${RTE_TARGET}/app
/test -c $Bitmask $EAL_PARAMS
403 # Run unit testpmd application.
408 echo " Enter hex bitmask of cores to execute testpmd app on"
409 echo " Example: to execute app on cores 0 to 7, enter 0xff"
413 sudo
${RTE_TARGET}/app
/testpmd
-c $Bitmask $EAL_PARAMS -- -i
417 # Print hugepage information.
421 grep -i huge
/proc
/meminfo
425 # Calls dpdk-devbind.py --status to show the devices and what they
426 # are all bound to, in terms of drivers.
430 if [ -d /sys
/module
/vfio_pci
-o -d /sys
/module
/igb_uio
]; then
431 ${RTE_SDK}/tools
/dpdk-devbind.py
--status
433 echo "# Please load the 'igb_uio' or 'vfio-pci' kernel module before "
434 echo "# querying or adjusting device bindings"
439 # Uses dpdk-devbind.py to move devices to work with vfio-pci
441 bind_devices_to_vfio
()
443 if [ -d /sys
/module
/vfio_pci
]; then
444 ${RTE_SDK}/tools
/dpdk-devbind.py
--status
446 echo -n "Enter PCI address of device to bind to VFIO driver: "
448 sudo
${RTE_SDK}/tools
/dpdk-devbind.py
-b vfio-pci
$PCI_PATH &&
451 echo "# Please load the 'vfio-pci' kernel module before querying or "
452 echo "# adjusting device bindings"
457 # Uses dpdk-devbind.py to move devices to work with igb_uio
459 bind_devices_to_igb_uio
()
461 if [ -d /sys
/module
/igb_uio
]; then
462 ${RTE_SDK}/tools
/dpdk-devbind.py
--status
464 echo -n "Enter PCI address of device to bind to IGB UIO driver: "
466 sudo
${RTE_SDK}/tools
/dpdk-devbind.py
-b igb_uio
$PCI_PATH && echo "OK"
468 echo "# Please load the 'igb_uio' kernel module before querying or "
469 echo "# adjusting device bindings"
474 # Uses dpdk-devbind.py to move devices to work with kernel drivers again
478 ${RTE_SDK}/tools
/dpdk-devbind.py
--status
480 echo -n "Enter PCI address of device to unbind: "
483 echo -n "Enter name of kernel driver to bind the device to: "
485 sudo
${RTE_SDK}/tools
/dpdk-devbind.py
-b $DRV $PCI_PATH && echo "OK"
489 # Options for building a target. Note that this step MUST be first as it sets
490 # up TARGETS[] starting from 1, and this is accessed in setup_target using the
491 # user entered option.
495 TITLE
="Select the DPDK environment to build"
497 for cfg
in config
/defconfig_
* ; do
498 cfg
=${cfg/config\/defconfig_/}
499 TEXT
[$CONFIG_NUM]="$cfg"
500 TARGETS
[$CONFIG_NUM]=$cfg
501 FUNC
[$CONFIG_NUM]="setup_target"
507 # Options for setting up environment.
511 TITLE
="Setup linuxapp environment"
513 TEXT
[1]="Insert IGB UIO module"
514 FUNC
[1]="load_igb_uio_module"
516 TEXT
[2]="Insert VFIO module"
517 FUNC
[2]="load_vfio_module"
519 TEXT
[3]="Insert KNI module"
520 FUNC
[3]="load_kni_module"
522 TEXT
[4]="Setup hugepage mappings for non-NUMA systems"
523 FUNC
[4]="set_non_numa_pages"
525 TEXT
[5]="Setup hugepage mappings for NUMA systems"
526 FUNC
[5]="set_numa_pages"
528 TEXT
[6]="Display current Ethernet/Crypto device settings"
529 FUNC
[6]="show_devices"
531 TEXT
[7]="Bind Ethernet/Crypto device to IGB UIO module"
532 FUNC
[7]="bind_devices_to_igb_uio"
534 TEXT
[8]="Bind Ethernet/Crypto device to VFIO module"
535 FUNC
[8]="bind_devices_to_vfio"
537 TEXT
[9]="Setup VFIO permissions"
538 FUNC
[9]="set_vfio_permissions"
542 # Options for running applications.
546 TITLE
="Run test application for linuxapp environment"
548 TEXT
[1]="Run test application (\$RTE_TARGET/app/test)"
549 FUNC
[1]="run_test_app"
551 TEXT
[2]="Run testpmd application in interactive mode (\$RTE_TARGET/app/testpmd)"
552 FUNC
[2]="run_testpmd_app"
562 TEXT
[1]="List hugepage info from /proc/meminfo"
563 FUNC
[1]="grep_meminfo"
568 # Options for cleaning up the system
572 TITLE
="Uninstall and system cleanup"
574 TEXT
[1]="Unbind devices from IGB UIO or VFIO driver"
575 FUNC
[1]="unbind_devices"
577 TEXT
[2]="Remove IGB UIO module"
578 FUNC
[2]="remove_igb_uio_module"
580 TEXT
[3]="Remove VFIO module"
581 FUNC
[3]="remove_vfio_module"
583 TEXT
[4]="Remove KNI module"
584 FUNC
[4]="remove_kni_module"
586 TEXT
[5]="Remove hugepage mappings"
587 FUNC
[5]="clear_huge_pages"
590 STEPS
[1]="step1_func"
591 STEPS
[2]="step2_func"
592 STEPS
[3]="step3_func"
593 STEPS
[4]="step4_func"
594 STEPS
[5]="step5_func"
598 while [ "$QUIT" == "0" ]; do
601 for s
in $
(seq ${#STEPS[@]}) ; do
604 echo "----------------------------------------------------------"
605 echo " Step $s: ${TITLE}"
606 echo "----------------------------------------------------------"
608 for i
in $
(seq ${#TEXT[@]}) ; do
609 echo "[$OPTION_NUM] ${TEXT[i]}"
610 OPTIONS
[$OPTION_NUM]=${FUNC[i]}
614 # Clear TEXT and FUNC arrays before next step
621 echo "[$OPTION_NUM] Exit Script"
622 OPTIONS
[$OPTION_NUM]="quit"
627 ${OPTIONS[our_entry]} ${our_entry}
629 if [ "$QUIT" == "0" ] ; then
631 echo -n "Press enter to continue ..."; read