]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/dpdk/usertools/dpdk-devbind.py
update sources to ceph Nautilus 14.2.1
[ceph.git] / ceph / src / spdk / dpdk / usertools / dpdk-devbind.py
1 #! /usr/bin/env python
2 # SPDX-License-Identifier: BSD-3-Clause
3 # Copyright(c) 2010-2014 Intel Corporation
4 #
5
6 import sys
7 import os
8 import getopt
9 import subprocess
10 from os.path import exists, abspath, dirname, basename
11
12 # The PCI base class for all devices
13 network_class = {'Class': '02', 'Vendor': None, 'Device': None,
14 'SVendor': None, 'SDevice': None}
15 encryption_class = {'Class': '10', 'Vendor': None, 'Device': None,
16 'SVendor': None, 'SDevice': None}
17 intel_processor_class = {'Class': '0b', 'Vendor': '8086', 'Device': None,
18 'SVendor': None, 'SDevice': None}
19 cavium_sso = {'Class': '08', 'Vendor': '177d', 'Device': 'a04b,a04d',
20 'SVendor': None, 'SDevice': None}
21 cavium_fpa = {'Class': '08', 'Vendor': '177d', 'Device': 'a053',
22 'SVendor': None, 'SDevice': None}
23 cavium_pkx = {'Class': '08', 'Vendor': '177d', 'Device': 'a0dd,a049',
24 'SVendor': None, 'SDevice': None}
25 cavium_tim = {'Class': '08', 'Vendor': '177d', 'Device': 'a051',
26 'SVendor': None, 'SDevice': None}
27 cavium_zip = {'Class': '12', 'Vendor': '177d', 'Device': 'a037',
28 'SVendor': None, 'SDevice': None}
29 avp_vnic = {'Class': '05', 'Vendor': '1af4', 'Device': '1110',
30 'SVendor': None, 'SDevice': None}
31
32 network_devices = [network_class, cavium_pkx, avp_vnic]
33 crypto_devices = [encryption_class, intel_processor_class]
34 eventdev_devices = [cavium_sso, cavium_tim]
35 mempool_devices = [cavium_fpa]
36 compress_devices = [cavium_zip]
37
38 # global dict ethernet devices present. Dictionary indexed by PCI address.
39 # Each device within this is itself a dictionary of device properties
40 devices = {}
41 # list of supported DPDK drivers
42 dpdk_drivers = ["igb_uio", "vfio-pci", "uio_pci_generic"]
43
44 # command-line arg flags
45 b_flag = None
46 status_flag = False
47 force_flag = False
48 args = []
49
50
51 def usage():
52 '''Print usage information for the program'''
53 argv0 = basename(sys.argv[0])
54 print("""
55 Usage:
56 ------
57
58 %(argv0)s [options] DEVICE1 DEVICE2 ....
59
60 where DEVICE1, DEVICE2 etc, are specified via PCI "domain:bus:slot.func" syntax
61 or "bus:slot.func" syntax. For devices bound to Linux kernel drivers, they may
62 also be referred to by Linux interface name e.g. eth0, eth1, em0, em1, etc.
63
64 Options:
65 --help, --usage:
66 Display usage information and quit
67
68 -s, --status:
69 Print the current status of all known network, crypto, event
70 and mempool devices.
71 For each device, it displays the PCI domain, bus, slot and function,
72 along with a text description of the device. Depending upon whether the
73 device is being used by a kernel driver, the igb_uio driver, or no
74 driver, other relevant information will be displayed:
75 * the Linux interface name e.g. if=eth0
76 * the driver being used e.g. drv=igb_uio
77 * any suitable drivers not currently using that device
78 e.g. unused=igb_uio
79 NOTE: if this flag is passed along with a bind/unbind option, the
80 status display will always occur after the other operations have taken
81 place.
82
83 --status-dev:
84 Print the status of given device group. Supported device groups are:
85 "net", "crypto", "event", "mempool" and "compress"
86
87 -b driver, --bind=driver:
88 Select the driver to use or \"none\" to unbind the device
89
90 -u, --unbind:
91 Unbind a device (Equivalent to \"-b none\")
92
93 --force:
94 By default, network devices which are used by Linux - as indicated by
95 having routes in the routing table - cannot be modified. Using the
96 --force flag overrides this behavior, allowing active links to be
97 forcibly unbound.
98 WARNING: This can lead to loss of network connection and should be used
99 with caution.
100
101 Examples:
102 ---------
103
104 To display current device status:
105 %(argv0)s --status
106
107 To display current network device status:
108 %(argv0)s --status-dev net
109
110 To bind eth1 from the current driver and move to use igb_uio
111 %(argv0)s --bind=igb_uio eth1
112
113 To unbind 0000:01:00.0 from using any driver
114 %(argv0)s -u 0000:01:00.0
115
116 To bind 0000:02:00.0 and 0000:02:00.1 to the ixgbe kernel driver
117 %(argv0)s -b ixgbe 02:00.0 02:00.1
118
119 """ % locals()) # replace items from local variables
120
121
122 # This is roughly compatible with check_output function in subprocess module
123 # which is only available in python 2.7.
124 def check_output(args, stderr=None):
125 '''Run a command and capture its output'''
126 return subprocess.Popen(args, stdout=subprocess.PIPE,
127 stderr=stderr).communicate()[0]
128
129
130 def check_modules():
131 '''Checks that igb_uio is loaded'''
132 global dpdk_drivers
133
134 # list of supported modules
135 mods = [{"Name": driver, "Found": False} for driver in dpdk_drivers]
136
137 # first check if module is loaded
138 try:
139 # Get list of sysfs modules (both built-in and dynamically loaded)
140 sysfs_path = '/sys/module/'
141
142 # Get the list of directories in sysfs_path
143 sysfs_mods = [os.path.join(sysfs_path, o) for o
144 in os.listdir(sysfs_path)
145 if os.path.isdir(os.path.join(sysfs_path, o))]
146
147 # Extract the last element of '/sys/module/abc' in the array
148 sysfs_mods = [a.split('/')[-1] for a in sysfs_mods]
149
150 # special case for vfio_pci (module is named vfio-pci,
151 # but its .ko is named vfio_pci)
152 sysfs_mods = [a if a != 'vfio_pci' else 'vfio-pci' for a in sysfs_mods]
153
154 for mod in mods:
155 if mod["Name"] in sysfs_mods:
156 mod["Found"] = True
157 except:
158 pass
159
160 # check if we have at least one loaded module
161 if True not in [mod["Found"] for mod in mods] and b_flag is not None:
162 if b_flag in dpdk_drivers:
163 print("Error - no supported modules(DPDK driver) are loaded")
164 sys.exit(1)
165 else:
166 print("Warning - no supported modules(DPDK driver) are loaded")
167
168 # change DPDK driver list to only contain drivers that are loaded
169 dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]]
170
171
172 def has_driver(dev_id):
173 '''return true if a device is assigned to a driver. False otherwise'''
174 return "Driver_str" in devices[dev_id]
175
176
177 def get_pci_device_details(dev_id, probe_lspci):
178 '''This function gets additional details for a PCI device'''
179 device = {}
180
181 if probe_lspci:
182 extra_info = check_output(["lspci", "-vmmks", dev_id]).splitlines()
183
184 # parse lspci details
185 for line in extra_info:
186 if len(line) == 0:
187 continue
188 name, value = line.decode().split("\t", 1)
189 name = name.strip(":") + "_str"
190 device[name] = value
191 # check for a unix interface name
192 device["Interface"] = ""
193 for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id):
194 if "net" in dirs:
195 device["Interface"] = \
196 ",".join(os.listdir(os.path.join(base, "net")))
197 break
198 # check if a port is used for ssh connection
199 device["Ssh_if"] = False
200 device["Active"] = ""
201
202 return device
203
204 def clear_data():
205 '''This function clears any old data'''
206 devices = {}
207
208 def get_device_details(devices_type):
209 '''This function populates the "devices" dictionary. The keys used are
210 the pci addresses (domain:bus:slot.func). The values are themselves
211 dictionaries - one for each NIC.'''
212 global devices
213 global dpdk_drivers
214
215 # first loop through and read details for all devices
216 # request machine readable format, with numeric IDs and String
217 dev = {}
218 dev_lines = check_output(["lspci", "-Dvmmnnk"]).splitlines()
219 for dev_line in dev_lines:
220 if len(dev_line) == 0:
221 if device_type_match(dev, devices_type):
222 # Replace "Driver" with "Driver_str" to have consistency of
223 # of dictionary key names
224 if "Driver" in dev.keys():
225 dev["Driver_str"] = dev.pop("Driver")
226 if "Module" in dev.keys():
227 dev["Module_str"] = dev.pop("Module")
228 # use dict to make copy of dev
229 devices[dev["Slot"]] = dict(dev)
230 # Clear previous device's data
231 dev = {}
232 else:
233 name, value = dev_line.decode().split("\t", 1)
234 value_list = value.rsplit(' ', 1)
235 if len(value_list) > 1:
236 # String stored in <name>_str
237 dev[name.rstrip(":") + '_str'] = value_list[0]
238 # Numeric IDs
239 dev[name.rstrip(":")] = value_list[len(value_list) - 1] \
240 .rstrip("]").lstrip("[")
241
242 if devices_type == network_devices:
243 # check what is the interface if any for an ssh connection if
244 # any to this host, so we can mark it later.
245 ssh_if = []
246 route = check_output(["ip", "-o", "route"])
247 # filter out all lines for 169.254 routes
248 route = "\n".join(filter(lambda ln: not ln.startswith("169.254"),
249 route.decode().splitlines()))
250 rt_info = route.split()
251 for i in range(len(rt_info) - 1):
252 if rt_info[i] == "dev":
253 ssh_if.append(rt_info[i+1])
254
255 # based on the basic info, get extended text details
256 for d in devices.keys():
257 if not device_type_match(devices[d], devices_type):
258 continue
259
260 # get additional info and add it to existing data
261 devices[d] = devices[d].copy()
262 # No need to probe lspci
263 devices[d].update(get_pci_device_details(d, False).items())
264
265 if devices_type == network_devices:
266 for _if in ssh_if:
267 if _if in devices[d]["Interface"].split(","):
268 devices[d]["Ssh_if"] = True
269 devices[d]["Active"] = "*Active*"
270 break
271
272 # add igb_uio to list of supporting modules if needed
273 if "Module_str" in devices[d]:
274 for driver in dpdk_drivers:
275 if driver not in devices[d]["Module_str"]:
276 devices[d]["Module_str"] = \
277 devices[d]["Module_str"] + ",%s" % driver
278 else:
279 devices[d]["Module_str"] = ",".join(dpdk_drivers)
280
281 # make sure the driver and module strings do not have any duplicates
282 if has_driver(d):
283 modules = devices[d]["Module_str"].split(",")
284 if devices[d]["Driver_str"] in modules:
285 modules.remove(devices[d]["Driver_str"])
286 devices[d]["Module_str"] = ",".join(modules)
287
288
289 def device_type_match(dev, devices_type):
290 for i in range(len(devices_type)):
291 param_count = len(
292 [x for x in devices_type[i].values() if x is not None])
293 match_count = 0
294 if dev["Class"][0:2] == devices_type[i]["Class"]:
295 match_count = match_count + 1
296 for key in devices_type[i].keys():
297 if key != 'Class' and devices_type[i][key]:
298 value_list = devices_type[i][key].split(',')
299 for value in value_list:
300 if value.strip(' ') == dev[key]:
301 match_count = match_count + 1
302 # count must be the number of non None parameters to match
303 if match_count == param_count:
304 return True
305 return False
306
307 def dev_id_from_dev_name(dev_name):
308 '''Take a device "name" - a string passed in by user to identify a NIC
309 device, and determine the device id - i.e. the domain:bus:slot.func - for
310 it, which can then be used to index into the devices array'''
311
312 # check if it's already a suitable index
313 if dev_name in devices:
314 return dev_name
315 # check if it's an index just missing the domain part
316 elif "0000:" + dev_name in devices:
317 return "0000:" + dev_name
318 else:
319 # check if it's an interface name, e.g. eth1
320 for d in devices.keys():
321 if dev_name in devices[d]["Interface"].split(","):
322 return devices[d]["Slot"]
323 # if nothing else matches - error
324 print("Unknown device: %s. "
325 "Please specify device in \"bus:slot.func\" format" % dev_name)
326 sys.exit(1)
327
328
329 def unbind_one(dev_id, force):
330 '''Unbind the device identified by "dev_id" from its current driver'''
331 dev = devices[dev_id]
332 if not has_driver(dev_id):
333 print("%s %s %s is not currently managed by any driver\n" %
334 (dev["Slot"], dev["Device_str"], dev["Interface"]))
335 return
336
337 # prevent us disconnecting ourselves
338 if dev["Ssh_if"] and not force:
339 print("Routing table indicates that interface %s is active. "
340 "Skipping unbind" % (dev_id))
341 return
342
343 # write to /sys to unbind
344 filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"]
345 try:
346 f = open(filename, "a")
347 except:
348 print("Error: unbind failed for %s - Cannot open %s"
349 % (dev_id, filename))
350 sys.exit(1)
351 f.write(dev_id)
352 f.close()
353
354
355 def bind_one(dev_id, driver, force):
356 '''Bind the device given by "dev_id" to the driver "driver". If the device
357 is already bound to a different driver, it will be unbound first'''
358 dev = devices[dev_id]
359 saved_driver = None # used to rollback any unbind in case of failure
360
361 # prevent disconnection of our ssh session
362 if dev["Ssh_if"] and not force:
363 print("Routing table indicates that interface %s is active. "
364 "Not modifying" % (dev_id))
365 return
366
367 # unbind any existing drivers we don't want
368 if has_driver(dev_id):
369 if dev["Driver_str"] == driver:
370 print("%s already bound to driver %s, skipping\n"
371 % (dev_id, driver))
372 return
373 else:
374 saved_driver = dev["Driver_str"]
375 unbind_one(dev_id, force)
376 dev["Driver_str"] = "" # clear driver string
377
378 # For kernels >= 3.15 driver_override can be used to specify the driver
379 # for a device rather than relying on the driver to provide a positive
380 # match of the device. The existing process of looking up
381 # the vendor and device ID, adding them to the driver new_id,
382 # will erroneously bind other devices too which has the additional burden
383 # of unbinding those devices
384 if driver in dpdk_drivers:
385 filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id
386 if os.path.exists(filename):
387 try:
388 f = open(filename, "w")
389 except:
390 print("Error: bind failed for %s - Cannot open %s"
391 % (dev_id, filename))
392 return
393 try:
394 f.write("%s" % driver)
395 f.close()
396 except:
397 print("Error: bind failed for %s - Cannot write driver %s to "
398 "PCI ID " % (dev_id, driver))
399 return
400 # For kernels < 3.15 use new_id to add PCI id's to the driver
401 else:
402 filename = "/sys/bus/pci/drivers/%s/new_id" % driver
403 try:
404 f = open(filename, "w")
405 except:
406 print("Error: bind failed for %s - Cannot open %s"
407 % (dev_id, filename))
408 return
409 try:
410 # Convert Device and Vendor Id to int to write to new_id
411 f.write("%04x %04x" % (int(dev["Vendor"],16),
412 int(dev["Device"], 16)))
413 f.close()
414 except:
415 print("Error: bind failed for %s - Cannot write new PCI ID to "
416 "driver %s" % (dev_id, driver))
417 return
418
419 # do the bind by writing to /sys
420 filename = "/sys/bus/pci/drivers/%s/bind" % driver
421 try:
422 f = open(filename, "a")
423 except:
424 print("Error: bind failed for %s - Cannot open %s"
425 % (dev_id, filename))
426 if saved_driver is not None: # restore any previous driver
427 bind_one(dev_id, saved_driver, force)
428 return
429 try:
430 f.write(dev_id)
431 f.close()
432 except:
433 # for some reason, closing dev_id after adding a new PCI ID to new_id
434 # results in IOError. however, if the device was successfully bound,
435 # we don't care for any errors and can safely ignore IOError
436 tmp = get_pci_device_details(dev_id, True)
437 if "Driver_str" in tmp and tmp["Driver_str"] == driver:
438 return
439 print("Error: bind failed for %s - Cannot bind to driver %s"
440 % (dev_id, driver))
441 if saved_driver is not None: # restore any previous driver
442 bind_one(dev_id, saved_driver, force)
443 return
444
445 # For kernels > 3.15 driver_override is used to bind a device to a driver.
446 # Before unbinding it, overwrite driver_override with empty string so that
447 # the device can be bound to any other driver
448 filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id
449 if os.path.exists(filename):
450 try:
451 f = open(filename, "w")
452 except:
453 print("Error: unbind failed for %s - Cannot open %s"
454 % (dev_id, filename))
455 sys.exit(1)
456 try:
457 f.write("\00")
458 f.close()
459 except:
460 print("Error: unbind failed for %s - Cannot open %s"
461 % (dev_id, filename))
462 sys.exit(1)
463
464
465 def unbind_all(dev_list, force=False):
466 """Unbind method, takes a list of device locations"""
467
468 if dev_list[0] == "dpdk":
469 for d in devices.keys():
470 if "Driver_str" in devices[d]:
471 if devices[d]["Driver_str"] in dpdk_drivers:
472 unbind_one(devices[d]["Slot"], force)
473 return
474
475 dev_list = map(dev_id_from_dev_name, dev_list)
476 for d in dev_list:
477 unbind_one(d, force)
478
479
480 def bind_all(dev_list, driver, force=False):
481 """Bind method, takes a list of device locations"""
482 global devices
483
484 dev_list = map(dev_id_from_dev_name, dev_list)
485
486 for d in dev_list:
487 bind_one(d, driver, force)
488
489 # For kernels < 3.15 when binding devices to a generic driver
490 # (i.e. one that doesn't have a PCI ID table) using new_id, some devices
491 # that are not bound to any other driver could be bound even if no one has
492 # asked them to. hence, we check the list of drivers again, and see if
493 # some of the previously-unbound devices were erroneously bound.
494 if not os.path.exists("/sys/bus/pci/devices/%s/driver_override" % d):
495 for d in devices.keys():
496 # skip devices that were already bound or that we know should be bound
497 if "Driver_str" in devices[d] or d in dev_list:
498 continue
499
500 # update information about this device
501 devices[d] = dict(devices[d].items() +
502 get_pci_device_details(d, True).items())
503
504 # check if updated information indicates that the device was bound
505 if "Driver_str" in devices[d]:
506 unbind_one(d, force)
507
508
509 def display_devices(title, dev_list, extra_params=None):
510 '''Displays to the user the details of a list of devices given in
511 "dev_list". The "extra_params" parameter, if given, should contain a string
512 with %()s fields in it for replacement by the named fields in each
513 device's dictionary.'''
514 strings = [] # this holds the strings to print. We sort before printing
515 print("\n%s" % title)
516 print("="*len(title))
517 if len(dev_list) == 0:
518 strings.append("<none>")
519 else:
520 for dev in dev_list:
521 if extra_params is not None:
522 strings.append("%s '%s %s' %s" % (dev["Slot"],
523 dev["Device_str"],
524 dev["Device"],
525 extra_params % dev))
526 else:
527 strings.append("%s '%s'" % (dev["Slot"], dev["Device_str"]))
528 # sort before printing, so that the entries appear in PCI order
529 strings.sort()
530 print("\n".join(strings)) # print one per line
531
532 def show_device_status(devices_type, device_name):
533 global dpdk_drivers
534 kernel_drv = []
535 dpdk_drv = []
536 no_drv = []
537
538 # split our list of network devices into the three categories above
539 for d in devices.keys():
540 if device_type_match(devices[d], devices_type):
541 if not has_driver(d):
542 no_drv.append(devices[d])
543 continue
544 if devices[d]["Driver_str"] in dpdk_drivers:
545 dpdk_drv.append(devices[d])
546 else:
547 kernel_drv.append(devices[d])
548
549 # print each category separately, so we can clearly see what's used by DPDK
550 display_devices("%s devices using DPDK-compatible driver" % device_name,
551 dpdk_drv, "drv=%(Driver_str)s unused=%(Module_str)s")
552 display_devices("%s devices using kernel driver" % device_name, kernel_drv,
553 "if=%(Interface)s drv=%(Driver_str)s "
554 "unused=%(Module_str)s %(Active)s")
555 display_devices("Other %s devices" % device_name, no_drv,
556 "unused=%(Module_str)s")
557
558 def show_status():
559 '''Function called when the script is passed the "--status" option.
560 Displays to the user what devices are bound to the igb_uio driver, the
561 kernel driver or to no driver'''
562
563 if status_dev == "net" or status_dev == "all":
564 show_device_status(network_devices, "Network")
565
566 if status_dev == "crypto" or status_dev == "all":
567 show_device_status(crypto_devices, "Crypto")
568
569 if status_dev == "event" or status_dev == "all":
570 show_device_status(eventdev_devices, "Eventdev")
571
572 if status_dev == "mempool" or status_dev == "all":
573 show_device_status(mempool_devices, "Mempool")
574
575 if status_dev == "compress" or status_dev == "all":
576 show_device_status(compress_devices , "Compress")
577
578
579 def parse_args():
580 '''Parses the command-line arguments given by the user and takes the
581 appropriate action for each'''
582 global b_flag
583 global status_flag
584 global status_dev
585 global force_flag
586 global args
587 if len(sys.argv) <= 1:
588 usage()
589 sys.exit(0)
590
591 try:
592 opts, args = getopt.getopt(sys.argv[1:], "b:us",
593 ["help", "usage", "status", "status-dev=",
594 "force", "bind=", "unbind", ])
595 except getopt.GetoptError as error:
596 print(str(error))
597 print("Run '%s --usage' for further information" % sys.argv[0])
598 sys.exit(1)
599
600 for opt, arg in opts:
601 if opt == "--help" or opt == "--usage":
602 usage()
603 sys.exit(0)
604 if opt == "--status-dev":
605 status_flag = True
606 status_dev = arg
607 if opt == "--status" or opt == "-s":
608 status_flag = True
609 status_dev = "all"
610 if opt == "--force":
611 force_flag = True
612 if opt == "-b" or opt == "-u" or opt == "--bind" or opt == "--unbind":
613 if b_flag is not None:
614 print("Error - Only one bind or unbind may be specified\n")
615 sys.exit(1)
616 if opt == "-u" or opt == "--unbind":
617 b_flag = "none"
618 else:
619 b_flag = arg
620
621
622 def do_arg_actions():
623 '''do the actual action requested by the user'''
624 global b_flag
625 global status_flag
626 global force_flag
627 global args
628
629 if b_flag is None and not status_flag:
630 print("Error: No action specified for devices."
631 "Please give a -b or -u option")
632 print("Run '%s --usage' for further information" % sys.argv[0])
633 sys.exit(1)
634
635 if b_flag is not None and len(args) == 0:
636 print("Error: No devices specified.")
637 print("Run '%s --usage' for further information" % sys.argv[0])
638 sys.exit(1)
639
640 if b_flag == "none" or b_flag == "None":
641 unbind_all(args, force_flag)
642 elif b_flag is not None:
643 bind_all(args, b_flag, force_flag)
644 if status_flag:
645 if b_flag is not None:
646 clear_data()
647 # refresh if we have changed anything
648 get_device_details(network_devices)
649 get_device_details(crypto_devices)
650 get_device_details(eventdev_devices)
651 get_device_details(mempool_devices)
652 get_device_details(compress_devices)
653 show_status()
654
655
656 def main():
657 '''program main function'''
658 parse_args()
659 check_modules()
660 clear_data()
661 get_device_details(network_devices)
662 get_device_details(crypto_devices)
663 get_device_details(eventdev_devices)
664 get_device_details(mempool_devices)
665 get_device_details(compress_devices)
666 do_arg_actions()
667
668 if __name__ == "__main__":
669 main()