utilities/bugtool/ovs-bugtool.in

   1 #! @PYTHON3@
   2
   3 # This library is free software; you can redistribute it and/or
   4 # modify it under the terms of version 2.1 of the GNU Lesser General Public
   5 # License as published by the Free Software Foundation.
   6 #
   7 # This library is distributed in the hope that it will be useful,
   8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
   9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  10 # Lesser General Public License for more details.
  11 #
  12 # You should have received a copy of the GNU Lesser General Public
  13 # License along with this library; if not, write to the Free Software
  14 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  15 #
  16 # Copyright (c) 2005, 2007 XenSource Ltd.
  17 # Copyright (c) 2010, 2011, 2012, 2013, 2015, 2016, 2017 Nicira, Inc.
  18
  19 #
  20 # To add new entries to the bugtool, you need to:
  21 #
  22 # Create a new capability.  These declare the new entry to the GUI, including
  23 # the expected size, time to collect, privacy implications, and whether the
  24 # capability should be selected by default.  One capability may refer to
  25 # multiple files, assuming that they can be reasonably grouped together, and
  26 # have the same privacy implications.  You need:
  27 #
  28 #   A new CAP_ constant.
  29 #   A cap() invocation to declare the capability.
  30 #
  31 # You then need to add calls to main() to collect the files.  These will
  32 # typically be calls to the helpers file_output(), tree_output(), cmd_output(),
  33 # or func_output().
  34 #
  35
  36 from __future__ import print_function
  37
  38 import StringIO
  39 import commands
  40 import fcntl
  41 import getopt
  42 import hashlib
  43 import os
  44 import platform
  45 import re
  46 import sys
  47 import tarfile
  48 import time
  49 import warnings
  50 import zipfile
  51 from select import select
  52 from signal import SIGTERM
  53 from subprocess import PIPE, Popen
  54
  55 from xml.dom.minidom import getDOMImplementation, parse
  56
  57 from six.moves import input
  58 warnings.filterwarnings(action="ignore", category=DeprecationWarning)
  59
  60 OS_RELEASE = platform.release()
  61
  62 #
  63 # Files & directories
  64 #
  65
  66 APT_SOURCES_LIST = "/etc/apt/sources.list"
  67 APT_SOURCES_LIST_D = "/etc/apt/sources.list.d"
  68 BUG_DIR = "/var/log/ovs-bugtool"
  69 PLUGIN_DIR = "@pkgdatadir@/bugtool-plugins"
  70 GRUB_CONFIG = '/boot/grub/menu.lst'
  71 BOOT_KERNEL = '/boot/vmlinuz-' + OS_RELEASE
  72 BOOT_INITRD = '/boot/initrd-' + OS_RELEASE + '.img'
  73 PROC_PARTITIONS = '/proc/partitions'
  74 FSTAB = '/etc/fstab'
  75 PROC_MOUNTS = '/proc/mounts'
  76 ISCSI_CONF = '/etc/iscsi/iscsid.conf'
  77 ISCSI_INITIATOR = '/etc/iscsi/initiatorname.iscsi'
  78 PROC_CPUINFO = '/proc/cpuinfo'
  79 PROC_MEMINFO = '/proc/meminfo'
  80 PROC_IOPORTS = '/proc/ioports'
  81 PROC_INTERRUPTS = '/proc/interrupts'
  82 PROC_SCSI = '/proc/scsi/scsi'
  83 PROC_VERSION = '/proc/version'
  84 PROC_MODULES = '/proc/modules'
  85 PROC_DEVICES = '/proc/devices'
  86 PROC_FILESYSTEMS = '/proc/filesystems'
  87 PROC_CMDLINE = '/proc/cmdline'
  88 PROC_CONFIG = '/proc/config.gz'
  89 PROC_USB_DEV = '/proc/bus/usb/devices'
  90 PROC_NET_BONDING_DIR = '/proc/net/bonding'
  91 IFCFG_RE = re.compile(r'^.*/ifcfg-.*')
  92 ROUTE_RE = re.compile(r'^.*/route-.*')
  93 SYSCONFIG_HWCONF = '/etc/sysconfig/hwconf'
  94 SYSCONFIG_NETWORK = '/etc/sysconfig/network'
  95 SYSCONFIG_NETWORK_SCRIPTS = '/etc/sysconfig/network-scripts'
  96 INTERFACES = '/etc/network/interfaces'
  97 INTERFACESD = '/etc/network/interfaces.d'
  98 PROC_NET_VLAN_DIR = '/proc/net/vlan'
  99 PROC_NET_SOFTNET_STAT = '/proc/net/softnet_stat'
 100 MODPROBE_CONF = '/etc/modprobe.conf'
 101 MODPROBE_DIR = '/etc/modprobe.d'
 102 RESOLV_CONF = '/etc/resolv.conf'
 103 MPP_CONF = '/etc/mpp.conf'
 104 MULTIPATH_CONF = '/etc/multipath.conf'
 105 NSSWITCH_CONF = '/etc/nsswitch.conf'
 106 NTP_CONF = '/etc/ntp.conf'
 107 IPTABLES_CONFIG = '/etc/sysconfig/iptables-config'
 108 HOSTS = '/etc/hosts'
 109 HOSTS_ALLOW = '/etc/hosts.allow'
 110 HOSTS_DENY = '/etc/hosts.deny'
 111 DHCP_LEASE_DIR = ['/var/lib/dhclient', '/var/lib/dhcp3']
 112 OPENVSWITCH_LOG_DIR = '@LOGDIR@/'
 113 OPENVSWITCH_DEFAULT_SWITCH = '/etc/default/openvswitch-switch'  # Debian
 114 OPENVSWITCH_SYSCONFIG_SWITCH = '/etc/sysconfig/openvswitch'     # RHEL
 115 OPENVSWITCH_CONF_DB = '@DBDIR@/conf.db'
 116 OPENVSWITCH_COMPACT_DB = '@DBDIR@/bugtool-compact-conf.db'
 117 OPENVSWITCH_VSWITCHD_PID = '@RUNDIR@/ovs-vswitchd.pid'
 118 VAR_LOG_DIR = '/var/log/'
 119 VAR_LOG_CORE_DIR = '/var/log/core'
 120 YUM_LOG = '/var/log/yum.log'
 121 YUM_REPOS_DIR = '/etc/yum.repos.d'
 122
 123 #
 124 # External programs
 125 #
 126
 127 os.environ['PATH'] = '/usr/local/sbin:/usr/local/bin:' \
 128                      '/usr/sbin:/usr/bin:/sbin:/bin:@pkgdatadir@/scripts'
 129 ARP = 'arp'
 130 CAT = 'cat'
 131 CHKCONFIG = 'chkconfig'
 132 DATE = 'date'
 133 DF = 'df'
 134 DMESG = 'dmesg'
 135 DMIDECODE = 'dmidecode'
 136 DMSETUP = 'dmsetup'
 137 DPKG_QUERY = 'dpkg-query'
 138 ETHTOOL = 'ethtool'
 139 FDISK = 'fdisk'
 140 FIND = 'find'
 141 IP = 'ip'
 142 IPTABLES = 'iptables'
 143 ISCSIADM = 'iscsiadm'
 144 LOSETUP = 'losetup'
 145 LS = 'ls'
 146 LSPCI = 'lspci'
 147 MODINFO = 'modinfo'
 148 MPPUTIL = 'mppUtil'
 149 MULTIPATHD = 'multipathd'
 150 NETSTAT = 'netstat'
 151 OVS_DPCTL = 'ovs-dpctl'
 152 OVS_OFCTL = 'ovs-ofctl'
 153 OVS_VSCTL = 'ovs-vsctl'
 154 PS = 'ps'
 155 ROUTE = 'route'
 156 RPM = 'rpm'
 157 SG_MAP = 'sg_map'
 158 SHA256_SUM = 'sha256sum'
 159 SYSCTL = 'sysctl'
 160 TC = 'tc'
 161 UPTIME = 'uptime'
 162 ZCAT = 'zcat'
 163
 164 #
 165 # PII -- Personally identifiable information.  Of particular concern are
 166 # things that would identify customers, or their network topology.
 167 # Passwords are never to be included in any bug report, regardless of any PII
 168 # declaration.
 169 #
 170 # NO            -- No PII will be in these entries.
 171 # YES           -- PII will likely or certainly be in these entries.
 172 # MAYBE         -- The user may wish to audit these entries for PII.
 173 # IF_CUSTOMIZED -- If the files are unmodified, then they will contain no PII,
 174 # but since we encourage customers to edit these files, PII may have been
 175 # introduced by the customer.  This is used in particular for the networking
 176 # scripts in dom0.
 177 #
 178
 179 PII_NO = 'no'
 180 PII_YES = 'yes'
 181 PII_MAYBE = 'maybe'
 182 PII_IF_CUSTOMIZED = 'if_customized'
 183 KEY = 0
 184 PII = 1
 185 MIN_SIZE = 2
 186 MAX_SIZE = 3
 187 MIN_TIME = 4
 188 MAX_TIME = 5
 189 MIME = 6
 190 CHECKED = 7
 191 HIDDEN = 8
 192
 193 MIME_DATA = 'application/data'
 194 MIME_TEXT = 'text/plain'
 195
 196 INVENTORY_XML_ROOT = "system-status-inventory"
 197 INVENTORY_XML_SUMMARY = 'system-summary'
 198 INVENTORY_XML_ELEMENT = 'inventory-entry'
 199 CAP_XML_ROOT = "system-status-capabilities"
 200 CAP_XML_ELEMENT = 'capability'
 201
 202
 203 CAP_BOOT_LOADER = 'boot-loader'
 204 CAP_DISK_INFO = 'disk-info'
 205 CAP_HARDWARE_INFO = 'hardware-info'
 206 CAP_KERNEL_INFO = 'kernel-info'
 207 CAP_LOSETUP_A = 'loopback-devices'
 208 CAP_MULTIPATH = 'multipath'
 209 CAP_NETWORK_CONFIG = 'network-config'
 210 CAP_NETWORK_INFO = 'network-info'
 211 CAP_NETWORK_STATUS = 'network-status'
 212 CAP_OPENVSWITCH_LOGS = 'ovs-system-logs'
 213 CAP_PROCESS_LIST = 'process-list'
 214 CAP_SYSTEM_LOGS = 'system-logs'
 215 CAP_SYSTEM_SERVICES = 'system-services'
 216 CAP_YUM = 'yum'
 217
 218 KB = 1024
 219 MB = 1024 * 1024
 220
 221 caps = {}
 222 cap_sizes = {}
 223 unlimited_data = False
 224 dbg = False
 225 # Default value for the number of days to collect logs.
 226 log_days = 20
 227 log_last_mod_time = None
 228 free_disk_space = None
 229 # Default value for delay between repeated commands
 230 command_delay = 10
 231
 232
 233 def cap(key, pii=PII_MAYBE, min_size=-1, max_size=-1, min_time=-1,
 234         max_time=-1, mime=MIME_TEXT, checked=True, hidden=False):
 235     caps[key] = (key, pii, min_size, max_size, min_time, max_time, mime,
 236                  checked, hidden)
 237     cap_sizes[key] = 0
 238
 239
 240 cap(CAP_BOOT_LOADER, PII_NO, max_size=3 * KB, max_time=5)
 241 cap(CAP_DISK_INFO, PII_MAYBE, max_size=50 * KB, max_time=20)
 242 cap(CAP_HARDWARE_INFO, PII_MAYBE, max_size=2 * MB, max_time=20)
 243 cap(CAP_KERNEL_INFO, PII_MAYBE, max_size=120 * KB, max_time=5)
 244 cap(CAP_LOSETUP_A, PII_MAYBE, max_size=KB, max_time=5)
 245 cap(CAP_MULTIPATH, PII_MAYBE, max_size=20 * KB, max_time=10)
 246 cap(CAP_NETWORK_CONFIG, PII_IF_CUSTOMIZED, min_size=0, max_size=5 * MB)
 247 cap(CAP_NETWORK_INFO, PII_YES, max_size=50 * MB, max_time=30)
 248 cap(CAP_NETWORK_STATUS, PII_YES, max_size=-1, max_time=30)
 249 cap(CAP_OPENVSWITCH_LOGS, PII_MAYBE, max_size=-1, max_time=5)
 250 cap(CAP_PROCESS_LIST, PII_YES, max_size=30 * KB, max_time=20)
 251 cap(CAP_SYSTEM_LOGS, PII_MAYBE, max_size=200 * MB, max_time=5)
 252 cap(CAP_SYSTEM_SERVICES, PII_NO, max_size=5 * KB, max_time=20)
 253 cap(CAP_YUM, PII_IF_CUSTOMIZED, max_size=10 * KB, max_time=30)
 254
 255 ANSWER_YES_TO_ALL = False
 256 SILENT_MODE = False
 257 entries = None
 258 data = {}
 259 dev_null = open('/dev/null', 'r+')
 260
 261
 262 def output(x):
 263     global SILENT_MODE
 264     if not SILENT_MODE:
 265         print(x)
 266
 267
 268 def output_ts(x):
 269     output("[%s]  %s" % (time.strftime("%x %X %Z"), x))
 270
 271
 272 def cmd_output(cap, args, label=None, filter=None,
 273                binary=False, repeat_count=1):
 274     if cap in entries:
 275         if not label:
 276             if isinstance(args, list):
 277                 a = [aa for aa in args]
 278                 a[0] = os.path.basename(a[0])
 279                 label = ' '.join(a)
 280             else:
 281                 label = args
 282         data[label] = {'cap': cap, 'cmd_args': args, 'filter': filter,
 283                        'binary': binary, 'repeat_count': repeat_count}
 284
 285
 286 def file_output(cap, path_list, newest_first=False, last_mod_time=None):
 287     """
 288     If newest_first is True, the list of files in path_list is sorted
 289     by file modification time in descending order, else its sorted
 290     in ascending order.
 291     """
 292     if cap in entries:
 293         path_entries = []
 294         for path in path_list:
 295             try:
 296                 s = os.stat(path)
 297             except OSError:
 298                 continue
 299             if last_mod_time is None or s.st_mtime >= last_mod_time:
 300                 path_entries.append((path, s))
 301
 302         def mtime(arg):
 303             (path, stat) = arg
 304             return stat.st_mtime
 305
 306         path_entries.sort(key=mtime, reverse=newest_first)
 307         for p in path_entries:
 308             if check_space(cap, p[0], p[1].st_size):
 309                 data[p] = {'cap': cap, 'filename': p[0]}
 310
 311
 312 def tree_output(cap, path, pattern=None, negate=False, newest_first=False,
 313                 last_mod_time=None):
 314     """
 315     Walks the directory tree rooted at path. Files in current dir are processed
 316     before files in sub-dirs.
 317     """
 318     if cap in entries:
 319         if os.path.exists(path):
 320             for root, dirs, files in os.walk(path):
 321                 fns = [fn for fn in [os.path.join(root, f) for f in files]
 322                        if os.path.isfile(fn) and matches(fn, pattern, negate)]
 323                 file_output(cap, fns, newest_first=newest_first,
 324                             last_mod_time=last_mod_time)
 325
 326
 327 def prefix_output(cap, prefix, newest_first=False, last_mod_time=None):
 328     """
 329     Output files with the same prefix.
 330     """
 331     fns = []
 332     for root, dirs, files in os.walk(os.path.dirname(prefix)):
 333         fns += [fn for fn in [os.path.join(root, f) for f in files]
 334                 if fn.startswith(prefix)]
 335     file_output(cap, fns, newest_first=newest_first,
 336                 last_mod_time=last_mod_time)
 337
 338
 339 def func_output(cap, label, func):
 340     if cap in entries:
 341         data[label] = {'cap': cap, 'func': func}
 342
 343
 344 def collect_data():
 345     first_run = True
 346
 347     while True:
 348         process_lists = {}
 349
 350         for (k, v) in data.items():
 351             cap = v['cap']
 352             if 'cmd_args' in v:
 353                 if 'output' not in v.keys():
 354                     v['output'] = StringIOmtime()
 355                 if v['repeat_count'] > 0:
 356                     if cap not in process_lists:
 357                         process_lists[cap] = []
 358                     process_lists[cap].append(
 359                             ProcOutput(v['cmd_args'], caps[cap][MAX_TIME],
 360                                        v['output'], v['filter'], v['binary']))
 361                     v['repeat_count'] -= 1
 362
 363         if bool(process_lists):
 364             if not first_run:
 365                 output_ts("Waiting %d sec between repeated commands" %
 366                           command_delay)
 367                 time.sleep(command_delay)
 368             else:
 369                 first_run = False
 370             run_procs(process_lists.values())
 371         else:
 372             break
 373
 374     for (k, v) in data.items():
 375         cap = v['cap']
 376         if 'filename' in v and v['filename'].startswith('/proc/'):
 377             # proc files must be read into memory
 378             try:
 379                 f = open(v['filename'], 'r')
 380                 s = f.read()
 381                 f.close()
 382                 if check_space(cap, v['filename'], len(s)):
 383                     v['output'] = StringIOmtime(s)
 384             except:
 385                 pass
 386         elif 'func' in v:
 387             try:
 388                 s = v['func'](cap)
 389             except Exception as e:
 390                 s = str(e)
 391             if check_space(cap, k, len(s)):
 392                 v['output'] = StringIOmtime(s)
 393
 394
 395 def main(argv=None):
 396     global ANSWER_YES_TO_ALL, SILENT_MODE
 397     global entries, data, dbg, unlimited_data, free_disk_space
 398     global log_days, log_last_mod_time, command_delay
 399
 400     # Filter flags
 401     only_ovs_info = False
 402     collect_all_info = True
 403
 404     if '--help' in sys.argv:
 405         print("""
 406 %(argv0)s: create status report bundles to assist in problem diagnosis
 407 usage: %(argv0)s OPTIONS
 408
 409 By default, %(argv0)s prompts for permission to collect each form of status
 410 information and produces a .tar.gz file as output.
 411
 412 The following options are available.
 413   --help                     display this help message, then exit
 414   -s, --silent               suppress most output to stdout
 415
 416 Options for categories of data to collect:
 417   --entries=CAP_A,CAP_B,...  set categories of data to collect
 418   --all                      collect all categories
 419   --ovs                      collect only directly OVS-related info
 420   --log-days=DAYS            collect DAYS worth of old logs
 421   --delay=DELAY              set delay between repeated command
 422   -y, --yestoall             suppress prompts to confirm collection
 423   --capabilities             print categories as XML on stdout, then exit
 424
 425 Output options:
 426   --output=FORMAT            set output format to one of tar tar.bz2 tar.gz zip
 427   --outfile=FILE             write output to FILE
 428   --outfd=FD                 write output to FD (requires --output=tar)
 429   --unlimited                ignore default limits on sizes of data collected
 430   --debug                    print ovs-bugtool debug info on stdout\
 431 """ % {'argv0': sys.argv[0]})
 432         sys.exit(0)
 433
 434     # we need access to privileged files, exit if we are not running as root
 435     if os.getuid() != 0:
 436         print("Error: ovs-bugtool must be run as root", file=sys.stderr)
 437         return 1
 438
 439     output_file = None
 440     output_type = 'tar.gz'
 441     output_fd = -1
 442
 443     if argv is None:
 444         argv = sys.argv
 445
 446     try:
 447         (options, params) = getopt.gnu_getopt(
 448             argv, 'sy', ['capabilities', 'silent', 'yestoall', 'entries=',
 449                          'output=', 'outfd=', 'outfile=', 'all', 'unlimited',
 450                          'debug', 'ovs', 'log-days=', 'delay='])
 451     except getopt.GetoptError as opterr:
 452         print(opterr, file=sys.stderr)
 453         return 2
 454
 455     try:
 456         load_plugins(True)
 457     except:
 458         pass
 459
 460     entries = [e for e in caps.keys() if caps[e][CHECKED]]
 461
 462     for (k, v) in options:
 463         if k == '--capabilities':
 464             update_capabilities()
 465             print_capabilities()
 466             return 0
 467
 468         if k == '--output':
 469             if v in ['tar', 'tar.bz2', 'tar.gz', 'zip']:
 470                 output_type = v
 471             else:
 472                 print("Invalid output format '%s'" % v, file=sys.stderr)
 473                 return 2
 474
 475         # "-s" or "--silent" means suppress output (except for the final
 476         # output filename at the end)
 477         if k in ['-s', '--silent']:
 478             SILENT_MODE = True
 479
 480         if k == '--entries' and v != '':
 481             entries = v.split(',')
 482
 483         # If the user runs the script with "-y" or "--yestoall" we don't ask
 484         # all the really annoying questions.
 485         if k in ['-y', '--yestoall']:
 486             ANSWER_YES_TO_ALL = True
 487
 488         if k == '--outfd':
 489             output_fd = int(v)
 490             try:
 491                 old = fcntl.fcntl(output_fd, fcntl.F_GETFD)
 492                 fcntl.fcntl(output_fd, fcntl.F_SETFD, old | fcntl.FD_CLOEXEC)
 493             except:
 494                 print("Invalid output file descriptor", output_fd,
 495                       file=sys.stderr)
 496                 return 2
 497
 498         if k == '--outfile':
 499             output_file = v
 500
 501         elif k == '--all':
 502             entries = caps.keys()
 503         elif k == '--unlimited':
 504             unlimited_data = True
 505         elif k == '--debug':
 506             dbg = True
 507             ProcOutput.debug = True
 508
 509         if k == '--ovs':
 510             only_ovs_info = True
 511             collect_all_info = False
 512
 513         if k == '--log-days':
 514             log_days = int(v)
 515
 516         if k == '--delay':
 517             command_delay = int(v)
 518
 519     if len(params) != 1:
 520         print("Invalid additional arguments", str(params), file=sys.stderr)
 521         return 2
 522
 523     if output_fd != -1 and output_type != 'tar':
 524         print("Option '--outfd' only valid with '--output=tar'",
 525               file=sys.stderr)
 526         return 2
 527
 528     if output_fd != -1 and output_file is not None:
 529         print("Cannot set both '--outfd' and '--outfile'", file=sys.stderr)
 530         return 2
 531
 532     if output_file is not None and not unlimited_data:
 533         free_disk_space = get_free_disk_space(output_file) * 90 / 100
 534
 535     log_last_mod_time = int(time.time()) - log_days * 86400
 536
 537     if ANSWER_YES_TO_ALL:
 538         output("Warning: '--yestoall' argument provided, will not prompt "
 539                "for individual files.")
 540
 541     output('''
 542 This application will collate dmesg output, details of the
 543 hardware configuration of your machine, information about the build of
 544 openvswitch that you are using, plus, if you allow it, various logs.
 545
 546 The collated information will be saved as a .%s for archiving or
 547 sending to a Technical Support Representative.
 548
 549 The logs may contain private information, and if you are at all
 550 worried about that, you should exit now, or you should explicitly
 551 exclude those logs from the archive.
 552
 553 ''' % output_type)
 554
 555     # assemble potential data
 556
 557     file_output(CAP_BOOT_LOADER, [GRUB_CONFIG])
 558     cmd_output(CAP_BOOT_LOADER, [LS, '-lR', '/boot'])
 559     cmd_output(CAP_BOOT_LOADER, [SHA256_SUM, BOOT_KERNEL, BOOT_INITRD],
 560                label='vmlinuz-initrd.sha256sum')
 561
 562     cmd_output(CAP_DISK_INFO, [FDISK, '-l'])
 563     file_output(CAP_DISK_INFO, [PROC_PARTITIONS, PROC_MOUNTS])
 564     file_output(CAP_DISK_INFO, [FSTAB, ISCSI_CONF, ISCSI_INITIATOR])
 565     cmd_output(CAP_DISK_INFO, [DF, '-alT'])
 566     cmd_output(CAP_DISK_INFO, [DF, '-alTi'])
 567     if len(pidof('iscsid')) != 0:
 568         cmd_output(CAP_DISK_INFO, [ISCSIADM, '-m', 'node'])
 569     cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/scsi_host'])
 570     cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/scsi_disk'])
 571     cmd_output(CAP_DISK_INFO, [LS, '-R', '/sys/class/fc_transport'])
 572     cmd_output(CAP_DISK_INFO, [SG_MAP, '-x'])
 573     func_output(CAP_DISK_INFO, 'scsi-hosts', dump_scsi_hosts)
 574
 575     file_output(CAP_HARDWARE_INFO,
 576                 [PROC_CPUINFO, PROC_MEMINFO, PROC_IOPORTS, PROC_INTERRUPTS])
 577     cmd_output(CAP_HARDWARE_INFO, [DMIDECODE])
 578     cmd_output(CAP_HARDWARE_INFO, [LSPCI, '-n'])
 579     cmd_output(CAP_HARDWARE_INFO, [LSPCI, '-vv'])
 580     file_output(CAP_HARDWARE_INFO, [PROC_USB_DEV, PROC_SCSI])
 581     file_output(CAP_HARDWARE_INFO, [SYSCONFIG_HWCONF])
 582     cmd_output(CAP_HARDWARE_INFO, [LS, '-lR', '/dev'])
 583
 584     file_output(CAP_KERNEL_INFO, [PROC_VERSION, PROC_MODULES,
 585                 PROC_DEVICES, PROC_FILESYSTEMS, PROC_CMDLINE])
 586     cmd_output(CAP_KERNEL_INFO, [ZCAT, PROC_CONFIG], label='config')
 587     cmd_output(CAP_KERNEL_INFO, [SYSCTL, '-A'])
 588     file_output(CAP_KERNEL_INFO, [MODPROBE_CONF])
 589     tree_output(CAP_KERNEL_INFO, MODPROBE_DIR)
 590     func_output(CAP_KERNEL_INFO, 'modinfo', module_info)
 591
 592     cmd_output(CAP_LOSETUP_A, [LOSETUP, '-a'])
 593
 594     file_output(CAP_MULTIPATH, [MULTIPATH_CONF, MPP_CONF])
 595     cmd_output(CAP_MULTIPATH, [DMSETUP, 'table'])
 596     func_output(CAP_MULTIPATH, 'multipathd_topology', multipathd_topology)
 597     cmd_output(CAP_MULTIPATH, [MPPUTIL, '-a'])
 598     if CAP_MULTIPATH in entries and collect_all_info:
 599         dump_rdac_groups(CAP_MULTIPATH)
 600
 601     tree_output(CAP_NETWORK_CONFIG, SYSCONFIG_NETWORK_SCRIPTS, IFCFG_RE)
 602     tree_output(CAP_NETWORK_CONFIG, SYSCONFIG_NETWORK_SCRIPTS, ROUTE_RE)
 603     tree_output(CAP_NETWORK_CONFIG, INTERFACESD)
 604     file_output(CAP_NETWORK_CONFIG, [SYSCONFIG_NETWORK, RESOLV_CONF,
 605                 NSSWITCH_CONF, HOSTS, INTERFACES])
 606     file_output(CAP_NETWORK_CONFIG,
 607                 [NTP_CONF, IPTABLES_CONFIG, HOSTS_ALLOW, HOSTS_DENY])
 608     file_output(CAP_NETWORK_CONFIG, [OPENVSWITCH_DEFAULT_SWITCH,
 609                 OPENVSWITCH_SYSCONFIG_SWITCH])
 610
 611     cmd_output(CAP_NETWORK_INFO, [IP, 'addr', 'show'])
 612     cmd_output(CAP_NETWORK_INFO, [ROUTE, '-n'])
 613     cmd_output(CAP_NETWORK_INFO, [ARP, '-n'])
 614     cmd_output(CAP_NETWORK_INFO, [NETSTAT, '-an'])
 615     for dir in DHCP_LEASE_DIR:
 616         tree_output(CAP_NETWORK_INFO, dir)
 617     for table in ['filter', 'nat', 'mangle', 'raw', 'security']:
 618         cmd_output(CAP_NETWORK_INFO, [IPTABLES, '-t', table, '-nL'])
 619     for p in os.listdir('/sys/class/net/'):
 620         try:
 621             f = open('/sys/class/net/%s/type' % p, 'r')
 622             t = f.readline()
 623             f.close()
 624             if os.path.islink('/sys/class/net/%s/device' % p) and int(t) == 1:
 625                 # ARPHRD_ETHER
 626                 cmd_output(CAP_NETWORK_INFO, [ETHTOOL, '-S', p])
 627                 if not p.startswith('vif') and not p.startswith('tap'):
 628                     cmd_output(CAP_NETWORK_INFO, [ETHTOOL, p])
 629                     cmd_output(CAP_NETWORK_INFO, [ETHTOOL, '-k', p])
 630                     cmd_output(CAP_NETWORK_INFO, [ETHTOOL, '-i', p])
 631                     cmd_output(CAP_NETWORK_INFO, [ETHTOOL, '-c', p])
 632             if int(t) == 1:
 633                 cmd_output(CAP_NETWORK_INFO,
 634                            [TC, '-s', '-d', 'class', 'show', 'dev', p])
 635         except:
 636             pass
 637     tree_output(CAP_NETWORK_INFO, PROC_NET_BONDING_DIR)
 638     tree_output(CAP_NETWORK_INFO, PROC_NET_VLAN_DIR)
 639     cmd_output(CAP_NETWORK_INFO, [TC, '-s', 'qdisc'])
 640     file_output(CAP_NETWORK_INFO, [PROC_NET_SOFTNET_STAT])
 641
 642     collect_ovsdb()
 643     if os.path.exists(OPENVSWITCH_VSWITCHD_PID):
 644         cmd_output(CAP_NETWORK_STATUS, [OVS_DPCTL, 'show', '-s'])
 645         for d in dp_list():
 646             cmd_output(CAP_NETWORK_STATUS, [OVS_DPCTL, 'dump-flows', '-m', d])
 647
 648     cmd_output(CAP_PROCESS_LIST, [PS, 'wwwaxf', '-eo',
 649                'pid,tty,stat,time,nice,psr,pcpu,pmem,nwchan,wchan:25,args'],
 650                label='process-tree')
 651     func_output(CAP_PROCESS_LIST, 'fd_usage', fd_usage)
 652
 653     system_logs = ([VAR_LOG_DIR + x for x in
 654         ['crit.log', 'kern.log', 'daemon.log', 'user.log',
 655         'syslog', 'messages', 'secure', 'debug', 'dmesg', 'boot']])
 656     for log in system_logs:
 657         prefix_output(CAP_SYSTEM_LOGS, log, last_mod_time=log_last_mod_time)
 658
 659     ovs_logs = ([OPENVSWITCH_LOG_DIR + x for x in
 660         ['ovs-vswitchd.log', 'ovsdb-server.log',
 661         'ovs-xapi-sync.log', 'ovs-ctl.log']])
 662     for log in ovs_logs:
 663         prefix_output(CAP_OPENVSWITCH_LOGS, log,
 664                       last_mod_time=log_last_mod_time)
 665
 666     cmd_output(CAP_SYSTEM_LOGS, [DMESG])
 667
 668     cmd_output(CAP_SYSTEM_SERVICES, [CHKCONFIG, '--list'])
 669
 670     tree_output(CAP_SYSTEM_LOGS, VAR_LOG_CORE_DIR)
 671
 672     file_output(CAP_YUM, [YUM_LOG])
 673     tree_output(CAP_YUM, YUM_REPOS_DIR)
 674     cmd_output(CAP_YUM, [RPM, '-qa'])
 675     file_output(CAP_YUM, [APT_SOURCES_LIST])
 676     tree_output(CAP_YUM, APT_SOURCES_LIST_D)
 677     cmd_output(CAP_YUM, [DPKG_QUERY, '-W',
 678                '-f=${Package} ${Version} ${Status}\n'], 'dpkg-packages')
 679
 680     # Filter out ovs relevant information if --ovs option passed
 681     # else collect all information
 682     filters = set()
 683     if only_ovs_info:
 684         filters.add('ovs')
 685         ovs_info_caps = [CAP_NETWORK_STATUS, CAP_SYSTEM_LOGS,
 686                          CAP_OPENVSWITCH_LOGS, CAP_NETWORK_CONFIG]
 687         ovs_info_list = ['process-tree']
 688         # We cannot use iteritems, since we modify 'data' as we pass through
 689         for (k, v) in data.items():
 690             cap = v['cap']
 691             if 'filename' in v:
 692                 info = k[0]
 693             else:
 694                 info = k
 695             if info not in ovs_info_list and cap not in ovs_info_caps:
 696                 del data[k]
 697
 698     if filters:
 699         filter = ",".join(filters)
 700     else:
 701         filter = None
 702
 703     try:
 704         load_plugins(filter=filter)
 705     except:
 706         pass
 707
 708     # permit the user to filter out data
 709     # We cannot use iteritems, since we modify 'data' as we pass through
 710     for (k, v) in sorted(data.items()):
 711         cap = v['cap']
 712         if 'filename' in v:
 713             key = k[0]
 714         else:
 715             key = k
 716         if not ANSWER_YES_TO_ALL and not yes("Include '%s'? [Y/n]: " % key):
 717             del data[k]
 718
 719     # collect selected data now
 720     output_ts('Running commands to collect data')
 721     collect_data()
 722
 723     subdir = "bug-report-%s" % time.strftime("%Y%m%d%H%M%S")
 724
 725     # include inventory
 726     data['inventory.xml'] = {'cap': None,
 727                         'output': StringIOmtime(make_inventory(data, subdir))}
 728
 729     # create archive
 730     if output_fd == -1:
 731         if output_file is None:
 732             dirname = BUG_DIR
 733         else:
 734             dirname = os.path.dirname(output_file)
 735         if dirname and not os.path.exists(dirname):
 736             try:
 737                 os.makedirs(dirname)
 738             except:
 739                 pass
 740
 741     if output_fd == -1:
 742         output_ts('Creating output file')
 743
 744     if output_type.startswith('tar'):
 745         make_tar(subdir, output_type, output_fd, output_file)
 746     else:
 747         make_zip(subdir, output_file)
 748
 749     if dbg:
 750         print("Category sizes (max, actual):\n", file=sys.stderr)
 751         for c in caps.keys():
 752             print("    %s (%d, %d)" % (c, caps[c][MAX_SIZE], cap_sizes[c]),
 753                   file=sys.stderr)
 754
 755     cleanup_ovsdb()
 756     return 0
 757
 758
 759 def dump_scsi_hosts(cap):
 760     output = ''
 761     scsi_list = os.listdir('/sys/class/scsi_host')
 762     scsi_list.sort()
 763
 764     for h in scsi_list:
 765         procname = ''
 766         try:
 767             f = open('/sys/class/scsi_host/%s/proc_name' % h)
 768             procname = f.readline().strip("\n")
 769             f.close()
 770         except:
 771             pass
 772         modelname = None
 773         try:
 774             f = open('/sys/class/scsi_host/%s/model_name' % h)
 775             modelname = f.readline().strip("\n")
 776             f.close()
 777         except:
 778             pass
 779
 780         output += "%s:\n" % h
 781         output += "    %s%s\n" \
 782                   % (procname, modelname and (" -> %s" % modelname) or '')
 783
 784     return output
 785
 786
 787 def module_info(cap):
 788     output = StringIO.StringIO()
 789     modules = open(PROC_MODULES, 'r')
 790     procs = []
 791
 792     for line in modules:
 793         module = line.split()[0]
 794         procs.append(ProcOutput([MODINFO, module],
 795                      caps[cap][MAX_TIME], output))
 796     modules.close()
 797
 798     run_procs([procs])
 799
 800     return output.getvalue()
 801
 802
 803 def multipathd_topology(cap):
 804     pipe = Popen([MULTIPATHD, '-k'], bufsize=1, stdin=PIPE,
 805                      stdout=PIPE, stderr=dev_null)
 806     stdout, stderr = pipe.communicate('show topology')
 807
 808     return stdout
 809
 810
 811 def dp_list():
 812     output = StringIO.StringIO()
 813     procs = [ProcOutput([OVS_DPCTL, 'dump-dps'],
 814              caps[CAP_NETWORK_STATUS][MAX_TIME], output)]
 815
 816     run_procs([procs])
 817
 818     if not procs[0].timed_out:
 819         return output.getvalue().splitlines()
 820     return []
 821
 822
 823 def collect_ovsdb():
 824     if not os.path.isfile(OPENVSWITCH_CONF_DB):
 825         return
 826
 827     max_size = 10 * MB
 828
 829     try:
 830         if os.path.getsize(OPENVSWITCH_CONF_DB) > max_size:
 831             if os.path.isfile(OPENVSWITCH_COMPACT_DB):
 832                 os.unlink(OPENVSWITCH_COMPACT_DB)
 833
 834             output = StringIO.StringIO()
 835             max_time = 5
 836             procs = [ProcOutput(['ovsdb-tool', 'compact',
 837                                 OPENVSWITCH_CONF_DB, OPENVSWITCH_COMPACT_DB],
 838                                 max_time, output)]
 839             run_procs([procs])
 840             file_output(CAP_NETWORK_STATUS, [OPENVSWITCH_COMPACT_DB])
 841         else:
 842             file_output(CAP_NETWORK_STATUS, [OPENVSWITCH_CONF_DB])
 843     except OSError:
 844         return
 845
 846
 847 def cleanup_ovsdb():
 848     try:
 849         if os.path.isfile(OPENVSWITCH_COMPACT_DB):
 850             os.unlink(OPENVSWITCH_COMPACT_DB)
 851     except:
 852         return
 853
 854
 855 def fd_usage(cap):
 856     output = ''
 857     fd_dict = {}
 858     for d in [p for p in os.listdir('/proc') if p.isdigit()]:
 859         try:
 860             fh = open('/proc/' + d + '/cmdline')
 861             name = fh.readline()
 862             num_fds = len(os.listdir(os.path.join('/proc/' + d + '/fd')))
 863             if num_fds > 0:
 864                 if num_fds not in fd_dict:
 865                     fd_dict[num_fds] = []
 866                 fd_dict[num_fds].append(name.replace('\0', ' ').strip())
 867         finally:
 868             fh.close()
 869     keys = fd_dict.keys()
 870     keys.sort(lambda a, b: int(b) - int(a))
 871     for k in keys:
 872         output += "%s: %s\n" % (k, str(fd_dict[k]))
 873     return output
 874
 875
 876 def dump_rdac_groups(cap):
 877     output = StringIO.StringIO()
 878     procs = [ProcOutput([MPPUTIL, '-a'], caps[cap][MAX_TIME], output)]
 879
 880     run_procs([procs])
 881
 882     if not procs[0].timed_out:
 883         proc_line = 0
 884         for line in output.getvalue().splitlines():
 885             if line.startswith('ID'):
 886                 proc_line = 2
 887             elif line.startswith('----'):
 888                 proc_line -= 1
 889             elif proc_line > 0:
 890                 group, _ = line.split(None, 1)
 891                 cmd_output(cap, [MPPUTIL, '-g', group])
 892
 893
 894 def load_plugins(just_capabilities=False, filter=None):
 895     global log_last_mod_time
 896
 897     def getText(nodelist):
 898         rc = ""
 899         for node in nodelist:
 900             if node.nodeType == node.TEXT_NODE:
 901                 rc += node.data
 902         return rc.encode()
 903
 904     def getBoolAttr(el, attr, default=False):
 905         ret = default
 906         val = el.getAttribute(attr).lower()
 907         if val in ['true', 'false', 'yes', 'no']:
 908             ret = val in ['true', 'yes']
 909         return ret
 910
 911     for dir in [d for d in os.listdir(PLUGIN_DIR)
 912                 if os.path.isdir(os.path.join(PLUGIN_DIR, d))]:
 913         if dir not in caps:
 914             if not os.path.exists("%s/%s.xml" % (PLUGIN_DIR, dir)):
 915                 continue
 916             xmldoc = parse("%s/%s.xml" % (PLUGIN_DIR, dir))
 917             assert xmldoc.documentElement.tagName == "capability"
 918
 919             pii, min_size, max_size, min_time, max_time, mime = \
 920                  PII_MAYBE, -1, -1, -1, -1, MIME_TEXT
 921
 922             if xmldoc.documentElement.getAttribute("pii") in \
 923                [PII_NO, PII_YES, PII_MAYBE, PII_IF_CUSTOMIZED]:
 924                 pii = xmldoc.documentElement.getAttribute("pii")
 925             if xmldoc.documentElement.getAttribute("min_size") != '':
 926                 min_size = int(
 927                             xmldoc.documentElement.getAttribute("min_size"))
 928             if xmldoc.documentElement.getAttribute("max_size") != '':
 929                 max_size = int(
 930                             xmldoc.documentElement.getAttribute("max_size"))
 931             if xmldoc.documentElement.getAttribute("min_time") != '':
 932                 min_time = int(xmldoc.documentElement.getAttribute("min_time"))
 933             if xmldoc.documentElement.getAttribute("max_time") != '':
 934                 max_time = int(xmldoc.documentElement.getAttribute("max_time"))
 935             if xmldoc.documentElement.getAttribute("mime") in \
 936                [MIME_DATA, MIME_TEXT]:
 937                 mime = xmldoc.documentElement.getAttribute("mime")
 938             checked = getBoolAttr(xmldoc.documentElement, 'checked', True)
 939             hidden = getBoolAttr(xmldoc.documentElement, 'hidden', False)
 940
 941             cap(dir, pii, min_size, max_size, min_time, max_time, mime,
 942                 checked, hidden)
 943
 944         if just_capabilities:
 945             continue
 946
 947         plugdir = os.path.join(PLUGIN_DIR, dir)
 948         for file in [f for f in os.listdir(plugdir) if f.endswith('.xml')]:
 949             xmldoc = parse(os.path.join(plugdir, file))
 950             assert xmldoc.documentElement.tagName == "collect"
 951
 952             for el in xmldoc.documentElement.getElementsByTagName("*"):
 953                 filters_tmp = el.getAttribute("filters")
 954                 if filters_tmp == '':
 955                     filters = []
 956                 else:
 957                     filters = filters_tmp.split(',')
 958                 if not(filter is None or filter in filters):
 959                     continue
 960                 if el.tagName == "files":
 961                     newest_first = getBoolAttr(el, 'newest_first')
 962                     if el.getAttribute("type") == "logs":
 963                         for fn in getText(el.childNodes).split():
 964                             prefix_output(dir, fn, newest_first=newest_first,
 965                                           last_mod_time=log_last_mod_time)
 966                     else:
 967                         file_output(dir, getText(el.childNodes).split(),
 968                                     newest_first=newest_first)
 969                 elif el.tagName == "directory":
 970                     pattern = el.getAttribute("pattern")
 971                     if pattern == '':
 972                         pattern = None
 973                     negate = getBoolAttr(el, 'negate')
 974                     newest_first = getBoolAttr(el, 'newest_first')
 975                     if el.getAttribute("type") == "logs":
 976                         tree_output(dir, getText(el.childNodes),
 977                                     pattern and re.compile(pattern) or None,
 978                                     negate=negate, newest_first=newest_first,
 979                                     last_mod_time=log_last_mod_time)
 980                     else:
 981                         tree_output(dir, getText(el.childNodes),
 982                                     pattern and re.compile(pattern) or None,
 983                                     negate=negate, newest_first=newest_first)
 984                 elif el.tagName == "command":
 985                     label = el.getAttribute("label")
 986                     if label == '':
 987                         label = None
 988                     binary = getBoolAttr(el, 'binary')
 989                     try:
 990                         repeat_count = int(el.getAttribute("repeat"))
 991                         if repeat_count > 1:
 992                             cmd_output(dir,
 993                                        DATE + ';' + getText(el.childNodes),
 994                                        label, binary=binary,
 995                                        repeat_count=repeat_count)
 996                         else:
 997                             cmd_output(dir, getText(el.childNodes),
 998                                        label, binary=binary,
 999                                        repeat_count=repeat_count)
1000                     except:
1001                         cmd_output(dir, getText(el.childNodes), label,
1002                                    binary=binary)
1003
1004
1005 def make_tar(subdir, suffix, output_fd, output_file):
1006     global SILENT_MODE, data
1007
1008     mode = 'w'
1009     if suffix == 'tar.bz2':
1010         mode = 'w:bz2'
1011     elif suffix == 'tar.gz':
1012         mode = 'w:gz'
1013
1014     if output_fd == -1:
1015         if output_file is None:
1016             filename = "%s/%s.%s" % (BUG_DIR, subdir, suffix)
1017         else:
1018             filename = output_file
1019         old_umask = os.umask(0o077)
1020         tf = tarfile.open(filename, mode)
1021         os.umask(old_umask)
1022     else:
1023         tf = tarfile.open(None, 'w', os.fdopen(output_fd, 'a'))
1024
1025     try:
1026         for (k, v) in data.items():
1027             try:
1028                 tar_filename = os.path.join(subdir, construct_filename(k, v))
1029                 ti = tarfile.TarInfo(tar_filename)
1030
1031                 ti.uname = 'root'
1032                 ti.gname = 'root'
1033
1034                 if 'output' in v:
1035                     ti.mtime = v['output'].mtime
1036                     ti.size = len(v['output'].getvalue())
1037                     v['output'].seek(0)
1038                     tf.addfile(ti, v['output'])
1039                 elif 'filename' in v:
1040                     s = os.stat(v['filename'])
1041                     ti.mtime = s.st_mtime
1042                     ti.size = s.st_size
1043                     tf.addfile(ti, open(v['filename']))
1044             except:
1045                 pass
1046     finally:
1047         tf.close()
1048
1049     if output_fd == -1:
1050         output('Writing tarball %s successful.' % filename)
1051         if SILENT_MODE:
1052             print(filename)
1053
1054
1055 def make_zip(subdir, output_file):
1056     global SILENT_MODE, data
1057
1058     if output_file is None:
1059         filename = "%s/%s.zip" % (BUG_DIR, subdir)
1060     else:
1061         filename = output_file
1062     old_umask = os.umask(0o077)
1063     zf = zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED)
1064     os.umask(old_umask)
1065
1066     try:
1067         for (k, v) in data.items():
1068             try:
1069                 dest = os.path.join(subdir, construct_filename(k, v))
1070
1071                 if 'output' in v:
1072                     zf.writestr(dest, v['output'].getvalue())
1073                 else:
1074                     if os.stat(v['filename']).st_size < 50:
1075                         compress_type = zipfile.ZIP_STORED
1076                     else:
1077                         compress_type = zipfile.ZIP_DEFLATED
1078                     zf.write(v['filename'], dest, compress_type)
1079             except:
1080                 pass
1081     finally:
1082         zf.close()
1083
1084     output('Writing archive %s successful.' % filename)
1085     if SILENT_MODE:
1086         print(filename)
1087
1088
1089 def make_inventory(inventory, subdir):
1090     document = getDOMImplementation().createDocument(
1091         None, INVENTORY_XML_ROOT, None)
1092
1093     # create summary entry
1094     s = document.createElement(INVENTORY_XML_SUMMARY)
1095     user = os.getenv('SUDO_USER', os.getenv('USER'))
1096     if user:
1097         s.setAttribute('user', user)
1098     s.setAttribute('date', time.strftime('%c'))
1099     s.setAttribute('hostname', platform.node())
1100     s.setAttribute('uname', ' '.join(platform.uname()))
1101     s.setAttribute('uptime', commands.getoutput(UPTIME))
1102     document.getElementsByTagName(INVENTORY_XML_ROOT)[0].appendChild(s)
1103
1104     map(lambda k_v: inventory_entry(document, subdir, k_v[0], k_v[1]),
1105         inventory.items())
1106     return document.toprettyxml()
1107
1108
1109 def inventory_entry(document, subdir, k, v):
1110     try:
1111         el = document.createElement(INVENTORY_XML_ELEMENT)
1112         el.setAttribute('capability', v['cap'])
1113         el.setAttribute('filename',
1114                         os.path.join(subdir, construct_filename(k, v)))
1115         el.setAttribute('sha256sum', sha256(v))
1116         document.getElementsByTagName(INVENTORY_XML_ROOT)[0].appendChild(el)
1117     except:
1118         pass
1119
1120
1121 def sha256(d):
1122     m = hashlib.sha256()
1123     if 'filename' in d:
1124         f = open(d['filename'])
1125         data = f.read(1024)
1126         while len(data) > 0:
1127             m.update(data)
1128             data = f.read(1024)
1129         f.close()
1130     elif 'output' in d:
1131         m.update(d['output'].getvalue())
1132     return m.hexdigest()
1133
1134
1135 def construct_filename(k, v):
1136     if 'filename' in v:
1137         if v['filename'][0] == '/':
1138             return v['filename'][1:]
1139         else:
1140             return v['filename']
1141     s = k.replace(' ', '-')
1142     s = s.replace('--', '-')
1143     s = s.replace('/', '%')
1144     if s.find('.') == -1:
1145         s += '.out'
1146
1147     return s
1148
1149
1150 def update_capabilities():
1151     pass
1152
1153
1154 def update_cap_size(cap, size):
1155     update_cap(cap, MIN_SIZE, size)
1156     update_cap(cap, MAX_SIZE, size)
1157     update_cap(cap, CHECKED, size > 0)
1158
1159
1160 def update_cap(cap, k, v):
1161     global caps
1162     temp = list(caps[cap])
1163     temp[k] = v
1164     caps[cap] = tuple(temp)
1165
1166
1167 def size_of_dir(d, pattern=None, negate=False):
1168     if os.path.isdir(d):
1169         return size_of_all([os.path.join(d, fn) for fn in os.listdir(d)],
1170                            pattern, negate)
1171     else:
1172         return 0
1173
1174
1175 def size_of_all(files, pattern=None, negate=False):
1176     return sum([size_of(f, pattern, negate) for f in files])
1177
1178
1179 def matches(f, pattern, negate):
1180     if negate:
1181         return not matches(f, pattern, False)
1182     else:
1183         return pattern is None or pattern.match(f)
1184
1185
1186 def size_of(f, pattern, negate):
1187     if os.path.isfile(f) and matches(f, pattern, negate):
1188         return os.stat(f)[6]
1189     else:
1190         return size_of_dir(f, pattern, negate)
1191
1192
1193 def print_capabilities():
1194     document = getDOMImplementation().createDocument(
1195         "ns", CAP_XML_ROOT, None)
1196     map(lambda key: capability(document, key),
1197         [k for k in caps.keys() if not caps[k][HIDDEN]])
1198     print(document.toprettyxml())
1199
1200
1201 def capability(document, key):
1202     c = caps[key]
1203     el = document.createElement(CAP_XML_ELEMENT)
1204     el.setAttribute('key', c[KEY])
1205     el.setAttribute('pii', c[PII])
1206     el.setAttribute('min-size', str(c[MIN_SIZE]))
1207     el.setAttribute('max-size', str(c[MAX_SIZE]))
1208     el.setAttribute('min-time', str(c[MIN_TIME]))
1209     el.setAttribute('max-time', str(c[MAX_TIME]))
1210     el.setAttribute('content-type', c[MIME])
1211     el.setAttribute('default-checked', c[CHECKED] and 'yes' or 'no')
1212     document.getElementsByTagName(CAP_XML_ROOT)[0].appendChild(el)
1213
1214
1215 def prettyDict(d):
1216     format = '%%-%ds: %%s' % max(map(len, [k for k, _ in d.items()]))
1217     return '\n'.join([format % i for i in d.items()]) + '\n'
1218
1219
1220 def yes(prompt):
1221     yn = input(prompt)
1222
1223     return len(yn) == 0 or yn.lower()[0] == 'y'
1224
1225
1226 partition_re = re.compile(r'(.*[0-9]+$)|(^xvd)')
1227
1228
1229 def disk_list():
1230     disks = []
1231     try:
1232         f = open('/proc/partitions')
1233         f.readline()
1234         f.readline()
1235         for line in f.readlines():
1236             (major, minor, blocks, name) = line.split()
1237             if int(major) < 254 and not partition_re.match(name):
1238                 disks.append(name)
1239         f.close()
1240     except:
1241         pass
1242     return disks
1243
1244
1245 class ProcOutput(object):
1246     debug = False
1247
1248     def __init__(self, command, max_time, inst=None, filter=None,
1249                  binary=False):
1250         self.command = command
1251         self.max_time = max_time
1252         self.inst = inst
1253         self.running = False
1254         self.status = None
1255         self.timed_out = False
1256         self.failed = False
1257         self.timeout = int(time.time()) + self.max_time
1258         self.filter = filter
1259         self.filter_state = {}
1260         if binary:
1261             self.bufsize = 1048576  # 1MB buffer
1262         else:
1263             self.bufsize = 1        # line buffered
1264
1265     def __del__(self):
1266         self.terminate()
1267
1268     def cmdAsStr(self):
1269         return isinstance(self.command, list) \
1270                 and ' '.join(self.command) or self.command
1271
1272     def run(self):
1273         self.timed_out = False
1274         try:
1275             if ProcOutput.debug:
1276                 output_ts("Starting '%s'" % self.cmdAsStr())
1277             self.proc = Popen(self.command, bufsize=self.bufsize,
1278                               stdin=dev_null, stdout=PIPE, stderr=dev_null,
1279                               shell=isinstance(self.command, str))
1280             old = fcntl.fcntl(self.proc.stdout.fileno(), fcntl.F_GETFD)
1281             fcntl.fcntl(self.proc.stdout.fileno(),
1282                         fcntl.F_SETFD, old | fcntl.FD_CLOEXEC)
1283             self.running = True
1284             self.failed = False
1285         except:
1286             output_ts("'%s' failed" % self.cmdAsStr())
1287             self.running = False
1288             self.failed = True
1289
1290     def terminate(self):
1291         if self.running:
1292             try:
1293                 self.proc.stdout.close()
1294                 os.kill(self.proc.pid, SIGTERM)
1295             except:
1296                 pass
1297             self.proc = None
1298             self.running = False
1299             self.status = SIGTERM
1300
1301     def read_line(self):
1302         assert self.running
1303         if self.bufsize == 1:
1304             line = self.proc.stdout.readline()
1305         else:
1306             line = self.proc.stdout.read(self.bufsize)
1307         if line == '':
1308             # process exited
1309             self.proc.stdout.close()
1310             self.status = self.proc.wait()
1311             self.proc = None
1312             self.running = False
1313         else:
1314             if self.filter:
1315                 line = self.filter(line, self.filter_state)
1316             if self.inst:
1317                 self.inst.write(line)
1318
1319
1320 def run_procs(procs):
1321     while True:
1322         pipes = []
1323         active_procs = []
1324
1325         for pp in procs:
1326             for p in pp:
1327                 if p.running:
1328                     active_procs.append(p)
1329                     pipes.append(p.proc.stdout)
1330                     break
1331                 elif p.status is None and not p.failed and not p.timed_out:
1332                     p.run()
1333                     if p.running:
1334                         active_procs.append(p)
1335                         pipes.append(p.proc.stdout)
1336                         break
1337
1338         if len(pipes) == 0:
1339             # all finished
1340             break
1341
1342         (i, o, x) = select(pipes, [], [], 1.0)
1343         now = int(time.time())
1344
1345         # handle process output
1346         for p in active_procs:
1347             if p.proc.stdout in i:
1348                 p.read_line()
1349
1350             # handle timeout
1351             if p.running and now > p.timeout:
1352                 output_ts("'%s' timed out" % p.cmdAsStr())
1353                 if p.inst:
1354                     p.inst.write("\n** timeout **\n")
1355                 p.timed_out = True
1356                 p.terminate()
1357
1358
1359 def pidof(name):
1360     pids = []
1361
1362     for d in [p for p in os.listdir('/proc') if p.isdigit()]:
1363         try:
1364             if os.path.basename(os.readlink('/proc/%s/exe' % d)) == name:
1365                 pids.append(int(d))
1366         except:
1367             pass
1368
1369     return pids
1370
1371
1372 def check_space(cap, name, size):
1373     global free_disk_space
1374     if free_disk_space is not None and size > free_disk_space:
1375         output("Omitting %s, out of disk space (requested: %u, allowed: %u)" %
1376                (name, size, free_disk_space))
1377         return False
1378     elif unlimited_data or caps[cap][MAX_SIZE] == -1 or \
1379              cap_sizes[cap] < caps[cap][MAX_SIZE]:
1380         cap_sizes[cap] += size
1381         if free_disk_space is not None:
1382             free_disk_space -= size
1383         return True
1384     else:
1385         output("Omitting %s, size constraint of %s exceeded" % (name, cap))
1386         return False
1387
1388
1389 def get_free_disk_space(path):
1390     path = os.path.abspath(path)
1391     while not os.path.exists(path):
1392         path = os.path.dirname(path)
1393     s = os.statvfs(path)
1394     return s.f_frsize * s.f_bfree
1395
1396
1397 class StringIOmtime(StringIO.StringIO):
1398     def __init__(self, buf=''):
1399         StringIO.StringIO.__init__(self, buf)
1400         self.mtime = time.time()
1401
1402     def write(self, s):
1403         StringIO.StringIO.write(self, s)
1404         self.mtime = time.time()
1405
1406
1407 if __name__ == "__main__":
1408     try:
1409         sys.exit(main())
1410     except KeyboardInterrupt:
1411         print("\nInterrupted.")
1412         sys.exit(3)