ceph/src/ceph-volume/ceph_volume/process.py

   1 from fcntl import fcntl, F_GETFL, F_SETFL
   2 from os import O_NONBLOCK, read
   3 import subprocess
   4 from select import select
   5 from ceph_volume import terminal
   6
   7 import logging
   8
   9 logger = logging.getLogger(__name__)
  10
  11
  12 def log_output(descriptor, message, terminal_logging):
  13     """
  14     log output to both the logger and the terminal if terminal_logging is
  15     enabled
  16     """
  17     if not message:
  18         return
  19     message = message.strip()
  20     line = '%s %s' % (descriptor, message)
  21     if terminal_logging:
  22         getattr(terminal, descriptor)(message)
  23     logger.info(line)
  24
  25
  26 def log_descriptors(reads, process, terminal_logging):
  27     """
  28     Helper to send output to the terminal while polling the subprocess
  29     """
  30     # these fcntl are set to O_NONBLOCK for the filedescriptors coming from
  31     # subprocess so that the logging does not block. Without these a prompt in
  32     # a subprocess output would hang and nothing would get printed. Note how
  33     # these are just set when logging subprocess, not globally.
  34     stdout_flags = fcntl(process.stdout, F_GETFL) # get current p.stdout flags
  35     stderr_flags = fcntl(process.stderr, F_GETFL) # get current p.stderr flags
  36     fcntl(process.stdout, F_SETFL, stdout_flags | O_NONBLOCK)
  37     fcntl(process.stderr, F_SETFL, stderr_flags | O_NONBLOCK)
  38     descriptor_names = {
  39         process.stdout.fileno(): 'stdout',
  40         process.stderr.fileno(): 'stderr'
  41     }
  42     for descriptor in reads:
  43         descriptor_name = descriptor_names[descriptor]
  44         try:
  45             log_output(descriptor_name, read(descriptor, 1024), terminal_logging)
  46         except (IOError, OSError):
  47             # nothing else to log
  48             pass
  49
  50
  51 def run(command, **kw):
  52     """
  53     A real-time-logging implementation of a remote subprocess.Popen call where
  54     a command is just executed on the remote end and no other handling is done.
  55
  56     :param command: The command to pass in to the remote subprocess.Popen as a list
  57     :param stop_on_error: If a nonzero exit status is return, it raises a ``RuntimeError``
  58     """
  59     stop_on_error = kw.pop('stop_on_error', True)
  60     command_msg = "Running command: %s" % ' '.join(command)
  61     stdin = kw.pop('stdin', None)
  62     logger.info(command_msg)
  63     terminal.write(command_msg)
  64     terminal_logging = kw.pop('terminal_logging', True)
  65
  66     process = subprocess.Popen(
  67         command,
  68         stdin=subprocess.PIPE,
  69         stdout=subprocess.PIPE,
  70         stderr=subprocess.PIPE,
  71         close_fds=True,
  72         **kw
  73     )
  74
  75     if stdin:
  76         process.communicate(stdin)
  77     while True:
  78         reads, _, _ = select(
  79             [process.stdout.fileno(), process.stderr.fileno()],
  80             [], []
  81         )
  82         log_descriptors(reads, process, terminal_logging)
  83
  84         if process.poll() is not None:
  85             # ensure we do not have anything pending in stdout or stderr
  86             log_descriptors(reads, process, terminal_logging)
  87
  88             break
  89
  90     returncode = process.wait()
  91     if returncode != 0:
  92         msg = "command returned non-zero exit status: %s" % returncode
  93         if stop_on_error:
  94             raise RuntimeError(msg)
  95         else:
  96             if terminal_logging:
  97                 terminal.warning(msg)
  98             logger.warning(msg)
  99
 100
 101 def call(command, **kw):
 102     """
 103     Similar to ``subprocess.Popen`` with the following changes:
 104
 105     * returns stdout, stderr, and exit code (vs. just the exit code)
 106     * logs the full contents of stderr and stdout (separately) to the file log
 107
 108     By default, no terminal output is given, not even the command that is going
 109     to run.
 110
 111     Useful when system calls are needed to act on output, and that same output
 112     shouldn't get displayed on the terminal.
 113
 114     :param terminal_verbose: Log command output to terminal, defaults to False, and
 115                              it is forcefully set to True if a return code is non-zero
 116     """
 117     terminal_verbose = kw.pop('terminal_verbose', False)
 118     command_msg = "Running command: %s" % ' '.join(command)
 119     stdin = kw.pop('stdin', None)
 120     logger.info(command_msg)
 121     terminal.write(command_msg)
 122
 123     process = subprocess.Popen(
 124         command,
 125         stdout=subprocess.PIPE,
 126         stderr=subprocess.PIPE,
 127         stdin=subprocess.PIPE,
 128         close_fds=True,
 129         **kw
 130     )
 131     if stdin:
 132         stdout_stream, stderr_stream = process.communicate(stdin)
 133     else:
 134         stdout_stream = process.stdout.read()
 135         stderr_stream = process.stderr.read()
 136     returncode = process.wait()
 137     if not isinstance(stdout_stream, str):
 138         stdout_stream = stdout_stream.decode('utf-8')
 139     if not isinstance(stderr_stream, str):
 140         stderr_stream = stderr_stream.decode('utf-8')
 141     stdout = stdout_stream.splitlines()
 142     stderr = stderr_stream.splitlines()
 143
 144     if returncode != 0:
 145         # set to true so that we can log the stderr/stdout that callers would
 146         # do anyway
 147         terminal_verbose = True
 148
 149     # the following can get a messed up order in the log if the system call
 150     # returns output with both stderr and stdout intermingled. This separates
 151     # that.
 152     for line in stdout:
 153         log_output('stdout', line, terminal_verbose)
 154     for line in stderr:
 155         log_output('stderr', line, terminal_verbose)
 156     return stdout, stderr, returncode