1 from fcntl
import fcntl
, F_GETFL
, F_SETFL
2 from os
import O_NONBLOCK
, read
4 from select
import select
5 from ceph_volume
import terminal
9 logger
= logging
.getLogger(__name__
)
12 def log_output(descriptor
, message
, terminal_logging
, logfile_logging
):
14 log output to both the logger and the terminal if terminal_logging is
19 message
= message
.strip()
20 line
= '%s %s' % (descriptor
, message
)
22 getattr(terminal
, descriptor
)(message
)
27 def log_descriptors(reads
, process
, terminal_logging
):
29 Helper to send output to the terminal while polling the subprocess
31 # these fcntl are set to O_NONBLOCK for the filedescriptors coming from
32 # subprocess so that the logging does not block. Without these a prompt in
33 # a subprocess output would hang and nothing would get printed. Note how
34 # these are just set when logging subprocess, not globally.
35 stdout_flags
= fcntl(process
.stdout
, F_GETFL
) # get current p.stdout flags
36 stderr_flags
= fcntl(process
.stderr
, F_GETFL
) # get current p.stderr flags
37 fcntl(process
.stdout
, F_SETFL
, stdout_flags | O_NONBLOCK
)
38 fcntl(process
.stderr
, F_SETFL
, stderr_flags | O_NONBLOCK
)
40 process
.stdout
.fileno(): 'stdout',
41 process
.stderr
.fileno(): 'stderr'
43 for descriptor
in reads
:
44 descriptor_name
= descriptor_names
[descriptor
]
46 log_output(descriptor_name
, read(descriptor
, 1024), terminal_logging
, True)
47 except (IOError, OSError):
52 def obfuscate(command_
, on
=None):
54 Certain commands that are useful to log might contain information that
55 should be replaced by '*' like when creating OSDs and the keyryings are
56 being passed, which should not be logged.
58 :param on: A string (will match a flag) or an integer (will match an index)
60 If matching on a flag (when ``on`` is a string) it will obfuscate on the
61 value for that flag. That is a command like ['ls', '-l', '/'] that calls
62 `obfuscate(command, on='-l')` will obfustace '/' which is the value for
65 The reason for `on` to allow either a string or an integer, altering
66 behavior for both is because it is easier for ``run`` and ``call`` to just
67 pop a value to obfuscate (vs. allowing an index or a flag)
70 msg
= "Running command: %s" % ' '.join(command
)
71 if on
in [None, False]:
74 if isinstance(on
, int):
79 index
= command
.index(on
) + 1
81 # if the flag just doesn't exist then it doesn't matter just return
86 command
[index
] = '*' * len(command
[index
])
87 except IndexError: # the index was completely out of range
90 return "Running command: %s" % ' '.join(command
)
93 def run(command
, **kw
):
95 A real-time-logging implementation of a remote subprocess.Popen call where
96 a command is just executed on the remote end and no other handling is done.
98 :param command: The command to pass in to the remote subprocess.Popen as a list
99 :param stop_on_error: If a nonzero exit status is return, it raises a ``RuntimeError``
100 :param fail_msg: If a nonzero exit status is returned this message will be included in the log
102 stop_on_error
= kw
.pop('stop_on_error', True)
103 command_msg
= obfuscate(command
, kw
.pop('obfuscate', None))
104 fail_msg
= kw
.pop('fail_msg', None)
105 logger
.info(command_msg
)
106 terminal
.write(command_msg
)
107 terminal_logging
= kw
.pop('terminal_logging', True)
109 process
= subprocess
.Popen(
111 stdout
=subprocess
.PIPE
,
112 stderr
=subprocess
.PIPE
,
118 reads
, _
, _
= select(
119 [process
.stdout
.fileno(), process
.stderr
.fileno()],
122 log_descriptors(reads
, process
, terminal_logging
)
124 if process
.poll() is not None:
125 # ensure we do not have anything pending in stdout or stderr
126 log_descriptors(reads
, process
, terminal_logging
)
130 returncode
= process
.wait()
132 msg
= "command returned non-zero exit status: %s" % returncode
134 logger
.warning(fail_msg
)
136 terminal
.warning(fail_msg
)
138 raise RuntimeError(msg
)
141 terminal
.warning(msg
)
145 def call(command
, **kw
):
147 Similar to ``subprocess.Popen`` with the following changes:
149 * returns stdout, stderr, and exit code (vs. just the exit code)
150 * logs the full contents of stderr and stdout (separately) to the file log
152 By default, no terminal output is given, not even the command that is going
155 Useful when system calls are needed to act on output, and that same output
156 shouldn't get displayed on the terminal.
158 Optionally, the command can be displayed on the terminal and the log file,
159 and log file output can be turned off. This is useful to prevent sensitive
160 output going to stderr/stdout and being captured on a log file.
162 :param terminal_verbose: Log command output to terminal, defaults to False, and
163 it is forcefully set to True if a return code is non-zero
164 :param logfile_verbose: Log stderr/stdout output to log file. Defaults to True
166 terminal_verbose
= kw
.pop('terminal_verbose', False)
167 logfile_verbose
= kw
.pop('logfile_verbose', True)
168 show_command
= kw
.pop('show_command', False)
169 command_msg
= "Running command: %s" % ' '.join(command
)
170 stdin
= kw
.pop('stdin', None)
171 logger
.info(command_msg
)
173 terminal
.write(command_msg
)
175 process
= subprocess
.Popen(
177 stdout
=subprocess
.PIPE
,
178 stderr
=subprocess
.PIPE
,
179 stdin
=subprocess
.PIPE
,
184 stdout_stream
, stderr_stream
= process
.communicate(stdin
)
186 stdout_stream
= process
.stdout
.read()
187 stderr_stream
= process
.stderr
.read()
188 returncode
= process
.wait()
189 if not isinstance(stdout_stream
, str):
190 stdout_stream
= stdout_stream
.decode('utf-8')
191 if not isinstance(stderr_stream
, str):
192 stderr_stream
= stderr_stream
.decode('utf-8')
193 stdout
= stdout_stream
.splitlines()
194 stderr
= stderr_stream
.splitlines()
197 # set to true so that we can log the stderr/stdout that callers would
199 terminal_verbose
= True
200 logfile_verbose
= True
202 # the following can get a messed up order in the log if the system call
203 # returns output with both stderr and stdout intermingled. This separates
206 log_output('stdout', line
, terminal_verbose
, logfile_verbose
)
208 log_output('stderr', line
, terminal_verbose
, logfile_verbose
)
209 return stdout
, stderr
, returncode