ceph/src/ceph-volume/ceph_volume/util/system.py

   1 import errno
   2 import logging
   3 import os
   4 import pwd
   5 import platform
   6 import tempfile
   7 import uuid
   8 import subprocess
   9 import threading
  10 from ceph_volume import process, terminal
  11 from . import as_string
  12
  13 # python2 has no FileNotFoundError
  14 try:
  15     FileNotFoundError
  16 except NameError:
  17     FileNotFoundError = OSError
  18
  19 logger = logging.getLogger(__name__)
  20 mlogger = terminal.MultiLogger(__name__)
  21
  22 # TODO: get these out of here and into a common area for others to consume
  23 if platform.system() == 'FreeBSD':
  24     FREEBSD = True
  25     DEFAULT_FS_TYPE = 'zfs'
  26     PROCDIR = '/compat/linux/proc'
  27     # FreeBSD does not have blockdevices any more
  28     BLOCKDIR = '/dev'
  29     ROOTGROUP = 'wheel'
  30 else:
  31     FREEBSD = False
  32     DEFAULT_FS_TYPE = 'xfs'
  33     PROCDIR = '/proc'
  34     BLOCKDIR = '/sys/block'
  35     ROOTGROUP = 'root'
  36
  37 host_rootfs = '/rootfs'
  38 run_host_cmd = [
  39         'nsenter',
  40         '--mount={}/proc/1/ns/mnt'.format(host_rootfs),
  41         '--ipc={}/proc/1/ns/ipc'.format(host_rootfs),
  42         '--net={}/proc/1/ns/net'.format(host_rootfs),
  43         '--uts={}/proc/1/ns/uts'.format(host_rootfs)
  44 ]
  45
  46 def generate_uuid():
  47     return str(uuid.uuid4())
  48
  49 def find_executable_on_host(locations=[], executable='', binary_check='/bin/ls'):
  50     paths = ['{}/{}'.format(location, executable) for location in locations]
  51     command = []
  52     command.extend(run_host_cmd + [binary_check] + paths)
  53     process = subprocess.Popen(
  54         command,
  55         stdout=subprocess.PIPE,
  56         stderr=subprocess.PIPE,
  57         stdin=subprocess.PIPE,
  58         close_fds=True
  59     )
  60     stdout = as_string(process.stdout.read())
  61     if stdout:
  62         executable_on_host = stdout.split('\n')[0]
  63         logger.info('Executable {} found on the host, will use {}'.format(executable, executable_on_host))
  64         return executable_on_host
  65     else:
  66         logger.warning('Executable {} not found on the host, will return {} as-is'.format(executable, executable))
  67         return executable
  68
  69 def which(executable, run_on_host=False):
  70     """find the location of an executable"""
  71     def _get_path(executable, locations):
  72         for location in locations:
  73             executable_path = os.path.join(location, executable)
  74             if os.path.exists(executable_path) and os.path.isfile(executable_path):
  75                 return executable_path
  76         return None
  77
  78     static_locations = (
  79         '/usr/local/bin',
  80         '/bin',
  81         '/usr/bin',
  82         '/usr/local/sbin',
  83         '/usr/sbin',
  84         '/sbin',
  85     )
  86
  87     if not run_on_host:
  88         path = os.getenv('PATH', '')
  89         path_locations = path.split(':')
  90         exec_in_path = _get_path(executable, path_locations)
  91         if exec_in_path:
  92             return exec_in_path
  93         mlogger.warning('Executable {} not in PATH: {}'.format(executable, path))
  94
  95         exec_in_static_locations = _get_path(executable, static_locations)
  96         if exec_in_static_locations:
  97             mlogger.warning('Found executable under {}, please ensure $PATH is set correctly!'.format(exec_in_static_locations))
  98             return exec_in_static_locations
  99     else:
 100         executable = find_executable_on_host(static_locations, executable)
 101
 102     # At this point, either `find_executable_on_host()` found an executable on the host
 103     # or we fallback to just returning the argument as-is, to prevent a hard fail, and
 104     # hoping that the system might have the executable somewhere custom
 105     return executable
 106
 107 def get_ceph_user_ids():
 108     """
 109     Return the id and gid of the ceph user
 110     """
 111     try:
 112         user = pwd.getpwnam('ceph')
 113     except KeyError:
 114         # is this even possible?
 115         raise RuntimeError('"ceph" user is not available in the current system')
 116     return user[2], user[3]
 117
 118
 119 def get_file_contents(path, default=''):
 120     contents = default
 121     if not os.path.exists(path):
 122         return contents
 123     try:
 124         with open(path, 'r') as open_file:
 125             contents = open_file.read().strip()
 126     except Exception:
 127         logger.exception('Failed to read contents from: %s' % path)
 128
 129     return contents
 130
 131
 132 def mkdir_p(path, chown=True):
 133     """
 134     A `mkdir -p` that defaults to chown the path to the ceph user
 135     """
 136     try:
 137         os.mkdir(path)
 138     except OSError as e:
 139         if e.errno == errno.EEXIST:
 140             pass
 141         else:
 142             raise
 143     if chown:
 144         uid, gid = get_ceph_user_ids()
 145         os.chown(path, uid, gid)
 146
 147
 148 def chown(path, recursive=True):
 149     """
 150     ``chown`` a path to the ceph user (uid and guid fetched at runtime)
 151     """
 152     uid, gid = get_ceph_user_ids()
 153     if os.path.islink(path):
 154         process.run(['chown', '-h', 'ceph:ceph', path])
 155         path = os.path.realpath(path)
 156     if recursive:
 157         process.run(['chown', '-R', 'ceph:ceph', path])
 158     else:
 159         os.chown(path, uid, gid)
 160
 161
 162 def is_binary(path):
 163     """
 164     Detect if a file path is a binary or not. Will falsely report as binary
 165     when utf-16 encoded. In the ceph universe there is no such risk (yet)
 166     """
 167     with open(path, 'rb') as fp:
 168         contents = fp.read(8192)
 169     if b'\x00' in contents:  # a null byte may signal binary
 170         return True
 171     return False
 172
 173
 174 class tmp_mount(object):
 175     """
 176     Temporarily mount a device on a temporary directory,
 177     and unmount it upon exit
 178
 179     When ``encrypted`` is set to ``True``, the exit method will call out to
 180     close the device so that it doesn't remain open after mounting. It is
 181     assumed that it will be open because otherwise it wouldn't be possible to
 182     mount in the first place
 183     """
 184
 185     def __init__(self, device, encrypted=False):
 186         self.device = device
 187         self.path = None
 188         self.encrypted = encrypted
 189
 190     def __enter__(self):
 191         self.path = tempfile.mkdtemp()
 192         process.run([
 193             'mount',
 194             '-v',
 195             self.device,
 196             self.path
 197         ])
 198         return self.path
 199
 200     def __exit__(self, exc_type, exc_val, exc_tb):
 201         process.run([
 202             'umount',
 203             '-v',
 204             self.path
 205         ])
 206         if self.encrypted:
 207             # avoid a circular import from the encryption module
 208             from ceph_volume.util import encryption
 209             encryption.dmcrypt_close(self.device)
 210
 211
 212 def unmount_tmpfs(path):
 213     """
 214     Removes the mount at the given path iff the path is a tmpfs mount point.
 215     Otherwise no action is taken.
 216     """
 217     _out, _err, rc = process.call(['findmnt', '-t', 'tmpfs', '-M', path])
 218     if rc != 0:
 219         logger.info('{} does not appear to be a tmpfs mount'.format(path))
 220     else:
 221         logger.info('Unmounting tmpfs path at {}'.format( path))
 222         unmount(path)
 223
 224
 225 def unmount(path):
 226     """
 227     Removes mounts at the given path
 228     """
 229     process.run([
 230         'umount',
 231         '-v',
 232         path,
 233     ])
 234
 235
 236 def path_is_mounted(path, destination=None):
 237     """
 238     Check if the given path is mounted
 239     """
 240     m = Mounts(paths=True)
 241     mounts = m.get_mounts()
 242     realpath = os.path.realpath(path)
 243     mounted_locations = mounts.get(realpath, [])
 244
 245     if destination:
 246         return destination in mounted_locations
 247     return mounted_locations != []
 248
 249
 250 def device_is_mounted(dev, destination=None):
 251     """
 252     Check if the given device is mounted, optionally validating that a
 253     destination exists
 254     """
 255     plain_mounts = Mounts(devices=True)
 256     realpath_mounts = Mounts(devices=True, realpath=True)
 257
 258     realpath_dev = os.path.realpath(dev) if dev.startswith('/') else dev
 259     destination = os.path.realpath(destination) if destination else None
 260     # plain mounts
 261     plain_dev_mounts = plain_mounts.get_mounts().get(dev, [])
 262     realpath_dev_mounts = plain_mounts.get_mounts().get(realpath_dev, [])
 263     # realpath mounts
 264     plain_dev_real_mounts = realpath_mounts.get_mounts().get(dev, [])
 265     realpath_dev_real_mounts = realpath_mounts.get_mounts().get(realpath_dev, [])
 266
 267     mount_locations = [
 268         plain_dev_mounts,
 269         realpath_dev_mounts,
 270         plain_dev_real_mounts,
 271         realpath_dev_real_mounts
 272     ]
 273
 274     for mounts in mount_locations:
 275         if mounts: # we have a matching mount
 276             if destination:
 277                 if destination in mounts:
 278                     logger.info(
 279                         '%s detected as mounted, exists at destination: %s', dev, destination
 280                     )
 281                     return True
 282             else:
 283                 logger.info('%s was found as mounted', dev)
 284                 return True
 285     logger.info('%s was not found as mounted', dev)
 286     return False
 287
 288 class Mounts(object):
 289     excluded_paths = []
 290
 291     def __init__(self, devices=False, paths=False, realpath=False):
 292         self.devices = devices
 293         self.paths = paths
 294         self.realpath = realpath
 295
 296     def safe_realpath(self, path, timeout=0.2):
 297         def _realpath(path, result):
 298             p = os.path.realpath(path)
 299             result.append(p)
 300
 301         result = []
 302         t = threading.Thread(target=_realpath, args=(path, result))
 303         t.setDaemon(True)
 304         t.start()
 305         t.join(timeout)
 306         if t.is_alive():
 307             return None
 308         return result[0]
 309
 310     def get_mounts(self):
 311         """
 312         Create a mapping of all available system mounts so that other helpers can
 313         detect nicely what path or device is mounted
 314
 315         It ignores (most of) non existing devices, but since some setups might need
 316         some extra device information, it will make an exception for:
 317
 318         - tmpfs
 319         - devtmpfs
 320         - /dev/root
 321
 322         If ``devices`` is set to ``True`` the mapping will be a device-to-path(s),
 323         if ``paths`` is set to ``True`` then the mapping will be
 324         a path-to-device(s)
 325
 326         :param realpath: Resolve devices to use their realpaths. This is useful for
 327         paths like LVM where more than one path can point to the same device
 328         """
 329         devices_mounted = {}
 330         paths_mounted = {}
 331         do_not_skip = ['tmpfs', 'devtmpfs', '/dev/root']
 332         default_to_devices = self.devices is False and self.paths is False
 333
 334
 335         with open(PROCDIR + '/mounts', 'rb') as mounts:
 336             proc_mounts = mounts.readlines()
 337
 338         for line in proc_mounts:
 339             fields = [as_string(f) for f in line.split()]
 340             if len(fields) < 3:
 341                 continue
 342             if fields[0] in Mounts.excluded_paths or \
 343                 fields[1] in Mounts.excluded_paths:
 344                 continue
 345             if self.realpath:
 346                 if fields[0].startswith('/'):
 347                     device = self.safe_realpath(fields[0])
 348                     if device is None:
 349                         logger.warning(f"Can't get realpath on {fields[0]}, skipping.")
 350                         Mounts.excluded_paths.append(fields[0])
 351                         continue
 352                 else:
 353                     device = fields[0]
 354             else:
 355                 device = fields[0]
 356             path = self.safe_realpath(fields[1])
 357             if path is None:
 358                 logger.warning(f"Can't get realpath on {fields[1]}, skipping.")
 359                 Mounts.excluded_paths.append(fields[1])
 360                 continue
 361             # only care about actual existing devices
 362             if not os.path.exists(device) or not device.startswith('/'):
 363                 if device not in do_not_skip:
 364                     continue
 365             if device in devices_mounted.keys():
 366                 devices_mounted[device].append(path)
 367             else:
 368                 devices_mounted[device] = [path]
 369             if path in paths_mounted.keys():
 370                 paths_mounted[path].append(device)
 371             else:
 372                 paths_mounted[path] = [device]
 373
 374         # Default to returning information for devices if
 375         if self.devices is True or default_to_devices:
 376             return devices_mounted
 377         else:
 378             return paths_mounted
 379
 380
 381 def set_context(path, recursive=False):
 382     """
 383     Calls ``restorecon`` to set the proper context on SELinux systems. Only if
 384     the ``restorecon`` executable is found anywhere in the path it will get
 385     called.
 386
 387     If the ``CEPH_VOLUME_SKIP_RESTORECON`` environment variable is set to
 388     any of: "1", "true", "yes" the call will be skipped as well.
 389
 390     Finally, if SELinux is not enabled, or not available in the system,
 391     ``restorecon`` will not be called. This is checked by calling out to the
 392     ``selinuxenabled`` executable. If that tool is not installed or returns
 393     a non-zero exit status then no further action is taken and this function
 394     will return.
 395     """
 396     skip = os.environ.get('CEPH_VOLUME_SKIP_RESTORECON', '')
 397     if skip.lower() in ['1', 'true', 'yes']:
 398         logger.info(
 399             'CEPH_VOLUME_SKIP_RESTORECON environ is set, will not call restorecon'
 400         )
 401         return
 402
 403     try:
 404         stdout, stderr, code = process.call(['selinuxenabled'],
 405                                             verbose_on_failure=False)
 406     except FileNotFoundError:
 407         logger.info('No SELinux found, skipping call to restorecon')
 408         return
 409
 410     if code != 0:
 411         logger.info('SELinux is not enabled, will not call restorecon')
 412         return
 413
 414     # restore selinux context to default policy values
 415     if which('restorecon').startswith('/'):
 416         if recursive:
 417             process.run(['restorecon', '-R', path])
 418         else:
 419             process.run(['restorecon', path])