ceph/qa/tasks/cephfs/kernel_mount.py

   1 from StringIO import StringIO
   2 import json
   3 import logging
   4 from textwrap import dedent
   5 from teuthology.orchestra.run import CommandFailedError
   6 from teuthology import misc
   7
   8 from teuthology.orchestra import remote as orchestra_remote
   9 from teuthology.orchestra import run
  10 from teuthology.contextutil import MaxWhileTries
  11 from .mount import CephFSMount
  12
  13 log = logging.getLogger(__name__)
  14
  15
  16 UMOUNT_TIMEOUT = 300
  17
  18
  19 class KernelMount(CephFSMount):
  20     def __init__(self, mons, test_dir, client_id, client_remote,
  21                  ipmi_user, ipmi_password, ipmi_domain):
  22         super(KernelMount, self).__init__(test_dir, client_id, client_remote)
  23         self.mons = mons
  24
  25         self.mounted = False
  26         self.ipmi_user = ipmi_user
  27         self.ipmi_password = ipmi_password
  28         self.ipmi_domain = ipmi_domain
  29
  30     def write_secret_file(self, remote, role, keyring, filename):
  31         """
  32         Stash the keyring in the filename specified.
  33         """
  34         remote.run(
  35             args=[
  36                 'adjust-ulimits',
  37                 'ceph-coverage',
  38                 '{tdir}/archive/coverage'.format(tdir=self.test_dir),
  39                 'ceph-authtool',
  40                 '--name={role}'.format(role=role),
  41                 '--print-key',
  42                 keyring,
  43                 run.Raw('>'),
  44                 filename,
  45             ],
  46             timeout=(5*60),
  47         )
  48
  49     def mount(self, mount_path=None, mount_fs_name=None):
  50         log.info('Mounting kclient client.{id} at {remote} {mnt}...'.format(
  51             id=self.client_id, remote=self.client_remote, mnt=self.mountpoint))
  52
  53         keyring = self.get_keyring_path()
  54         secret = '{tdir}/ceph.data/client.{id}.secret'.format(tdir=self.test_dir, id=self.client_id)
  55         self.write_secret_file(self.client_remote, 'client.{id}'.format(id=self.client_id),
  56                                keyring, secret)
  57
  58         self.client_remote.run(
  59             args=[
  60                 'mkdir',
  61                 '--',
  62                 self.mountpoint,
  63             ],
  64             timeout=(5*60),
  65         )
  66
  67         if mount_path is None:
  68             mount_path = "/"
  69
  70         opts = 'name={id},secretfile={secret},norequire_active_mds'.format(id=self.client_id,
  71                                                       secret=secret)
  72
  73         if mount_fs_name is not None:
  74             opts += ",mds_namespace={0}".format(mount_fs_name)
  75
  76         self.client_remote.run(
  77             args=[
  78                 'sudo',
  79                 'adjust-ulimits',
  80                 'ceph-coverage',
  81                 '{tdir}/archive/coverage'.format(tdir=self.test_dir),
  82                 '/sbin/mount.ceph',
  83                 '{mons}:{mount_path}'.format(mons=','.join(self.mons), mount_path=mount_path),
  84                 self.mountpoint,
  85                 '-v',
  86                 '-o',
  87                 opts
  88             ],
  89             timeout=(30*60),
  90         )
  91
  92         self.client_remote.run(
  93             args=['sudo', 'chmod', '1777', self.mountpoint], timeout=(5*60))
  94
  95         self.mounted = True
  96
  97     def umount(self, force=False):
  98         log.debug('Unmounting client client.{id}...'.format(id=self.client_id))
  99
 100         cmd=['sudo', 'umount', self.mountpoint]
 101         if force:
 102             cmd.append('-f')
 103
 104         try:
 105             self.client_remote.run(args=cmd, timeout=(15*60))
 106         except Exception as e:
 107             self.client_remote.run(args=[
 108                 'sudo',
 109                 run.Raw('PATH=/usr/sbin:$PATH'),
 110                 'lsof',
 111                 run.Raw(';'),
 112                 'ps', 'auxf',
 113             ], timeout=(15*60))
 114             raise e
 115
 116         rproc = self.client_remote.run(
 117             args=[
 118                 'rmdir',
 119                 '--',
 120                 self.mountpoint,
 121             ],
 122             wait=False
 123         )
 124         run.wait([rproc], UMOUNT_TIMEOUT)
 125         self.mounted = False
 126
 127     def cleanup(self):
 128         pass
 129
 130     def umount_wait(self, force=False, require_clean=False, timeout=900):
 131         """
 132         Unlike the fuse client, the kernel client's umount is immediate
 133         """
 134         if not self.is_mounted():
 135             return
 136
 137         try:
 138             self.umount(force)
 139         except (CommandFailedError, MaxWhileTries):
 140             if not force:
 141                 raise
 142
 143             self.kill()
 144             self.kill_cleanup()
 145
 146         self.mounted = False
 147
 148     def is_mounted(self):
 149         return self.mounted
 150
 151     def wait_until_mounted(self):
 152         """
 153         Unlike the fuse client, the kernel client is up and running as soon
 154         as the initial mount() function returns.
 155         """
 156         assert self.mounted
 157
 158     def teardown(self):
 159         super(KernelMount, self).teardown()
 160         if self.mounted:
 161             self.umount()
 162
 163     def kill(self):
 164         """
 165         The Ceph kernel client doesn't have a mechanism to kill itself (doing
 166         that in side the kernel would be weird anyway), so we reboot the whole node
 167         to get the same effect.
 168
 169         We use IPMI to reboot, because we don't want the client to send any
 170         releases of capabilities.
 171         """
 172
 173         con = orchestra_remote.getRemoteConsole(self.client_remote.hostname,
 174                                                 self.ipmi_user,
 175                                                 self.ipmi_password,
 176                                                 self.ipmi_domain)
 177         con.power_off()
 178
 179         self.mounted = False
 180
 181     def kill_cleanup(self):
 182         assert not self.mounted
 183
 184         con = orchestra_remote.getRemoteConsole(self.client_remote.hostname,
 185                                                 self.ipmi_user,
 186                                                 self.ipmi_password,
 187                                                 self.ipmi_domain)
 188         con.power_on()
 189
 190         # Wait for node to come back up after reboot
 191         misc.reconnect(None, 300, [self.client_remote])
 192
 193         # Remove mount directory
 194         self.client_remote.run(
 195             args=[
 196                 'rmdir',
 197                 '--',
 198                 self.mountpoint,
 199             ],
 200             timeout=(5*60),
 201         )
 202
 203     def _find_debug_dir(self):
 204         """
 205         Find the debugfs folder for this mount
 206         """
 207         pyscript = dedent("""
 208             import glob
 209             import os
 210             import json
 211
 212             def get_id_to_dir():
 213                 result = {}
 214                 for dir in glob.glob("/sys/kernel/debug/ceph/*"):
 215                     mds_sessions_lines = open(os.path.join(dir, "mds_sessions")).readlines()
 216                     client_id = mds_sessions_lines[1].split()[1].strip('"')
 217
 218                     result[client_id] = dir
 219                 return result
 220
 221             print json.dumps(get_id_to_dir())
 222             """)
 223
 224         p = self.client_remote.run(args=[
 225             'sudo', 'python', '-c', pyscript
 226         ], stdout=StringIO(), timeout=(5*60))
 227         client_id_to_dir = json.loads(p.stdout.getvalue())
 228
 229         try:
 230             return client_id_to_dir[self.client_id]
 231         except KeyError:
 232             log.error("Client id '{0}' debug dir not found (clients seen were: {1})".format(
 233                 self.client_id, ",".join(client_id_to_dir.keys())
 234             ))
 235             raise
 236
 237     def _read_debug_file(self, filename):
 238         debug_dir = self._find_debug_dir()
 239
 240         pyscript = dedent("""
 241             import os
 242
 243             print open(os.path.join("{debug_dir}", "{filename}")).read()
 244             """).format(debug_dir=debug_dir, filename=filename)
 245
 246         p = self.client_remote.run(args=[
 247             'sudo', 'python', '-c', pyscript
 248         ], stdout=StringIO(), timeout=(5*60))
 249         return p.stdout.getvalue()
 250
 251     def get_global_id(self):
 252         """
 253         Look up the CephFS client ID for this mount, using debugfs.
 254         """
 255
 256         assert self.mounted
 257
 258         mds_sessions = self._read_debug_file("mds_sessions")
 259         lines = mds_sessions.split("\n")
 260         return int(lines[0].split()[1])
 261
 262     def get_osd_epoch(self):
 263         """
 264         Return 2-tuple of osd_epoch, osd_epoch_barrier
 265         """
 266         osd_map = self._read_debug_file("osdmap")
 267         lines = osd_map.split("\n")
 268         first_line_tokens = lines[0].split()
 269         epoch, barrier = int(first_line_tokens[1]), int(first_line_tokens[3])
 270
 271         return epoch, barrier