ceph/qa/tasks/ceph_test_case.py

   1
   2 import unittest
   3 from unittest import case
   4 import time
   5 import logging
   6
   7 from teuthology.orchestra.run import CommandFailedError
   8
   9 log = logging.getLogger(__name__)
  10
  11
  12 class CephTestCase(unittest.TestCase):
  13     """
  14     For test tasks that want to define a structured set of
  15     tests implemented in python.  Subclass this with appropriate
  16     helpers for the subsystem you're testing.
  17     """
  18
  19     # Environment references
  20     mounts = None
  21     fs = None
  22     recovery_fs = None
  23     ceph_cluster = None
  24     mds_cluster = None
  25     mgr_cluster = None
  26     ctx = None
  27
  28     mon_manager = None
  29
  30     # Declarative test requirements: subclasses should override these to indicate
  31     # their special needs.  If not met, tests will be skipped.
  32     REQUIRE_MEMSTORE = False
  33
  34     def setUp(self):
  35         self.ceph_cluster.mon_manager.raw_cluster_cmd("log",
  36             "Starting test {0}".format(self.id()))
  37
  38         if self.REQUIRE_MEMSTORE:
  39             objectstore = self.ceph_cluster.get_config("osd_objectstore", "osd")
  40             if objectstore != "memstore":
  41                 # You certainly *could* run this on a real OSD, but you don't want to sit
  42                 # here for hours waiting for the test to fill up a 1TB drive!
  43                 raise case.SkipTest("Require `memstore` OSD backend (test " \
  44                         "would take too long on full sized OSDs")
  45
  46
  47
  48     def tearDown(self):
  49         self.ceph_cluster.mon_manager.raw_cluster_cmd("log",
  50             "Ended test {0}".format(self.id()))
  51
  52     def assert_cluster_log(self, expected_pattern, invert_match=False,
  53                            timeout=10, watch_channel=None):
  54         """
  55         Context manager.  Assert that during execution, or up to 5 seconds later,
  56         the Ceph cluster log emits a message matching the expected pattern.
  57
  58         :param expected_pattern: A string that you expect to see in the log output
  59         :type expected_pattern: str
  60         :param watch_channel: Specifies the channel to be watched. This can be
  61                               'cluster', 'audit', ...
  62         :type watch_channel: str
  63         """
  64
  65         ceph_manager = self.ceph_cluster.mon_manager
  66
  67         class ContextManager(object):
  68             def match(self):
  69                 found = expected_pattern in self.watcher_process.stdout.getvalue()
  70                 if invert_match:
  71                     return not found
  72
  73                 return found
  74
  75             def __enter__(self):
  76                 self.watcher_process = ceph_manager.run_ceph_w(watch_channel)
  77
  78             def __exit__(self, exc_type, exc_val, exc_tb):
  79                 if not self.watcher_process.finished:
  80                     # Check if we got an early match, wait a bit if we didn't
  81                     if self.match():
  82                         return
  83                     else:
  84                         log.debug("No log hits yet, waiting...")
  85                         # Default monc tick interval is 10s, so wait that long and
  86                         # then some grace
  87                         time.sleep(5 + timeout)
  88
  89                 self.watcher_process.stdin.close()
  90                 try:
  91                     self.watcher_process.wait()
  92                 except CommandFailedError:
  93                     pass
  94
  95                 if not self.match():
  96                     log.error("Log output: \n{0}\n".format(self.watcher_process.stdout.getvalue()))
  97                     raise AssertionError("Expected log message not found: '{0}'".format(expected_pattern))
  98
  99         return ContextManager()
 100
 101     def wait_for_health(self, pattern, timeout):
 102         """
 103         Wait until 'ceph health' contains messages matching the pattern
 104         """
 105         def seen_health_warning():
 106             health = self.ceph_cluster.mon_manager.get_mon_health()
 107             codes = [s for s in health['checks']]
 108             summary_strings = [s[1]['summary']['message'] for s in health['checks'].iteritems()]
 109             if len(summary_strings) == 0:
 110                 log.debug("Not expected number of summary strings ({0})".format(summary_strings))
 111                 return False
 112             else:
 113                 for ss in summary_strings:
 114                     if pattern in ss:
 115                          return True
 116                 if pattern in codes:
 117                     return True
 118
 119             log.debug("Not found expected summary strings yet ({0})".format(summary_strings))
 120             return False
 121
 122         self.wait_until_true(seen_health_warning, timeout)
 123
 124     def wait_for_health_clear(self, timeout):
 125         """
 126         Wait until `ceph health` returns no messages
 127         """
 128         def is_clear():
 129             health = self.ceph_cluster.mon_manager.get_mon_health()
 130             return len(health['checks']) == 0
 131
 132         self.wait_until_true(is_clear, timeout)
 133
 134     def wait_until_equal(self, get_fn, expect_val, timeout, reject_fn=None):
 135         period = 5
 136         elapsed = 0
 137         while True:
 138             val = get_fn()
 139             if val == expect_val:
 140                 return
 141             elif reject_fn and reject_fn(val):
 142                 raise RuntimeError("wait_until_equal: forbidden value {0} seen".format(val))
 143             else:
 144                 if elapsed >= timeout:
 145                     raise RuntimeError("Timed out after {0} seconds waiting for {1} (currently {2})".format(
 146                         elapsed, expect_val, val
 147                     ))
 148                 else:
 149                     log.debug("wait_until_equal: {0} != {1}, waiting...".format(val, expect_val))
 150                 time.sleep(period)
 151                 elapsed += period
 152
 153         log.debug("wait_until_equal: success")
 154
 155     @classmethod
 156     def wait_until_true(cls, condition, timeout, period=5):
 157         elapsed = 0
 158         while True:
 159             if condition():
 160                 log.debug("wait_until_true: success in {0}s".format(elapsed))
 161                 return
 162             else:
 163                 if elapsed >= timeout:
 164                     raise RuntimeError("Timed out after {0}s".format(elapsed))
 165                 else:
 166                     log.debug("wait_until_true: waiting...")
 167                 time.sleep(period)
 168                 elapsed += period
 169
 170