ceph/qa/tasks/ceph_test_case.py

   1
   2 import unittest
   3 import time
   4 import logging
   5
   6 from teuthology.orchestra.run import CommandFailedError
   7
   8 log = logging.getLogger(__name__)
   9
  10
  11 class CephTestCase(unittest.TestCase):
  12     """
  13     For test tasks that want to define a structured set of
  14     tests implemented in python.  Subclass this with appropriate
  15     helpers for the subsystem you're testing.
  16     """
  17
  18     # Environment references
  19     mounts = None
  20     fs = None
  21     ceph_cluster = None
  22     mds_cluster = None
  23     mgr_cluster = None
  24     ctx = None
  25
  26     mon_manager = None
  27
  28     def setUp(self):
  29         self.ceph_cluster.mon_manager.raw_cluster_cmd("log",
  30             "Starting test {0}".format(self.id()))
  31
  32     def tearDown(self):
  33         self.ceph_cluster.mon_manager.raw_cluster_cmd("log",
  34             "Ended test {0}".format(self.id()))
  35
  36     def assert_cluster_log(self, expected_pattern, invert_match=False, timeout=10):
  37         """
  38         Context manager.  Assert that during execution, or up to 5 seconds later,
  39         the Ceph cluster log emits a message matching the expected pattern.
  40
  41         :param expected_pattern: a string that you expect to see in the log output
  42         """
  43
  44         ceph_manager = self.ceph_cluster.mon_manager
  45
  46         class ContextManager(object):
  47             def match(self):
  48                 found = expected_pattern in self.watcher_process.stdout.getvalue()
  49                 if invert_match:
  50                     return not found
  51
  52                 return found
  53
  54             def __enter__(self):
  55                 self.watcher_process = ceph_manager.run_ceph_w()
  56
  57             def __exit__(self, exc_type, exc_val, exc_tb):
  58                 if not self.watcher_process.finished:
  59                     # Check if we got an early match, wait a bit if we didn't
  60                     if self.match():
  61                         return
  62                     else:
  63                         log.debug("No log hits yet, waiting...")
  64                         # Default monc tick interval is 10s, so wait that long and
  65                         # then some grace
  66                         time.sleep(5 + timeout)
  67
  68                 self.watcher_process.stdin.close()
  69                 try:
  70                     self.watcher_process.wait()
  71                 except CommandFailedError:
  72                     pass
  73
  74                 if not self.match():
  75                     log.error("Log output: \n{0}\n".format(self.watcher_process.stdout.getvalue()))
  76                     raise AssertionError("Expected log message not found: '{0}'".format(expected_pattern))
  77
  78         return ContextManager()
  79
  80     def wait_for_health(self, pattern, timeout):
  81         """
  82         Wait until 'ceph health' contains messages matching the pattern
  83         """
  84         def seen_health_warning():
  85             health = self.ceph_cluster.mon_manager.get_mon_health()
  86             summary_strings = [s['summary'] for s in health['summary']]
  87             if len(summary_strings) == 0:
  88                 log.debug("Not expected number of summary strings ({0})".format(summary_strings))
  89                 return False
  90             else:
  91                 for ss in summary_strings:
  92                     if pattern in ss:
  93                          return True
  94
  95             log.debug("Not found expected summary strings yet ({0})".format(summary_strings))
  96             return False
  97
  98         self.wait_until_true(seen_health_warning, timeout)
  99
 100     def wait_for_health_clear(self, timeout):
 101         """
 102         Wait until `ceph health` returns no messages
 103         """
 104         def is_clear():
 105             health = self.ceph_cluster.mon_manager.get_mon_health()
 106             return len(health['summary']) == 0
 107
 108         self.wait_until_true(is_clear, timeout)
 109
 110     def wait_until_equal(self, get_fn, expect_val, timeout, reject_fn=None):
 111         period = 5
 112         elapsed = 0
 113         while True:
 114             val = get_fn()
 115             if val == expect_val:
 116                 return
 117             elif reject_fn and reject_fn(val):
 118                 raise RuntimeError("wait_until_equal: forbidden value {0} seen".format(val))
 119             else:
 120                 if elapsed >= timeout:
 121                     raise RuntimeError("Timed out after {0} seconds waiting for {1} (currently {2})".format(
 122                         elapsed, expect_val, val
 123                     ))
 124                 else:
 125                     log.debug("wait_until_equal: {0} != {1}, waiting...".format(val, expect_val))
 126                 time.sleep(period)
 127                 elapsed += period
 128
 129         log.debug("wait_until_equal: success")
 130
 131     def wait_until_true(self, condition, timeout):
 132         period = 5
 133         elapsed = 0
 134         while True:
 135             if condition():
 136                 log.debug("wait_until_true: success in {0}s".format(elapsed))
 137                 return
 138             else:
 139                 if elapsed >= timeout:
 140                     raise RuntimeError("Timed out after {0}s".format(elapsed))
 141                 else:
 142                     log.debug("wait_until_true: waiting...")
 143                 time.sleep(period)
 144                 elapsed += period
 145
 146