]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | |
2 | import unittest | |
3 | import time | |
4 | import logging | |
5 | ||
6 | from teuthology.orchestra.run import CommandFailedError | |
7 | ||
8 | log = logging.getLogger(__name__) | |
9 | ||
10 | ||
11 | class CephTestCase(unittest.TestCase): | |
12 | """ | |
13 | For test tasks that want to define a structured set of | |
14 | tests implemented in python. Subclass this with appropriate | |
15 | helpers for the subsystem you're testing. | |
16 | """ | |
17 | ||
18 | # Environment references | |
19 | mounts = None | |
20 | fs = None | |
21 | ceph_cluster = None | |
22 | mds_cluster = None | |
23 | mgr_cluster = None | |
24 | ctx = None | |
25 | ||
26 | mon_manager = None | |
27 | ||
28 | def setUp(self): | |
29 | self.ceph_cluster.mon_manager.raw_cluster_cmd("log", | |
30 | "Starting test {0}".format(self.id())) | |
31 | ||
32 | def tearDown(self): | |
33 | self.ceph_cluster.mon_manager.raw_cluster_cmd("log", | |
34 | "Ended test {0}".format(self.id())) | |
35 | ||
36 | def assert_cluster_log(self, expected_pattern, invert_match=False, timeout=10): | |
37 | """ | |
38 | Context manager. Assert that during execution, or up to 5 seconds later, | |
39 | the Ceph cluster log emits a message matching the expected pattern. | |
40 | ||
41 | :param expected_pattern: a string that you expect to see in the log output | |
42 | """ | |
43 | ||
44 | ceph_manager = self.ceph_cluster.mon_manager | |
45 | ||
46 | class ContextManager(object): | |
47 | def match(self): | |
48 | found = expected_pattern in self.watcher_process.stdout.getvalue() | |
49 | if invert_match: | |
50 | return not found | |
51 | ||
52 | return found | |
53 | ||
54 | def __enter__(self): | |
55 | self.watcher_process = ceph_manager.run_ceph_w() | |
56 | ||
57 | def __exit__(self, exc_type, exc_val, exc_tb): | |
58 | if not self.watcher_process.finished: | |
59 | # Check if we got an early match, wait a bit if we didn't | |
60 | if self.match(): | |
61 | return | |
62 | else: | |
63 | log.debug("No log hits yet, waiting...") | |
64 | # Default monc tick interval is 10s, so wait that long and | |
65 | # then some grace | |
66 | time.sleep(5 + timeout) | |
67 | ||
68 | self.watcher_process.stdin.close() | |
69 | try: | |
70 | self.watcher_process.wait() | |
71 | except CommandFailedError: | |
72 | pass | |
73 | ||
74 | if not self.match(): | |
75 | log.error("Log output: \n{0}\n".format(self.watcher_process.stdout.getvalue())) | |
76 | raise AssertionError("Expected log message not found: '{0}'".format(expected_pattern)) | |
77 | ||
78 | return ContextManager() | |
79 | ||
80 | def wait_for_health(self, pattern, timeout): | |
81 | """ | |
82 | Wait until 'ceph health' contains messages matching the pattern | |
83 | """ | |
84 | def seen_health_warning(): | |
85 | health = self.ceph_cluster.mon_manager.get_mon_health() | |
224ce89b | 86 | codes = [s for s in health['checks']] |
d2e6a577 | 87 | summary_strings = [s[1]['summary']['message'] for s in health['checks'].iteritems()] |
7c673cae FG |
88 | if len(summary_strings) == 0: |
89 | log.debug("Not expected number of summary strings ({0})".format(summary_strings)) | |
90 | return False | |
91 | else: | |
92 | for ss in summary_strings: | |
93 | if pattern in ss: | |
94 | return True | |
224ce89b WB |
95 | if pattern in codes: |
96 | return True | |
7c673cae FG |
97 | |
98 | log.debug("Not found expected summary strings yet ({0})".format(summary_strings)) | |
99 | return False | |
100 | ||
101 | self.wait_until_true(seen_health_warning, timeout) | |
102 | ||
103 | def wait_for_health_clear(self, timeout): | |
104 | """ | |
105 | Wait until `ceph health` returns no messages | |
106 | """ | |
107 | def is_clear(): | |
108 | health = self.ceph_cluster.mon_manager.get_mon_health() | |
224ce89b | 109 | return len(health['checks']) == 0 |
7c673cae FG |
110 | |
111 | self.wait_until_true(is_clear, timeout) | |
112 | ||
113 | def wait_until_equal(self, get_fn, expect_val, timeout, reject_fn=None): | |
114 | period = 5 | |
115 | elapsed = 0 | |
116 | while True: | |
117 | val = get_fn() | |
118 | if val == expect_val: | |
119 | return | |
120 | elif reject_fn and reject_fn(val): | |
121 | raise RuntimeError("wait_until_equal: forbidden value {0} seen".format(val)) | |
122 | else: | |
123 | if elapsed >= timeout: | |
124 | raise RuntimeError("Timed out after {0} seconds waiting for {1} (currently {2})".format( | |
125 | elapsed, expect_val, val | |
126 | )) | |
127 | else: | |
128 | log.debug("wait_until_equal: {0} != {1}, waiting...".format(val, expect_val)) | |
129 | time.sleep(period) | |
130 | elapsed += period | |
131 | ||
132 | log.debug("wait_until_equal: success") | |
133 | ||
134 | def wait_until_true(self, condition, timeout): | |
135 | period = 5 | |
136 | elapsed = 0 | |
137 | while True: | |
138 | if condition(): | |
139 | log.debug("wait_until_true: success in {0}s".format(elapsed)) | |
140 | return | |
141 | else: | |
142 | if elapsed >= timeout: | |
143 | raise RuntimeError("Timed out after {0}s".format(elapsed)) | |
144 | else: | |
145 | log.debug("wait_until_true: waiting...") | |
146 | time.sleep(period) | |
147 | elapsed += period | |
148 | ||
149 |