]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | |
2 | """ | |
3 | High level status display commands | |
4 | """ | |
5 | ||
6 | from collections import defaultdict | |
7 | from prettytable import PrettyTable | |
8 | import prettytable | |
9 | ||
10 | from mgr_module import MgrModule | |
11 | ||
12 | ||
13 | class Module(MgrModule): | |
14 | COMMANDS = [ | |
15 | { | |
16 | "cmd": "fs status " | |
17 | "name=fs,type=CephString,req=false", | |
18 | "desc": "Show the status of a CephFS filesystem", | |
19 | "perm": "r" | |
20 | }, | |
21 | { | |
22 | "cmd": "osd perf " | |
23 | "name=bucket,type=CephString,req=false", | |
24 | "desc": "Show the status of OSDs within a bucket, or all", | |
25 | "perm": "r" | |
26 | }, | |
27 | ] | |
28 | ||
29 | ( | |
30 | BLACK, | |
31 | RED, | |
32 | GREEN, | |
33 | YELLOW, | |
34 | BLUE, | |
35 | MAGENTA, | |
36 | CYAN, | |
37 | GRAY | |
38 | ) = range(8) | |
39 | ||
40 | RESET_SEQ = "\033[0m" | |
41 | COLOR_SEQ = "\033[1;%dm" | |
42 | COLOR_DARK_SEQ = "\033[0;%dm" | |
43 | BOLD_SEQ = "\033[1m" | |
44 | UNDERLINE_SEQ = "\033[4m" | |
45 | ||
46 | def colorize(self, msg, color, dark=False): | |
47 | """ | |
48 | Decorate `msg` with escape sequences to give the requested color | |
49 | """ | |
50 | return (self.COLOR_DARK_SEQ if dark else self.COLOR_SEQ) % (30 + color) \ | |
51 | + msg + self.RESET_SEQ | |
52 | ||
53 | def bold(self, msg): | |
54 | """ | |
55 | Decorate `msg` with escape sequences to make it appear bold | |
56 | """ | |
57 | return self.BOLD_SEQ + msg + self.RESET_SEQ | |
58 | ||
59 | def format_dimless(self, n, width, colored=True): | |
60 | """ | |
61 | Format a number without units, so as to fit into `width` characters, substituting | |
62 | an appropriate unit suffix. | |
63 | """ | |
64 | units = [' ', 'k', 'M', 'G', 'T', 'P'] | |
65 | unit = 0 | |
66 | while len("%s" % (int(n) // (1000**unit))) > width - 1: | |
67 | unit += 1 | |
68 | ||
69 | if unit > 0: | |
70 | truncated_float = ("%f" % (n / (1000.0 ** unit)))[0:width - 1] | |
71 | if truncated_float[-1] == '.': | |
72 | truncated_float = " " + truncated_float[0:-1] | |
73 | else: | |
74 | truncated_float = "%{wid}d".format(wid=width-1) % n | |
75 | formatted = "%s%s" % (truncated_float, units[unit]) | |
76 | ||
77 | if colored: | |
78 | if n == 0: | |
79 | color = self.BLACK, False | |
80 | else: | |
81 | color = self.YELLOW, False | |
82 | return self.bold(self.colorize(formatted[0:-1], color[0], color[1])) \ | |
83 | + self.bold(self.colorize(formatted[-1], self.BLACK, False)) | |
84 | else: | |
85 | return formatted | |
86 | ||
87 | def get_latest(self, daemon_type, daemon_name, stat): | |
88 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] | |
89 | #self.log.error("get_latest {0} data={1}".format(stat, data)) | |
90 | if data: | |
91 | return data[-1][1] | |
92 | else: | |
93 | return 0 | |
94 | ||
95 | def get_rate(self, daemon_type, daemon_name, stat): | |
96 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] | |
97 | ||
98 | #self.log.error("get_latest {0} data={1}".format(stat, data)) | |
99 | if data and len(data) > 1: | |
100 | return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0]) | |
101 | else: | |
102 | return 0 | |
103 | ||
104 | def handle_fs_status(self, cmd): | |
105 | output = "" | |
106 | ||
107 | mds_versions = defaultdict(list) | |
108 | ||
109 | fsmap = self.get("fs_map") | |
110 | for filesystem in fsmap['filesystems']: | |
111 | rank_table = PrettyTable( | |
112 | ("Rank", "State", "MDS", "Activity", "dns", "inos"), | |
113 | hrules=prettytable.FRAME | |
114 | ) | |
115 | ||
116 | mdsmap = filesystem['mdsmap'] | |
117 | ||
118 | client_count = 0 | |
119 | ||
120 | for rank in mdsmap["in"]: | |
121 | up = "mds_{0}".format(rank) in mdsmap["up"] | |
122 | if up: | |
123 | gid = mdsmap['up']["mds_{0}".format(rank)] | |
124 | info = mdsmap['info']['gid_{0}'.format(gid)] | |
31f18b77 | 125 | dns = self.get_latest("mds", info['name'], "mds_mem.dn") |
7c673cae FG |
126 | inos = self.get_latest("mds", info['name'], "mds_mem.ino") |
127 | ||
128 | if rank == 0: | |
129 | client_count = self.get_latest("mds", info['name'], | |
130 | "mds_sessions.session_count") | |
131 | elif client_count == 0: | |
132 | # In case rank 0 was down, look at another rank's | |
133 | # sessionmap to get an indication of clients. | |
134 | client_count = self.get_latest("mds", info['name'], | |
135 | "mds_sessions.session_count") | |
136 | ||
137 | laggy = "laggy_since" in info | |
138 | ||
139 | state = info['state'].split(":")[1] | |
140 | if laggy: | |
141 | state += "(laggy)" | |
142 | if state == "active" and not laggy: | |
143 | c_state = self.colorize(state, self.GREEN) | |
144 | else: | |
145 | c_state = self.colorize(state, self.YELLOW) | |
146 | ||
147 | # Populate based on context of state, e.g. client | |
148 | # ops for an active daemon, replay progress, reconnect | |
149 | # progress | |
150 | activity = "" | |
151 | ||
152 | if state == "active": | |
153 | activity = "Reqs: " + self.format_dimless( | |
154 | self.get_rate("mds", info['name'], "mds_server.handle_client_request"), | |
155 | 5 | |
156 | ) + "/s" | |
157 | ||
158 | metadata = self.get_metadata('mds', info['name']) | |
159 | mds_versions[metadata.get('ceph_version', "unknown")].append(info['name']) | |
160 | rank_table.add_row([ | |
161 | self.bold(rank.__str__()), c_state, info['name'], | |
162 | activity, | |
163 | self.format_dimless(dns, 5), | |
164 | self.format_dimless(inos, 5) | |
165 | ]) | |
166 | ||
167 | else: | |
168 | rank_table.add_row([ | |
169 | rank, "failed", "", "", "" | |
170 | ]) | |
171 | ||
172 | # Find the standby replays | |
173 | for gid_str, daemon_info in mdsmap['info'].iteritems(): | |
174 | if daemon_info['state'] != "up:standby-replay": | |
175 | continue | |
176 | ||
177 | inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino") | |
31f18b77 | 178 | dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn") |
7c673cae FG |
179 | |
180 | activity = "Evts: " + self.format_dimless( | |
181 | self.get_rate("mds", daemon_info['name'], "mds_log.replay"), | |
182 | 5 | |
183 | ) + "/s" | |
184 | ||
185 | rank_table.add_row([ | |
186 | "{0}-s".format(daemon_info['rank']), "standby-replay", | |
187 | daemon_info['name'], activity, | |
188 | self.format_dimless(dns, 5), | |
189 | self.format_dimless(inos, 5) | |
190 | ]) | |
191 | ||
192 | df = self.get("df") | |
193 | pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) | |
194 | osdmap = self.get("osd_map") | |
195 | pools = dict([(p['pool'], p) for p in osdmap['pools']]) | |
196 | metadata_pool_id = mdsmap['metadata_pool'] | |
197 | data_pool_ids = mdsmap['data_pools'] | |
198 | ||
199 | pools_table = PrettyTable(["Pool", "type", "used", "avail"]) | |
200 | for pool_id in [metadata_pool_id] + data_pool_ids: | |
201 | pool_type = "metadata" if pool_id == metadata_pool_id else "data" | |
202 | stats = pool_stats[pool_id] | |
203 | pools_table.add_row([ | |
204 | pools[pool_id]['pool_name'], pool_type, | |
205 | self.format_dimless(stats['bytes_used'], 5), | |
206 | self.format_dimless(stats['max_avail'], 5) | |
207 | ]) | |
208 | ||
209 | output += "{0} - {1} clients\n".format( | |
210 | mdsmap['fs_name'], client_count) | |
211 | output += "=" * len(mdsmap['fs_name']) + "\n" | |
212 | output += rank_table.get_string() | |
213 | output += "\n" + pools_table.get_string() + "\n" | |
214 | ||
215 | standby_table = PrettyTable(["Standby MDS"]) | |
216 | for standby in fsmap['standbys']: | |
217 | metadata = self.get_metadata('mds', standby['name']) | |
218 | mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name']) | |
219 | ||
220 | standby_table.add_row([standby['name']]) | |
221 | ||
222 | output += "\n" + standby_table.get_string() + "\n" | |
223 | ||
224 | if len(mds_versions) == 1: | |
225 | output += "MDS version: {0}".format(mds_versions.keys()[0]) | |
226 | else: | |
227 | version_table = PrettyTable(["version", "daemons"]) | |
228 | for version, daemons in mds_versions.iteritems(): | |
229 | version_table.add_row([ | |
230 | version, | |
231 | ", ".join(daemons) | |
232 | ]) | |
233 | output += version_table.get_string() + "\n" | |
234 | ||
235 | return 0, "", output | |
236 | ||
237 | def handle_osd_perf(self, cmd): | |
238 | osd_table = PrettyTable(['id', 'host', 'used', 'avail', 'wr ops', 'wr data', 'rd ops', 'rd data']) | |
239 | osdmap = self.get("osd_map") | |
240 | ||
241 | # Build dict of OSD ID to stats | |
242 | osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']]) | |
243 | ||
244 | for osd in osdmap['osds']: | |
245 | osd_id = osd['osd'] | |
246 | metadata = self.get_metadata('osd', "%s" % osd_id) | |
247 | stats = osd_stats[osd_id] | |
248 | ||
249 | osd_table.add_row([osd_id, metadata['hostname'], | |
250 | self.format_dimless(stats['kb_used'] * 1024, 5), | |
251 | self.format_dimless(stats['kb_avail'] * 1024, 5), | |
252 | self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") + | |
253 | self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5), | |
254 | self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5), | |
255 | self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5), | |
256 | self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5), | |
257 | ]) | |
258 | ||
259 | return 0, "", osd_table.get_string() | |
260 | ||
261 | def handle_command(self, cmd): | |
262 | self.log.error("handle_command") | |
263 | ||
264 | if cmd['prefix'] == "fs status": | |
265 | return self.handle_fs_status(cmd) | |
266 | elif cmd['prefix'] == "osd perf": | |
267 | return self.handle_osd_perf(cmd) | |
268 | else: | |
269 | # mgr should respect our self.COMMANDS and not call us for | |
270 | # any prefix we don't advertise | |
271 | raise NotImplementedError(cmd['prefix']) |