]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | |
2 | """ | |
3 | High level status display commands | |
4 | """ | |
5 | ||
6 | from collections import defaultdict | |
7 | from prettytable import PrettyTable | |
8 | import prettytable | |
224ce89b WB |
9 | import fnmatch |
10 | import errno | |
7c673cae FG |
11 | |
12 | from mgr_module import MgrModule | |
13 | ||
14 | ||
15 | class Module(MgrModule): | |
16 | COMMANDS = [ | |
17 | { | |
18 | "cmd": "fs status " | |
19 | "name=fs,type=CephString,req=false", | |
20 | "desc": "Show the status of a CephFS filesystem", | |
21 | "perm": "r" | |
22 | }, | |
23 | { | |
224ce89b | 24 | "cmd": "osd status " |
7c673cae FG |
25 | "name=bucket,type=CephString,req=false", |
26 | "desc": "Show the status of OSDs within a bucket, or all", | |
27 | "perm": "r" | |
28 | }, | |
29 | ] | |
30 | ||
31 | ( | |
32 | BLACK, | |
33 | RED, | |
34 | GREEN, | |
35 | YELLOW, | |
36 | BLUE, | |
37 | MAGENTA, | |
38 | CYAN, | |
39 | GRAY | |
40 | ) = range(8) | |
41 | ||
42 | RESET_SEQ = "\033[0m" | |
43 | COLOR_SEQ = "\033[1;%dm" | |
44 | COLOR_DARK_SEQ = "\033[0;%dm" | |
45 | BOLD_SEQ = "\033[1m" | |
46 | UNDERLINE_SEQ = "\033[4m" | |
47 | ||
48 | def colorize(self, msg, color, dark=False): | |
49 | """ | |
50 | Decorate `msg` with escape sequences to give the requested color | |
51 | """ | |
52 | return (self.COLOR_DARK_SEQ if dark else self.COLOR_SEQ) % (30 + color) \ | |
53 | + msg + self.RESET_SEQ | |
54 | ||
55 | def bold(self, msg): | |
56 | """ | |
57 | Decorate `msg` with escape sequences to make it appear bold | |
58 | """ | |
59 | return self.BOLD_SEQ + msg + self.RESET_SEQ | |
60 | ||
61 | def format_dimless(self, n, width, colored=True): | |
62 | """ | |
63 | Format a number without units, so as to fit into `width` characters, substituting | |
64 | an appropriate unit suffix. | |
65 | """ | |
66 | units = [' ', 'k', 'M', 'G', 'T', 'P'] | |
67 | unit = 0 | |
68 | while len("%s" % (int(n) // (1000**unit))) > width - 1: | |
69 | unit += 1 | |
70 | ||
71 | if unit > 0: | |
72 | truncated_float = ("%f" % (n / (1000.0 ** unit)))[0:width - 1] | |
73 | if truncated_float[-1] == '.': | |
74 | truncated_float = " " + truncated_float[0:-1] | |
75 | else: | |
76 | truncated_float = "%{wid}d".format(wid=width-1) % n | |
77 | formatted = "%s%s" % (truncated_float, units[unit]) | |
78 | ||
79 | if colored: | |
80 | if n == 0: | |
81 | color = self.BLACK, False | |
82 | else: | |
83 | color = self.YELLOW, False | |
84 | return self.bold(self.colorize(formatted[0:-1], color[0], color[1])) \ | |
85 | + self.bold(self.colorize(formatted[-1], self.BLACK, False)) | |
86 | else: | |
87 | return formatted | |
88 | ||
89 | def get_latest(self, daemon_type, daemon_name, stat): | |
90 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] | |
91 | #self.log.error("get_latest {0} data={1}".format(stat, data)) | |
92 | if data: | |
93 | return data[-1][1] | |
94 | else: | |
95 | return 0 | |
96 | ||
97 | def get_rate(self, daemon_type, daemon_name, stat): | |
98 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] | |
99 | ||
100 | #self.log.error("get_latest {0} data={1}".format(stat, data)) | |
101 | if data and len(data) > 1: | |
102 | return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0]) | |
103 | else: | |
104 | return 0 | |
105 | ||
106 | def handle_fs_status(self, cmd): | |
107 | output = "" | |
108 | ||
224ce89b WB |
109 | fs_filter = cmd.get('fs', None) |
110 | ||
7c673cae FG |
111 | mds_versions = defaultdict(list) |
112 | ||
113 | fsmap = self.get("fs_map") | |
114 | for filesystem in fsmap['filesystems']: | |
224ce89b WB |
115 | if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter: |
116 | continue | |
117 | ||
7c673cae FG |
118 | rank_table = PrettyTable( |
119 | ("Rank", "State", "MDS", "Activity", "dns", "inos"), | |
120 | hrules=prettytable.FRAME | |
121 | ) | |
122 | ||
123 | mdsmap = filesystem['mdsmap'] | |
124 | ||
125 | client_count = 0 | |
126 | ||
127 | for rank in mdsmap["in"]: | |
128 | up = "mds_{0}".format(rank) in mdsmap["up"] | |
129 | if up: | |
130 | gid = mdsmap['up']["mds_{0}".format(rank)] | |
131 | info = mdsmap['info']['gid_{0}'.format(gid)] | |
31f18b77 | 132 | dns = self.get_latest("mds", info['name'], "mds_mem.dn") |
7c673cae FG |
133 | inos = self.get_latest("mds", info['name'], "mds_mem.ino") |
134 | ||
135 | if rank == 0: | |
136 | client_count = self.get_latest("mds", info['name'], | |
137 | "mds_sessions.session_count") | |
138 | elif client_count == 0: | |
139 | # In case rank 0 was down, look at another rank's | |
140 | # sessionmap to get an indication of clients. | |
141 | client_count = self.get_latest("mds", info['name'], | |
142 | "mds_sessions.session_count") | |
143 | ||
144 | laggy = "laggy_since" in info | |
145 | ||
146 | state = info['state'].split(":")[1] | |
147 | if laggy: | |
148 | state += "(laggy)" | |
149 | if state == "active" and not laggy: | |
150 | c_state = self.colorize(state, self.GREEN) | |
151 | else: | |
152 | c_state = self.colorize(state, self.YELLOW) | |
153 | ||
154 | # Populate based on context of state, e.g. client | |
155 | # ops for an active daemon, replay progress, reconnect | |
156 | # progress | |
157 | activity = "" | |
158 | ||
159 | if state == "active": | |
160 | activity = "Reqs: " + self.format_dimless( | |
161 | self.get_rate("mds", info['name'], "mds_server.handle_client_request"), | |
162 | 5 | |
163 | ) + "/s" | |
164 | ||
165 | metadata = self.get_metadata('mds', info['name']) | |
166 | mds_versions[metadata.get('ceph_version', "unknown")].append(info['name']) | |
167 | rank_table.add_row([ | |
168 | self.bold(rank.__str__()), c_state, info['name'], | |
169 | activity, | |
170 | self.format_dimless(dns, 5), | |
171 | self.format_dimless(inos, 5) | |
172 | ]) | |
173 | ||
174 | else: | |
175 | rank_table.add_row([ | |
c07f9fc5 | 176 | rank, "failed", "", "", "", "" |
7c673cae FG |
177 | ]) |
178 | ||
179 | # Find the standby replays | |
180 | for gid_str, daemon_info in mdsmap['info'].iteritems(): | |
181 | if daemon_info['state'] != "up:standby-replay": | |
182 | continue | |
183 | ||
184 | inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino") | |
31f18b77 | 185 | dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn") |
7c673cae FG |
186 | |
187 | activity = "Evts: " + self.format_dimless( | |
188 | self.get_rate("mds", daemon_info['name'], "mds_log.replay"), | |
189 | 5 | |
190 | ) + "/s" | |
191 | ||
192 | rank_table.add_row([ | |
193 | "{0}-s".format(daemon_info['rank']), "standby-replay", | |
194 | daemon_info['name'], activity, | |
195 | self.format_dimless(dns, 5), | |
196 | self.format_dimless(inos, 5) | |
197 | ]) | |
198 | ||
199 | df = self.get("df") | |
200 | pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) | |
201 | osdmap = self.get("osd_map") | |
202 | pools = dict([(p['pool'], p) for p in osdmap['pools']]) | |
203 | metadata_pool_id = mdsmap['metadata_pool'] | |
204 | data_pool_ids = mdsmap['data_pools'] | |
205 | ||
206 | pools_table = PrettyTable(["Pool", "type", "used", "avail"]) | |
207 | for pool_id in [metadata_pool_id] + data_pool_ids: | |
208 | pool_type = "metadata" if pool_id == metadata_pool_id else "data" | |
209 | stats = pool_stats[pool_id] | |
210 | pools_table.add_row([ | |
211 | pools[pool_id]['pool_name'], pool_type, | |
212 | self.format_dimless(stats['bytes_used'], 5), | |
213 | self.format_dimless(stats['max_avail'], 5) | |
214 | ]) | |
215 | ||
216 | output += "{0} - {1} clients\n".format( | |
217 | mdsmap['fs_name'], client_count) | |
218 | output += "=" * len(mdsmap['fs_name']) + "\n" | |
219 | output += rank_table.get_string() | |
220 | output += "\n" + pools_table.get_string() + "\n" | |
221 | ||
222 | standby_table = PrettyTable(["Standby MDS"]) | |
223 | for standby in fsmap['standbys']: | |
224 | metadata = self.get_metadata('mds', standby['name']) | |
225 | mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name']) | |
226 | ||
227 | standby_table.add_row([standby['name']]) | |
228 | ||
229 | output += "\n" + standby_table.get_string() + "\n" | |
230 | ||
231 | if len(mds_versions) == 1: | |
232 | output += "MDS version: {0}".format(mds_versions.keys()[0]) | |
233 | else: | |
234 | version_table = PrettyTable(["version", "daemons"]) | |
235 | for version, daemons in mds_versions.iteritems(): | |
236 | version_table.add_row([ | |
237 | version, | |
238 | ", ".join(daemons) | |
239 | ]) | |
240 | output += version_table.get_string() + "\n" | |
241 | ||
242 | return 0, "", output | |
243 | ||
224ce89b | 244 | def handle_osd_status(self, cmd): |
7c673cae FG |
245 | osd_table = PrettyTable(['id', 'host', 'used', 'avail', 'wr ops', 'wr data', 'rd ops', 'rd data']) |
246 | osdmap = self.get("osd_map") | |
247 | ||
224ce89b WB |
248 | filter_osds = set() |
249 | bucket_filter = None | |
250 | if 'bucket' in cmd: | |
251 | self.log.debug("Filtering to bucket '{0}'".format(cmd['bucket'])) | |
252 | bucket_filter = cmd['bucket'] | |
253 | crush = self.get("osd_map_crush") | |
254 | found = False | |
255 | for bucket in crush['buckets']: | |
256 | if fnmatch.fnmatch(bucket['name'], bucket_filter): | |
257 | found = True | |
258 | filter_osds.update([i['id'] for i in bucket['items']]) | |
259 | ||
260 | if not found: | |
261 | msg = "Bucket '{0}' not found".format(bucket_filter) | |
262 | return errno.ENOENT, msg, "" | |
263 | ||
7c673cae FG |
264 | # Build dict of OSD ID to stats |
265 | osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']]) | |
266 | ||
267 | for osd in osdmap['osds']: | |
268 | osd_id = osd['osd'] | |
224ce89b WB |
269 | if bucket_filter and osd_id not in filter_osds: |
270 | continue | |
271 | ||
7c673cae FG |
272 | metadata = self.get_metadata('osd', "%s" % osd_id) |
273 | stats = osd_stats[osd_id] | |
274 | ||
275 | osd_table.add_row([osd_id, metadata['hostname'], | |
276 | self.format_dimless(stats['kb_used'] * 1024, 5), | |
277 | self.format_dimless(stats['kb_avail'] * 1024, 5), | |
278 | self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") + | |
279 | self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5), | |
280 | self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5), | |
281 | self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5), | |
282 | self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5), | |
283 | ]) | |
284 | ||
285 | return 0, "", osd_table.get_string() | |
286 | ||
287 | def handle_command(self, cmd): | |
288 | self.log.error("handle_command") | |
289 | ||
290 | if cmd['prefix'] == "fs status": | |
291 | return self.handle_fs_status(cmd) | |
224ce89b WB |
292 | elif cmd['prefix'] == "osd status": |
293 | return self.handle_osd_status(cmd) | |
7c673cae FG |
294 | else: |
295 | # mgr should respect our self.COMMANDS and not call us for | |
296 | # any prefix we don't advertise | |
297 | raise NotImplementedError(cmd['prefix']) |