]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | |
2 | """ | |
3 | High level status display commands | |
4 | """ | |
5 | ||
6 | from collections import defaultdict | |
7 | from prettytable import PrettyTable | |
224ce89b | 8 | import errno |
1adf2230 | 9 | import fnmatch |
11fdf7f2 | 10 | import mgr_util |
1adf2230 AA |
11 | import prettytable |
12 | import six | |
9f95a23c | 13 | import json |
7c673cae FG |
14 | |
15 | from mgr_module import MgrModule | |
16 | ||
17 | ||
18 | class Module(MgrModule): | |
19 | COMMANDS = [ | |
20 | { | |
21 | "cmd": "fs status " | |
22 | "name=fs,type=CephString,req=false", | |
23 | "desc": "Show the status of a CephFS filesystem", | |
24 | "perm": "r" | |
25 | }, | |
26 | { | |
224ce89b | 27 | "cmd": "osd status " |
7c673cae FG |
28 | "name=bucket,type=CephString,req=false", |
29 | "desc": "Show the status of OSDs within a bucket, or all", | |
30 | "perm": "r" | |
31 | }, | |
32 | ] | |
33 | ||
3efd9988 | 34 | |
7c673cae FG |
35 | def get_latest(self, daemon_type, daemon_name, stat): |
36 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] | |
37 | #self.log.error("get_latest {0} data={1}".format(stat, data)) | |
38 | if data: | |
39 | return data[-1][1] | |
40 | else: | |
41 | return 0 | |
42 | ||
43 | def get_rate(self, daemon_type, daemon_name, stat): | |
44 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] | |
45 | ||
46 | #self.log.error("get_latest {0} data={1}".format(stat, data)) | |
9f95a23c | 47 | if data and len(data) > 1 and data[-1][0] != data[-2][0]: |
7c673cae FG |
48 | return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0]) |
49 | else: | |
50 | return 0 | |
51 | ||
52 | def handle_fs_status(self, cmd): | |
53 | output = "" | |
9f95a23c TL |
54 | json_output = defaultdict(list) |
55 | output_format = cmd.get('format', 'plain') | |
7c673cae | 56 | |
224ce89b WB |
57 | fs_filter = cmd.get('fs', None) |
58 | ||
7c673cae FG |
59 | mds_versions = defaultdict(list) |
60 | ||
61 | fsmap = self.get("fs_map") | |
62 | for filesystem in fsmap['filesystems']: | |
224ce89b WB |
63 | if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter: |
64 | continue | |
65 | ||
7c673cae | 66 | rank_table = PrettyTable( |
9f95a23c TL |
67 | ("RANK", "STATE", "MDS", "ACTIVITY", "DNS", "INOS"), |
68 | border=False, | |
7c673cae | 69 | ) |
9f95a23c TL |
70 | rank_table.left_padding_width = 0 |
71 | rank_table.right_padding_width = 2 | |
7c673cae FG |
72 | |
73 | mdsmap = filesystem['mdsmap'] | |
74 | ||
75 | client_count = 0 | |
76 | ||
77 | for rank in mdsmap["in"]: | |
78 | up = "mds_{0}".format(rank) in mdsmap["up"] | |
79 | if up: | |
80 | gid = mdsmap['up']["mds_{0}".format(rank)] | |
81 | info = mdsmap['info']['gid_{0}'.format(gid)] | |
31f18b77 | 82 | dns = self.get_latest("mds", info['name'], "mds_mem.dn") |
7c673cae FG |
83 | inos = self.get_latest("mds", info['name'], "mds_mem.ino") |
84 | ||
85 | if rank == 0: | |
86 | client_count = self.get_latest("mds", info['name'], | |
87 | "mds_sessions.session_count") | |
88 | elif client_count == 0: | |
89 | # In case rank 0 was down, look at another rank's | |
90 | # sessionmap to get an indication of clients. | |
91 | client_count = self.get_latest("mds", info['name'], | |
92 | "mds_sessions.session_count") | |
93 | ||
94 | laggy = "laggy_since" in info | |
95 | ||
96 | state = info['state'].split(":")[1] | |
97 | if laggy: | |
98 | state += "(laggy)" | |
99 | if state == "active" and not laggy: | |
11fdf7f2 | 100 | c_state = mgr_util.colorize(state, mgr_util.GREEN) |
7c673cae | 101 | else: |
11fdf7f2 | 102 | c_state = mgr_util.colorize(state, mgr_util.YELLOW) |
7c673cae FG |
103 | |
104 | # Populate based on context of state, e.g. client | |
105 | # ops for an active daemon, replay progress, reconnect | |
106 | # progress | |
107 | activity = "" | |
108 | ||
109 | if state == "active": | |
9f95a23c TL |
110 | rate = self.get_rate("mds", info['name'], "mds_server.handle_client_request") |
111 | if output_format not in ('json', 'json-pretty'): | |
112 | activity = "Reqs: " + mgr_util.format_dimless(rate, 5) + "/s" | |
7c673cae FG |
113 | |
114 | metadata = self.get_metadata('mds', info['name']) | |
115 | mds_versions[metadata.get('ceph_version', "unknown")].append(info['name']) | |
9f95a23c TL |
116 | if output_format in ('json', 'json-pretty'): |
117 | json_output['mdsmap'].append({ | |
118 | 'rank': rank, | |
119 | 'name': info['name'], | |
120 | 'state': state, | |
121 | 'rate': rate if state == "active" else "0", | |
122 | 'dns': dns, | |
123 | 'inos': inos | |
124 | }) | |
125 | else: | |
126 | rank_table.add_row([ | |
127 | mgr_util.bold(rank.__str__()), c_state, info['name'], | |
128 | activity, | |
129 | mgr_util.format_dimless(dns, 5), | |
130 | mgr_util.format_dimless(inos, 5) | |
131 | ]) | |
7c673cae | 132 | else: |
9f95a23c TL |
133 | if output_format in ('json', 'json-pretty'): |
134 | json_output['mdsmap'].append({ | |
135 | 'rank': rank, | |
136 | 'state': "failed" | |
137 | }) | |
138 | else: | |
139 | rank_table.add_row([ | |
140 | rank, "failed", "", "", "", "" | |
141 | ]) | |
7c673cae FG |
142 | |
143 | # Find the standby replays | |
1adf2230 | 144 | for gid_str, daemon_info in six.iteritems(mdsmap['info']): |
7c673cae FG |
145 | if daemon_info['state'] != "up:standby-replay": |
146 | continue | |
147 | ||
148 | inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino") | |
31f18b77 | 149 | dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn") |
7c673cae | 150 | |
9f95a23c TL |
151 | events = self.get_rate("mds", daemon_info['name'], "mds_log.replayed") |
152 | if output_format not in ('json', 'json-pretty'): | |
153 | activity = "Evts: " + mgr_util.format_dimless(events, 5) + "/s" | |
7c673cae | 154 | |
11fdf7f2 TL |
155 | metadata = self.get_metadata('mds', daemon_info['name']) |
156 | mds_versions[metadata.get('ceph_version', "unknown")].append(daemon_info['name']) | |
157 | ||
9f95a23c TL |
158 | if output_format in ('json', 'json-pretty'): |
159 | json_output['mdsmap'].append({ | |
160 | 'rank': rank, | |
161 | 'name': daemon_info['name'], | |
162 | 'state': 'standby-replay', | |
163 | 'events': events, | |
164 | 'dns': 5, | |
165 | 'inos': 5 | |
166 | }) | |
167 | else: | |
168 | rank_table.add_row([ | |
169 | "{0}-s".format(daemon_info['rank']), "standby-replay", | |
170 | daemon_info['name'], activity, | |
171 | mgr_util.format_dimless(dns, 5), | |
172 | mgr_util.format_dimless(inos, 5) | |
173 | ]) | |
7c673cae FG |
174 | |
175 | df = self.get("df") | |
176 | pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) | |
177 | osdmap = self.get("osd_map") | |
178 | pools = dict([(p['pool'], p) for p in osdmap['pools']]) | |
179 | metadata_pool_id = mdsmap['metadata_pool'] | |
180 | data_pool_ids = mdsmap['data_pools'] | |
181 | ||
9f95a23c TL |
182 | pools_table = PrettyTable(["POOL", "TYPE", "USED", "AVAIL"], |
183 | border=False) | |
184 | pools_table.left_padding_width = 0 | |
185 | pools_table.right_padding_width = 2 | |
7c673cae FG |
186 | for pool_id in [metadata_pool_id] + data_pool_ids: |
187 | pool_type = "metadata" if pool_id == metadata_pool_id else "data" | |
188 | stats = pool_stats[pool_id] | |
9f95a23c TL |
189 | |
190 | if output_format in ('json', 'json-pretty'): | |
191 | json_output['pools'].append({ | |
192 | 'id': pool_id, | |
193 | 'name': pools[pool_id]['pool_name'], | |
194 | 'type': pool_type, | |
195 | 'used': stats['bytes_used'], | |
196 | 'avail': stats['max_avail'] | |
197 | }) | |
198 | else: | |
199 | pools_table.add_row([ | |
200 | pools[pool_id]['pool_name'], pool_type, | |
201 | mgr_util.format_bytes(stats['bytes_used'], 5), | |
202 | mgr_util.format_bytes(stats['max_avail'], 5) | |
203 | ]) | |
204 | ||
205 | if output_format in ('json', 'json-pretty'): | |
206 | json_output['clients'].append({ | |
207 | 'fs': mdsmap['fs_name'], | |
208 | 'clients': client_count, | |
209 | }) | |
210 | else: | |
211 | output += "{0} - {1} clients\n".format( | |
212 | mdsmap['fs_name'], client_count) | |
213 | output += "=" * len(mdsmap['fs_name']) + "\n" | |
214 | output += rank_table.get_string() | |
215 | output += "\n" + pools_table.get_string() + "\n" | |
216 | ||
217 | if not output and not json_output and fs_filter is not None: | |
11fdf7f2 TL |
218 | return errno.EINVAL, "", "Invalid filesystem: " + fs_filter |
219 | ||
9f95a23c TL |
220 | standby_table = PrettyTable(["STANDBY MDS"], border=False) |
221 | standby_table.left_padding_width = 0 | |
222 | standby_table.right_padding_width = 2 | |
7c673cae FG |
223 | for standby in fsmap['standbys']: |
224 | metadata = self.get_metadata('mds', standby['name']) | |
225 | mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name']) | |
226 | ||
9f95a23c TL |
227 | if output_format in ('json', 'json-pretty'): |
228 | json_output['mdsmap'].append({ | |
229 | 'name': standby['name'], | |
230 | 'state': "standby" | |
231 | }) | |
232 | else: | |
233 | standby_table.add_row([standby['name']]) | |
7c673cae | 234 | |
9f95a23c TL |
235 | if output_format not in ('json', 'json-pretty'): |
236 | output += "\n" + standby_table.get_string() + "\n" | |
7c673cae FG |
237 | |
238 | if len(mds_versions) == 1: | |
9f95a23c TL |
239 | if output_format in ('json', 'json-pretty'): |
240 | json_output['mds_version'] = list(mds_versions)[0] | |
241 | else: | |
242 | output += "MDS version: {0}".format(list(mds_versions)[0]) | |
7c673cae | 243 | else: |
9f95a23c TL |
244 | version_table = PrettyTable(["VERSION", "DAEMONS"], |
245 | border=False) | |
246 | version_table.left_padding_width = 0 | |
247 | version_table.right_padding_width = 2 | |
1adf2230 | 248 | for version, daemons in six.iteritems(mds_versions): |
9f95a23c TL |
249 | if output_format in ('json', 'json-pretty'): |
250 | json_output['mds_version'].append({ | |
251 | 'version': version, | |
252 | 'daemons': daemons | |
253 | }) | |
254 | else: | |
255 | version_table.add_row([ | |
256 | version, | |
257 | ", ".join(daemons) | |
258 | ]) | |
259 | if output_format not in ('json', 'json-pretty'): | |
260 | output += version_table.get_string() + "\n" | |
7c673cae | 261 | |
9f95a23c TL |
262 | if output_format == "json": |
263 | return 0, "", json.dumps(json_output, sort_keys=True) | |
264 | elif output_format == "json-pretty": | |
265 | return 0, "", json.dumps(json_output, sort_keys=True, indent=4, separators=(',', ': ')) | |
266 | else: | |
267 | return 0, output, "" | |
7c673cae | 268 | |
224ce89b | 269 | def handle_osd_status(self, cmd): |
9f95a23c TL |
270 | osd_table = PrettyTable(['ID', 'HOST', 'USED', 'AVAIL', 'WR OPS', |
271 | 'WR DATA', 'RD OPS', 'RD DATA', 'STATE'], | |
272 | border=False) | |
273 | osd_table.align['ID'] = 'r' | |
274 | osd_table.align['HOST'] = 'l' | |
275 | osd_table.align['USED'] = 'r' | |
276 | osd_table.align['AVAIL'] = 'r' | |
277 | osd_table.align['WR OPS'] = 'r' | |
278 | osd_table.align['WR DATA'] = 'r' | |
279 | osd_table.align['RD OPS'] = 'r' | |
280 | osd_table.align['RD DATA'] = 'r' | |
281 | osd_table.align['STATE'] = 'l' | |
282 | osd_table.left_padding_width = 0 | |
283 | osd_table.right_padding_width = 2 | |
7c673cae FG |
284 | osdmap = self.get("osd_map") |
285 | ||
224ce89b WB |
286 | filter_osds = set() |
287 | bucket_filter = None | |
288 | if 'bucket' in cmd: | |
289 | self.log.debug("Filtering to bucket '{0}'".format(cmd['bucket'])) | |
290 | bucket_filter = cmd['bucket'] | |
291 | crush = self.get("osd_map_crush") | |
292 | found = False | |
293 | for bucket in crush['buckets']: | |
294 | if fnmatch.fnmatch(bucket['name'], bucket_filter): | |
295 | found = True | |
296 | filter_osds.update([i['id'] for i in bucket['items']]) | |
297 | ||
298 | if not found: | |
299 | msg = "Bucket '{0}' not found".format(bucket_filter) | |
300 | return errno.ENOENT, msg, "" | |
301 | ||
7c673cae FG |
302 | # Build dict of OSD ID to stats |
303 | osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']]) | |
304 | ||
305 | for osd in osdmap['osds']: | |
306 | osd_id = osd['osd'] | |
224ce89b WB |
307 | if bucket_filter and osd_id not in filter_osds: |
308 | continue | |
309 | ||
b32b8144 FG |
310 | hostname = "" |
311 | kb_used = 0 | |
312 | kb_avail = 0 | |
7c673cae | 313 | |
b32b8144 FG |
314 | if osd_id in osd_stats: |
315 | metadata = self.get_metadata('osd', "%s" % osd_id) | |
316 | stats = osd_stats[osd_id] | |
317 | hostname = metadata['hostname'] | |
318 | kb_used = stats['kb_used'] * 1024 | |
319 | kb_avail = stats['kb_avail'] * 1024 | |
320 | ||
321 | osd_table.add_row([osd_id, hostname, | |
11fdf7f2 TL |
322 | mgr_util.format_bytes(kb_used, 5), |
323 | mgr_util.format_bytes(kb_avail, 5), | |
324 | mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") + | |
7c673cae | 325 | self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5), |
11fdf7f2 TL |
326 | mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5), |
327 | mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5), | |
328 | mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5), | |
b32b8144 | 329 | ','.join(osd['state']), |
7c673cae FG |
330 | ]) |
331 | ||
11fdf7f2 | 332 | return 0, osd_table.get_string(), "" |
7c673cae | 333 | |
11fdf7f2 | 334 | def handle_command(self, inbuf, cmd): |
7c673cae FG |
335 | self.log.error("handle_command") |
336 | ||
337 | if cmd['prefix'] == "fs status": | |
338 | return self.handle_fs_status(cmd) | |
224ce89b WB |
339 | elif cmd['prefix'] == "osd status": |
340 | return self.handle_osd_status(cmd) | |
7c673cae FG |
341 | else: |
342 | # mgr should respect our self.COMMANDS and not call us for | |
343 | # any prefix we don't advertise | |
344 | raise NotImplementedError(cmd['prefix']) |