]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | |
2 | """ | |
3 | High level status display commands | |
4 | """ | |
5 | ||
6 | from collections import defaultdict | |
7 | from prettytable import PrettyTable | |
1e59de90 | 8 | from typing import Any, Dict, List, Optional, Tuple, Union |
224ce89b | 9 | import errno |
1adf2230 | 10 | import fnmatch |
11fdf7f2 | 11 | import mgr_util |
9f95a23c | 12 | import json |
7c673cae | 13 | |
20effc67 | 14 | from mgr_module import CLIReadCommand, MgrModule, HandleCommandResult |
7c673cae FG |
15 | |
16 | ||
17 | class Module(MgrModule): | |
20effc67 | 18 | def get_latest(self, daemon_type: str, daemon_name: str, stat: str) -> int: |
7c673cae | 19 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] |
7c673cae FG |
20 | if data: |
21 | return data[-1][1] | |
22 | else: | |
23 | return 0 | |
24 | ||
20effc67 | 25 | def get_rate(self, daemon_type: str, daemon_name: str, stat: str) -> int: |
7c673cae | 26 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] |
2a845540 | 27 | if data and len(data) > 1 and (int(data[-1][0] - data[-2][0]) != 0): |
20effc67 | 28 | return (data[-1][1] - data[-2][1]) // int(data[-1][0] - data[-2][0]) |
7c673cae FG |
29 | else: |
30 | return 0 | |
31 | ||
20effc67 TL |
32 | @CLIReadCommand("fs status") |
33 | def handle_fs_status(self, | |
34 | fs: Optional[str] = None, | |
35 | format: str = 'plain') -> Tuple[int, str, str]: | |
36 | """ | |
37 | Show the status of a CephFS filesystem | |
38 | """ | |
7c673cae | 39 | output = "" |
20effc67 TL |
40 | json_output: Dict[str, List[Dict[str, Union[int, str, List[str]]]]] = \ |
41 | dict(mdsmap=[], | |
42 | pools=[], | |
43 | clients=[], | |
44 | mds_version=[]) | |
45 | output_format = format | |
7c673cae | 46 | |
20effc67 | 47 | fs_filter = fs |
224ce89b | 48 | |
7c673cae FG |
49 | mds_versions = defaultdict(list) |
50 | ||
51 | fsmap = self.get("fs_map") | |
52 | for filesystem in fsmap['filesystems']: | |
224ce89b WB |
53 | if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter: |
54 | continue | |
55 | ||
7c673cae | 56 | rank_table = PrettyTable( |
f67539c2 | 57 | ("RANK", "STATE", "MDS", "ACTIVITY", "DNS", "INOS", "DIRS", "CAPS"), |
9f95a23c | 58 | border=False, |
7c673cae | 59 | ) |
9f95a23c TL |
60 | rank_table.left_padding_width = 0 |
61 | rank_table.right_padding_width = 2 | |
7c673cae FG |
62 | |
63 | mdsmap = filesystem['mdsmap'] | |
64 | ||
65 | client_count = 0 | |
66 | ||
67 | for rank in mdsmap["in"]: | |
68 | up = "mds_{0}".format(rank) in mdsmap["up"] | |
69 | if up: | |
70 | gid = mdsmap['up']["mds_{0}".format(rank)] | |
71 | info = mdsmap['info']['gid_{0}'.format(gid)] | |
31f18b77 | 72 | dns = self.get_latest("mds", info['name'], "mds_mem.dn") |
7c673cae | 73 | inos = self.get_latest("mds", info['name'], "mds_mem.ino") |
f67539c2 TL |
74 | dirs = self.get_latest("mds", info['name'], "mds_mem.dir") |
75 | caps = self.get_latest("mds", info['name'], "mds_mem.cap") | |
7c673cae FG |
76 | |
77 | if rank == 0: | |
78 | client_count = self.get_latest("mds", info['name'], | |
79 | "mds_sessions.session_count") | |
80 | elif client_count == 0: | |
81 | # In case rank 0 was down, look at another rank's | |
82 | # sessionmap to get an indication of clients. | |
83 | client_count = self.get_latest("mds", info['name'], | |
84 | "mds_sessions.session_count") | |
85 | ||
86 | laggy = "laggy_since" in info | |
87 | ||
88 | state = info['state'].split(":")[1] | |
89 | if laggy: | |
90 | state += "(laggy)" | |
91 | if state == "active" and not laggy: | |
11fdf7f2 | 92 | c_state = mgr_util.colorize(state, mgr_util.GREEN) |
7c673cae | 93 | else: |
11fdf7f2 | 94 | c_state = mgr_util.colorize(state, mgr_util.YELLOW) |
7c673cae FG |
95 | |
96 | # Populate based on context of state, e.g. client | |
97 | # ops for an active daemon, replay progress, reconnect | |
98 | # progress | |
99 | activity = "" | |
100 | ||
101 | if state == "active": | |
20effc67 TL |
102 | rate = self.get_rate("mds", info['name'], |
103 | "mds_server.handle_client_request") | |
9f95a23c TL |
104 | if output_format not in ('json', 'json-pretty'): |
105 | activity = "Reqs: " + mgr_util.format_dimless(rate, 5) + "/s" | |
7c673cae | 106 | |
20effc67 TL |
107 | metadata = self.get_metadata('mds', info['name'], |
108 | default=defaultdict(lambda: 'unknown')) | |
109 | assert metadata | |
f6b5b4d7 TL |
110 | mds_versions[metadata['ceph_version']].append(info['name']) |
111 | ||
9f95a23c TL |
112 | if output_format in ('json', 'json-pretty'): |
113 | json_output['mdsmap'].append({ | |
114 | 'rank': rank, | |
115 | 'name': info['name'], | |
116 | 'state': state, | |
117 | 'rate': rate if state == "active" else "0", | |
118 | 'dns': dns, | |
f67539c2 TL |
119 | 'inos': inos, |
120 | 'dirs': dirs, | |
121 | 'caps': caps | |
9f95a23c TL |
122 | }) |
123 | else: | |
124 | rank_table.add_row([ | |
125 | mgr_util.bold(rank.__str__()), c_state, info['name'], | |
126 | activity, | |
127 | mgr_util.format_dimless(dns, 5), | |
f67539c2 TL |
128 | mgr_util.format_dimless(inos, 5), |
129 | mgr_util.format_dimless(dirs, 5), | |
130 | mgr_util.format_dimless(caps, 5) | |
9f95a23c | 131 | ]) |
7c673cae | 132 | else: |
9f95a23c TL |
133 | if output_format in ('json', 'json-pretty'): |
134 | json_output['mdsmap'].append({ | |
135 | 'rank': rank, | |
136 | 'state': "failed" | |
137 | }) | |
138 | else: | |
139 | rank_table.add_row([ | |
f67539c2 | 140 | rank, "failed", "", "", "", "", "", "" |
9f95a23c | 141 | ]) |
7c673cae FG |
142 | |
143 | # Find the standby replays | |
f67539c2 | 144 | for gid_str, daemon_info in mdsmap['info'].items(): |
7c673cae FG |
145 | if daemon_info['state'] != "up:standby-replay": |
146 | continue | |
147 | ||
148 | inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino") | |
31f18b77 | 149 | dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn") |
f67539c2 TL |
150 | dirs = self.get_latest("mds", daemon_info['name'], "mds_mem.dir") |
151 | caps = self.get_latest("mds", daemon_info['name'], "mds_mem.cap") | |
7c673cae | 152 | |
9f95a23c TL |
153 | events = self.get_rate("mds", daemon_info['name'], "mds_log.replayed") |
154 | if output_format not in ('json', 'json-pretty'): | |
155 | activity = "Evts: " + mgr_util.format_dimless(events, 5) + "/s" | |
7c673cae | 156 | |
20effc67 TL |
157 | metadata = self.get_metadata('mds', daemon_info['name'], |
158 | default=defaultdict(lambda: 'unknown')) | |
159 | assert metadata | |
f6b5b4d7 | 160 | mds_versions[metadata['ceph_version']].append(daemon_info['name']) |
11fdf7f2 | 161 | |
9f95a23c TL |
162 | if output_format in ('json', 'json-pretty'): |
163 | json_output['mdsmap'].append({ | |
164 | 'rank': rank, | |
165 | 'name': daemon_info['name'], | |
166 | 'state': 'standby-replay', | |
167 | 'events': events, | |
168 | 'dns': 5, | |
f67539c2 TL |
169 | 'inos': 5, |
170 | 'dirs': 5, | |
171 | 'caps': 5 | |
9f95a23c TL |
172 | }) |
173 | else: | |
174 | rank_table.add_row([ | |
175 | "{0}-s".format(daemon_info['rank']), "standby-replay", | |
176 | daemon_info['name'], activity, | |
177 | mgr_util.format_dimless(dns, 5), | |
f67539c2 TL |
178 | mgr_util.format_dimless(inos, 5), |
179 | mgr_util.format_dimless(dirs, 5), | |
180 | mgr_util.format_dimless(caps, 5) | |
9f95a23c | 181 | ]) |
7c673cae FG |
182 | |
183 | df = self.get("df") | |
184 | pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) | |
185 | osdmap = self.get("osd_map") | |
186 | pools = dict([(p['pool'], p) for p in osdmap['pools']]) | |
187 | metadata_pool_id = mdsmap['metadata_pool'] | |
188 | data_pool_ids = mdsmap['data_pools'] | |
189 | ||
9f95a23c TL |
190 | pools_table = PrettyTable(["POOL", "TYPE", "USED", "AVAIL"], |
191 | border=False) | |
192 | pools_table.left_padding_width = 0 | |
193 | pools_table.right_padding_width = 2 | |
7c673cae FG |
194 | for pool_id in [metadata_pool_id] + data_pool_ids: |
195 | pool_type = "metadata" if pool_id == metadata_pool_id else "data" | |
196 | stats = pool_stats[pool_id] | |
20effc67 | 197 | |
9f95a23c TL |
198 | if output_format in ('json', 'json-pretty'): |
199 | json_output['pools'].append({ | |
200 | 'id': pool_id, | |
201 | 'name': pools[pool_id]['pool_name'], | |
202 | 'type': pool_type, | |
203 | 'used': stats['bytes_used'], | |
204 | 'avail': stats['max_avail'] | |
205 | }) | |
206 | else: | |
207 | pools_table.add_row([ | |
208 | pools[pool_id]['pool_name'], pool_type, | |
209 | mgr_util.format_bytes(stats['bytes_used'], 5), | |
210 | mgr_util.format_bytes(stats['max_avail'], 5) | |
211 | ]) | |
20effc67 | 212 | |
9f95a23c TL |
213 | if output_format in ('json', 'json-pretty'): |
214 | json_output['clients'].append({ | |
215 | 'fs': mdsmap['fs_name'], | |
216 | 'clients': client_count, | |
217 | }) | |
218 | else: | |
219 | output += "{0} - {1} clients\n".format( | |
220 | mdsmap['fs_name'], client_count) | |
221 | output += "=" * len(mdsmap['fs_name']) + "\n" | |
222 | output += rank_table.get_string() | |
223 | output += "\n" + pools_table.get_string() + "\n" | |
224 | ||
225 | if not output and not json_output and fs_filter is not None: | |
11fdf7f2 TL |
226 | return errno.EINVAL, "", "Invalid filesystem: " + fs_filter |
227 | ||
9f95a23c TL |
228 | standby_table = PrettyTable(["STANDBY MDS"], border=False) |
229 | standby_table.left_padding_width = 0 | |
230 | standby_table.right_padding_width = 2 | |
7c673cae | 231 | for standby in fsmap['standbys']: |
20effc67 TL |
232 | metadata = self.get_metadata('mds', standby['name'], |
233 | default=defaultdict(lambda: 'unknown')) | |
234 | assert metadata | |
f6b5b4d7 | 235 | mds_versions[metadata['ceph_version']].append(standby['name']) |
7c673cae | 236 | |
9f95a23c TL |
237 | if output_format in ('json', 'json-pretty'): |
238 | json_output['mdsmap'].append({ | |
239 | 'name': standby['name'], | |
240 | 'state': "standby" | |
241 | }) | |
242 | else: | |
243 | standby_table.add_row([standby['name']]) | |
7c673cae | 244 | |
9f95a23c TL |
245 | if output_format not in ('json', 'json-pretty'): |
246 | output += "\n" + standby_table.get_string() + "\n" | |
7c673cae FG |
247 | |
248 | if len(mds_versions) == 1: | |
9f95a23c | 249 | if output_format in ('json', 'json-pretty'): |
20effc67 TL |
250 | json_output['mds_version'] = [dict(version=k, daemon=v) |
251 | for k, v in mds_versions.items()] | |
9f95a23c | 252 | else: |
20effc67 | 253 | output += "MDS version: {0}".format([*mds_versions][0]) |
7c673cae | 254 | else: |
9f95a23c TL |
255 | version_table = PrettyTable(["VERSION", "DAEMONS"], |
256 | border=False) | |
257 | version_table.left_padding_width = 0 | |
258 | version_table.right_padding_width = 2 | |
f67539c2 | 259 | for version, daemons in mds_versions.items(): |
9f95a23c TL |
260 | if output_format in ('json', 'json-pretty'): |
261 | json_output['mds_version'].append({ | |
262 | 'version': version, | |
263 | 'daemons': daemons | |
264 | }) | |
265 | else: | |
266 | version_table.add_row([ | |
267 | version, | |
268 | ", ".join(daemons) | |
269 | ]) | |
270 | if output_format not in ('json', 'json-pretty'): | |
271 | output += version_table.get_string() + "\n" | |
7c673cae | 272 | |
9f95a23c | 273 | if output_format == "json": |
e306af50 | 274 | return HandleCommandResult(stdout=json.dumps(json_output, sort_keys=True)) |
9f95a23c | 275 | elif output_format == "json-pretty": |
20effc67 TL |
276 | return HandleCommandResult(stdout=json.dumps(json_output, sort_keys=True, indent=4, |
277 | separators=(',', ': '))) | |
9f95a23c | 278 | else: |
e306af50 | 279 | return HandleCommandResult(stdout=output) |
7c673cae | 280 | |
20effc67 | 281 | @CLIReadCommand("osd status") |
1e59de90 | 282 | def handle_osd_status(self, bucket: Optional[str] = None, format: str = 'plain') -> Tuple[int, str, str]: |
20effc67 TL |
283 | """ |
284 | Show the status of OSDs within a bucket, or all | |
285 | """ | |
1e59de90 TL |
286 | json_output: Dict[str, List[Any]] = \ |
287 | dict(OSDs=[]) | |
288 | output_format = format | |
289 | ||
9f95a23c TL |
290 | osd_table = PrettyTable(['ID', 'HOST', 'USED', 'AVAIL', 'WR OPS', |
291 | 'WR DATA', 'RD OPS', 'RD DATA', 'STATE'], | |
292 | border=False) | |
293 | osd_table.align['ID'] = 'r' | |
294 | osd_table.align['HOST'] = 'l' | |
295 | osd_table.align['USED'] = 'r' | |
296 | osd_table.align['AVAIL'] = 'r' | |
297 | osd_table.align['WR OPS'] = 'r' | |
298 | osd_table.align['WR DATA'] = 'r' | |
299 | osd_table.align['RD OPS'] = 'r' | |
300 | osd_table.align['RD DATA'] = 'r' | |
301 | osd_table.align['STATE'] = 'l' | |
302 | osd_table.left_padding_width = 0 | |
303 | osd_table.right_padding_width = 2 | |
7c673cae FG |
304 | osdmap = self.get("osd_map") |
305 | ||
224ce89b WB |
306 | filter_osds = set() |
307 | bucket_filter = None | |
20effc67 TL |
308 | if bucket is not None: |
309 | self.log.debug(f"Filtering to bucket '{bucket}'") | |
310 | bucket_filter = bucket | |
224ce89b WB |
311 | crush = self.get("osd_map_crush") |
312 | found = False | |
20effc67 TL |
313 | for bucket_ in crush['buckets']: |
314 | if fnmatch.fnmatch(bucket_['name'], bucket_filter): | |
224ce89b | 315 | found = True |
20effc67 | 316 | filter_osds.update([i['id'] for i in bucket_['items']]) |
224ce89b WB |
317 | |
318 | if not found: | |
319 | msg = "Bucket '{0}' not found".format(bucket_filter) | |
320 | return errno.ENOENT, msg, "" | |
321 | ||
7c673cae FG |
322 | # Build dict of OSD ID to stats |
323 | osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']]) | |
324 | ||
325 | for osd in osdmap['osds']: | |
326 | osd_id = osd['osd'] | |
224ce89b WB |
327 | if bucket_filter and osd_id not in filter_osds: |
328 | continue | |
329 | ||
b32b8144 FG |
330 | hostname = "" |
331 | kb_used = 0 | |
332 | kb_avail = 0 | |
7c673cae | 333 | |
b32b8144 | 334 | if osd_id in osd_stats: |
20effc67 | 335 | metadata = self.get_metadata('osd', str(osd_id), default=defaultdict(str)) |
b32b8144 | 336 | stats = osd_stats[osd_id] |
20effc67 | 337 | assert metadata |
b32b8144 FG |
338 | hostname = metadata['hostname'] |
339 | kb_used = stats['kb_used'] * 1024 | |
340 | kb_avail = stats['kb_avail'] * 1024 | |
341 | ||
20effc67 TL |
342 | wr_ops_rate = (self.get_rate("osd", osd_id.__str__(), "osd.op_w") + |
343 | self.get_rate("osd", osd_id.__str__(), "osd.op_rw")) | |
344 | wr_byte_rate = self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes") | |
345 | rd_ops_rate = self.get_rate("osd", osd_id.__str__(), "osd.op_r") | |
346 | rd_byte_rate = self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes") | |
b32b8144 | 347 | osd_table.add_row([osd_id, hostname, |
11fdf7f2 TL |
348 | mgr_util.format_bytes(kb_used, 5), |
349 | mgr_util.format_bytes(kb_avail, 5), | |
20effc67 TL |
350 | mgr_util.format_dimless(wr_ops_rate, 5), |
351 | mgr_util.format_bytes(wr_byte_rate, 5), | |
352 | mgr_util.format_dimless(rd_ops_rate, 5), | |
353 | mgr_util.format_bytes(rd_byte_rate, 5), | |
b32b8144 | 354 | ','.join(osd['state']), |
7c673cae | 355 | ]) |
1e59de90 TL |
356 | if output_format in ('json', 'json-pretty'): |
357 | json_output['OSDs'].append({ | |
358 | 'id': osd_id, | |
359 | 'host name': hostname, | |
360 | 'kb used' : kb_used, | |
361 | 'kb available':kb_avail, | |
362 | 'write ops rate': wr_ops_rate, | |
363 | 'write byte rate': wr_byte_rate, | |
364 | 'read ops rate': rd_ops_rate, | |
365 | 'read byte rate': rd_byte_rate, | |
366 | 'state': osd['state'] | |
367 | }) | |
7c673cae | 368 | |
1e59de90 TL |
369 | if output_format == "json": |
370 | return 0, json.dumps(json_output, sort_keys=True) , "" | |
371 | elif output_format == "json-pretty": | |
372 | return 0, json.dumps(json_output, sort_keys=True,indent=4,separators=(',', ': ')) , "" | |
373 | else: | |
374 | return 0, osd_table.get_string(), "" |