ceph/src/pybind/mgr/status/module.py

   1
   2 """
   3 High level status display commands
   4 """
   5
   6 from collections import defaultdict
   7 from prettytable import PrettyTable
   8 from typing import Any, Dict, List, Optional, Tuple, Union
   9 import errno
  10 import fnmatch
  11 import mgr_util
  12 import json
  13
  14 from mgr_module import CLIReadCommand, MgrModule, HandleCommandResult
  15
  16
  17 class Module(MgrModule):
  18     def get_latest(self, daemon_type: str, daemon_name: str, stat: str) -> int:
  19         data = self.get_counter(daemon_type, daemon_name, stat)[stat]
  20         if data:
  21             return data[-1][1]
  22         else:
  23             return 0
  24
  25     def get_rate(self, daemon_type: str, daemon_name: str, stat: str) -> int:
  26         data = self.get_counter(daemon_type, daemon_name, stat)[stat]
  27         if data and len(data) > 1 and (int(data[-1][0] - data[-2][0]) != 0):
  28             return (data[-1][1] - data[-2][1]) // int(data[-1][0] - data[-2][0])
  29         else:
  30             return 0
  31
  32     @CLIReadCommand("fs status")
  33     def handle_fs_status(self,
  34                          fs: Optional[str] = None,
  35                          format: str = 'plain') -> Tuple[int, str, str]:
  36         """
  37         Show the status of a CephFS filesystem
  38         """
  39         output = ""
  40         json_output: Dict[str, List[Dict[str, Union[int, str, List[str]]]]] = \
  41             dict(mdsmap=[],
  42                  pools=[],
  43                  clients=[],
  44                  mds_version=[])
  45         output_format = format
  46
  47         fs_filter = fs
  48
  49         mds_versions = defaultdict(list)
  50
  51         fsmap = self.get("fs_map")
  52         for filesystem in fsmap['filesystems']:
  53             if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter:
  54                 continue
  55
  56             rank_table = PrettyTable(
  57                 ("RANK", "STATE", "MDS", "ACTIVITY", "DNS", "INOS", "DIRS", "CAPS"),
  58                 border=False,
  59             )
  60             rank_table.left_padding_width = 0
  61             rank_table.right_padding_width = 2
  62
  63             mdsmap = filesystem['mdsmap']
  64
  65             client_count = 0
  66
  67             for rank in mdsmap["in"]:
  68                 up = "mds_{0}".format(rank) in mdsmap["up"]
  69                 if up:
  70                     gid = mdsmap['up']["mds_{0}".format(rank)]
  71                     info = mdsmap['info']['gid_{0}'.format(gid)]
  72                     dns = self.get_latest("mds", info['name'], "mds_mem.dn")
  73                     inos = self.get_latest("mds", info['name'], "mds_mem.ino")
  74                     dirs = self.get_latest("mds", info['name'], "mds_mem.dir")
  75                     caps = self.get_latest("mds", info['name'], "mds_mem.cap")
  76
  77                     if rank == 0:
  78                         client_count = self.get_latest("mds", info['name'],
  79                                                        "mds_sessions.session_count")
  80                     elif client_count == 0:
  81                         # In case rank 0 was down, look at another rank's
  82                         # sessionmap to get an indication of clients.
  83                         client_count = self.get_latest("mds", info['name'],
  84                                                        "mds_sessions.session_count")
  85
  86                     laggy = "laggy_since" in info
  87
  88                     state = info['state'].split(":")[1]
  89                     if laggy:
  90                         state += "(laggy)"
  91                     if state == "active" and not laggy:
  92                         c_state = mgr_util.colorize(state, mgr_util.GREEN)
  93                     else:
  94                         c_state = mgr_util.colorize(state, mgr_util.YELLOW)
  95
  96                     # Populate based on context of state, e.g. client
  97                     # ops for an active daemon, replay progress, reconnect
  98                     # progress
  99                     activity = ""
 100
 101                     if state == "active":
 102                         rate = self.get_rate("mds", info['name'],
 103                                              "mds_server.handle_client_request")
 104                         if output_format not in ('json', 'json-pretty'):
 105                             activity = "Reqs: " + mgr_util.format_dimless(rate, 5) + "/s"
 106
 107                     metadata = self.get_metadata('mds', info['name'],
 108                                                  default=defaultdict(lambda: 'unknown'))
 109                     assert metadata
 110                     mds_versions[metadata['ceph_version']].append(info['name'])
 111
 112                     if output_format in ('json', 'json-pretty'):
 113                         json_output['mdsmap'].append({
 114                             'rank': rank,
 115                             'name': info['name'],
 116                             'state': state,
 117                             'rate': rate if state == "active" else "0",
 118                             'dns': dns,
 119                             'inos': inos,
 120                             'dirs': dirs,
 121                             'caps': caps
 122                         })
 123                     else:
 124                         rank_table.add_row([
 125                             mgr_util.bold(rank.__str__()), c_state, info['name'],
 126                             activity,
 127                             mgr_util.format_dimless(dns, 5),
 128                             mgr_util.format_dimless(inos, 5),
 129                             mgr_util.format_dimless(dirs, 5),
 130                             mgr_util.format_dimless(caps, 5)
 131                         ])
 132                 else:
 133                     if output_format in ('json', 'json-pretty'):
 134                         json_output['mdsmap'].append({
 135                             'rank': rank,
 136                             'state': "failed"
 137                         })
 138                     else:
 139                         rank_table.add_row([
 140                             rank, "failed", "", "", "", "", "", ""
 141                         ])
 142
 143             # Find the standby replays
 144             for gid_str, daemon_info in mdsmap['info'].items():
 145                 if daemon_info['state'] != "up:standby-replay":
 146                     continue
 147
 148                 inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino")
 149                 dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn")
 150                 dirs = self.get_latest("mds", daemon_info['name'], "mds_mem.dir")
 151                 caps = self.get_latest("mds", daemon_info['name'], "mds_mem.cap")
 152
 153                 events = self.get_rate("mds", daemon_info['name'], "mds_log.replayed")
 154                 if output_format not in ('json', 'json-pretty'):
 155                     activity = "Evts: " + mgr_util.format_dimless(events, 5) + "/s"
 156
 157                 metadata = self.get_metadata('mds', daemon_info['name'],
 158                                              default=defaultdict(lambda: 'unknown'))
 159                 assert metadata
 160                 mds_versions[metadata['ceph_version']].append(daemon_info['name'])
 161
 162                 if output_format in ('json', 'json-pretty'):
 163                     json_output['mdsmap'].append({
 164                         'rank': rank,
 165                         'name': daemon_info['name'],
 166                         'state': 'standby-replay',
 167                         'events': events,
 168                         'dns': 5,
 169                         'inos': 5,
 170                         'dirs': 5,
 171                         'caps': 5
 172                     })
 173                 else:
 174                     rank_table.add_row([
 175                         "{0}-s".format(daemon_info['rank']), "standby-replay",
 176                         daemon_info['name'], activity,
 177                         mgr_util.format_dimless(dns, 5),
 178                         mgr_util.format_dimless(inos, 5),
 179                         mgr_util.format_dimless(dirs, 5),
 180                         mgr_util.format_dimless(caps, 5)
 181                     ])
 182
 183             df = self.get("df")
 184             pool_stats = dict([(p['id'], p['stats']) for p in df['pools']])
 185             osdmap = self.get("osd_map")
 186             pools = dict([(p['pool'], p) for p in osdmap['pools']])
 187             metadata_pool_id = mdsmap['metadata_pool']
 188             data_pool_ids = mdsmap['data_pools']
 189
 190             pools_table = PrettyTable(["POOL", "TYPE", "USED", "AVAIL"],
 191                                       border=False)
 192             pools_table.left_padding_width = 0
 193             pools_table.right_padding_width = 2
 194             for pool_id in [metadata_pool_id] + data_pool_ids:
 195                 pool_type = "metadata" if pool_id == metadata_pool_id else "data"
 196                 stats = pool_stats[pool_id]
 197
 198                 if output_format in ('json', 'json-pretty'):
 199                     json_output['pools'].append({
 200                         'id': pool_id,
 201                         'name': pools[pool_id]['pool_name'],
 202                         'type': pool_type,
 203                         'used': stats['bytes_used'],
 204                         'avail': stats['max_avail']
 205                     })
 206                 else:
 207                     pools_table.add_row([
 208                         pools[pool_id]['pool_name'], pool_type,
 209                         mgr_util.format_bytes(stats['bytes_used'], 5),
 210                         mgr_util.format_bytes(stats['max_avail'], 5)
 211                     ])
 212
 213             if output_format in ('json', 'json-pretty'):
 214                 json_output['clients'].append({
 215                     'fs': mdsmap['fs_name'],
 216                     'clients': client_count,
 217                 })
 218             else:
 219                 output += "{0} - {1} clients\n".format(
 220                     mdsmap['fs_name'], client_count)
 221                 output += "=" * len(mdsmap['fs_name']) + "\n"
 222                 output += rank_table.get_string()
 223                 output += "\n" + pools_table.get_string() + "\n"
 224
 225         if not output and not json_output and fs_filter is not None:
 226             return errno.EINVAL, "", "Invalid filesystem: " + fs_filter
 227
 228         standby_table = PrettyTable(["STANDBY MDS"], border=False)
 229         standby_table.left_padding_width = 0
 230         standby_table.right_padding_width = 2
 231         for standby in fsmap['standbys']:
 232             metadata = self.get_metadata('mds', standby['name'],
 233                                          default=defaultdict(lambda: 'unknown'))
 234             assert metadata
 235             mds_versions[metadata['ceph_version']].append(standby['name'])
 236
 237             if output_format in ('json', 'json-pretty'):
 238                 json_output['mdsmap'].append({
 239                     'name': standby['name'],
 240                     'state': "standby"
 241                 })
 242             else:
 243                 standby_table.add_row([standby['name']])
 244
 245         if output_format not in ('json', 'json-pretty'):
 246             output += "\n" + standby_table.get_string() + "\n"
 247
 248         if len(mds_versions) == 1:
 249             if output_format in ('json', 'json-pretty'):
 250                 json_output['mds_version'] = [dict(version=k, daemon=v)
 251                                               for k, v in mds_versions.items()]
 252             else:
 253                 output += "MDS version: {0}".format([*mds_versions][0])
 254         else:
 255             version_table = PrettyTable(["VERSION", "DAEMONS"],
 256                                         border=False)
 257             version_table.left_padding_width = 0
 258             version_table.right_padding_width = 2
 259             for version, daemons in mds_versions.items():
 260                 if output_format in ('json', 'json-pretty'):
 261                     json_output['mds_version'].append({
 262                         'version': version,
 263                         'daemons': daemons
 264                     })
 265                 else:
 266                     version_table.add_row([
 267                         version,
 268                         ", ".join(daemons)
 269                     ])
 270             if output_format not in ('json', 'json-pretty'):
 271                 output += version_table.get_string() + "\n"
 272
 273         if output_format == "json":
 274             return HandleCommandResult(stdout=json.dumps(json_output, sort_keys=True))
 275         elif output_format == "json-pretty":
 276             return HandleCommandResult(stdout=json.dumps(json_output, sort_keys=True, indent=4,
 277                                                          separators=(',', ': ')))
 278         else:
 279             return HandleCommandResult(stdout=output)
 280
 281     @CLIReadCommand("osd status")
 282     def handle_osd_status(self, bucket: Optional[str] = None, format: str = 'plain') -> Tuple[int, str, str]:
 283         """
 284         Show the status of OSDs within a bucket, or all
 285         """
 286         json_output: Dict[str, List[Any]] = \
 287             dict(OSDs=[])
 288         output_format = format
 289
 290         osd_table = PrettyTable(['ID', 'HOST', 'USED', 'AVAIL', 'WR OPS',
 291                                  'WR DATA', 'RD OPS', 'RD DATA', 'STATE'],
 292                                 border=False)
 293         osd_table.align['ID'] = 'r'
 294         osd_table.align['HOST'] = 'l'
 295         osd_table.align['USED'] = 'r'
 296         osd_table.align['AVAIL'] = 'r'
 297         osd_table.align['WR OPS'] = 'r'
 298         osd_table.align['WR DATA'] = 'r'
 299         osd_table.align['RD OPS'] = 'r'
 300         osd_table.align['RD DATA'] = 'r'
 301         osd_table.align['STATE'] = 'l'
 302         osd_table.left_padding_width = 0
 303         osd_table.right_padding_width = 2
 304         osdmap = self.get("osd_map")
 305
 306         filter_osds = set()
 307         bucket_filter = None
 308         if bucket is not None:
 309             self.log.debug(f"Filtering to bucket '{bucket}'")
 310             bucket_filter = bucket
 311             crush = self.get("osd_map_crush")
 312             found = False
 313             for bucket_ in crush['buckets']:
 314                 if fnmatch.fnmatch(bucket_['name'], bucket_filter):
 315                     found = True
 316                     filter_osds.update([i['id'] for i in bucket_['items']])
 317
 318             if not found:
 319                 msg = "Bucket '{0}' not found".format(bucket_filter)
 320                 return errno.ENOENT, msg, ""
 321
 322         # Build dict of OSD ID to stats
 323         osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']])
 324
 325         for osd in osdmap['osds']:
 326             osd_id = osd['osd']
 327             if bucket_filter and osd_id not in filter_osds:
 328                 continue
 329
 330             hostname = ""
 331             kb_used = 0
 332             kb_avail = 0
 333
 334             if osd_id in osd_stats:
 335                 metadata = self.get_metadata('osd', str(osd_id), default=defaultdict(str))
 336                 stats = osd_stats[osd_id]
 337                 assert metadata
 338                 hostname = metadata['hostname']
 339                 kb_used = stats['kb_used'] * 1024
 340                 kb_avail = stats['kb_avail'] * 1024
 341
 342             wr_ops_rate = (self.get_rate("osd", osd_id.__str__(), "osd.op_w") +
 343                            self.get_rate("osd", osd_id.__str__(), "osd.op_rw"))
 344             wr_byte_rate = self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes")
 345             rd_ops_rate = self.get_rate("osd", osd_id.__str__(), "osd.op_r")
 346             rd_byte_rate = self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes")
 347             osd_table.add_row([osd_id, hostname,
 348                                mgr_util.format_bytes(kb_used, 5),
 349                                mgr_util.format_bytes(kb_avail, 5),
 350                                mgr_util.format_dimless(wr_ops_rate, 5),
 351                                mgr_util.format_bytes(wr_byte_rate, 5),
 352                                mgr_util.format_dimless(rd_ops_rate, 5),
 353                                mgr_util.format_bytes(rd_byte_rate, 5),
 354                                ','.join(osd['state']),
 355                                ])
 356             if output_format in ('json', 'json-pretty'):
 357                 json_output['OSDs'].append({
 358                                         'id': osd_id,
 359                                         'host name': hostname,
 360                                         'kb used' : kb_used,
 361                                         'kb available':kb_avail,
 362                                         'write ops rate': wr_ops_rate,
 363                                         'write byte rate': wr_byte_rate,
 364                                         'read ops rate': rd_ops_rate,
 365                                         'read byte rate': rd_byte_rate,
 366                                         'state': osd['state']
 367                                       })
 368
 369         if output_format == "json":
 370             return 0, json.dumps(json_output, sort_keys=True) , ""
 371         elif output_format == "json-pretty":
 372             return 0, json.dumps(json_output, sort_keys=True,indent=4,separators=(',', ': ')) , ""
 373         else:
 374             return 0, osd_table.get_string(), ""