ceph/src/pybind/mgr/status/module.py

   1
   2 """
   3 High level status display commands
   4 """
   5
   6 from collections import defaultdict
   7 from prettytable import PrettyTable
   8 import errno
   9 import fnmatch
  10 import mgr_util
  11 import prettytable
  12 import six
  13 import json
  14
  15 from mgr_module import MgrModule, HandleCommandResult
  16
  17
  18 class Module(MgrModule):
  19     COMMANDS = [
  20         {
  21             "cmd": "fs status "
  22                    "name=fs,type=CephString,req=false",
  23             "desc": "Show the status of a CephFS filesystem",
  24             "perm": "r"
  25         },
  26         {
  27             "cmd": "osd status "
  28                    "name=bucket,type=CephString,req=false",
  29             "desc": "Show the status of OSDs within a bucket, or all",
  30             "perm": "r"
  31         },
  32     ]
  33
  34
  35     def get_latest(self, daemon_type, daemon_name, stat):
  36         data = self.get_counter(daemon_type, daemon_name, stat)[stat]
  37         #self.log.error("get_latest {0} data={1}".format(stat, data))
  38         if data:
  39             return data[-1][1]
  40         else:
  41             return 0
  42
  43     def get_rate(self, daemon_type, daemon_name, stat):
  44         data = self.get_counter(daemon_type, daemon_name, stat)[stat]
  45
  46         #self.log.error("get_latest {0} data={1}".format(stat, data))
  47         if data and len(data) > 1 and data[-1][0] != data[-2][0]:
  48             return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0])
  49         else:
  50             return 0
  51
  52     def handle_fs_status(self, cmd):
  53         output = ""
  54         json_output = defaultdict(list)
  55         output_format = cmd.get('format', 'plain')
  56
  57         fs_filter = cmd.get('fs', None)
  58
  59         mds_versions = defaultdict(list)
  60
  61         fsmap = self.get("fs_map")
  62         for filesystem in fsmap['filesystems']:
  63             if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter:
  64                 continue
  65
  66             rank_table = PrettyTable(
  67                 ("RANK", "STATE", "MDS", "ACTIVITY", "DNS", "INOS"),
  68                 border=False,
  69             )
  70             rank_table.left_padding_width = 0
  71             rank_table.right_padding_width = 2
  72
  73             mdsmap = filesystem['mdsmap']
  74
  75             client_count = 0
  76
  77             for rank in mdsmap["in"]:
  78                 up = "mds_{0}".format(rank) in mdsmap["up"]
  79                 if up:
  80                     gid = mdsmap['up']["mds_{0}".format(rank)]
  81                     info = mdsmap['info']['gid_{0}'.format(gid)]
  82                     dns = self.get_latest("mds", info['name'], "mds_mem.dn")
  83                     inos = self.get_latest("mds", info['name'], "mds_mem.ino")
  84
  85                     if rank == 0:
  86                         client_count = self.get_latest("mds", info['name'],
  87                                                        "mds_sessions.session_count")
  88                     elif client_count == 0:
  89                         # In case rank 0 was down, look at another rank's
  90                         # sessionmap to get an indication of clients.
  91                         client_count = self.get_latest("mds", info['name'],
  92                                                        "mds_sessions.session_count")
  93
  94                     laggy = "laggy_since" in info
  95
  96                     state = info['state'].split(":")[1]
  97                     if laggy:
  98                         state += "(laggy)"
  99                     if state == "active" and not laggy:
 100                         c_state = mgr_util.colorize(state, mgr_util.GREEN)
 101                     else:
 102                         c_state = mgr_util.colorize(state, mgr_util.YELLOW)
 103
 104                     # Populate based on context of state, e.g. client
 105                     # ops for an active daemon, replay progress, reconnect
 106                     # progress
 107                     activity = ""
 108
 109                     if state == "active":
 110                         rate = self.get_rate("mds", info['name'], "mds_server.handle_client_request")
 111                         if output_format not in ('json', 'json-pretty'):
 112                             activity = "Reqs: " + mgr_util.format_dimless(rate, 5) + "/s"
 113
 114                     defaults = defaultdict(lambda: None, {'version' : 'unknown'})
 115                     metadata = self.get_metadata('mds', info['name'], default=defaults)
 116                     mds_versions[metadata['ceph_version']].append(info['name'])
 117
 118                     if output_format in ('json', 'json-pretty'):
 119                         json_output['mdsmap'].append({
 120                             'rank': rank,
 121                             'name': info['name'],
 122                             'state': state,
 123                             'rate': rate if state == "active" else "0",
 124                             'dns': dns,
 125                             'inos': inos
 126                         })
 127                     else:
 128                         rank_table.add_row([
 129                             mgr_util.bold(rank.__str__()), c_state, info['name'],
 130                             activity,
 131                             mgr_util.format_dimless(dns, 5),
 132                             mgr_util.format_dimless(inos, 5)
 133                         ])
 134                 else:
 135                     if output_format in ('json', 'json-pretty'):
 136                         json_output['mdsmap'].append({
 137                             'rank': rank,
 138                             'state': "failed"
 139                         })
 140                     else:
 141                         rank_table.add_row([
 142                             rank, "failed", "", "", "", ""
 143                         ])
 144
 145             # Find the standby replays
 146             for gid_str, daemon_info in six.iteritems(mdsmap['info']):
 147                 if daemon_info['state'] != "up:standby-replay":
 148                     continue
 149
 150                 inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino")
 151                 dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn")
 152
 153                 events = self.get_rate("mds", daemon_info['name'], "mds_log.replayed")
 154                 if output_format not in ('json', 'json-pretty'):
 155                     activity = "Evts: " + mgr_util.format_dimless(events, 5) + "/s"
 156
 157                 defaults = defaultdict(lambda: None, {'version' : 'unknown'})
 158                 metadata = self.get_metadata('mds', daemon_info['name'], default=defaults)
 159                 mds_versions[metadata['ceph_version']].append(daemon_info['name'])
 160
 161                 if output_format in ('json', 'json-pretty'):
 162                     json_output['mdsmap'].append({
 163                         'rank': rank,
 164                         'name': daemon_info['name'],
 165                         'state': 'standby-replay',
 166                         'events': events,
 167                         'dns': 5,
 168                         'inos': 5
 169                     })
 170                 else:
 171                     rank_table.add_row([
 172                         "{0}-s".format(daemon_info['rank']), "standby-replay",
 173                         daemon_info['name'], activity,
 174                         mgr_util.format_dimless(dns, 5),
 175                         mgr_util.format_dimless(inos, 5)
 176                     ])
 177
 178             df = self.get("df")
 179             pool_stats = dict([(p['id'], p['stats']) for p in df['pools']])
 180             osdmap = self.get("osd_map")
 181             pools = dict([(p['pool'], p) for p in osdmap['pools']])
 182             metadata_pool_id = mdsmap['metadata_pool']
 183             data_pool_ids = mdsmap['data_pools']
 184
 185             pools_table = PrettyTable(["POOL", "TYPE", "USED", "AVAIL"],
 186                                       border=False)
 187             pools_table.left_padding_width = 0
 188             pools_table.right_padding_width = 2
 189             for pool_id in [metadata_pool_id] + data_pool_ids:
 190                 pool_type = "metadata" if pool_id == metadata_pool_id else "data"
 191                 stats = pool_stats[pool_id]
 192
 193                 if output_format in ('json', 'json-pretty'):
 194                     json_output['pools'].append({
 195                         'id': pool_id,
 196                         'name': pools[pool_id]['pool_name'],
 197                         'type': pool_type,
 198                         'used': stats['bytes_used'],
 199                         'avail': stats['max_avail']
 200                     })
 201                 else:
 202                     pools_table.add_row([
 203                         pools[pool_id]['pool_name'], pool_type,
 204                         mgr_util.format_bytes(stats['bytes_used'], 5),
 205                         mgr_util.format_bytes(stats['max_avail'], 5)
 206                     ])
 207
 208             if output_format in ('json', 'json-pretty'):
 209                 json_output['clients'].append({
 210                     'fs': mdsmap['fs_name'],
 211                     'clients': client_count,
 212                 })
 213             else:
 214                 output += "{0} - {1} clients\n".format(
 215                     mdsmap['fs_name'], client_count)
 216                 output += "=" * len(mdsmap['fs_name']) + "\n"
 217                 output += rank_table.get_string()
 218                 output += "\n" + pools_table.get_string() + "\n"
 219
 220         if not output and not json_output and fs_filter is not None:
 221             return errno.EINVAL, "", "Invalid filesystem: " + fs_filter
 222
 223         standby_table = PrettyTable(["STANDBY MDS"], border=False)
 224         standby_table.left_padding_width = 0
 225         standby_table.right_padding_width = 2
 226         for standby in fsmap['standbys']:
 227             defaults = defaultdict(lambda: None, {'version' : 'unknown'})
 228             metadata = self.get_metadata('mds', standby['name'], default=defaults)
 229             mds_versions[metadata['ceph_version']].append(standby['name'])
 230
 231             if output_format in ('json', 'json-pretty'):
 232                 json_output['mdsmap'].append({
 233                     'name': standby['name'],
 234                     'state': "standby"
 235                 })
 236             else:
 237                 standby_table.add_row([standby['name']])
 238
 239         if output_format not in ('json', 'json-pretty'):
 240             output += "\n" + standby_table.get_string() + "\n"
 241
 242         if len(mds_versions) == 1:
 243             if output_format in ('json', 'json-pretty'):
 244                 json_output['mds_version'] = list(mds_versions)[0]
 245             else:
 246                 output += "MDS version: {0}".format(list(mds_versions)[0])
 247         else:
 248             version_table = PrettyTable(["VERSION", "DAEMONS"],
 249                                         border=False)
 250             version_table.left_padding_width = 0
 251             version_table.right_padding_width = 2
 252             for version, daemons in six.iteritems(mds_versions):
 253                 if output_format in ('json', 'json-pretty'):
 254                     json_output['mds_version'].append({
 255                         'version': version,
 256                         'daemons': daemons
 257                     })
 258                 else:
 259                     version_table.add_row([
 260                         version,
 261                         ", ".join(daemons)
 262                     ])
 263             if output_format not in ('json', 'json-pretty'):
 264                 output += version_table.get_string() + "\n"
 265
 266         if output_format == "json":
 267             return HandleCommandResult(stdout=json.dumps(json_output, sort_keys=True))
 268         elif output_format == "json-pretty":
 269             return HandleCommandResult(stdout=json.dumps(json_output, sort_keys=True, indent=4, separators=(',', ': ')))
 270         else:
 271             return HandleCommandResult(stdout=output)
 272
 273     def handle_osd_status(self, cmd):
 274         osd_table = PrettyTable(['ID', 'HOST', 'USED', 'AVAIL', 'WR OPS',
 275                                  'WR DATA', 'RD OPS', 'RD DATA', 'STATE'],
 276                                 border=False)
 277         osd_table.align['ID'] = 'r'
 278         osd_table.align['HOST'] = 'l'
 279         osd_table.align['USED'] = 'r'
 280         osd_table.align['AVAIL'] = 'r'
 281         osd_table.align['WR OPS'] = 'r'
 282         osd_table.align['WR DATA'] = 'r'
 283         osd_table.align['RD OPS'] = 'r'
 284         osd_table.align['RD DATA'] = 'r'
 285         osd_table.align['STATE'] = 'l'
 286         osd_table.left_padding_width = 0
 287         osd_table.right_padding_width = 2
 288         osdmap = self.get("osd_map")
 289
 290         filter_osds = set()
 291         bucket_filter = None
 292         if 'bucket' in cmd:
 293             self.log.debug("Filtering to bucket '{0}'".format(cmd['bucket']))
 294             bucket_filter = cmd['bucket']
 295             crush = self.get("osd_map_crush")
 296             found = False
 297             for bucket in crush['buckets']:
 298                 if fnmatch.fnmatch(bucket['name'], bucket_filter):
 299                     found = True
 300                     filter_osds.update([i['id'] for i in bucket['items']])
 301
 302             if not found:
 303                 msg = "Bucket '{0}' not found".format(bucket_filter)
 304                 return errno.ENOENT, msg, ""
 305
 306         # Build dict of OSD ID to stats
 307         osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']])
 308
 309         for osd in osdmap['osds']:
 310             osd_id = osd['osd']
 311             if bucket_filter and osd_id not in filter_osds:
 312                 continue
 313
 314             hostname = ""
 315             kb_used = 0
 316             kb_avail = 0
 317
 318             if osd_id in osd_stats:
 319                 defaults = defaultdict(lambda: None, {'hostname' : ''})
 320                 metadata = self.get_metadata('osd', str(osd_id), default=defaults)
 321                 stats = osd_stats[osd_id]
 322                 hostname = metadata['hostname']
 323                 kb_used = stats['kb_used'] * 1024
 324                 kb_avail = stats['kb_avail'] * 1024
 325
 326             osd_table.add_row([osd_id, hostname,
 327                                mgr_util.format_bytes(kb_used, 5),
 328                                mgr_util.format_bytes(kb_avail, 5),
 329                                mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") +
 330                                self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5),
 331                                mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5),
 332                                mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5),
 333                                mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5),
 334                                ','.join(osd['state']),
 335                                ])
 336
 337         return 0, osd_table.get_string(), ""
 338
 339     def handle_command(self, inbuf, cmd):
 340         self.log.error("handle_command")
 341
 342         if cmd['prefix'] == "fs status":
 343             return self.handle_fs_status(cmd)
 344         elif cmd['prefix'] == "osd status":
 345             return self.handle_osd_status(cmd)
 346         else:
 347             # mgr should respect our self.COMMANDS and not call us for
 348             # any prefix we don't advertise
 349             raise NotImplementedError(cmd['prefix'])