ceph/src/pybind/mgr/status/module.py

   1
   2 """
   3 High level status display commands
   4 """
   5
   6 from collections import defaultdict
   7 from prettytable import PrettyTable
   8 import errno
   9 import fnmatch
  10 import prettytable
  11 import six
  12
  13 from mgr_module import MgrModule
  14
  15
  16 class Module(MgrModule):
  17     COMMANDS = [
  18         {
  19             "cmd": "fs status "
  20                    "name=fs,type=CephString,req=false",
  21             "desc": "Show the status of a CephFS filesystem",
  22             "perm": "r"
  23         },
  24         {
  25             "cmd": "osd status "
  26                    "name=bucket,type=CephString,req=false",
  27             "desc": "Show the status of OSDs within a bucket, or all",
  28             "perm": "r"
  29         },
  30     ]
  31
  32     (
  33         BLACK,
  34         RED,
  35         GREEN,
  36         YELLOW,
  37         BLUE,
  38         MAGENTA,
  39         CYAN,
  40         GRAY
  41     ) = range(8)
  42
  43     RESET_SEQ = "\033[0m"
  44     COLOR_SEQ = "\033[1;%dm"
  45     COLOR_DARK_SEQ = "\033[0;%dm"
  46     BOLD_SEQ = "\033[1m"
  47     UNDERLINE_SEQ = "\033[4m"
  48
  49     def colorize(self, msg, color, dark=False):
  50         """
  51         Decorate `msg` with escape sequences to give the requested color
  52         """
  53         return (self.COLOR_DARK_SEQ if dark else self.COLOR_SEQ) % (30 + color) \
  54                + msg + self.RESET_SEQ
  55
  56     def bold(self, msg):
  57         """
  58         Decorate `msg` with escape sequences to make it appear bold
  59         """
  60         return self.BOLD_SEQ + msg + self.RESET_SEQ
  61
  62     def format_units(self, n, width, colored, decimal):
  63         """
  64         Format a number without units, so as to fit into `width` characters, substituting
  65         an appropriate unit suffix.
  66
  67         Use decimal for dimensionless things, use base 2 (decimal=False) for byte sizes/rates.
  68         """
  69
  70         factor = 1000 if decimal else 1024
  71         units = [' ', 'k', 'M', 'G', 'T', 'P']
  72         unit = 0
  73         while len("%s" % (int(n) // (factor**unit))) > width - 1:
  74             unit += 1
  75
  76         if unit > 0:
  77             truncated_float = ("%f" % (n / (float(factor) ** unit)))[0:width - 1]
  78             if truncated_float[-1] == '.':
  79                 truncated_float = " " + truncated_float[0:-1]
  80         else:
  81             truncated_float = "%{wid}d".format(wid=width-1) % n
  82         formatted = "%s%s" % (truncated_float, units[unit])
  83
  84         if colored:
  85             if n == 0:
  86                 color = self.BLACK, False
  87             else:
  88                 color = self.YELLOW, False
  89             return self.bold(self.colorize(formatted[0:-1], color[0], color[1])) \
  90                 + self.bold(self.colorize(formatted[-1], self.BLACK, False))
  91         else:
  92             return formatted
  93
  94     def format_dimless(self, n, width, colored=True):
  95         return self.format_units(n, width, colored, decimal=True)
  96
  97     def format_bytes(self, n, width, colored=True):
  98         return self.format_units(n, width, colored, decimal=False)
  99
 100     def get_latest(self, daemon_type, daemon_name, stat):
 101         data = self.get_counter(daemon_type, daemon_name, stat)[stat]
 102         #self.log.error("get_latest {0} data={1}".format(stat, data))
 103         if data:
 104             return data[-1][1]
 105         else:
 106             return 0
 107
 108     def get_rate(self, daemon_type, daemon_name, stat):
 109         data = self.get_counter(daemon_type, daemon_name, stat)[stat]
 110
 111         #self.log.error("get_latest {0} data={1}".format(stat, data))
 112         if data and len(data) > 1:
 113             return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0])
 114         else:
 115             return 0
 116
 117     def handle_fs_status(self, cmd):
 118         output = ""
 119
 120         fs_filter = cmd.get('fs', None)
 121
 122         mds_versions = defaultdict(list)
 123
 124         fsmap = self.get("fs_map")
 125         for filesystem in fsmap['filesystems']:
 126             if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter:
 127                 continue
 128
 129             rank_table = PrettyTable(
 130                 ("Rank", "State", "MDS", "Activity", "dns", "inos"),
 131                 hrules=prettytable.FRAME
 132             )
 133
 134             mdsmap = filesystem['mdsmap']
 135
 136             client_count = 0
 137
 138             for rank in mdsmap["in"]:
 139                 up = "mds_{0}".format(rank) in mdsmap["up"]
 140                 if up:
 141                     gid = mdsmap['up']["mds_{0}".format(rank)]
 142                     info = mdsmap['info']['gid_{0}'.format(gid)]
 143                     dns = self.get_latest("mds", info['name'], "mds_mem.dn")
 144                     inos = self.get_latest("mds", info['name'], "mds_mem.ino")
 145
 146                     if rank == 0:
 147                         client_count = self.get_latest("mds", info['name'],
 148                                                        "mds_sessions.session_count")
 149                     elif client_count == 0:
 150                         # In case rank 0 was down, look at another rank's
 151                         # sessionmap to get an indication of clients.
 152                         client_count = self.get_latest("mds", info['name'],
 153                                                        "mds_sessions.session_count")
 154
 155                     laggy = "laggy_since" in info
 156
 157                     state = info['state'].split(":")[1]
 158                     if laggy:
 159                         state += "(laggy)"
 160                     if state == "active" and not laggy:
 161                         c_state = self.colorize(state, self.GREEN)
 162                     else:
 163                         c_state = self.colorize(state, self.YELLOW)
 164
 165                     # Populate based on context of state, e.g. client
 166                     # ops for an active daemon, replay progress, reconnect
 167                     # progress
 168                     activity = ""
 169
 170                     if state == "active":
 171                         activity = "Reqs: " + self.format_dimless(
 172                             self.get_rate("mds", info['name'], "mds_server.handle_client_request"),
 173                             5
 174                         ) + "/s"
 175
 176                     metadata = self.get_metadata('mds', info['name'])
 177                     mds_versions[metadata.get('ceph_version', "unknown")].append(info['name'])
 178                     rank_table.add_row([
 179                         self.bold(rank.__str__()), c_state, info['name'],
 180                         activity,
 181                         self.format_dimless(dns, 5),
 182                         self.format_dimless(inos, 5)
 183                     ])
 184
 185                 else:
 186                     rank_table.add_row([
 187                         rank, "failed", "", "", "", ""
 188                     ])
 189
 190             # Find the standby replays
 191             for gid_str, daemon_info in six.iteritems(mdsmap['info']):
 192                 if daemon_info['state'] != "up:standby-replay":
 193                     continue
 194
 195                 inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino")
 196                 dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn")
 197
 198                 activity = "Evts: " + self.format_dimless(
 199                     self.get_rate("mds", daemon_info['name'], "mds_log.replay"),
 200                     5
 201                 ) + "/s"
 202
 203                 rank_table.add_row([
 204                     "{0}-s".format(daemon_info['rank']), "standby-replay",
 205                     daemon_info['name'], activity,
 206                     self.format_dimless(dns, 5),
 207                     self.format_dimless(inos, 5)
 208                 ])
 209
 210             df = self.get("df")
 211             pool_stats = dict([(p['id'], p['stats']) for p in df['pools']])
 212             osdmap = self.get("osd_map")
 213             pools = dict([(p['pool'], p) for p in osdmap['pools']])
 214             metadata_pool_id = mdsmap['metadata_pool']
 215             data_pool_ids = mdsmap['data_pools']
 216
 217             pools_table = PrettyTable(["Pool", "type", "used", "avail"])
 218             for pool_id in [metadata_pool_id] + data_pool_ids:
 219                 pool_type = "metadata" if pool_id == metadata_pool_id else "data"
 220                 stats = pool_stats[pool_id]
 221                 pools_table.add_row([
 222                     pools[pool_id]['pool_name'], pool_type,
 223                     self.format_bytes(stats['bytes_used'], 5),
 224                     self.format_bytes(stats['max_avail'], 5)
 225                 ])
 226
 227             output += "{0} - {1} clients\n".format(
 228                 mdsmap['fs_name'], client_count)
 229             output += "=" * len(mdsmap['fs_name']) + "\n"
 230             output += rank_table.get_string()
 231             output += "\n" + pools_table.get_string() + "\n"
 232
 233         standby_table = PrettyTable(["Standby MDS"])
 234         for standby in fsmap['standbys']:
 235             metadata = self.get_metadata('mds', standby['name'])
 236             mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name'])
 237
 238             standby_table.add_row([standby['name']])
 239
 240         output += "\n" + standby_table.get_string() + "\n"
 241
 242         if len(mds_versions) == 1:
 243             output += "MDS version: {0}".format(mds_versions.keys()[0])
 244         else:
 245             version_table = PrettyTable(["version", "daemons"])
 246             for version, daemons in six.iteritems(mds_versions):
 247                 version_table.add_row([
 248                     version,
 249                     ", ".join(daemons)
 250                 ])
 251             output += version_table.get_string() + "\n"
 252
 253         return 0, "", output
 254
 255     def handle_osd_status(self, cmd):
 256         osd_table = PrettyTable(['id', 'host', 'used', 'avail', 'wr ops', 'wr data', 'rd ops', 'rd data', 'state'])
 257         osdmap = self.get("osd_map")
 258
 259         filter_osds = set()
 260         bucket_filter = None
 261         if 'bucket' in cmd:
 262             self.log.debug("Filtering to bucket '{0}'".format(cmd['bucket']))
 263             bucket_filter = cmd['bucket']
 264             crush = self.get("osd_map_crush")
 265             found = False
 266             for bucket in crush['buckets']:
 267                 if fnmatch.fnmatch(bucket['name'], bucket_filter):
 268                     found = True
 269                     filter_osds.update([i['id'] for i in bucket['items']])
 270
 271             if not found:
 272                 msg = "Bucket '{0}' not found".format(bucket_filter)
 273                 return errno.ENOENT, msg, ""
 274
 275         # Build dict of OSD ID to stats
 276         osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']])
 277
 278         for osd in osdmap['osds']:
 279             osd_id = osd['osd']
 280             if bucket_filter and osd_id not in filter_osds:
 281                 continue
 282
 283             hostname = ""
 284             kb_used = 0
 285             kb_avail = 0
 286
 287             if osd_id in osd_stats:
 288                 metadata = self.get_metadata('osd', "%s" % osd_id)
 289                 stats = osd_stats[osd_id]
 290                 hostname = metadata['hostname']
 291                 kb_used = stats['kb_used'] * 1024
 292                 kb_avail = stats['kb_avail'] * 1024
 293
 294             osd_table.add_row([osd_id, hostname,
 295                                self.format_bytes(kb_used, 5),
 296                                self.format_bytes(kb_avail, 5),
 297                                self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") +
 298                                self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5),
 299                                self.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5),
 300                                self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5),
 301                                self.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5),
 302                                ','.join(osd['state']),
 303                                ])
 304
 305         return 0, "", osd_table.get_string()
 306
 307     def handle_command(self, cmd):
 308         self.log.error("handle_command")
 309
 310         if cmd['prefix'] == "fs status":
 311             return self.handle_fs_status(cmd)
 312         elif cmd['prefix'] == "osd status":
 313             return self.handle_osd_status(cmd)
 314         else:
 315             # mgr should respect our self.COMMANDS and not call us for
 316             # any prefix we don't advertise
 317             raise NotImplementedError(cmd['prefix'])