ceph/src/pybind/mgr/status/module.py

   1
   2 """
   3 High level status display commands
   4 """
   5
   6 from collections import defaultdict
   7 from prettytable import PrettyTable
   8 import prettytable
   9 import fnmatch
  10 import errno
  11
  12 from mgr_module import MgrModule
  13
  14
  15 class Module(MgrModule):
  16     COMMANDS = [
  17         {
  18             "cmd": "fs status "
  19                    "name=fs,type=CephString,req=false",
  20             "desc": "Show the status of a CephFS filesystem",
  21             "perm": "r"
  22         },
  23         {
  24             "cmd": "osd status "
  25                    "name=bucket,type=CephString,req=false",
  26             "desc": "Show the status of OSDs within a bucket, or all",
  27             "perm": "r"
  28         },
  29     ]
  30
  31     (
  32         BLACK,
  33         RED,
  34         GREEN,
  35         YELLOW,
  36         BLUE,
  37         MAGENTA,
  38         CYAN,
  39         GRAY
  40     ) = range(8)
  41
  42     RESET_SEQ = "\033[0m"
  43     COLOR_SEQ = "\033[1;%dm"
  44     COLOR_DARK_SEQ = "\033[0;%dm"
  45     BOLD_SEQ = "\033[1m"
  46     UNDERLINE_SEQ = "\033[4m"
  47
  48     def colorize(self, msg, color, dark=False):
  49         """
  50         Decorate `msg` with escape sequences to give the requested color
  51         """
  52         return (self.COLOR_DARK_SEQ if dark else self.COLOR_SEQ) % (30 + color) \
  53                + msg + self.RESET_SEQ
  54
  55     def bold(self, msg):
  56         """
  57         Decorate `msg` with escape sequences to make it appear bold
  58         """
  59         return self.BOLD_SEQ + msg + self.RESET_SEQ
  60
  61     def format_dimless(self, n, width, colored=True):
  62         """
  63         Format a number without units, so as to fit into `width` characters, substituting
  64         an appropriate unit suffix.
  65         """
  66         units = [' ', 'k', 'M', 'G', 'T', 'P']
  67         unit = 0
  68         while len("%s" % (int(n) // (1000**unit))) > width - 1:
  69             unit += 1
  70
  71         if unit > 0:
  72             truncated_float = ("%f" % (n / (1000.0 ** unit)))[0:width - 1]
  73             if truncated_float[-1] == '.':
  74                 truncated_float = " " + truncated_float[0:-1]
  75         else:
  76             truncated_float = "%{wid}d".format(wid=width-1) % n
  77         formatted = "%s%s" % (truncated_float, units[unit])
  78
  79         if colored:
  80             if n == 0:
  81                 color = self.BLACK, False
  82             else:
  83                 color = self.YELLOW, False
  84             return self.bold(self.colorize(formatted[0:-1], color[0], color[1])) \
  85                 + self.bold(self.colorize(formatted[-1], self.BLACK, False))
  86         else:
  87             return formatted
  88
  89     def get_latest(self, daemon_type, daemon_name, stat):
  90         data = self.get_counter(daemon_type, daemon_name, stat)[stat]
  91         #self.log.error("get_latest {0} data={1}".format(stat, data))
  92         if data:
  93             return data[-1][1]
  94         else:
  95             return 0
  96
  97     def get_rate(self, daemon_type, daemon_name, stat):
  98         data = self.get_counter(daemon_type, daemon_name, stat)[stat]
  99
 100         #self.log.error("get_latest {0} data={1}".format(stat, data))
 101         if data and len(data) > 1:
 102             return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0])
 103         else:
 104             return 0
 105
 106     def handle_fs_status(self, cmd):
 107         output = ""
 108
 109         fs_filter = cmd.get('fs', None)
 110
 111         mds_versions = defaultdict(list)
 112
 113         fsmap = self.get("fs_map")
 114         for filesystem in fsmap['filesystems']:
 115             if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter:
 116                 continue
 117
 118             rank_table = PrettyTable(
 119                 ("Rank", "State", "MDS", "Activity", "dns", "inos"),
 120                 hrules=prettytable.FRAME
 121             )
 122
 123             mdsmap = filesystem['mdsmap']
 124
 125             client_count = 0
 126
 127             for rank in mdsmap["in"]:
 128                 up = "mds_{0}".format(rank) in mdsmap["up"]
 129                 if up:
 130                     gid = mdsmap['up']["mds_{0}".format(rank)]
 131                     info = mdsmap['info']['gid_{0}'.format(gid)]
 132                     dns = self.get_latest("mds", info['name'], "mds_mem.dn")
 133                     inos = self.get_latest("mds", info['name'], "mds_mem.ino")
 134
 135                     if rank == 0:
 136                         client_count = self.get_latest("mds", info['name'],
 137                                                        "mds_sessions.session_count")
 138                     elif client_count == 0:
 139                         # In case rank 0 was down, look at another rank's
 140                         # sessionmap to get an indication of clients.
 141                         client_count = self.get_latest("mds", info['name'],
 142                                                        "mds_sessions.session_count")
 143
 144                     laggy = "laggy_since" in info
 145
 146                     state = info['state'].split(":")[1]
 147                     if laggy:
 148                         state += "(laggy)"
 149                     if state == "active" and not laggy:
 150                         c_state = self.colorize(state, self.GREEN)
 151                     else:
 152                         c_state = self.colorize(state, self.YELLOW)
 153
 154                     # Populate based on context of state, e.g. client
 155                     # ops for an active daemon, replay progress, reconnect
 156                     # progress
 157                     activity = ""
 158
 159                     if state == "active":
 160                         activity = "Reqs: " + self.format_dimless(
 161                             self.get_rate("mds", info['name'], "mds_server.handle_client_request"),
 162                             5
 163                         ) + "/s"
 164
 165                     metadata = self.get_metadata('mds', info['name'])
 166                     mds_versions[metadata.get('ceph_version', "unknown")].append(info['name'])
 167                     rank_table.add_row([
 168                         self.bold(rank.__str__()), c_state, info['name'],
 169                         activity,
 170                         self.format_dimless(dns, 5),
 171                         self.format_dimless(inos, 5)
 172                     ])
 173
 174                 else:
 175                     rank_table.add_row([
 176                         rank, "failed", "", "", ""
 177                     ])
 178
 179             # Find the standby replays
 180             for gid_str, daemon_info in mdsmap['info'].iteritems():
 181                 if daemon_info['state'] != "up:standby-replay":
 182                     continue
 183
 184                 inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino")
 185                 dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn")
 186
 187                 activity = "Evts: " + self.format_dimless(
 188                     self.get_rate("mds", daemon_info['name'], "mds_log.replay"),
 189                     5
 190                 ) + "/s"
 191
 192                 rank_table.add_row([
 193                     "{0}-s".format(daemon_info['rank']), "standby-replay",
 194                     daemon_info['name'], activity,
 195                     self.format_dimless(dns, 5),
 196                     self.format_dimless(inos, 5)
 197                 ])
 198
 199             df = self.get("df")
 200             pool_stats = dict([(p['id'], p['stats']) for p in df['pools']])
 201             osdmap = self.get("osd_map")
 202             pools = dict([(p['pool'], p) for p in osdmap['pools']])
 203             metadata_pool_id = mdsmap['metadata_pool']
 204             data_pool_ids = mdsmap['data_pools']
 205
 206             pools_table = PrettyTable(["Pool", "type", "used", "avail"])
 207             for pool_id in [metadata_pool_id] + data_pool_ids:
 208                 pool_type = "metadata" if pool_id == metadata_pool_id else "data"
 209                 stats = pool_stats[pool_id]
 210                 pools_table.add_row([
 211                     pools[pool_id]['pool_name'], pool_type,
 212                     self.format_dimless(stats['bytes_used'], 5),
 213                     self.format_dimless(stats['max_avail'], 5)
 214                 ])
 215
 216             output += "{0} - {1} clients\n".format(
 217                 mdsmap['fs_name'], client_count)
 218             output += "=" * len(mdsmap['fs_name']) + "\n"
 219             output += rank_table.get_string()
 220             output += "\n" + pools_table.get_string() + "\n"
 221
 222         standby_table = PrettyTable(["Standby MDS"])
 223         for standby in fsmap['standbys']:
 224             metadata = self.get_metadata('mds', standby['name'])
 225             mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name'])
 226
 227             standby_table.add_row([standby['name']])
 228
 229         output += "\n" + standby_table.get_string() + "\n"
 230
 231         if len(mds_versions) == 1:
 232             output += "MDS version: {0}".format(mds_versions.keys()[0])
 233         else:
 234             version_table = PrettyTable(["version", "daemons"])
 235             for version, daemons in mds_versions.iteritems():
 236                 version_table.add_row([
 237                     version,
 238                     ", ".join(daemons)
 239                 ])
 240             output += version_table.get_string() + "\n"
 241
 242         return 0, "", output
 243
 244     def handle_osd_status(self, cmd):
 245         osd_table = PrettyTable(['id', 'host', 'used', 'avail', 'wr ops', 'wr data', 'rd ops', 'rd data'])
 246         osdmap = self.get("osd_map")
 247
 248         filter_osds = set()
 249         bucket_filter = None
 250         if 'bucket' in cmd:
 251             self.log.debug("Filtering to bucket '{0}'".format(cmd['bucket']))
 252             bucket_filter = cmd['bucket']
 253             crush = self.get("osd_map_crush")
 254             found = False
 255             for bucket in crush['buckets']:
 256                 if fnmatch.fnmatch(bucket['name'], bucket_filter):
 257                     found = True
 258                     filter_osds.update([i['id'] for i in bucket['items']])
 259
 260             if not found:
 261                 msg = "Bucket '{0}' not found".format(bucket_filter)
 262                 return errno.ENOENT, msg, ""
 263
 264         # Build dict of OSD ID to stats
 265         osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']])
 266
 267         for osd in osdmap['osds']:
 268             osd_id = osd['osd']
 269             if bucket_filter and osd_id not in filter_osds:
 270                 continue
 271
 272             metadata = self.get_metadata('osd', "%s" % osd_id)
 273             stats = osd_stats[osd_id]
 274
 275             osd_table.add_row([osd_id, metadata['hostname'],
 276                                self.format_dimless(stats['kb_used'] * 1024, 5),
 277                                self.format_dimless(stats['kb_avail'] * 1024, 5),
 278                                self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") +
 279                                self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5),
 280                                self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5),
 281                                self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5),
 282                                self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5),
 283                                ])
 284
 285         return 0, "", osd_table.get_string()
 286
 287     def handle_command(self, cmd):
 288         self.log.error("handle_command")
 289
 290         if cmd['prefix'] == "fs status":
 291             return self.handle_fs_status(cmd)
 292         elif cmd['prefix'] == "osd status":
 293             return self.handle_osd_status(cmd)
 294         else:
 295             # mgr should respect our self.COMMANDS and not call us for
 296             # any prefix we don't advertise
 297             raise NotImplementedError(cmd['prefix'])