[ceph.git] / ceph / src / pybind / mgr / status / module.py


"""
High level status display commands
"""

from collections import defaultdict
from prettytable import PrettyTable
import prettytable
import fnmatch
import errno

from mgr_module import MgrModule


class Module(MgrModule):
    COMMANDS = [
        {
            "cmd": "fs status "
                   "name=fs,type=CephString,req=false",
            "desc": "Show the status of a CephFS filesystem",
            "perm": "r"
        },
        {
            "cmd": "osd status "
                   "name=bucket,type=CephString,req=false",
            "desc": "Show the status of OSDs within a bucket, or all",
            "perm": "r"
        },
    ]

    (
        BLACK,
        RED,
        GREEN,
        YELLOW,
        BLUE,
        MAGENTA,
        CYAN,
        GRAY
    ) = range(8)

    RESET_SEQ = "\033[0m"
    COLOR_SEQ = "\033[1;%dm"
    COLOR_DARK_SEQ = "\033[0;%dm"
    BOLD_SEQ = "\033[1m"
    UNDERLINE_SEQ = "\033[4m"

    def colorize(self, msg, color, dark=False):
        """
        Decorate `msg` with escape sequences to give the requested color
        """
        return (self.COLOR_DARK_SEQ if dark else self.COLOR_SEQ) % (30 + color) \
               + msg + self.RESET_SEQ

    def bold(self, msg):
        """
        Decorate `msg` with escape sequences to make it appear bold
        """
        return self.BOLD_SEQ + msg + self.RESET_SEQ

    def format_units(self, n, width, colored, decimal):
        """
        Format a number without units, so as to fit into `width` characters, substituting
        an appropriate unit suffix.
        
        Use decimal for dimensionless things, use base 2 (decimal=False) for byte sizes/rates.
        """
        
        factor = 1000 if decimal else 1024
        units = [' ', 'k', 'M', 'G', 'T', 'P']
        unit = 0
        while len("%s" % (int(n) // (factor**unit))) > width - 1:
            unit += 1

        if unit > 0:
            truncated_float = ("%f" % (n / (float(factor) ** unit)))[0:width - 1]
            if truncated_float[-1] == '.':
                truncated_float = " " + truncated_float[0:-1]
        else:
            truncated_float = "%{wid}d".format(wid=width-1) % n
        formatted = "%s%s" % (truncated_float, units[unit])

        if colored:
            if n == 0:
                color = self.BLACK, False
            else:
                color = self.YELLOW, False
            return self.bold(self.colorize(formatted[0:-1], color[0], color[1])) \
                + self.bold(self.colorize(formatted[-1], self.BLACK, False))
        else:
            return formatted

    def format_dimless(self, n, width, colored=True):
        return self.format_units(n, width, colored, decimal=True)
    
    def format_bytes(self, n, width, colored=True):
        return self.format_units(n, width, colored, decimal=False)
        
    def get_latest(self, daemon_type, daemon_name, stat):
        data = self.get_counter(daemon_type, daemon_name, stat)[stat]
        #self.log.error("get_latest {0} data={1}".format(stat, data))
        if data:
            return data[-1][1]
        else:
            return 0

    def get_rate(self, daemon_type, daemon_name, stat):
        data = self.get_counter(daemon_type, daemon_name, stat)[stat]

        #self.log.error("get_latest {0} data={1}".format(stat, data))
        if data and len(data) > 1:
            return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0])
        else:
            return 0

    def handle_fs_status(self, cmd):
        output = ""

        fs_filter = cmd.get('fs', None)

        mds_versions = defaultdict(list)

        fsmap = self.get("fs_map")
        for filesystem in fsmap['filesystems']:
            if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter:
                continue

            rank_table = PrettyTable(
                ("Rank", "State", "MDS", "Activity", "dns", "inos"),
                hrules=prettytable.FRAME
            )

            mdsmap = filesystem['mdsmap']

            client_count = 0

            for rank in mdsmap["in"]:
                up = "mds_{0}".format(rank) in mdsmap["up"]
                if up:
                    gid = mdsmap['up']["mds_{0}".format(rank)]
                    info = mdsmap['info']['gid_{0}'.format(gid)]
                    dns = self.get_latest("mds", info['name'], "mds_mem.dn")
                    inos = self.get_latest("mds", info['name'], "mds_mem.ino")

                    if rank == 0:
                        client_count = self.get_latest("mds", info['name'],
                                                       "mds_sessions.session_count")
                    elif client_count == 0:
                        # In case rank 0 was down, look at another rank's
                        # sessionmap to get an indication of clients.
                        client_count = self.get_latest("mds", info['name'],
                                                       "mds_sessions.session_count")

                    laggy = "laggy_since" in info

                    state = info['state'].split(":")[1]
                    if laggy:
                        state += "(laggy)"
                    if state == "active" and not laggy:
                        c_state = self.colorize(state, self.GREEN)
                    else:
                        c_state = self.colorize(state, self.YELLOW)

                    # Populate based on context of state, e.g. client
                    # ops for an active daemon, replay progress, reconnect
                    # progress
                    activity = ""

                    if state == "active":
                        activity = "Reqs: " + self.format_dimless(
                            self.get_rate("mds", info['name'], "mds_server.handle_client_request"),
                            5
                        ) + "/s"

                    metadata = self.get_metadata('mds', info['name'])
                    mds_versions[metadata.get('ceph_version', "unknown")].append(info['name'])
                    rank_table.add_row([
                        self.bold(rank.__str__()), c_state, info['name'],
                        activity,
                        self.format_dimless(dns, 5),
                        self.format_dimless(inos, 5)
                    ])

                else:
                    rank_table.add_row([
                        rank, "failed", "", "", "", ""
                    ])

            # Find the standby replays
            for gid_str, daemon_info in mdsmap['info'].iteritems():
                if daemon_info['state'] != "up:standby-replay":
                    continue

                inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino")
                dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn")

                activity = "Evts: " + self.format_dimless(
                    self.get_rate("mds", daemon_info['name'], "mds_log.replay"),
                    5
                ) + "/s"

                rank_table.add_row([
                    "{0}-s".format(daemon_info['rank']), "standby-replay",
                    daemon_info['name'], activity,
                    self.format_dimless(dns, 5),
                    self.format_dimless(inos, 5)
                ])

            df = self.get("df")
            pool_stats = dict([(p['id'], p['stats']) for p in df['pools']])
            osdmap = self.get("osd_map")
            pools = dict([(p['pool'], p) for p in osdmap['pools']])
            metadata_pool_id = mdsmap['metadata_pool']
            data_pool_ids = mdsmap['data_pools']

            pools_table = PrettyTable(["Pool", "type", "used", "avail"])
            for pool_id in [metadata_pool_id] + data_pool_ids:
                pool_type = "metadata" if pool_id == metadata_pool_id else "data"
                stats = pool_stats[pool_id]
                pools_table.add_row([
                    pools[pool_id]['pool_name'], pool_type,
                    self.format_bytes(stats['bytes_used'], 5),
                    self.format_bytes(stats['max_avail'], 5)
                ])

            output += "{0} - {1} clients\n".format(
                mdsmap['fs_name'], client_count)
            output += "=" * len(mdsmap['fs_name']) + "\n"
            output += rank_table.get_string()
            output += "\n" + pools_table.get_string() + "\n"

        standby_table = PrettyTable(["Standby MDS"])
        for standby in fsmap['standbys']:
            metadata = self.get_metadata('mds', standby['name'])
            mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name'])

            standby_table.add_row([standby['name']])

        output += "\n" + standby_table.get_string() + "\n"

        if len(mds_versions) == 1:
            output += "MDS version: {0}".format(mds_versions.keys()[0])
        else:
            version_table = PrettyTable(["version", "daemons"])
            for version, daemons in mds_versions.iteritems():
                version_table.add_row([
                    version,
                    ", ".join(daemons)
                ])
            output += version_table.get_string() + "\n"

        return 0, "", output

    def handle_osd_status(self, cmd):
        osd_table = PrettyTable(['id', 'host', 'used', 'avail', 'wr ops', 'wr data', 'rd ops', 'rd data'])
        osdmap = self.get("osd_map")

        filter_osds = set()
        bucket_filter = None
        if 'bucket' in cmd:
            self.log.debug("Filtering to bucket '{0}'".format(cmd['bucket']))
            bucket_filter = cmd['bucket']
            crush = self.get("osd_map_crush")
            found = False
            for bucket in crush['buckets']:
                if fnmatch.fnmatch(bucket['name'], bucket_filter):
                    found = True
                    filter_osds.update([i['id'] for i in bucket['items']])

            if not found:
                msg = "Bucket '{0}' not found".format(bucket_filter)
                return errno.ENOENT, msg, ""

        # Build dict of OSD ID to stats
        osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']])

        for osd in osdmap['osds']:
            osd_id = osd['osd']
            if bucket_filter and osd_id not in filter_osds:
                continue

            metadata = self.get_metadata('osd', "%s" % osd_id)
            stats = osd_stats[osd_id]

            osd_table.add_row([osd_id, metadata['hostname'],
                               self.format_bytes(stats['kb_used'] * 1024, 5),
                               self.format_bytes(stats['kb_avail'] * 1024, 5),
                               self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") +
                               self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5),
                               self.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5),
                               self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5),
                               self.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5),
                               ])

        return 0, "", osd_table.get_string()

    def handle_command(self, cmd):
        self.log.error("handle_command")

        if cmd['prefix'] == "fs status":
            return self.handle_fs_status(cmd)
        elif cmd['prefix'] == "osd status":
            return self.handle_osd_status(cmd)
        else:
            # mgr should respect our self.COMMANDS and not call us for
            # any prefix we don't advertise
            raise NotImplementedError(cmd['prefix'])
Commit	Line	Data
7c673cae FG	1
	2	"""
	3	High level status display commands
	4	"""
	5
	6	from collections import defaultdict
	7	from prettytable import PrettyTable
	8	import prettytable
224ce89b WB	9	import fnmatch
224ce89b WB	10	import errno
7c673cae FG	11
	12	from mgr_module import MgrModule
	13
	14
	15	class Module(MgrModule):
	16	COMMANDS = [
	17	{
	18	"cmd": "fs status "
	19	"name=fs,type=CephString,req=false",
	20	"desc": "Show the status of a CephFS filesystem",
	21	"perm": "r"
	22	},
	23	{
224ce89b	24	"cmd": "osd status "
7c673cae FG	25	"name=bucket,type=CephString,req=false",
	26	"desc": "Show the status of OSDs within a bucket, or all",
	27	"perm": "r"
	28	},
	29	]
	30
	31	(
	32	BLACK,
	33	RED,
	34	GREEN,
	35	YELLOW,
	36	BLUE,
	37	MAGENTA,
	38	CYAN,
	39	GRAY
	40	) = range(8)
	41
	42	RESET_SEQ = "\033[0m"
	43	COLOR_SEQ = "\033[1;%dm"
	44	COLOR_DARK_SEQ = "\033[0;%dm"
	45	BOLD_SEQ = "\033[1m"
	46	UNDERLINE_SEQ = "\033[4m"
	47
	48	def colorize(self, msg, color, dark=False):
	49	"""
	50	Decorate `msg` with escape sequences to give the requested color
	51	"""
	52	return (self.COLOR_DARK_SEQ if dark else self.COLOR_SEQ) % (30 + color) \
	53	+ msg + self.RESET_SEQ
	54
	55	def bold(self, msg):
	56	"""
	57	Decorate `msg` with escape sequences to make it appear bold
	58	"""
	59	return self.BOLD_SEQ + msg + self.RESET_SEQ
	60
3efd9988	61	def format_units(self, n, width, colored, decimal):
7c673cae FG	62	"""
	63	Format a number without units, so as to fit into `width` characters, substituting
	64	an appropriate unit suffix.
3efd9988 FG	65
3efd9988 FG	66	Use decimal for dimensionless things, use base 2 (decimal=False) for byte sizes/rates.
7c673cae	67	"""
3efd9988 FG	68
3efd9988 FG	69	factor = 1000 if decimal else 1024
7c673cae FG	70	units = [' ', 'k', 'M', 'G', 'T', 'P']
7c673cae FG	71	unit = 0
3efd9988	72	while len("%s" % (int(n) // (factor**unit))) > width - 1:
7c673cae FG	73	unit += 1
	74
	75	if unit > 0:
3efd9988	76	truncated_float = ("%f" % (n / (float(factor) ** unit)))[0:width - 1]
7c673cae FG	77	if truncated_float[-1] == '.':
	78	truncated_float = " " + truncated_float[0:-1]
	79	else:
	80	truncated_float = "%{wid}d".format(wid=width-1) % n
	81	formatted = "%s%s" % (truncated_float, units[unit])
	82
	83	if colored:
	84	if n == 0:
	85	color = self.BLACK, False
	86	else:
	87	color = self.YELLOW, False
	88	return self.bold(self.colorize(formatted[0:-1], color[0], color[1])) \
	89	+ self.bold(self.colorize(formatted[-1], self.BLACK, False))
	90	else:
	91	return formatted
	92
3efd9988 FG	93	def format_dimless(self, n, width, colored=True):
	94	return self.format_units(n, width, colored, decimal=True)
	95
	96	def format_bytes(self, n, width, colored=True):
	97	return self.format_units(n, width, colored, decimal=False)
	98
7c673cae FG	99	def get_latest(self, daemon_type, daemon_name, stat):
	100	data = self.get_counter(daemon_type, daemon_name, stat)[stat]
	101	#self.log.error("get_latest {0} data={1}".format(stat, data))
	102	if data:
	103	return data[-1][1]
	104	else:
	105	return 0
	106
	107	def get_rate(self, daemon_type, daemon_name, stat):
	108	data = self.get_counter(daemon_type, daemon_name, stat)[stat]
	109
	110	#self.log.error("get_latest {0} data={1}".format(stat, data))
	111	if data and len(data) > 1:
	112	return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0])
	113	else:
	114	return 0
	115
	116	def handle_fs_status(self, cmd):
	117	output = ""
	118
224ce89b WB	119	fs_filter = cmd.get('fs', None)
224ce89b WB	120
7c673cae FG	121	mds_versions = defaultdict(list)
	122
	123	fsmap = self.get("fs_map")
	124	for filesystem in fsmap['filesystems']:
224ce89b WB	125	if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter:
	126	continue
	127
7c673cae FG	128	rank_table = PrettyTable(
	129	("Rank", "State", "MDS", "Activity", "dns", "inos"),
	130	hrules=prettytable.FRAME
	131	)
	132
	133	mdsmap = filesystem['mdsmap']
	134
	135	client_count = 0
	136
	137	for rank in mdsmap["in"]:
	138	up = "mds_{0}".format(rank) in mdsmap["up"]
	139	if up:
	140	gid = mdsmap['up']["mds_{0}".format(rank)]
	141	info = mdsmap['info']['gid_{0}'.format(gid)]
31f18b77	142	dns = self.get_latest("mds", info['name'], "mds_mem.dn")
7c673cae FG	143	inos = self.get_latest("mds", info['name'], "mds_mem.ino")
	144
	145	if rank == 0:
	146	client_count = self.get_latest("mds", info['name'],
	147	"mds_sessions.session_count")
	148	elif client_count == 0:
	149	# In case rank 0 was down, look at another rank's
	150	# sessionmap to get an indication of clients.
	151	client_count = self.get_latest("mds", info['name'],
	152	"mds_sessions.session_count")
	153
	154	laggy = "laggy_since" in info
	155
	156	state = info['state'].split(":")[1]
	157	if laggy:
	158	state += "(laggy)"
	159	if state == "active" and not laggy:
	160	c_state = self.colorize(state, self.GREEN)
	161	else:
	162	c_state = self.colorize(state, self.YELLOW)
	163
	164	# Populate based on context of state, e.g. client
	165	# ops for an active daemon, replay progress, reconnect
	166	# progress
	167	activity = ""
	168
	169	if state == "active":
	170	activity = "Reqs: " + self.format_dimless(
	171	self.get_rate("mds", info['name'], "mds_server.handle_client_request"),
	172	5
	173	) + "/s"
	174
	175	metadata = self.get_metadata('mds', info['name'])
	176	mds_versions[metadata.get('ceph_version', "unknown")].append(info['name'])
	177	rank_table.add_row([
	178	self.bold(rank.__str__()), c_state, info['name'],
	179	activity,
	180	self.format_dimless(dns, 5),
	181	self.format_dimless(inos, 5)
	182	])
	183
	184	else:
	185	rank_table.add_row([
c07f9fc5	186	rank, "failed", "", "", "", ""
7c673cae FG	187	])
	188
	189	# Find the standby replays
	190	for gid_str, daemon_info in mdsmap['info'].iteritems():
	191	if daemon_info['state'] != "up:standby-replay":
	192	continue
	193
	194	inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino")
31f18b77	195	dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn")
7c673cae FG	196
	197	activity = "Evts: " + self.format_dimless(
	198	self.get_rate("mds", daemon_info['name'], "mds_log.replay"),
	199	5
	200	) + "/s"
	201
	202	rank_table.add_row([
	203	"{0}-s".format(daemon_info['rank']), "standby-replay",
	204	daemon_info['name'], activity,
	205	self.format_dimless(dns, 5),
	206	self.format_dimless(inos, 5)
	207	])
	208
	209	df = self.get("df")
	210	pool_stats = dict([(p['id'], p['stats']) for p in df['pools']])
	211	osdmap = self.get("osd_map")
	212	pools = dict([(p['pool'], p) for p in osdmap['pools']])
	213	metadata_pool_id = mdsmap['metadata_pool']
	214	data_pool_ids = mdsmap['data_pools']
	215
	216	pools_table = PrettyTable(["Pool", "type", "used", "avail"])
	217	for pool_id in [metadata_pool_id] + data_pool_ids:
	218	pool_type = "metadata" if pool_id == metadata_pool_id else "data"
	219	stats = pool_stats[pool_id]
	220	pools_table.add_row([
	221	pools[pool_id]['pool_name'], pool_type,
3efd9988 FG	222	self.format_bytes(stats['bytes_used'], 5),
3efd9988 FG	223	self.format_bytes(stats['max_avail'], 5)
7c673cae FG	224	])
	225
	226	output += "{0} - {1} clients\n".format(
	227	mdsmap['fs_name'], client_count)
	228	output += "=" * len(mdsmap['fs_name']) + "\n"
	229	output += rank_table.get_string()
	230	output += "\n" + pools_table.get_string() + "\n"
	231
	232	standby_table = PrettyTable(["Standby MDS"])
	233	for standby in fsmap['standbys']:
	234	metadata = self.get_metadata('mds', standby['name'])
	235	mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name'])
	236
	237	standby_table.add_row([standby['name']])
	238
	239	output += "\n" + standby_table.get_string() + "\n"
	240
	241	if len(mds_versions) == 1:
	242	output += "MDS version: {0}".format(mds_versions.keys()[0])
	243	else:
	244	version_table = PrettyTable(["version", "daemons"])
	245	for version, daemons in mds_versions.iteritems():
	246	version_table.add_row([
	247	version,
	248	", ".join(daemons)
	249	])
	250	output += version_table.get_string() + "\n"
	251
	252	return 0, "", output
	253
224ce89b	254	def handle_osd_status(self, cmd):
7c673cae FG	255	osd_table = PrettyTable(['id', 'host', 'used', 'avail', 'wr ops', 'wr data', 'rd ops', 'rd data'])
	256	osdmap = self.get("osd_map")
	257
224ce89b WB	258	filter_osds = set()
	259	bucket_filter = None
	260	if 'bucket' in cmd:
	261	self.log.debug("Filtering to bucket '{0}'".format(cmd['bucket']))
	262	bucket_filter = cmd['bucket']
	263	crush = self.get("osd_map_crush")
	264	found = False
	265	for bucket in crush['buckets']:
	266	if fnmatch.fnmatch(bucket['name'], bucket_filter):
	267	found = True
	268	filter_osds.update([i['id'] for i in bucket['items']])
	269
	270	if not found:
	271	msg = "Bucket '{0}' not found".format(bucket_filter)
	272	return errno.ENOENT, msg, ""
	273
7c673cae FG	274	# Build dict of OSD ID to stats
	275	osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']])
	276
	277	for osd in osdmap['osds']:
	278	osd_id = osd['osd']
224ce89b WB	279	if bucket_filter and osd_id not in filter_osds:
	280	continue
	281
7c673cae FG	282	metadata = self.get_metadata('osd', "%s" % osd_id)
	283	stats = osd_stats[osd_id]
	284
	285	osd_table.add_row([osd_id, metadata['hostname'],
3efd9988 FG	286	self.format_bytes(stats['kb_used'] * 1024, 5),
3efd9988 FG	287	self.format_bytes(stats['kb_avail'] * 1024, 5),
7c673cae FG	288	self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") +
7c673cae FG	289	self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5),
3efd9988	290	self.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5),
7c673cae	291	self.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5),
3efd9988	292	self.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5),
7c673cae FG	293	])
	294
	295	return 0, "", osd_table.get_string()
	296
	297	def handle_command(self, cmd):
	298	self.log.error("handle_command")
	299
	300	if cmd['prefix'] == "fs status":
	301	return self.handle_fs_status(cmd)
224ce89b WB	302	elif cmd['prefix'] == "osd status":
224ce89b WB	303	return self.handle_osd_status(cmd)
7c673cae FG	304	else:
	305	# mgr should respect our self.COMMANDS and not call us for
	306	# any prefix we don't advertise
	307	raise NotImplementedError(cmd['prefix'])