[ceph.git] / ceph / src / pybind / mgr / status / module.py


"""
High level status display commands
"""

from collections import defaultdict
from prettytable import PrettyTable
import errno
import fnmatch
import mgr_util
import prettytable
import six

from mgr_module import MgrModule


class Module(MgrModule):
    COMMANDS = [
        {
            "cmd": "fs status "
                   "name=fs,type=CephString,req=false",
            "desc": "Show the status of a CephFS filesystem",
            "perm": "r"
        },
        {
            "cmd": "osd status "
                   "name=bucket,type=CephString,req=false",
            "desc": "Show the status of OSDs within a bucket, or all",
            "perm": "r"
        },
    ]

        
    def get_latest(self, daemon_type, daemon_name, stat):
        data = self.get_counter(daemon_type, daemon_name, stat)[stat]
        #self.log.error("get_latest {0} data={1}".format(stat, data))
        if data:
            return data[-1][1]
        else:
            return 0

    def get_rate(self, daemon_type, daemon_name, stat):
        data = self.get_counter(daemon_type, daemon_name, stat)[stat]

        #self.log.error("get_latest {0} data={1}".format(stat, data))
        if data and len(data) > 1:
            return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0])
        else:
            return 0

    def handle_fs_status(self, cmd):
        output = ""

        fs_filter = cmd.get('fs', None)

        mds_versions = defaultdict(list)

        fsmap = self.get("fs_map")
        for filesystem in fsmap['filesystems']:
            if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter:
                continue

            rank_table = PrettyTable(
                ("Rank", "State", "MDS", "Activity", "dns", "inos"),
                hrules=prettytable.FRAME
            )

            mdsmap = filesystem['mdsmap']

            client_count = 0

            for rank in mdsmap["in"]:
                up = "mds_{0}".format(rank) in mdsmap["up"]
                if up:
                    gid = mdsmap['up']["mds_{0}".format(rank)]
                    info = mdsmap['info']['gid_{0}'.format(gid)]
                    dns = self.get_latest("mds", info['name'], "mds_mem.dn")
                    inos = self.get_latest("mds", info['name'], "mds_mem.ino")

                    if rank == 0:
                        client_count = self.get_latest("mds", info['name'],
                                                       "mds_sessions.session_count")
                    elif client_count == 0:
                        # In case rank 0 was down, look at another rank's
                        # sessionmap to get an indication of clients.
                        client_count = self.get_latest("mds", info['name'],
                                                       "mds_sessions.session_count")

                    laggy = "laggy_since" in info

                    state = info['state'].split(":")[1]
                    if laggy:
                        state += "(laggy)"
                    if state == "active" and not laggy:
                        c_state = mgr_util.colorize(state, mgr_util.GREEN)
                    else:
                        c_state = mgr_util.colorize(state, mgr_util.YELLOW)

                    # Populate based on context of state, e.g. client
                    # ops for an active daemon, replay progress, reconnect
                    # progress
                    activity = ""

                    if state == "active":
                        activity = "Reqs: " + mgr_util.format_dimless(
                            self.get_rate("mds", info['name'], "mds_server.handle_client_request"),
                            5
                        ) + "/s"

                    metadata = self.get_metadata('mds', info['name'])
                    mds_versions[metadata.get('ceph_version', "unknown")].append(info['name'])
                    rank_table.add_row([
                        mgr_util.bold(rank.__str__()), c_state, info['name'],
                        activity,
                        mgr_util.format_dimless(dns, 5),
                        mgr_util.format_dimless(inos, 5)
                    ])

                else:
                    rank_table.add_row([
                        rank, "failed", "", "", "", ""
                    ])

            # Find the standby replays
            for gid_str, daemon_info in six.iteritems(mdsmap['info']):
                if daemon_info['state'] != "up:standby-replay":
                    continue

                inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino")
                dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn")

                activity = "Evts: " + mgr_util.format_dimless(
                    self.get_rate("mds", daemon_info['name'], "mds_log.replayed"),
                    5
                ) + "/s"

                metadata = self.get_metadata('mds', daemon_info['name'])
                mds_versions[metadata.get('ceph_version', "unknown")].append(daemon_info['name'])

                rank_table.add_row([
                    "{0}-s".format(daemon_info['rank']), "standby-replay",
                    daemon_info['name'], activity,
                    mgr_util.format_dimless(dns, 5),
                    mgr_util.format_dimless(inos, 5)
                ])

            df = self.get("df")
            pool_stats = dict([(p['id'], p['stats']) for p in df['pools']])
            osdmap = self.get("osd_map")
            pools = dict([(p['pool'], p) for p in osdmap['pools']])
            metadata_pool_id = mdsmap['metadata_pool']
            data_pool_ids = mdsmap['data_pools']

            pools_table = PrettyTable(["Pool", "type", "used", "avail"])
            for pool_id in [metadata_pool_id] + data_pool_ids:
                pool_type = "metadata" if pool_id == metadata_pool_id else "data"
                stats = pool_stats[pool_id]
                pools_table.add_row([
                    pools[pool_id]['pool_name'], pool_type,
                    mgr_util.format_bytes(stats['bytes_used'], 5),
                    mgr_util.format_bytes(stats['max_avail'], 5)
                ])

            output += "{0} - {1} clients\n".format(
                mdsmap['fs_name'], client_count)
            output += "=" * len(mdsmap['fs_name']) + "\n"
            output += rank_table.get_string()
            output += "\n" + pools_table.get_string() + "\n"

        if not output and fs_filter is not None:
            return errno.EINVAL, "", "Invalid filesystem: " + fs_filter

        standby_table = PrettyTable(["Standby MDS"])
        for standby in fsmap['standbys']:
            metadata = self.get_metadata('mds', standby['name'])
            mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name'])

            standby_table.add_row([standby['name']])

        output += "\n" + standby_table.get_string() + "\n"

        if len(mds_versions) == 1:
            output += "MDS version: {0}".format(list(mds_versions)[0])
        else:
            version_table = PrettyTable(["version", "daemons"])
            for version, daemons in six.iteritems(mds_versions):
                version_table.add_row([
                    version,
                    ", ".join(daemons)
                ])
            output += version_table.get_string() + "\n"

        return 0, output, ""

    def handle_osd_status(self, cmd):
        osd_table = PrettyTable(['id', 'host', 'used', 'avail', 'wr ops', 'wr data', 'rd ops', 'rd data', 'state'])
        osdmap = self.get("osd_map")

        filter_osds = set()
        bucket_filter = None
        if 'bucket' in cmd:
            self.log.debug("Filtering to bucket '{0}'".format(cmd['bucket']))
            bucket_filter = cmd['bucket']
            crush = self.get("osd_map_crush")
            found = False
            for bucket in crush['buckets']:
                if fnmatch.fnmatch(bucket['name'], bucket_filter):
                    found = True
                    filter_osds.update([i['id'] for i in bucket['items']])

            if not found:
                msg = "Bucket '{0}' not found".format(bucket_filter)
                return errno.ENOENT, msg, ""

        # Build dict of OSD ID to stats
        osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']])

        for osd in osdmap['osds']:
            osd_id = osd['osd']
            if bucket_filter and osd_id not in filter_osds:
                continue

            hostname = ""
            kb_used = 0
            kb_avail = 0

            if osd_id in osd_stats:
                metadata = self.get_metadata('osd', "%s" % osd_id)
                stats = osd_stats[osd_id]
                hostname = metadata['hostname']
                kb_used = stats['kb_used'] * 1024
                kb_avail = stats['kb_avail'] * 1024

            osd_table.add_row([osd_id, hostname,
                               mgr_util.format_bytes(kb_used, 5),
                               mgr_util.format_bytes(kb_avail, 5),
                               mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") +
                               self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5),
                               mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5),
                               mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5),
                               mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5),
                               ','.join(osd['state']),
                               ])

        return 0, osd_table.get_string(), ""

    def handle_command(self, inbuf, cmd):
        self.log.error("handle_command")

        if cmd['prefix'] == "fs status":
            return self.handle_fs_status(cmd)
        elif cmd['prefix'] == "osd status":
            return self.handle_osd_status(cmd)
        else:
            # mgr should respect our self.COMMANDS and not call us for
            # any prefix we don't advertise
            raise NotImplementedError(cmd['prefix'])
Commit	Line	Data
7c673cae FG	1
	2	"""
	3	High level status display commands
	4	"""
	5
	6	from collections import defaultdict
	7	from prettytable import PrettyTable
224ce89b	8	import errno
1adf2230	9	import fnmatch
11fdf7f2	10	import mgr_util
1adf2230 AA	11	import prettytable
1adf2230 AA	12	import six
7c673cae FG	13
	14	from mgr_module import MgrModule
	15
	16
	17	class Module(MgrModule):
	18	COMMANDS = [
	19	{
	20	"cmd": "fs status "
	21	"name=fs,type=CephString,req=false",
	22	"desc": "Show the status of a CephFS filesystem",
	23	"perm": "r"
	24	},
	25	{
224ce89b	26	"cmd": "osd status "
7c673cae FG	27	"name=bucket,type=CephString,req=false",
	28	"desc": "Show the status of OSDs within a bucket, or all",
	29	"perm": "r"
	30	},
	31	]
	32
3efd9988	33
7c673cae FG	34	def get_latest(self, daemon_type, daemon_name, stat):
	35	data = self.get_counter(daemon_type, daemon_name, stat)[stat]
	36	#self.log.error("get_latest {0} data={1}".format(stat, data))
	37	if data:
	38	return data[-1][1]
	39	else:
	40	return 0
	41
	42	def get_rate(self, daemon_type, daemon_name, stat):
	43	data = self.get_counter(daemon_type, daemon_name, stat)[stat]
	44
	45	#self.log.error("get_latest {0} data={1}".format(stat, data))
	46	if data and len(data) > 1:
	47	return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0])
	48	else:
	49	return 0
	50
	51	def handle_fs_status(self, cmd):
	52	output = ""
	53
224ce89b WB	54	fs_filter = cmd.get('fs', None)
224ce89b WB	55
7c673cae FG	56	mds_versions = defaultdict(list)
	57
	58	fsmap = self.get("fs_map")
	59	for filesystem in fsmap['filesystems']:
224ce89b WB	60	if fs_filter and filesystem['mdsmap']['fs_name'] != fs_filter:
	61	continue
	62
7c673cae FG	63	rank_table = PrettyTable(
	64	("Rank", "State", "MDS", "Activity", "dns", "inos"),
	65	hrules=prettytable.FRAME
	66	)
	67
	68	mdsmap = filesystem['mdsmap']
	69
	70	client_count = 0
	71
	72	for rank in mdsmap["in"]:
	73	up = "mds_{0}".format(rank) in mdsmap["up"]
	74	if up:
	75	gid = mdsmap['up']["mds_{0}".format(rank)]
	76	info = mdsmap['info']['gid_{0}'.format(gid)]
31f18b77	77	dns = self.get_latest("mds", info['name'], "mds_mem.dn")
7c673cae FG	78	inos = self.get_latest("mds", info['name'], "mds_mem.ino")
	79
	80	if rank == 0:
	81	client_count = self.get_latest("mds", info['name'],
	82	"mds_sessions.session_count")
	83	elif client_count == 0:
	84	# In case rank 0 was down, look at another rank's
	85	# sessionmap to get an indication of clients.
	86	client_count = self.get_latest("mds", info['name'],
	87	"mds_sessions.session_count")
	88
	89	laggy = "laggy_since" in info
	90
	91	state = info['state'].split(":")[1]
	92	if laggy:
	93	state += "(laggy)"
	94	if state == "active" and not laggy:
11fdf7f2	95	c_state = mgr_util.colorize(state, mgr_util.GREEN)
7c673cae	96	else:
11fdf7f2	97	c_state = mgr_util.colorize(state, mgr_util.YELLOW)
7c673cae FG	98
	99	# Populate based on context of state, e.g. client
	100	# ops for an active daemon, replay progress, reconnect
	101	# progress
	102	activity = ""
	103
	104	if state == "active":
11fdf7f2	105	activity = "Reqs: " + mgr_util.format_dimless(
7c673cae FG	106	self.get_rate("mds", info['name'], "mds_server.handle_client_request"),
	107	5
	108	) + "/s"
	109
	110	metadata = self.get_metadata('mds', info['name'])
	111	mds_versions[metadata.get('ceph_version', "unknown")].append(info['name'])
	112	rank_table.add_row([
11fdf7f2	113	mgr_util.bold(rank.__str__()), c_state, info['name'],
7c673cae	114	activity,
11fdf7f2 TL	115	mgr_util.format_dimless(dns, 5),
11fdf7f2 TL	116	mgr_util.format_dimless(inos, 5)
7c673cae FG	117	])
	118
	119	else:
	120	rank_table.add_row([
c07f9fc5	121	rank, "failed", "", "", "", ""
7c673cae FG	122	])
	123
	124	# Find the standby replays
1adf2230	125	for gid_str, daemon_info in six.iteritems(mdsmap['info']):
7c673cae FG	126	if daemon_info['state'] != "up:standby-replay":
	127	continue
	128
	129	inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino")
31f18b77	130	dns = self.get_latest("mds", daemon_info['name'], "mds_mem.dn")
7c673cae	131
11fdf7f2	132	activity = "Evts: " + mgr_util.format_dimless(
f64942e4	133	self.get_rate("mds", daemon_info['name'], "mds_log.replayed"),
7c673cae FG	134	5
	135	) + "/s"
	136
11fdf7f2 TL	137	metadata = self.get_metadata('mds', daemon_info['name'])
	138	mds_versions[metadata.get('ceph_version', "unknown")].append(daemon_info['name'])
	139
7c673cae FG	140	rank_table.add_row([
	141	"{0}-s".format(daemon_info['rank']), "standby-replay",
	142	daemon_info['name'], activity,
11fdf7f2 TL	143	mgr_util.format_dimless(dns, 5),
11fdf7f2 TL	144	mgr_util.format_dimless(inos, 5)
7c673cae FG	145	])
	146
	147	df = self.get("df")
	148	pool_stats = dict([(p['id'], p['stats']) for p in df['pools']])
	149	osdmap = self.get("osd_map")
	150	pools = dict([(p['pool'], p) for p in osdmap['pools']])
	151	metadata_pool_id = mdsmap['metadata_pool']
	152	data_pool_ids = mdsmap['data_pools']
	153
	154	pools_table = PrettyTable(["Pool", "type", "used", "avail"])
	155	for pool_id in [metadata_pool_id] + data_pool_ids:
	156	pool_type = "metadata" if pool_id == metadata_pool_id else "data"
	157	stats = pool_stats[pool_id]
	158	pools_table.add_row([
	159	pools[pool_id]['pool_name'], pool_type,
11fdf7f2 TL	160	mgr_util.format_bytes(stats['bytes_used'], 5),
11fdf7f2 TL	161	mgr_util.format_bytes(stats['max_avail'], 5)
7c673cae FG	162	])
	163
	164	output += "{0} - {1} clients\n".format(
	165	mdsmap['fs_name'], client_count)
	166	output += "=" * len(mdsmap['fs_name']) + "\n"
	167	output += rank_table.get_string()
	168	output += "\n" + pools_table.get_string() + "\n"
	169
11fdf7f2 TL	170	if not output and fs_filter is not None:
	171	return errno.EINVAL, "", "Invalid filesystem: " + fs_filter
	172
7c673cae FG	173	standby_table = PrettyTable(["Standby MDS"])
	174	for standby in fsmap['standbys']:
	175	metadata = self.get_metadata('mds', standby['name'])
	176	mds_versions[metadata.get('ceph_version', "unknown")].append(standby['name'])
	177
	178	standby_table.add_row([standby['name']])
	179
	180	output += "\n" + standby_table.get_string() + "\n"
	181
	182	if len(mds_versions) == 1:
f64942e4	183	output += "MDS version: {0}".format(list(mds_versions)[0])
7c673cae FG	184	else:
7c673cae FG	185	version_table = PrettyTable(["version", "daemons"])
1adf2230	186	for version, daemons in six.iteritems(mds_versions):
7c673cae FG	187	version_table.add_row([
	188	version,
	189	", ".join(daemons)
	190	])
	191	output += version_table.get_string() + "\n"
	192
11fdf7f2	193	return 0, output, ""
7c673cae	194
224ce89b	195	def handle_osd_status(self, cmd):
b32b8144	196	osd_table = PrettyTable(['id', 'host', 'used', 'avail', 'wr ops', 'wr data', 'rd ops', 'rd data', 'state'])
7c673cae FG	197	osdmap = self.get("osd_map")
7c673cae FG	198
224ce89b WB	199	filter_osds = set()
	200	bucket_filter = None
	201	if 'bucket' in cmd:
	202	self.log.debug("Filtering to bucket '{0}'".format(cmd['bucket']))
	203	bucket_filter = cmd['bucket']
	204	crush = self.get("osd_map_crush")
	205	found = False
	206	for bucket in crush['buckets']:
	207	if fnmatch.fnmatch(bucket['name'], bucket_filter):
	208	found = True
	209	filter_osds.update([i['id'] for i in bucket['items']])
	210
	211	if not found:
	212	msg = "Bucket '{0}' not found".format(bucket_filter)
	213	return errno.ENOENT, msg, ""
	214
7c673cae FG	215	# Build dict of OSD ID to stats
	216	osd_stats = dict([(o['osd'], o) for o in self.get("osd_stats")['osd_stats']])
	217
	218	for osd in osdmap['osds']:
	219	osd_id = osd['osd']
224ce89b WB	220	if bucket_filter and osd_id not in filter_osds:
	221	continue
	222
b32b8144 FG	223	hostname = ""
	224	kb_used = 0
	225	kb_avail = 0
7c673cae	226
b32b8144 FG	227	if osd_id in osd_stats:
	228	metadata = self.get_metadata('osd', "%s" % osd_id)
	229	stats = osd_stats[osd_id]
	230	hostname = metadata['hostname']
	231	kb_used = stats['kb_used'] * 1024
	232	kb_avail = stats['kb_avail'] * 1024
	233
	234	osd_table.add_row([osd_id, hostname,
11fdf7f2 TL	235	mgr_util.format_bytes(kb_used, 5),
	236	mgr_util.format_bytes(kb_avail, 5),
	237	mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_w") +
7c673cae	238	self.get_rate("osd", osd_id.__str__(), "osd.op_rw"), 5),
11fdf7f2 TL	239	mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_in_bytes"), 5),
	240	mgr_util.format_dimless(self.get_rate("osd", osd_id.__str__(), "osd.op_r"), 5),
	241	mgr_util.format_bytes(self.get_rate("osd", osd_id.__str__(), "osd.op_out_bytes"), 5),
b32b8144	242	','.join(osd['state']),
7c673cae FG	243	])
7c673cae FG	244
11fdf7f2	245	return 0, osd_table.get_string(), ""
7c673cae	246
11fdf7f2	247	def handle_command(self, inbuf, cmd):
7c673cae FG	248	self.log.error("handle_command")
	249
	250	if cmd['prefix'] == "fs status":
	251	return self.handle_fs_status(cmd)
224ce89b WB	252	elif cmd['prefix'] == "osd status":
224ce89b WB	253	return self.handle_osd_status(cmd)
7c673cae FG	254	else:
	255	# mgr should respect our self.COMMANDS and not call us for
	256	# any prefix we don't advertise
	257	raise NotImplementedError(cmd['prefix'])