[ceph.git] / ceph / src / ceph-volume / ceph_volume / devices / lvm / listing.py

from __future__ import print_function
import argparse
import json
import logging
from textwrap import dedent
from ceph_volume import decorators
from ceph_volume.util import disk
from ceph_volume.api import lvm as api
from ceph_volume.exceptions import MultipleLVsError

logger = logging.getLogger(__name__)


osd_list_header_template = """\n
{osd_id:=^20}"""


osd_device_header_template = """

  {type: <13} {path}
"""

device_metadata_item_template = """
      {tag_name: <25} {value}"""


def readable_tag(tag):
    actual_name = tag.split('.')[-1]
    return actual_name.replace('_', ' ')


def pretty_report(report):
    output = []
    for _id, devices in sorted(report.items()):
        output.append(
            osd_list_header_template.format(osd_id=" osd.%s " % _id)
        )
        for device in devices:
            output.append(
                osd_device_header_template.format(
                    type='[%s]' % device['type'],
                    path=device['path']
                )
            )
            for tag_name, value in sorted(device.get('tags', {}).items()):
                output.append(
                    device_metadata_item_template.format(
                        tag_name=readable_tag(tag_name),
                        value=value
                    )
                )
            if not device.get('devices'):
                continue
            else:
                output.append(
                    device_metadata_item_template.format(
                        tag_name='devices',
                        value=','.join(device['devices'])
                    )
                )

    print(''.join(output))


def direct_report():
    """
    Other non-cli consumers of listing information will want to consume the
    report without the need to parse arguments or other flags. This helper
    bypasses the need to deal with the class interface which is meant for cli
    handling.
    """
    _list = List([])
    # this is crucial: make sure that all paths will reflect current
    # information. In the case of a system that has migrated, the disks will
    # have changed paths
    _list.update()
    return _list.full_report()


class List(object):

    help = 'list logical volumes and devices associated with Ceph'

    def __init__(self, argv):
        self.argv = argv

    @property
    def pvs(self):
        """
        To avoid having to make an LVM API call for every single item being
        reported, the call gets set only once, using that stored call for
        subsequent calls
        """
        if getattr(self, '_pvs', None) is not None:
            return self._pvs
        self._pvs = api.get_api_pvs()
        return self._pvs

    def match_devices(self, lv_uuid):
        """
        It is possible to have more than one PV reported *with the same name*,
        to avoid incorrect or duplicate contents we correlated the lv uuid to
        the one on the physical device.
        """
        devices = []
        for device in self.pvs:
            if device.get('lv_uuid') == lv_uuid:
                devices.append(device['pv_name'])
        return devices

    @decorators.needs_root
    def list(self, args):
        # ensure everything is up to date before calling out
        # to list lv's
        lvs = self.update()
        report = self.generate(args, lvs)
        if args.format == 'json':
            # If the report is empty, we don't return a non-zero exit status
            # because it is assumed this is going to be consumed by automated
            # systems like ceph-ansible which would be forced to ignore the
            # non-zero exit status if all they need is the information in the
            # JSON object
            print(json.dumps(report, indent=4, sort_keys=True))
        else:
            if not report:
                raise SystemExit('No valid Ceph devices found')
            pretty_report(report)

    def update(self):
        """
        Ensure all journal devices are up to date if they aren't a logical
        volume
        """
        lvs = api.Volumes()
        for lv in lvs:
            try:
                lv.tags['ceph.osd_id']
            except KeyError:
                # only consider ceph-based logical volumes, everything else
                # will get ignored
                continue

            for device_type in ['journal', 'block', 'wal', 'db']:
                device_name = 'ceph.%s_device' % device_type
                device_uuid = lv.tags.get('ceph.%s_uuid' % device_type)
                if not device_uuid:
                    # bluestore will not have a journal, filestore will not have
                    # a block/wal/db, so we must skip if not present
                    continue
                disk_device = disk.get_device_from_partuuid(device_uuid)
                if disk_device:
                    if lv.tags[device_name] != disk_device:
                        # this means that the device has changed, so it must be updated
                        # on the API to reflect this
                        lv.set_tags({device_name: disk_device})
        return lvs

    def generate(self, args, lvs=None):
        """
        Generate reports for an individual device or for all Ceph-related
        devices, logical or physical, as long as they have been prepared by
        this tool before and contain enough metadata.
        """
        if args.device:
            return self.single_report(args.device, lvs)
        else:
            return self.full_report(lvs)

    def single_report(self, device, lvs=None):
        """
        Generate a report for a single device. This can be either a logical
        volume in the form of vg/lv or a device with an absolute path like
        /dev/sda1 or /dev/sda
        """
        if lvs is None:
            lvs = api.Volumes()
        report = {}
        lv = api.get_lv_from_argument(device)

        # check if there was a pv created with the
        # name of device
        pv = api.get_pv(pv_name=device)
        if pv and not lv:
            try:
                lv = api.get_lv(vg_name=pv.vg_name)
            except MultipleLVsError:
                lvs.filter(vg_name=pv.vg_name)
                return self.full_report(lvs=lvs)

        if lv:
            try:
                _id = lv.tags['ceph.osd_id']
            except KeyError:
                logger.warning('device is not part of ceph: %s', device)
                return report

            report.setdefault(_id, [])
            lv_report = lv.as_dict()
            lv_report['devices'] = self.match_devices(lv.lv_uuid)
            report[_id].append(lv_report)

        else:
            # this has to be a journal/wal/db device (not a logical volume) so try
            # to find the PARTUUID that should be stored in the OSD logical
            # volume
            for device_type in ['journal', 'block', 'wal', 'db']:
                device_tag_name = 'ceph.%s_device' % device_type
                device_tag_uuid = 'ceph.%s_uuid' % device_type
                associated_lv = lvs.get(lv_tags={device_tag_name: device})
                if associated_lv:
                    _id = associated_lv.tags['ceph.osd_id']
                    uuid = associated_lv.tags[device_tag_uuid]

                    report.setdefault(_id, [])
                    report[_id].append(
                        {
                            'tags': {'PARTUUID': uuid},
                            'type': device_type,
                            'path': device,
                        }
                    )
        return report

    def full_report(self, lvs=None):
        """
        Generate a report for all the logical volumes and associated devices
        that have been previously prepared by Ceph
        """
        if lvs is None:
            lvs = api.Volumes()
        report = {}

        for lv in lvs:
            try:
                _id = lv.tags['ceph.osd_id']
            except KeyError:
                # only consider ceph-based logical volumes, everything else
                # will get ignored
                continue

            report.setdefault(_id, [])
            lv_report = lv.as_dict()
            lv_report['devices'] = self.match_devices(lv.lv_uuid)
            report[_id].append(lv_report)

            for device_type in ['journal', 'block', 'wal', 'db']:
                device_uuid = lv.tags.get('ceph.%s_uuid' % device_type)
                if not device_uuid:
                    # bluestore will not have a journal, filestore will not have
                    # a block/wal/db, so we must skip if not present
                    continue
                if not api.get_lv(lv_uuid=device_uuid, lvs=lvs):
                    # means we have a regular device, so query blkid
                    disk_device = disk.get_device_from_partuuid(device_uuid)
                    if disk_device:
                        report[_id].append(
                            {
                                'tags': {'PARTUUID': device_uuid},
                                'type': device_type,
                                'path': disk_device,
                            }
                        )

        return report

    def main(self):
        sub_command_help = dedent("""
        List devices or logical volumes associated with Ceph. An association is
        determined if a device has information relating to an OSD. This is
        verified by querying LVM's metadata and correlating it with devices.

        The lvs associated with the OSD need to have been prepared previously,
        so that all needed tags and metadata exist.

        Full listing of all system devices associated with a cluster::

            ceph-volume lvm list

        List a particular device, reporting all metadata about it::

            ceph-volume lvm list /dev/sda1

        List a logical volume, along with all its metadata (vg is a volume
        group, and lv the logical volume name)::

            ceph-volume lvm list {vg/lv}
        """)
        parser = argparse.ArgumentParser(
            prog='ceph-volume lvm list',
            formatter_class=argparse.RawDescriptionHelpFormatter,
            description=sub_command_help,
        )

        parser.add_argument(
            'device',
            metavar='DEVICE',
            nargs='?',
            help='Path to an lv (as vg/lv) or to a device like /dev/sda1'
        )

        parser.add_argument(
            '--format',
            help='output format, defaults to "pretty"',
            default='pretty',
            choices=['json', 'pretty'],
        )

        args = parser.parse_args(self.argv)
        self.list(args)
Commit	Line	Data
3efd9988 FG	1	from __future__ import print_function
	2	import argparse
	3	import json
	4	import logging
	5	from textwrap import dedent
	6	from ceph_volume import decorators
	7	from ceph_volume.util import disk
	8	from ceph_volume.api import lvm as api
94b18763	9	from ceph_volume.exceptions import MultipleLVsError
3efd9988 FG	10
	11	logger = logging.getLogger(__name__)
	12
	13
	14	osd_list_header_template = """\n
	15	{osd_id:=^20}"""
	16
	17
	18	osd_device_header_template = """
	19
11fdf7f2	20	{type: <13} {path}
3efd9988 FG	21	"""
	22
	23	device_metadata_item_template = """
	24	{tag_name: <25} {value}"""
	25
	26
	27	def readable_tag(tag):
	28	actual_name = tag.split('.')[-1]
	29	return actual_name.replace('_', ' ')
	30
	31
	32	def pretty_report(report):
	33	output = []
11fdf7f2	34	for _id, devices in sorted(report.items()):
3efd9988 FG	35	output.append(
	36	osd_list_header_template.format(osd_id=" osd.%s " % _id)
	37	)
	38	for device in devices:
	39	output.append(
	40	osd_device_header_template.format(
11fdf7f2	41	type='[%s]' % device['type'],
3efd9988 FG	42	path=device['path']
	43	)
	44	)
11fdf7f2	45	for tag_name, value in sorted(device.get('tags', {}).items()):
3efd9988 FG	46	output.append(
	47	device_metadata_item_template.format(
	48	tag_name=readable_tag(tag_name),
	49	value=value
	50	)
	51	)
1adf2230 AA	52	if not device.get('devices'):
	53	continue
	54	else:
	55	output.append(
	56	device_metadata_item_template.format(
	57	tag_name='devices',
	58	value=','.join(device['devices'])
	59	)
	60	)
28e407b8	61
3efd9988 FG	62	print(''.join(output))
	63
	64
94b18763 FG	65	def direct_report():
	66	"""
	67	Other non-cli consumers of listing information will want to consume the
	68	report without the need to parse arguments or other flags. This helper
	69	bypasses the need to deal with the class interface which is meant for cli
	70	handling.
	71	"""
	72	_list = List([])
	73	# this is crucial: make sure that all paths will reflect current
	74	# information. In the case of a system that has migrated, the disks will
	75	# have changed paths
	76	_list.update()
	77	return _list.full_report()
	78
	79
3efd9988 FG	80	class List(object):
	81
	82	help = 'list logical volumes and devices associated with Ceph'
	83
	84	def __init__(self, argv):
	85	self.argv = argv
	86
28e407b8 AA	87	@property
	88	def pvs(self):
	89	"""
	90	To avoid having to make an LVM API call for every single item being
	91	reported, the call gets set only once, using that stored call for
	92	subsequent calls
	93	"""
	94	if getattr(self, '_pvs', None) is not None:
	95	return self._pvs
	96	self._pvs = api.get_api_pvs()
	97	return self._pvs
	98
	99	def match_devices(self, lv_uuid):
	100	"""
	101	It is possible to have more than one PV reported with the same name,
	102	to avoid incorrect or duplicate contents we correlated the lv uuid to
	103	the one on the physical device.
	104	"""
	105	devices = []
	106	for device in self.pvs:
	107	if device.get('lv_uuid') == lv_uuid:
	108	devices.append(device['pv_name'])
	109	return devices
	110
3efd9988 FG	111	@decorators.needs_root
	112	def list(self, args):
	113	# ensure everything is up to date before calling out
	114	# to list lv's
eafe8130 TL	115	lvs = self.update()
eafe8130 TL	116	report = self.generate(args, lvs)
3efd9988 FG	117	if args.format == 'json':
	118	# If the report is empty, we don't return a non-zero exit status
	119	# because it is assumed this is going to be consumed by automated
	120	# systems like ceph-ansible which would be forced to ignore the
	121	# non-zero exit status if all they need is the information in the
	122	# JSON object
	123	print(json.dumps(report, indent=4, sort_keys=True))
	124	else:
	125	if not report:
	126	raise SystemExit('No valid Ceph devices found')
	127	pretty_report(report)
	128
	129	def update(self):
	130	"""
	131	Ensure all journal devices are up to date if they aren't a logical
	132	volume
	133	"""
	134	lvs = api.Volumes()
	135	for lv in lvs:
	136	try:
	137	lv.tags['ceph.osd_id']
	138	except KeyError:
	139	# only consider ceph-based logical volumes, everything else
	140	# will get ignored
	141	continue
	142
	143	for device_type in ['journal', 'block', 'wal', 'db']:
	144	device_name = 'ceph.%s_device' % device_type
	145	device_uuid = lv.tags.get('ceph.%s_uuid' % device_type)
	146	if not device_uuid:
	147	# bluestore will not have a journal, filestore will not have
	148	# a block/wal/db, so we must skip if not present
	149	continue
	150	disk_device = disk.get_device_from_partuuid(device_uuid)
	151	if disk_device:
	152	if lv.tags[device_name] != disk_device:
	153	# this means that the device has changed, so it must be updated
	154	# on the API to reflect this
	155	lv.set_tags({device_name: disk_device})
eafe8130	156	return lvs
3efd9988	157
eafe8130	158	def generate(self, args, lvs=None):
3efd9988 FG	159	"""
	160	Generate reports for an individual device or for all Ceph-related
	161	devices, logical or physical, as long as they have been prepared by
	162	this tool before and contain enough metadata.
	163	"""
	164	if args.device:
eafe8130	165	return self.single_report(args.device, lvs)
3efd9988	166	else:
eafe8130	167	return self.full_report(lvs)
3efd9988	168
eafe8130	169	def single_report(self, device, lvs=None):
3efd9988 FG	170	"""
	171	Generate a report for a single device. This can be either a logical
	172	volume in the form of vg/lv or a device with an absolute path like
94b18763	173	/dev/sda1 or /dev/sda
3efd9988	174	"""
eafe8130 TL	175	if lvs is None:
eafe8130 TL	176	lvs = api.Volumes()
3efd9988 FG	177	report = {}
3efd9988 FG	178	lv = api.get_lv_from_argument(device)
94b18763 FG	179
	180	# check if there was a pv created with the
	181	# name of device
	182	pv = api.get_pv(pv_name=device)
	183	if pv and not lv:
	184	try:
	185	lv = api.get_lv(vg_name=pv.vg_name)
	186	except MultipleLVsError:
	187	lvs.filter(vg_name=pv.vg_name)
	188	return self.full_report(lvs=lvs)
	189
3efd9988 FG	190	if lv:
	191	try:
	192	_id = lv.tags['ceph.osd_id']
	193	except KeyError:
	194	logger.warning('device is not part of ceph: %s', device)
	195	return report
	196
	197	report.setdefault(_id, [])
28e407b8 AA	198	lv_report = lv.as_dict()
	199	lv_report['devices'] = self.match_devices(lv.lv_uuid)
	200	report[_id].append(lv_report)
3efd9988 FG	201
	202	else:
	203	# this has to be a journal/wal/db device (not a logical volume) so try
	204	# to find the PARTUUID that should be stored in the OSD logical
	205	# volume
	206	for device_type in ['journal', 'block', 'wal', 'db']:
	207	device_tag_name = 'ceph.%s_device' % device_type
	208	device_tag_uuid = 'ceph.%s_uuid' % device_type
	209	associated_lv = lvs.get(lv_tags={device_tag_name: device})
	210	if associated_lv:
	211	_id = associated_lv.tags['ceph.osd_id']
	212	uuid = associated_lv.tags[device_tag_uuid]
	213
	214	report.setdefault(_id, [])
	215	report[_id].append(
	216	{
	217	'tags': {'PARTUUID': uuid},
	218	'type': device_type,
	219	'path': device,
	220	}
	221	)
	222	return report
	223
94b18763	224	def full_report(self, lvs=None):
3efd9988 FG	225	"""
	226	Generate a report for all the logical volumes and associated devices
	227	that have been previously prepared by Ceph
	228	"""
94b18763 FG	229	if lvs is None:
94b18763 FG	230	lvs = api.Volumes()
3efd9988	231	report = {}
eafe8130	232
3efd9988 FG	233	for lv in lvs:
	234	try:
	235	_id = lv.tags['ceph.osd_id']
	236	except KeyError:
	237	# only consider ceph-based logical volumes, everything else
	238	# will get ignored
	239	continue
	240
	241	report.setdefault(_id, [])
28e407b8 AA	242	lv_report = lv.as_dict()
	243	lv_report['devices'] = self.match_devices(lv.lv_uuid)
	244	report[_id].append(lv_report)
3efd9988 FG	245
	246	for device_type in ['journal', 'block', 'wal', 'db']:
	247	device_uuid = lv.tags.get('ceph.%s_uuid' % device_type)
	248	if not device_uuid:
	249	# bluestore will not have a journal, filestore will not have
	250	# a block/wal/db, so we must skip if not present
	251	continue
eafe8130	252	if not api.get_lv(lv_uuid=device_uuid, lvs=lvs):
3efd9988 FG	253	# means we have a regular device, so query blkid
	254	disk_device = disk.get_device_from_partuuid(device_uuid)
	255	if disk_device:
	256	report[_id].append(
	257	{
	258	'tags': {'PARTUUID': device_uuid},
	259	'type': device_type,
	260	'path': disk_device,
	261	}
	262	)
	263
	264	return report
	265
	266	def main(self):
	267	sub_command_help = dedent("""
	268	List devices or logical volumes associated with Ceph. An association is
	269	determined if a device has information relating to an OSD. This is
	270	verified by querying LVM's metadata and correlating it with devices.
	271
	272	The lvs associated with the OSD need to have been prepared previously,
	273	so that all needed tags and metadata exist.
	274
	275	Full listing of all system devices associated with a cluster::
	276
	277	ceph-volume lvm list
	278
	279	List a particular device, reporting all metadata about it::
	280
	281	ceph-volume lvm list /dev/sda1
	282
	283	List a logical volume, along with all its metadata (vg is a volume
	284	group, and lv the logical volume name)::
	285
	286	ceph-volume lvm list {vg/lv}
	287	""")
	288	parser = argparse.ArgumentParser(
	289	prog='ceph-volume lvm list',
	290	formatter_class=argparse.RawDescriptionHelpFormatter,
	291	description=sub_command_help,
	292	)
	293
	294	parser.add_argument(
	295	'device',
	296	metavar='DEVICE',
	297	nargs='?',
	298	help='Path to an lv (as vg/lv) or to a device like /dev/sda1'
	299	)
	300
	301	parser.add_argument(
	302	'--format',
	303	help='output format, defaults to "pretty"',
	304	default='pretty',
	305	choices=['json', 'pretty'],
	306	)
	307
	308	args = parser.parse_args(self.argv)
	309	self.list(args)