ceph/src/ceph-volume/ceph_volume/devices/lvm/api.py

   1 """
   2 API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention
   3 that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides
   4 set of utilities for interacting with LVM.
   5 """
   6 from ceph_volume import process
   7 from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError, MultiplePVsError
   8
   9
  10 def _output_parser(output, fields):
  11     """
  12     Newer versions of LVM allow ``--reportformat=json``, but older versions,
  13     like the one included in Xenial do not. LVM has the ability to filter and
  14     format its output so we assume the output will be in a format this parser
  15     can handle (using ',' as a delimiter)
  16
  17     :param fields: A string, possibly using ',' to group many items, as it
  18                    would be used on the CLI
  19     :param output: The CLI output from the LVM call
  20     """
  21     field_items = fields.split(',')
  22     report = []
  23     for line in output:
  24         # clear the leading/trailing whitespace
  25         line = line.strip()
  26
  27         # remove the extra '"' in each field
  28         line = line.replace('"', '')
  29
  30         # prevent moving forward with empty contents
  31         if not line:
  32             continue
  33
  34         # spliting on ';' because that is what the lvm call uses as
  35         # '--separator'
  36         output_items = [i.strip() for i in line.split(';')]
  37         # map the output to the fiels
  38         report.append(
  39             dict(zip(field_items, output_items))
  40         )
  41
  42     return report
  43
  44
  45 def parse_tags(lv_tags):
  46     """
  47     Return a dictionary mapping of all the tags associated with
  48     a Volume from the comma-separated tags coming from the LVM API
  49
  50     Input look like::
  51
  52        "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0"
  53
  54     For the above example, the expected return value would be::
  55
  56         {
  57             "ceph.osd_fsid": "aaa-fff-bbbb",
  58             "ceph.osd_id": "0"
  59         }
  60     """
  61     if not lv_tags:
  62         return {}
  63     tag_mapping = {}
  64     tags = lv_tags.split(',')
  65     for tag_assignment in tags:
  66         key, value = tag_assignment.split('=', 1)
  67         tag_mapping[key] = value
  68
  69     return tag_mapping
  70
  71
  72 def get_api_vgs():
  73     """
  74     Return the list of group volumes available in the system using flags to
  75     include common metadata associated with them
  76
  77     Command and sample delimeted output, should look like::
  78
  79         $ sudo vgs --noheadings --separator=';' \
  80           -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free
  81           ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m
  82           osd_vg;3;1;0;wz--n-;29.21g;9.21g
  83
  84     """
  85     fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free'
  86     stdout, stderr, returncode = process.call(
  87         ['sudo', 'vgs', '--noheadings', '--separator=";"', '-o', fields]
  88     )
  89     return _output_parser(stdout, fields)
  90
  91
  92 def get_api_lvs():
  93     """
  94     Return the list of logical volumes available in the system using flags to include common
  95     metadata associated with them
  96
  97     Command and delimeted output, should look like::
  98
  99         $ sudo lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name
 100           ;/dev/ubuntubox-vg/root;root;ubuntubox-vg
 101           ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg
 102
 103     """
 104     fields = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid'
 105     stdout, stderr, returncode = process.call(
 106         ['sudo', 'lvs', '--noheadings', '--separator=";"', '-o', fields]
 107     )
 108     return _output_parser(stdout, fields)
 109
 110
 111 def get_api_pvs():
 112     """
 113     Return the list of physical volumes configured for lvm and available in the
 114     system using flags to include common metadata associated with them like the uuid
 115
 116     Command and delimeted output, should look like::
 117
 118         $ sudo pvs --noheadings --separator=';' -o pv_name,pv_tags,pv_uuid
 119           /dev/sda1;;
 120           /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D
 121
 122     """
 123     fields = 'pv_name,pv_tags,pv_uuid'
 124
 125     # note the use of `pvs -a` which will return every physical volume including
 126     # ones that have not been initialized as "pv" by LVM
 127     stdout, stderr, returncode = process.call(
 128         ['sudo', 'pvs', '-a', '--no-heading', '--separator=";"', '-o', fields]
 129     )
 130
 131     return _output_parser(stdout, fields)
 132
 133
 134 def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
 135     """
 136     Return a matching lv for the current system, requiring ``lv_name``,
 137     ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv
 138     is found.
 139
 140     It is useful to use ``tags`` when trying to find a specific logical volume,
 141     but it can also lead to multiple lvs being found, since a lot of metadata
 142     is shared between lvs of a distinct OSD.
 143     """
 144     if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
 145         return None
 146     lvs = Volumes()
 147     return lvs.get(
 148         lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_uuid=lv_uuid,
 149         lv_tags=lv_tags
 150     )
 151
 152
 153 def get_pv(pv_name=None, pv_uuid=None, pv_tags=None):
 154     """
 155     Return a matching pv (physical volume) for the current system, requiring
 156     ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one
 157     pv is found.
 158     """
 159     if not any([pv_name, pv_uuid, pv_tags]):
 160         return None
 161     pvs = PVolumes()
 162     return pvs.get(pv_name=pv_name, pv_uuid=pv_uuid, pv_tags=pv_tags)
 163
 164
 165 def create_pv(device):
 166     """
 167     Create a physical volume from a device, useful when devices need to be later mapped
 168     to journals.
 169     """
 170     process.run([
 171         'sudo',
 172         'pvcreate',
 173         '-v',  # verbose
 174         '-f',  # force it
 175         '--yes', # answer yes to any prompts
 176         device
 177     ])
 178
 179
 180 def create_lv(name, group, size=None, **tags):
 181     """
 182     Create a Logical Volume in a Volume Group. Command looks like::
 183
 184         lvcreate -L 50G -n gfslv vg0
 185
 186     ``name``, ``group``, and ``size`` are required. Tags are optional and are "translated" to include
 187     the prefixes for the Ceph LVM tag API.
 188
 189     """
 190     # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations
 191     type_path_tag = {
 192         'journal': 'ceph.journal_device',
 193         'data': 'ceph.data_device',
 194         'block': 'ceph.block',
 195         'wal': 'ceph.wal',
 196         'db': 'ceph.db',
 197         'lockbox': 'ceph.lockbox_device',
 198     }
 199     if size:
 200         process.run([
 201             'sudo',
 202             'lvcreate',
 203             '--yes',
 204             '-L',
 205             '%sG' % size,
 206             '-n', name, group
 207         ])
 208     # create the lv with all the space available, this is needed because the
 209     # system call is different for LVM
 210     else:
 211         process.run([
 212             'sudo',
 213             'lvcreate',
 214             '--yes',
 215             '-l',
 216             '100%FREE',
 217             '-n', name, group
 218         ])
 219
 220     lv = get_lv(lv_name=name, vg_name=group)
 221     ceph_tags = {}
 222     for k, v in tags.items():
 223         ceph_tags['ceph.%s' % k] = v
 224     lv.set_tags(ceph_tags)
 225
 226     # when creating a distinct type, the caller doesn't know what the path will
 227     # be so this function will set it after creation using the mapping
 228     path_tag = type_path_tag[tags['type']]
 229     lv.set_tags(
 230         {path_tag: lv.lv_path}
 231     )
 232     return lv
 233
 234
 235 def get_vg(vg_name=None, vg_tags=None):
 236     """
 237     Return a matching vg for the current system, requires ``vg_name`` or
 238     ``tags``. Raises an error if more than one vg is found.
 239
 240     It is useful to use ``tags`` when trying to find a specific volume group,
 241     but it can also lead to multiple vgs being found.
 242     """
 243     if not any([vg_name, vg_tags]):
 244         return None
 245     vgs = VolumeGroups()
 246     return vgs.get(vg_name=vg_name, vg_tags=vg_tags)
 247
 248
 249 class VolumeGroups(list):
 250     """
 251     A list of all known volume groups for the current system, with the ability
 252     to filter them via keyword arguments.
 253     """
 254
 255     def __init__(self):
 256         self._populate()
 257
 258     def _populate(self):
 259         # get all the vgs in the current system
 260         for vg_item in get_api_vgs():
 261             self.append(VolumeGroup(**vg_item))
 262
 263     def _purge(self):
 264         """
 265         Deplete all the items in the list, used internally only so that we can
 266         dynamically allocate the items when filtering without the concern of
 267         messing up the contents
 268         """
 269         self[:] = []
 270
 271     def _filter(self, vg_name=None, vg_tags=None):
 272         """
 273         The actual method that filters using a new list. Useful so that other
 274         methods that do not want to alter the contents of the list (e.g.
 275         ``self.find``) can operate safely.
 276
 277         .. note:: ``vg_tags`` is not yet implemented
 278         """
 279         filtered = [i for i in self]
 280         if vg_name:
 281             filtered = [i for i in filtered if i.vg_name == vg_name]
 282
 283         # at this point, `filtered` has either all the volumes in self or is an
 284         # actual filtered list if any filters were applied
 285         if vg_tags:
 286             tag_filtered = []
 287             for volume in filtered:
 288                 matches = all(volume.tags.get(k) == str(v) for k, v in vg_tags.items())
 289                 if matches:
 290                     tag_filtered.append(volume)
 291             return tag_filtered
 292
 293         return filtered
 294
 295     def filter(self, vg_name=None, vg_tags=None):
 296         """
 297         Filter out groups on top level attributes like ``vg_name`` or by
 298         ``vg_tags`` where a dict is required. For example, to find a Ceph group
 299         with dmcache as the type, the filter would look like::
 300
 301             vg_tags={'ceph.type': 'dmcache'}
 302
 303         .. warning:: These tags are not documented because they are currently
 304                      unused, but are here to maintain API consistency
 305         """
 306         if not any([vg_name, vg_tags]):
 307             raise TypeError('.filter() requires vg_name or vg_tags (none given)')
 308         # first find the filtered volumes with the values in self
 309         filtered_groups = self._filter(
 310             vg_name=vg_name,
 311             vg_tags=vg_tags
 312         )
 313         # then purge everything
 314         self._purge()
 315         # and add the filtered items
 316         self.extend(filtered_groups)
 317
 318     def get(self, vg_name=None, vg_tags=None):
 319         """
 320         This is a bit expensive, since it will try to filter out all the
 321         matching items in the list, filter them out applying anything that was
 322         added and return the matching item.
 323
 324         This method does *not* alter the list, and it will raise an error if
 325         multiple VGs are matched
 326
 327         It is useful to use ``tags`` when trying to find a specific volume group,
 328         but it can also lead to multiple vgs being found (although unlikely)
 329         """
 330         if not any([vg_name, vg_tags]):
 331             return None
 332         vgs = self._filter(
 333             vg_name=vg_name,
 334             vg_tags=vg_tags
 335         )
 336         if not vgs:
 337             return None
 338         if len(vgs) > 1:
 339             # this is probably never going to happen, but it is here to keep
 340             # the API code consistent
 341             raise MultipleVGsError(vg_name)
 342         return vgs[0]
 343
 344
 345 class Volumes(list):
 346     """
 347     A list of all known (logical) volumes for the current system, with the ability
 348     to filter them via keyword arguments.
 349     """
 350
 351     def __init__(self):
 352         self._populate()
 353
 354     def _populate(self):
 355         # get all the lvs in the current system
 356         for lv_item in get_api_lvs():
 357             self.append(Volume(**lv_item))
 358
 359     def _purge(self):
 360         """
 361         Deplete all the items in the list, used internally only so that we can
 362         dynamically allocate the items when filtering without the concern of
 363         messing up the contents
 364         """
 365         self[:] = []
 366
 367     def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
 368         """
 369         The actual method that filters using a new list. Useful so that other
 370         methods that do not want to alter the contents of the list (e.g.
 371         ``self.find``) can operate safely.
 372         """
 373         filtered = [i for i in self]
 374         if lv_name:
 375             filtered = [i for i in filtered if i.lv_name == lv_name]
 376
 377         if vg_name:
 378             filtered = [i for i in filtered if i.vg_name == vg_name]
 379
 380         if lv_uuid:
 381             filtered = [i for i in filtered if i.lv_uuid == lv_uuid]
 382
 383         if lv_path:
 384             filtered = [i for i in filtered if i.lv_path == lv_path]
 385
 386         # at this point, `filtered` has either all the volumes in self or is an
 387         # actual filtered list if any filters were applied
 388         if lv_tags:
 389             tag_filtered = []
 390             for volume in filtered:
 391                 # all the tags we got need to match on the volume
 392                 matches = all(volume.tags.get(k) == str(v) for k, v in lv_tags.items())
 393                 if matches:
 394                     tag_filtered.append(volume)
 395             return tag_filtered
 396
 397         return filtered
 398
 399     def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
 400         """
 401         Filter out volumes on top level attributes like ``lv_name`` or by
 402         ``lv_tags`` where a dict is required. For example, to find a volume
 403         that has an OSD ID of 0, the filter would look like::
 404
 405             lv_tags={'ceph.osd_id': '0'}
 406
 407         """
 408         if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
 409             raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)')
 410         # first find the filtered volumes with the values in self
 411         filtered_volumes = self._filter(
 412             lv_name=lv_name,
 413             vg_name=vg_name,
 414             lv_path=lv_path,
 415             lv_uuid=lv_uuid,
 416             lv_tags=lv_tags
 417         )
 418         # then purge everything
 419         self._purge()
 420         # and add the filtered items
 421         self.extend(filtered_volumes)
 422
 423     def get(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
 424         """
 425         This is a bit expensive, since it will try to filter out all the
 426         matching items in the list, filter them out applying anything that was
 427         added and return the matching item.
 428
 429         This method does *not* alter the list, and it will raise an error if
 430         multiple LVs are matched
 431
 432         It is useful to use ``tags`` when trying to find a specific logical volume,
 433         but it can also lead to multiple lvs being found, since a lot of metadata
 434         is shared between lvs of a distinct OSD.
 435         """
 436         if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
 437             return None
 438         lvs = self._filter(
 439             lv_name=lv_name,
 440             vg_name=vg_name,
 441             lv_path=lv_path,
 442             lv_uuid=lv_uuid,
 443             lv_tags=lv_tags
 444         )
 445         if not lvs:
 446             return None
 447         if len(lvs) > 1:
 448             raise MultipleLVsError(lv_name, lv_path)
 449         return lvs[0]
 450
 451
 452 class PVolumes(list):
 453     """
 454     A list of all known (physical) volumes for the current system, with the ability
 455     to filter them via keyword arguments.
 456     """
 457
 458     def __init__(self):
 459         self._populate()
 460
 461     def _populate(self):
 462         # get all the pvs in the current system
 463         for pv_item in get_api_pvs():
 464             self.append(PVolume(**pv_item))
 465
 466     def _purge(self):
 467         """
 468         Deplete all the items in the list, used internally only so that we can
 469         dynamically allocate the items when filtering without the concern of
 470         messing up the contents
 471         """
 472         self[:] = []
 473
 474     def _filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
 475         """
 476         The actual method that filters using a new list. Useful so that other
 477         methods that do not want to alter the contents of the list (e.g.
 478         ``self.find``) can operate safely.
 479         """
 480         filtered = [i for i in self]
 481         if pv_name:
 482             filtered = [i for i in filtered if i.pv_name == pv_name]
 483
 484         if pv_uuid:
 485             filtered = [i for i in filtered if i.pv_uuid == pv_uuid]
 486
 487         # at this point, `filtered` has either all the physical volumes in self
 488         # or is an actual filtered list if any filters were applied
 489         if pv_tags:
 490             tag_filtered = []
 491             for pvolume in filtered:
 492                 matches = all(pvolume.tags.get(k) == str(v) for k, v in pv_tags.items())
 493                 if matches:
 494                     tag_filtered.append(pvolume)
 495             # return the tag_filtered pvolumes here, the `filtered` list is no
 496             # longer useable
 497             return tag_filtered
 498
 499         return filtered
 500
 501     def filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
 502         """
 503         Filter out volumes on top level attributes like ``pv_name`` or by
 504         ``pv_tags`` where a dict is required. For example, to find a physical volume
 505         that has an OSD ID of 0, the filter would look like::
 506
 507             pv_tags={'ceph.osd_id': '0'}
 508
 509         """
 510         if not any([pv_name, pv_uuid, pv_tags]):
 511             raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags (none given)')
 512         # first find the filtered volumes with the values in self
 513         filtered_volumes = self._filter(
 514             pv_name=pv_name,
 515             pv_uuid=pv_uuid,
 516             pv_tags=pv_tags
 517         )
 518         # then purge everything
 519         self._purge()
 520         # and add the filtered items
 521         self.extend(filtered_volumes)
 522
 523     def get(self, pv_name=None, pv_uuid=None, pv_tags=None):
 524         """
 525         This is a bit expensive, since it will try to filter out all the
 526         matching items in the list, filter them out applying anything that was
 527         added and return the matching item.
 528
 529         This method does *not* alter the list, and it will raise an error if
 530         multiple pvs are matched
 531
 532         It is useful to use ``tags`` when trying to find a specific logical volume,
 533         but it can also lead to multiple pvs being found, since a lot of metadata
 534         is shared between pvs of a distinct OSD.
 535         """
 536         if not any([pv_name, pv_uuid, pv_tags]):
 537             return None
 538         pvs = self._filter(
 539             pv_name=pv_name,
 540             pv_uuid=pv_uuid,
 541             pv_tags=pv_tags
 542         )
 543         if not pvs:
 544             return None
 545         if len(pvs) > 1:
 546             raise MultiplePVsError(pv_name)
 547         return pvs[0]
 548
 549
 550 class VolumeGroup(object):
 551     """
 552     Represents an LVM group, with some top-level attributes like ``vg_name``
 553     """
 554
 555     def __init__(self, **kw):
 556         for k, v in kw.items():
 557             setattr(self, k, v)
 558         self.name = kw['vg_name']
 559         self.tags = parse_tags(kw.get('vg_tags', ''))
 560
 561     def __str__(self):
 562         return '<%s>' % self.name
 563
 564     def __repr__(self):
 565         return self.__str__()
 566
 567
 568 class Volume(object):
 569     """
 570     Represents a Logical Volume from LVM, with some top-level attributes like
 571     ``lv_name`` and parsed tags as a dictionary of key/value pairs.
 572     """
 573
 574     def __init__(self, **kw):
 575         for k, v in kw.items():
 576             setattr(self, k, v)
 577         self.lv_api = kw
 578         self.name = kw['lv_name']
 579         self.tags = parse_tags(kw['lv_tags'])
 580
 581     def __str__(self):
 582         return '<%s>' % self.lv_api['lv_path']
 583
 584     def __repr__(self):
 585         return self.__str__()
 586
 587     def set_tags(self, tags):
 588         """
 589         :param tags: A dictionary of tag names and values, like::
 590
 591             {
 592                 "ceph.osd_fsid": "aaa-fff-bbbb",
 593                 "ceph.osd_id": "0"
 594             }
 595
 596         At the end of all modifications, the tags are refreshed to reflect
 597         LVM's most current view.
 598         """
 599         for k, v in tags.items():
 600             self.set_tag(k, v)
 601         # after setting all the tags, refresh them for the current object, use the
 602         # lv_* identifiers to filter because those shouldn't change
 603         lv_object = get_lv(lv_name=self.lv_name, lv_path=self.lv_path)
 604         self.tags = lv_object.tags
 605
 606     def set_tag(self, key, value):
 607         """
 608         Set the key/value pair as an LVM tag. Does not "refresh" the values of
 609         the current object for its tags. Meant to be a "fire and forget" type
 610         of modification.
 611         """
 612         # remove it first if it exists
 613         if self.tags.get(key):
 614             current_value = self.tags[key]
 615             tag = "%s=%s" % (key, current_value)
 616             process.call(['sudo', 'lvchange', '--deltag', tag, self.lv_api['lv_path']])
 617
 618         process.call(
 619             [
 620                 'sudo', 'lvchange',
 621                 '--addtag', '%s=%s' % (key, value), self.lv_path
 622             ]
 623         )
 624
 625
 626 class PVolume(object):
 627     """
 628     Represents a Physical Volume from LVM, with some top-level attributes like
 629     ``pv_name`` and parsed tags as a dictionary of key/value pairs.
 630     """
 631
 632     def __init__(self, **kw):
 633         for k, v in kw.items():
 634             setattr(self, k, v)
 635         self.pv_api = kw
 636         self.name = kw['pv_name']
 637         self.tags = parse_tags(kw['pv_tags'])
 638
 639     def __str__(self):
 640         return '<%s>' % self.pv_api['pv_name']
 641
 642     def __repr__(self):
 643         return self.__str__()
 644
 645     def set_tags(self, tags):
 646         """
 647         :param tags: A dictionary of tag names and values, like::
 648
 649             {
 650                 "ceph.osd_fsid": "aaa-fff-bbbb",
 651                 "ceph.osd_id": "0"
 652             }
 653
 654         At the end of all modifications, the tags are refreshed to reflect
 655         LVM's most current view.
 656         """
 657         for k, v in tags.items():
 658             self.set_tag(k, v)
 659         # after setting all the tags, refresh them for the current object, use the
 660         # pv_* identifiers to filter because those shouldn't change
 661         pv_object = get_pv(pv_name=self.pv_name, pv_uuid=self.pv_uuid)
 662         self.tags = pv_object.tags
 663
 664     def set_tag(self, key, value):
 665         """
 666         Set the key/value pair as an LVM tag. Does not "refresh" the values of
 667         the current object for its tags. Meant to be a "fire and forget" type
 668         of modification.
 669
 670         **warning**: Altering tags on a PV has to be done ensuring that the
 671         device is actually the one intended. ``pv_name`` is *not* a persistent
 672         value, only ``pv_uuid`` is. Using ``pv_uuid`` is the best way to make
 673         sure the device getting changed is the one needed.
 674         """
 675         # remove it first if it exists
 676         if self.tags.get(key):
 677             current_value = self.tags[key]
 678             tag = "%s=%s" % (key, current_value)
 679             process.call(['sudo', 'pvchange', '--deltag', tag, self.pv_name])
 680
 681         process.call(
 682             [
 683                 'sudo', 'pvchange',
 684                 '--addtag', '%s=%s' % (key, value), self.pv_name
 685             ]
 686         )