ceph/src/ceph-volume/ceph_volume/devices/lvm/api.py

   1 """
   2 API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention
   3 that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides
   4 set of utilities for interacting with LVM.
   5 """
   6 from ceph_volume import process
   7 from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError
   8
   9
  10 def _output_parser(output, fields):
  11     """
  12     Newer versions of LVM allow ``--reportformat=json``, but older versions,
  13     like the one included in Xenial do not. LVM has the ability to filter and
  14     format its output so we assume the output will be in a format this parser
  15     can handle (using ',' as a delimiter)
  16
  17     :param fields: A string, possibly using ',' to group many items, as it
  18                    would be used on the CLI
  19     :param output: The CLI output from the LVM call
  20     """
  21     field_items = fields.split(',')
  22     report = []
  23     for line in output:
  24         # clear the leading/trailing whitespace
  25         line = line.strip()
  26
  27         # remove the extra '"' in each field
  28         line = line.replace('"', '')
  29
  30         # prevent moving forward with empty contents
  31         if not line:
  32             continue
  33
  34         # spliting on ';' because that is what the lvm call uses as
  35         # '--separator'
  36         output_items = [i.strip() for i in line.split(';')]
  37         # map the output to the fiels
  38         report.append(
  39             dict(zip(field_items, output_items))
  40         )
  41
  42     return report
  43
  44
  45 def parse_tags(lv_tags):
  46     """
  47     Return a dictionary mapping of all the tags associated with
  48     a Volume from the comma-separated tags coming from the LVM API
  49
  50     Input look like::
  51
  52        "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0"
  53
  54     For the above example, the expected return value would be::
  55
  56         {
  57             "ceph.osd_fsid": "aaa-fff-bbbb",
  58             "ceph.osd_id": "0"
  59         }
  60     """
  61     if not lv_tags:
  62         return {}
  63     tag_mapping = {}
  64     tags = lv_tags.split(',')
  65     for tag_assignment in tags:
  66         key, value = tag_assignment.split('=', 1)
  67         tag_mapping[key] = value
  68
  69     return tag_mapping
  70
  71
  72 def get_api_vgs():
  73     """
  74     Return the list of group volumes available in the system using flags to
  75     include common metadata associated with them
  76
  77     Command and sample delimeted output, should look like::
  78
  79         $ sudo vgs --noheadings --separator=';' \
  80           -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free
  81           ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m
  82           osd_vg;3;1;0;wz--n-;29.21g;9.21g
  83
  84     """
  85     fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free'
  86     stdout, stderr, returncode = process.call(
  87         ['sudo', 'vgs', '--noheadings', '--separator=";"', '-o', fields]
  88     )
  89     return _output_parser(stdout, fields)
  90
  91
  92 def get_api_lvs():
  93     """
  94     Return the list of logical volumes available in the system using flags to include common
  95     metadata associated with them
  96
  97     Command and delimeted output, should look like::
  98
  99         $ sudo lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name
 100           ;/dev/ubuntubox-vg/root;root;ubuntubox-vg
 101           ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg
 102
 103     """
 104     fields = 'lv_tags,lv_path,lv_name,vg_name'
 105     stdout, stderr, returncode = process.call(
 106         ['sudo', 'lvs', '--noheadings', '--separator=";"', '-o', fields]
 107     )
 108     return _output_parser(stdout, fields)
 109
 110
 111 def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_tags=None):
 112     """
 113     Return a matching lv for the current system, requiring ``lv_name``,
 114     ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv
 115     is found.
 116
 117     It is useful to use ``tags`` when trying to find a specific logical volume,
 118     but it can also lead to multiple lvs being found, since a lot of metadata
 119     is shared between lvs of a distinct OSD.
 120     """
 121     if not any([lv_name, vg_name, lv_path, lv_tags]):
 122         return None
 123     lvs = Volumes()
 124     return lvs.get(lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_tags=lv_tags)
 125
 126
 127 def create_lv(name, group, size=None, **tags):
 128     """
 129     Create a Logical Volume in a Volume Group. Command looks like::
 130
 131         lvcreate -L 50G -n gfslv vg0
 132
 133     ``name``, ``group``, and ``size`` are required. Tags are optional and are "translated" to include
 134     the prefixes for the Ceph LVM tag API.
 135
 136     """
 137     # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations
 138     type_path_tag = {
 139         'journal': 'ceph.journal_device',
 140         'data': 'ceph.data_device',
 141         'block': 'ceph.block',
 142         'wal': 'ceph.wal',
 143         'db': 'ceph.db',
 144         'lockbox': 'ceph.lockbox_device',
 145     }
 146     if size:
 147         process.run([
 148             'sudo',
 149             'lvcreate',
 150             '--yes',
 151             '-L',
 152             '%sG' % size,
 153             '-n', name, group
 154         ])
 155     # create the lv with all the space available, this is needed because the
 156     # system call is different for LVM
 157     else:
 158         process.run([
 159             'sudo',
 160             'lvcreate',
 161             '--yes',
 162             '-l',
 163             '100%FREE',
 164             '-n', name, group
 165         ])
 166
 167     lv = get_lv(lv_name=name, vg_name=group)
 168     ceph_tags = {}
 169     for k, v in tags.items():
 170         ceph_tags['ceph.%s' % k] = v
 171     lv.set_tags(ceph_tags)
 172
 173     # when creating a distinct type, the caller doesn't know what the path will
 174     # be so this function will set it after creation using the mapping
 175     path_tag = type_path_tag[tags['type']]
 176     lv.set_tags(
 177         {path_tag: lv.lv_path}
 178     )
 179     return lv
 180
 181
 182 def get_vg(vg_name=None, vg_tags=None):
 183     """
 184     Return a matching vg for the current system, requires ``vg_name`` or
 185     ``tags``. Raises an error if more than one vg is found.
 186
 187     It is useful to use ``tags`` when trying to find a specific volume group,
 188     but it can also lead to multiple vgs being found.
 189     """
 190     if not any([vg_name, vg_tags]):
 191         return None
 192     vgs = VolumeGroups()
 193     return vgs.get(vg_name=vg_name, vg_tags=vg_tags)
 194
 195
 196 class VolumeGroups(list):
 197     """
 198     A list of all known volume groups for the current system, with the ability
 199     to filter them via keyword arguments.
 200     """
 201
 202     def __init__(self):
 203         self._populate()
 204
 205     def _populate(self):
 206         # get all the vgs in the current system
 207         for vg_item in get_api_vgs():
 208             self.append(VolumeGroup(**vg_item))
 209
 210     def _purge(self):
 211         """
 212         Deplete all the items in the list, used internally only so that we can
 213         dynamically allocate the items when filtering without the concern of
 214         messing up the contents
 215         """
 216         self[:] = []
 217
 218     def _filter(self, vg_name=None, vg_tags=None):
 219         """
 220         The actual method that filters using a new list. Useful so that other
 221         methods that do not want to alter the contents of the list (e.g.
 222         ``self.find``) can operate safely.
 223
 224         .. note:: ``vg_tags`` is not yet implemented
 225         """
 226         filtered = [i for i in self]
 227         if vg_name:
 228             filtered = [i for i in filtered if i.vg_name == vg_name]
 229
 230         # at this point, `filtered` has either all the volumes in self or is an
 231         # actual filtered list if any filters were applied
 232         if vg_tags:
 233             tag_filtered = []
 234             for k, v in vg_tags.items():
 235                 for volume in filtered:
 236                     if volume.tags.get(k) == str(v):
 237                         if volume not in tag_filtered:
 238                             tag_filtered.append(volume)
 239             # return the tag_filtered volumes here, the `filtered` list is no
 240             # longer useable
 241             return tag_filtered
 242
 243         return filtered
 244
 245     def filter(self, vg_name=None, vg_tags=None):
 246         """
 247         Filter out groups on top level attributes like ``vg_name`` or by
 248         ``vg_tags`` where a dict is required. For example, to find a Ceph group
 249         with dmcache as the type, the filter would look like::
 250
 251             vg_tags={'ceph.type': 'dmcache'}
 252
 253         .. warning:: These tags are not documented because they are currently
 254                      unused, but are here to maintain API consistency
 255         """
 256         if not any([vg_name, vg_tags]):
 257             raise TypeError('.filter() requires vg_name or vg_tags (none given)')
 258         # first find the filtered volumes with the values in self
 259         filtered_groups = self._filter(
 260             vg_name=vg_name,
 261             vg_tags=vg_tags
 262         )
 263         # then purge everything
 264         self._purge()
 265         # and add the filtered items
 266         self.extend(filtered_groups)
 267
 268     def get(self, vg_name=None, vg_tags=None):
 269         """
 270         This is a bit expensive, since it will try to filter out all the
 271         matching items in the list, filter them out applying anything that was
 272         added and return the matching item.
 273
 274         This method does *not* alter the list, and it will raise an error if
 275         multiple VGs are matched
 276
 277         It is useful to use ``tags`` when trying to find a specific volume group,
 278         but it can also lead to multiple vgs being found (although unlikely)
 279         """
 280         if not any([vg_name, vg_tags]):
 281             return None
 282         vgs = self._filter(
 283             vg_name=vg_name,
 284             vg_tags=vg_tags
 285         )
 286         if not vgs:
 287             return None
 288         if len(vgs) > 1:
 289             # this is probably never going to happen, but it is here to keep
 290             # the API code consistent
 291             raise MultipleVGsError(vg_name)
 292         return vgs[0]
 293
 294
 295 class Volumes(list):
 296     """
 297     A list of all known (logical) volumes for the current system, with the ability
 298     to filter them via keyword arguments.
 299     """
 300
 301     def __init__(self):
 302         self._populate()
 303
 304     def _populate(self):
 305         # get all the lvs in the current system
 306         for lv_item in get_api_lvs():
 307             self.append(Volume(**lv_item))
 308
 309     def _purge(self):
 310         """
 311         Deplete all the items in the list, used internally only so that we can
 312         dynamically allocate the items when filtering without the concern of
 313         messing up the contents
 314         """
 315         self[:] = []
 316
 317     def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_tags=None):
 318         """
 319         The actual method that filters using a new list. Useful so that other
 320         methods that do not want to alter the contents of the list (e.g.
 321         ``self.find``) can operate safely.
 322         """
 323         filtered = [i for i in self]
 324         if lv_name:
 325             filtered = [i for i in filtered if i.lv_name == lv_name]
 326
 327         if vg_name:
 328             filtered = [i for i in filtered if i.vg_name == vg_name]
 329
 330         if lv_path:
 331             filtered = [i for i in filtered if i.lv_path == lv_path]
 332
 333         # at this point, `filtered` has either all the volumes in self or is an
 334         # actual filtered list if any filters were applied
 335         if lv_tags:
 336             tag_filtered = []
 337             for k, v in lv_tags.items():
 338                 for volume in filtered:
 339                     if volume.tags.get(k) == str(v):
 340                         if volume not in tag_filtered:
 341                             tag_filtered.append(volume)
 342             # return the tag_filtered volumes here, the `filtered` list is no
 343             # longer useable
 344             return tag_filtered
 345
 346         return filtered
 347
 348     def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_tags=None):
 349         """
 350         Filter out volumes on top level attributes like ``lv_name`` or by
 351         ``lv_tags`` where a dict is required. For example, to find a volume
 352         that has an OSD ID of 0, the filter would look like::
 353
 354             lv_tags={'ceph.osd_id': '0'}
 355
 356         """
 357         if not any([lv_name, vg_name, lv_path, lv_tags]):
 358             raise TypeError('.filter() requires lv_name, vg_name, lv_path, or tags (none given)')
 359         # first find the filtered volumes with the values in self
 360         filtered_volumes = self._filter(
 361             lv_name=lv_name,
 362             vg_name=vg_name,
 363             lv_path=lv_path,
 364             lv_tags=lv_tags
 365         )
 366         # then purge everything
 367         self._purge()
 368         # and add the filtered items
 369         self.extend(filtered_volumes)
 370
 371     def get(self, lv_name=None, vg_name=None, lv_path=None, lv_tags=None):
 372         """
 373         This is a bit expensive, since it will try to filter out all the
 374         matching items in the list, filter them out applying anything that was
 375         added and return the matching item.
 376
 377         This method does *not* alter the list, and it will raise an error if
 378         multiple LVs are matched
 379
 380         It is useful to use ``tags`` when trying to find a specific logical volume,
 381         but it can also lead to multiple lvs being found, since a lot of metadata
 382         is shared between lvs of a distinct OSD.
 383         """
 384         if not any([lv_name, vg_name, lv_path, lv_tags]):
 385             return None
 386         lvs = self._filter(
 387             lv_name=lv_name,
 388             vg_name=vg_name,
 389             lv_path=lv_path,
 390             lv_tags=lv_tags
 391         )
 392         if not lvs:
 393             return None
 394         if len(lvs) > 1:
 395             raise MultipleLVsError(lv_name, lv_path)
 396         return lvs[0]
 397
 398
 399 class VolumeGroup(object):
 400     """
 401     Represents an LVM group, with some top-level attributes like ``vg_name``
 402     """
 403
 404     def __init__(self, **kw):
 405         for k, v in kw.items():
 406             setattr(self, k, v)
 407         self.name = kw['vg_name']
 408         self.tags = parse_tags(kw.get('vg_tags', ''))
 409
 410     def __str__(self):
 411         return '<%s>' % self.name
 412
 413     def __repr__(self):
 414         return self.__str__()
 415
 416
 417 class Volume(object):
 418     """
 419     Represents a Logical Volume from LVM, with some top-level attributes like
 420     ``lv_name`` and parsed tags as a dictionary of key/value pairs.
 421     """
 422
 423     def __init__(self, **kw):
 424         for k, v in kw.items():
 425             setattr(self, k, v)
 426         self.lv_api = kw
 427         self.name = kw['lv_name']
 428         self.tags = parse_tags(kw['lv_tags'])
 429
 430     def __str__(self):
 431         return '<%s>' % self.lv_api['lv_path']
 432
 433     def __repr__(self):
 434         return self.__str__()
 435
 436     def set_tags(self, tags):
 437         """
 438         :param tags: A dictionary of tag names and values, like::
 439
 440             {
 441                 "ceph.osd_fsid": "aaa-fff-bbbb",
 442                 "ceph.osd_id": "0"
 443             }
 444
 445         At the end of all modifications, the tags are refreshed to reflect
 446         LVM's most current view.
 447         """
 448         for k, v in tags.items():
 449             self.set_tag(k, v)
 450         # after setting all the tags, refresh them for the current object, use the
 451         # lv_* identifiers to filter because those shouldn't change
 452         lv_object = get_lv(lv_name=self.lv_name, lv_path=self.lv_path)
 453         self.tags = lv_object.tags
 454
 455     def set_tag(self, key, value):
 456         """
 457         Set the key/value pair as an LVM tag. Does not "refresh" the values of
 458         the current object for its tags. Meant to be a "fire and forget" type
 459         of modification.
 460         """
 461         # remove it first if it exists
 462         if self.tags.get(key):
 463             current_value = self.tags[key]
 464             tag = "%s=%s" % (key, current_value)
 465             process.call(['sudo', 'lvchange', '--deltag', tag, self.lv_api['lv_path']])
 466
 467         process.call(
 468             [
 469                 'sudo', 'lvchange',
 470                 '--addtag', '%s=%s' % (key, value), self.lv_path
 471             ]
 472         )