]>
git.proxmox.com Git - ceph.git/blob - ceph/src/ceph-volume/ceph_volume/api/lvm.py
2 API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention
3 that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides
4 set of utilities for interacting with LVM.
6 from ceph_volume
import process
7 from ceph_volume
.exceptions
import MultipleLVsError
, MultipleVGsError
, MultiplePVsError
10 def _output_parser(output
, fields
):
12 Newer versions of LVM allow ``--reportformat=json``, but older versions,
13 like the one included in Xenial do not. LVM has the ability to filter and
14 format its output so we assume the output will be in a format this parser
15 can handle (using ',' as a delimiter)
17 :param fields: A string, possibly using ',' to group many items, as it
18 would be used on the CLI
19 :param output: The CLI output from the LVM call
21 field_items
= fields
.split(',')
24 # clear the leading/trailing whitespace
27 # remove the extra '"' in each field
28 line
= line
.replace('"', '')
30 # prevent moving forward with empty contents
34 # spliting on ';' because that is what the lvm call uses as
36 output_items
= [i
.strip() for i
in line
.split(';')]
37 # map the output to the fiels
39 dict(zip(field_items
, output_items
))
45 def parse_tags(lv_tags
):
47 Return a dictionary mapping of all the tags associated with
48 a Volume from the comma-separated tags coming from the LVM API
52 "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0"
54 For the above example, the expected return value would be::
57 "ceph.osd_fsid": "aaa-fff-bbbb",
64 tags
= lv_tags
.split(',')
65 for tag_assignment
in tags
:
66 key
, value
= tag_assignment
.split('=', 1)
67 tag_mapping
[key
] = value
74 Return the list of group volumes available in the system using flags to
75 include common metadata associated with them
77 Command and sample delimeted output, should look like::
79 $ sudo vgs --noheadings --separator=';' \
80 -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free
81 ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m
82 osd_vg;3;1;0;wz--n-;29.21g;9.21g
85 fields
= 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free'
86 stdout
, stderr
, returncode
= process
.call(
87 ['sudo', 'vgs', '--noheadings', '--separator=";"', '-o', fields
]
89 return _output_parser(stdout
, fields
)
94 Return the list of logical volumes available in the system using flags to include common
95 metadata associated with them
97 Command and delimeted output, should look like::
99 $ sudo lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name
100 ;/dev/ubuntubox-vg/root;root;ubuntubox-vg
101 ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg
104 fields
= 'lv_tags,lv_path,lv_name,vg_name,lv_uuid'
105 stdout
, stderr
, returncode
= process
.call(
106 ['sudo', 'lvs', '--noheadings', '--separator=";"', '-o', fields
]
108 return _output_parser(stdout
, fields
)
113 Return the list of physical volumes configured for lvm and available in the
114 system using flags to include common metadata associated with them like the uuid
116 Command and delimeted output, should look like::
118 $ sudo pvs --noheadings --separator=';' -o pv_name,pv_tags,pv_uuid
120 /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D
123 fields
= 'pv_name,pv_tags,pv_uuid'
125 # note the use of `pvs -a` which will return every physical volume including
126 # ones that have not been initialized as "pv" by LVM
127 stdout
, stderr
, returncode
= process
.call(
128 ['sudo', 'pvs', '-a', '--no-heading', '--separator=";"', '-o', fields
]
131 return _output_parser(stdout
, fields
)
134 def get_lv_from_argument(argument
):
136 Helper proxy function that consumes a possible logical volume passed in from the CLI
137 in the form of `vg/lv`, but with some validation so that an argument that is a full
138 path to a device can be ignored
140 if argument
.startswith('/'):
141 lv
= get_lv(lv_path
=argument
)
144 vg_name
, lv_name
= argument
.split('/')
145 except (ValueError, AttributeError):
147 return get_lv(lv_name
=lv_name
, vg_name
=vg_name
)
150 def get_lv(lv_name
=None, vg_name
=None, lv_path
=None, lv_uuid
=None, lv_tags
=None):
152 Return a matching lv for the current system, requiring ``lv_name``,
153 ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv
156 It is useful to use ``tags`` when trying to find a specific logical volume,
157 but it can also lead to multiple lvs being found, since a lot of metadata
158 is shared between lvs of a distinct OSD.
160 if not any([lv_name
, vg_name
, lv_path
, lv_uuid
, lv_tags
]):
164 lv_name
=lv_name
, vg_name
=vg_name
, lv_path
=lv_path
, lv_uuid
=lv_uuid
,
169 def get_pv(pv_name
=None, pv_uuid
=None, pv_tags
=None):
171 Return a matching pv (physical volume) for the current system, requiring
172 ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one
175 if not any([pv_name
, pv_uuid
, pv_tags
]):
178 return pvs
.get(pv_name
=pv_name
, pv_uuid
=pv_uuid
, pv_tags
=pv_tags
)
181 def create_pv(device
):
183 Create a physical volume from a device, useful when devices need to be later mapped
191 '--yes', # answer yes to any prompts
196 def create_vg(name
, *devices
):
198 Create a Volume Group. Command looks like::
200 vgcreate --force --yes group_name device
202 Once created the volume group is returned as a ``VolumeGroup`` object
209 name
] + list(devices
)
212 vg
= get_vg(vg_name
=name
)
218 Removes a logical volume given it's absolute path.
220 Will return True if the lv is successfully removed or
221 raises a RuntimeError if the removal fails.
223 stdout
, stderr
, returncode
= process
.call(
232 terminal_verbose
=True,
235 raise RuntimeError("Unable to remove %s".format(path
))
239 def create_lv(name
, group
, size
=None, tags
=None):
241 Create a Logical Volume in a Volume Group. Command looks like::
243 lvcreate -L 50G -n gfslv vg0
245 ``name``, ``group``, are required. If ``size`` is provided it must follow
246 lvm's size notation (like 1G, or 20M). Tags are an optional dictionary and is expected to
247 conform to the convention of prefixing them with "ceph." like::
249 {"ceph.block_device": "/dev/ceph/osd-1"}
251 # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations
253 'journal': 'ceph.journal_device',
254 'data': 'ceph.data_device',
255 'block': 'ceph.block_device',
256 'wal': 'ceph.wal_device',
257 'db': 'ceph.db_device',
258 'lockbox': 'ceph.lockbox_device', # XXX might not ever need this lockbox sorcery
269 # create the lv with all the space available, this is needed because the
270 # system call is different for LVM
281 lv
= get_lv(lv_name
=name
, vg_name
=group
)
284 # when creating a distinct type, the caller doesn't know what the path will
285 # be so this function will set it after creation using the mapping
286 path_tag
= type_path_tag
.get(tags
.get('ceph.type'))
289 {path_tag
: lv
.lv_path
}
294 def get_vg(vg_name
=None, vg_tags
=None):
296 Return a matching vg for the current system, requires ``vg_name`` or
297 ``tags``. Raises an error if more than one vg is found.
299 It is useful to use ``tags`` when trying to find a specific volume group,
300 but it can also lead to multiple vgs being found.
302 if not any([vg_name
, vg_tags
]):
305 return vgs
.get(vg_name
=vg_name
, vg_tags
=vg_tags
)
308 class VolumeGroups(list):
310 A list of all known volume groups for the current system, with the ability
311 to filter them via keyword arguments.
318 # get all the vgs in the current system
319 for vg_item
in get_api_vgs():
320 self
.append(VolumeGroup(**vg_item
))
324 Deplete all the items in the list, used internally only so that we can
325 dynamically allocate the items when filtering without the concern of
326 messing up the contents
330 def _filter(self
, vg_name
=None, vg_tags
=None):
332 The actual method that filters using a new list. Useful so that other
333 methods that do not want to alter the contents of the list (e.g.
334 ``self.find``) can operate safely.
336 .. note:: ``vg_tags`` is not yet implemented
338 filtered
= [i
for i
in self
]
340 filtered
= [i
for i
in filtered
if i
.vg_name
== vg_name
]
342 # at this point, `filtered` has either all the volumes in self or is an
343 # actual filtered list if any filters were applied
346 for volume
in filtered
:
347 matches
= all(volume
.tags
.get(k
) == str(v
) for k
, v
in vg_tags
.items())
349 tag_filtered
.append(volume
)
354 def filter(self
, vg_name
=None, vg_tags
=None):
356 Filter out groups on top level attributes like ``vg_name`` or by
357 ``vg_tags`` where a dict is required. For example, to find a Ceph group
358 with dmcache as the type, the filter would look like::
360 vg_tags={'ceph.type': 'dmcache'}
362 .. warning:: These tags are not documented because they are currently
363 unused, but are here to maintain API consistency
365 if not any([vg_name
, vg_tags
]):
366 raise TypeError('.filter() requires vg_name or vg_tags (none given)')
367 # first find the filtered volumes with the values in self
368 filtered_groups
= self
._filter
(
372 # then purge everything
374 # and add the filtered items
375 self
.extend(filtered_groups
)
377 def get(self
, vg_name
=None, vg_tags
=None):
379 This is a bit expensive, since it will try to filter out all the
380 matching items in the list, filter them out applying anything that was
381 added and return the matching item.
383 This method does *not* alter the list, and it will raise an error if
384 multiple VGs are matched
386 It is useful to use ``tags`` when trying to find a specific volume group,
387 but it can also lead to multiple vgs being found (although unlikely)
389 if not any([vg_name
, vg_tags
]):
398 # this is probably never going to happen, but it is here to keep
399 # the API code consistent
400 raise MultipleVGsError(vg_name
)
406 A list of all known (logical) volumes for the current system, with the ability
407 to filter them via keyword arguments.
414 # get all the lvs in the current system
415 for lv_item
in get_api_lvs():
416 self
.append(Volume(**lv_item
))
420 Deplete all the items in the list, used internally only so that we can
421 dynamically allocate the items when filtering without the concern of
422 messing up the contents
426 def _filter(self
, lv_name
=None, vg_name
=None, lv_path
=None, lv_uuid
=None, lv_tags
=None):
428 The actual method that filters using a new list. Useful so that other
429 methods that do not want to alter the contents of the list (e.g.
430 ``self.find``) can operate safely.
432 filtered
= [i
for i
in self
]
434 filtered
= [i
for i
in filtered
if i
.lv_name
== lv_name
]
437 filtered
= [i
for i
in filtered
if i
.vg_name
== vg_name
]
440 filtered
= [i
for i
in filtered
if i
.lv_uuid
== lv_uuid
]
443 filtered
= [i
for i
in filtered
if i
.lv_path
== lv_path
]
445 # at this point, `filtered` has either all the volumes in self or is an
446 # actual filtered list if any filters were applied
449 for volume
in filtered
:
450 # all the tags we got need to match on the volume
451 matches
= all(volume
.tags
.get(k
) == str(v
) for k
, v
in lv_tags
.items())
453 tag_filtered
.append(volume
)
458 def filter(self
, lv_name
=None, vg_name
=None, lv_path
=None, lv_uuid
=None, lv_tags
=None):
460 Filter out volumes on top level attributes like ``lv_name`` or by
461 ``lv_tags`` where a dict is required. For example, to find a volume
462 that has an OSD ID of 0, the filter would look like::
464 lv_tags={'ceph.osd_id': '0'}
467 if not any([lv_name
, vg_name
, lv_path
, lv_uuid
, lv_tags
]):
468 raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)')
469 # first find the filtered volumes with the values in self
470 filtered_volumes
= self
._filter
(
477 # then purge everything
479 # and add the filtered items
480 self
.extend(filtered_volumes
)
482 def get(self
, lv_name
=None, vg_name
=None, lv_path
=None, lv_uuid
=None, lv_tags
=None):
484 This is a bit expensive, since it will try to filter out all the
485 matching items in the list, filter them out applying anything that was
486 added and return the matching item.
488 This method does *not* alter the list, and it will raise an error if
489 multiple LVs are matched
491 It is useful to use ``tags`` when trying to find a specific logical volume,
492 but it can also lead to multiple lvs being found, since a lot of metadata
493 is shared between lvs of a distinct OSD.
495 if not any([lv_name
, vg_name
, lv_path
, lv_uuid
, lv_tags
]):
507 raise MultipleLVsError(lv_name
, lv_path
)
511 class PVolumes(list):
513 A list of all known (physical) volumes for the current system, with the ability
514 to filter them via keyword arguments.
521 # get all the pvs in the current system
522 for pv_item
in get_api_pvs():
523 self
.append(PVolume(**pv_item
))
527 Deplete all the items in the list, used internally only so that we can
528 dynamically allocate the items when filtering without the concern of
529 messing up the contents
533 def _filter(self
, pv_name
=None, pv_uuid
=None, pv_tags
=None):
535 The actual method that filters using a new list. Useful so that other
536 methods that do not want to alter the contents of the list (e.g.
537 ``self.find``) can operate safely.
539 filtered
= [i
for i
in self
]
541 filtered
= [i
for i
in filtered
if i
.pv_name
== pv_name
]
544 filtered
= [i
for i
in filtered
if i
.pv_uuid
== pv_uuid
]
546 # at this point, `filtered` has either all the physical volumes in self
547 # or is an actual filtered list if any filters were applied
550 for pvolume
in filtered
:
551 matches
= all(pvolume
.tags
.get(k
) == str(v
) for k
, v
in pv_tags
.items())
553 tag_filtered
.append(pvolume
)
554 # return the tag_filtered pvolumes here, the `filtered` list is no
560 def filter(self
, pv_name
=None, pv_uuid
=None, pv_tags
=None):
562 Filter out volumes on top level attributes like ``pv_name`` or by
563 ``pv_tags`` where a dict is required. For example, to find a physical volume
564 that has an OSD ID of 0, the filter would look like::
566 pv_tags={'ceph.osd_id': '0'}
569 if not any([pv_name
, pv_uuid
, pv_tags
]):
570 raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags (none given)')
571 # first find the filtered volumes with the values in self
572 filtered_volumes
= self
._filter
(
577 # then purge everything
579 # and add the filtered items
580 self
.extend(filtered_volumes
)
582 def get(self
, pv_name
=None, pv_uuid
=None, pv_tags
=None):
584 This is a bit expensive, since it will try to filter out all the
585 matching items in the list, filter them out applying anything that was
586 added and return the matching item.
588 This method does *not* alter the list, and it will raise an error if
589 multiple pvs are matched
591 It is useful to use ``tags`` when trying to find a specific logical volume,
592 but it can also lead to multiple pvs being found, since a lot of metadata
593 is shared between pvs of a distinct OSD.
595 if not any([pv_name
, pv_uuid
, pv_tags
]):
605 raise MultiplePVsError(pv_name
)
609 class VolumeGroup(object):
611 Represents an LVM group, with some top-level attributes like ``vg_name``
614 def __init__(self
, **kw
):
615 for k
, v
in kw
.items():
617 self
.name
= kw
['vg_name']
618 self
.tags
= parse_tags(kw
.get('vg_tags', ''))
621 return '<%s>' % self
.name
624 return self
.__str
__()
627 class Volume(object):
629 Represents a Logical Volume from LVM, with some top-level attributes like
630 ``lv_name`` and parsed tags as a dictionary of key/value pairs.
633 def __init__(self
, **kw
):
634 for k
, v
in kw
.items():
637 self
.name
= kw
['lv_name']
638 self
.tags
= parse_tags(kw
['lv_tags'])
641 return '<%s>' % self
.lv_api
['lv_path']
644 return self
.__str
__()
648 obj
.update(self
.lv_api
)
649 obj
['tags'] = self
.tags
650 obj
['name'] = self
.name
651 obj
['type'] = self
.tags
['ceph.type']
652 obj
['path'] = self
.lv_path
655 def clear_tags(self
):
657 Removes all tags from the Logical Volume.
659 for k
, v
in self
.tags
.items():
660 tag
= "%s=%s" % (k
, v
)
661 process
.run(['sudo', 'lvchange', '--deltag', tag
, self
.lv_path
])
663 def set_tags(self
, tags
):
665 :param tags: A dictionary of tag names and values, like::
668 "ceph.osd_fsid": "aaa-fff-bbbb",
672 At the end of all modifications, the tags are refreshed to reflect
673 LVM's most current view.
675 for k
, v
in tags
.items():
677 # after setting all the tags, refresh them for the current object, use the
678 # lv_* identifiers to filter because those shouldn't change
679 lv_object
= get_lv(lv_name
=self
.lv_name
, lv_path
=self
.lv_path
)
680 self
.tags
= lv_object
.tags
682 def set_tag(self
, key
, value
):
684 Set the key/value pair as an LVM tag. Does not "refresh" the values of
685 the current object for its tags. Meant to be a "fire and forget" type
688 # remove it first if it exists
689 if self
.tags
.get(key
):
690 current_value
= self
.tags
[key
]
691 tag
= "%s=%s" % (key
, current_value
)
692 process
.call(['sudo', 'lvchange', '--deltag', tag
, self
.lv_api
['lv_path']])
697 '--addtag', '%s=%s' % (key
, value
), self
.lv_path
702 class PVolume(object):
704 Represents a Physical Volume from LVM, with some top-level attributes like
705 ``pv_name`` and parsed tags as a dictionary of key/value pairs.
708 def __init__(self
, **kw
):
709 for k
, v
in kw
.items():
712 self
.name
= kw
['pv_name']
713 self
.tags
= parse_tags(kw
['pv_tags'])
716 return '<%s>' % self
.pv_api
['pv_name']
719 return self
.__str
__()
721 def set_tags(self
, tags
):
723 :param tags: A dictionary of tag names and values, like::
726 "ceph.osd_fsid": "aaa-fff-bbbb",
730 At the end of all modifications, the tags are refreshed to reflect
731 LVM's most current view.
733 for k
, v
in tags
.items():
735 # after setting all the tags, refresh them for the current object, use the
736 # pv_* identifiers to filter because those shouldn't change
737 pv_object
= get_pv(pv_name
=self
.pv_name
, pv_uuid
=self
.pv_uuid
)
738 self
.tags
= pv_object
.tags
740 def set_tag(self
, key
, value
):
742 Set the key/value pair as an LVM tag. Does not "refresh" the values of
743 the current object for its tags. Meant to be a "fire and forget" type
746 **warning**: Altering tags on a PV has to be done ensuring that the
747 device is actually the one intended. ``pv_name`` is *not* a persistent
748 value, only ``pv_uuid`` is. Using ``pv_uuid`` is the best way to make
749 sure the device getting changed is the one needed.
751 # remove it first if it exists
752 if self
.tags
.get(key
):
753 current_value
= self
.tags
[key
]
754 tag
= "%s=%s" % (key
, current_value
)
755 process
.call(['sudo', 'pvchange', '--deltag', tag
, self
.pv_name
])
760 '--addtag', '%s=%s' % (key
, value
), self
.pv_name