]> git.proxmox.com Git - ceph.git/blob - ceph/src/ceph-volume/ceph_volume/api/lvm.py
update sources to 12.2.2
[ceph.git] / ceph / src / ceph-volume / ceph_volume / api / lvm.py
1 """
2 API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention
3 that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides
4 set of utilities for interacting with LVM.
5 """
6 from ceph_volume import process
7 from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError, MultiplePVsError
8
9
10 def _output_parser(output, fields):
11 """
12 Newer versions of LVM allow ``--reportformat=json``, but older versions,
13 like the one included in Xenial do not. LVM has the ability to filter and
14 format its output so we assume the output will be in a format this parser
15 can handle (using ',' as a delimiter)
16
17 :param fields: A string, possibly using ',' to group many items, as it
18 would be used on the CLI
19 :param output: The CLI output from the LVM call
20 """
21 field_items = fields.split(',')
22 report = []
23 for line in output:
24 # clear the leading/trailing whitespace
25 line = line.strip()
26
27 # remove the extra '"' in each field
28 line = line.replace('"', '')
29
30 # prevent moving forward with empty contents
31 if not line:
32 continue
33
34 # spliting on ';' because that is what the lvm call uses as
35 # '--separator'
36 output_items = [i.strip() for i in line.split(';')]
37 # map the output to the fiels
38 report.append(
39 dict(zip(field_items, output_items))
40 )
41
42 return report
43
44
45 def parse_tags(lv_tags):
46 """
47 Return a dictionary mapping of all the tags associated with
48 a Volume from the comma-separated tags coming from the LVM API
49
50 Input look like::
51
52 "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0"
53
54 For the above example, the expected return value would be::
55
56 {
57 "ceph.osd_fsid": "aaa-fff-bbbb",
58 "ceph.osd_id": "0"
59 }
60 """
61 if not lv_tags:
62 return {}
63 tag_mapping = {}
64 tags = lv_tags.split(',')
65 for tag_assignment in tags:
66 key, value = tag_assignment.split('=', 1)
67 tag_mapping[key] = value
68
69 return tag_mapping
70
71
72 def get_api_vgs():
73 """
74 Return the list of group volumes available in the system using flags to
75 include common metadata associated with them
76
77 Command and sample delimeted output, should look like::
78
79 $ sudo vgs --noheadings --separator=';' \
80 -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free
81 ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m
82 osd_vg;3;1;0;wz--n-;29.21g;9.21g
83
84 """
85 fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free'
86 stdout, stderr, returncode = process.call(
87 ['sudo', 'vgs', '--noheadings', '--separator=";"', '-o', fields]
88 )
89 return _output_parser(stdout, fields)
90
91
92 def get_api_lvs():
93 """
94 Return the list of logical volumes available in the system using flags to include common
95 metadata associated with them
96
97 Command and delimeted output, should look like::
98
99 $ sudo lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name
100 ;/dev/ubuntubox-vg/root;root;ubuntubox-vg
101 ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg
102
103 """
104 fields = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid'
105 stdout, stderr, returncode = process.call(
106 ['sudo', 'lvs', '--noheadings', '--separator=";"', '-o', fields]
107 )
108 return _output_parser(stdout, fields)
109
110
111 def get_api_pvs():
112 """
113 Return the list of physical volumes configured for lvm and available in the
114 system using flags to include common metadata associated with them like the uuid
115
116 Command and delimeted output, should look like::
117
118 $ sudo pvs --noheadings --separator=';' -o pv_name,pv_tags,pv_uuid
119 /dev/sda1;;
120 /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D
121
122 """
123 fields = 'pv_name,pv_tags,pv_uuid'
124
125 # note the use of `pvs -a` which will return every physical volume including
126 # ones that have not been initialized as "pv" by LVM
127 stdout, stderr, returncode = process.call(
128 ['sudo', 'pvs', '-a', '--no-heading', '--separator=";"', '-o', fields]
129 )
130
131 return _output_parser(stdout, fields)
132
133
134 def get_lv_from_argument(argument):
135 """
136 Helper proxy function that consumes a possible logical volume passed in from the CLI
137 in the form of `vg/lv`, but with some validation so that an argument that is a full
138 path to a device can be ignored
139 """
140 if argument.startswith('/'):
141 lv = get_lv(lv_path=argument)
142 return lv
143 try:
144 vg_name, lv_name = argument.split('/')
145 except (ValueError, AttributeError):
146 return None
147 return get_lv(lv_name=lv_name, vg_name=vg_name)
148
149
150 def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
151 """
152 Return a matching lv for the current system, requiring ``lv_name``,
153 ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv
154 is found.
155
156 It is useful to use ``tags`` when trying to find a specific logical volume,
157 but it can also lead to multiple lvs being found, since a lot of metadata
158 is shared between lvs of a distinct OSD.
159 """
160 if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
161 return None
162 lvs = Volumes()
163 return lvs.get(
164 lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_uuid=lv_uuid,
165 lv_tags=lv_tags
166 )
167
168
169 def get_pv(pv_name=None, pv_uuid=None, pv_tags=None):
170 """
171 Return a matching pv (physical volume) for the current system, requiring
172 ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one
173 pv is found.
174 """
175 if not any([pv_name, pv_uuid, pv_tags]):
176 return None
177 pvs = PVolumes()
178 return pvs.get(pv_name=pv_name, pv_uuid=pv_uuid, pv_tags=pv_tags)
179
180
181 def create_pv(device):
182 """
183 Create a physical volume from a device, useful when devices need to be later mapped
184 to journals.
185 """
186 process.run([
187 'sudo',
188 'pvcreate',
189 '-v', # verbose
190 '-f', # force it
191 '--yes', # answer yes to any prompts
192 device
193 ])
194
195
196 def create_vg(name, *devices):
197 """
198 Create a Volume Group. Command looks like::
199
200 vgcreate --force --yes group_name device
201
202 Once created the volume group is returned as a ``VolumeGroup`` object
203 """
204 process.run([
205 'sudo',
206 'vgcreate',
207 '--force',
208 '--yes',
209 name] + list(devices)
210 )
211
212 vg = get_vg(vg_name=name)
213 return vg
214
215
216 def remove_lv(path):
217 """
218 Removes a logical volume given it's absolute path.
219
220 Will return True if the lv is successfully removed or
221 raises a RuntimeError if the removal fails.
222 """
223 stdout, stderr, returncode = process.call(
224 [
225 'sudo',
226 'lvremove',
227 '-v', # verbose
228 '-f', # force it
229 path
230 ],
231 show_command=True,
232 terminal_verbose=True,
233 )
234 if returncode != 0:
235 raise RuntimeError("Unable to remove %s".format(path))
236 return True
237
238
239 def create_lv(name, group, size=None, tags=None):
240 """
241 Create a Logical Volume in a Volume Group. Command looks like::
242
243 lvcreate -L 50G -n gfslv vg0
244
245 ``name``, ``group``, are required. If ``size`` is provided it must follow
246 lvm's size notation (like 1G, or 20M). Tags are an optional dictionary and is expected to
247 conform to the convention of prefixing them with "ceph." like::
248
249 {"ceph.block_device": "/dev/ceph/osd-1"}
250 """
251 # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations
252 type_path_tag = {
253 'journal': 'ceph.journal_device',
254 'data': 'ceph.data_device',
255 'block': 'ceph.block_device',
256 'wal': 'ceph.wal_device',
257 'db': 'ceph.db_device',
258 'lockbox': 'ceph.lockbox_device', # XXX might not ever need this lockbox sorcery
259 }
260 if size:
261 process.run([
262 'sudo',
263 'lvcreate',
264 '--yes',
265 '-L',
266 '%s' % size,
267 '-n', name, group
268 ])
269 # create the lv with all the space available, this is needed because the
270 # system call is different for LVM
271 else:
272 process.run([
273 'sudo',
274 'lvcreate',
275 '--yes',
276 '-l',
277 '100%FREE',
278 '-n', name, group
279 ])
280
281 lv = get_lv(lv_name=name, vg_name=group)
282 lv.set_tags(tags)
283
284 # when creating a distinct type, the caller doesn't know what the path will
285 # be so this function will set it after creation using the mapping
286 path_tag = type_path_tag.get(tags.get('ceph.type'))
287 if path_tag:
288 lv.set_tags(
289 {path_tag: lv.lv_path}
290 )
291 return lv
292
293
294 def get_vg(vg_name=None, vg_tags=None):
295 """
296 Return a matching vg for the current system, requires ``vg_name`` or
297 ``tags``. Raises an error if more than one vg is found.
298
299 It is useful to use ``tags`` when trying to find a specific volume group,
300 but it can also lead to multiple vgs being found.
301 """
302 if not any([vg_name, vg_tags]):
303 return None
304 vgs = VolumeGroups()
305 return vgs.get(vg_name=vg_name, vg_tags=vg_tags)
306
307
308 class VolumeGroups(list):
309 """
310 A list of all known volume groups for the current system, with the ability
311 to filter them via keyword arguments.
312 """
313
314 def __init__(self):
315 self._populate()
316
317 def _populate(self):
318 # get all the vgs in the current system
319 for vg_item in get_api_vgs():
320 self.append(VolumeGroup(**vg_item))
321
322 def _purge(self):
323 """
324 Deplete all the items in the list, used internally only so that we can
325 dynamically allocate the items when filtering without the concern of
326 messing up the contents
327 """
328 self[:] = []
329
330 def _filter(self, vg_name=None, vg_tags=None):
331 """
332 The actual method that filters using a new list. Useful so that other
333 methods that do not want to alter the contents of the list (e.g.
334 ``self.find``) can operate safely.
335
336 .. note:: ``vg_tags`` is not yet implemented
337 """
338 filtered = [i for i in self]
339 if vg_name:
340 filtered = [i for i in filtered if i.vg_name == vg_name]
341
342 # at this point, `filtered` has either all the volumes in self or is an
343 # actual filtered list if any filters were applied
344 if vg_tags:
345 tag_filtered = []
346 for volume in filtered:
347 matches = all(volume.tags.get(k) == str(v) for k, v in vg_tags.items())
348 if matches:
349 tag_filtered.append(volume)
350 return tag_filtered
351
352 return filtered
353
354 def filter(self, vg_name=None, vg_tags=None):
355 """
356 Filter out groups on top level attributes like ``vg_name`` or by
357 ``vg_tags`` where a dict is required. For example, to find a Ceph group
358 with dmcache as the type, the filter would look like::
359
360 vg_tags={'ceph.type': 'dmcache'}
361
362 .. warning:: These tags are not documented because they are currently
363 unused, but are here to maintain API consistency
364 """
365 if not any([vg_name, vg_tags]):
366 raise TypeError('.filter() requires vg_name or vg_tags (none given)')
367 # first find the filtered volumes with the values in self
368 filtered_groups = self._filter(
369 vg_name=vg_name,
370 vg_tags=vg_tags
371 )
372 # then purge everything
373 self._purge()
374 # and add the filtered items
375 self.extend(filtered_groups)
376
377 def get(self, vg_name=None, vg_tags=None):
378 """
379 This is a bit expensive, since it will try to filter out all the
380 matching items in the list, filter them out applying anything that was
381 added and return the matching item.
382
383 This method does *not* alter the list, and it will raise an error if
384 multiple VGs are matched
385
386 It is useful to use ``tags`` when trying to find a specific volume group,
387 but it can also lead to multiple vgs being found (although unlikely)
388 """
389 if not any([vg_name, vg_tags]):
390 return None
391 vgs = self._filter(
392 vg_name=vg_name,
393 vg_tags=vg_tags
394 )
395 if not vgs:
396 return None
397 if len(vgs) > 1:
398 # this is probably never going to happen, but it is here to keep
399 # the API code consistent
400 raise MultipleVGsError(vg_name)
401 return vgs[0]
402
403
404 class Volumes(list):
405 """
406 A list of all known (logical) volumes for the current system, with the ability
407 to filter them via keyword arguments.
408 """
409
410 def __init__(self):
411 self._populate()
412
413 def _populate(self):
414 # get all the lvs in the current system
415 for lv_item in get_api_lvs():
416 self.append(Volume(**lv_item))
417
418 def _purge(self):
419 """
420 Deplete all the items in the list, used internally only so that we can
421 dynamically allocate the items when filtering without the concern of
422 messing up the contents
423 """
424 self[:] = []
425
426 def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
427 """
428 The actual method that filters using a new list. Useful so that other
429 methods that do not want to alter the contents of the list (e.g.
430 ``self.find``) can operate safely.
431 """
432 filtered = [i for i in self]
433 if lv_name:
434 filtered = [i for i in filtered if i.lv_name == lv_name]
435
436 if vg_name:
437 filtered = [i for i in filtered if i.vg_name == vg_name]
438
439 if lv_uuid:
440 filtered = [i for i in filtered if i.lv_uuid == lv_uuid]
441
442 if lv_path:
443 filtered = [i for i in filtered if i.lv_path == lv_path]
444
445 # at this point, `filtered` has either all the volumes in self or is an
446 # actual filtered list if any filters were applied
447 if lv_tags:
448 tag_filtered = []
449 for volume in filtered:
450 # all the tags we got need to match on the volume
451 matches = all(volume.tags.get(k) == str(v) for k, v in lv_tags.items())
452 if matches:
453 tag_filtered.append(volume)
454 return tag_filtered
455
456 return filtered
457
458 def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
459 """
460 Filter out volumes on top level attributes like ``lv_name`` or by
461 ``lv_tags`` where a dict is required. For example, to find a volume
462 that has an OSD ID of 0, the filter would look like::
463
464 lv_tags={'ceph.osd_id': '0'}
465
466 """
467 if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
468 raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)')
469 # first find the filtered volumes with the values in self
470 filtered_volumes = self._filter(
471 lv_name=lv_name,
472 vg_name=vg_name,
473 lv_path=lv_path,
474 lv_uuid=lv_uuid,
475 lv_tags=lv_tags
476 )
477 # then purge everything
478 self._purge()
479 # and add the filtered items
480 self.extend(filtered_volumes)
481
482 def get(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None):
483 """
484 This is a bit expensive, since it will try to filter out all the
485 matching items in the list, filter them out applying anything that was
486 added and return the matching item.
487
488 This method does *not* alter the list, and it will raise an error if
489 multiple LVs are matched
490
491 It is useful to use ``tags`` when trying to find a specific logical volume,
492 but it can also lead to multiple lvs being found, since a lot of metadata
493 is shared between lvs of a distinct OSD.
494 """
495 if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]):
496 return None
497 lvs = self._filter(
498 lv_name=lv_name,
499 vg_name=vg_name,
500 lv_path=lv_path,
501 lv_uuid=lv_uuid,
502 lv_tags=lv_tags
503 )
504 if not lvs:
505 return None
506 if len(lvs) > 1:
507 raise MultipleLVsError(lv_name, lv_path)
508 return lvs[0]
509
510
511 class PVolumes(list):
512 """
513 A list of all known (physical) volumes for the current system, with the ability
514 to filter them via keyword arguments.
515 """
516
517 def __init__(self):
518 self._populate()
519
520 def _populate(self):
521 # get all the pvs in the current system
522 for pv_item in get_api_pvs():
523 self.append(PVolume(**pv_item))
524
525 def _purge(self):
526 """
527 Deplete all the items in the list, used internally only so that we can
528 dynamically allocate the items when filtering without the concern of
529 messing up the contents
530 """
531 self[:] = []
532
533 def _filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
534 """
535 The actual method that filters using a new list. Useful so that other
536 methods that do not want to alter the contents of the list (e.g.
537 ``self.find``) can operate safely.
538 """
539 filtered = [i for i in self]
540 if pv_name:
541 filtered = [i for i in filtered if i.pv_name == pv_name]
542
543 if pv_uuid:
544 filtered = [i for i in filtered if i.pv_uuid == pv_uuid]
545
546 # at this point, `filtered` has either all the physical volumes in self
547 # or is an actual filtered list if any filters were applied
548 if pv_tags:
549 tag_filtered = []
550 for pvolume in filtered:
551 matches = all(pvolume.tags.get(k) == str(v) for k, v in pv_tags.items())
552 if matches:
553 tag_filtered.append(pvolume)
554 # return the tag_filtered pvolumes here, the `filtered` list is no
555 # longer useable
556 return tag_filtered
557
558 return filtered
559
560 def filter(self, pv_name=None, pv_uuid=None, pv_tags=None):
561 """
562 Filter out volumes on top level attributes like ``pv_name`` or by
563 ``pv_tags`` where a dict is required. For example, to find a physical volume
564 that has an OSD ID of 0, the filter would look like::
565
566 pv_tags={'ceph.osd_id': '0'}
567
568 """
569 if not any([pv_name, pv_uuid, pv_tags]):
570 raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags (none given)')
571 # first find the filtered volumes with the values in self
572 filtered_volumes = self._filter(
573 pv_name=pv_name,
574 pv_uuid=pv_uuid,
575 pv_tags=pv_tags
576 )
577 # then purge everything
578 self._purge()
579 # and add the filtered items
580 self.extend(filtered_volumes)
581
582 def get(self, pv_name=None, pv_uuid=None, pv_tags=None):
583 """
584 This is a bit expensive, since it will try to filter out all the
585 matching items in the list, filter them out applying anything that was
586 added and return the matching item.
587
588 This method does *not* alter the list, and it will raise an error if
589 multiple pvs are matched
590
591 It is useful to use ``tags`` when trying to find a specific logical volume,
592 but it can also lead to multiple pvs being found, since a lot of metadata
593 is shared between pvs of a distinct OSD.
594 """
595 if not any([pv_name, pv_uuid, pv_tags]):
596 return None
597 pvs = self._filter(
598 pv_name=pv_name,
599 pv_uuid=pv_uuid,
600 pv_tags=pv_tags
601 )
602 if not pvs:
603 return None
604 if len(pvs) > 1:
605 raise MultiplePVsError(pv_name)
606 return pvs[0]
607
608
609 class VolumeGroup(object):
610 """
611 Represents an LVM group, with some top-level attributes like ``vg_name``
612 """
613
614 def __init__(self, **kw):
615 for k, v in kw.items():
616 setattr(self, k, v)
617 self.name = kw['vg_name']
618 self.tags = parse_tags(kw.get('vg_tags', ''))
619
620 def __str__(self):
621 return '<%s>' % self.name
622
623 def __repr__(self):
624 return self.__str__()
625
626
627 class Volume(object):
628 """
629 Represents a Logical Volume from LVM, with some top-level attributes like
630 ``lv_name`` and parsed tags as a dictionary of key/value pairs.
631 """
632
633 def __init__(self, **kw):
634 for k, v in kw.items():
635 setattr(self, k, v)
636 self.lv_api = kw
637 self.name = kw['lv_name']
638 self.tags = parse_tags(kw['lv_tags'])
639
640 def __str__(self):
641 return '<%s>' % self.lv_api['lv_path']
642
643 def __repr__(self):
644 return self.__str__()
645
646 def as_dict(self):
647 obj = {}
648 obj.update(self.lv_api)
649 obj['tags'] = self.tags
650 obj['name'] = self.name
651 obj['type'] = self.tags['ceph.type']
652 obj['path'] = self.lv_path
653 return obj
654
655 def clear_tags(self):
656 """
657 Removes all tags from the Logical Volume.
658 """
659 for k, v in self.tags.items():
660 tag = "%s=%s" % (k, v)
661 process.run(['sudo', 'lvchange', '--deltag', tag, self.lv_path])
662
663 def set_tags(self, tags):
664 """
665 :param tags: A dictionary of tag names and values, like::
666
667 {
668 "ceph.osd_fsid": "aaa-fff-bbbb",
669 "ceph.osd_id": "0"
670 }
671
672 At the end of all modifications, the tags are refreshed to reflect
673 LVM's most current view.
674 """
675 for k, v in tags.items():
676 self.set_tag(k, v)
677 # after setting all the tags, refresh them for the current object, use the
678 # lv_* identifiers to filter because those shouldn't change
679 lv_object = get_lv(lv_name=self.lv_name, lv_path=self.lv_path)
680 self.tags = lv_object.tags
681
682 def set_tag(self, key, value):
683 """
684 Set the key/value pair as an LVM tag. Does not "refresh" the values of
685 the current object for its tags. Meant to be a "fire and forget" type
686 of modification.
687 """
688 # remove it first if it exists
689 if self.tags.get(key):
690 current_value = self.tags[key]
691 tag = "%s=%s" % (key, current_value)
692 process.call(['sudo', 'lvchange', '--deltag', tag, self.lv_api['lv_path']])
693
694 process.call(
695 [
696 'sudo', 'lvchange',
697 '--addtag', '%s=%s' % (key, value), self.lv_path
698 ]
699 )
700
701
702 class PVolume(object):
703 """
704 Represents a Physical Volume from LVM, with some top-level attributes like
705 ``pv_name`` and parsed tags as a dictionary of key/value pairs.
706 """
707
708 def __init__(self, **kw):
709 for k, v in kw.items():
710 setattr(self, k, v)
711 self.pv_api = kw
712 self.name = kw['pv_name']
713 self.tags = parse_tags(kw['pv_tags'])
714
715 def __str__(self):
716 return '<%s>' % self.pv_api['pv_name']
717
718 def __repr__(self):
719 return self.__str__()
720
721 def set_tags(self, tags):
722 """
723 :param tags: A dictionary of tag names and values, like::
724
725 {
726 "ceph.osd_fsid": "aaa-fff-bbbb",
727 "ceph.osd_id": "0"
728 }
729
730 At the end of all modifications, the tags are refreshed to reflect
731 LVM's most current view.
732 """
733 for k, v in tags.items():
734 self.set_tag(k, v)
735 # after setting all the tags, refresh them for the current object, use the
736 # pv_* identifiers to filter because those shouldn't change
737 pv_object = get_pv(pv_name=self.pv_name, pv_uuid=self.pv_uuid)
738 self.tags = pv_object.tags
739
740 def set_tag(self, key, value):
741 """
742 Set the key/value pair as an LVM tag. Does not "refresh" the values of
743 the current object for its tags. Meant to be a "fire and forget" type
744 of modification.
745
746 **warning**: Altering tags on a PV has to be done ensuring that the
747 device is actually the one intended. ``pv_name`` is *not* a persistent
748 value, only ``pv_uuid`` is. Using ``pv_uuid`` is the best way to make
749 sure the device getting changed is the one needed.
750 """
751 # remove it first if it exists
752 if self.tags.get(key):
753 current_value = self.tags[key]
754 tag = "%s=%s" % (key, current_value)
755 process.call(['sudo', 'pvchange', '--deltag', tag, self.pv_name])
756
757 process.call(
758 [
759 'sudo', 'pvchange',
760 '--addtag', '%s=%s' % (key, value), self.pv_name
761 ]
762 )