]>
Commit | Line | Data |
---|---|---|
d2e6a577 FG |
1 | """ |
2 | API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention | |
3 | that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides | |
4 | set of utilities for interacting with LVM. | |
5 | """ | |
94b18763 FG |
6 | import logging |
7 | import os | |
1adf2230 | 8 | import uuid |
e306af50 | 9 | from itertools import repeat |
1adf2230 AA |
10 | from math import floor |
11 | from ceph_volume import process, util | |
12 | from ceph_volume.exceptions import ( | |
13 | MultipleLVsError, MultipleVGsError, | |
14 | MultiplePVsError, SizeAllocationError | |
15 | ) | |
d2e6a577 | 16 | |
94b18763 FG |
17 | logger = logging.getLogger(__name__) |
18 | ||
d2e6a577 | 19 | |
b5b8bbf5 FG |
20 | def _output_parser(output, fields): |
21 | """ | |
22 | Newer versions of LVM allow ``--reportformat=json``, but older versions, | |
23 | like the one included in Xenial do not. LVM has the ability to filter and | |
24 | format its output so we assume the output will be in a format this parser | |
92f5a8d4 | 25 | can handle (using ';' as a delimiter) |
b5b8bbf5 FG |
26 | |
27 | :param fields: A string, possibly using ',' to group many items, as it | |
28 | would be used on the CLI | |
29 | :param output: The CLI output from the LVM call | |
30 | """ | |
31 | field_items = fields.split(',') | |
32 | report = [] | |
33 | for line in output: | |
34 | # clear the leading/trailing whitespace | |
35 | line = line.strip() | |
36 | ||
37 | # remove the extra '"' in each field | |
38 | line = line.replace('"', '') | |
39 | ||
40 | # prevent moving forward with empty contents | |
41 | if not line: | |
42 | continue | |
43 | ||
11fdf7f2 | 44 | # splitting on ';' because that is what the lvm call uses as |
b5b8bbf5 FG |
45 | # '--separator' |
46 | output_items = [i.strip() for i in line.split(';')] | |
92f5a8d4 | 47 | # map the output to the fields |
b5b8bbf5 FG |
48 | report.append( |
49 | dict(zip(field_items, output_items)) | |
50 | ) | |
51 | ||
52 | return report | |
53 | ||
54 | ||
1adf2230 AA |
55 | def _splitname_parser(line): |
56 | """ | |
57 | Parses the output from ``dmsetup splitname``, that should contain prefixes | |
58 | (--nameprefixes) and set the separator to ";" | |
59 | ||
60 | Output for /dev/mapper/vg-lv will usually look like:: | |
61 | ||
62 | DM_VG_NAME='/dev/mapper/vg';DM_LV_NAME='lv';DM_LV_LAYER='' | |
63 | ||
64 | ||
65 | The ``VG_NAME`` will usually not be what other callers need (e.g. just 'vg' | |
66 | in the example), so this utility will split ``/dev/mapper/`` out, so that | |
67 | the actual volume group name is kept | |
68 | ||
69 | :returns: dictionary with stripped prefixes | |
70 | """ | |
1adf2230 | 71 | parsed = {} |
81eedcae TL |
72 | try: |
73 | parts = line[0].split(';') | |
74 | except IndexError: | |
75 | logger.exception('Unable to parse mapper device: %s', line) | |
76 | return parsed | |
77 | ||
1adf2230 AA |
78 | for part in parts: |
79 | part = part.replace("'", '') | |
80 | key, value = part.split('=') | |
81 | if 'DM_VG_NAME' in key: | |
82 | value = value.split('/dev/mapper/')[-1] | |
83 | key = key.split('DM_')[-1] | |
84 | parsed[key] = value | |
85 | ||
86 | return parsed | |
87 | ||
88 | ||
89 | def sizing(device_size, parts=None, size=None): | |
90 | """ | |
91 | Calculate proper sizing to fully utilize the volume group in the most | |
92 | efficient way possible. To prevent situations where LVM might accept | |
93 | a percentage that is beyond the vg's capabilities, it will refuse with | |
94 | an error when requesting a larger-than-possible parameter, in addition | |
95 | to rounding down calculations. | |
96 | ||
97 | A dictionary with different sizing parameters is returned, to make it | |
98 | easier for others to choose what they need in order to create logical | |
99 | volumes:: | |
100 | ||
101 | >>> sizing(100, parts=2) | |
102 | >>> {'parts': 2, 'percentages': 50, 'sizes': 50} | |
103 | ||
104 | """ | |
105 | if parts is not None and size is not None: | |
106 | raise ValueError( | |
107 | "Cannot process sizing with both parts (%s) and size (%s)" % (parts, size) | |
108 | ) | |
109 | ||
110 | if size and size > device_size: | |
111 | raise SizeAllocationError(size, device_size) | |
112 | ||
113 | def get_percentage(parts): | |
114 | return int(floor(100 / float(parts))) | |
115 | ||
116 | if parts is not None: | |
117 | # Prevent parts being 0, falling back to 1 (100% usage) | |
118 | parts = parts or 1 | |
119 | percentages = get_percentage(parts) | |
120 | ||
121 | if size: | |
122 | parts = int(device_size / size) or 1 | |
123 | percentages = get_percentage(parts) | |
124 | ||
125 | sizes = device_size / parts if parts else int(floor(device_size)) | |
126 | ||
127 | return { | |
128 | 'parts': parts, | |
129 | 'percentages': percentages, | |
92f5a8d4 | 130 | 'sizes': int(sizes/1024/1024/1024), |
1adf2230 AA |
131 | } |
132 | ||
133 | ||
d2e6a577 FG |
134 | def parse_tags(lv_tags): |
135 | """ | |
136 | Return a dictionary mapping of all the tags associated with | |
137 | a Volume from the comma-separated tags coming from the LVM API | |
138 | ||
139 | Input look like:: | |
140 | ||
141 | "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0" | |
142 | ||
143 | For the above example, the expected return value would be:: | |
144 | ||
145 | { | |
146 | "ceph.osd_fsid": "aaa-fff-bbbb", | |
147 | "ceph.osd_id": "0" | |
148 | } | |
149 | """ | |
150 | if not lv_tags: | |
151 | return {} | |
152 | tag_mapping = {} | |
153 | tags = lv_tags.split(',') | |
154 | for tag_assignment in tags: | |
b32b8144 FG |
155 | if not tag_assignment.startswith('ceph.'): |
156 | continue | |
d2e6a577 FG |
157 | key, value = tag_assignment.split('=', 1) |
158 | tag_mapping[key] = value | |
159 | ||
160 | return tag_mapping | |
161 | ||
162 | ||
94b18763 FG |
163 | def _vdo_parents(devices): |
164 | """ | |
165 | It is possible we didn't get a logical volume, or a mapper path, but | |
166 | a device like /dev/sda2, to resolve this, we must look at all the slaves of | |
167 | every single device in /sys/block and if any of those devices is related to | |
168 | VDO devices, then we can add the parent | |
169 | """ | |
170 | parent_devices = [] | |
171 | for parent in os.listdir('/sys/block'): | |
172 | for slave in os.listdir('/sys/block/%s/slaves' % parent): | |
173 | if slave in devices: | |
174 | parent_devices.append('/dev/%s' % parent) | |
175 | parent_devices.append(parent) | |
176 | return parent_devices | |
177 | ||
178 | ||
179 | def _vdo_slaves(vdo_names): | |
180 | """ | |
181 | find all the slaves associated with each vdo name (from realpath) by going | |
182 | into /sys/block/<realpath>/slaves | |
183 | """ | |
184 | devices = [] | |
185 | for vdo_name in vdo_names: | |
186 | mapper_path = '/dev/mapper/%s' % vdo_name | |
187 | if not os.path.exists(mapper_path): | |
188 | continue | |
189 | # resolve the realpath and realname of the vdo mapper | |
190 | vdo_realpath = os.path.realpath(mapper_path) | |
191 | vdo_realname = vdo_realpath.split('/')[-1] | |
192 | slaves_path = '/sys/block/%s/slaves' % vdo_realname | |
193 | if not os.path.exists(slaves_path): | |
194 | continue | |
195 | devices.append(vdo_realpath) | |
196 | devices.append(mapper_path) | |
197 | devices.append(vdo_realname) | |
198 | for slave in os.listdir(slaves_path): | |
199 | devices.append('/dev/%s' % slave) | |
200 | devices.append(slave) | |
201 | return devices | |
202 | ||
203 | ||
204 | def _is_vdo(path): | |
205 | """ | |
206 | A VDO device can be composed from many different devices, go through each | |
207 | one of those devices and its slaves (if any) and correlate them back to | |
208 | /dev/mapper and their realpaths, and then check if they appear as part of | |
209 | /sys/kvdo/<name>/statistics | |
210 | ||
211 | From the realpath of a logical volume, determine if it is a VDO device or | |
212 | not, by correlating it to the presence of the name in | |
213 | /sys/kvdo/<name>/statistics and all the previously captured devices | |
214 | """ | |
215 | if not os.path.isdir('/sys/kvdo'): | |
216 | return False | |
217 | realpath = os.path.realpath(path) | |
218 | realpath_name = realpath.split('/')[-1] | |
219 | devices = [] | |
220 | vdo_names = set() | |
221 | # get all the vdo names | |
222 | for dirname in os.listdir('/sys/kvdo/'): | |
223 | if os.path.isdir('/sys/kvdo/%s/statistics' % dirname): | |
224 | vdo_names.add(dirname) | |
225 | ||
226 | # find all the slaves associated with each vdo name (from realpath) by | |
227 | # going into /sys/block/<realpath>/slaves | |
228 | devices.extend(_vdo_slaves(vdo_names)) | |
229 | ||
230 | # Find all possible parents, looking into slaves that are related to VDO | |
231 | devices.extend(_vdo_parents(devices)) | |
232 | ||
233 | return any([ | |
234 | path in devices, | |
235 | realpath in devices, | |
236 | realpath_name in devices]) | |
237 | ||
238 | ||
239 | def is_vdo(path): | |
240 | """ | |
241 | Detect if a path is backed by VDO, proxying the actual call to _is_vdo so | |
242 | that we can prevent an exception breaking OSD creation. If an exception is | |
243 | raised, it will get captured and logged to file, while returning | |
244 | a ``False``. | |
245 | """ | |
246 | try: | |
247 | if _is_vdo(path): | |
248 | return '1' | |
249 | return '0' | |
250 | except Exception: | |
251 | logger.exception('Unable to properly detect device as VDO: %s', path) | |
252 | return '0' | |
253 | ||
254 | ||
1adf2230 AA |
255 | def dmsetup_splitname(dev): |
256 | """ | |
257 | Run ``dmsetup splitname`` and parse the results. | |
258 | ||
259 | .. warning:: This call does not ensure that the device is correct or that | |
260 | it exists. ``dmsetup`` will happily take a non existing path and still | |
261 | return a 0 exit status. | |
262 | """ | |
263 | command = [ | |
264 | 'dmsetup', 'splitname', '--noheadings', | |
265 | "--separator=';'", '--nameprefixes', dev | |
266 | ] | |
267 | out, err, rc = process.call(command) | |
268 | return _splitname_parser(out) | |
269 | ||
270 | ||
92f5a8d4 TL |
271 | def is_ceph_device(lv): |
272 | try: | |
273 | lv.tags['ceph.osd_id'] | |
274 | except (KeyError, AttributeError): | |
275 | logger.warning('device is not part of ceph: %s', lv) | |
276 | return False | |
277 | ||
278 | if lv.tags['ceph.osd_id'] == 'null': | |
279 | return False | |
280 | else: | |
281 | return True | |
282 | ||
283 | ||
eafe8130 TL |
284 | #################################### |
285 | # | |
286 | # Code for LVM Physical Volumes | |
287 | # | |
288 | ################################ | |
d2e6a577 | 289 | |
92f5a8d4 | 290 | PV_FIELDS = 'pv_name,pv_tags,pv_uuid,vg_name,lv_uuid' |
d2e6a577 | 291 | |
181888fb FG |
292 | def get_api_pvs(): |
293 | """ | |
294 | Return the list of physical volumes configured for lvm and available in the | |
295 | system using flags to include common metadata associated with them like the uuid | |
296 | ||
b32b8144 FG |
297 | This will only return physical volumes set up to work with LVM. |
298 | ||
94b18763 | 299 | Command and delimited output should look like:: |
181888fb | 300 | |
94b18763 | 301 | $ pvs --noheadings --readonly --separator=';' -o pv_name,pv_tags,pv_uuid |
181888fb FG |
302 | /dev/sda1;; |
303 | /dev/sdv;;07A4F654-4162-4600-8EB3-88D1E42F368D | |
304 | ||
305 | """ | |
181888fb | 306 | stdout, stderr, returncode = process.call( |
92f5a8d4 TL |
307 | ['pvs', '--no-heading', '--readonly', '--separator=";"', '-o', |
308 | PV_FIELDS], | |
91327a77 | 309 | verbose_on_failure=False |
181888fb FG |
310 | ) |
311 | ||
92f5a8d4 | 312 | return _output_parser(stdout, PV_FIELDS) |
181888fb FG |
313 | |
314 | ||
eafe8130 | 315 | class PVolume(object): |
3efd9988 | 316 | """ |
eafe8130 TL |
317 | Represents a Physical Volume from LVM, with some top-level attributes like |
318 | ``pv_name`` and parsed tags as a dictionary of key/value pairs. | |
3efd9988 | 319 | """ |
3efd9988 | 320 | |
eafe8130 TL |
321 | def __init__(self, **kw): |
322 | for k, v in kw.items(): | |
323 | setattr(self, k, v) | |
324 | self.pv_api = kw | |
325 | self.name = kw['pv_name'] | |
326 | self.tags = parse_tags(kw['pv_tags']) | |
3efd9988 | 327 | |
eafe8130 TL |
328 | def __str__(self): |
329 | return '<%s>' % self.pv_api['pv_name'] | |
d2e6a577 | 330 | |
eafe8130 TL |
331 | def __repr__(self): |
332 | return self.__str__() | |
333 | ||
334 | def set_tags(self, tags): | |
335 | """ | |
336 | :param tags: A dictionary of tag names and values, like:: | |
337 | ||
338 | { | |
339 | "ceph.osd_fsid": "aaa-fff-bbbb", | |
340 | "ceph.osd_id": "0" | |
341 | } | |
342 | ||
343 | At the end of all modifications, the tags are refreshed to reflect | |
344 | LVM's most current view. | |
345 | """ | |
346 | for k, v in tags.items(): | |
347 | self.set_tag(k, v) | |
348 | # after setting all the tags, refresh them for the current object, use the | |
349 | # pv_* identifiers to filter because those shouldn't change | |
350 | pv_object = get_pv(pv_name=self.pv_name, pv_uuid=self.pv_uuid) | |
351 | self.tags = pv_object.tags | |
352 | ||
353 | def set_tag(self, key, value): | |
354 | """ | |
355 | Set the key/value pair as an LVM tag. Does not "refresh" the values of | |
356 | the current object for its tags. Meant to be a "fire and forget" type | |
357 | of modification. | |
358 | ||
359 | **warning**: Altering tags on a PV has to be done ensuring that the | |
360 | device is actually the one intended. ``pv_name`` is *not* a persistent | |
361 | value, only ``pv_uuid`` is. Using ``pv_uuid`` is the best way to make | |
362 | sure the device getting changed is the one needed. | |
363 | """ | |
364 | # remove it first if it exists | |
365 | if self.tags.get(key): | |
366 | current_value = self.tags[key] | |
367 | tag = "%s=%s" % (key, current_value) | |
368 | process.call(['pvchange', '--deltag', tag, self.pv_name]) | |
369 | ||
370 | process.call( | |
371 | [ | |
372 | 'pvchange', | |
373 | '--addtag', '%s=%s' % (key, value), self.pv_name | |
374 | ] | |
375 | ) | |
181888fb FG |
376 | |
377 | ||
eafe8130 | 378 | class PVolumes(list): |
181888fb | 379 | """ |
eafe8130 TL |
380 | A list of all known (physical) volumes for the current system, with the ability |
381 | to filter them via keyword arguments. | |
181888fb | 382 | """ |
eafe8130 TL |
383 | |
384 | def __init__(self, populate=True): | |
385 | if populate: | |
386 | self._populate() | |
387 | ||
388 | def _populate(self): | |
389 | # get all the pvs in the current system | |
390 | for pv_item in get_api_pvs(): | |
391 | self.append(PVolume(**pv_item)) | |
392 | ||
393 | def _purge(self): | |
394 | """ | |
395 | Deplete all the items in the list, used internally only so that we can | |
396 | dynamically allocate the items when filtering without the concern of | |
397 | messing up the contents | |
398 | """ | |
399 | self[:] = [] | |
400 | ||
401 | def _filter(self, pv_name=None, pv_uuid=None, pv_tags=None): | |
402 | """ | |
403 | The actual method that filters using a new list. Useful so that other | |
404 | methods that do not want to alter the contents of the list (e.g. | |
405 | ``self.find``) can operate safely. | |
406 | """ | |
407 | filtered = [i for i in self] | |
408 | if pv_name: | |
409 | filtered = [i for i in filtered if i.pv_name == pv_name] | |
410 | ||
411 | if pv_uuid: | |
412 | filtered = [i for i in filtered if i.pv_uuid == pv_uuid] | |
413 | ||
414 | # at this point, `filtered` has either all the physical volumes in self | |
415 | # or is an actual filtered list if any filters were applied | |
416 | if pv_tags: | |
417 | tag_filtered = [] | |
418 | for pvolume in filtered: | |
419 | matches = all(pvolume.tags.get(k) == str(v) for k, v in pv_tags.items()) | |
420 | if matches: | |
421 | tag_filtered.append(pvolume) | |
422 | # return the tag_filtered pvolumes here, the `filtered` list is no | |
423 | # longer usable | |
424 | return tag_filtered | |
425 | ||
426 | return filtered | |
427 | ||
428 | def filter(self, pv_name=None, pv_uuid=None, pv_tags=None): | |
429 | """ | |
430 | Filter out volumes on top level attributes like ``pv_name`` or by | |
431 | ``pv_tags`` where a dict is required. For example, to find a physical | |
432 | volume that has an OSD ID of 0, the filter would look like:: | |
433 | ||
434 | pv_tags={'ceph.osd_id': '0'} | |
435 | ||
436 | """ | |
437 | if not any([pv_name, pv_uuid, pv_tags]): | |
438 | raise TypeError('.filter() requires pv_name, pv_uuid, or pv_tags' | |
439 | '(none given)') | |
440 | ||
441 | filtered_pvs = PVolumes(populate=False) | |
442 | filtered_pvs.extend(self._filter(pv_name, pv_uuid, pv_tags)) | |
443 | return filtered_pvs | |
444 | ||
445 | def get(self, pv_name=None, pv_uuid=None, pv_tags=None): | |
446 | """ | |
447 | This is a bit expensive, since it will try to filter out all the | |
448 | matching items in the list, filter them out applying anything that was | |
449 | added and return the matching item. | |
450 | ||
451 | This method does *not* alter the list, and it will raise an error if | |
452 | multiple pvs are matched | |
453 | ||
454 | It is useful to use ``tags`` when trying to find a specific logical volume, | |
455 | but it can also lead to multiple pvs being found, since a lot of metadata | |
456 | is shared between pvs of a distinct OSD. | |
457 | """ | |
458 | if not any([pv_name, pv_uuid, pv_tags]): | |
459 | return None | |
460 | pvs = self._filter( | |
461 | pv_name=pv_name, | |
462 | pv_uuid=pv_uuid, | |
463 | pv_tags=pv_tags | |
464 | ) | |
465 | if not pvs: | |
466 | return None | |
467 | if len(pvs) > 1 and pv_tags: | |
468 | raise MultiplePVsError(pv_name) | |
469 | return pvs[0] | |
181888fb FG |
470 | |
471 | ||
472 | def create_pv(device): | |
473 | """ | |
474 | Create a physical volume from a device, useful when devices need to be later mapped | |
475 | to journals. | |
476 | """ | |
477 | process.run([ | |
181888fb FG |
478 | 'pvcreate', |
479 | '-v', # verbose | |
480 | '-f', # force it | |
481 | '--yes', # answer yes to any prompts | |
482 | device | |
483 | ]) | |
d2e6a577 FG |
484 | |
485 | ||
eafe8130 | 486 | def remove_pv(pv_name): |
3efd9988 | 487 | """ |
eafe8130 TL |
488 | Removes a physical volume using a double `-f` to prevent prompts and fully |
489 | remove anything related to LVM. This is tremendously destructive, but so is all other actions | |
490 | when zapping a device. | |
3efd9988 | 491 | |
eafe8130 TL |
492 | In the case where multiple PVs are found, it will ignore that fact and |
493 | continue with the removal, specifically in the case of messages like:: | |
3efd9988 | 494 | |
eafe8130 | 495 | WARNING: PV $UUID /dev/DEV-1 was already found on /dev/DEV-2 |
1adf2230 | 496 | |
eafe8130 TL |
497 | These situations can be avoided with custom filtering rules, which this API |
498 | cannot handle while accommodating custom user filters. | |
3efd9988 | 499 | """ |
eafe8130 TL |
500 | fail_msg = "Unable to remove vg %s" % pv_name |
501 | process.run( | |
502 | [ | |
503 | 'pvremove', | |
504 | '-v', # verbose | |
505 | '-f', # force it | |
506 | '-f', # force it | |
507 | pv_name | |
508 | ], | |
509 | fail_msg=fail_msg, | |
510 | ) | |
3efd9988 FG |
511 | |
512 | ||
eafe8130 | 513 | def get_pv(pv_name=None, pv_uuid=None, pv_tags=None, pvs=None): |
1adf2230 | 514 | """ |
eafe8130 TL |
515 | Return a matching pv (physical volume) for the current system, requiring |
516 | ``pv_name``, ``pv_uuid``, or ``pv_tags``. Raises an error if more than one | |
517 | pv is found. | |
518 | """ | |
519 | if not any([pv_name, pv_uuid, pv_tags]): | |
520 | return None | |
521 | if pvs is None or len(pvs) == 0: | |
522 | pvs = PVolumes() | |
1adf2230 | 523 | |
eafe8130 | 524 | return pvs.get(pv_name=pv_name, pv_uuid=pv_uuid, pv_tags=pv_tags) |
1adf2230 | 525 | |
1adf2230 | 526 | |
eafe8130 TL |
527 | ################################ |
528 | # | |
529 | # Code for LVM Volume Groups | |
530 | # | |
531 | ############################# | |
1adf2230 | 532 | |
92f5a8d4 TL |
533 | VG_FIELDS = 'vg_name,pv_count,lv_count,vg_attr,vg_extent_count,vg_free_count,vg_extent_size' |
534 | VG_CMD_OPTIONS = ['--noheadings', '--readonly', '--units=b', '--nosuffix', '--separator=";"'] | |
535 | ||
1adf2230 | 536 | |
eafe8130 | 537 | def get_api_vgs(): |
81eedcae | 538 | """ |
eafe8130 TL |
539 | Return the list of group volumes available in the system using flags to |
540 | include common metadata associated with them | |
81eedcae | 541 | |
eafe8130 | 542 | Command and sample delimited output should look like:: |
81eedcae | 543 | |
92f5a8d4 TL |
544 | $ vgs --noheadings --units=b --readonly --separator=';' \ |
545 | -o vg_name,pv_count,lv_count,vg_attr,vg_free_count,vg_extent_size | |
546 | ubuntubox-vg;1;2;wz--n-;12; | |
eafe8130 TL |
547 | |
548 | To normalize sizing, the units are forced in 'g' which is equivalent to | |
549 | gigabytes, which uses multiples of 1024 (as opposed to 1000) | |
81eedcae | 550 | """ |
eafe8130 | 551 | stdout, stderr, returncode = process.call( |
92f5a8d4 | 552 | ['vgs'] + VG_CMD_OPTIONS + ['-o', VG_FIELDS], |
eafe8130 | 553 | verbose_on_failure=False |
81eedcae | 554 | ) |
92f5a8d4 | 555 | return _output_parser(stdout, VG_FIELDS) |
81eedcae TL |
556 | |
557 | ||
eafe8130 | 558 | class VolumeGroup(object): |
b32b8144 | 559 | """ |
eafe8130 | 560 | Represents an LVM group, with some top-level attributes like ``vg_name`` |
b32b8144 | 561 | """ |
b32b8144 | 562 | |
eafe8130 TL |
563 | def __init__(self, **kw): |
564 | for k, v in kw.items(): | |
565 | setattr(self, k, v) | |
566 | self.name = kw['vg_name'] | |
9f95a23c TL |
567 | if not self.name: |
568 | raise ValueError('VolumeGroup must have a non-empty name') | |
eafe8130 | 569 | self.tags = parse_tags(kw.get('vg_tags', '')) |
b32b8144 | 570 | |
eafe8130 TL |
571 | def __str__(self): |
572 | return '<%s>' % self.name | |
91327a77 | 573 | |
eafe8130 TL |
574 | def __repr__(self): |
575 | return self.__str__() | |
91327a77 | 576 | |
eafe8130 TL |
577 | @property |
578 | def free(self): | |
579 | """ | |
92f5a8d4 | 580 | Return free space in VG in bytes |
eafe8130 | 581 | """ |
92f5a8d4 | 582 | return int(self.vg_extent_size) * int(self.vg_free_count) |
3efd9988 | 583 | |
eafe8130 TL |
584 | @property |
585 | def size(self): | |
586 | """ | |
92f5a8d4 | 587 | Returns VG size in bytes |
eafe8130 | 588 | """ |
92f5a8d4 | 589 | return int(self.vg_extent_size) * int(self.vg_extent_count) |
91327a77 | 590 | |
eafe8130 TL |
591 | def sizing(self, parts=None, size=None): |
592 | """ | |
593 | Calculate proper sizing to fully utilize the volume group in the most | |
594 | efficient way possible. To prevent situations where LVM might accept | |
595 | a percentage that is beyond the vg's capabilities, it will refuse with | |
596 | an error when requesting a larger-than-possible parameter, in addition | |
597 | to rounding down calculations. | |
3efd9988 | 598 | |
eafe8130 TL |
599 | A dictionary with different sizing parameters is returned, to make it |
600 | easier for others to choose what they need in order to create logical | |
601 | volumes:: | |
3efd9988 | 602 | |
eafe8130 TL |
603 | >>> data_vg.free |
604 | 1024 | |
605 | >>> data_vg.sizing(parts=4) | |
606 | {'parts': 4, 'sizes': 256, 'percentages': 25} | |
607 | >>> data_vg.sizing(size=512) | |
608 | {'parts': 2, 'sizes': 512, 'percentages': 50} | |
d2e6a577 | 609 | |
d2e6a577 | 610 | |
eafe8130 TL |
611 | :param parts: Number of parts to create LVs from |
612 | :param size: Size in gigabytes to divide the VG into | |
d2e6a577 | 613 | |
eafe8130 TL |
614 | :raises SizeAllocationError: When requested size cannot be allocated with |
615 | :raises ValueError: If both ``parts`` and ``size`` are given | |
616 | """ | |
617 | if parts is not None and size is not None: | |
618 | raise ValueError( | |
619 | "Cannot process sizing with both parts (%s) and size (%s)" % (parts, size) | |
620 | ) | |
1adf2230 | 621 | |
eafe8130 TL |
622 | # if size is given we need to map that to extents so that we avoid |
623 | # issues when trying to get this right with a size in gigabytes find | |
624 | # the percentage first, cheating, because these values are thrown out | |
625 | vg_free_count = util.str_to_int(self.vg_free_count) | |
626 | ||
627 | if size: | |
92f5a8d4 TL |
628 | size = size * 1024 * 1024 * 1024 |
629 | extents = int(size / int(self.vg_extent_size)) | |
eafe8130 TL |
630 | disk_sizing = sizing(self.free, size=size, parts=parts) |
631 | else: | |
632 | if parts is not None: | |
633 | # Prevent parts being 0, falling back to 1 (100% usage) | |
634 | parts = parts or 1 | |
635 | size = int(self.free / parts) | |
636 | extents = size * vg_free_count / self.free | |
637 | disk_sizing = sizing(self.free, parts=parts) | |
638 | ||
639 | extent_sizing = sizing(vg_free_count, size=extents) | |
640 | ||
641 | disk_sizing['extents'] = int(extents) | |
642 | disk_sizing['percentages'] = extent_sizing['percentages'] | |
643 | return disk_sizing | |
644 | ||
92f5a8d4 TL |
645 | def bytes_to_extents(self, size): |
646 | ''' | |
647 | Return a how many extents we can fit into a size in bytes. | |
648 | ''' | |
649 | return int(size / int(self.vg_extent_size)) | |
650 | ||
651 | def slots_to_extents(self, slots): | |
652 | ''' | |
653 | Return how many extents fit the VG slot times | |
654 | ''' | |
655 | return int(int(self.vg_free_count) / slots) | |
656 | ||
eafe8130 TL |
657 | |
658 | class VolumeGroups(list): | |
659 | """ | |
660 | A list of all known volume groups for the current system, with the ability | |
661 | to filter them via keyword arguments. | |
d2e6a577 | 662 | """ |
1adf2230 | 663 | |
eafe8130 TL |
664 | def __init__(self, populate=True): |
665 | if populate: | |
666 | self._populate() | |
d2e6a577 FG |
667 | |
668 | def _populate(self): | |
669 | # get all the vgs in the current system | |
670 | for vg_item in get_api_vgs(): | |
671 | self.append(VolumeGroup(**vg_item)) | |
672 | ||
673 | def _purge(self): | |
674 | """ | |
675 | Deplete all the items in the list, used internally only so that we can | |
676 | dynamically allocate the items when filtering without the concern of | |
677 | messing up the contents | |
678 | """ | |
679 | self[:] = [] | |
680 | ||
681 | def _filter(self, vg_name=None, vg_tags=None): | |
682 | """ | |
683 | The actual method that filters using a new list. Useful so that other | |
684 | methods that do not want to alter the contents of the list (e.g. | |
685 | ``self.find``) can operate safely. | |
686 | ||
687 | .. note:: ``vg_tags`` is not yet implemented | |
688 | """ | |
689 | filtered = [i for i in self] | |
690 | if vg_name: | |
691 | filtered = [i for i in filtered if i.vg_name == vg_name] | |
692 | ||
693 | # at this point, `filtered` has either all the volumes in self or is an | |
694 | # actual filtered list if any filters were applied | |
695 | if vg_tags: | |
696 | tag_filtered = [] | |
181888fb FG |
697 | for volume in filtered: |
698 | matches = all(volume.tags.get(k) == str(v) for k, v in vg_tags.items()) | |
699 | if matches: | |
700 | tag_filtered.append(volume) | |
d2e6a577 FG |
701 | return tag_filtered |
702 | ||
703 | return filtered | |
704 | ||
705 | def filter(self, vg_name=None, vg_tags=None): | |
706 | """ | |
707 | Filter out groups on top level attributes like ``vg_name`` or by | |
708 | ``vg_tags`` where a dict is required. For example, to find a Ceph group | |
709 | with dmcache as the type, the filter would look like:: | |
710 | ||
711 | vg_tags={'ceph.type': 'dmcache'} | |
712 | ||
713 | .. warning:: These tags are not documented because they are currently | |
714 | unused, but are here to maintain API consistency | |
715 | """ | |
716 | if not any([vg_name, vg_tags]): | |
717 | raise TypeError('.filter() requires vg_name or vg_tags (none given)') | |
eafe8130 TL |
718 | |
719 | filtered_vgs = VolumeGroups(populate=False) | |
720 | filtered_vgs.extend(self._filter(vg_name, vg_tags)) | |
721 | return filtered_vgs | |
d2e6a577 FG |
722 | |
723 | def get(self, vg_name=None, vg_tags=None): | |
724 | """ | |
725 | This is a bit expensive, since it will try to filter out all the | |
726 | matching items in the list, filter them out applying anything that was | |
727 | added and return the matching item. | |
728 | ||
729 | This method does *not* alter the list, and it will raise an error if | |
730 | multiple VGs are matched | |
731 | ||
732 | It is useful to use ``tags`` when trying to find a specific volume group, | |
733 | but it can also lead to multiple vgs being found (although unlikely) | |
734 | """ | |
735 | if not any([vg_name, vg_tags]): | |
736 | return None | |
737 | vgs = self._filter( | |
738 | vg_name=vg_name, | |
739 | vg_tags=vg_tags | |
740 | ) | |
741 | if not vgs: | |
742 | return None | |
743 | if len(vgs) > 1: | |
744 | # this is probably never going to happen, but it is here to keep | |
745 | # the API code consistent | |
746 | raise MultipleVGsError(vg_name) | |
747 | return vgs[0] | |
748 | ||
749 | ||
eafe8130 | 750 | def create_vg(devices, name=None, name_prefix=None): |
d2e6a577 | 751 | """ |
eafe8130 | 752 | Create a Volume Group. Command looks like:: |
d2e6a577 | 753 | |
eafe8130 | 754 | vgcreate --force --yes group_name device |
d2e6a577 | 755 | |
eafe8130 | 756 | Once created the volume group is returned as a ``VolumeGroup`` object |
d2e6a577 | 757 | |
eafe8130 TL |
758 | :param devices: A list of devices to create a VG. Optionally, a single |
759 | device (as a string) can be used. | |
760 | :param name: Optionally set the name of the VG, defaults to 'ceph-{uuid}' | |
761 | :param name_prefix: Optionally prefix the name of the VG, which will get combined | |
762 | with a UUID string | |
763 | """ | |
764 | if isinstance(devices, set): | |
765 | devices = list(devices) | |
766 | if not isinstance(devices, list): | |
767 | devices = [devices] | |
768 | if name_prefix: | |
769 | name = "%s-%s" % (name_prefix, str(uuid.uuid4())) | |
770 | elif name is None: | |
771 | name = "ceph-%s" % str(uuid.uuid4()) | |
772 | process.run([ | |
773 | 'vgcreate', | |
eafe8130 TL |
774 | '--force', |
775 | '--yes', | |
776 | name] + devices | |
777 | ) | |
d2e6a577 | 778 | |
eafe8130 TL |
779 | vg = get_vg(vg_name=name) |
780 | return vg | |
d2e6a577 | 781 | |
d2e6a577 | 782 | |
eafe8130 TL |
783 | def extend_vg(vg, devices): |
784 | """ | |
785 | Extend a Volume Group. Command looks like:: | |
181888fb | 786 | |
eafe8130 | 787 | vgextend --force --yes group_name [device, ...] |
d2e6a577 | 788 | |
eafe8130 | 789 | Once created the volume group is extended and returned as a ``VolumeGroup`` object |
d2e6a577 | 790 | |
eafe8130 TL |
791 | :param vg: A VolumeGroup object |
792 | :param devices: A list of devices to extend the VG. Optionally, a single | |
793 | device (as a string) can be used. | |
794 | """ | |
795 | if not isinstance(devices, list): | |
796 | devices = [devices] | |
797 | process.run([ | |
798 | 'vgextend', | |
799 | '--force', | |
800 | '--yes', | |
801 | vg.name] + devices | |
802 | ) | |
d2e6a577 | 803 | |
eafe8130 TL |
804 | vg = get_vg(vg_name=vg.name) |
805 | return vg | |
d2e6a577 | 806 | |
d2e6a577 | 807 | |
eafe8130 TL |
808 | def reduce_vg(vg, devices): |
809 | """ | |
810 | Reduce a Volume Group. Command looks like:: | |
d2e6a577 | 811 | |
eafe8130 | 812 | vgreduce --force --yes group_name [device, ...] |
d2e6a577 | 813 | |
eafe8130 TL |
814 | :param vg: A VolumeGroup object |
815 | :param devices: A list of devices to remove from the VG. Optionally, a | |
816 | single device (as a string) can be used. | |
817 | """ | |
818 | if not isinstance(devices, list): | |
819 | devices = [devices] | |
820 | process.run([ | |
821 | 'vgreduce', | |
822 | '--force', | |
823 | '--yes', | |
824 | vg.name] + devices | |
825 | ) | |
d2e6a577 | 826 | |
eafe8130 TL |
827 | vg = get_vg(vg_name=vg.name) |
828 | return vg | |
d2e6a577 FG |
829 | |
830 | ||
eafe8130 | 831 | def remove_vg(vg_name): |
181888fb | 832 | """ |
eafe8130 | 833 | Removes a volume group. |
181888fb | 834 | """ |
eafe8130 TL |
835 | if not vg_name: |
836 | logger.warning('Skipping removal of invalid VG name: "%s"', vg_name) | |
837 | return | |
838 | fail_msg = "Unable to remove vg %s" % vg_name | |
839 | process.run( | |
840 | [ | |
841 | 'vgremove', | |
842 | '-v', # verbose | |
843 | '-f', # force it | |
844 | vg_name | |
845 | ], | |
846 | fail_msg=fail_msg, | |
847 | ) | |
181888fb FG |
848 | |
849 | ||
eafe8130 | 850 | def get_vg(vg_name=None, vg_tags=None, vgs=None): |
d2e6a577 | 851 | """ |
eafe8130 TL |
852 | Return a matching vg for the current system, requires ``vg_name`` or |
853 | ``tags``. Raises an error if more than one vg is found. | |
1adf2230 | 854 | |
eafe8130 TL |
855 | It is useful to use ``tags`` when trying to find a specific volume group, |
856 | but it can also lead to multiple vgs being found. | |
857 | """ | |
858 | if not any([vg_name, vg_tags]): | |
859 | return None | |
860 | if vgs is None or len(vgs) == 0: | |
861 | vgs = VolumeGroups() | |
1adf2230 | 862 | |
eafe8130 | 863 | return vgs.get(vg_name=vg_name, vg_tags=vg_tags) |
1adf2230 AA |
864 | |
865 | ||
92f5a8d4 TL |
866 | def get_device_vgs(device, name_prefix=''): |
867 | stdout, stderr, returncode = process.call( | |
868 | ['pvs'] + VG_CMD_OPTIONS + ['-o', VG_FIELDS, device], | |
869 | verbose_on_failure=False | |
870 | ) | |
871 | vgs = _output_parser(stdout, VG_FIELDS) | |
9f95a23c | 872 | return [VolumeGroup(**vg) for vg in vgs if vg['vg_name'] and vg['vg_name'].startswith(name_prefix)] |
92f5a8d4 TL |
873 | |
874 | ||
eafe8130 TL |
875 | ################################# |
876 | # | |
877 | # Code for LVM Logical Volumes | |
878 | # | |
879 | ############################### | |
1adf2230 | 880 | |
92f5a8d4 TL |
881 | LV_FIELDS = 'lv_tags,lv_path,lv_name,vg_name,lv_uuid,lv_size' |
882 | LV_CMD_OPTIONS = ['--noheadings', '--readonly', '--separator=";"', '-a'] | |
1adf2230 | 883 | |
eafe8130 TL |
884 | def get_api_lvs(): |
885 | """ | |
886 | Return the list of logical volumes available in the system using flags to include common | |
887 | metadata associated with them | |
1adf2230 | 888 | |
eafe8130 | 889 | Command and delimited output should look like:: |
1adf2230 | 890 | |
eafe8130 TL |
891 | $ lvs --noheadings --readonly --separator=';' -a -o lv_tags,lv_path,lv_name,vg_name |
892 | ;/dev/ubuntubox-vg/root;root;ubuntubox-vg | |
893 | ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg | |
1adf2230 | 894 | |
eafe8130 | 895 | """ |
eafe8130 | 896 | stdout, stderr, returncode = process.call( |
92f5a8d4 | 897 | ['lvs'] + LV_CMD_OPTIONS + ['-o', LV_FIELDS], |
eafe8130 TL |
898 | verbose_on_failure=False |
899 | ) | |
92f5a8d4 | 900 | return _output_parser(stdout, LV_FIELDS) |
1adf2230 | 901 | |
d2e6a577 FG |
902 | |
903 | class Volume(object): | |
904 | """ | |
905 | Represents a Logical Volume from LVM, with some top-level attributes like | |
906 | ``lv_name`` and parsed tags as a dictionary of key/value pairs. | |
907 | """ | |
908 | ||
909 | def __init__(self, **kw): | |
910 | for k, v in kw.items(): | |
911 | setattr(self, k, v) | |
912 | self.lv_api = kw | |
913 | self.name = kw['lv_name'] | |
9f95a23c TL |
914 | if not self.name: |
915 | raise ValueError('Volume must have a non-empty name') | |
d2e6a577 | 916 | self.tags = parse_tags(kw['lv_tags']) |
3a9019d9 | 917 | self.encrypted = self.tags.get('ceph.encrypted', '0') == '1' |
91327a77 | 918 | self.used_by_ceph = 'ceph.osd_id' in self.tags |
d2e6a577 FG |
919 | |
920 | def __str__(self): | |
921 | return '<%s>' % self.lv_api['lv_path'] | |
922 | ||
923 | def __repr__(self): | |
924 | return self.__str__() | |
925 | ||
3efd9988 FG |
926 | def as_dict(self): |
927 | obj = {} | |
928 | obj.update(self.lv_api) | |
929 | obj['tags'] = self.tags | |
930 | obj['name'] = self.name | |
931 | obj['type'] = self.tags['ceph.type'] | |
932 | obj['path'] = self.lv_path | |
933 | return obj | |
934 | ||
91327a77 AA |
935 | def report(self): |
936 | if not self.used_by_ceph: | |
937 | return { | |
938 | 'name': self.lv_name, | |
939 | 'comment': 'not used by ceph' | |
940 | } | |
941 | else: | |
942 | type_ = self.tags['ceph.type'] | |
943 | report = { | |
944 | 'name': self.lv_name, | |
945 | 'osd_id': self.tags['ceph.osd_id'], | |
946 | 'cluster_name': self.tags['ceph.cluster_name'], | |
947 | 'type': type_, | |
948 | 'osd_fsid': self.tags['ceph.osd_fsid'], | |
949 | 'cluster_fsid': self.tags['ceph.cluster_fsid'], | |
e306af50 | 950 | 'osdspec_affinity': self.tags.get('ceph.osdspec_affinity', ''), |
91327a77 AA |
951 | } |
952 | type_uuid = '{}_uuid'.format(type_) | |
953 | report[type_uuid] = self.tags['ceph.{}'.format(type_uuid)] | |
954 | return report | |
955 | ||
e306af50 TL |
956 | def _format_tag_args(self, op, tags): |
957 | tag_args = ['{}={}'.format(k, v) for k, v in tags.items()] | |
958 | # weird but efficient way of ziping two lists and getting a flat list | |
959 | return list(sum(zip(repeat(op), tag_args), ())) | |
960 | ||
961 | def clear_tags(self, keys=None): | |
3efd9988 | 962 | """ |
e306af50 | 963 | Removes all or passed tags from the Logical Volume. |
3efd9988 | 964 | """ |
e306af50 TL |
965 | if not keys: |
966 | keys = self.tags.keys() | |
967 | ||
968 | del_tags = {k: self.tags[k] for k in keys if k in self.tags} | |
969 | if not del_tags: | |
970 | # nothing to clear | |
971 | return | |
972 | del_tag_args = self._format_tag_args('--deltag', del_tags) | |
973 | # --deltag returns successful even if the to be deleted tag is not set | |
974 | process.call(['lvchange'] + del_tag_args + [self.lv_path]) | |
975 | for k in del_tags.keys(): | |
976 | del self.tags[k] | |
81eedcae | 977 | |
3efd9988 | 978 | |
d2e6a577 FG |
979 | def set_tags(self, tags): |
980 | """ | |
981 | :param tags: A dictionary of tag names and values, like:: | |
982 | ||
983 | { | |
984 | "ceph.osd_fsid": "aaa-fff-bbbb", | |
985 | "ceph.osd_id": "0" | |
986 | } | |
987 | ||
988 | At the end of all modifications, the tags are refreshed to reflect | |
989 | LVM's most current view. | |
990 | """ | |
e306af50 TL |
991 | self.clear_tags(tags.keys()) |
992 | add_tag_args = self._format_tag_args('--addtag', tags) | |
993 | process.call(['lvchange'] + add_tag_args + [self.lv_path]) | |
d2e6a577 | 994 | for k, v in tags.items(): |
e306af50 | 995 | self.tags[k] = v |
81eedcae TL |
996 | |
997 | ||
998 | def clear_tag(self, key): | |
999 | if self.tags.get(key): | |
1000 | current_value = self.tags[key] | |
1001 | tag = "%s=%s" % (key, current_value) | |
1002 | process.call(['lvchange', '--deltag', tag, self.lv_path]) | |
1003 | del self.tags[key] | |
1004 | ||
d2e6a577 FG |
1005 | |
1006 | def set_tag(self, key, value): | |
1007 | """ | |
81eedcae | 1008 | Set the key/value pair as an LVM tag. |
d2e6a577 FG |
1009 | """ |
1010 | # remove it first if it exists | |
81eedcae | 1011 | self.clear_tag(key) |
d2e6a577 FG |
1012 | |
1013 | process.call( | |
1014 | [ | |
b32b8144 | 1015 | 'lvchange', |
d2e6a577 FG |
1016 | '--addtag', '%s=%s' % (key, value), self.lv_path |
1017 | ] | |
1018 | ) | |
81eedcae | 1019 | self.tags[key] = value |
181888fb | 1020 | |
92f5a8d4 TL |
1021 | def deactivate(self): |
1022 | """ | |
1023 | Deactivate the LV by calling lvchange -an | |
1024 | """ | |
1025 | process.call(['lvchange', '-an', self.lv_path]) | |
1026 | ||
181888fb | 1027 | |
eafe8130 | 1028 | class Volumes(list): |
181888fb | 1029 | """ |
eafe8130 TL |
1030 | A list of all known (logical) volumes for the current system, with the ability |
1031 | to filter them via keyword arguments. | |
181888fb FG |
1032 | """ |
1033 | ||
eafe8130 TL |
1034 | def __init__(self): |
1035 | self._populate() | |
181888fb | 1036 | |
eafe8130 TL |
1037 | def _populate(self): |
1038 | # get all the lvs in the current system | |
1039 | for lv_item in get_api_lvs(): | |
1040 | self.append(Volume(**lv_item)) | |
181888fb | 1041 | |
eafe8130 | 1042 | def _purge(self): |
181888fb | 1043 | """ |
eafe8130 TL |
1044 | Delete all the items in the list, used internally only so that we can |
1045 | dynamically allocate the items when filtering without the concern of | |
1046 | messing up the contents | |
1047 | """ | |
1048 | self[:] = [] | |
181888fb | 1049 | |
eafe8130 TL |
1050 | def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): |
1051 | """ | |
1052 | The actual method that filters using a new list. Useful so that other | |
1053 | methods that do not want to alter the contents of the list (e.g. | |
1054 | ``self.find``) can operate safely. | |
1055 | """ | |
1056 | filtered = [i for i in self] | |
1057 | if lv_name: | |
1058 | filtered = [i for i in filtered if i.lv_name == lv_name] | |
181888fb | 1059 | |
eafe8130 TL |
1060 | if vg_name: |
1061 | filtered = [i for i in filtered if i.vg_name == vg_name] | |
1062 | ||
1063 | if lv_uuid: | |
1064 | filtered = [i for i in filtered if i.lv_uuid == lv_uuid] | |
1065 | ||
1066 | if lv_path: | |
1067 | filtered = [i for i in filtered if i.lv_path == lv_path] | |
1068 | ||
1069 | # at this point, `filtered` has either all the volumes in self or is an | |
1070 | # actual filtered list if any filters were applied | |
1071 | if lv_tags: | |
1072 | tag_filtered = [] | |
1073 | for volume in filtered: | |
1074 | # all the tags we got need to match on the volume | |
1075 | matches = all(volume.tags.get(k) == str(v) for k, v in lv_tags.items()) | |
1076 | if matches: | |
1077 | tag_filtered.append(volume) | |
1078 | return tag_filtered | |
1079 | ||
1080 | return filtered | |
1081 | ||
1082 | def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): | |
181888fb | 1083 | """ |
eafe8130 TL |
1084 | Filter out volumes on top level attributes like ``lv_name`` or by |
1085 | ``lv_tags`` where a dict is required. For example, to find a volume | |
1086 | that has an OSD ID of 0, the filter would look like:: | |
1087 | ||
1088 | lv_tags={'ceph.osd_id': '0'} | |
181888fb | 1089 | |
181888fb | 1090 | """ |
eafe8130 TL |
1091 | if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): |
1092 | raise TypeError('.filter() requires lv_name, vg_name, lv_path, lv_uuid, or tags (none given)') | |
1093 | # first find the filtered volumes with the values in self | |
1094 | filtered_volumes = self._filter( | |
1095 | lv_name=lv_name, | |
1096 | vg_name=vg_name, | |
1097 | lv_path=lv_path, | |
1098 | lv_uuid=lv_uuid, | |
1099 | lv_tags=lv_tags | |
1100 | ) | |
1101 | # then purge everything | |
1102 | self._purge() | |
1103 | # and add the filtered items | |
1104 | self.extend(filtered_volumes) | |
181888fb | 1105 | |
eafe8130 | 1106 | def get(self, lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None): |
181888fb | 1107 | """ |
eafe8130 TL |
1108 | This is a bit expensive, since it will try to filter out all the |
1109 | matching items in the list, filter them out applying anything that was | |
1110 | added and return the matching item. | |
181888fb | 1111 | |
eafe8130 TL |
1112 | This method does *not* alter the list, and it will raise an error if |
1113 | multiple LVs are matched | |
1114 | ||
1115 | It is useful to use ``tags`` when trying to find a specific logical volume, | |
1116 | but it can also lead to multiple lvs being found, since a lot of metadata | |
1117 | is shared between lvs of a distinct OSD. | |
1118 | """ | |
1119 | if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): | |
1120 | return None | |
1121 | lvs = self._filter( | |
1122 | lv_name=lv_name, | |
1123 | vg_name=vg_name, | |
1124 | lv_path=lv_path, | |
1125 | lv_uuid=lv_uuid, | |
1126 | lv_tags=lv_tags | |
181888fb | 1127 | ) |
eafe8130 TL |
1128 | if not lvs: |
1129 | return None | |
1130 | if len(lvs) > 1: | |
1131 | raise MultipleLVsError(lv_name, lv_path) | |
1132 | return lvs[0] | |
1133 | ||
1134 | ||
92f5a8d4 TL |
1135 | def create_lv(name_prefix, |
1136 | uuid, | |
1137 | vg=None, | |
1138 | device=None, | |
1139 | slots=None, | |
1140 | extents=None, | |
1141 | size=None, | |
1142 | tags=None): | |
eafe8130 TL |
1143 | """ |
1144 | Create a Logical Volume in a Volume Group. Command looks like:: | |
1145 | ||
1146 | lvcreate -L 50G -n gfslv vg0 | |
1147 | ||
92f5a8d4 TL |
1148 | ``name_prefix`` is required. If ``size`` is provided its expected to be a |
1149 | byte count. Tags are an optional dictionary and is expected to | |
eafe8130 TL |
1150 | conform to the convention of prefixing them with "ceph." like:: |
1151 | ||
1152 | {"ceph.block_device": "/dev/ceph/osd-1"} | |
1153 | ||
92f5a8d4 TL |
1154 | :param name_prefix: name prefix for the LV, typically somehting like ceph-osd-block |
1155 | :param uuid: UUID to ensure uniqueness; is combined with name_prefix to | |
1156 | form the LV name | |
1157 | :param vg: optional, pass an existing VG to create LV | |
1158 | :param device: optional, device to use. Either device of vg must be passed | |
1159 | :param slots: optional, number of slots to divide vg up, LV will occupy one | |
1160 | one slot if enough space is available | |
1161 | :param extends: optional, how many lvm extends to use, supersedes slots | |
1162 | :param size: optional, target LV size in bytes, supersedes extents, | |
1163 | resulting LV might be smaller depending on extent | |
1164 | size of the underlying VG | |
1165 | :param tags: optional, a dict of lvm tags to set on the LV | |
1166 | """ | |
1167 | name = '{}-{}'.format(name_prefix, uuid) | |
1168 | if not vg: | |
1169 | if not device: | |
1170 | raise RuntimeError("Must either specify vg or device, none given") | |
1171 | # check if a vgs starting with ceph already exists | |
1172 | vgs = get_device_vgs(device, 'ceph') | |
1173 | if vgs: | |
1174 | vg = vgs[0] | |
1175 | else: | |
1176 | # create on if not | |
1177 | vg = create_vg(device, name_prefix='ceph') | |
1178 | assert(vg) | |
eafe8130 | 1179 | |
eafe8130 | 1180 | if size: |
92f5a8d4 TL |
1181 | extents = vg.bytes_to_extents(size) |
1182 | logger.debug('size was passed: {} -> {}'.format(size, extents)) | |
1183 | elif slots and not extents: | |
1184 | extents = vg.slots_to_extents(slots) | |
1185 | logger.debug('slots was passed: {} -> {}'.format(slots, extents)) | |
1186 | ||
1187 | if extents: | |
eafe8130 TL |
1188 | command = [ |
1189 | 'lvcreate', | |
1190 | '--yes', | |
1191 | '-l', | |
92f5a8d4 TL |
1192 | '{}'.format(extents), |
1193 | '-n', name, vg.vg_name | |
eafe8130 TL |
1194 | ] |
1195 | # create the lv with all the space available, this is needed because the | |
1196 | # system call is different for LVM | |
1197 | else: | |
1198 | command = [ | |
1199 | 'lvcreate', | |
1200 | '--yes', | |
1201 | '-l', | |
1202 | '100%FREE', | |
92f5a8d4 | 1203 | '-n', name, vg.vg_name |
eafe8130 | 1204 | ] |
eafe8130 TL |
1205 | process.run(command) |
1206 | ||
92f5a8d4 | 1207 | lv = get_lv(lv_name=name, vg_name=vg.vg_name) |
eafe8130 | 1208 | |
92f5a8d4 TL |
1209 | if tags is None: |
1210 | tags = { | |
1211 | "ceph.osd_id": "null", | |
1212 | "ceph.type": "null", | |
1213 | "ceph.cluster_fsid": "null", | |
1214 | "ceph.osd_fsid": "null", | |
1215 | } | |
eafe8130 TL |
1216 | # when creating a distinct type, the caller doesn't know what the path will |
1217 | # be so this function will set it after creation using the mapping | |
92f5a8d4 TL |
1218 | # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations |
1219 | type_path_tag = { | |
1220 | 'journal': 'ceph.journal_device', | |
1221 | 'data': 'ceph.data_device', | |
1222 | 'block': 'ceph.block_device', | |
1223 | 'wal': 'ceph.wal_device', | |
1224 | 'db': 'ceph.db_device', | |
1225 | 'lockbox': 'ceph.lockbox_device', # XXX might not ever need this lockbox sorcery | |
1226 | } | |
eafe8130 TL |
1227 | path_tag = type_path_tag.get(tags.get('ceph.type')) |
1228 | if path_tag: | |
92f5a8d4 TL |
1229 | tags.update({path_tag: lv.lv_path}) |
1230 | ||
1231 | lv.set_tags(tags) | |
1232 | ||
eafe8130 TL |
1233 | return lv |
1234 | ||
1235 | ||
1236 | def remove_lv(lv): | |
1237 | """ | |
1238 | Removes a logical volume given it's absolute path. | |
1239 | ||
1240 | Will return True if the lv is successfully removed or | |
1241 | raises a RuntimeError if the removal fails. | |
1242 | ||
1243 | :param lv: A ``Volume`` object or the path for an LV | |
1244 | """ | |
1245 | if isinstance(lv, Volume): | |
1246 | path = lv.lv_path | |
1247 | else: | |
1248 | path = lv | |
1249 | ||
1250 | stdout, stderr, returncode = process.call( | |
1251 | [ | |
1252 | 'lvremove', | |
1253 | '-v', # verbose | |
1254 | '-f', # force it | |
1255 | path | |
1256 | ], | |
1257 | show_command=True, | |
1258 | terminal_verbose=True, | |
1259 | ) | |
1260 | if returncode != 0: | |
1261 | raise RuntimeError("Unable to remove %s" % path) | |
1262 | return True | |
1263 | ||
1264 | ||
1265 | def is_lv(dev, lvs=None): | |
1266 | """ | |
1267 | Boolean to detect if a device is an LV or not. | |
1268 | """ | |
1269 | splitname = dmsetup_splitname(dev) | |
1270 | # Allowing to optionally pass `lvs` can help reduce repetitive checks for | |
1271 | # multiple devices at once. | |
1272 | if lvs is None or len(lvs) == 0: | |
1273 | lvs = Volumes() | |
1274 | ||
1275 | if splitname.get('LV_NAME'): | |
1276 | lvs.filter(lv_name=splitname['LV_NAME'], vg_name=splitname['VG_NAME']) | |
1277 | return len(lvs) > 0 | |
1278 | return False | |
1279 | ||
92f5a8d4 TL |
1280 | def get_lv_by_name(name): |
1281 | stdout, stderr, returncode = process.call( | |
1282 | ['lvs', '--noheadings', '-o', LV_FIELDS, '-S', | |
1283 | 'lv_name={}'.format(name)], | |
1284 | verbose_on_failure=False | |
1285 | ) | |
1286 | lvs = _output_parser(stdout, LV_FIELDS) | |
1287 | return [Volume(**lv) for lv in lvs] | |
1288 | ||
1289 | def get_lvs_by_tag(lv_tag): | |
1290 | stdout, stderr, returncode = process.call( | |
1291 | ['lvs', '--noheadings', '--separator=";"', '-a', '-o', LV_FIELDS, '-S', | |
1292 | 'lv_tags={{{}}}'.format(lv_tag)], | |
1293 | verbose_on_failure=False | |
1294 | ) | |
1295 | lvs = _output_parser(stdout, LV_FIELDS) | |
1296 | return [Volume(**lv) for lv in lvs] | |
eafe8130 TL |
1297 | |
1298 | def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_uuid=None, lv_tags=None, lvs=None): | |
1299 | """ | |
1300 | Return a matching lv for the current system, requiring ``lv_name``, | |
1301 | ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv | |
1302 | is found. | |
1303 | ||
1304 | It is useful to use ``tags`` when trying to find a specific logical volume, | |
1305 | but it can also lead to multiple lvs being found, since a lot of metadata | |
1306 | is shared between lvs of a distinct OSD. | |
1307 | """ | |
1308 | if not any([lv_name, vg_name, lv_path, lv_uuid, lv_tags]): | |
1309 | return None | |
1310 | if lvs is None: | |
1311 | lvs = Volumes() | |
1312 | return lvs.get( | |
1313 | lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_uuid=lv_uuid, | |
1314 | lv_tags=lv_tags | |
1315 | ) | |
1316 | ||
1317 | ||
1318 | def get_lv_from_argument(argument): | |
1319 | """ | |
1320 | Helper proxy function that consumes a possible logical volume passed in from the CLI | |
1321 | in the form of `vg/lv`, but with some validation so that an argument that is a full | |
1322 | path to a device can be ignored | |
1323 | """ | |
1324 | if argument.startswith('/'): | |
1325 | lv = get_lv(lv_path=argument) | |
1326 | return lv | |
1327 | try: | |
1328 | vg_name, lv_name = argument.split('/') | |
1329 | except (ValueError, AttributeError): | |
1330 | return None | |
1331 | return get_lv(lv_name=lv_name, vg_name=vg_name) | |
1332 | ||
1333 | ||
1334 | def create_lvs(volume_group, parts=None, size=None, name_prefix='ceph-lv'): | |
1335 | """ | |
1336 | Create multiple Logical Volumes from a Volume Group by calculating the | |
1337 | proper extents from ``parts`` or ``size``. A custom prefix can be used | |
1338 | (defaults to ``ceph-lv``), these names are always suffixed with a uuid. | |
1339 | ||
1340 | LV creation in ceph-volume will require tags, this is expected to be | |
1341 | pre-computed by callers who know Ceph metadata like OSD IDs and FSIDs. It | |
1342 | will probably not be the case when mass-creating LVs, so common/default | |
1343 | tags will be set to ``"null"``. | |
1344 | ||
1345 | .. note:: LVs that are not in use can be detected by querying LVM for tags that are | |
1346 | set to ``"null"``. | |
1347 | ||
1348 | :param volume_group: The volume group (vg) to use for LV creation | |
1349 | :type group: ``VolumeGroup()`` object | |
1350 | :param parts: Number of LVs to create *instead of* ``size``. | |
1351 | :type parts: int | |
1352 | :param size: Size (in gigabytes) of LVs to create, e.g. "as many 10gb LVs as possible" | |
1353 | :type size: int | |
1354 | :param extents: The number of LVM extents to use to create the LV. Useful if looking to have | |
1355 | accurate LV sizes (LVM rounds sizes otherwise) | |
1356 | """ | |
1357 | if parts is None and size is None: | |
1358 | # fallback to just one part (using 100% of the vg) | |
1359 | parts = 1 | |
1360 | lvs = [] | |
1361 | tags = { | |
1362 | "ceph.osd_id": "null", | |
1363 | "ceph.type": "null", | |
1364 | "ceph.cluster_fsid": "null", | |
1365 | "ceph.osd_fsid": "null", | |
1366 | } | |
1367 | sizing = volume_group.sizing(parts=parts, size=size) | |
1368 | for part in range(0, sizing['parts']): | |
1369 | size = sizing['sizes'] | |
1370 | extents = sizing['extents'] | |
eafe8130 | 1371 | lvs.append( |
92f5a8d4 | 1372 | create_lv(name_prefix, uuid.uuid4(), vg=volume_group, extents=extents, tags=tags) |
eafe8130 TL |
1373 | ) |
1374 | return lvs | |
92f5a8d4 TL |
1375 | |
1376 | ||
1377 | def get_device_lvs(device, name_prefix=''): | |
1378 | stdout, stderr, returncode = process.call( | |
1379 | ['pvs'] + LV_CMD_OPTIONS + ['-o', LV_FIELDS, device], | |
1380 | verbose_on_failure=False | |
1381 | ) | |
1382 | lvs = _output_parser(stdout, LV_FIELDS) | |
9f95a23c TL |
1383 | return [Volume(**lv) for lv in lvs if lv['lv_name'] and |
1384 | lv['lv_name'].startswith(name_prefix)] | |
92f5a8d4 TL |
1385 | |
1386 | ||
1387 | ############################################################# | |
1388 | # | |
1389 | # New methods to get PVs, LVs, and VGs. | |
1390 | # Later, these can be easily merged with get_api_* methods | |
1391 | # | |
1392 | ########################################################### | |
1393 | ||
1394 | def convert_filters_to_str(filters): | |
1395 | """ | |
1396 | Convert filter args from dictionary to following format - | |
1397 | filters={filter_name=filter_val,...} | |
1398 | """ | |
1399 | if not filters: | |
1400 | return filters | |
1401 | ||
1402 | filter_arg = '' | |
1403 | for k, v in filters.items(): | |
1404 | filter_arg += k + '=' + v + ',' | |
1405 | # get rid of extra comma at the end | |
1406 | filter_arg = filter_arg[:len(filter_arg) - 1] | |
1407 | ||
1408 | return filter_arg | |
1409 | ||
1410 | def convert_tags_to_str(tags): | |
1411 | """ | |
1412 | Convert tags from dictionary to following format - | |
1413 | tags={tag_name=tag_val,...} | |
1414 | """ | |
1415 | if not tags: | |
1416 | return tags | |
1417 | ||
1418 | tag_arg = 'tags={' | |
1419 | for k, v in tags.items(): | |
1420 | tag_arg += k + '=' + v + ',' | |
1421 | # get rid of extra comma at the end | |
1422 | tag_arg = tag_arg[:len(tag_arg) - 1] + '}' | |
1423 | ||
1424 | return tag_arg | |
1425 | ||
1426 | def make_filters_lvmcmd_ready(filters, tags): | |
1427 | """ | |
1428 | Convert filters (including tags) from dictionary to following format - | |
1429 | filter_name=filter_val...,tags={tag_name=tag_val,...} | |
1430 | ||
1431 | The command will look as follows = | |
1432 | lvs -S filter_name=filter_val...,tags={tag_name=tag_val,...} | |
1433 | """ | |
1434 | filters = convert_filters_to_str(filters) | |
1435 | tags = convert_tags_to_str(tags) | |
1436 | ||
1437 | if filters and tags: | |
1438 | return filters + ',' + tags | |
1439 | if filters and not tags: | |
1440 | return filters | |
1441 | if not filters and tags: | |
1442 | return tags | |
1443 | else: | |
1444 | return '' | |
1445 | ||
1446 | def get_pvs(fields=PV_FIELDS, filters='', tags=None): | |
1447 | """ | |
1448 | Return a list of PVs that are available on the system and match the | |
1449 | filters and tags passed. Argument filters takes a dictionary containing | |
1450 | arguments required by -S option of LVM. Passing a list of LVM tags can be | |
1451 | quite tricky to pass as a dictionary within dictionary, therefore pass | |
1452 | dictionary of tags via tags argument and tricky part will be taken care of | |
1453 | by the helper methods. | |
1454 | ||
1455 | :param fields: string containing list of fields to be displayed by the | |
1456 | pvs command | |
1457 | :param sep: string containing separator to be used between two fields | |
1458 | :param filters: dictionary containing LVM filters | |
1459 | :param tags: dictionary containng LVM tags | |
1460 | :returns: list of class PVolume object representing pvs on the system | |
1461 | """ | |
1462 | filters = make_filters_lvmcmd_ready(filters, tags) | |
1463 | args = ['pvs', '--no-heading', '--readonly', '--separator=";"', '-S', | |
1464 | filters, '-o', fields] | |
1465 | ||
1466 | stdout, stderr, returncode = process.call(args, verbose_on_failure=False) | |
1467 | pvs_report = _output_parser(stdout, fields) | |
1468 | return [PVolume(**pv_report) for pv_report in pvs_report] | |
1469 | ||
1470 | def get_first_pv(fields=PV_FIELDS, filters=None, tags=None): | |
1471 | """ | |
1472 | Wrapper of get_pv meant to be a convenience method to avoid the phrase:: | |
1473 | pvs = get_pvs() | |
1474 | if len(pvs) >= 1: | |
1475 | pv = pvs[0] | |
1476 | """ | |
1477 | pvs = get_pvs(fields=fields, filters=filters, tags=tags) | |
1478 | return pvs[0] if len(pvs) > 0 else [] | |
1479 | ||
1480 | def get_vgs(fields=VG_FIELDS, filters='', tags=None): | |
1481 | """ | |
1482 | Return a list of VGs that are available on the system and match the | |
1483 | filters and tags passed. Argument filters takes a dictionary containing | |
1484 | arguments required by -S option of LVM. Passing a list of LVM tags can be | |
1485 | quite tricky to pass as a dictionary within dictionary, therefore pass | |
1486 | dictionary of tags via tags argument and tricky part will be taken care of | |
1487 | by the helper methods. | |
1488 | ||
1489 | :param fields: string containing list of fields to be displayed by the | |
1490 | vgs command | |
1491 | :param sep: string containing separator to be used between two fields | |
1492 | :param filters: dictionary containing LVM filters | |
1493 | :param tags: dictionary containng LVM tags | |
1494 | :returns: list of class VolumeGroup object representing vgs on the system | |
1495 | """ | |
1496 | filters = make_filters_lvmcmd_ready(filters, tags) | |
1497 | args = ['vgs'] + VG_CMD_OPTIONS + ['-S', filters, '-o', fields] | |
1498 | ||
1499 | stdout, stderr, returncode = process.call(args, verbose_on_failure=False) | |
1500 | vgs_report =_output_parser(stdout, fields) | |
1501 | return [VolumeGroup(**vg_report) for vg_report in vgs_report] | |
1502 | ||
1503 | def get_first_vg(fields=VG_FIELDS, filters=None, tags=None): | |
1504 | """ | |
1505 | Wrapper of get_vg meant to be a convenience method to avoid the phrase:: | |
1506 | vgs = get_vgs() | |
1507 | if len(vgs) >= 1: | |
1508 | vg = vgs[0] | |
1509 | """ | |
1510 | vgs = get_vgs(fields=fields, filters=filters, tags=tags) | |
1511 | return vgs[0] if len(vgs) > 0 else [] | |
1512 | ||
1513 | def get_lvs(fields=LV_FIELDS, filters='', tags=None): | |
1514 | """ | |
1515 | Return a list of LVs that are available on the system and match the | |
1516 | filters and tags passed. Argument filters takes a dictionary containing | |
1517 | arguments required by -S option of LVM. Passing a list of LVM tags can be | |
1518 | quite tricky to pass as a dictionary within dictionary, therefore pass | |
1519 | dictionary of tags via tags argument and tricky part will be taken care of | |
1520 | by the helper methods. | |
1521 | ||
1522 | :param fields: string containing list of fields to be displayed by the | |
1523 | lvs command | |
1524 | :param sep: string containing separator to be used between two fields | |
1525 | :param filters: dictionary containing LVM filters | |
1526 | :param tags: dictionary containng LVM tags | |
1527 | :returns: list of class Volume object representing LVs on the system | |
1528 | """ | |
1529 | filters = make_filters_lvmcmd_ready(filters, tags) | |
1530 | args = ['lvs'] + LV_CMD_OPTIONS + ['-S', filters, '-o', fields] | |
1531 | ||
1532 | stdout, stderr, returncode = process.call(args, verbose_on_failure=False) | |
1533 | lvs_report = _output_parser(stdout, fields) | |
1534 | return [Volume(**lv_report) for lv_report in lvs_report] | |
1535 | ||
1536 | def get_first_lv(fields=LV_FIELDS, filters=None, tags=None): | |
1537 | """ | |
1538 | Wrapper of get_lv meant to be a convenience method to avoid the phrase:: | |
1539 | lvs = get_lvs() | |
1540 | if len(lvs) >= 1: | |
1541 | lv = lvs[0] | |
1542 | """ | |
1543 | lvs = get_lvs(fields=fields, filters=filters, tags=tags) | |
1544 | return lvs[0] if len(lvs) > 0 else [] |