]>
Commit | Line | Data |
---|---|---|
d2e6a577 FG |
1 | """ |
2 | API for CRUD lvm tag operations. Follows the Ceph LVM tag naming convention | |
3 | that prefixes tags with ``ceph.`` and uses ``=`` for assignment, and provides | |
4 | set of utilities for interacting with LVM. | |
5 | """ | |
d2e6a577 FG |
6 | from ceph_volume import process |
7 | from ceph_volume.exceptions import MultipleLVsError, MultipleVGsError | |
8 | ||
9 | ||
b5b8bbf5 FG |
10 | def _output_parser(output, fields): |
11 | """ | |
12 | Newer versions of LVM allow ``--reportformat=json``, but older versions, | |
13 | like the one included in Xenial do not. LVM has the ability to filter and | |
14 | format its output so we assume the output will be in a format this parser | |
15 | can handle (using ',' as a delimiter) | |
16 | ||
17 | :param fields: A string, possibly using ',' to group many items, as it | |
18 | would be used on the CLI | |
19 | :param output: The CLI output from the LVM call | |
20 | """ | |
21 | field_items = fields.split(',') | |
22 | report = [] | |
23 | for line in output: | |
24 | # clear the leading/trailing whitespace | |
25 | line = line.strip() | |
26 | ||
27 | # remove the extra '"' in each field | |
28 | line = line.replace('"', '') | |
29 | ||
30 | # prevent moving forward with empty contents | |
31 | if not line: | |
32 | continue | |
33 | ||
34 | # spliting on ';' because that is what the lvm call uses as | |
35 | # '--separator' | |
36 | output_items = [i.strip() for i in line.split(';')] | |
37 | # map the output to the fiels | |
38 | report.append( | |
39 | dict(zip(field_items, output_items)) | |
40 | ) | |
41 | ||
42 | return report | |
43 | ||
44 | ||
d2e6a577 FG |
45 | def parse_tags(lv_tags): |
46 | """ | |
47 | Return a dictionary mapping of all the tags associated with | |
48 | a Volume from the comma-separated tags coming from the LVM API | |
49 | ||
50 | Input look like:: | |
51 | ||
52 | "ceph.osd_fsid=aaa-fff-bbbb,ceph.osd_id=0" | |
53 | ||
54 | For the above example, the expected return value would be:: | |
55 | ||
56 | { | |
57 | "ceph.osd_fsid": "aaa-fff-bbbb", | |
58 | "ceph.osd_id": "0" | |
59 | } | |
60 | """ | |
61 | if not lv_tags: | |
62 | return {} | |
63 | tag_mapping = {} | |
64 | tags = lv_tags.split(',') | |
65 | for tag_assignment in tags: | |
66 | key, value = tag_assignment.split('=', 1) | |
67 | tag_mapping[key] = value | |
68 | ||
69 | return tag_mapping | |
70 | ||
71 | ||
72 | def get_api_vgs(): | |
73 | """ | |
b5b8bbf5 FG |
74 | Return the list of group volumes available in the system using flags to |
75 | include common metadata associated with them | |
d2e6a577 | 76 | |
b5b8bbf5 | 77 | Command and sample delimeted output, should look like:: |
d2e6a577 | 78 | |
b5b8bbf5 FG |
79 | $ sudo vgs --noheadings --separator=';' \ |
80 | -o vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free | |
81 | ubuntubox-vg;1;2;0;wz--n-;299.52g;12.00m | |
82 | osd_vg;3;1;0;wz--n-;29.21g;9.21g | |
d2e6a577 FG |
83 | |
84 | """ | |
b5b8bbf5 | 85 | fields = 'vg_name,pv_count,lv_count,snap_count,vg_attr,vg_size,vg_free' |
d2e6a577 | 86 | stdout, stderr, returncode = process.call( |
b5b8bbf5 | 87 | ['sudo', 'vgs', '--noheadings', '--separator=";"', '-o', fields] |
d2e6a577 | 88 | ) |
b5b8bbf5 | 89 | return _output_parser(stdout, fields) |
d2e6a577 FG |
90 | |
91 | ||
92 | def get_api_lvs(): | |
93 | """ | |
94 | Return the list of logical volumes available in the system using flags to include common | |
95 | metadata associated with them | |
96 | ||
b5b8bbf5 | 97 | Command and delimeted output, should look like:: |
d2e6a577 | 98 | |
b5b8bbf5 FG |
99 | $ sudo lvs --noheadings --separator=';' -o lv_tags,lv_path,lv_name,vg_name |
100 | ;/dev/ubuntubox-vg/root;root;ubuntubox-vg | |
101 | ;/dev/ubuntubox-vg/swap_1;swap_1;ubuntubox-vg | |
d2e6a577 FG |
102 | |
103 | """ | |
b5b8bbf5 | 104 | fields = 'lv_tags,lv_path,lv_name,vg_name' |
d2e6a577 | 105 | stdout, stderr, returncode = process.call( |
b5b8bbf5 FG |
106 | ['sudo', 'lvs', '--noheadings', '--separator=";"', '-o', fields] |
107 | ) | |
108 | return _output_parser(stdout, fields) | |
d2e6a577 FG |
109 | |
110 | ||
111 | def get_lv(lv_name=None, vg_name=None, lv_path=None, lv_tags=None): | |
112 | """ | |
113 | Return a matching lv for the current system, requiring ``lv_name``, | |
114 | ``vg_name``, ``lv_path`` or ``tags``. Raises an error if more than one lv | |
115 | is found. | |
116 | ||
117 | It is useful to use ``tags`` when trying to find a specific logical volume, | |
118 | but it can also lead to multiple lvs being found, since a lot of metadata | |
119 | is shared between lvs of a distinct OSD. | |
120 | """ | |
121 | if not any([lv_name, vg_name, lv_path, lv_tags]): | |
122 | return None | |
123 | lvs = Volumes() | |
124 | return lvs.get(lv_name=lv_name, vg_name=vg_name, lv_path=lv_path, lv_tags=lv_tags) | |
125 | ||
126 | ||
127 | def create_lv(name, group, size=None, **tags): | |
128 | """ | |
129 | Create a Logical Volume in a Volume Group. Command looks like:: | |
130 | ||
131 | lvcreate -L 50G -n gfslv vg0 | |
132 | ||
133 | ``name``, ``group``, and ``size`` are required. Tags are optional and are "translated" to include | |
134 | the prefixes for the Ceph LVM tag API. | |
135 | ||
136 | """ | |
137 | # XXX add CEPH_VOLUME_LVM_DEBUG to enable -vvvv on lv operations | |
138 | type_path_tag = { | |
139 | 'journal': 'ceph.journal_device', | |
140 | 'data': 'ceph.data_device', | |
141 | 'block': 'ceph.block', | |
142 | 'wal': 'ceph.wal', | |
143 | 'db': 'ceph.db', | |
144 | 'lockbox': 'ceph.lockbox_device', | |
145 | } | |
146 | if size: | |
147 | process.run([ | |
148 | 'sudo', | |
149 | 'lvcreate', | |
150 | '--yes', | |
151 | '-L', | |
152 | '%sG' % size, | |
153 | '-n', name, group | |
154 | ]) | |
155 | # create the lv with all the space available, this is needed because the | |
156 | # system call is different for LVM | |
157 | else: | |
158 | process.run([ | |
159 | 'sudo', | |
160 | 'lvcreate', | |
161 | '--yes', | |
162 | '-l', | |
163 | '100%FREE', | |
164 | '-n', name, group | |
165 | ]) | |
166 | ||
167 | lv = get_lv(lv_name=name, vg_name=group) | |
168 | ceph_tags = {} | |
169 | for k, v in tags.items(): | |
170 | ceph_tags['ceph.%s' % k] = v | |
171 | lv.set_tags(ceph_tags) | |
172 | ||
173 | # when creating a distinct type, the caller doesn't know what the path will | |
174 | # be so this function will set it after creation using the mapping | |
175 | path_tag = type_path_tag[tags['type']] | |
176 | lv.set_tags( | |
177 | {path_tag: lv.lv_path} | |
178 | ) | |
179 | return lv | |
180 | ||
181 | ||
182 | def get_vg(vg_name=None, vg_tags=None): | |
183 | """ | |
184 | Return a matching vg for the current system, requires ``vg_name`` or | |
185 | ``tags``. Raises an error if more than one vg is found. | |
186 | ||
187 | It is useful to use ``tags`` when trying to find a specific volume group, | |
188 | but it can also lead to multiple vgs being found. | |
189 | """ | |
190 | if not any([vg_name, vg_tags]): | |
191 | return None | |
192 | vgs = VolumeGroups() | |
193 | return vgs.get(vg_name=vg_name, vg_tags=vg_tags) | |
194 | ||
195 | ||
196 | class VolumeGroups(list): | |
197 | """ | |
198 | A list of all known volume groups for the current system, with the ability | |
199 | to filter them via keyword arguments. | |
200 | """ | |
201 | ||
202 | def __init__(self): | |
203 | self._populate() | |
204 | ||
205 | def _populate(self): | |
206 | # get all the vgs in the current system | |
207 | for vg_item in get_api_vgs(): | |
208 | self.append(VolumeGroup(**vg_item)) | |
209 | ||
210 | def _purge(self): | |
211 | """ | |
212 | Deplete all the items in the list, used internally only so that we can | |
213 | dynamically allocate the items when filtering without the concern of | |
214 | messing up the contents | |
215 | """ | |
216 | self[:] = [] | |
217 | ||
218 | def _filter(self, vg_name=None, vg_tags=None): | |
219 | """ | |
220 | The actual method that filters using a new list. Useful so that other | |
221 | methods that do not want to alter the contents of the list (e.g. | |
222 | ``self.find``) can operate safely. | |
223 | ||
224 | .. note:: ``vg_tags`` is not yet implemented | |
225 | """ | |
226 | filtered = [i for i in self] | |
227 | if vg_name: | |
228 | filtered = [i for i in filtered if i.vg_name == vg_name] | |
229 | ||
230 | # at this point, `filtered` has either all the volumes in self or is an | |
231 | # actual filtered list if any filters were applied | |
232 | if vg_tags: | |
233 | tag_filtered = [] | |
234 | for k, v in vg_tags.items(): | |
235 | for volume in filtered: | |
236 | if volume.tags.get(k) == str(v): | |
237 | if volume not in tag_filtered: | |
238 | tag_filtered.append(volume) | |
239 | # return the tag_filtered volumes here, the `filtered` list is no | |
240 | # longer useable | |
241 | return tag_filtered | |
242 | ||
243 | return filtered | |
244 | ||
245 | def filter(self, vg_name=None, vg_tags=None): | |
246 | """ | |
247 | Filter out groups on top level attributes like ``vg_name`` or by | |
248 | ``vg_tags`` where a dict is required. For example, to find a Ceph group | |
249 | with dmcache as the type, the filter would look like:: | |
250 | ||
251 | vg_tags={'ceph.type': 'dmcache'} | |
252 | ||
253 | .. warning:: These tags are not documented because they are currently | |
254 | unused, but are here to maintain API consistency | |
255 | """ | |
256 | if not any([vg_name, vg_tags]): | |
257 | raise TypeError('.filter() requires vg_name or vg_tags (none given)') | |
258 | # first find the filtered volumes with the values in self | |
259 | filtered_groups = self._filter( | |
260 | vg_name=vg_name, | |
261 | vg_tags=vg_tags | |
262 | ) | |
263 | # then purge everything | |
264 | self._purge() | |
265 | # and add the filtered items | |
266 | self.extend(filtered_groups) | |
267 | ||
268 | def get(self, vg_name=None, vg_tags=None): | |
269 | """ | |
270 | This is a bit expensive, since it will try to filter out all the | |
271 | matching items in the list, filter them out applying anything that was | |
272 | added and return the matching item. | |
273 | ||
274 | This method does *not* alter the list, and it will raise an error if | |
275 | multiple VGs are matched | |
276 | ||
277 | It is useful to use ``tags`` when trying to find a specific volume group, | |
278 | but it can also lead to multiple vgs being found (although unlikely) | |
279 | """ | |
280 | if not any([vg_name, vg_tags]): | |
281 | return None | |
282 | vgs = self._filter( | |
283 | vg_name=vg_name, | |
284 | vg_tags=vg_tags | |
285 | ) | |
286 | if not vgs: | |
287 | return None | |
288 | if len(vgs) > 1: | |
289 | # this is probably never going to happen, but it is here to keep | |
290 | # the API code consistent | |
291 | raise MultipleVGsError(vg_name) | |
292 | return vgs[0] | |
293 | ||
294 | ||
295 | class Volumes(list): | |
296 | """ | |
297 | A list of all known (logical) volumes for the current system, with the ability | |
298 | to filter them via keyword arguments. | |
299 | """ | |
300 | ||
301 | def __init__(self): | |
302 | self._populate() | |
303 | ||
304 | def _populate(self): | |
305 | # get all the lvs in the current system | |
306 | for lv_item in get_api_lvs(): | |
307 | self.append(Volume(**lv_item)) | |
308 | ||
309 | def _purge(self): | |
310 | """ | |
311 | Deplete all the items in the list, used internally only so that we can | |
312 | dynamically allocate the items when filtering without the concern of | |
313 | messing up the contents | |
314 | """ | |
315 | self[:] = [] | |
316 | ||
317 | def _filter(self, lv_name=None, vg_name=None, lv_path=None, lv_tags=None): | |
318 | """ | |
319 | The actual method that filters using a new list. Useful so that other | |
320 | methods that do not want to alter the contents of the list (e.g. | |
321 | ``self.find``) can operate safely. | |
322 | """ | |
323 | filtered = [i for i in self] | |
324 | if lv_name: | |
325 | filtered = [i for i in filtered if i.lv_name == lv_name] | |
326 | ||
327 | if vg_name: | |
328 | filtered = [i for i in filtered if i.vg_name == vg_name] | |
329 | ||
330 | if lv_path: | |
331 | filtered = [i for i in filtered if i.lv_path == lv_path] | |
332 | ||
333 | # at this point, `filtered` has either all the volumes in self or is an | |
334 | # actual filtered list if any filters were applied | |
335 | if lv_tags: | |
336 | tag_filtered = [] | |
337 | for k, v in lv_tags.items(): | |
338 | for volume in filtered: | |
339 | if volume.tags.get(k) == str(v): | |
340 | if volume not in tag_filtered: | |
341 | tag_filtered.append(volume) | |
342 | # return the tag_filtered volumes here, the `filtered` list is no | |
343 | # longer useable | |
344 | return tag_filtered | |
345 | ||
346 | return filtered | |
347 | ||
348 | def filter(self, lv_name=None, vg_name=None, lv_path=None, lv_tags=None): | |
349 | """ | |
350 | Filter out volumes on top level attributes like ``lv_name`` or by | |
351 | ``lv_tags`` where a dict is required. For example, to find a volume | |
352 | that has an OSD ID of 0, the filter would look like:: | |
353 | ||
354 | lv_tags={'ceph.osd_id': '0'} | |
355 | ||
356 | """ | |
357 | if not any([lv_name, vg_name, lv_path, lv_tags]): | |
358 | raise TypeError('.filter() requires lv_name, vg_name, lv_path, or tags (none given)') | |
359 | # first find the filtered volumes with the values in self | |
360 | filtered_volumes = self._filter( | |
361 | lv_name=lv_name, | |
362 | vg_name=vg_name, | |
363 | lv_path=lv_path, | |
364 | lv_tags=lv_tags | |
365 | ) | |
366 | # then purge everything | |
367 | self._purge() | |
368 | # and add the filtered items | |
369 | self.extend(filtered_volumes) | |
370 | ||
371 | def get(self, lv_name=None, vg_name=None, lv_path=None, lv_tags=None): | |
372 | """ | |
373 | This is a bit expensive, since it will try to filter out all the | |
374 | matching items in the list, filter them out applying anything that was | |
375 | added and return the matching item. | |
376 | ||
377 | This method does *not* alter the list, and it will raise an error if | |
378 | multiple LVs are matched | |
379 | ||
380 | It is useful to use ``tags`` when trying to find a specific logical volume, | |
381 | but it can also lead to multiple lvs being found, since a lot of metadata | |
382 | is shared between lvs of a distinct OSD. | |
383 | """ | |
384 | if not any([lv_name, vg_name, lv_path, lv_tags]): | |
385 | return None | |
386 | lvs = self._filter( | |
387 | lv_name=lv_name, | |
388 | vg_name=vg_name, | |
389 | lv_path=lv_path, | |
390 | lv_tags=lv_tags | |
391 | ) | |
392 | if not lvs: | |
393 | return None | |
394 | if len(lvs) > 1: | |
395 | raise MultipleLVsError(lv_name, lv_path) | |
396 | return lvs[0] | |
397 | ||
398 | ||
399 | class VolumeGroup(object): | |
400 | """ | |
401 | Represents an LVM group, with some top-level attributes like ``vg_name`` | |
402 | """ | |
403 | ||
404 | def __init__(self, **kw): | |
405 | for k, v in kw.items(): | |
406 | setattr(self, k, v) | |
407 | self.name = kw['vg_name'] | |
408 | self.tags = parse_tags(kw.get('vg_tags', '')) | |
409 | ||
410 | def __str__(self): | |
411 | return '<%s>' % self.name | |
412 | ||
413 | def __repr__(self): | |
414 | return self.__str__() | |
415 | ||
416 | ||
417 | class Volume(object): | |
418 | """ | |
419 | Represents a Logical Volume from LVM, with some top-level attributes like | |
420 | ``lv_name`` and parsed tags as a dictionary of key/value pairs. | |
421 | """ | |
422 | ||
423 | def __init__(self, **kw): | |
424 | for k, v in kw.items(): | |
425 | setattr(self, k, v) | |
426 | self.lv_api = kw | |
427 | self.name = kw['lv_name'] | |
428 | self.tags = parse_tags(kw['lv_tags']) | |
429 | ||
430 | def __str__(self): | |
431 | return '<%s>' % self.lv_api['lv_path'] | |
432 | ||
433 | def __repr__(self): | |
434 | return self.__str__() | |
435 | ||
436 | def set_tags(self, tags): | |
437 | """ | |
438 | :param tags: A dictionary of tag names and values, like:: | |
439 | ||
440 | { | |
441 | "ceph.osd_fsid": "aaa-fff-bbbb", | |
442 | "ceph.osd_id": "0" | |
443 | } | |
444 | ||
445 | At the end of all modifications, the tags are refreshed to reflect | |
446 | LVM's most current view. | |
447 | """ | |
448 | for k, v in tags.items(): | |
449 | self.set_tag(k, v) | |
450 | # after setting all the tags, refresh them for the current object, use the | |
451 | # lv_* identifiers to filter because those shouldn't change | |
452 | lv_object = get_lv(lv_name=self.lv_name, lv_path=self.lv_path) | |
453 | self.tags = lv_object.tags | |
454 | ||
455 | def set_tag(self, key, value): | |
456 | """ | |
457 | Set the key/value pair as an LVM tag. Does not "refresh" the values of | |
458 | the current object for its tags. Meant to be a "fire and forget" type | |
459 | of modification. | |
460 | """ | |
461 | # remove it first if it exists | |
462 | if self.tags.get(key): | |
463 | current_value = self.tags[key] | |
464 | tag = "%s=%s" % (key, current_value) | |
465 | process.call(['sudo', 'lvchange', '--deltag', tag, self.lv_api['lv_path']]) | |
466 | ||
467 | process.call( | |
468 | [ | |
469 | 'sudo', 'lvchange', | |
470 | '--addtag', '%s=%s' % (key, value), self.lv_path | |
471 | ] | |
472 | ) |