]>
Commit | Line | Data |
---|---|---|
d2e6a577 FG |
1 | from __future__ import print_function |
2 | import json | |
b32b8144 | 3 | import logging |
3efd9988 | 4 | import uuid |
d2e6a577 FG |
5 | from textwrap import dedent |
6 | from ceph_volume.util import prepare as prepare_utils | |
b32b8144 | 7 | from ceph_volume.util import encryption as encryption_utils |
181888fb | 8 | from ceph_volume.util import system, disk |
3a9019d9 | 9 | from ceph_volume.util.arg_validators import exclude_group_options |
181888fb | 10 | from ceph_volume import conf, decorators, terminal |
3efd9988 | 11 | from ceph_volume.api import lvm as api |
b32b8144 | 12 | from .common import prepare_parser, rollback_osd |
d2e6a577 FG |
13 | |
14 | ||
b32b8144 FG |
15 | logger = logging.getLogger(__name__) |
16 | ||
17 | ||
18 | def prepare_dmcrypt(key, device, device_type, tags): | |
19 | """ | |
20 | Helper for devices that are encrypted. The operations needed for | |
21 | block, db, wal, or data/journal devices are all the same | |
22 | """ | |
23 | if not device: | |
24 | return '' | |
25 | tag_name = 'ceph.%s_uuid' % device_type | |
26 | uuid = tags[tag_name] | |
27 | # format data device | |
28 | encryption_utils.luks_format( | |
29 | key, | |
30 | device | |
31 | ) | |
32 | encryption_utils.luks_open( | |
33 | key, | |
34 | device, | |
35 | uuid | |
36 | ) | |
37 | ||
38 | return '/dev/mapper/%s' % uuid | |
39 | ||
40 | ||
41 | def prepare_filestore(device, journal, secrets, tags, osd_id, fsid): | |
d2e6a577 | 42 | """ |
3efd9988 | 43 | :param device: The name of the logical volume to work with |
d2e6a577 FG |
44 | :param journal: similar to device but can also be a regular/plain disk |
45 | :param secrets: A dict with the secrets needed to create the osd (e.g. cephx) | |
46 | :param id_: The OSD id | |
47 | :param fsid: The OSD fsid, also known as the OSD UUID | |
48 | """ | |
49 | cephx_secret = secrets.get('cephx_secret', prepare_utils.create_key()) | |
d2e6a577 | 50 | |
b32b8144 FG |
51 | # encryption-only operations |
52 | if secrets.get('dmcrypt_key'): | |
53 | # format and open ('decrypt' devices) and re-assign the device and journal | |
54 | # variables so that the rest of the process can use the mapper paths | |
55 | key = secrets['dmcrypt_key'] | |
56 | device = prepare_dmcrypt(key, device, 'data', tags) | |
57 | journal = prepare_dmcrypt(key, journal, 'journal', tags) | |
58 | ||
d2e6a577 | 59 | # create the directory |
3efd9988 | 60 | prepare_utils.create_osd_path(osd_id) |
d2e6a577 FG |
61 | # format the device |
62 | prepare_utils.format_device(device) | |
63 | # mount the data device | |
64 | prepare_utils.mount_osd(device, osd_id) | |
65 | # symlink the journal | |
66 | prepare_utils.link_journal(journal, osd_id) | |
67 | # get the latest monmap | |
68 | prepare_utils.get_monmap(osd_id) | |
69 | # prepare the osd filesystem | |
3efd9988 | 70 | prepare_utils.osd_mkfs_filestore(osd_id, fsid) |
d2e6a577 FG |
71 | # write the OSD keyring if it doesn't exist already |
72 | prepare_utils.write_keyring(osd_id, cephx_secret) | |
b32b8144 FG |
73 | if secrets.get('dmcrypt_key'): |
74 | # if the device is going to get activated right away, this can be done | |
75 | # here, otherwise it will be recreated | |
76 | encryption_utils.write_lockbox_keyring( | |
77 | osd_id, | |
78 | fsid, | |
79 | tags['ceph.cephx_lockbox_secret'] | |
80 | ) | |
d2e6a577 FG |
81 | |
82 | ||
b32b8144 | 83 | def prepare_bluestore(block, wal, db, secrets, tags, osd_id, fsid): |
3efd9988 FG |
84 | """ |
85 | :param block: The name of the logical volume for the bluestore data | |
86 | :param wal: a regular/plain disk or logical volume, to be used for block.wal | |
87 | :param db: a regular/plain disk or logical volume, to be used for block.db | |
88 | :param secrets: A dict with the secrets needed to create the osd (e.g. cephx) | |
89 | :param id_: The OSD id | |
90 | :param fsid: The OSD fsid, also known as the OSD UUID | |
91 | """ | |
92 | cephx_secret = secrets.get('cephx_secret', prepare_utils.create_key()) | |
b32b8144 FG |
93 | # encryption-only operations |
94 | if secrets.get('dmcrypt_key'): | |
95 | # If encrypted, there is no need to create the lockbox keyring file because | |
96 | # bluestore re-creates the files and does not have support for other files | |
97 | # like the custom lockbox one. This will need to be done on activation. | |
98 | # format and open ('decrypt' devices) and re-assign the device and journal | |
99 | # variables so that the rest of the process can use the mapper paths | |
100 | key = secrets['dmcrypt_key'] | |
101 | block = prepare_dmcrypt(key, block, 'block', tags) | |
102 | wal = prepare_dmcrypt(key, wal, 'wal', tags) | |
103 | db = prepare_dmcrypt(key, db, 'db', tags) | |
3efd9988 | 104 | |
3efd9988 FG |
105 | # create the directory |
106 | prepare_utils.create_osd_path(osd_id, tmpfs=True) | |
107 | # symlink the block | |
108 | prepare_utils.link_block(block, osd_id) | |
109 | # get the latest monmap | |
110 | prepare_utils.get_monmap(osd_id) | |
111 | # write the OSD keyring if it doesn't exist already | |
112 | prepare_utils.write_keyring(osd_id, cephx_secret) | |
113 | # prepare the osd filesystem | |
114 | prepare_utils.osd_mkfs_bluestore( | |
115 | osd_id, fsid, | |
116 | keyring=cephx_secret, | |
117 | wal=wal, | |
118 | db=db | |
119 | ) | |
d2e6a577 FG |
120 | |
121 | ||
122 | class Prepare(object): | |
123 | ||
124 | help = 'Format an LVM device and associate it with an OSD' | |
125 | ||
126 | def __init__(self, argv): | |
127 | self.argv = argv | |
b32b8144 | 128 | self.osd_id = None |
d2e6a577 | 129 | |
3efd9988 | 130 | def get_ptuuid(self, argument): |
181888fb FG |
131 | uuid = disk.get_partuuid(argument) |
132 | if not uuid: | |
133 | terminal.error('blkid could not detect a PARTUUID for device: %s' % argument) | |
3efd9988 | 134 | raise RuntimeError('unable to use device') |
181888fb FG |
135 | return uuid |
136 | ||
3efd9988 | 137 | def get_lv(self, argument): |
b5b8bbf5 | 138 | """ |
3efd9988 FG |
139 | Perform some parsing of the command-line value so that the process |
140 | can determine correctly if it got a device path or an lv. | |
141 | ||
142 | :param argument: The command-line value that will need to be split to | |
143 | retrieve the actual lv | |
b5b8bbf5 FG |
144 | """ |
145 | try: | |
146 | vg_name, lv_name = argument.split('/') | |
147 | except (ValueError, AttributeError): | |
148 | return None | |
149 | return api.get_lv(lv_name=lv_name, vg_name=vg_name) | |
150 | ||
3efd9988 FG |
151 | def setup_device(self, device_type, device_name, tags): |
152 | """ | |
153 | Check if ``device`` is an lv, if so, set the tags, making sure to | |
154 | update the tags with the lv_uuid and lv_path which the incoming tags | |
155 | will not have. | |
156 | ||
157 | If the device is not a logical volume, then retrieve the partition UUID | |
158 | by querying ``blkid`` | |
159 | """ | |
160 | if device_name is None: | |
161 | return '', '', tags | |
162 | tags['ceph.type'] = device_type | |
163 | lv = self.get_lv(device_name) | |
164 | if lv: | |
165 | uuid = lv.lv_uuid | |
166 | path = lv.lv_path | |
167 | tags['ceph.%s_uuid' % device_type] = uuid | |
168 | tags['ceph.%s_device' % device_type] = path | |
169 | lv.set_tags(tags) | |
170 | else: | |
171 | # otherwise assume this is a regular disk partition | |
172 | uuid = self.get_ptuuid(device_name) | |
173 | path = device_name | |
174 | tags['ceph.%s_uuid' % device_type] = uuid | |
175 | tags['ceph.%s_device' % device_type] = path | |
176 | return path, uuid, tags | |
177 | ||
178 | def prepare_device(self, arg, device_type, cluster_fsid, osd_fsid): | |
179 | """ | |
180 | Check if ``arg`` is a device or partition to create an LV out of it | |
181 | with a distinct volume group name, assigning LV tags on it and | |
182 | ultimately, returning the logical volume object. Failing to detect | |
183 | a device or partition will result in error. | |
184 | ||
185 | :param arg: The value of ``--data`` when parsing args | |
186 | :param device_type: Usually, either ``data`` or ``block`` (filestore vs. bluestore) | |
187 | :param cluster_fsid: The cluster fsid/uuid | |
188 | :param osd_fsid: The OSD fsid/uuid | |
189 | """ | |
190 | if disk.is_partition(arg) or disk.is_device(arg): | |
191 | # we must create a vg, and then a single lv | |
192 | vg_name = "ceph-%s" % cluster_fsid | |
193 | if api.get_vg(vg_name=vg_name): | |
194 | # means we already have a group for this, make a different one | |
195 | # XXX this could end up being annoying for an operator, maybe? | |
196 | vg_name = "ceph-%s" % str(uuid.uuid4()) | |
197 | api.create_vg(vg_name, arg) | |
198 | lv_name = "osd-%s-%s" % (device_type, osd_fsid) | |
199 | return api.create_lv( | |
200 | lv_name, | |
201 | vg_name, # the volume group | |
202 | tags={'ceph.type': device_type}) | |
203 | else: | |
204 | error = [ | |
b32b8144 FG |
205 | 'Cannot use device (%s).' % arg, |
206 | 'A vg/lv path or an existing device is needed'] | |
3efd9988 FG |
207 | raise RuntimeError(' '.join(error)) |
208 | ||
209 | raise RuntimeError('no data logical volume found with: %s' % arg) | |
210 | ||
b32b8144 FG |
211 | def safe_prepare(self, args): |
212 | """ | |
213 | An intermediate step between `main()` and `prepare()` so that we can | |
214 | capture the `self.osd_id` in case we need to rollback | |
215 | """ | |
216 | try: | |
217 | self.prepare(args) | |
218 | except Exception: | |
219 | logger.error('lvm prepare was unable to complete') | |
220 | logger.info('will rollback OSD ID creation') | |
221 | rollback_osd(args, self.osd_id) | |
222 | raise | |
223 | terminal.success("ceph-volume lvm prepare successful for: %s" % args.data) | |
224 | ||
d2e6a577 FG |
225 | @decorators.needs_root |
226 | def prepare(self, args): | |
227 | # FIXME we don't allow re-using a keyring, we always generate one for the | |
228 | # OSD, this needs to be fixed. This could either be a file (!) or a string | |
229 | # (!!) or some flags that we would need to compound into a dict so that we | |
230 | # can convert to JSON (!!!) | |
231 | secrets = {'cephx_secret': prepare_utils.create_key()} | |
b32b8144 FG |
232 | cephx_lockbox_secret = '' |
233 | encrypted = 1 if args.dmcrypt else 0 | |
234 | cephx_lockbox_secret = '' if not encrypted else prepare_utils.create_key() | |
235 | ||
236 | if encrypted: | |
237 | secrets['dmcrypt_key'] = encryption_utils.create_dmcrypt_key() | |
238 | secrets['cephx_lockbox_secret'] = cephx_lockbox_secret | |
d2e6a577 FG |
239 | |
240 | cluster_fsid = conf.ceph.get('global', 'fsid') | |
3efd9988 | 241 | osd_fsid = args.osd_fsid or system.generate_uuid() |
b32b8144 FG |
242 | crush_device_class = args.crush_device_class |
243 | if crush_device_class: | |
244 | secrets['crush_device_class'] = crush_device_class | |
245 | # reuse a given ID if it exists, otherwise create a new ID | |
246 | self.osd_id = prepare_utils.create_id(osd_fsid, json.dumps(secrets), osd_id=args.osd_id) | |
247 | tags = { | |
248 | 'ceph.osd_fsid': osd_fsid, | |
249 | 'ceph.osd_id': self.osd_id, | |
250 | 'ceph.cluster_fsid': cluster_fsid, | |
251 | 'ceph.cluster_name': conf.cluster, | |
252 | 'ceph.crush_device_class': crush_device_class, | |
253 | } | |
d2e6a577 | 254 | if args.filestore: |
b5b8bbf5 FG |
255 | if not args.journal: |
256 | raise RuntimeError('--journal is required when using --filestore') | |
b5b8bbf5 | 257 | |
3efd9988 FG |
258 | data_lv = self.get_lv(args.data) |
259 | if not data_lv: | |
260 | data_lv = self.prepare_device(args.data, 'data', cluster_fsid, osd_fsid) | |
181888fb | 261 | |
b32b8144 FG |
262 | tags['ceph.data_device'] = data_lv.lv_path |
263 | tags['ceph.data_uuid'] = data_lv.lv_uuid | |
264 | tags['ceph.cephx_lockbox_secret'] = cephx_lockbox_secret | |
265 | tags['ceph.encrypted'] = encrypted | |
3efd9988 FG |
266 | |
267 | journal_device, journal_uuid, tags = self.setup_device('journal', args.journal, tags) | |
268 | ||
269 | tags['ceph.type'] = 'data' | |
270 | data_lv.set_tags(tags) | |
d2e6a577 FG |
271 | |
272 | prepare_filestore( | |
273 | data_lv.lv_path, | |
274 | journal_device, | |
275 | secrets, | |
b32b8144 FG |
276 | tags, |
277 | self.osd_id, | |
278 | osd_fsid, | |
d2e6a577 FG |
279 | ) |
280 | elif args.bluestore: | |
3efd9988 FG |
281 | block_lv = self.get_lv(args.data) |
282 | if not block_lv: | |
283 | block_lv = self.prepare_device(args.data, 'block', cluster_fsid, osd_fsid) | |
284 | ||
b32b8144 FG |
285 | tags['ceph.block_device'] = block_lv.lv_path |
286 | tags['ceph.block_uuid'] = block_lv.lv_uuid | |
287 | tags['ceph.cephx_lockbox_secret'] = cephx_lockbox_secret | |
288 | tags['ceph.encrypted'] = encrypted | |
3efd9988 FG |
289 | |
290 | wal_device, wal_uuid, tags = self.setup_device('wal', args.block_wal, tags) | |
291 | db_device, db_uuid, tags = self.setup_device('db', args.block_db, tags) | |
292 | ||
293 | tags['ceph.type'] = 'block' | |
294 | block_lv.set_tags(tags) | |
295 | ||
296 | prepare_bluestore( | |
297 | block_lv.lv_path, | |
298 | wal_device, | |
299 | db_device, | |
300 | secrets, | |
b32b8144 FG |
301 | tags, |
302 | self.osd_id, | |
303 | osd_fsid, | |
3efd9988 | 304 | ) |
d2e6a577 FG |
305 | |
306 | def main(self): | |
307 | sub_command_help = dedent(""" | |
308 | Prepare an OSD by assigning an ID and FSID, registering them with the | |
309 | cluster with an ID and FSID, formatting and mounting the volume, and | |
310 | finally by adding all the metadata to the logical volumes using LVM | |
311 | tags, so that it can later be discovered. | |
312 | ||
313 | Once the OSD is ready, an ad-hoc systemd unit will be enabled so that | |
314 | it can later get activated and the OSD daemon can get started. | |
315 | ||
316 | Most basic Usage looks like (journal will be collocated from the same volume group): | |
317 | ||
318 | ceph-volume lvm prepare --data {volume group name} | |
319 | ||
b32b8144 | 320 | Encryption is supported via dmcrypt and the --dmcrypt flag. |
d2e6a577 FG |
321 | |
322 | Example calls for supported scenarios: | |
323 | ||
324 | Dedicated volume group for Journal(s) | |
325 | ------------------------------------- | |
326 | ||
327 | Existing logical volume (lv) or device: | |
328 | ||
3efd9988 | 329 | ceph-volume lvm prepare --filestore --data {vg/lv} --journal /path/to/device |
d2e6a577 FG |
330 | |
331 | Or: | |
332 | ||
3efd9988 FG |
333 | ceph-volume lvm prepare --filestore --data {vg/lv} --journal {vg/lv} |
334 | ||
335 | Existing block device, that will be made a group and logical volume: | |
336 | ||
337 | ceph-volume lvm prepare --filestore --data /path/to/device --journal {vg/lv} | |
338 | ||
339 | Bluestore | |
340 | --------- | |
341 | ||
342 | Existing logical volume (lv): | |
343 | ||
344 | ceph-volume lvm prepare --bluestore --data {vg/lv} | |
345 | ||
346 | Existing block device, that will be made a group and logical volume: | |
d2e6a577 | 347 | |
3efd9988 | 348 | ceph-volume lvm prepare --bluestore --data /path/to/device |
d2e6a577 | 349 | |
3efd9988 | 350 | Optionally, can consume db and wal devices or logical volumes: |
d2e6a577 | 351 | |
3efd9988 | 352 | ceph-volume lvm prepare --bluestore --data {vg/lv} --block.wal {device} --block-db {vg/lv} |
d2e6a577 FG |
353 | """) |
354 | parser = prepare_parser( | |
355 | prog='ceph-volume lvm prepare', | |
356 | description=sub_command_help, | |
357 | ) | |
358 | if len(self.argv) == 0: | |
359 | print(sub_command_help) | |
360 | return | |
3a9019d9 | 361 | exclude_group_options(parser, argv=self.argv, groups=['filestore', 'bluestore']) |
d2e6a577 | 362 | args = parser.parse_args(self.argv) |
3efd9988 FG |
363 | # Default to bluestore here since defaulting it in add_argument may |
364 | # cause both to be True | |
b32b8144 | 365 | if not args.bluestore and not args.filestore: |
3efd9988 | 366 | args.bluestore = True |
b32b8144 | 367 | self.safe_prepare(args) |