]>
Commit | Line | Data |
---|---|---|
d2e6a577 FG |
1 | from __future__ import print_function |
2 | import argparse | |
3efd9988 FG |
3 | import logging |
4 | import os | |
d2e6a577 | 5 | from textwrap import dedent |
1adf2230 | 6 | from ceph_volume import process, conf, decorators, terminal, __release__ |
181888fb | 7 | from ceph_volume.util import system, disk |
3efd9988 | 8 | from ceph_volume.util import prepare as prepare_utils |
b32b8144 | 9 | from ceph_volume.util import encryption as encryption_utils |
d2e6a577 | 10 | from ceph_volume.systemd import systemctl |
3efd9988 | 11 | from ceph_volume.api import lvm as api |
94b18763 | 12 | from .listing import direct_report |
3efd9988 FG |
13 | |
14 | ||
15 | logger = logging.getLogger(__name__) | |
d2e6a577 FG |
16 | |
17 | ||
94b18763 | 18 | def activate_filestore(lvs, no_systemd=False): |
d2e6a577 FG |
19 | # find the osd |
20 | osd_lv = lvs.get(lv_tags={'ceph.type': 'data'}) | |
3efd9988 FG |
21 | if not osd_lv: |
22 | raise RuntimeError('Unable to find a data LV for filestore activation') | |
b32b8144 | 23 | is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1' |
94b18763 | 24 | is_vdo = osd_lv.tags.get('ceph.vdo', '0') |
b32b8144 | 25 | |
d2e6a577 | 26 | osd_id = osd_lv.tags['ceph.osd_id'] |
3efd9988 | 27 | conf.cluster = osd_lv.tags['ceph.cluster_name'] |
d2e6a577 FG |
28 | # it may have a volume with a journal |
29 | osd_journal_lv = lvs.get(lv_tags={'ceph.type': 'journal'}) | |
30 | # TODO: add sensible error reporting if this is ever the case | |
31 | # blow up with a KeyError if this doesn't exist | |
32 | osd_fsid = osd_lv.tags['ceph.osd_fsid'] | |
33 | if not osd_journal_lv: | |
181888fb FG |
34 | # must be a disk partition, by quering blkid by the uuid we are ensuring that the |
35 | # device path is always correct | |
b32b8144 FG |
36 | journal_uuid = osd_lv.tags['ceph.journal_uuid'] |
37 | osd_journal = disk.get_device_from_partuuid(journal_uuid) | |
d2e6a577 | 38 | else: |
b32b8144 | 39 | journal_uuid = osd_journal_lv.lv_uuid |
181888fb | 40 | osd_journal = osd_lv.tags['ceph.journal_device'] |
d2e6a577 FG |
41 | |
42 | if not osd_journal: | |
43 | raise RuntimeError('unable to detect an lv or device journal for OSD %s' % osd_id) | |
44 | ||
b32b8144 FG |
45 | # this is done here, so that previous checks that ensure path availability |
46 | # and correctness can still be enforced, and report if any issues are found | |
47 | if is_encrypted: | |
48 | lockbox_secret = osd_lv.tags['ceph.cephx_lockbox_secret'] | |
49 | # this keyring writing is idempotent | |
50 | encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret) | |
51 | dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid) | |
52 | encryption_utils.luks_open(dmcrypt_secret, osd_lv.lv_path, osd_lv.lv_uuid) | |
53 | encryption_utils.luks_open(dmcrypt_secret, osd_journal, journal_uuid) | |
54 | ||
55 | osd_journal = '/dev/mapper/%s' % journal_uuid | |
56 | source = '/dev/mapper/%s' % osd_lv.lv_uuid | |
57 | else: | |
58 | source = osd_lv.lv_path | |
94b18763 | 59 | |
d2e6a577 | 60 | # mount the osd |
d2e6a577 | 61 | destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) |
3efd9988 | 62 | if not system.device_is_mounted(source, destination=destination): |
94b18763 | 63 | prepare_utils.mount_osd(source, osd_id, is_vdo=is_vdo) |
d2e6a577 | 64 | |
181888fb FG |
65 | # always re-do the symlink regardless if it exists, so that the journal |
66 | # device path that may have changed can be mapped correctly every time | |
67 | destination = '/var/lib/ceph/osd/%s-%s/journal' % (conf.cluster, osd_id) | |
b32b8144 | 68 | process.run(['ln', '-snf', osd_journal, destination]) |
d2e6a577 FG |
69 | |
70 | # make sure that the journal has proper permissions | |
71 | system.chown(osd_journal) | |
72 | ||
94b18763 FG |
73 | if no_systemd is False: |
74 | # enable the ceph-volume unit for this OSD | |
75 | systemctl.enable_volume(osd_id, osd_fsid, 'lvm') | |
d2e6a577 | 76 | |
1adf2230 AA |
77 | # enable the OSD |
78 | systemctl.enable_osd(osd_id) | |
79 | ||
94b18763 FG |
80 | # start the OSD |
81 | systemctl.start_osd(osd_id) | |
b32b8144 | 82 | terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id) |
d2e6a577 FG |
83 | |
84 | ||
b32b8144 | 85 | def get_osd_device_path(osd_lv, lvs, device_type, dmcrypt_secret=None): |
3efd9988 FG |
86 | """ |
87 | ``device_type`` can be one of ``db``, ``wal`` or ``block`` so that | |
88 | we can query ``lvs`` (a ``Volumes`` object) and fallback to querying the uuid | |
89 | if that is not present. | |
90 | ||
91 | Return a path if possible, failing to do that a ``None``, since some of these devices | |
92 | are optional | |
93 | """ | |
94 | osd_lv = lvs.get(lv_tags={'ceph.type': 'block'}) | |
b32b8144 FG |
95 | is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1' |
96 | logger.debug('Found block device (%s) with encryption: %s', osd_lv.name, is_encrypted) | |
3efd9988 FG |
97 | uuid_tag = 'ceph.%s_uuid' % device_type |
98 | device_uuid = osd_lv.tags.get(uuid_tag) | |
99 | if not device_uuid: | |
100 | return None | |
101 | ||
102 | device_lv = lvs.get(lv_uuid=device_uuid) | |
103 | if device_lv: | |
b32b8144 FG |
104 | if is_encrypted: |
105 | encryption_utils.luks_open(dmcrypt_secret, device_lv.lv_path, device_uuid) | |
106 | return '/dev/mapper/%s' % device_uuid | |
3efd9988 FG |
107 | return device_lv.lv_path |
108 | else: | |
109 | # this could be a regular device, so query it with blkid | |
110 | physical_device = disk.get_device_from_partuuid(device_uuid) | |
b32b8144 FG |
111 | if physical_device and is_encrypted: |
112 | encryption_utils.luks_open(dmcrypt_secret, physical_device, device_uuid) | |
113 | return '/dev/mapper/%s' % device_uuid | |
3efd9988 FG |
114 | return physical_device or None |
115 | return None | |
116 | ||
117 | ||
94b18763 | 118 | def activate_bluestore(lvs, no_systemd=False): |
3efd9988 FG |
119 | # find the osd |
120 | osd_lv = lvs.get(lv_tags={'ceph.type': 'block'}) | |
94b18763 FG |
121 | if not osd_lv: |
122 | raise RuntimeError('could not find a bluestore OSD to activate') | |
b32b8144 FG |
123 | is_encrypted = osd_lv.tags.get('ceph.encrypted', '0') == '1' |
124 | dmcrypt_secret = None | |
3efd9988 FG |
125 | osd_id = osd_lv.tags['ceph.osd_id'] |
126 | conf.cluster = osd_lv.tags['ceph.cluster_name'] | |
127 | osd_fsid = osd_lv.tags['ceph.osd_fsid'] | |
3efd9988 FG |
128 | |
129 | # mount on tmpfs the osd directory | |
130 | osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) | |
131 | if not system.path_is_mounted(osd_path): | |
132 | # mkdir -p and mount as tmpfs | |
133 | prepare_utils.create_osd_path(osd_id, tmpfs=True) | |
134 | # XXX This needs to be removed once ceph-bluestore-tool can deal with | |
135 | # symlinks that exist in the osd dir | |
136 | for link_name in ['block', 'block.db', 'block.wal']: | |
137 | link_path = os.path.join(osd_path, link_name) | |
138 | if os.path.exists(link_path): | |
139 | os.unlink(os.path.join(osd_path, link_name)) | |
b32b8144 FG |
140 | # encryption is handled here, before priming the OSD dir |
141 | if is_encrypted: | |
142 | osd_lv_path = '/dev/mapper/%s' % osd_lv.lv_uuid | |
143 | lockbox_secret = osd_lv.tags['ceph.cephx_lockbox_secret'] | |
144 | encryption_utils.write_lockbox_keyring(osd_id, osd_fsid, lockbox_secret) | |
145 | dmcrypt_secret = encryption_utils.get_dmcrypt_key(osd_id, osd_fsid) | |
146 | encryption_utils.luks_open(dmcrypt_secret, osd_lv.lv_path, osd_lv.lv_uuid) | |
147 | else: | |
148 | osd_lv_path = osd_lv.lv_path | |
149 | ||
150 | db_device_path = get_osd_device_path(osd_lv, lvs, 'db', dmcrypt_secret=dmcrypt_secret) | |
151 | wal_device_path = get_osd_device_path(osd_lv, lvs, 'wal', dmcrypt_secret=dmcrypt_secret) | |
152 | ||
3efd9988 | 153 | # Once symlinks are removed, the osd dir can be 'primed again. |
1adf2230 | 154 | prime_command = [ |
b32b8144 FG |
155 | 'ceph-bluestore-tool', '--cluster=%s' % conf.cluster, |
156 | 'prime-osd-dir', '--dev', osd_lv_path, | |
1adf2230 AA |
157 | '--path', osd_path] |
158 | ||
159 | if __release__ != "luminous": | |
160 | # mon-config changes are not available in Luminous | |
161 | prime_command.append('--no-mon-config') | |
162 | ||
163 | process.run(prime_command) | |
3efd9988 FG |
164 | # always re-do the symlink regardless if it exists, so that the block, |
165 | # block.wal, and block.db devices that may have changed can be mapped | |
166 | # correctly every time | |
b32b8144 | 167 | process.run(['ln', '-snf', osd_lv_path, os.path.join(osd_path, 'block')]) |
3efd9988 FG |
168 | system.chown(os.path.join(osd_path, 'block')) |
169 | system.chown(osd_path) | |
170 | if db_device_path: | |
171 | destination = os.path.join(osd_path, 'block.db') | |
b32b8144 | 172 | process.run(['ln', '-snf', db_device_path, destination]) |
3efd9988 | 173 | system.chown(db_device_path) |
28e407b8 | 174 | system.chown(destination) |
3efd9988 FG |
175 | if wal_device_path: |
176 | destination = os.path.join(osd_path, 'block.wal') | |
b32b8144 | 177 | process.run(['ln', '-snf', wal_device_path, destination]) |
3efd9988 | 178 | system.chown(wal_device_path) |
28e407b8 | 179 | system.chown(destination) |
3efd9988 | 180 | |
94b18763 FG |
181 | if no_systemd is False: |
182 | # enable the ceph-volume unit for this OSD | |
183 | systemctl.enable_volume(osd_id, osd_fsid, 'lvm') | |
3efd9988 | 184 | |
1adf2230 AA |
185 | # enable the OSD |
186 | systemctl.enable_osd(osd_id) | |
187 | ||
94b18763 FG |
188 | # start the OSD |
189 | systemctl.start_osd(osd_id) | |
b32b8144 | 190 | terminal.success("ceph-volume lvm activate successful for osd ID: %s" % osd_id) |
d2e6a577 FG |
191 | |
192 | ||
193 | class Activate(object): | |
194 | ||
195 | help = 'Discover and mount the LVM device associated with an OSD ID and start the Ceph OSD' | |
196 | ||
197 | def __init__(self, argv): | |
198 | self.argv = argv | |
199 | ||
200 | @decorators.needs_root | |
94b18763 FG |
201 | def activate_all(self, args): |
202 | listed_osds = direct_report() | |
203 | osds = {} | |
204 | for osd_id, devices in listed_osds.items(): | |
205 | # the metadata for all devices in each OSD will contain | |
206 | # the FSID which is required for activation | |
207 | for device in devices: | |
208 | fsid = device.get('tags', {}).get('ceph.osd_fsid') | |
209 | if fsid: | |
210 | osds[fsid] = osd_id | |
211 | break | |
212 | if not osds: | |
213 | terminal.warning('Was unable to find any OSDs to activate') | |
214 | terminal.warning('Verify OSDs are present with "ceph-volume lvm list"') | |
215 | return | |
216 | for osd_fsid, osd_id in osds.items(): | |
217 | if systemctl.osd_is_active(osd_id): | |
218 | terminal.warning( | |
219 | 'OSD ID %s FSID %s process is active. Skipping activation' % (osd_id, osd_fsid) | |
220 | ) | |
221 | else: | |
222 | terminal.info('Activating OSD ID %s FSID %s' % (osd_id, osd_fsid)) | |
223 | self.activate(args, osd_id=osd_id, osd_fsid=osd_fsid) | |
224 | ||
225 | @decorators.needs_root | |
226 | def activate(self, args, osd_id=None, osd_fsid=None): | |
227 | """ | |
228 | :param args: The parsed arguments coming from the CLI | |
229 | :param osd_id: When activating all, this gets populated with an existing OSD ID | |
230 | :param osd_fsid: When activating all, this gets populated with an existing OSD FSID | |
231 | """ | |
232 | osd_id = osd_id if osd_id is not None else args.osd_id | |
233 | osd_fsid = osd_fsid if osd_fsid is not None else args.osd_fsid | |
234 | ||
d2e6a577 FG |
235 | lvs = api.Volumes() |
236 | # filter them down for the OSD ID and FSID we need to activate | |
94b18763 FG |
237 | if osd_id and osd_fsid: |
238 | lvs.filter(lv_tags={'ceph.osd_id': osd_id, 'ceph.osd_fsid': osd_fsid}) | |
239 | elif osd_fsid and not osd_id: | |
240 | lvs.filter(lv_tags={'ceph.osd_fsid': osd_fsid}) | |
d2e6a577 | 241 | if not lvs: |
94b18763 | 242 | raise RuntimeError('could not find osd.%s with fsid %s' % (osd_id, osd_fsid)) |
3efd9988 FG |
243 | # This argument is only available when passed in directly or via |
244 | # systemd, not when ``create`` is being used | |
245 | if getattr(args, 'auto_detect_objectstore', False): | |
246 | logger.info('auto detecting objectstore') | |
247 | # may get multiple lvs, so can't do lvs.get() calls here | |
248 | for lv in lvs: | |
249 | has_journal = lv.tags.get('ceph.journal_uuid') | |
250 | if has_journal: | |
251 | logger.info('found a journal associated with the OSD, assuming filestore') | |
252 | return activate_filestore(lvs) | |
253 | logger.info('unable to find a journal associated with the OSD, assuming bluestore') | |
254 | return activate_bluestore(lvs) | |
255 | if args.bluestore: | |
94b18763 | 256 | activate_bluestore(lvs, no_systemd=args.no_systemd) |
3efd9988 | 257 | elif args.filestore: |
94b18763 | 258 | activate_filestore(lvs, no_systemd=args.no_systemd) |
d2e6a577 FG |
259 | |
260 | def main(self): | |
261 | sub_command_help = dedent(""" | |
262 | Activate OSDs by discovering them with LVM and mounting them in their | |
263 | appropriate destination: | |
264 | ||
265 | ceph-volume lvm activate {ID} {FSID} | |
266 | ||
267 | The lvs associated with the OSD need to have been prepared previously, | |
268 | so that all needed tags and metadata exist. | |
269 | ||
94b18763 FG |
270 | When migrating OSDs, or a multiple-osd activation is needed, the |
271 | ``--all`` flag can be used instead of the individual ID and FSID: | |
272 | ||
273 | ceph-volume lvm activate --all | |
274 | ||
d2e6a577 FG |
275 | """) |
276 | parser = argparse.ArgumentParser( | |
277 | prog='ceph-volume lvm activate', | |
278 | formatter_class=argparse.RawDescriptionHelpFormatter, | |
279 | description=sub_command_help, | |
280 | ) | |
281 | ||
282 | parser.add_argument( | |
283 | 'osd_id', | |
284 | metavar='ID', | |
285 | nargs='?', | |
286 | help='The ID of the OSD, usually an integer, like 0' | |
287 | ) | |
288 | parser.add_argument( | |
289 | 'osd_fsid', | |
290 | metavar='FSID', | |
291 | nargs='?', | |
292 | help='The FSID of the OSD, similar to a SHA1' | |
293 | ) | |
3efd9988 FG |
294 | parser.add_argument( |
295 | '--auto-detect-objectstore', | |
296 | action='store_true', | |
297 | help='Autodetect the objectstore by inspecting the OSD', | |
298 | ) | |
d2e6a577 FG |
299 | parser.add_argument( |
300 | '--bluestore', | |
3efd9988 | 301 | action='store_true', |
94b18763 | 302 | help='bluestore objectstore (default)', |
d2e6a577 FG |
303 | ) |
304 | parser.add_argument( | |
305 | '--filestore', | |
3efd9988 | 306 | action='store_true', |
94b18763 FG |
307 | help='filestore objectstore', |
308 | ) | |
309 | parser.add_argument( | |
310 | '--all', | |
311 | dest='activate_all', | |
312 | action='store_true', | |
313 | help='Activate all OSDs found in the system', | |
314 | ) | |
315 | parser.add_argument( | |
316 | '--no-systemd', | |
317 | dest='no_systemd', | |
318 | action='store_true', | |
319 | help='Skip creating and enabling systemd units and starting OSD services', | |
d2e6a577 FG |
320 | ) |
321 | if len(self.argv) == 0: | |
322 | print(sub_command_help) | |
323 | return | |
324 | args = parser.parse_args(self.argv) | |
3efd9988 FG |
325 | # Default to bluestore here since defaulting it in add_argument may |
326 | # cause both to be True | |
327 | if not args.bluestore and not args.filestore: | |
328 | args.bluestore = True | |
94b18763 FG |
329 | if args.activate_all: |
330 | self.activate_all(args) | |
331 | else: | |
332 | self.activate(args) |