[ceph.git] / ceph / src / pybind / mgr / dashboard / rbd_mirroring.py


import json
import re
import rados
import rbd
from remote_view_cache import RemoteViewCache

class DaemonsAndPools(RemoteViewCache):
    def _get(self):
        daemons = self.get_daemons()
        return {
            'daemons': daemons,
            'pools': self.get_pools(daemons)
        }

    def get_daemons(self):
        daemons = []
        for server in self._module.list_servers():
            for service in server['services']:
                if service['type'] == 'rbd-mirror':
                    metadata = self._module.get_metadata('rbd-mirror',
                                                         service['id'])
                    status = self._module.get_daemon_status('rbd-mirror',
                                                            service['id'])
                    try:
                        status = json.loads(status['json'])
                    except:
                        status = {}

                    # extract per-daemon service data and health
                    daemon = {
                        'id': service['id'],
                        'instance_id': metadata['instance_id'],
                        'version': metadata['ceph_version'],
                        'server_hostname': server['hostname'],
                        'service': service,
                        'server': server,
                        'metadata': metadata,
                        'status': status
                    }
                    daemon = dict(daemon, **self.get_daemon_health(daemon))
                    daemons.append(daemon)

        return sorted(daemons, key=lambda k: k['id'])

    def get_daemon_health(self, daemon):
        health = {
            'health_color': 'info',
            'health' : 'Unknown'
        }
        for pool_id, pool_data in daemon['status'].items():
            if (health['health'] != 'error' and
                [k for k,v in pool_data.get('callouts', {}).items() if v['level'] == 'error']):
                health = {
                    'health_color': 'error',
                    'health': 'Error'
                }
            elif (health['health'] != 'error' and
                  [k for k,v in pool_data.get('callouts', {}).items() if v['level'] == 'warning']):
                health = {
                    'health_color': 'warning',
                    'health': 'Warning'
                }
            elif health['health_color'] == 'info':
                health = {
                    'health_color': 'success',
                    'health': 'OK'
                }
        return health

    def get_pools(self, daemons):
        status, pool_names = self._module.rbd_pool_ls.get()
        if pool_names is None:
            self.log.warning("Failed to get RBD pool list")
            return {}

        pool_stats = {}
        rbdctx = rbd.RBD()
        for pool_name in pool_names:
            self.log.debug("Constructing IOCtx " + pool_name)
            try:
                ioctx = self._module.rados.open_ioctx(pool_name)
            except:
                self.log.exception("Failed to open pool " + pool_name)
                continue

            try:
                mirror_mode = rbdctx.mirror_mode_get(ioctx)
            except:
                self.log.exception("Failed to query mirror mode " + pool_name)

            stats = {}
            if mirror_mode == rbd.RBD_MIRROR_MODE_DISABLED:
                continue
            elif mirror_mode == rbd.RBD_MIRROR_MODE_IMAGE:
                mirror_mode = "image"
            elif mirror_mode == rbd.RBD_MIRROR_MODE_POOL:
                mirror_mode = "pool"
            else:
                mirror_mode = "unknown"
                stats['health_color'] = "warning"
                stats['health'] = "Warning"

            pool_stats[pool_name] = dict(stats, **{
                'mirror_mode': mirror_mode
            })

        for daemon in daemons:
            for pool_id, pool_data in daemon['status'].items():
                stats = pool_stats.get(pool_data['name'], None)
                if stats is None:
                    continue

                if pool_data.get('leader', False):
                    # leader instance stores image counts
                    stats['leader_id'] = daemon['metadata']['instance_id']
                    stats['image_local_count'] = pool_data.get('image_local_count', 0)
                    stats['image_remote_count'] = pool_data.get('image_remote_count', 0)

                if (stats.get('health_color', '') != 'error' and
                    pool_data.get('image_error_count', 0) > 0):
                    stats['health_color'] = 'error'
                    stats['health'] = 'Error'
                elif (stats.get('health_color', '') != 'error' and
                      pool_data.get('image_warning_count', 0) > 0):
                    stats['health_color'] = 'warning'
                    stats['health'] = 'Warning'
                elif stats.get('health', None) is None:
                    stats['health_color'] = 'success'
                    stats['health'] = 'OK'

        for name, stats in pool_stats.items():
            if stats.get('health', None) is None:
                # daemon doesn't know about pool
                stats['health_color'] = 'error'
                stats['health'] = 'Error'
            elif stats.get('leader_id', None) is None:
                # no daemons are managing the pool as leader instance
                stats['health_color'] = 'warning'
                stats['health'] = 'Warning'
        return pool_stats


class PoolDatum(RemoteViewCache):
    def __init__(self, module_inst, pool_name):
        super(PoolDatum, self).__init__(module_inst)
        self.pool_name = pool_name

    def _get(self):
        data = {}
        self.log.debug("Constructing IOCtx " + self.pool_name)
        try:
            ioctx = self._module.rados.open_ioctx(self.pool_name)
        except:
            self.log.exception("Failed to open pool " + pool_name)
            return None

        mirror_state = {
            'down': {
                'health': 'issue',
                'state_color': 'warning',
                'state': 'Unknown',
                'description': None
            },
            rbd.MIRROR_IMAGE_STATUS_STATE_UNKNOWN: {
                'health': 'issue',
                'state_color': 'warning',
                'state': 'Unknown'
            },
            rbd.MIRROR_IMAGE_STATUS_STATE_ERROR: {
                'health': 'issue',
                'state_color': 'error',
                'state': 'Error'
            },
            rbd.MIRROR_IMAGE_STATUS_STATE_SYNCING: {
                'health': 'syncing'
            },
            rbd.MIRROR_IMAGE_STATUS_STATE_STARTING_REPLAY: {
                'health': 'ok',
                'state_color': 'success',
                'state': 'Starting'
            },
            rbd.MIRROR_IMAGE_STATUS_STATE_REPLAYING: {
                'health': 'ok',
                'state_color': 'success',
                'state': 'Replaying'
            },
            rbd.MIRROR_IMAGE_STATUS_STATE_STOPPING_REPLAY: {
                'health': 'ok',
                'state_color': 'success',
                'state': 'Stopping'
            },
            rbd.MIRROR_IMAGE_STATUS_STATE_STOPPED: {
                'health': 'ok',
                'state_color': 'info',
                'state': 'Primary'
            }
        }

        rbdctx = rbd.RBD()
        try:
            mirror_image_status = rbdctx.mirror_image_status_list(ioctx)
            data['mirror_images'] = sorted([
                dict({
                    'name': image['name'],
                    'description': image['description']
                }, **mirror_state['down' if not image['up'] else image['state']])
                for image in mirror_image_status
            ], key=lambda k: k['name'])
        except rbd.ImageNotFound:
            pass
        except:
            self.log.exception("Failed to list mirror image status " + self.pool_name)

        return data

class Toplevel(RemoteViewCache):
    def __init__(self, module_inst, daemons_and_pools):
        super(Toplevel, self).__init__(module_inst)
        self.daemons_and_pools = daemons_and_pools

    def _get(self):
        status, data = self.daemons_and_pools.get()
        if data is None:
            self.log.warning("Failed to get rbd-mirror daemons and pools")
            daemons = {}
        daemons = data.get('daemons', [])
        pools = data.get('pools', {})

        warnings = 0
        errors = 0
        for daemon in daemons:
            if daemon['health_color'] == 'error':
                errors += 1
            elif daemon['health_color'] == 'warning':
                warnings += 1
        for pool_name, pool in pools.items():
            if pool['health_color'] == 'error':
                errors += 1
            elif pool['health_color'] == 'warning':
                warnings += 1
        return {'warnings': warnings, 'errors': errors}


class ContentData(RemoteViewCache):
    def __init__(self, module_inst, daemons_and_pools, pool_data):
        super(ContentData, self).__init__(module_inst)

        self.daemons_and_pools = daemons_and_pools
        self.pool_data = pool_data

    def _get(self):
        status, pool_names = self._module.rbd_pool_ls.get()
        if pool_names is None:
            self.log.warning("Failed to get RBD pool list")
            return None

        status, data = self.daemons_and_pools.get()
        if data is None:
            self.log.warning("Failed to get rbd-mirror daemons list")
            data = {}
        daemons = data.get('daemons', [])
        pool_stats = data.get('pools', {})

        pools = []
        image_error = []
        image_syncing = []
        image_ready = []
        for pool_name in pool_names:
            pool = self.get_pool_datum(pool_name) or {}
            stats = pool_stats.get(pool_name, {})
            if stats.get('mirror_mode', None) is None:
                continue

            mirror_images = pool.get('mirror_images', [])
            for mirror_image in mirror_images:
                image = {
                    'pool_name': pool_name,
                    'name': mirror_image['name']
                }

                if mirror_image['health'] == 'ok':
                    image.update({
                        'state_color': mirror_image['state_color'],
                        'state': mirror_image['state'],
                        'description': mirror_image['description']
                    })
                    image_ready.append(image)
                elif mirror_image['health'] == 'syncing':
                    p = re.compile("bootstrapping, IMAGE_COPY/COPY_OBJECT (.*)%")
                    image.update({
                        'progress': (p.findall(mirror_image['description']) or [0])[0]
                    })
                    image_syncing.append(image)
                else:
                    image.update({
                        'state_color': mirror_image['state_color'],
                        'state': mirror_image['state'],
                        'description': mirror_image['description']
                    })
                    image_error.append(image)

            pools.append(dict({
                'name': pool_name
            }, **stats))

        return {
            'daemons': daemons,
            'pools' : pools,
            'image_error': image_error,
            'image_syncing': image_syncing,
            'image_ready': image_ready
        }

    def get_pool_datum(self, pool_name):
        pool_datum = self.pool_data.get(pool_name, None)
        if pool_datum is None:
            pool_datum = PoolDatum(self._module, pool_name)
            self.pool_data[pool_name] = pool_datum

        status, value = pool_datum.get()
        return value

class Controller:
    def __init__(self, module_inst):
        self.daemons_and_pools = DaemonsAndPools(module_inst)
        self.pool_data = {}
        self.toplevel = Toplevel(module_inst, self.daemons_and_pools)
        self.content_data = ContentData(module_inst, self.daemons_and_pools,
                                        self.pool_data)
Commit	Line	Data
c07f9fc5 FG	1
	2	import json
	3	import re
	4	import rados
	5	import rbd
	6	from remote_view_cache import RemoteViewCache
	7
	8	class DaemonsAndPools(RemoteViewCache):
	9	def _get(self):
	10	daemons = self.get_daemons()
	11	return {
	12	'daemons': daemons,
	13	'pools': self.get_pools(daemons)
	14	}
	15
	16	def get_daemons(self):
	17	daemons = []
	18	for server in self._module.list_servers():
	19	for service in server['services']:
	20	if service['type'] == 'rbd-mirror':
	21	metadata = self._module.get_metadata('rbd-mirror',
	22	service['id'])
	23	status = self._module.get_daemon_status('rbd-mirror',
	24	service['id'])
	25	try:
	26	status = json.loads(status['json'])
	27	except:
	28	status = {}
	29
	30	# extract per-daemon service data and health
	31	daemon = {
	32	'id': service['id'],
	33	'instance_id': metadata['instance_id'],
	34	'version': metadata['ceph_version'],
	35	'server_hostname': server['hostname'],
	36	'service': service,
	37	'server': server,
	38	'metadata': metadata,
	39	'status': status
	40	}
	41	daemon = dict(daemon, **self.get_daemon_health(daemon))
	42	daemons.append(daemon)
	43
	44	return sorted(daemons, key=lambda k: k['id'])
	45
	46	def get_daemon_health(self, daemon):
	47	health = {
	48	'health_color': 'info',
	49	'health' : 'Unknown'
	50	}
	51	for pool_id, pool_data in daemon['status'].items():
	52	if (health['health'] != 'error' and
	53	[k for k,v in pool_data.get('callouts', {}).items() if v['level'] == 'error']):
	54	health = {
	55	'health_color': 'error',
	56	'health': 'Error'
	57	}
	58	elif (health['health'] != 'error' and
	59	[k for k,v in pool_data.get('callouts', {}).items() if v['level'] == 'warning']):
	60	health = {
	61	'health_color': 'warning',
	62	'health': 'Warning'
	63	}
	64	elif health['health_color'] == 'info':
65	health = {
66	'health_color': 'success',
67	'health': 'OK'
68	}
69	return health
70
71	def get_pools(self, daemons):
72	status, pool_names = self._module.rbd_pool_ls.get()
73	if pool_names is None:
74	self.log.warning("Failed to get RBD pool list")
75	return {}
76
77	pool_stats = {}
78	rbdctx = rbd.RBD()
79	for pool_name in pool_names:
80	self.log.debug("Constructing IOCtx " + pool_name)
81	try:
82	ioctx = self._module.rados.open_ioctx(pool_name)
83	except:
84	self.log.exception("Failed to open pool " + pool_name)
85	continue
86
87	try:
88	mirror_mode = rbdctx.mirror_mode_get(ioctx)
89	except:
90	self.log.exception("Failed to query mirror mode " + pool_name)
91
92	stats = {}
93	if mirror_mode == rbd.RBD_MIRROR_MODE_DISABLED:
94	continue
95	elif mirror_mode == rbd.RBD_MIRROR_MODE_IMAGE:
96	mirror_mode = "image"
97	elif mirror_mode == rbd.RBD_MIRROR_MODE_POOL:
98	mirror_mode = "pool"
99	else:
100	mirror_mode = "unknown"
101	stats['health_color'] = "warning"
102	stats['health'] = "Warning"
103
104	pool_stats[pool_name] = dict(stats, **{
105	'mirror_mode': mirror_mode
106	})
107
108	for daemon in daemons:
109	for pool_id, pool_data in daemon['status'].items():
110	stats = pool_stats.get(pool_data['name'], None)
111	if stats is None:
112	continue
113
114	if pool_data.get('leader', False):
115	# leader instance stores image counts
116	stats['leader_id'] = daemon['metadata']['instance_id']
117	stats['image_local_count'] = pool_data.get('image_local_count', 0)
118	stats['image_remote_count'] = pool_data.get('image_remote_count', 0)
119
120	if (stats.get('health_color', '') != 'error' and
121	pool_data.get('image_error_count', 0) > 0):
122	stats['health_color'] = 'error'
123	stats['health'] = 'Error'
124	elif (stats.get('health_color', '') != 'error' and
125	pool_data.get('image_warning_count', 0) > 0):
126	stats['health_color'] = 'warning'
127	stats['health'] = 'Warning'
128	elif stats.get('health', None) is None:
129	stats['health_color'] = 'success'
130	stats['health'] = 'OK'
131
132	for name, stats in pool_stats.items():
133	if stats.get('health', None) is None:
134	# daemon doesn't know about pool
135	stats['health_color'] = 'error'
136	stats['health'] = 'Error'
137	elif stats.get('leader_id', None) is None:
138	# no daemons are managing the pool as leader instance
139	stats['health_color'] = 'warning'
140	stats['health'] = 'Warning'
141	return pool_stats
142
143
144	class PoolDatum(RemoteViewCache):
145	def __init__(self, module_inst, pool_name):
146	super(PoolDatum, self).__init__(module_inst)
147	self.pool_name = pool_name
148
149	def _get(self):
150	data = {}
151	self.log.debug("Constructing IOCtx " + self.pool_name)
152	try:
153	ioctx = self._module.rados.open_ioctx(self.pool_name)
154	except:
155	self.log.exception("Failed to open pool " + pool_name)
156	return None
157
158	mirror_state = {
159	'down': {
160	'health': 'issue',
161	'state_color': 'warning',
162	'state': 'Unknown',
163	'description': None
164	},
165	rbd.MIRROR_IMAGE_STATUS_STATE_UNKNOWN: {
166	'health': 'issue',
167	'state_color': 'warning',
168	'state': 'Unknown'
169	},
170	rbd.MIRROR_IMAGE_STATUS_STATE_ERROR: {
171	'health': 'issue',
172	'state_color': 'error',
173	'state': 'Error'
174	},
175	rbd.MIRROR_IMAGE_STATUS_STATE_SYNCING: {
176	'health': 'syncing'
177	},
178	rbd.MIRROR_IMAGE_STATUS_STATE_STARTING_REPLAY: {
179	'health': 'ok',
180	'state_color': 'success',
181	'state': 'Starting'
182	},
183	rbd.MIRROR_IMAGE_STATUS_STATE_REPLAYING: {
184	'health': 'ok',
185	'state_color': 'success',
186	'state': 'Replaying'
187	},
188	rbd.MIRROR_IMAGE_STATUS_STATE_STOPPING_REPLAY: {
189	'health': 'ok',
190	'state_color': 'success',
191	'state': 'Stopping'
192	},
193	rbd.MIRROR_IMAGE_STATUS_STATE_STOPPED: {
194	'health': 'ok',
195	'state_color': 'info',
196	'state': 'Primary'
197	}
198	}
199
200	rbdctx = rbd.RBD()
201	try:
202	mirror_image_status = rbdctx.mirror_image_status_list(ioctx)
203	data['mirror_images'] = sorted([
204	dict({
205	'name': image['name'],
206	'description': image['description']
207	}, **mirror_state['down' if not image['up'] else image['state']])
208	for image in mirror_image_status
209	], key=lambda k: k['name'])
210	except rbd.ImageNotFound:
211	pass
212	except:
213	self.log.exception("Failed to list mirror image status " + self.pool_name)
214
215	return data
216
217	class Toplevel(RemoteViewCache):
218	def __init__(self, module_inst, daemons_and_pools):
219	super(Toplevel, self).__init__(module_inst)
220	self.daemons_and_pools = daemons_and_pools
221
222	def _get(self):
223	status, data = self.daemons_and_pools.get()
224	if data is None:
225	self.log.warning("Failed to get rbd-mirror daemons and pools")
226	daemons = {}
227	daemons = data.get('daemons', [])
228	pools = data.get('pools', {})
229
230	warnings = 0
231	errors = 0
232	for daemon in daemons:
233	if daemon['health_color'] == 'error':
234	errors += 1
235	elif daemon['health_color'] == 'warning':
236	warnings += 1
237	for pool_name, pool in pools.items():
238	if pool['health_color'] == 'error':
239	errors += 1
240	elif pool['health_color'] == 'warning':
241	warnings += 1
242	return {'warnings': warnings, 'errors': errors}
243
244
245	class ContentData(RemoteViewCache):
246	def __init__(self, module_inst, daemons_and_pools, pool_data):
247	super(ContentData, self).__init__(module_inst)
248
249	self.daemons_and_pools = daemons_and_pools
250	self.pool_data = pool_data
251
252	def _get(self):
253	status, pool_names = self._module.rbd_pool_ls.get()
254	if pool_names is None:
255	self.log.warning("Failed to get RBD pool list")
256	return None
257
258	status, data = self.daemons_and_pools.get()
259	if data is None:
260	self.log.warning("Failed to get rbd-mirror daemons list")
261	data = {}
262	daemons = data.get('daemons', [])
263	pool_stats = data.get('pools', {})
264
265	pools = []
266	image_error = []
267	image_syncing = []
268	image_ready = []
269	for pool_name in pool_names:
270	pool = self.get_pool_datum(pool_name) or {}
271	stats = pool_stats.get(pool_name, {})
272	if stats.get('mirror_mode', None) is None:
273	continue
274
275	mirror_images = pool.get('mirror_images', [])
276	for mirror_image in mirror_images:
277	image = {
278	'pool_name': pool_name,
279	'name': mirror_image['name']
280	}
281
282	if mirror_image['health'] == 'ok':
283	image.update({
284	'state_color': mirror_image['state_color'],
285	'state': mirror_image['state'],
286	'description': mirror_image['description']
287	})
288	image_ready.append(image)
289	elif mirror_image['health'] == 'syncing':
290	p = re.compile("bootstrapping, IMAGE_COPY/COPY_OBJECT (.*)%")
291	image.update({
292	'progress': (p.findall(mirror_image['description']) or [0])[0]
293	})
294	image_syncing.append(image)
295	else:
296	image.update({
297	'state_color': mirror_image['state_color'],
298	'state': mirror_image['state'],
299	'description': mirror_image['description']
300	})
301	image_error.append(image)
302
303	pools.append(dict({
304	'name': pool_name
305	}, **stats))
306
307	return {
308	'daemons': daemons,
309	'pools' : pools,
310	'image_error': image_error,
311	'image_syncing': image_syncing,
312	'image_ready': image_ready
313	}
314
315	def get_pool_datum(self, pool_name):
316	pool_datum = self.pool_data.get(pool_name, None)
317	if pool_datum is None:
318	pool_datum = PoolDatum(self._module, pool_name)
319	self.pool_data[pool_name] = pool_datum
320
321	status, value = pool_datum.get()
322	return value
323
324	class Controller:
325	def __init__(self, module_inst):
326	self.daemons_and_pools = DaemonsAndPools(module_inst)
327	self.pool_data = {}
328	self.toplevel = Toplevel(module_inst, self.daemons_and_pools)
329	self.content_data = ContentData(module_inst, self.daemons_and_pools,
330	self.pool_data)
331