]>
Commit | Line | Data |
---|---|---|
31f18b77 FG |
1 | |
2 | """ | |
3 | Demonstrate writing a Ceph web interface inside a mgr module. | |
4 | """ | |
5 | ||
6 | # We must share a global reference to this instance, because it is the | |
7 | # gatekeeper to all accesses to data from the C++ side (e.g. the REST API | |
8 | # request handlers need to see it) | |
9 | from collections import defaultdict | |
10 | import collections | |
11 | ||
12 | _global_instance = {'plugin': None} | |
13 | def global_instance(): | |
14 | assert _global_instance['plugin'] is not None | |
15 | return _global_instance['plugin'] | |
16 | ||
17 | ||
18 | import os | |
19 | import logging | |
20 | import logging.config | |
21 | import json | |
22 | import sys | |
23 | import time | |
24 | import threading | |
25 | ||
26 | import cherrypy | |
27 | import jinja2 | |
28 | ||
29 | from mgr_module import MgrModule, CommandResult | |
30 | ||
31 | from types import OsdMap, NotFound, Config, FsMap, MonMap, \ | |
32 | PgSummary, Health, MonStatus | |
33 | ||
34 | import rados | |
c07f9fc5 FG |
35 | import rbd_iscsi |
36 | import rbd_mirroring | |
224ce89b | 37 | from rbd_ls import RbdLs, RbdPoolLs |
31f18b77 FG |
38 | from cephfs_clients import CephFSClients |
39 | ||
31f18b77 FG |
40 | log = logging.getLogger("dashboard") |
41 | ||
42 | ||
43 | # How many cluster log lines shall we hold onto in our | |
44 | # python module for the convenience of the GUI? | |
45 | LOG_BUFFER_SIZE = 30 | |
46 | ||
47 | # cherrypy likes to sys.exit on error. don't let it take us down too! | |
48 | def os_exit_noop(): | |
49 | pass | |
50 | ||
51 | os._exit = os_exit_noop | |
52 | ||
53 | ||
54 | def recurse_refs(root, path): | |
55 | if isinstance(root, dict): | |
56 | for k, v in root.items(): | |
57 | recurse_refs(v, path + "->%s" % k) | |
58 | elif isinstance(root, list): | |
59 | for n, i in enumerate(root): | |
60 | recurse_refs(i, path + "[%d]" % n) | |
61 | ||
62 | log.info("%s %d (%s)" % (path, sys.getrefcount(root), root.__class__)) | |
63 | ||
64 | ||
65 | class Module(MgrModule): | |
66 | def __init__(self, *args, **kwargs): | |
67 | super(Module, self).__init__(*args, **kwargs) | |
68 | _global_instance['plugin'] = self | |
69 | self.log.info("Constructing module {0}: instance {1}".format( | |
70 | __name__, _global_instance)) | |
71 | ||
72 | self.log_primed = False | |
73 | self.log_buffer = collections.deque(maxlen=LOG_BUFFER_SIZE) | |
74 | self.audit_buffer = collections.deque(maxlen=LOG_BUFFER_SIZE) | |
75 | ||
76 | # Keep a librados instance for those that need it. | |
77 | self._rados = None | |
78 | ||
79 | # Stateful instances of RbdLs, hold cached results. Key to dict | |
80 | # is pool name. | |
81 | self.rbd_ls = {} | |
82 | ||
224ce89b WB |
83 | # Stateful instance of RbdPoolLs, hold cached list of RBD |
84 | # pools | |
85 | self.rbd_pool_ls = RbdPoolLs(self) | |
86 | ||
c07f9fc5 FG |
87 | # Stateful instance of RbdISCSI |
88 | self.rbd_iscsi = rbd_iscsi.Controller(self) | |
89 | ||
90 | # Stateful instance of RbdMirroring, hold cached results. | |
91 | self.rbd_mirroring = rbd_mirroring.Controller(self) | |
92 | ||
31f18b77 FG |
93 | # Stateful instances of CephFSClients, hold cached results. Key to |
94 | # dict is FSCID | |
95 | self.cephfs_clients = {} | |
96 | ||
97 | # A short history of pool df stats | |
98 | self.pool_stats = defaultdict(lambda: defaultdict( | |
99 | lambda: collections.deque(maxlen=10))) | |
100 | ||
101 | @property | |
102 | def rados(self): | |
103 | """ | |
104 | A librados instance to be shared by any classes within | |
105 | this mgr module that want one. | |
106 | """ | |
107 | if self._rados: | |
108 | return self._rados | |
109 | ||
110 | from mgr_module import ceph_state | |
111 | ctx_capsule = ceph_state.get_context() | |
112 | self._rados = rados.Rados(context=ctx_capsule) | |
113 | self._rados.connect() | |
114 | ||
115 | return self._rados | |
116 | ||
31f18b77 FG |
117 | def update_pool_stats(self): |
118 | df = global_instance().get("df") | |
119 | pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) | |
120 | now = time.time() | |
121 | for pool_id, stats in pool_stats.items(): | |
122 | for stat_name, stat_val in stats.items(): | |
123 | self.pool_stats[pool_id][stat_name].appendleft((now, stat_val)) | |
124 | ||
125 | def notify(self, notify_type, notify_val): | |
126 | if notify_type == "clog": | |
127 | # Only store log messages once we've done our initial load, | |
128 | # so that we don't end up duplicating. | |
129 | if self.log_primed: | |
130 | if notify_val['channel'] == "audit": | |
131 | self.audit_buffer.appendleft(notify_val) | |
132 | else: | |
133 | self.log_buffer.appendleft(notify_val) | |
134 | elif notify_type == "pg_summary": | |
135 | self.update_pool_stats() | |
136 | else: | |
137 | pass | |
138 | ||
139 | def get_sync_object(self, object_type, path=None): | |
140 | if object_type == OsdMap: | |
141 | data = self.get("osd_map") | |
142 | ||
143 | assert data is not None | |
144 | ||
145 | data['tree'] = self.get("osd_map_tree") | |
146 | data['crush'] = self.get("osd_map_crush") | |
147 | data['crush_map_text'] = self.get("osd_map_crush_map_text") | |
148 | data['osd_metadata'] = self.get("osd_metadata") | |
149 | obj = OsdMap(data) | |
150 | elif object_type == Config: | |
151 | data = self.get("config") | |
152 | obj = Config( data) | |
153 | elif object_type == MonMap: | |
154 | data = self.get("mon_map") | |
155 | obj = MonMap(data) | |
156 | elif object_type == FsMap: | |
157 | data = self.get("fs_map") | |
158 | obj = FsMap(data) | |
159 | elif object_type == PgSummary: | |
160 | data = self.get("pg_summary") | |
161 | self.log.debug("JSON: {0}".format(data)) | |
162 | obj = PgSummary(data) | |
163 | elif object_type == Health: | |
164 | data = self.get("health") | |
165 | obj = Health(json.loads(data['json'])) | |
166 | elif object_type == MonStatus: | |
167 | data = self.get("mon_status") | |
168 | obj = MonStatus(json.loads(data['json'])) | |
169 | else: | |
170 | raise NotImplementedError(object_type) | |
171 | ||
172 | # TODO: move 'path' handling up into C++ land so that we only | |
173 | # Pythonize the part we're interested in | |
174 | if path: | |
175 | try: | |
176 | for part in path: | |
177 | if isinstance(obj, dict): | |
178 | obj = obj[part] | |
179 | else: | |
180 | obj = getattr(obj, part) | |
181 | except (AttributeError, KeyError): | |
182 | raise NotFound(object_type, path) | |
183 | ||
184 | return obj | |
185 | ||
186 | def shutdown(self): | |
187 | log.info("Stopping server...") | |
188 | cherrypy.engine.exit() | |
189 | log.info("Stopped server") | |
190 | ||
191 | log.info("Stopping librados...") | |
192 | if self._rados: | |
193 | self._rados.shutdown() | |
194 | log.info("Stopped librados.") | |
195 | ||
196 | def get_latest(self, daemon_type, daemon_name, stat): | |
197 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] | |
198 | if data: | |
199 | return data[-1][1] | |
200 | else: | |
201 | return 0 | |
202 | ||
203 | def get_rate(self, daemon_type, daemon_name, stat): | |
204 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] | |
205 | ||
206 | if data and len(data) > 1: | |
207 | return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0]) | |
208 | else: | |
209 | return 0 | |
210 | ||
211 | def format_dimless(self, n, width, colored=True): | |
212 | """ | |
213 | Format a number without units, so as to fit into `width` characters, substituting | |
214 | an appropriate unit suffix. | |
215 | """ | |
216 | units = [' ', 'k', 'M', 'G', 'T', 'P'] | |
217 | unit = 0 | |
218 | while len("%s" % (int(n) // (1000**unit))) > width - 1: | |
219 | unit += 1 | |
220 | ||
221 | if unit > 0: | |
222 | truncated_float = ("%f" % (n / (1000.0 ** unit)))[0:width - 1] | |
223 | if truncated_float[-1] == '.': | |
224 | truncated_float = " " + truncated_float[0:-1] | |
225 | else: | |
226 | truncated_float = "%{wid}d".format(wid=width-1) % n | |
227 | formatted = "%s%s" % (truncated_float, units[unit]) | |
228 | ||
229 | if colored: | |
230 | # TODO: html equivalent | |
231 | # if n == 0: | |
232 | # color = self.BLACK, False | |
233 | # else: | |
234 | # color = self.YELLOW, False | |
235 | # return self.bold(self.colorize(formatted[0:-1], color[0], color[1])) \ | |
236 | # + self.bold(self.colorize(formatted[-1], self.BLACK, False)) | |
237 | return formatted | |
238 | else: | |
239 | return formatted | |
240 | ||
241 | def fs_status(self, fs_id): | |
242 | mds_versions = defaultdict(list) | |
243 | ||
244 | fsmap = self.get("fs_map") | |
245 | filesystem = None | |
246 | for fs in fsmap['filesystems']: | |
247 | if fs['id'] == fs_id: | |
248 | filesystem = fs | |
249 | break | |
250 | ||
251 | rank_table = [] | |
252 | ||
253 | mdsmap = filesystem['mdsmap'] | |
254 | ||
255 | client_count = 0 | |
256 | ||
257 | for rank in mdsmap["in"]: | |
258 | up = "mds_{0}".format(rank) in mdsmap["up"] | |
259 | if up: | |
260 | gid = mdsmap['up']["mds_{0}".format(rank)] | |
261 | info = mdsmap['info']['gid_{0}'.format(gid)] | |
262 | dns = self.get_latest("mds", info['name'], "mds.inodes") | |
263 | inos = self.get_latest("mds", info['name'], "mds_mem.ino") | |
264 | ||
265 | if rank == 0: | |
266 | client_count = self.get_latest("mds", info['name'], | |
267 | "mds_sessions.session_count") | |
268 | elif client_count == 0: | |
269 | # In case rank 0 was down, look at another rank's | |
270 | # sessionmap to get an indication of clients. | |
271 | client_count = self.get_latest("mds", info['name'], | |
272 | "mds_sessions.session_count") | |
273 | ||
274 | laggy = "laggy_since" in info | |
275 | ||
276 | state = info['state'].split(":")[1] | |
277 | if laggy: | |
278 | state += "(laggy)" | |
279 | ||
280 | # if state == "active" and not laggy: | |
281 | # c_state = self.colorize(state, self.GREEN) | |
282 | # else: | |
283 | # c_state = self.colorize(state, self.YELLOW) | |
284 | ||
285 | # Populate based on context of state, e.g. client | |
286 | # ops for an active daemon, replay progress, reconnect | |
287 | # progress | |
288 | activity = "" | |
289 | ||
290 | if state == "active": | |
291 | activity = "Reqs: " + self.format_dimless( | |
292 | self.get_rate("mds", info['name'], "mds_server.handle_client_request"), | |
293 | 5 | |
294 | ) + "/s" | |
295 | ||
296 | metadata = self.get_metadata('mds', info['name']) | |
297 | mds_versions[metadata['ceph_version']].append(info['name']) | |
298 | rank_table.append( | |
299 | { | |
300 | "rank": rank, | |
301 | "state": state, | |
302 | "mds": info['name'], | |
303 | "activity": activity, | |
304 | "dns": dns, | |
305 | "inos": inos | |
306 | } | |
307 | ) | |
308 | ||
309 | else: | |
310 | rank_table.append( | |
311 | { | |
312 | "rank": rank, | |
313 | "state": "failed", | |
314 | "mds": "", | |
315 | "activity": "", | |
316 | "dns": 0, | |
317 | "inos": 0 | |
318 | } | |
319 | ) | |
320 | ||
321 | # Find the standby replays | |
322 | for gid_str, daemon_info in mdsmap['info'].iteritems(): | |
323 | if daemon_info['state'] != "up:standby-replay": | |
324 | continue | |
325 | ||
326 | inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino") | |
327 | dns = self.get_latest("mds", daemon_info['name'], "mds.inodes") | |
328 | ||
329 | activity = "Evts: " + self.format_dimless( | |
330 | self.get_rate("mds", daemon_info['name'], "mds_log.replay"), | |
331 | 5 | |
332 | ) + "/s" | |
333 | ||
334 | rank_table.append( | |
335 | { | |
336 | "rank": "{0}-s".format(daemon_info['rank']), | |
337 | "state": "standby-replay", | |
338 | "mds": daemon_info['name'], | |
339 | "activity": activity, | |
340 | "dns": dns, | |
341 | "inos": inos | |
342 | } | |
343 | ) | |
344 | ||
345 | df = self.get("df") | |
346 | pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) | |
347 | osdmap = self.get("osd_map") | |
348 | pools = dict([(p['pool'], p) for p in osdmap['pools']]) | |
349 | metadata_pool_id = mdsmap['metadata_pool'] | |
350 | data_pool_ids = mdsmap['data_pools'] | |
351 | ||
352 | pools_table = [] | |
353 | for pool_id in [metadata_pool_id] + data_pool_ids: | |
354 | pool_type = "metadata" if pool_id == metadata_pool_id else "data" | |
355 | stats = pool_stats[pool_id] | |
356 | pools_table.append({ | |
357 | "pool": pools[pool_id]['pool_name'], | |
358 | "type": pool_type, | |
359 | "used": stats['bytes_used'], | |
360 | "avail": stats['max_avail'] | |
361 | }) | |
362 | ||
363 | standby_table = [] | |
364 | for standby in fsmap['standbys']: | |
365 | metadata = self.get_metadata('mds', standby['name']) | |
366 | mds_versions[metadata['ceph_version']].append(standby['name']) | |
367 | ||
368 | standby_table.append({ | |
369 | 'name': standby['name'] | |
370 | }) | |
371 | ||
372 | return { | |
373 | "filesystem": { | |
374 | "id": fs_id, | |
375 | "name": mdsmap['fs_name'], | |
376 | "client_count": client_count, | |
377 | "clients_url": "/clients/{0}/".format(fs_id), | |
378 | "ranks": rank_table, | |
379 | "pools": pools_table | |
380 | }, | |
381 | "standbys": standby_table, | |
382 | "versions": mds_versions | |
383 | } | |
384 | ||
385 | def serve(self): | |
386 | current_dir = os.path.dirname(os.path.abspath(__file__)) | |
387 | ||
388 | jinja_loader = jinja2.FileSystemLoader(current_dir) | |
389 | env = jinja2.Environment(loader=jinja_loader) | |
390 | ||
391 | result = CommandResult("") | |
392 | self.send_command(result, "mon", "", json.dumps({ | |
393 | "prefix":"log last", | |
394 | "format": "json" | |
395 | }), "") | |
396 | r, outb, outs = result.wait() | |
397 | if r != 0: | |
398 | # Oh well. We won't let this stop us though. | |
399 | self.log.error("Error fetching log history (r={0}, \"{1}\")".format( | |
400 | r, outs)) | |
401 | else: | |
402 | try: | |
403 | lines = json.loads(outb) | |
404 | except ValueError: | |
405 | self.log.error("Error decoding log history") | |
406 | else: | |
407 | for l in lines: | |
408 | if l['channel'] == 'audit': | |
409 | self.audit_buffer.appendleft(l) | |
410 | else: | |
411 | self.log_buffer.appendleft(l) | |
412 | ||
413 | self.log_primed = True | |
414 | ||
c07f9fc5 FG |
415 | class EndPoint(object): |
416 | def _health_data(self): | |
417 | health = global_instance().get_sync_object(Health).data | |
418 | # Transform the `checks` dict into a list for the convenience | |
419 | # of rendering from javascript. | |
420 | checks = [] | |
421 | for k, v in health['checks'].iteritems(): | |
422 | v['type'] = k | |
423 | checks.append(v) | |
424 | ||
425 | checks = sorted(checks, cmp=lambda a, b: a['severity'] > b['severity']) | |
426 | ||
427 | health['checks'] = checks | |
428 | ||
429 | return health | |
430 | ||
31f18b77 FG |
431 | def _toplevel_data(self): |
432 | """ | |
433 | Data consumed by the base.html template | |
434 | """ | |
224ce89b WB |
435 | status, data = global_instance().rbd_pool_ls.get() |
436 | if data is None: | |
437 | log.warning("Failed to get RBD pool list") | |
438 | data = [] | |
439 | ||
440 | rbd_pools = sorted([ | |
441 | { | |
442 | "name": name, | |
c07f9fc5 | 443 | "url": "/rbd_pool/{0}/".format(name) |
224ce89b WB |
444 | } |
445 | for name in data | |
446 | ], key=lambda k: k['name']) | |
447 | ||
c07f9fc5 FG |
448 | status, rbd_mirroring = global_instance().rbd_mirroring.toplevel.get() |
449 | if rbd_mirroring is None: | |
450 | log.warning("Failed to get RBD mirroring summary") | |
451 | rbd_mirroring = {} | |
452 | ||
31f18b77 FG |
453 | fsmap = global_instance().get_sync_object(FsMap) |
454 | filesystems = [ | |
455 | { | |
456 | "id": f['id'], | |
457 | "name": f['mdsmap']['fs_name'], | |
458 | "url": "/filesystem/{0}/".format(f['id']) | |
459 | } | |
460 | for f in fsmap.data['filesystems'] | |
461 | ] | |
462 | ||
463 | return { | |
224ce89b | 464 | 'rbd_pools': rbd_pools, |
c07f9fc5 | 465 | 'rbd_mirroring': rbd_mirroring, |
224ce89b | 466 | 'health_status': self._health_data()['status'], |
31f18b77 FG |
467 | 'filesystems': filesystems |
468 | } | |
469 | ||
c07f9fc5 | 470 | class Root(EndPoint): |
31f18b77 FG |
471 | @cherrypy.expose |
472 | def filesystem(self, fs_id): | |
473 | template = env.get_template("filesystem.html") | |
474 | ||
475 | toplevel_data = self._toplevel_data() | |
476 | ||
477 | content_data = { | |
478 | "fs_status": global_instance().fs_status(int(fs_id)) | |
479 | } | |
480 | ||
481 | return template.render( | |
482 | ceph_version=global_instance().version, | |
c07f9fc5 | 483 | path_info=cherrypy.request.path_info, |
31f18b77 FG |
484 | toplevel_data=json.dumps(toplevel_data, indent=2), |
485 | content_data=json.dumps(content_data, indent=2) | |
486 | ) | |
487 | ||
488 | @cherrypy.expose | |
489 | @cherrypy.tools.json_out() | |
490 | def filesystem_data(self, fs_id): | |
491 | return global_instance().fs_status(int(fs_id)) | |
492 | ||
31f18b77 FG |
493 | def _clients(self, fs_id): |
494 | cephfs_clients = global_instance().cephfs_clients.get(fs_id, None) | |
495 | if cephfs_clients is None: | |
496 | cephfs_clients = CephFSClients(global_instance(), fs_id) | |
497 | global_instance().cephfs_clients[fs_id] = cephfs_clients | |
498 | ||
499 | status, clients = cephfs_clients.get() | |
500 | #TODO do something sensible with status | |
501 | ||
502 | # Decorate the metadata with some fields that will be | |
503 | # indepdendent of whether it's a kernel or userspace | |
504 | # client, so that the javascript doesn't have to grok that. | |
505 | for client in clients: | |
506 | if "ceph_version" in client['client_metadata']: | |
507 | client['type'] = "userspace" | |
508 | client['version'] = client['client_metadata']['ceph_version'] | |
509 | client['hostname'] = client['client_metadata']['hostname'] | |
510 | elif "kernel_version" in client['client_metadata']: | |
511 | client['type'] = "kernel" | |
224ce89b | 512 | client['version'] = client['client_metadata']['kernel_version'] |
31f18b77 FG |
513 | client['hostname'] = client['client_metadata']['hostname'] |
514 | else: | |
515 | client['type'] = "unknown" | |
516 | client['version'] = "" | |
517 | client['hostname'] = "" | |
518 | ||
519 | return clients | |
520 | ||
521 | @cherrypy.expose | |
224ce89b WB |
522 | def clients(self, fscid_str): |
523 | try: | |
524 | fscid = int(fscid_str) | |
525 | except ValueError: | |
526 | raise cherrypy.HTTPError(400, | |
527 | "Invalid filesystem id {0}".format(fscid_str)) | |
528 | ||
529 | try: | |
530 | fs_name = FsMap(global_instance().get( | |
531 | "fs_map")).get_filesystem(fscid)['mdsmap']['fs_name'] | |
532 | except NotFound: | |
533 | log.warning("Missing FSCID, dumping fsmap:\n{0}".format( | |
534 | json.dumps(global_instance().get("fs_map"), indent=2) | |
535 | )) | |
536 | raise cherrypy.HTTPError(404, | |
537 | "No filesystem with id {0}".format(fscid)) | |
538 | ||
539 | clients = self._clients(fscid) | |
31f18b77 FG |
540 | global_instance().log.debug(json.dumps(clients, indent=2)) |
541 | content_data = { | |
542 | "clients": clients, | |
224ce89b WB |
543 | "fs_name": fs_name, |
544 | "fscid": fscid, | |
545 | "fs_url": "/filesystem/" + fscid_str + "/" | |
31f18b77 FG |
546 | } |
547 | ||
224ce89b | 548 | template = env.get_template("clients.html") |
31f18b77 FG |
549 | return template.render( |
550 | ceph_version=global_instance().version, | |
c07f9fc5 | 551 | path_info=cherrypy.request.path_info, |
224ce89b | 552 | toplevel_data=json.dumps(self._toplevel_data(), indent=2), |
31f18b77 FG |
553 | content_data=json.dumps(content_data, indent=2) |
554 | ) | |
555 | ||
556 | @cherrypy.expose | |
557 | @cherrypy.tools.json_out() | |
558 | def clients_data(self, fs_id): | |
559 | return self._clients(int(fs_id)) | |
560 | ||
c07f9fc5 | 561 | def _rbd_pool(self, pool_name): |
31f18b77 FG |
562 | rbd_ls = global_instance().rbd_ls.get(pool_name, None) |
563 | if rbd_ls is None: | |
564 | rbd_ls = RbdLs(global_instance(), pool_name) | |
565 | global_instance().rbd_ls[pool_name] = rbd_ls | |
566 | ||
567 | status, value = rbd_ls.get() | |
568 | ||
569 | interval = 5 | |
570 | ||
571 | wait = interval - rbd_ls.latency | |
572 | def wait_and_load(): | |
573 | time.sleep(wait) | |
574 | rbd_ls.get() | |
575 | ||
576 | threading.Thread(target=wait_and_load).start() | |
577 | ||
578 | assert status != RbdLs.VALUE_NONE # FIXME bubble status up to UI | |
579 | return value | |
580 | ||
581 | @cherrypy.expose | |
c07f9fc5 FG |
582 | def rbd_pool(self, pool_name): |
583 | template = env.get_template("rbd_pool.html") | |
31f18b77 FG |
584 | |
585 | toplevel_data = self._toplevel_data() | |
586 | ||
c07f9fc5 | 587 | images = self._rbd_pool(pool_name) |
31f18b77 FG |
588 | content_data = { |
589 | "images": images, | |
590 | "pool_name": pool_name | |
591 | } | |
592 | ||
593 | return template.render( | |
594 | ceph_version=global_instance().version, | |
c07f9fc5 FG |
595 | path_info=cherrypy.request.path_info, |
596 | toplevel_data=json.dumps(toplevel_data, indent=2), | |
597 | content_data=json.dumps(content_data, indent=2) | |
598 | ) | |
599 | ||
600 | @cherrypy.expose | |
601 | @cherrypy.tools.json_out() | |
602 | def rbd_pool_data(self, pool_name): | |
603 | return self._rbd_pool(pool_name) | |
604 | ||
605 | def _rbd_mirroring(self): | |
606 | status, data = global_instance().rbd_mirroring.content_data.get() | |
607 | if data is None: | |
608 | log.warning("Failed to get RBD mirroring status") | |
609 | return {} | |
610 | return data | |
611 | ||
612 | @cherrypy.expose | |
613 | def rbd_mirroring(self): | |
614 | template = env.get_template("rbd_mirroring.html") | |
615 | ||
616 | toplevel_data = self._toplevel_data() | |
617 | content_data = self._rbd_mirroring() | |
618 | ||
619 | return template.render( | |
620 | ceph_version=global_instance().version, | |
621 | path_info=cherrypy.request.path_info, | |
622 | toplevel_data=json.dumps(toplevel_data, indent=2), | |
623 | content_data=json.dumps(content_data, indent=2) | |
624 | ) | |
625 | ||
626 | @cherrypy.expose | |
627 | @cherrypy.tools.json_out() | |
628 | def rbd_mirroring_data(self): | |
629 | return self._rbd_mirroring() | |
630 | ||
631 | def _rbd_iscsi(self): | |
632 | status, data = global_instance().rbd_iscsi.content_data.get() | |
633 | if data is None: | |
634 | log.warning("Failed to get RBD iSCSI status") | |
635 | return {} | |
636 | return data | |
637 | ||
638 | @cherrypy.expose | |
639 | def rbd_iscsi(self): | |
640 | template = env.get_template("rbd_iscsi.html") | |
641 | ||
642 | toplevel_data = self._toplevel_data() | |
643 | content_data = self._rbd_iscsi() | |
644 | ||
645 | return template.render( | |
646 | ceph_version=global_instance().version, | |
647 | path_info=cherrypy.request.path_info, | |
31f18b77 FG |
648 | toplevel_data=json.dumps(toplevel_data, indent=2), |
649 | content_data=json.dumps(content_data, indent=2) | |
650 | ) | |
651 | ||
652 | @cherrypy.expose | |
653 | @cherrypy.tools.json_out() | |
c07f9fc5 FG |
654 | def rbd_iscsi_data(self): |
655 | return self._rbd_iscsi() | |
31f18b77 FG |
656 | |
657 | @cherrypy.expose | |
658 | def health(self): | |
659 | template = env.get_template("health.html") | |
660 | return template.render( | |
661 | ceph_version=global_instance().version, | |
c07f9fc5 | 662 | path_info=cherrypy.request.path_info, |
31f18b77 FG |
663 | toplevel_data=json.dumps(self._toplevel_data(), indent=2), |
664 | content_data=json.dumps(self._health(), indent=2) | |
665 | ) | |
666 | ||
667 | @cherrypy.expose | |
668 | def servers(self): | |
669 | template = env.get_template("servers.html") | |
670 | return template.render( | |
671 | ceph_version=global_instance().version, | |
c07f9fc5 | 672 | path_info=cherrypy.request.path_info, |
31f18b77 FG |
673 | toplevel_data=json.dumps(self._toplevel_data(), indent=2), |
674 | content_data=json.dumps(self._servers(), indent=2) | |
675 | ) | |
676 | ||
677 | def _servers(self): | |
31f18b77 FG |
678 | return { |
679 | 'servers': global_instance().list_servers() | |
680 | } | |
681 | ||
682 | @cherrypy.expose | |
683 | @cherrypy.tools.json_out() | |
684 | def servers_data(self): | |
685 | return self._servers() | |
686 | ||
687 | def _health(self): | |
688 | # Fuse osdmap with pg_summary to get description of pools | |
689 | # including their PG states | |
690 | osd_map = global_instance().get_sync_object(OsdMap).data | |
691 | pg_summary = global_instance().get_sync_object(PgSummary).data | |
692 | pools = [] | |
693 | ||
694 | if len(global_instance().pool_stats) == 0: | |
695 | global_instance().update_pool_stats() | |
696 | ||
697 | for pool in osd_map['pools']: | |
698 | pool['pg_status'] = pg_summary['by_pool'][pool['pool'].__str__()] | |
699 | stats = global_instance().pool_stats[pool['pool']] | |
700 | s = {} | |
701 | ||
702 | def get_rate(series): | |
703 | if len(series) >= 2: | |
704 | return (float(series[0][1]) - float(series[1][1])) / (float(series[0][0]) - float(series[1][0])) | |
705 | else: | |
706 | return 0 | |
707 | ||
708 | for stat_name, stat_series in stats.items(): | |
709 | s[stat_name] = { | |
710 | 'latest': stat_series[0][1], | |
711 | 'rate': get_rate(stat_series), | |
712 | 'series': [i for i in stat_series] | |
713 | } | |
714 | pool['stats'] = s | |
715 | pools.append(pool) | |
716 | ||
717 | # Not needed, skip the effort of transmitting this | |
718 | # to UI | |
719 | del osd_map['pg_temp'] | |
720 | ||
224ce89b WB |
721 | df = global_instance().get("df") |
722 | df['stats']['total_objects'] = sum( | |
723 | [p['stats']['objects'] for p in df['pools']]) | |
724 | ||
31f18b77 | 725 | return { |
224ce89b | 726 | "health": self._health_data(), |
31f18b77 FG |
727 | "mon_status": global_instance().get_sync_object( |
728 | MonStatus).data, | |
224ce89b | 729 | "fs_map": global_instance().get_sync_object(FsMap).data, |
31f18b77 FG |
730 | "osd_map": osd_map, |
731 | "clog": list(global_instance().log_buffer), | |
732 | "audit_log": list(global_instance().audit_buffer), | |
224ce89b WB |
733 | "pools": pools, |
734 | "mgr_map": global_instance().get("mgr_map"), | |
735 | "df": df | |
31f18b77 FG |
736 | } |
737 | ||
738 | @cherrypy.expose | |
739 | @cherrypy.tools.json_out() | |
740 | def health_data(self): | |
741 | return self._health() | |
742 | ||
743 | @cherrypy.expose | |
744 | def index(self): | |
745 | return self.health() | |
746 | ||
747 | @cherrypy.expose | |
748 | @cherrypy.tools.json_out() | |
749 | def toplevel_data(self): | |
750 | return self._toplevel_data() | |
751 | ||
752 | def _get_mds_names(self, filesystem_id=None): | |
753 | names = [] | |
754 | ||
755 | fsmap = global_instance().get("fs_map") | |
756 | for fs in fsmap['filesystems']: | |
757 | if filesystem_id is not None and fs['id'] != filesystem_id: | |
758 | continue | |
759 | names.extend([info['name'] for _, info in fs['mdsmap']['info'].items()]) | |
760 | ||
761 | if filesystem_id is None: | |
762 | names.extend(info['name'] for info in fsmap['standbys']) | |
763 | ||
764 | return names | |
765 | ||
766 | @cherrypy.expose | |
767 | @cherrypy.tools.json_out() | |
768 | def mds_counters(self, fs_id): | |
769 | """ | |
770 | Result format: map of daemon name to map of counter to list of datapoints | |
771 | """ | |
772 | ||
773 | # Opinionated list of interesting performance counters for the GUI -- | |
774 | # if you need something else just add it. See how simple life is | |
775 | # when you don't have to write general purpose APIs? | |
776 | counters = [ | |
777 | "mds_server.handle_client_request", | |
778 | "mds_log.ev", | |
779 | "mds_cache.num_strays", | |
780 | "mds.exported", | |
781 | "mds.exported_inodes", | |
782 | "mds.imported", | |
783 | "mds.imported_inodes", | |
784 | "mds.inodes", | |
785 | "mds.caps", | |
786 | "mds.subtrees" | |
787 | ] | |
788 | ||
789 | result = {} | |
790 | mds_names = self._get_mds_names(int(fs_id)) | |
791 | ||
792 | for mds_name in mds_names: | |
793 | result[mds_name] = {} | |
794 | for counter in counters: | |
795 | data = global_instance().get_counter("mds", mds_name, counter) | |
796 | if data is not None: | |
797 | result[mds_name][counter] = data[counter] | |
798 | else: | |
799 | result[mds_name][counter] = [] | |
800 | ||
801 | return dict(result) | |
802 | ||
c07f9fc5 FG |
803 | @cherrypy.expose |
804 | @cherrypy.tools.json_out() | |
805 | def get_counter(self, type, id, path): | |
806 | return global_instance().get_counter(type, id, path) | |
807 | ||
808 | @cherrypy.expose | |
809 | @cherrypy.tools.json_out() | |
810 | def get_perf_schema(self, **args): | |
811 | type = args.get('type', '') | |
812 | id = args.get('id', '') | |
813 | schema = global_instance().get_perf_schema(type, id) | |
814 | ret = dict() | |
815 | for k1 in schema.keys(): # 'perf_schema' | |
816 | ret[k1] = collections.OrderedDict() | |
817 | for k2 in sorted(schema[k1].keys()): | |
818 | sorted_dict = collections.OrderedDict( | |
819 | sorted(schema[k1][k2].items(), key=lambda i: i[0]) | |
820 | ) | |
821 | ret[k1][k2] = sorted_dict | |
822 | return ret | |
823 | ||
224ce89b WB |
824 | server_addr = self.get_localized_config('server_addr', '::') |
825 | server_port = self.get_localized_config('server_port', '7000') | |
31f18b77 | 826 | if server_addr is None: |
c07f9fc5 | 827 | raise RuntimeError('no server_addr configured; try "ceph config-key set mgr/dashboard/server_addr <ip>"') |
31f18b77 FG |
828 | log.info("server_addr: %s server_port: %s" % (server_addr, server_port)) |
829 | cherrypy.config.update({ | |
830 | 'server.socket_host': server_addr, | |
831 | 'server.socket_port': int(server_port), | |
832 | 'engine.autoreload.on': False | |
833 | }) | |
834 | ||
835 | static_dir = os.path.join(current_dir, 'static') | |
836 | conf = { | |
837 | "/static": { | |
838 | "tools.staticdir.on": True, | |
839 | 'tools.staticdir.dir': static_dir | |
840 | } | |
841 | } | |
842 | log.info("Serving static from {0}".format(static_dir)) | |
c07f9fc5 FG |
843 | |
844 | class OSDEndpoint(EndPoint): | |
845 | def _osd(self, osd_id): | |
846 | osd_id = int(osd_id) | |
847 | ||
848 | osd_map = global_instance().get("osd_map") | |
849 | ||
850 | osd = None | |
851 | for o in osd_map['osds']: | |
852 | if o['osd'] == osd_id: | |
853 | osd = o | |
854 | break | |
855 | ||
856 | assert osd is not None # TODO 400 | |
857 | ||
858 | osd_spec = "{0}".format(osd_id) | |
859 | ||
860 | osd_metadata = global_instance().get_metadata( | |
861 | "osd", osd_spec) | |
862 | ||
863 | result = CommandResult("") | |
864 | global_instance().send_command(result, "osd", osd_spec, | |
865 | json.dumps({ | |
866 | "prefix": "perf histogram dump", | |
867 | }), | |
868 | "") | |
869 | r, outb, outs = result.wait() | |
870 | assert r == 0 | |
871 | histogram = json.loads(outb) | |
872 | ||
873 | return { | |
874 | "osd": osd, | |
875 | "osd_metadata": osd_metadata, | |
876 | "osd_histogram": histogram | |
877 | } | |
878 | ||
879 | @cherrypy.expose | |
880 | def perf(self, osd_id): | |
881 | template = env.get_template("osd_perf.html") | |
882 | toplevel_data = self._toplevel_data() | |
883 | ||
884 | return template.render( | |
885 | ceph_version=global_instance().version, | |
886 | path_info='/osd' + cherrypy.request.path_info, | |
887 | toplevel_data=json.dumps(toplevel_data, indent=2), | |
888 | content_data=json.dumps(self._osd(osd_id), indent=2) | |
889 | ) | |
890 | ||
891 | @cherrypy.expose | |
892 | @cherrypy.tools.json_out() | |
893 | def perf_data(self, osd_id): | |
894 | return self._osd(osd_id) | |
895 | ||
896 | @cherrypy.expose | |
897 | @cherrypy.tools.json_out() | |
898 | def list_data(self): | |
899 | return self._osds_by_server() | |
900 | ||
901 | def _osd_summary(self, osd_id, osd_info): | |
902 | """ | |
903 | The info used for displaying an OSD in a table | |
904 | """ | |
905 | ||
906 | osd_spec = "{0}".format(osd_id) | |
907 | ||
908 | result = {} | |
909 | result['id'] = osd_id | |
910 | result['stats'] = {} | |
911 | result['stats_history'] = {} | |
912 | ||
913 | # Counter stats | |
914 | for s in ['osd.op_w', 'osd.op_in_bytes', 'osd.op_r', 'osd.op_out_bytes']: | |
915 | result['stats'][s.split(".")[1]] = global_instance().get_rate('osd', osd_spec, s) | |
916 | result['stats_history'][s.split(".")[1]] = \ | |
917 | global_instance().get_counter('osd', osd_spec, s)[s] | |
918 | ||
919 | # Gauge stats | |
920 | for s in ["osd.numpg", "osd.stat_bytes", "osd.stat_bytes_used"]: | |
921 | result['stats'][s.split(".")[1]] = global_instance().get_latest('osd', osd_spec, s) | |
922 | ||
923 | result['up'] = osd_info['up'] | |
924 | result['in'] = osd_info['in'] | |
925 | ||
926 | result['url'] = "/osd/perf/{0}".format(osd_id) | |
927 | ||
928 | return result | |
929 | ||
930 | def _osds_by_server(self): | |
931 | result = defaultdict(list) | |
932 | servers = global_instance().list_servers() | |
933 | ||
934 | osd_map = global_instance().get_sync_object(OsdMap) | |
935 | ||
936 | for server in servers: | |
937 | hostname = server['hostname'] | |
938 | services = server['services'] | |
939 | first = True | |
940 | for s in services: | |
941 | if s["type"] == "osd": | |
942 | osd_id = int(s["id"]) | |
943 | # If metadata doesn't tally with osdmap, drop it. | |
944 | if osd_id not in osd_map.osds_by_id: | |
945 | global_instance().log.warn( | |
946 | "OSD service {0} missing in OSDMap, stale metadata?".format(osd_id)) | |
947 | continue | |
948 | summary = self._osd_summary(osd_id, | |
949 | osd_map.osds_by_id[osd_id]) | |
950 | ||
951 | if first: | |
952 | # A little helper for rendering | |
953 | summary['first'] = True | |
954 | first = False | |
955 | result[hostname].append(summary) | |
956 | ||
957 | global_instance().log.warn("result.size {0} servers.size {1}".format( | |
958 | len(result), len(servers) | |
959 | )) | |
960 | ||
961 | # Return list form for convenience of rendering | |
962 | return result.items() | |
963 | ||
964 | @cherrypy.expose | |
965 | def index(self): | |
966 | """ | |
967 | List of all OSDS grouped by host | |
968 | :return: | |
969 | """ | |
970 | ||
971 | template = env.get_template("osds.html") | |
972 | toplevel_data = self._toplevel_data() | |
973 | ||
974 | content_data = { | |
975 | "osds_by_server": self._osds_by_server() | |
976 | } | |
977 | ||
978 | return template.render( | |
979 | ceph_version=global_instance().version, | |
980 | path_info='/osd' + cherrypy.request.path_info, | |
981 | toplevel_data=json.dumps(toplevel_data, indent=2), | |
982 | content_data=json.dumps(content_data, indent=2) | |
983 | ) | |
984 | ||
31f18b77 | 985 | cherrypy.tree.mount(Root(), "/", conf) |
c07f9fc5 | 986 | cherrypy.tree.mount(OSDEndpoint(), "/osd", conf) |
31f18b77 FG |
987 | |
988 | log.info("Starting engine...") | |
989 | cherrypy.engine.start() | |
990 | log.info("Waiting for engine...") | |
991 | cherrypy.engine.block() | |
992 | log.info("Engine done.") |