]>
Commit | Line | Data |
---|---|---|
31f18b77 FG |
1 | |
2 | """ | |
3 | Demonstrate writing a Ceph web interface inside a mgr module. | |
4 | """ | |
5 | ||
6 | # We must share a global reference to this instance, because it is the | |
7 | # gatekeeper to all accesses to data from the C++ side (e.g. the REST API | |
8 | # request handlers need to see it) | |
9 | from collections import defaultdict | |
10 | import collections | |
11 | ||
12 | _global_instance = {'plugin': None} | |
13 | def global_instance(): | |
14 | assert _global_instance['plugin'] is not None | |
15 | return _global_instance['plugin'] | |
16 | ||
17 | ||
18 | import os | |
19 | import logging | |
20 | import logging.config | |
21 | import json | |
22 | import sys | |
23 | import time | |
24 | import threading | |
3efd9988 | 25 | import socket |
31f18b77 FG |
26 | |
27 | import cherrypy | |
28 | import jinja2 | |
29 | ||
3efd9988 | 30 | from mgr_module import MgrModule, MgrStandbyModule, CommandResult |
31f18b77 FG |
31 | |
32 | from types import OsdMap, NotFound, Config, FsMap, MonMap, \ | |
33 | PgSummary, Health, MonStatus | |
34 | ||
35 | import rados | |
c07f9fc5 FG |
36 | import rbd_iscsi |
37 | import rbd_mirroring | |
224ce89b | 38 | from rbd_ls import RbdLs, RbdPoolLs |
31f18b77 FG |
39 | from cephfs_clients import CephFSClients |
40 | ||
31f18b77 FG |
41 | log = logging.getLogger("dashboard") |
42 | ||
43 | ||
44 | # How many cluster log lines shall we hold onto in our | |
45 | # python module for the convenience of the GUI? | |
46 | LOG_BUFFER_SIZE = 30 | |
47 | ||
48 | # cherrypy likes to sys.exit on error. don't let it take us down too! | |
3efd9988 | 49 | def os_exit_noop(*args, **kwargs): |
31f18b77 FG |
50 | pass |
51 | ||
52 | os._exit = os_exit_noop | |
53 | ||
54 | ||
55 | def recurse_refs(root, path): | |
56 | if isinstance(root, dict): | |
57 | for k, v in root.items(): | |
58 | recurse_refs(v, path + "->%s" % k) | |
59 | elif isinstance(root, list): | |
60 | for n, i in enumerate(root): | |
61 | recurse_refs(i, path + "[%d]" % n) | |
62 | ||
63 | log.info("%s %d (%s)" % (path, sys.getrefcount(root), root.__class__)) | |
64 | ||
3efd9988 FG |
65 | def get_prefixed_url(url): |
66 | return global_instance().url_prefix + url | |
67 | ||
68 | ||
69 | ||
70 | class StandbyModule(MgrStandbyModule): | |
71 | def serve(self): | |
72 | server_addr = self.get_localized_config('server_addr', '::') | |
73 | server_port = self.get_localized_config('server_port', '7000') | |
74 | if server_addr is None: | |
75 | raise RuntimeError('no server_addr configured; try "ceph config-key set mgr/dashboard/server_addr <ip>"') | |
76 | log.info("server_addr: %s server_port: %s" % (server_addr, server_port)) | |
77 | cherrypy.config.update({ | |
78 | 'server.socket_host': server_addr, | |
79 | 'server.socket_port': int(server_port), | |
80 | 'engine.autoreload.on': False | |
81 | }) | |
82 | ||
83 | current_dir = os.path.dirname(os.path.abspath(__file__)) | |
84 | jinja_loader = jinja2.FileSystemLoader(current_dir) | |
85 | env = jinja2.Environment(loader=jinja_loader) | |
86 | ||
87 | module = self | |
88 | ||
89 | class Root(object): | |
90 | @cherrypy.expose | |
91 | def index(self): | |
92 | active_uri = module.get_active_uri() | |
93 | if active_uri: | |
94 | log.info("Redirecting to active '{0}'".format(active_uri)) | |
95 | raise cherrypy.HTTPRedirect(active_uri) | |
96 | else: | |
97 | template = env.get_template("standby.html") | |
98 | return template.render(delay=5) | |
99 | ||
100 | cherrypy.tree.mount(Root(), "/", {}) | |
101 | log.info("Starting engine...") | |
102 | cherrypy.engine.start() | |
103 | log.info("Waiting for engine...") | |
104 | cherrypy.engine.wait(state=cherrypy.engine.states.STOPPED) | |
105 | log.info("Engine done.") | |
106 | ||
107 | def shutdown(self): | |
108 | log.info("Stopping server...") | |
109 | cherrypy.engine.wait(state=cherrypy.engine.states.STARTED) | |
110 | cherrypy.engine.stop() | |
111 | log.info("Stopped server") | |
112 | ||
31f18b77 FG |
113 | |
114 | class Module(MgrModule): | |
115 | def __init__(self, *args, **kwargs): | |
116 | super(Module, self).__init__(*args, **kwargs) | |
117 | _global_instance['plugin'] = self | |
118 | self.log.info("Constructing module {0}: instance {1}".format( | |
119 | __name__, _global_instance)) | |
120 | ||
121 | self.log_primed = False | |
122 | self.log_buffer = collections.deque(maxlen=LOG_BUFFER_SIZE) | |
123 | self.audit_buffer = collections.deque(maxlen=LOG_BUFFER_SIZE) | |
124 | ||
125 | # Keep a librados instance for those that need it. | |
126 | self._rados = None | |
127 | ||
128 | # Stateful instances of RbdLs, hold cached results. Key to dict | |
129 | # is pool name. | |
130 | self.rbd_ls = {} | |
131 | ||
224ce89b WB |
132 | # Stateful instance of RbdPoolLs, hold cached list of RBD |
133 | # pools | |
134 | self.rbd_pool_ls = RbdPoolLs(self) | |
135 | ||
c07f9fc5 FG |
136 | # Stateful instance of RbdISCSI |
137 | self.rbd_iscsi = rbd_iscsi.Controller(self) | |
138 | ||
139 | # Stateful instance of RbdMirroring, hold cached results. | |
140 | self.rbd_mirroring = rbd_mirroring.Controller(self) | |
141 | ||
31f18b77 FG |
142 | # Stateful instances of CephFSClients, hold cached results. Key to |
143 | # dict is FSCID | |
144 | self.cephfs_clients = {} | |
145 | ||
146 | # A short history of pool df stats | |
147 | self.pool_stats = defaultdict(lambda: defaultdict( | |
148 | lambda: collections.deque(maxlen=10))) | |
149 | ||
3efd9988 FG |
150 | # A prefix for all URLs to use the dashboard with a reverse http proxy |
151 | self.url_prefix = '' | |
152 | ||
31f18b77 FG |
153 | @property |
154 | def rados(self): | |
155 | """ | |
156 | A librados instance to be shared by any classes within | |
157 | this mgr module that want one. | |
158 | """ | |
159 | if self._rados: | |
160 | return self._rados | |
161 | ||
3efd9988 | 162 | ctx_capsule = self.get_context() |
31f18b77 FG |
163 | self._rados = rados.Rados(context=ctx_capsule) |
164 | self._rados.connect() | |
165 | ||
166 | return self._rados | |
167 | ||
31f18b77 FG |
168 | def update_pool_stats(self): |
169 | df = global_instance().get("df") | |
170 | pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) | |
171 | now = time.time() | |
172 | for pool_id, stats in pool_stats.items(): | |
173 | for stat_name, stat_val in stats.items(): | |
174 | self.pool_stats[pool_id][stat_name].appendleft((now, stat_val)) | |
175 | ||
176 | def notify(self, notify_type, notify_val): | |
177 | if notify_type == "clog": | |
178 | # Only store log messages once we've done our initial load, | |
179 | # so that we don't end up duplicating. | |
180 | if self.log_primed: | |
181 | if notify_val['channel'] == "audit": | |
182 | self.audit_buffer.appendleft(notify_val) | |
183 | else: | |
184 | self.log_buffer.appendleft(notify_val) | |
185 | elif notify_type == "pg_summary": | |
186 | self.update_pool_stats() | |
187 | else: | |
188 | pass | |
189 | ||
190 | def get_sync_object(self, object_type, path=None): | |
191 | if object_type == OsdMap: | |
192 | data = self.get("osd_map") | |
193 | ||
194 | assert data is not None | |
195 | ||
196 | data['tree'] = self.get("osd_map_tree") | |
197 | data['crush'] = self.get("osd_map_crush") | |
198 | data['crush_map_text'] = self.get("osd_map_crush_map_text") | |
199 | data['osd_metadata'] = self.get("osd_metadata") | |
200 | obj = OsdMap(data) | |
201 | elif object_type == Config: | |
202 | data = self.get("config") | |
203 | obj = Config( data) | |
204 | elif object_type == MonMap: | |
205 | data = self.get("mon_map") | |
206 | obj = MonMap(data) | |
207 | elif object_type == FsMap: | |
208 | data = self.get("fs_map") | |
209 | obj = FsMap(data) | |
210 | elif object_type == PgSummary: | |
211 | data = self.get("pg_summary") | |
212 | self.log.debug("JSON: {0}".format(data)) | |
213 | obj = PgSummary(data) | |
214 | elif object_type == Health: | |
215 | data = self.get("health") | |
216 | obj = Health(json.loads(data['json'])) | |
217 | elif object_type == MonStatus: | |
218 | data = self.get("mon_status") | |
219 | obj = MonStatus(json.loads(data['json'])) | |
220 | else: | |
221 | raise NotImplementedError(object_type) | |
222 | ||
223 | # TODO: move 'path' handling up into C++ land so that we only | |
224 | # Pythonize the part we're interested in | |
225 | if path: | |
226 | try: | |
227 | for part in path: | |
228 | if isinstance(obj, dict): | |
229 | obj = obj[part] | |
230 | else: | |
231 | obj = getattr(obj, part) | |
232 | except (AttributeError, KeyError): | |
233 | raise NotFound(object_type, path) | |
234 | ||
235 | return obj | |
236 | ||
237 | def shutdown(self): | |
238 | log.info("Stopping server...") | |
239 | cherrypy.engine.exit() | |
240 | log.info("Stopped server") | |
241 | ||
242 | log.info("Stopping librados...") | |
243 | if self._rados: | |
244 | self._rados.shutdown() | |
245 | log.info("Stopped librados.") | |
246 | ||
247 | def get_latest(self, daemon_type, daemon_name, stat): | |
248 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] | |
249 | if data: | |
250 | return data[-1][1] | |
251 | else: | |
252 | return 0 | |
253 | ||
254 | def get_rate(self, daemon_type, daemon_name, stat): | |
255 | data = self.get_counter(daemon_type, daemon_name, stat)[stat] | |
256 | ||
257 | if data and len(data) > 1: | |
258 | return (data[-1][1] - data[-2][1]) / float(data[-1][0] - data[-2][0]) | |
259 | else: | |
260 | return 0 | |
261 | ||
262 | def format_dimless(self, n, width, colored=True): | |
263 | """ | |
264 | Format a number without units, so as to fit into `width` characters, substituting | |
265 | an appropriate unit suffix. | |
266 | """ | |
267 | units = [' ', 'k', 'M', 'G', 'T', 'P'] | |
268 | unit = 0 | |
269 | while len("%s" % (int(n) // (1000**unit))) > width - 1: | |
270 | unit += 1 | |
271 | ||
272 | if unit > 0: | |
273 | truncated_float = ("%f" % (n / (1000.0 ** unit)))[0:width - 1] | |
274 | if truncated_float[-1] == '.': | |
275 | truncated_float = " " + truncated_float[0:-1] | |
276 | else: | |
277 | truncated_float = "%{wid}d".format(wid=width-1) % n | |
278 | formatted = "%s%s" % (truncated_float, units[unit]) | |
279 | ||
280 | if colored: | |
281 | # TODO: html equivalent | |
282 | # if n == 0: | |
283 | # color = self.BLACK, False | |
284 | # else: | |
285 | # color = self.YELLOW, False | |
286 | # return self.bold(self.colorize(formatted[0:-1], color[0], color[1])) \ | |
287 | # + self.bold(self.colorize(formatted[-1], self.BLACK, False)) | |
288 | return formatted | |
289 | else: | |
290 | return formatted | |
291 | ||
292 | def fs_status(self, fs_id): | |
293 | mds_versions = defaultdict(list) | |
294 | ||
295 | fsmap = self.get("fs_map") | |
296 | filesystem = None | |
297 | for fs in fsmap['filesystems']: | |
298 | if fs['id'] == fs_id: | |
299 | filesystem = fs | |
300 | break | |
301 | ||
302 | rank_table = [] | |
303 | ||
304 | mdsmap = filesystem['mdsmap'] | |
305 | ||
306 | client_count = 0 | |
307 | ||
308 | for rank in mdsmap["in"]: | |
309 | up = "mds_{0}".format(rank) in mdsmap["up"] | |
310 | if up: | |
311 | gid = mdsmap['up']["mds_{0}".format(rank)] | |
312 | info = mdsmap['info']['gid_{0}'.format(gid)] | |
313 | dns = self.get_latest("mds", info['name'], "mds.inodes") | |
314 | inos = self.get_latest("mds", info['name'], "mds_mem.ino") | |
315 | ||
316 | if rank == 0: | |
317 | client_count = self.get_latest("mds", info['name'], | |
318 | "mds_sessions.session_count") | |
319 | elif client_count == 0: | |
320 | # In case rank 0 was down, look at another rank's | |
321 | # sessionmap to get an indication of clients. | |
322 | client_count = self.get_latest("mds", info['name'], | |
323 | "mds_sessions.session_count") | |
324 | ||
325 | laggy = "laggy_since" in info | |
326 | ||
327 | state = info['state'].split(":")[1] | |
328 | if laggy: | |
329 | state += "(laggy)" | |
330 | ||
331 | # if state == "active" and not laggy: | |
332 | # c_state = self.colorize(state, self.GREEN) | |
333 | # else: | |
334 | # c_state = self.colorize(state, self.YELLOW) | |
335 | ||
336 | # Populate based on context of state, e.g. client | |
337 | # ops for an active daemon, replay progress, reconnect | |
338 | # progress | |
339 | activity = "" | |
340 | ||
341 | if state == "active": | |
342 | activity = "Reqs: " + self.format_dimless( | |
343 | self.get_rate("mds", info['name'], "mds_server.handle_client_request"), | |
344 | 5 | |
345 | ) + "/s" | |
346 | ||
347 | metadata = self.get_metadata('mds', info['name']) | |
181888fb | 348 | mds_versions[metadata.get('ceph_version', 'unknown')].append(info['name']) |
31f18b77 FG |
349 | rank_table.append( |
350 | { | |
351 | "rank": rank, | |
352 | "state": state, | |
353 | "mds": info['name'], | |
354 | "activity": activity, | |
355 | "dns": dns, | |
356 | "inos": inos | |
357 | } | |
358 | ) | |
359 | ||
360 | else: | |
361 | rank_table.append( | |
362 | { | |
363 | "rank": rank, | |
364 | "state": "failed", | |
365 | "mds": "", | |
366 | "activity": "", | |
367 | "dns": 0, | |
368 | "inos": 0 | |
369 | } | |
370 | ) | |
371 | ||
372 | # Find the standby replays | |
373 | for gid_str, daemon_info in mdsmap['info'].iteritems(): | |
374 | if daemon_info['state'] != "up:standby-replay": | |
375 | continue | |
376 | ||
377 | inos = self.get_latest("mds", daemon_info['name'], "mds_mem.ino") | |
378 | dns = self.get_latest("mds", daemon_info['name'], "mds.inodes") | |
379 | ||
380 | activity = "Evts: " + self.format_dimless( | |
381 | self.get_rate("mds", daemon_info['name'], "mds_log.replay"), | |
382 | 5 | |
383 | ) + "/s" | |
384 | ||
385 | rank_table.append( | |
386 | { | |
387 | "rank": "{0}-s".format(daemon_info['rank']), | |
388 | "state": "standby-replay", | |
389 | "mds": daemon_info['name'], | |
390 | "activity": activity, | |
391 | "dns": dns, | |
392 | "inos": inos | |
393 | } | |
394 | ) | |
395 | ||
396 | df = self.get("df") | |
397 | pool_stats = dict([(p['id'], p['stats']) for p in df['pools']]) | |
398 | osdmap = self.get("osd_map") | |
399 | pools = dict([(p['pool'], p) for p in osdmap['pools']]) | |
400 | metadata_pool_id = mdsmap['metadata_pool'] | |
401 | data_pool_ids = mdsmap['data_pools'] | |
402 | ||
403 | pools_table = [] | |
404 | for pool_id in [metadata_pool_id] + data_pool_ids: | |
405 | pool_type = "metadata" if pool_id == metadata_pool_id else "data" | |
406 | stats = pool_stats[pool_id] | |
407 | pools_table.append({ | |
408 | "pool": pools[pool_id]['pool_name'], | |
409 | "type": pool_type, | |
410 | "used": stats['bytes_used'], | |
411 | "avail": stats['max_avail'] | |
412 | }) | |
413 | ||
414 | standby_table = [] | |
415 | for standby in fsmap['standbys']: | |
416 | metadata = self.get_metadata('mds', standby['name']) | |
181888fb | 417 | mds_versions[metadata.get('ceph_version', 'unknown')].append(standby['name']) |
31f18b77 FG |
418 | |
419 | standby_table.append({ | |
420 | 'name': standby['name'] | |
421 | }) | |
422 | ||
423 | return { | |
424 | "filesystem": { | |
425 | "id": fs_id, | |
426 | "name": mdsmap['fs_name'], | |
427 | "client_count": client_count, | |
3efd9988 | 428 | "clients_url": get_prefixed_url("/clients/{0}/".format(fs_id)), |
31f18b77 FG |
429 | "ranks": rank_table, |
430 | "pools": pools_table | |
431 | }, | |
432 | "standbys": standby_table, | |
433 | "versions": mds_versions | |
434 | } | |
435 | ||
436 | def serve(self): | |
437 | current_dir = os.path.dirname(os.path.abspath(__file__)) | |
438 | ||
439 | jinja_loader = jinja2.FileSystemLoader(current_dir) | |
440 | env = jinja2.Environment(loader=jinja_loader) | |
441 | ||
442 | result = CommandResult("") | |
443 | self.send_command(result, "mon", "", json.dumps({ | |
444 | "prefix":"log last", | |
445 | "format": "json" | |
446 | }), "") | |
447 | r, outb, outs = result.wait() | |
448 | if r != 0: | |
449 | # Oh well. We won't let this stop us though. | |
450 | self.log.error("Error fetching log history (r={0}, \"{1}\")".format( | |
451 | r, outs)) | |
452 | else: | |
453 | try: | |
454 | lines = json.loads(outb) | |
455 | except ValueError: | |
456 | self.log.error("Error decoding log history") | |
457 | else: | |
458 | for l in lines: | |
459 | if l['channel'] == 'audit': | |
460 | self.audit_buffer.appendleft(l) | |
461 | else: | |
462 | self.log_buffer.appendleft(l) | |
463 | ||
464 | self.log_primed = True | |
465 | ||
c07f9fc5 FG |
466 | class EndPoint(object): |
467 | def _health_data(self): | |
468 | health = global_instance().get_sync_object(Health).data | |
469 | # Transform the `checks` dict into a list for the convenience | |
470 | # of rendering from javascript. | |
471 | checks = [] | |
472 | for k, v in health['checks'].iteritems(): | |
473 | v['type'] = k | |
474 | checks.append(v) | |
475 | ||
476 | checks = sorted(checks, cmp=lambda a, b: a['severity'] > b['severity']) | |
477 | ||
478 | health['checks'] = checks | |
479 | ||
480 | return health | |
481 | ||
31f18b77 FG |
482 | def _toplevel_data(self): |
483 | """ | |
484 | Data consumed by the base.html template | |
485 | """ | |
224ce89b WB |
486 | status, data = global_instance().rbd_pool_ls.get() |
487 | if data is None: | |
488 | log.warning("Failed to get RBD pool list") | |
489 | data = [] | |
490 | ||
491 | rbd_pools = sorted([ | |
492 | { | |
493 | "name": name, | |
3efd9988 | 494 | "url": get_prefixed_url("/rbd_pool/{0}/".format(name)) |
224ce89b WB |
495 | } |
496 | for name in data | |
497 | ], key=lambda k: k['name']) | |
498 | ||
c07f9fc5 FG |
499 | status, rbd_mirroring = global_instance().rbd_mirroring.toplevel.get() |
500 | if rbd_mirroring is None: | |
501 | log.warning("Failed to get RBD mirroring summary") | |
502 | rbd_mirroring = {} | |
503 | ||
31f18b77 FG |
504 | fsmap = global_instance().get_sync_object(FsMap) |
505 | filesystems = [ | |
506 | { | |
507 | "id": f['id'], | |
508 | "name": f['mdsmap']['fs_name'], | |
3efd9988 | 509 | "url": get_prefixed_url("/filesystem/{0}/".format(f['id'])) |
31f18b77 FG |
510 | } |
511 | for f in fsmap.data['filesystems'] | |
512 | ] | |
513 | ||
514 | return { | |
224ce89b | 515 | 'rbd_pools': rbd_pools, |
c07f9fc5 | 516 | 'rbd_mirroring': rbd_mirroring, |
224ce89b | 517 | 'health_status': self._health_data()['status'], |
31f18b77 FG |
518 | 'filesystems': filesystems |
519 | } | |
520 | ||
c07f9fc5 | 521 | class Root(EndPoint): |
31f18b77 FG |
522 | @cherrypy.expose |
523 | def filesystem(self, fs_id): | |
524 | template = env.get_template("filesystem.html") | |
525 | ||
526 | toplevel_data = self._toplevel_data() | |
527 | ||
528 | content_data = { | |
529 | "fs_status": global_instance().fs_status(int(fs_id)) | |
530 | } | |
531 | ||
532 | return template.render( | |
3efd9988 | 533 | url_prefix = global_instance().url_prefix, |
31f18b77 | 534 | ceph_version=global_instance().version, |
c07f9fc5 | 535 | path_info=cherrypy.request.path_info, |
31f18b77 FG |
536 | toplevel_data=json.dumps(toplevel_data, indent=2), |
537 | content_data=json.dumps(content_data, indent=2) | |
538 | ) | |
539 | ||
540 | @cherrypy.expose | |
541 | @cherrypy.tools.json_out() | |
542 | def filesystem_data(self, fs_id): | |
543 | return global_instance().fs_status(int(fs_id)) | |
544 | ||
31f18b77 FG |
545 | def _clients(self, fs_id): |
546 | cephfs_clients = global_instance().cephfs_clients.get(fs_id, None) | |
547 | if cephfs_clients is None: | |
548 | cephfs_clients = CephFSClients(global_instance(), fs_id) | |
549 | global_instance().cephfs_clients[fs_id] = cephfs_clients | |
550 | ||
551 | status, clients = cephfs_clients.get() | |
552 | #TODO do something sensible with status | |
553 | ||
554 | # Decorate the metadata with some fields that will be | |
555 | # indepdendent of whether it's a kernel or userspace | |
556 | # client, so that the javascript doesn't have to grok that. | |
557 | for client in clients: | |
558 | if "ceph_version" in client['client_metadata']: | |
559 | client['type'] = "userspace" | |
560 | client['version'] = client['client_metadata']['ceph_version'] | |
561 | client['hostname'] = client['client_metadata']['hostname'] | |
562 | elif "kernel_version" in client['client_metadata']: | |
563 | client['type'] = "kernel" | |
224ce89b | 564 | client['version'] = client['client_metadata']['kernel_version'] |
31f18b77 FG |
565 | client['hostname'] = client['client_metadata']['hostname'] |
566 | else: | |
567 | client['type'] = "unknown" | |
568 | client['version'] = "" | |
569 | client['hostname'] = "" | |
570 | ||
571 | return clients | |
572 | ||
573 | @cherrypy.expose | |
224ce89b WB |
574 | def clients(self, fscid_str): |
575 | try: | |
576 | fscid = int(fscid_str) | |
577 | except ValueError: | |
578 | raise cherrypy.HTTPError(400, | |
579 | "Invalid filesystem id {0}".format(fscid_str)) | |
580 | ||
581 | try: | |
582 | fs_name = FsMap(global_instance().get( | |
583 | "fs_map")).get_filesystem(fscid)['mdsmap']['fs_name'] | |
584 | except NotFound: | |
585 | log.warning("Missing FSCID, dumping fsmap:\n{0}".format( | |
586 | json.dumps(global_instance().get("fs_map"), indent=2) | |
587 | )) | |
588 | raise cherrypy.HTTPError(404, | |
589 | "No filesystem with id {0}".format(fscid)) | |
590 | ||
591 | clients = self._clients(fscid) | |
31f18b77 FG |
592 | global_instance().log.debug(json.dumps(clients, indent=2)) |
593 | content_data = { | |
594 | "clients": clients, | |
224ce89b WB |
595 | "fs_name": fs_name, |
596 | "fscid": fscid, | |
3efd9988 | 597 | "fs_url": get_prefixed_url("/filesystem/" + fscid_str + "/") |
31f18b77 FG |
598 | } |
599 | ||
224ce89b | 600 | template = env.get_template("clients.html") |
31f18b77 | 601 | return template.render( |
3efd9988 | 602 | url_prefix = global_instance().url_prefix, |
31f18b77 | 603 | ceph_version=global_instance().version, |
c07f9fc5 | 604 | path_info=cherrypy.request.path_info, |
224ce89b | 605 | toplevel_data=json.dumps(self._toplevel_data(), indent=2), |
31f18b77 FG |
606 | content_data=json.dumps(content_data, indent=2) |
607 | ) | |
608 | ||
609 | @cherrypy.expose | |
610 | @cherrypy.tools.json_out() | |
611 | def clients_data(self, fs_id): | |
612 | return self._clients(int(fs_id)) | |
613 | ||
c07f9fc5 | 614 | def _rbd_pool(self, pool_name): |
31f18b77 FG |
615 | rbd_ls = global_instance().rbd_ls.get(pool_name, None) |
616 | if rbd_ls is None: | |
617 | rbd_ls = RbdLs(global_instance(), pool_name) | |
618 | global_instance().rbd_ls[pool_name] = rbd_ls | |
619 | ||
620 | status, value = rbd_ls.get() | |
621 | ||
622 | interval = 5 | |
623 | ||
624 | wait = interval - rbd_ls.latency | |
625 | def wait_and_load(): | |
626 | time.sleep(wait) | |
627 | rbd_ls.get() | |
628 | ||
629 | threading.Thread(target=wait_and_load).start() | |
630 | ||
631 | assert status != RbdLs.VALUE_NONE # FIXME bubble status up to UI | |
632 | return value | |
633 | ||
634 | @cherrypy.expose | |
c07f9fc5 FG |
635 | def rbd_pool(self, pool_name): |
636 | template = env.get_template("rbd_pool.html") | |
31f18b77 FG |
637 | |
638 | toplevel_data = self._toplevel_data() | |
639 | ||
c07f9fc5 | 640 | images = self._rbd_pool(pool_name) |
31f18b77 FG |
641 | content_data = { |
642 | "images": images, | |
643 | "pool_name": pool_name | |
644 | } | |
645 | ||
646 | return template.render( | |
3efd9988 | 647 | url_prefix = global_instance().url_prefix, |
31f18b77 | 648 | ceph_version=global_instance().version, |
c07f9fc5 FG |
649 | path_info=cherrypy.request.path_info, |
650 | toplevel_data=json.dumps(toplevel_data, indent=2), | |
651 | content_data=json.dumps(content_data, indent=2) | |
652 | ) | |
653 | ||
654 | @cherrypy.expose | |
655 | @cherrypy.tools.json_out() | |
656 | def rbd_pool_data(self, pool_name): | |
657 | return self._rbd_pool(pool_name) | |
658 | ||
659 | def _rbd_mirroring(self): | |
660 | status, data = global_instance().rbd_mirroring.content_data.get() | |
661 | if data is None: | |
662 | log.warning("Failed to get RBD mirroring status") | |
663 | return {} | |
664 | return data | |
665 | ||
666 | @cherrypy.expose | |
667 | def rbd_mirroring(self): | |
668 | template = env.get_template("rbd_mirroring.html") | |
669 | ||
670 | toplevel_data = self._toplevel_data() | |
671 | content_data = self._rbd_mirroring() | |
672 | ||
673 | return template.render( | |
3efd9988 | 674 | url_prefix = global_instance().url_prefix, |
c07f9fc5 FG |
675 | ceph_version=global_instance().version, |
676 | path_info=cherrypy.request.path_info, | |
677 | toplevel_data=json.dumps(toplevel_data, indent=2), | |
678 | content_data=json.dumps(content_data, indent=2) | |
679 | ) | |
680 | ||
681 | @cherrypy.expose | |
682 | @cherrypy.tools.json_out() | |
683 | def rbd_mirroring_data(self): | |
684 | return self._rbd_mirroring() | |
685 | ||
686 | def _rbd_iscsi(self): | |
687 | status, data = global_instance().rbd_iscsi.content_data.get() | |
688 | if data is None: | |
689 | log.warning("Failed to get RBD iSCSI status") | |
690 | return {} | |
691 | return data | |
692 | ||
693 | @cherrypy.expose | |
694 | def rbd_iscsi(self): | |
695 | template = env.get_template("rbd_iscsi.html") | |
696 | ||
697 | toplevel_data = self._toplevel_data() | |
698 | content_data = self._rbd_iscsi() | |
699 | ||
700 | return template.render( | |
3efd9988 | 701 | url_prefix = global_instance().url_prefix, |
c07f9fc5 FG |
702 | ceph_version=global_instance().version, |
703 | path_info=cherrypy.request.path_info, | |
31f18b77 FG |
704 | toplevel_data=json.dumps(toplevel_data, indent=2), |
705 | content_data=json.dumps(content_data, indent=2) | |
706 | ) | |
707 | ||
708 | @cherrypy.expose | |
709 | @cherrypy.tools.json_out() | |
c07f9fc5 FG |
710 | def rbd_iscsi_data(self): |
711 | return self._rbd_iscsi() | |
31f18b77 FG |
712 | |
713 | @cherrypy.expose | |
714 | def health(self): | |
715 | template = env.get_template("health.html") | |
716 | return template.render( | |
3efd9988 | 717 | url_prefix = global_instance().url_prefix, |
31f18b77 | 718 | ceph_version=global_instance().version, |
c07f9fc5 | 719 | path_info=cherrypy.request.path_info, |
31f18b77 FG |
720 | toplevel_data=json.dumps(self._toplevel_data(), indent=2), |
721 | content_data=json.dumps(self._health(), indent=2) | |
722 | ) | |
723 | ||
724 | @cherrypy.expose | |
725 | def servers(self): | |
726 | template = env.get_template("servers.html") | |
727 | return template.render( | |
3efd9988 | 728 | url_prefix = global_instance().url_prefix, |
31f18b77 | 729 | ceph_version=global_instance().version, |
c07f9fc5 | 730 | path_info=cherrypy.request.path_info, |
31f18b77 FG |
731 | toplevel_data=json.dumps(self._toplevel_data(), indent=2), |
732 | content_data=json.dumps(self._servers(), indent=2) | |
733 | ) | |
734 | ||
735 | def _servers(self): | |
31f18b77 FG |
736 | return { |
737 | 'servers': global_instance().list_servers() | |
738 | } | |
739 | ||
740 | @cherrypy.expose | |
741 | @cherrypy.tools.json_out() | |
742 | def servers_data(self): | |
743 | return self._servers() | |
744 | ||
745 | def _health(self): | |
746 | # Fuse osdmap with pg_summary to get description of pools | |
747 | # including their PG states | |
748 | osd_map = global_instance().get_sync_object(OsdMap).data | |
749 | pg_summary = global_instance().get_sync_object(PgSummary).data | |
750 | pools = [] | |
751 | ||
752 | if len(global_instance().pool_stats) == 0: | |
753 | global_instance().update_pool_stats() | |
754 | ||
755 | for pool in osd_map['pools']: | |
756 | pool['pg_status'] = pg_summary['by_pool'][pool['pool'].__str__()] | |
757 | stats = global_instance().pool_stats[pool['pool']] | |
758 | s = {} | |
759 | ||
760 | def get_rate(series): | |
761 | if len(series) >= 2: | |
762 | return (float(series[0][1]) - float(series[1][1])) / (float(series[0][0]) - float(series[1][0])) | |
763 | else: | |
764 | return 0 | |
765 | ||
766 | for stat_name, stat_series in stats.items(): | |
767 | s[stat_name] = { | |
768 | 'latest': stat_series[0][1], | |
769 | 'rate': get_rate(stat_series), | |
770 | 'series': [i for i in stat_series] | |
771 | } | |
772 | pool['stats'] = s | |
773 | pools.append(pool) | |
774 | ||
775 | # Not needed, skip the effort of transmitting this | |
776 | # to UI | |
777 | del osd_map['pg_temp'] | |
778 | ||
224ce89b WB |
779 | df = global_instance().get("df") |
780 | df['stats']['total_objects'] = sum( | |
781 | [p['stats']['objects'] for p in df['pools']]) | |
782 | ||
31f18b77 | 783 | return { |
224ce89b | 784 | "health": self._health_data(), |
31f18b77 FG |
785 | "mon_status": global_instance().get_sync_object( |
786 | MonStatus).data, | |
224ce89b | 787 | "fs_map": global_instance().get_sync_object(FsMap).data, |
31f18b77 FG |
788 | "osd_map": osd_map, |
789 | "clog": list(global_instance().log_buffer), | |
790 | "audit_log": list(global_instance().audit_buffer), | |
224ce89b WB |
791 | "pools": pools, |
792 | "mgr_map": global_instance().get("mgr_map"), | |
793 | "df": df | |
31f18b77 FG |
794 | } |
795 | ||
796 | @cherrypy.expose | |
797 | @cherrypy.tools.json_out() | |
798 | def health_data(self): | |
799 | return self._health() | |
800 | ||
801 | @cherrypy.expose | |
802 | def index(self): | |
803 | return self.health() | |
804 | ||
805 | @cherrypy.expose | |
806 | @cherrypy.tools.json_out() | |
807 | def toplevel_data(self): | |
808 | return self._toplevel_data() | |
809 | ||
810 | def _get_mds_names(self, filesystem_id=None): | |
811 | names = [] | |
812 | ||
813 | fsmap = global_instance().get("fs_map") | |
814 | for fs in fsmap['filesystems']: | |
815 | if filesystem_id is not None and fs['id'] != filesystem_id: | |
816 | continue | |
817 | names.extend([info['name'] for _, info in fs['mdsmap']['info'].items()]) | |
818 | ||
819 | if filesystem_id is None: | |
820 | names.extend(info['name'] for info in fsmap['standbys']) | |
821 | ||
822 | return names | |
823 | ||
824 | @cherrypy.expose | |
825 | @cherrypy.tools.json_out() | |
826 | def mds_counters(self, fs_id): | |
827 | """ | |
828 | Result format: map of daemon name to map of counter to list of datapoints | |
829 | """ | |
830 | ||
831 | # Opinionated list of interesting performance counters for the GUI -- | |
832 | # if you need something else just add it. See how simple life is | |
833 | # when you don't have to write general purpose APIs? | |
834 | counters = [ | |
835 | "mds_server.handle_client_request", | |
836 | "mds_log.ev", | |
837 | "mds_cache.num_strays", | |
838 | "mds.exported", | |
839 | "mds.exported_inodes", | |
840 | "mds.imported", | |
841 | "mds.imported_inodes", | |
842 | "mds.inodes", | |
843 | "mds.caps", | |
844 | "mds.subtrees" | |
845 | ] | |
846 | ||
847 | result = {} | |
848 | mds_names = self._get_mds_names(int(fs_id)) | |
849 | ||
850 | for mds_name in mds_names: | |
851 | result[mds_name] = {} | |
852 | for counter in counters: | |
853 | data = global_instance().get_counter("mds", mds_name, counter) | |
854 | if data is not None: | |
855 | result[mds_name][counter] = data[counter] | |
856 | else: | |
857 | result[mds_name][counter] = [] | |
858 | ||
859 | return dict(result) | |
860 | ||
c07f9fc5 FG |
861 | @cherrypy.expose |
862 | @cherrypy.tools.json_out() | |
863 | def get_counter(self, type, id, path): | |
864 | return global_instance().get_counter(type, id, path) | |
865 | ||
866 | @cherrypy.expose | |
867 | @cherrypy.tools.json_out() | |
868 | def get_perf_schema(self, **args): | |
869 | type = args.get('type', '') | |
870 | id = args.get('id', '') | |
871 | schema = global_instance().get_perf_schema(type, id) | |
872 | ret = dict() | |
873 | for k1 in schema.keys(): # 'perf_schema' | |
874 | ret[k1] = collections.OrderedDict() | |
875 | for k2 in sorted(schema[k1].keys()): | |
876 | sorted_dict = collections.OrderedDict( | |
877 | sorted(schema[k1][k2].items(), key=lambda i: i[0]) | |
878 | ) | |
879 | ret[k1][k2] = sorted_dict | |
880 | return ret | |
881 | ||
3efd9988 FG |
882 | url_prefix = self.get_config('url_prefix') |
883 | if url_prefix == None: | |
884 | url_prefix = '' | |
885 | else: | |
886 | if len(url_prefix) != 0: | |
887 | if url_prefix[0] != '/': | |
888 | url_prefix = '/'+url_prefix | |
889 | if url_prefix[-1] == '/': | |
890 | url_prefix = url_prefix[:-1] | |
891 | self.url_prefix = url_prefix | |
892 | ||
224ce89b WB |
893 | server_addr = self.get_localized_config('server_addr', '::') |
894 | server_port = self.get_localized_config('server_port', '7000') | |
31f18b77 | 895 | if server_addr is None: |
c07f9fc5 | 896 | raise RuntimeError('no server_addr configured; try "ceph config-key set mgr/dashboard/server_addr <ip>"') |
31f18b77 FG |
897 | log.info("server_addr: %s server_port: %s" % (server_addr, server_port)) |
898 | cherrypy.config.update({ | |
899 | 'server.socket_host': server_addr, | |
900 | 'server.socket_port': int(server_port), | |
901 | 'engine.autoreload.on': False | |
902 | }) | |
903 | ||
3efd9988 FG |
904 | osdmap = self.get_osdmap() |
905 | log.info("latest osdmap is %d" % osdmap.get_epoch()) | |
906 | ||
907 | # Publish the URI that others may use to access the service we're | |
908 | # about to start serving | |
909 | self.set_uri("http://{0}:{1}/".format( | |
910 | socket.getfqdn() if server_addr == "::" else server_addr, | |
911 | server_port | |
912 | )) | |
913 | ||
31f18b77 FG |
914 | static_dir = os.path.join(current_dir, 'static') |
915 | conf = { | |
916 | "/static": { | |
917 | "tools.staticdir.on": True, | |
918 | 'tools.staticdir.dir': static_dir | |
919 | } | |
920 | } | |
921 | log.info("Serving static from {0}".format(static_dir)) | |
c07f9fc5 FG |
922 | |
923 | class OSDEndpoint(EndPoint): | |
924 | def _osd(self, osd_id): | |
925 | osd_id = int(osd_id) | |
926 | ||
927 | osd_map = global_instance().get("osd_map") | |
928 | ||
929 | osd = None | |
930 | for o in osd_map['osds']: | |
931 | if o['osd'] == osd_id: | |
932 | osd = o | |
933 | break | |
934 | ||
935 | assert osd is not None # TODO 400 | |
936 | ||
937 | osd_spec = "{0}".format(osd_id) | |
938 | ||
939 | osd_metadata = global_instance().get_metadata( | |
940 | "osd", osd_spec) | |
941 | ||
942 | result = CommandResult("") | |
943 | global_instance().send_command(result, "osd", osd_spec, | |
944 | json.dumps({ | |
945 | "prefix": "perf histogram dump", | |
946 | }), | |
947 | "") | |
948 | r, outb, outs = result.wait() | |
949 | assert r == 0 | |
950 | histogram = json.loads(outb) | |
951 | ||
952 | return { | |
953 | "osd": osd, | |
954 | "osd_metadata": osd_metadata, | |
955 | "osd_histogram": histogram | |
956 | } | |
957 | ||
958 | @cherrypy.expose | |
959 | def perf(self, osd_id): | |
960 | template = env.get_template("osd_perf.html") | |
961 | toplevel_data = self._toplevel_data() | |
962 | ||
963 | return template.render( | |
3efd9988 | 964 | url_prefix = global_instance().url_prefix, |
c07f9fc5 FG |
965 | ceph_version=global_instance().version, |
966 | path_info='/osd' + cherrypy.request.path_info, | |
967 | toplevel_data=json.dumps(toplevel_data, indent=2), | |
968 | content_data=json.dumps(self._osd(osd_id), indent=2) | |
969 | ) | |
970 | ||
971 | @cherrypy.expose | |
972 | @cherrypy.tools.json_out() | |
973 | def perf_data(self, osd_id): | |
974 | return self._osd(osd_id) | |
975 | ||
976 | @cherrypy.expose | |
977 | @cherrypy.tools.json_out() | |
978 | def list_data(self): | |
979 | return self._osds_by_server() | |
980 | ||
981 | def _osd_summary(self, osd_id, osd_info): | |
982 | """ | |
983 | The info used for displaying an OSD in a table | |
984 | """ | |
985 | ||
986 | osd_spec = "{0}".format(osd_id) | |
987 | ||
988 | result = {} | |
989 | result['id'] = osd_id | |
990 | result['stats'] = {} | |
991 | result['stats_history'] = {} | |
992 | ||
993 | # Counter stats | |
994 | for s in ['osd.op_w', 'osd.op_in_bytes', 'osd.op_r', 'osd.op_out_bytes']: | |
995 | result['stats'][s.split(".")[1]] = global_instance().get_rate('osd', osd_spec, s) | |
996 | result['stats_history'][s.split(".")[1]] = \ | |
997 | global_instance().get_counter('osd', osd_spec, s)[s] | |
998 | ||
999 | # Gauge stats | |
1000 | for s in ["osd.numpg", "osd.stat_bytes", "osd.stat_bytes_used"]: | |
1001 | result['stats'][s.split(".")[1]] = global_instance().get_latest('osd', osd_spec, s) | |
1002 | ||
1003 | result['up'] = osd_info['up'] | |
1004 | result['in'] = osd_info['in'] | |
1005 | ||
3efd9988 | 1006 | result['url'] = get_prefixed_url("/osd/perf/{0}".format(osd_id)) |
c07f9fc5 FG |
1007 | |
1008 | return result | |
1009 | ||
1010 | def _osds_by_server(self): | |
1011 | result = defaultdict(list) | |
1012 | servers = global_instance().list_servers() | |
1013 | ||
1014 | osd_map = global_instance().get_sync_object(OsdMap) | |
1015 | ||
1016 | for server in servers: | |
1017 | hostname = server['hostname'] | |
1018 | services = server['services'] | |
c07f9fc5 FG |
1019 | for s in services: |
1020 | if s["type"] == "osd": | |
1021 | osd_id = int(s["id"]) | |
1022 | # If metadata doesn't tally with osdmap, drop it. | |
1023 | if osd_id not in osd_map.osds_by_id: | |
1024 | global_instance().log.warn( | |
1025 | "OSD service {0} missing in OSDMap, stale metadata?".format(osd_id)) | |
1026 | continue | |
1027 | summary = self._osd_summary(osd_id, | |
1028 | osd_map.osds_by_id[osd_id]) | |
1029 | ||
c07f9fc5 FG |
1030 | result[hostname].append(summary) |
1031 | ||
3efd9988 FG |
1032 | result[hostname].sort(key=lambda a: a['id']) |
1033 | if len(result[hostname]): | |
1034 | result[hostname][0]['first'] = True | |
1035 | ||
c07f9fc5 FG |
1036 | global_instance().log.warn("result.size {0} servers.size {1}".format( |
1037 | len(result), len(servers) | |
1038 | )) | |
1039 | ||
1040 | # Return list form for convenience of rendering | |
3efd9988 | 1041 | return sorted(result.items(), key=lambda a: a[0]) |
c07f9fc5 FG |
1042 | |
1043 | @cherrypy.expose | |
1044 | def index(self): | |
1045 | """ | |
1046 | List of all OSDS grouped by host | |
1047 | :return: | |
1048 | """ | |
1049 | ||
1050 | template = env.get_template("osds.html") | |
1051 | toplevel_data = self._toplevel_data() | |
1052 | ||
1053 | content_data = { | |
1054 | "osds_by_server": self._osds_by_server() | |
1055 | } | |
1056 | ||
1057 | return template.render( | |
3efd9988 | 1058 | url_prefix = global_instance().url_prefix, |
c07f9fc5 FG |
1059 | ceph_version=global_instance().version, |
1060 | path_info='/osd' + cherrypy.request.path_info, | |
1061 | toplevel_data=json.dumps(toplevel_data, indent=2), | |
1062 | content_data=json.dumps(content_data, indent=2) | |
1063 | ) | |
1064 | ||
3efd9988 FG |
1065 | cherrypy.tree.mount(Root(), get_prefixed_url("/"), conf) |
1066 | cherrypy.tree.mount(OSDEndpoint(), get_prefixed_url("/osd"), conf) | |
31f18b77 | 1067 | |
3efd9988 FG |
1068 | log.info("Starting engine on {0}:{1}...".format( |
1069 | server_addr, server_port)) | |
31f18b77 FG |
1070 | cherrypy.engine.start() |
1071 | log.info("Waiting for engine...") | |
1072 | cherrypy.engine.block() | |
1073 | log.info("Engine done.") |