ceph/src/mgr/MgrStandby.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2016 John Spray <john.spray@redhat.com>
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  */
  13
  14 #include <Python.h>
  15
  16 #include "common/errno.h"
  17 #include "common/signal.h"
  18 #include "include/compat.h"
  19
  20 #include "include/stringify.h"
  21 #include "global/global_context.h"
  22 #include "global/signal_handler.h"
  23
  24 #include "mgr/MgrContext.h"
  25
  26 #include "messages/MMgrBeacon.h"
  27 #include "messages/MMgrMap.h"
  28 #include "Mgr.h"
  29
  30 #include "MgrStandby.h"
  31
  32 #define dout_context g_ceph_context
  33 #define dout_subsys ceph_subsys_mgr
  34 #undef dout_prefix
  35 #define dout_prefix *_dout << "mgr " << __func__ << " "
  36
  37
  38 MgrStandby::MgrStandby(int argc, const char **argv) :
  39   Dispatcher(g_ceph_context),
  40   monc{g_ceph_context},
  41   client_messenger(Messenger::create_client_messenger(g_ceph_context, "mgr")),
  42   objecter{g_ceph_context, client_messenger.get(), &monc, NULL, 0, 0},
  43   client{client_messenger.get(), &monc, &objecter},
  44   log_client(g_ceph_context, client_messenger.get(), &monc.monmap, LogClient::NO_FLAGS),
  45   clog(log_client.create_channel(CLOG_CHANNEL_CLUSTER)),
  46   audit_clog(log_client.create_channel(CLOG_CHANNEL_AUDIT)),
  47   lock("MgrStandby::lock"),
  48   timer(g_ceph_context, lock),
  49   active_mgr(nullptr),
  50   orig_argc(argc),
  51   orig_argv(argv)
  52 {
  53 }
  54
  55 MgrStandby::~MgrStandby() = default;
  56
  57 const char** MgrStandby::get_tracked_conf_keys() const
  58 {
  59   static const char* KEYS[] = {
  60     // clog & admin clog
  61     "clog_to_monitors",
  62     "clog_to_syslog",
  63     "clog_to_syslog_facility",
  64     "clog_to_syslog_level",
  65     "osd_objectstore_fuse",
  66     "clog_to_graylog",
  67     "clog_to_graylog_host",
  68     "clog_to_graylog_port",
  69     "host",
  70     "fsid",
  71     NULL
  72   };
  73   return KEYS;
  74 }
  75
  76 void MgrStandby::handle_conf_change(
  77   const struct md_config_t *conf,
  78   const std::set <std::string> &changed)
  79 {
  80   if (changed.count("clog_to_monitors") ||
  81       changed.count("clog_to_syslog") ||
  82       changed.count("clog_to_syslog_level") ||
  83       changed.count("clog_to_syslog_facility") ||
  84       changed.count("clog_to_graylog") ||
  85       changed.count("clog_to_graylog_host") ||
  86       changed.count("clog_to_graylog_port") ||
  87       changed.count("host") ||
  88       changed.count("fsid")) {
  89     _update_log_config();
  90   }
  91 }
  92
  93 int MgrStandby::init()
  94 {
  95   Mutex::Locker l(lock);
  96
  97   // Initialize Messenger
  98   client_messenger->add_dispatcher_tail(this);
  99   client_messenger->add_dispatcher_head(&objecter);
 100   client_messenger->add_dispatcher_tail(&client);
 101   client_messenger->start();
 102
 103   // Initialize MonClient
 104   if (monc.build_initial_monmap() < 0) {
 105     client_messenger->shutdown();
 106     client_messenger->wait();
 107     return -1;
 108   }
 109
 110   monc.sub_want("mgrmap", 0, 0);
 111
 112   monc.set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD
 113       |CEPH_ENTITY_TYPE_MDS|CEPH_ENTITY_TYPE_MGR);
 114   monc.set_messenger(client_messenger.get());
 115   int r = monc.init();
 116   if (r < 0) {
 117     monc.shutdown();
 118     client_messenger->shutdown();
 119     client_messenger->wait();
 120     return r;
 121   }
 122   r = monc.authenticate();
 123   if (r < 0) {
 124     derr << "Authentication failed, did you specify a mgr ID with a valid keyring?" << dendl;
 125     monc.shutdown();
 126     client_messenger->shutdown();
 127     client_messenger->wait();
 128     return r;
 129   }
 130
 131   client_t whoami = monc.get_global_id();
 132   client_messenger->set_myname(entity_name_t::CLIENT(whoami.v));
 133   monc.set_log_client(&log_client);
 134   _update_log_config();
 135   objecter.set_client_incarnation(0);
 136   objecter.init();
 137   objecter.start();
 138   client.init();
 139   timer.init();
 140
 141   tick();
 142
 143   dout(4) << "Complete." << dendl;
 144   return 0;
 145 }
 146
 147 void MgrStandby::send_beacon()
 148 {
 149   assert(lock.is_locked_by_me());
 150   dout(1) << state_str() << dendl;
 151   dout(10) << "sending beacon as gid " << monc.get_global_id() << dendl;
 152
 153   bool available = active_mgr != nullptr && active_mgr->is_initialized();
 154   auto addr = available ? active_mgr->get_server_addr() : entity_addr_t();
 155   MMgrBeacon *m = new MMgrBeacon(monc.get_fsid(),
 156                                  monc.get_global_id(),
 157                                  g_conf->name.get_id(),
 158                                  addr,
 159                                  available);
 160
 161   monc.send_mon_message(m);
 162 }
 163
 164 void MgrStandby::tick()
 165 {
 166   dout(0) << __func__ << dendl;
 167   send_beacon();
 168
 169   if (active_mgr) {
 170     active_mgr->tick();
 171   }
 172
 173   timer.add_event_after(g_conf->mgr_tick_period, new FunctionContext(
 174         [this](int r){
 175           tick();
 176         }
 177   ));
 178 }
 179
 180 void MgrStandby::handle_signal(int signum)
 181 {
 182   Mutex::Locker l(lock);
 183   assert(signum == SIGINT || signum == SIGTERM);
 184   derr << "*** Got signal " << sig_str(signum) << " ***" << dendl;
 185   shutdown();
 186 }
 187
 188 void MgrStandby::shutdown()
 189 {
 190   // Expect already to be locked as we're called from signal handler
 191   assert(lock.is_locked_by_me());
 192
 193   // stop sending beacon first, i use monc to talk with monitors
 194   timer.shutdown();
 195   // client uses monc and objecter
 196   client.shutdown();
 197   // stop monc, so mon won't be able to instruct me to shutdown/activate after
 198   // the active_mgr is stopped
 199   monc.shutdown();
 200   if (active_mgr) {
 201     active_mgr->shutdown();
 202   }
 203   // objecter is used by monc and active_mgr
 204   objecter.shutdown();
 205   // client_messenger is used by all of them, so stop it in the end
 206   client_messenger->shutdown();
 207 }
 208
 209 void MgrStandby::respawn()
 210 {
 211   char *new_argv[orig_argc+1];
 212   dout(1) << " e: '" << orig_argv[0] << "'" << dendl;
 213   for (int i=0; i<orig_argc; i++) {
 214     new_argv[i] = (char *)orig_argv[i];
 215     dout(1) << " " << i << ": '" << orig_argv[i] << "'" << dendl;
 216   }
 217   new_argv[orig_argc] = NULL;
 218
 219   /* Determine the path to our executable, test if Linux /proc/self/exe exists.
 220    * This allows us to exec the same executable even if it has since been
 221    * unlinked.
 222    */
 223   char exe_path[PATH_MAX] = "";
 224   if (readlink(PROCPREFIX "/proc/self/exe", exe_path, PATH_MAX-1) == -1) {
 225     /* Print CWD for the user's interest */
 226     char buf[PATH_MAX];
 227     char *cwd = getcwd(buf, sizeof(buf));
 228     assert(cwd);
 229     dout(1) << " cwd " << cwd << dendl;
 230
 231     /* Fall back to a best-effort: just running in our CWD */
 232     strncpy(exe_path, orig_argv[0], PATH_MAX-1);
 233   } else {
 234     dout(1) << "respawning with exe " << exe_path << dendl;
 235     strcpy(exe_path, PROCPREFIX "/proc/self/exe");
 236   }
 237
 238   dout(1) << " exe_path " << exe_path << dendl;
 239
 240   unblock_all_signals(NULL);
 241   execv(exe_path, new_argv);
 242
 243   derr << "respawn execv " << orig_argv[0]
 244        << " failed with " << cpp_strerror(errno) << dendl;
 245   ceph_abort();
 246 }
 247
 248 void MgrStandby::_update_log_config()
 249 {
 250   map<string,string> log_to_monitors;
 251   map<string,string> log_to_syslog;
 252   map<string,string> log_channel;
 253   map<string,string> log_prio;
 254   map<string,string> log_to_graylog;
 255   map<string,string> log_to_graylog_host;
 256   map<string,string> log_to_graylog_port;
 257   uuid_d fsid;
 258   string host;
 259
 260   if (parse_log_client_options(cct, log_to_monitors, log_to_syslog,
 261                                log_channel, log_prio, log_to_graylog,
 262                                log_to_graylog_host, log_to_graylog_port,
 263                                fsid, host) == 0) {
 264     clog->update_config(log_to_monitors, log_to_syslog,
 265                         log_channel, log_prio, log_to_graylog,
 266                         log_to_graylog_host, log_to_graylog_port,
 267                         fsid, host);
 268     audit_clog->update_config(log_to_monitors, log_to_syslog,
 269                               log_channel, log_prio, log_to_graylog,
 270                               log_to_graylog_host, log_to_graylog_port,
 271                               fsid, host);
 272   }
 273 }
 274
 275 void MgrStandby::handle_mgr_map(MMgrMap* mmap)
 276 {
 277   auto map = mmap->get_map();
 278   dout(4) << "received map epoch " << map.get_epoch() << dendl;
 279   const bool active_in_map = map.active_gid == monc.get_global_id();
 280   dout(4) << "active in map: " << active_in_map
 281           << " active is " << map.active_gid << dendl;
 282   if (active_in_map) {
 283     if (!active_mgr) {
 284       dout(1) << "Activating!" << dendl;
 285       active_mgr.reset(new Mgr(&monc, client_messenger.get(), &objecter,
 286                                &client, clog, audit_clog));
 287       active_mgr->background_init();
 288       dout(1) << "I am now active" << dendl;
 289     } else {
 290       dout(10) << "I was already active" << dendl;
 291     }
 292   } else {
 293     if (active_mgr != nullptr) {
 294       derr << "I was active but no longer am" << dendl;
 295       respawn();
 296     }
 297   }
 298
 299   mmap->put();
 300 }
 301
 302 bool MgrStandby::ms_dispatch(Message *m)
 303 {
 304   Mutex::Locker l(lock);
 305   dout(4) << state_str() << " " << *m << dendl;
 306
 307   if (m->get_type() == MSG_MGR_MAP) {
 308     handle_mgr_map(static_cast<MMgrMap*>(m));
 309     return true;
 310   } else if (active_mgr) {
 311     auto am = active_mgr;
 312     lock.Unlock();
 313     bool handled = am->ms_dispatch(m);
 314     lock.Lock();
 315     return handled;
 316   } else {
 317     return false;
 318   }
 319 }
 320
 321
 322 bool MgrStandby::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer,
 323                          bool force_new)
 324 {
 325   if (dest_type == CEPH_ENTITY_TYPE_MON)
 326     return true;
 327
 328   if (force_new) {
 329     if (monc.wait_auth_rotating(10) < 0)
 330       return false;
 331   }
 332
 333   *authorizer = monc.build_authorizer(dest_type);
 334   return *authorizer != NULL;
 335 }
 336
 337 bool MgrStandby::ms_handle_refused(Connection *con)
 338 {
 339   // do nothing for now
 340   return false;
 341 }
 342
 343 // A reference for use by the signal handler
 344 static MgrStandby *signal_mgr = nullptr;
 345
 346 static void handle_mgr_signal(int signum)
 347 {
 348   if (signal_mgr) {
 349     signal_mgr->handle_signal(signum);
 350   }
 351 }
 352
 353 int MgrStandby::main(vector<const char *> args)
 354 {
 355   // Enable signal handlers
 356   signal_mgr = this;
 357   init_async_signal_handler();
 358   register_async_signal_handler(SIGHUP, sighup_handler);
 359   register_async_signal_handler_oneshot(SIGINT, handle_mgr_signal);
 360   register_async_signal_handler_oneshot(SIGTERM, handle_mgr_signal);
 361
 362   client_messenger->wait();
 363
 364   // Disable signal handlers
 365   unregister_async_signal_handler(SIGHUP, sighup_handler);
 366   unregister_async_signal_handler(SIGINT, handle_mgr_signal);
 367   unregister_async_signal_handler(SIGTERM, handle_mgr_signal);
 368   shutdown_async_signal_handler();
 369   signal_mgr = nullptr;
 370
 371   return 0;
 372 }
 373
 374
 375 std::string MgrStandby::state_str()
 376 {
 377   return active_mgr == nullptr ? "standby" : "active";
 378 }
 379