ceph/src/crimson/osd/main_config_bootstrap_helpers.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "crimson/osd/main_config_bootstrap_helpers.h"
   5
   6 #include <seastar/core/print.hh>
   7 #include <seastar/core/prometheus.hh>
   8 #include <seastar/core/thread.hh>
   9 #include <seastar/http/httpd.hh>
  10 #include <seastar/net/inet_address.hh>
  11 #include <seastar/util/closeable.hh>
  12 #include <seastar/util/defer.hh>
  13 #include <seastar/util/std-compat.hh>
  14
  15 #include "common/ceph_argparse.h"
  16 #include "common/config_tracker.h"
  17 #include "crimson/common/buffer_io.h"
  18 #include "crimson/common/config_proxy.h"
  19 #include "crimson/common/fatal_signal.h"
  20 #include "crimson/mon/MonClient.h"
  21 #include "crimson/net/Messenger.h"
  22 #include "crimson/osd/main_config_bootstrap_helpers.h"
  23
  24 using namespace std::literals;
  25 using crimson::common::local_conf;
  26 using crimson::common::sharded_conf;
  27 using crimson::common::sharded_perf_coll;
  28
  29 static seastar::logger& logger()
  30 {
  31   return crimson::get_logger(ceph_subsys_osd);
  32 }
  33
  34 namespace crimson::osd {
  35
  36 void usage(const char* prog)
  37 {
  38   std::cout << "usage: " << prog << std::endl;
  39   generic_server_usage();
  40 }
  41
  42
  43 seastar::future<> populate_config_from_mon()
  44 {
  45   logger().info("populating config from monitor");
  46   // i don't have any client before joining the cluster, so no need to have
  47   // a proper auth handler
  48   class DummyAuthHandler : public crimson::common::AuthHandler {
  49   public:
  50     void handle_authentication(const EntityName& name,
  51                                const AuthCapsInfo& caps)
  52     {}
  53   };
  54   return seastar::async([] {
  55     auto auth_handler = std::make_unique<DummyAuthHandler>();
  56     auto msgr = crimson::net::Messenger::create(entity_name_t::CLIENT(),
  57                                                 "temp_mon_client",
  58                                                 get_nonce());
  59     crimson::mon::Client monc{*msgr, *auth_handler};
  60     msgr->set_auth_client(&monc);
  61     msgr->start({&monc}).get();
  62     auto stop_msgr = seastar::defer([&] {
  63       msgr->stop();
  64       msgr->shutdown().get();
  65     });
  66     monc.start().handle_exception([] (auto ep) {
  67       fmt::print(std::cerr, "FATAL: unable to connect to cluster: {}\n", ep);
  68       return seastar::make_exception_future<>(ep);
  69     }).get();
  70     auto stop_monc = seastar::defer([&] {
  71       monc.stop().get();
  72     });
  73     monc.sub_want("config", 0, 0);
  74     monc.renew_subs().get();
  75     // wait for monmap and config
  76     monc.wait_for_config().get();
  77     auto fsid = monc.get_fsid().to_string();
  78     local_conf().set_val("fsid", fsid).get();
  79     logger().debug("{}: got config from monitor, fsid {}", __func__, fsid);
  80   });
  81 }
  82
  83 static tl::expected<early_config_t, int>
  84 _get_early_config(int argc, const char *argv[])
  85 {
  86   early_config_t ret;
  87
  88   // pull off ceph configs the stuff from early_args
  89   std::vector<const char *> early_args;
  90   early_args.insert(
  91     std::end(early_args),
  92     argv, argv + argc);
  93
  94   ret.init_params = ceph_argparse_early_args(
  95     early_args,
  96     CEPH_ENTITY_TYPE_OSD,
  97     &ret.cluster_name,
  98     &ret.conf_file_list);
  99
 100   if (ceph_argparse_need_usage(early_args)) {
 101     usage(argv[0]);
 102     exit(0);
 103   }
 104
 105   seastar::app_template::config app_cfg;
 106   app_cfg.name = "Crimson-startup";
 107   app_cfg.auto_handle_sigint_sigterm = false;
 108   seastar::app_template app(std::move(app_cfg));
 109   const char *bootstrap_args[] = { argv[0], "--smp", "1" };
 110   int r = app.run(
 111     sizeof(bootstrap_args) / sizeof(bootstrap_args[0]),
 112     const_cast<char**>(bootstrap_args),
 113     [argc, argv, &ret, &early_args] {
 114       return seastar::async([argc, argv, &ret, &early_args] {
 115         seastar::global_logger_registry().set_all_loggers_level(
 116           seastar::log_level::debug);
 117         sharded_conf().start(
 118           ret.init_params.name, ret.cluster_name).get();
 119         local_conf().start().get();
 120         auto stop_conf = seastar::deferred_stop(sharded_conf());
 121
 122         sharded_perf_coll().start().get();
 123         auto stop_perf_coll = seastar::deferred_stop(sharded_perf_coll());
 124
 125         local_conf().parse_env().get();
 126         local_conf().parse_argv(early_args).get();
 127         local_conf().parse_config_files(ret.conf_file_list).get();
 128
 129         if (local_conf()->no_mon_config) {
 130           logger().info("bypassing the config fetch due to --no-mon-config");
 131         } else {
 132           populate_config_from_mon().get();
 133         }
 134
 135         // get ceph configs
 136         std::set_difference(
 137           argv, argv + argc,
 138           std::begin(early_args),
 139           std::end(early_args),
 140           std::back_inserter(ret.ceph_args));
 141
 142         ret.early_args.insert(
 143           std::end(ret.early_args),
 144           std::begin(early_args),
 145           std::end(early_args));
 146
 147         if (auto found = std::find_if(
 148               std::begin(early_args),
 149               std::end(early_args),
 150               [](auto* arg) { return "--smp"sv == arg; });
 151             found == std::end(early_args)) {
 152
 153           // Set --smp based on crimson_seastar_smp config option
 154           ret.early_args.emplace_back("--smp");
 155
 156           auto smp_config = local_conf().get_val<uint64_t>(
 157             "crimson_seastar_smp");
 158
 159           ret.early_args.emplace_back(fmt::format("{}", smp_config));
 160           logger().info("get_early_config: set --smp {}", smp_config);
 161         }
 162         return 0;
 163       });
 164     });
 165   if (r < 0) {
 166     return tl::unexpected(r);
 167   }
 168   return ret;
 169 }
 170
 171 /* get_early_config handles obtaining config parameters required prior
 172  * to reactor startup.  Most deployment mechanisms (cephadm for one)
 173  * rely on pulling configs from the monitor rather than shipping around
 174  * config files, so this process needs to support pulling config options
 175  * from the monitors.
 176  *
 177  * Of particular interest are config params related to the seastar
 178  * reactor itself which can't be modified after the reactor has been
 179  * started -- like the number of cores to use (smp::count).  Contacting
 180  * the monitors, however, requires a MonClient, which in turn needs a
 181  * running reactor.
 182  *
 183  * Unfortunately, seastar doesn't clean up thread local state
 184  * associated with seastar::smp task queues etc, so we can't
 185  * start a reactor, stop it, and restart it in the same thread
 186  * without an impractical amount of cleanup in seastar.
 187  *
 188  * More unfortunately, starting a reactor in a seperate thread
 189  * and then joining the thread still doesn't avoid all global state,
 190  * I observed tasks from the previous reactor incarnation nevertheless
 191  * continuing to run in the new one resulting in a crash as they access
 192  * freed memory.
 193  *
 194  * The approach taken here, therefore, is to actually fork, start a
 195  * reactor in the child process, encode the resulting early_config_t,
 196  * and send it back to the parent process.
 197  */
 198 tl::expected<early_config_t, int>
 199 get_early_config(int argc, const char *argv[])
 200 {
 201   int pipes[2];
 202   int r = pipe2(pipes, 0);
 203   if (r < 0) {
 204     std::cerr << "get_early_config: failed to create pipes: "
 205               << -errno << std::endl;
 206     return tl::unexpected(-errno);
 207   }
 208
 209   pid_t worker = fork();
 210   if (worker < 0) {
 211     close(pipes[0]);
 212     close(pipes[1]);
 213     std::cerr << "get_early_config: failed to fork: "
 214               << -errno << std::endl;
 215     return tl::unexpected(-errno);
 216   } else if (worker == 0) { // child
 217     close(pipes[0]);
 218     auto ret = _get_early_config(argc, argv);
 219     if (ret.has_value()) {
 220       bufferlist bl;
 221       ::encode(ret.value(), bl);
 222       r = bl.write_fd(pipes[1]);
 223       close(pipes[1]);
 224       if (r < 0) {
 225         std::cerr << "get_early_config: child failed to write_fd: "
 226                   << r << std::endl;
 227         exit(-r);
 228       } else {
 229         exit(0);
 230       }
 231     } else {
 232       std::cerr << "get_early_config: child failed: "
 233                 << -ret.error() << std::endl;
 234       exit(-ret.error());
 235     }
 236     return tl::unexpected(-1);
 237   } else { // parent
 238     close(pipes[1]);
 239
 240     bufferlist bl;
 241     early_config_t ret;
 242     while ((r = bl.read_fd(pipes[0], 1024)) > 0);
 243     close(pipes[0]);
 244
 245     // ignore error, we'll propogate error based on read and decode
 246     waitpid(worker, nullptr, 0);
 247
 248     if (r < 0) {
 249       std::cerr << "get_early_config: parent failed to read from pipe: "
 250                 << r << std::endl;
 251       return tl::unexpected(r);
 252     }
 253     try {
 254       auto bliter = bl.cbegin();
 255       ::decode(ret, bliter);
 256       return ret;
 257     } catch (...) {
 258       std::cerr << "get_early_config: parent failed to decode" << std::endl;
 259       return tl::unexpected(-EINVAL);
 260     }
 261   }
 262 }
 263
 264 }