ceph/src/crimson/osd/main.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include <sys/types.h>
   5 #include <unistd.h>
   6
   7 #include <iostream>
   8 #include <fstream>
   9 #include <random>
  10
  11 #include <seastar/core/app-template.hh>
  12 #include <seastar/core/print.hh>
  13 #include <seastar/core/prometheus.hh>
  14 #include <seastar/core/thread.hh>
  15 #include <seastar/http/httpd.hh>
  16 #include <seastar/net/inet_address.hh>
  17 #include <seastar/util/closeable.hh>
  18 #include <seastar/util/defer.hh>
  19 #include <seastar/util/std-compat.hh>
  20
  21 #include "auth/KeyRing.h"
  22 #include "common/ceph_argparse.h"
  23 #include "common/config_tracker.h"
  24 #include "crimson/common/buffer_io.h"
  25 #include "crimson/common/config_proxy.h"
  26 #include "crimson/common/fatal_signal.h"
  27 #include "crimson/mon/MonClient.h"
  28 #include "crimson/net/Messenger.h"
  29 #include "crimson/osd/stop_signal.h"
  30 #include "crimson/osd/main_config_bootstrap_helpers.h"
  31 #include "global/pidfile.h"
  32 #include "osd.h"
  33
  34 using namespace std::literals;
  35 namespace bpo = boost::program_options;
  36 using crimson::common::local_conf;
  37 using crimson::common::sharded_conf;
  38 using crimson::common::sharded_perf_coll;
  39
  40 static seastar::logger& logger()
  41 {
  42   return crimson::get_logger(ceph_subsys_osd);
  43 }
  44
  45 seastar::future<> make_keyring()
  46 {
  47   const auto path = local_conf().get_val<std::string>("keyring");
  48   return seastar::file_exists(path).then([path](bool exists) {
  49     KeyRing keyring;
  50     EntityName name{local_conf()->name};
  51     EntityAuth auth;
  52     if (exists &&
  53         keyring.load(nullptr, path) == 0 &&
  54         keyring.get_auth(name, auth)) {
  55       fmt::print(std::cerr, "already have key in keyring: {}\n", path);
  56       return seastar::now();
  57     } else {
  58       CephContext temp_cct{};
  59       auth.key.create(&temp_cct, CEPH_CRYPTO_AES);
  60       keyring.add(name, auth);
  61       bufferlist bl;
  62       keyring.encode_plaintext(bl);
  63       const auto permissions = (seastar::file_permissions::user_read |
  64                               seastar::file_permissions::user_write);
  65       return crimson::write_file(std::move(bl), path, permissions);
  66     }
  67   }).handle_exception_type([path](const std::filesystem::filesystem_error& e) {
  68     fmt::print(std::cerr, "FATAL: writing new keyring to {}: {}\n", path, e.what());
  69     throw e;
  70   });
  71 }
  72
  73 static std::ofstream maybe_set_logger()
  74 {
  75   std::ofstream log_file_stream;
  76   if (auto log_file = local_conf()->log_file; !log_file.empty()) {
  77     log_file_stream.open(log_file, std::ios::app | std::ios::out);
  78     try {
  79       seastar::throw_system_error_on(log_file_stream.fail());
  80     } catch (const std::system_error& e) {
  81       ceph_abort_msg(fmt::format("unable to open log file: {}", e.what()));
  82     }
  83     logger().set_ostream(log_file_stream);
  84   }
  85   return log_file_stream;
  86 }
  87
  88 int main(int argc, const char* argv[])
  89 {
  90   auto early_config_result = crimson::osd::get_early_config(argc, argv);
  91   if (!early_config_result.has_value()) {
  92     int r = early_config_result.error();
  93     std::cerr << "do_early_config returned error: " << r << std::endl;
  94     return r;
  95   }
  96   auto &early_config = early_config_result.value();
  97
  98   auto seastar_n_early_args = early_config.get_early_args();
  99   auto config_proxy_args = early_config.get_ceph_args();
 100
 101   seastar::app_template::config app_cfg;
 102   app_cfg.name = "Crimson";
 103   app_cfg.auto_handle_sigint_sigterm = false;
 104   seastar::app_template app(std::move(app_cfg));
 105   app.add_options()
 106     ("mkkey", "generate a new secret key. "
 107               "This is normally used in combination with --mkfs")
 108     ("mkfs", "create a [new] data directory")
 109     ("debug", "enable debug output on all loggers")
 110     ("trace", "enable trace output on all loggers")
 111     ("osdspec-affinity", bpo::value<std::string>()->default_value(std::string{}),
 112      "set affinity to an osdspec")
 113     ("prometheus_port", bpo::value<uint16_t>()->default_value(0),
 114      "Prometheus port. Set to zero to disable")
 115     ("prometheus_address", bpo::value<std::string>()->default_value("0.0.0.0"),
 116      "Prometheus listening address")
 117     ("prometheus_prefix", bpo::value<std::string>()->default_value("osd"),
 118      "Prometheus metrics prefix");
 119
 120   try {
 121     return app.run(
 122       seastar_n_early_args.size(),
 123       const_cast<char**>(seastar_n_early_args.data()),
 124       [&] {
 125       auto& config = app.configuration();
 126       return seastar::async([&] {
 127         try {
 128           FatalSignal fatal_signal;
 129           seastar_apps_lib::stop_signal should_stop;
 130           if (config.count("debug")) {
 131             seastar::global_logger_registry().set_all_loggers_level(
 132               seastar::log_level::debug
 133             );
 134           }
 135           if (config.count("trace")) {
 136             seastar::global_logger_registry().set_all_loggers_level(
 137               seastar::log_level::trace
 138             );
 139           }
 140           sharded_conf().start(
 141             early_config.init_params.name, early_config.cluster_name).get();
 142           local_conf().start().get();
 143           auto stop_conf = seastar::deferred_stop(sharded_conf());
 144           sharded_perf_coll().start().get();
 145           auto stop_perf_coll = seastar::deferred_stop(sharded_perf_coll());
 146           local_conf().parse_config_files(early_config.conf_file_list).get();
 147           local_conf().parse_env().get();
 148           local_conf().parse_argv(config_proxy_args).get();
 149           auto log_file_stream = maybe_set_logger();
 150           auto reset_logger = seastar::defer([] {
 151             logger().set_ostream(std::cerr);
 152           });
 153           if (const auto ret = pidfile_write(local_conf()->pid_file);
 154               ret == -EACCES || ret == -EAGAIN) {
 155             ceph_abort_msg(
 156               "likely there is another crimson-osd instance with the same id");
 157           } else if (ret < 0) {
 158             ceph_abort_msg(fmt::format("pidfile_write failed with {} {}",
 159                                        ret, cpp_strerror(-ret)));
 160           }
 161           // just ignore SIGHUP, we don't reread settings. keep in mind signals
 162           // handled by S* must be blocked for alien threads (see AlienStore).
 163           seastar::engine().handle_signal(SIGHUP, [] {});
 164
 165           // start prometheus API server
 166           seastar::httpd::http_server_control prom_server;
 167           std::any stop_prometheus;
 168           if (uint16_t prom_port = config["prometheus_port"].as<uint16_t>();
 169               prom_port != 0) {
 170             prom_server.start("prometheus").get();
 171             stop_prometheus = seastar::make_shared(seastar::deferred_stop(prom_server));
 172
 173             seastar::prometheus::config prom_config;
 174             prom_config.prefix = config["prometheus_prefix"].as<std::string>();
 175             seastar::prometheus::start(prom_server, prom_config).get();
 176             seastar::net::inet_address prom_addr(config["prometheus_address"].as<std::string>());
 177             prom_server.listen(seastar::socket_address{prom_addr, prom_port})
 178               .handle_exception([=] (auto ep) {
 179               std::cerr << seastar::format("Could not start Prometheus API server on {}:{}: {}\n",
 180                                            prom_addr, prom_port, ep);
 181               return seastar::make_exception_future(ep);
 182             }).get();
 183           }
 184
 185           const int whoami = std::stoi(local_conf()->name.get_id());
 186           const auto nonce = crimson::osd::get_nonce();
 187           crimson::net::MessengerRef cluster_msgr, client_msgr;
 188           crimson::net::MessengerRef hb_front_msgr, hb_back_msgr;
 189           for (auto [msgr, name] : {make_pair(std::ref(cluster_msgr), "cluster"s),
 190                                     make_pair(std::ref(client_msgr), "client"s),
 191                                     make_pair(std::ref(hb_front_msgr), "hb_front"s),
 192                                     make_pair(std::ref(hb_back_msgr), "hb_back"s)}) {
 193             msgr = crimson::net::Messenger::create(entity_name_t::OSD(whoami),
 194                                                    name,
 195                                                    nonce);
 196           }
 197           auto store = crimson::os::FuturizedStore::create(
 198             local_conf().get_val<std::string>("osd_objectstore"),
 199             local_conf().get_val<std::string>("osd_data"),
 200             local_conf().get_config_values());
 201
 202           crimson::osd::OSD osd(
 203             whoami, nonce, std::ref(should_stop.abort_source()),
 204             std::ref(*store), cluster_msgr, client_msgr,
 205             hb_front_msgr, hb_back_msgr);
 206
 207           if (config.count("mkkey")) {
 208             make_keyring().get();
 209           }
 210           if (local_conf()->no_mon_config) {
 211             logger().info("bypassing the config fetch due to --no-mon-config");
 212           } else {
 213             crimson::osd::populate_config_from_mon().get();
 214           }
 215           if (config.count("mkfs")) {
 216             auto osd_uuid = local_conf().get_val<uuid_d>("osd_uuid");
 217             if (osd_uuid.is_zero()) {
 218               // use a random osd uuid if not specified
 219               osd_uuid.generate_random();
 220             }
 221             osd.mkfs(
 222               *store,
 223               whoami,
 224               osd_uuid,
 225               local_conf().get_val<uuid_d>("fsid"),
 226               config["osdspec-affinity"].as<std::string>()).get();
 227           }
 228           if (config.count("mkkey") || config.count("mkfs")) {
 229             return EXIT_SUCCESS;
 230           } else {
 231             osd.start().get();
 232           }
 233           logger().info("crimson startup completed");
 234           should_stop.wait().get();
 235           logger().info("crimson shutting down");
 236           osd.stop().get();
 237           // stop()s registered using defer() are called here
 238         } catch (...) {
 239           logger().error("startup failed: {}", std::current_exception());
 240           return EXIT_FAILURE;
 241         }
 242         logger().info("crimson shutdown complete");
 243         return EXIT_SUCCESS;
 244       });
 245     });
 246   } catch (...) {
 247     fmt::print(std::cerr, "FATAL: Exception during startup, aborting: {}\n", std::current_exception());
 248     return EXIT_FAILURE;
 249   }
 250 }
 251
 252 /*
 253  * Local Variables:
 254  * compile-command: "make -j4 \
 255  * -C ../../../build \
 256  * crimson-osd"
 257  * End:
 258  */