1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
2 // vim: ts=8 sw=2 smarttab
11 #include <seastar/core/app-template.hh>
12 #include <seastar/core/print.hh>
13 #include <seastar/core/prometheus.hh>
14 #include <seastar/core/thread.hh>
15 #include <seastar/http/httpd.hh>
16 #include <seastar/net/inet_address.hh>
17 #include <seastar/util/closeable.hh>
18 #include <seastar/util/defer.hh>
19 #include <seastar/util/std-compat.hh>
21 #include "auth/KeyRing.h"
22 #include "common/ceph_argparse.h"
23 #include "common/config_tracker.h"
24 #include "crimson/common/buffer_io.h"
25 #include "crimson/common/config_proxy.h"
26 #include "crimson/common/fatal_signal.h"
27 #include "crimson/mon/MonClient.h"
28 #include "crimson/net/Messenger.h"
29 #include "crimson/osd/stop_signal.h"
30 #include "crimson/osd/main_config_bootstrap_helpers.h"
31 #include "global/pidfile.h"
34 using namespace std::literals
;
35 namespace bpo
= boost::program_options
;
36 using crimson::common::local_conf
;
37 using crimson::common::sharded_conf
;
38 using crimson::common::sharded_perf_coll
;
40 static seastar::logger
& logger()
42 return crimson::get_logger(ceph_subsys_osd
);
45 seastar::future
<> make_keyring()
47 const auto path
= local_conf().get_val
<std::string
>("keyring");
48 return seastar::file_exists(path
).then([path
](bool exists
) {
50 EntityName name
{local_conf()->name
};
53 keyring
.load(nullptr, path
) == 0 &&
54 keyring
.get_auth(name
, auth
)) {
55 fmt::print(std::cerr
, "already have key in keyring: {}\n", path
);
56 return seastar::now();
58 CephContext temp_cct
{};
59 auth
.key
.create(&temp_cct
, CEPH_CRYPTO_AES
);
60 keyring
.add(name
, auth
);
62 keyring
.encode_plaintext(bl
);
63 const auto permissions
= (seastar::file_permissions::user_read
|
64 seastar::file_permissions::user_write
);
65 return crimson::write_file(std::move(bl
), path
, permissions
);
67 }).handle_exception_type([path
](const std::filesystem::filesystem_error
& e
) {
68 fmt::print(std::cerr
, "FATAL: writing new keyring to {}: {}\n", path
, e
.what());
73 static std::ofstream
maybe_set_logger()
75 std::ofstream log_file_stream
;
76 if (auto log_file
= local_conf()->log_file
; !log_file
.empty()) {
77 log_file_stream
.open(log_file
, std::ios::app
| std::ios::out
);
79 seastar::throw_system_error_on(log_file_stream
.fail());
80 } catch (const std::system_error
& e
) {
81 ceph_abort_msg(fmt::format("unable to open log file: {}", e
.what()));
83 logger().set_ostream(log_file_stream
);
85 return log_file_stream
;
88 int main(int argc
, const char* argv
[])
90 auto early_config_result
= crimson::osd::get_early_config(argc
, argv
);
91 if (!early_config_result
.has_value()) {
92 int r
= early_config_result
.error();
93 std::cerr
<< "do_early_config returned error: " << r
<< std::endl
;
96 auto &early_config
= early_config_result
.value();
98 auto seastar_n_early_args
= early_config
.get_early_args();
99 auto config_proxy_args
= early_config
.get_ceph_args();
101 seastar::app_template::config app_cfg
;
102 app_cfg
.name
= "Crimson";
103 app_cfg
.auto_handle_sigint_sigterm
= false;
104 seastar::app_template
app(std::move(app_cfg
));
106 ("mkkey", "generate a new secret key. "
107 "This is normally used in combination with --mkfs")
108 ("mkfs", "create a [new] data directory")
109 ("debug", "enable debug output on all loggers")
110 ("trace", "enable trace output on all loggers")
111 ("osdspec-affinity", bpo::value
<std::string
>()->default_value(std::string
{}),
112 "set affinity to an osdspec")
113 ("prometheus_port", bpo::value
<uint16_t>()->default_value(0),
114 "Prometheus port. Set to zero to disable")
115 ("prometheus_address", bpo::value
<std::string
>()->default_value("0.0.0.0"),
116 "Prometheus listening address")
117 ("prometheus_prefix", bpo::value
<std::string
>()->default_value("osd"),
118 "Prometheus metrics prefix");
122 seastar_n_early_args
.size(),
123 const_cast<char**>(seastar_n_early_args
.data()),
125 auto& config
= app
.configuration();
126 return seastar::async([&] {
128 FatalSignal fatal_signal
;
129 seastar_apps_lib::stop_signal should_stop
;
130 if (config
.count("debug")) {
131 seastar::global_logger_registry().set_all_loggers_level(
132 seastar::log_level::debug
135 if (config
.count("trace")) {
136 seastar::global_logger_registry().set_all_loggers_level(
137 seastar::log_level::trace
140 sharded_conf().start(
141 early_config
.init_params
.name
, early_config
.cluster_name
).get();
142 local_conf().start().get();
143 auto stop_conf
= seastar::deferred_stop(sharded_conf());
144 sharded_perf_coll().start().get();
145 auto stop_perf_coll
= seastar::deferred_stop(sharded_perf_coll());
146 local_conf().parse_config_files(early_config
.conf_file_list
).get();
147 local_conf().parse_env().get();
148 local_conf().parse_argv(config_proxy_args
).get();
149 auto log_file_stream
= maybe_set_logger();
150 auto reset_logger
= seastar::defer([] {
151 logger().set_ostream(std::cerr
);
153 if (const auto ret
= pidfile_write(local_conf()->pid_file
);
154 ret
== -EACCES
|| ret
== -EAGAIN
) {
156 "likely there is another crimson-osd instance with the same id");
157 } else if (ret
< 0) {
158 ceph_abort_msg(fmt::format("pidfile_write failed with {} {}",
159 ret
, cpp_strerror(-ret
)));
161 // just ignore SIGHUP, we don't reread settings. keep in mind signals
162 // handled by S* must be blocked for alien threads (see AlienStore).
163 seastar::engine().handle_signal(SIGHUP
, [] {});
165 // start prometheus API server
166 seastar::httpd::http_server_control prom_server
;
167 std::any stop_prometheus
;
168 if (uint16_t prom_port
= config
["prometheus_port"].as
<uint16_t>();
170 prom_server
.start("prometheus").get();
171 stop_prometheus
= seastar::make_shared(seastar::deferred_stop(prom_server
));
173 seastar::prometheus::config prom_config
;
174 prom_config
.prefix
= config
["prometheus_prefix"].as
<std::string
>();
175 seastar::prometheus::start(prom_server
, prom_config
).get();
176 seastar::net::inet_address
prom_addr(config
["prometheus_address"].as
<std::string
>());
177 prom_server
.listen(seastar::socket_address
{prom_addr
, prom_port
})
178 .handle_exception([=] (auto ep
) {
179 std::cerr
<< seastar::format("Could not start Prometheus API server on {}:{}: {}\n",
180 prom_addr
, prom_port
, ep
);
181 return seastar::make_exception_future(ep
);
185 const int whoami
= std::stoi(local_conf()->name
.get_id());
186 const auto nonce
= crimson::osd::get_nonce();
187 crimson::net::MessengerRef cluster_msgr
, client_msgr
;
188 crimson::net::MessengerRef hb_front_msgr
, hb_back_msgr
;
189 for (auto [msgr
, name
] : {make_pair(std::ref(cluster_msgr
), "cluster"s
),
190 make_pair(std::ref(client_msgr
), "client"s
),
191 make_pair(std::ref(hb_front_msgr
), "hb_front"s
),
192 make_pair(std::ref(hb_back_msgr
), "hb_back"s
)}) {
193 msgr
= crimson::net::Messenger::create(entity_name_t::OSD(whoami
),
198 auto store
= crimson::os::FuturizedStore::create(
199 local_conf().get_val
<std::string
>("osd_objectstore"),
200 local_conf().get_val
<std::string
>("osd_data"),
201 local_conf().get_config_values());
203 crimson::osd::OSD
osd(
204 whoami
, nonce
, std::ref(should_stop
.abort_source()),
205 std::ref(*store
), cluster_msgr
, client_msgr
,
206 hb_front_msgr
, hb_back_msgr
);
208 if (config
.count("mkkey")) {
209 make_keyring().get();
211 if (local_conf()->no_mon_config
) {
212 logger().info("bypassing the config fetch due to --no-mon-config");
214 crimson::osd::populate_config_from_mon().get();
216 if (config
.count("mkfs")) {
217 auto osd_uuid
= local_conf().get_val
<uuid_d
>("osd_uuid");
218 if (osd_uuid
.is_zero()) {
219 // use a random osd uuid if not specified
220 osd_uuid
.generate_random();
226 local_conf().get_val
<uuid_d
>("fsid"),
227 config
["osdspec-affinity"].as
<std::string
>()).get();
229 if (config
.count("mkkey") || config
.count("mkfs")) {
234 logger().info("crimson startup completed");
235 should_stop
.wait().get();
236 logger().info("crimson shutting down");
238 // stop()s registered using defer() are called here
240 logger().error("startup failed: {}", std::current_exception());
243 logger().info("crimson shutdown complete");
248 fmt::print(std::cerr
, "FATAL: Exception during startup, aborting: {}\n", std::current_exception());
255 * compile-command: "make -j4 \
256 * -C ../../../build \