]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
11fdf7f2 TL |
4 | #include "MonClient.h" |
5 | ||
6 | #include <random> | |
7 | ||
8 | #include <seastar/core/future-util.hh> | |
9 | #include <seastar/core/lowres_clock.hh> | |
9f95a23c | 10 | #include <seastar/core/shared_future.hh> |
11fdf7f2 TL |
11 | #include <seastar/util/log.hh> |
12 | ||
13 | #include "auth/AuthClientHandler.h" | |
11fdf7f2 TL |
14 | #include "auth/RotatingKeyRing.h" |
15 | ||
16 | #include "common/hostname.h" | |
17 | ||
18 | #include "crimson/auth/KeyRing.h" | |
19 | #include "crimson/common/config_proxy.h" | |
20 | #include "crimson/common/log.h" | |
21 | #include "crimson/net/Connection.h" | |
22 | #include "crimson/net/Errors.h" | |
23 | #include "crimson/net/Messenger.h" | |
24 | ||
25 | #include "messages/MAuth.h" | |
26 | #include "messages/MAuthReply.h" | |
27 | #include "messages/MConfig.h" | |
28 | #include "messages/MLogAck.h" | |
29 | #include "messages/MMonCommand.h" | |
30 | #include "messages/MMonCommandAck.h" | |
9f95a23c | 31 | #include "messages/MMonGetMap.h" |
11fdf7f2 TL |
32 | #include "messages/MMonGetVersion.h" |
33 | #include "messages/MMonGetVersionReply.h" | |
34 | #include "messages/MMonMap.h" | |
35 | #include "messages/MMonSubscribe.h" | |
36 | #include "messages/MMonSubscribeAck.h" | |
37 | ||
38 | namespace { | |
39 | seastar::logger& logger() | |
40 | { | |
9f95a23c | 41 | return crimson::get_logger(ceph_subsys_monc); |
11fdf7f2 TL |
42 | } |
43 | } | |
44 | ||
9f95a23c | 45 | namespace crimson::mon { |
11fdf7f2 | 46 | |
9f95a23c | 47 | using crimson::common::local_conf; |
11fdf7f2 TL |
48 | |
49 | class Connection { | |
50 | public: | |
9f95a23c TL |
51 | Connection(const AuthRegistry& auth_registry, |
52 | crimson::net::ConnectionRef conn, | |
53 | KeyRing* keyring); | |
54 | enum class AuthResult { | |
55 | success = 0, | |
56 | failure, | |
57 | canceled | |
58 | }; | |
11fdf7f2 | 59 | seastar::future<> handle_auth_reply(Ref<MAuthReply> m); |
9f95a23c TL |
60 | // v1 |
61 | seastar::future<AuthResult> authenticate_v1( | |
62 | epoch_t epoch, | |
63 | const EntityName& name, | |
64 | uint32_t want_keys); | |
65 | // v2 | |
66 | seastar::future<AuthResult> authenticate_v2(); | |
67 | auth::AuthClient::auth_request_t | |
68 | get_auth_request(const EntityName& name, | |
69 | uint32_t want_keys); | |
70 | using secret_t = string; | |
71 | tuple<CryptoKey, secret_t, bufferlist> | |
72 | handle_auth_reply_more(const ceph::buffer::list& bl); | |
73 | tuple<CryptoKey, secret_t, int> | |
74 | handle_auth_done(uint64_t new_global_id, | |
75 | const ceph::buffer::list& bl); | |
76 | int handle_auth_bad_method(uint32_t old_auth_method, | |
77 | int result, | |
78 | const std::vector<uint32_t>& allowed_methods, | |
79 | const std::vector<uint32_t>& allowed_modes); | |
80 | ||
81 | // v1 and v2 | |
11fdf7f2 TL |
82 | seastar::future<> close(); |
83 | bool is_my_peer(const entity_addr_t& addr) const; | |
9f95a23c TL |
84 | AuthAuthorizer* get_authorizer(entity_type_t peer) const; |
85 | KeyStore& get_keys(); | |
11fdf7f2 | 86 | seastar::future<> renew_tickets(); |
9f95a23c TL |
87 | seastar::future<> renew_rotating_keyring(); |
88 | ||
89 | crimson::net::ConnectionRef get_conn(); | |
11fdf7f2 TL |
90 | |
91 | private: | |
92 | seastar::future<> setup_session(epoch_t epoch, | |
9f95a23c TL |
93 | const EntityName& name); |
94 | std::unique_ptr<AuthClientHandler> create_auth(crimson::auth::method_t, | |
95 | uint64_t global_id, | |
96 | const EntityName& name, | |
97 | uint32_t want_keys); | |
98 | enum class request_t { | |
99 | rotating, | |
100 | general, | |
101 | }; | |
102 | seastar::future<std::optional<AuthResult>> do_auth_single(request_t); | |
103 | seastar::future<AuthResult> do_auth(request_t); | |
11fdf7f2 TL |
104 | |
105 | private: | |
106 | bool closed = false; | |
9f95a23c TL |
107 | // v1 |
108 | seastar::shared_promise<Ref<MAuthReply>> reply; | |
109 | // v2 | |
110 | using clock_t = seastar::lowres_system_clock; | |
111 | clock_t::time_point auth_start; | |
112 | crimson::auth::method_t auth_method = 0; | |
113 | seastar::promise<AuthResult> auth_done; | |
114 | // v1 and v2 | |
115 | const AuthRegistry& auth_registry; | |
116 | crimson::net::ConnectionRef conn; | |
11fdf7f2 | 117 | std::unique_ptr<AuthClientHandler> auth; |
9f95a23c TL |
118 | std::unique_ptr<RotatingKeyRing> rotating_keyring; |
119 | uint64_t global_id = 0; | |
120 | clock_t::time_point last_rotating_renew_sent; | |
11fdf7f2 TL |
121 | }; |
122 | ||
9f95a23c TL |
123 | Connection::Connection(const AuthRegistry& auth_registry, |
124 | crimson::net::ConnectionRef conn, | |
11fdf7f2 | 125 | KeyRing* keyring) |
9f95a23c TL |
126 | : auth_registry{auth_registry}, |
127 | conn{conn}, | |
128 | rotating_keyring{ | |
129 | std::make_unique<RotatingKeyRing>(nullptr, | |
130 | CEPH_ENTITY_TYPE_OSD, | |
131 | keyring)} | |
11fdf7f2 TL |
132 | {} |
133 | ||
134 | seastar::future<> Connection::handle_auth_reply(Ref<MAuthReply> m) | |
135 | { | |
136 | reply.set_value(m); | |
9f95a23c | 137 | reply = {}; |
11fdf7f2 TL |
138 | return seastar::now(); |
139 | } | |
140 | ||
141 | seastar::future<> Connection::renew_tickets() | |
142 | { | |
143 | if (auth->need_tickets()) { | |
9f95a23c TL |
144 | return do_auth(request_t::general).then([](AuthResult r) { |
145 | if (r != AuthResult::success) { | |
146 | throw std::system_error( | |
147 | make_error_code( | |
148 | crimson::net::error::negotiation_failure)); | |
11fdf7f2 TL |
149 | } |
150 | }); | |
151 | } | |
152 | return seastar::now(); | |
153 | } | |
154 | ||
9f95a23c TL |
155 | seastar::future<> Connection::renew_rotating_keyring() |
156 | { | |
157 | auto now = clock_t::now(); | |
158 | auto ttl = std::chrono::seconds{ | |
159 | static_cast<long>(crimson::common::local_conf()->auth_service_ticket_ttl)}; | |
160 | auto cutoff = now - ttl / 4; | |
161 | if (!rotating_keyring->need_new_secrets(utime_t(cutoff))) { | |
162 | return seastar::now(); | |
163 | } | |
164 | if (now - last_rotating_renew_sent < std::chrono::seconds{1}) { | |
165 | logger().info("renew_rotating_keyring called too often"); | |
166 | return seastar::now(); | |
167 | } | |
168 | last_rotating_renew_sent = now; | |
169 | return do_auth(request_t::rotating).then([](AuthResult r) { | |
170 | if (r != AuthResult::success) { | |
171 | throw std::system_error(make_error_code( | |
172 | crimson::net::error::negotiation_failure)); | |
173 | } | |
174 | }); | |
175 | } | |
176 | ||
177 | AuthAuthorizer* Connection::get_authorizer(entity_type_t peer) const | |
178 | { | |
179 | if (auth) { | |
180 | return auth->build_authorizer(peer); | |
181 | } else { | |
182 | return nullptr; | |
183 | } | |
184 | } | |
185 | ||
186 | KeyStore& Connection::get_keys() { | |
187 | return *rotating_keyring; | |
188 | } | |
189 | ||
11fdf7f2 | 190 | std::unique_ptr<AuthClientHandler> |
9f95a23c TL |
191 | Connection::create_auth(crimson::auth::method_t protocol, |
192 | uint64_t global_id, | |
11fdf7f2 TL |
193 | const EntityName& name, |
194 | uint32_t want_keys) | |
195 | { | |
9f95a23c | 196 | static crimson::common::CephContext cct; |
11fdf7f2 TL |
197 | std::unique_ptr<AuthClientHandler> auth; |
198 | auth.reset(AuthClientHandler::create(&cct, | |
9f95a23c TL |
199 | protocol, |
200 | rotating_keyring.get())); | |
11fdf7f2 | 201 | if (!auth) { |
9f95a23c | 202 | logger().error("no handler for protocol {}", protocol); |
11fdf7f2 | 203 | throw std::system_error(make_error_code( |
9f95a23c | 204 | crimson::net::error::negotiation_failure)); |
11fdf7f2 TL |
205 | } |
206 | auth->init(name); | |
207 | auth->set_want_keys(want_keys); | |
208 | auth->set_global_id(global_id); | |
11fdf7f2 TL |
209 | return auth; |
210 | } | |
211 | ||
212 | seastar::future<> | |
213 | Connection::setup_session(epoch_t epoch, | |
11fdf7f2 TL |
214 | const EntityName& name) |
215 | { | |
9f95a23c TL |
216 | auto m = ceph::make_message<MAuth>(); |
217 | m->protocol = CEPH_AUTH_UNKNOWN; | |
11fdf7f2 TL |
218 | m->monmap_epoch = epoch; |
219 | __u8 struct_v = 1; | |
220 | encode(struct_v, m->auth_payload); | |
9f95a23c TL |
221 | std::vector<crimson::auth::method_t> auth_methods; |
222 | auth_registry.get_supported_methods(conn->get_peer_type(), &auth_methods); | |
223 | encode(auth_methods, m->auth_payload); | |
11fdf7f2 TL |
224 | encode(name, m->auth_payload); |
225 | encode(global_id, m->auth_payload); | |
226 | return conn->send(m); | |
227 | } | |
228 | ||
9f95a23c TL |
229 | seastar::future<std::optional<Connection::AuthResult>> |
230 | Connection::do_auth_single(Connection::request_t what) | |
11fdf7f2 TL |
231 | { |
232 | auto m = make_message<MAuth>(); | |
233 | m->protocol = auth->get_protocol(); | |
234 | auth->prepare_build_request(); | |
9f95a23c TL |
235 | switch (what) { |
236 | case request_t::rotating: | |
237 | auth->build_rotating_request(m->auth_payload); | |
238 | break; | |
239 | case request_t::general: | |
240 | if (int ret = auth->build_request(m->auth_payload); ret) { | |
241 | logger().error("missing/bad key for '{}'", local_conf()->name); | |
242 | throw std::system_error(make_error_code( | |
243 | crimson::net::error::negotiation_failure)); | |
244 | } | |
245 | break; | |
246 | default: | |
247 | assert(0); | |
11fdf7f2 TL |
248 | } |
249 | logger().info("sending {}", *m); | |
250 | return conn->send(m).then([this] { | |
251 | logger().info("waiting"); | |
9f95a23c | 252 | return reply.get_shared_future(); |
11fdf7f2 | 253 | }).then([this] (Ref<MAuthReply> m) { |
9f95a23c TL |
254 | if (!m) { |
255 | ceph_assert(closed); | |
256 | logger().info("do_auth: connection closed"); | |
257 | return seastar::make_ready_future<std::optional<Connection::AuthResult>>( | |
258 | std::make_optional(AuthResult::canceled)); | |
259 | } | |
260 | logger().info( | |
261 | "do_auth: mon {} => {} returns {}: {}", | |
262 | conn->get_messenger()->get_myaddr(), | |
263 | conn->get_peer_addr(), *m, m->result); | |
11fdf7f2 TL |
264 | auto p = m->result_bl.cbegin(); |
265 | auto ret = auth->handle_response(m->result, p, | |
11fdf7f2 TL |
266 | nullptr, nullptr); |
267 | if (ret != 0 && ret != -EAGAIN) { | |
9f95a23c TL |
268 | logger().error( |
269 | "do_auth: got error {} on mon {}", | |
270 | ret, | |
271 | conn->get_peer_addr()); | |
11fdf7f2 | 272 | } |
9f95a23c TL |
273 | return seastar::make_ready_future<std::optional<Connection::AuthResult>>( |
274 | ret == -EAGAIN | |
275 | ? std::nullopt | |
276 | : std::make_optional(ret == 0 | |
277 | ? AuthResult::success | |
278 | : AuthResult::failure | |
279 | )); | |
11fdf7f2 TL |
280 | }); |
281 | } | |
282 | ||
9f95a23c TL |
283 | seastar::future<Connection::AuthResult> |
284 | Connection::do_auth(Connection::request_t what) { | |
285 | return seastar::repeat_until_value([this, what]() { | |
286 | return do_auth_single(what); | |
287 | }); | |
288 | } | |
289 | ||
290 | seastar::future<Connection::AuthResult> | |
291 | Connection::authenticate_v1(epoch_t epoch, | |
292 | const EntityName& name, | |
293 | uint32_t want_keys) | |
11fdf7f2 | 294 | { |
9f95a23c TL |
295 | return conn->keepalive().then([epoch, name, this] { |
296 | return setup_session(epoch, name); | |
11fdf7f2 | 297 | }).then([this] { |
9f95a23c | 298 | return reply.get_shared_future(); |
11fdf7f2 | 299 | }).then([name, want_keys, this](Ref<MAuthReply> m) { |
9f95a23c TL |
300 | if (!m) { |
301 | logger().error("authenticate_v1 canceled on {}", name); | |
302 | return seastar::make_ready_future<AuthResult>(AuthResult::canceled); | |
303 | } | |
11fdf7f2 | 304 | global_id = m->global_id; |
9f95a23c | 305 | auth = create_auth(m->protocol, m->global_id, name, want_keys); |
11fdf7f2 TL |
306 | switch (auto p = m->result_bl.cbegin(); |
307 | auth->handle_response(m->result, p, | |
11fdf7f2 TL |
308 | nullptr, nullptr)) { |
309 | case 0: | |
310 | // none | |
9f95a23c | 311 | return seastar::make_ready_future<AuthResult>(AuthResult::success); |
11fdf7f2 TL |
312 | case -EAGAIN: |
313 | // cephx | |
9f95a23c | 314 | return do_auth(request_t::general); |
11fdf7f2 TL |
315 | default: |
316 | ceph_assert_always(0); | |
317 | } | |
9f95a23c TL |
318 | }).handle_exception([](auto ep) { |
319 | logger().error("authenticate_v1 failed with {}", ep); | |
320 | return seastar::make_ready_future<AuthResult>(AuthResult::canceled); | |
321 | }); | |
322 | } | |
323 | ||
324 | seastar::future<Connection::AuthResult> Connection::authenticate_v2() | |
325 | { | |
326 | auth_start = seastar::lowres_system_clock::now(); | |
327 | return conn->send(make_message<MMonGetMap>()).then([this] { | |
328 | return auth_done.get_future(); | |
11fdf7f2 TL |
329 | }); |
330 | } | |
331 | ||
9f95a23c TL |
332 | auth::AuthClient::auth_request_t |
333 | Connection::get_auth_request(const EntityName& entity_name, | |
334 | uint32_t want_keys) | |
335 | { | |
336 | // choose method | |
337 | auth_method = [&] { | |
338 | std::vector<crimson::auth::method_t> methods; | |
339 | auth_registry.get_supported_methods(conn->get_peer_type(), &methods); | |
340 | if (methods.empty()) { | |
341 | logger().info("get_auth_request no methods is supported"); | |
342 | throw crimson::auth::error("no methods is supported"); | |
343 | } | |
344 | return methods.front(); | |
345 | }(); | |
346 | ||
347 | std::vector<uint32_t> modes; | |
348 | auth_registry.get_supported_modes(conn->get_peer_type(), auth_method, | |
349 | &modes); | |
350 | logger().info("method {} preferred_modes {}", auth_method, modes); | |
351 | if (modes.empty()) { | |
352 | throw crimson::auth::error("no modes is supported"); | |
353 | } | |
354 | auth = create_auth(auth_method, global_id, entity_name, want_keys); | |
355 | ||
356 | using ceph::encode; | |
357 | bufferlist bl; | |
358 | // initial request includes some boilerplate... | |
359 | encode((char)AUTH_MODE_MON, bl); | |
360 | encode(entity_name, bl); | |
361 | encode(global_id, bl); | |
362 | // and (maybe) some method-specific initial payload | |
363 | auth->build_initial_request(&bl); | |
364 | return {auth_method, modes, bl}; | |
365 | } | |
366 | ||
367 | tuple<CryptoKey, Connection::secret_t, bufferlist> | |
368 | Connection::handle_auth_reply_more(const ceph::buffer::list& payload) | |
369 | { | |
370 | CryptoKey session_key; | |
371 | secret_t connection_secret; | |
372 | bufferlist reply; | |
373 | auto p = payload.cbegin(); | |
374 | int r = auth->handle_response(0, p, &session_key, &connection_secret); | |
375 | if (r == -EAGAIN) { | |
376 | auth->prepare_build_request(); | |
377 | auth->build_request(reply); | |
378 | logger().info(" responding with {} bytes", reply.length()); | |
379 | return {session_key, connection_secret, reply}; | |
380 | } else if (r < 0) { | |
381 | logger().error(" handle_response returned {}", r); | |
382 | throw crimson::auth::error("unable to build auth"); | |
383 | } else { | |
384 | logger().info("authenticated!"); | |
385 | std::terminate(); | |
386 | } | |
387 | } | |
388 | ||
389 | tuple<CryptoKey, Connection::secret_t, int> | |
390 | Connection::handle_auth_done(uint64_t new_global_id, | |
391 | const ceph::buffer::list& payload) | |
392 | { | |
393 | global_id = new_global_id; | |
394 | auth->set_global_id(global_id); | |
395 | auto p = payload.begin(); | |
396 | CryptoKey session_key; | |
397 | secret_t connection_secret; | |
398 | int r = auth->handle_response(0, p, &session_key, &connection_secret); | |
399 | conn->set_last_keepalive_ack(auth_start); | |
400 | auth_done.set_value(AuthResult::success); | |
401 | return {session_key, connection_secret, r}; | |
402 | } | |
403 | ||
404 | int Connection::handle_auth_bad_method(uint32_t old_auth_method, | |
405 | int result, | |
406 | const std::vector<uint32_t>& allowed_methods, | |
407 | const std::vector<uint32_t>& allowed_modes) | |
408 | { | |
409 | logger().info("old_auth_method {} result {} allowed_methods {}", | |
410 | old_auth_method, cpp_strerror(result), allowed_methods); | |
411 | std::vector<uint32_t> auth_supported; | |
412 | auth_registry.get_supported_methods(conn->get_peer_type(), &auth_supported); | |
413 | auto p = std::find(auth_supported.begin(), auth_supported.end(), | |
414 | old_auth_method); | |
415 | assert(p != auth_supported.end()); | |
416 | p = std::find_first_of(std::next(p), auth_supported.end(), | |
417 | allowed_methods.begin(), allowed_methods.end()); | |
418 | if (p == auth_supported.end()) { | |
419 | logger().error("server allowed_methods {} but i only support {}", | |
420 | allowed_methods, auth_supported); | |
421 | auth_done.set_exception(std::system_error(make_error_code( | |
422 | crimson::net::error::negotiation_failure))); | |
423 | return -EACCES; | |
424 | } | |
425 | auth_method = *p; | |
426 | logger().info("will try {} next", auth_method); | |
427 | return 0; | |
428 | } | |
429 | ||
11fdf7f2 TL |
430 | seastar::future<> Connection::close() |
431 | { | |
9f95a23c TL |
432 | reply.set_value(Ref<MAuthReply>(nullptr)); |
433 | reply = {}; | |
434 | auth_done.set_value(AuthResult::canceled); | |
435 | auth_done = {}; | |
11fdf7f2 TL |
436 | if (conn && !std::exchange(closed, true)) { |
437 | return conn->close(); | |
438 | } else { | |
439 | return seastar::now(); | |
440 | } | |
441 | } | |
442 | ||
443 | bool Connection::is_my_peer(const entity_addr_t& addr) const | |
444 | { | |
9f95a23c | 445 | ceph_assert(conn); |
11fdf7f2 TL |
446 | return conn->get_peer_addr() == addr; |
447 | } | |
448 | ||
9f95a23c | 449 | crimson::net::ConnectionRef Connection::get_conn() { |
11fdf7f2 TL |
450 | return conn; |
451 | } | |
452 | ||
9f95a23c TL |
453 | Client::Client(crimson::net::Messenger& messenger, |
454 | crimson::common::AuthHandler& auth_handler) | |
11fdf7f2 TL |
455 | // currently, crimson is OSD-only |
456 | : want_keys{CEPH_ENTITY_TYPE_MON | | |
457 | CEPH_ENTITY_TYPE_OSD | | |
458 | CEPH_ENTITY_TYPE_MGR}, | |
459 | timer{[this] { tick(); }}, | |
9f95a23c TL |
460 | msgr{messenger}, |
461 | auth_registry{&cct}, | |
462 | auth_handler{auth_handler} | |
11fdf7f2 TL |
463 | {} |
464 | ||
465 | Client::Client(Client&&) = default; | |
466 | Client::~Client() = default; | |
467 | ||
468 | seastar::future<> Client::start() { | |
9f95a23c TL |
469 | entity_name = crimson::common::local_conf()->name; |
470 | auth_registry.refresh_config(); | |
11fdf7f2 | 471 | return load_keyring().then([this] { |
9f95a23c | 472 | return monmap.build_initial(crimson::common::local_conf(), false); |
11fdf7f2 TL |
473 | }).then([this] { |
474 | return authenticate(); | |
9f95a23c TL |
475 | }).then([this] { |
476 | auto interval = | |
477 | std::chrono::duration_cast<seastar::lowres_clock::duration>( | |
478 | std::chrono::duration<double>( | |
479 | local_conf().get_val<double>("mon_client_ping_interval"))); | |
480 | timer.arm_periodic(interval); | |
11fdf7f2 TL |
481 | }); |
482 | } | |
483 | ||
484 | seastar::future<> Client::load_keyring() | |
485 | { | |
9f95a23c | 486 | if (!auth_registry.is_supported_method(msgr.get_mytype(), CEPH_AUTH_CEPHX)) { |
11fdf7f2 TL |
487 | return seastar::now(); |
488 | } else { | |
9f95a23c TL |
489 | return crimson::auth::load_from_keyring(&keyring).then([](KeyRing* keyring) { |
490 | return crimson::auth::load_from_keyfile(keyring); | |
11fdf7f2 | 491 | }).then([](KeyRing* keyring) { |
9f95a23c | 492 | return crimson::auth::load_from_key(keyring); |
11fdf7f2 TL |
493 | }).then([](KeyRing*) { |
494 | return seastar::now(); | |
495 | }); | |
496 | } | |
497 | } | |
498 | ||
499 | void Client::tick() | |
500 | { | |
9f95a23c TL |
501 | (void) seastar::with_gate(tick_gate, [this] { |
502 | if (active_con) { | |
503 | return seastar::when_all_succeed(active_con->get_conn()->keepalive(), | |
504 | active_con->renew_tickets(), | |
505 | active_con->renew_rotating_keyring()); | |
506 | } else { | |
507 | return seastar::now(); | |
508 | } | |
11fdf7f2 TL |
509 | }); |
510 | } | |
511 | ||
512 | bool Client::is_hunting() const { | |
513 | return !active_con; | |
514 | } | |
515 | ||
516 | seastar::future<> | |
9f95a23c | 517 | Client::ms_dispatch(crimson::net::Connection* conn, MessageRef m) |
11fdf7f2 | 518 | { |
11fdf7f2 TL |
519 | // we only care about these message types |
520 | switch (m->get_type()) { | |
521 | case CEPH_MSG_MON_MAP: | |
522 | return handle_monmap(conn, boost::static_pointer_cast<MMonMap>(m)); | |
523 | case CEPH_MSG_AUTH_REPLY: | |
524 | return handle_auth_reply( | |
525 | conn, boost::static_pointer_cast<MAuthReply>(m)); | |
526 | case CEPH_MSG_MON_SUBSCRIBE_ACK: | |
527 | return handle_subscribe_ack( | |
528 | boost::static_pointer_cast<MMonSubscribeAck>(m)); | |
529 | case CEPH_MSG_MON_GET_VERSION_REPLY: | |
530 | return handle_get_version_reply( | |
531 | boost::static_pointer_cast<MMonGetVersionReply>(m)); | |
532 | case MSG_MON_COMMAND_ACK: | |
533 | return handle_mon_command_ack( | |
534 | boost::static_pointer_cast<MMonCommandAck>(m)); | |
535 | case MSG_LOGACK: | |
536 | return handle_log_ack( | |
537 | boost::static_pointer_cast<MLogAck>(m)); | |
538 | case MSG_CONFIG: | |
539 | return handle_config( | |
540 | boost::static_pointer_cast<MConfig>(m)); | |
541 | default: | |
542 | return seastar::now(); | |
543 | } | |
544 | } | |
545 | ||
9f95a23c | 546 | seastar::future<> Client::ms_handle_reset(crimson::net::ConnectionRef conn) |
11fdf7f2 TL |
547 | { |
548 | auto found = std::find_if(pending_conns.begin(), pending_conns.end(), | |
549 | [peer_addr = conn->get_peer_addr()](auto& mc) { | |
9f95a23c | 550 | return mc->is_my_peer(peer_addr); |
11fdf7f2 TL |
551 | }); |
552 | if (found != pending_conns.end()) { | |
553 | logger().warn("pending conn reset by {}", conn->get_peer_addr()); | |
9f95a23c | 554 | return (*found)->close(); |
11fdf7f2 TL |
555 | } else if (active_con && active_con->is_my_peer(conn->get_peer_addr())) { |
556 | logger().warn("active conn reset {}", conn->get_peer_addr()); | |
557 | active_con.reset(); | |
558 | return reopen_session(-1); | |
559 | } else { | |
560 | logger().error("unknown reset from {}", conn->get_peer_addr()); | |
561 | return seastar::now(); | |
562 | } | |
563 | } | |
564 | ||
9f95a23c TL |
565 | std::pair<std::vector<uint32_t>, std::vector<uint32_t>> |
566 | Client::get_supported_auth_methods(int peer_type) | |
567 | { | |
568 | std::vector<uint32_t> methods; | |
569 | std::vector<uint32_t> modes; | |
570 | auth_registry.get_supported_methods(peer_type, &methods, &modes); | |
571 | return {methods, modes}; | |
572 | } | |
573 | ||
574 | uint32_t Client::pick_con_mode(int peer_type, | |
575 | uint32_t auth_method, | |
576 | const std::vector<uint32_t>& preferred_modes) | |
577 | { | |
578 | return auth_registry.pick_mode(peer_type, auth_method, preferred_modes); | |
579 | } | |
580 | ||
581 | AuthAuthorizeHandler* Client::get_auth_authorize_handler(int peer_type, | |
582 | int auth_method) | |
583 | { | |
584 | return auth_registry.get_handler(peer_type, auth_method); | |
585 | } | |
586 | ||
587 | ||
588 | int Client::handle_auth_request(crimson::net::ConnectionRef con, | |
589 | AuthConnectionMetaRef auth_meta, | |
590 | bool more, | |
591 | uint32_t auth_method, | |
592 | const ceph::bufferlist& payload, | |
593 | ceph::bufferlist *reply) | |
594 | { | |
595 | // for some channels prior to nautilus (osd heartbeat), we tolerate the lack of | |
596 | // an authorizer. | |
597 | if (payload.length() == 0) { | |
598 | if (con->get_messenger()->get_require_authorizer()) { | |
599 | return -EACCES; | |
600 | } else { | |
601 | auth_handler.handle_authentication({}, {}); | |
602 | return 1; | |
603 | } | |
604 | } | |
605 | auth_meta->auth_mode = payload[0]; | |
606 | if (auth_meta->auth_mode < AUTH_MODE_AUTHORIZER || | |
607 | auth_meta->auth_mode > AUTH_MODE_AUTHORIZER_MAX) { | |
608 | return -EACCES; | |
609 | } | |
610 | AuthAuthorizeHandler* ah = get_auth_authorize_handler(con->get_peer_type(), | |
611 | auth_method); | |
612 | if (!ah) { | |
613 | logger().error("no AuthAuthorizeHandler found for auth method: {}", | |
614 | auth_method); | |
615 | return -EOPNOTSUPP; | |
616 | } | |
617 | auto authorizer_challenge = &auth_meta->authorizer_challenge; | |
618 | ceph_assert(active_con); | |
619 | if (!HAVE_FEATURE(active_con->get_conn()->get_features(), CEPHX_V2)) { | |
620 | if (local_conf().get_val<uint64_t>("cephx_service_require_version") >= 2) { | |
621 | return -EACCES; | |
622 | } | |
623 | authorizer_challenge = nullptr; | |
624 | } | |
625 | bool was_challenge = (bool)auth_meta->authorizer_challenge; | |
626 | EntityName name; | |
627 | AuthCapsInfo caps_info; | |
628 | bool is_valid = ah->verify_authorizer( | |
629 | &cct, | |
630 | active_con->get_keys(), | |
631 | payload, | |
632 | auth_meta->get_connection_secret_length(), | |
633 | reply, | |
634 | &name, | |
635 | &active_con->get_conn()->peer_global_id, | |
636 | &caps_info, | |
637 | &auth_meta->session_key, | |
638 | &auth_meta->connection_secret, | |
639 | authorizer_challenge); | |
640 | if (is_valid) { | |
641 | auth_handler.handle_authentication(name, caps_info); | |
642 | return 1; | |
643 | } | |
644 | if (!more && !was_challenge && auth_meta->authorizer_challenge) { | |
645 | logger().info("added challenge on {}", con); | |
646 | return 0; | |
647 | } else { | |
648 | logger().info("bad authorizer on {}", con); | |
649 | return -EACCES; | |
650 | } | |
651 | } | |
652 | ||
653 | auth::AuthClient::auth_request_t | |
654 | Client::get_auth_request(crimson::net::ConnectionRef con, | |
655 | AuthConnectionMetaRef auth_meta) | |
656 | { | |
657 | logger().info("get_auth_request(con={}, auth_method={})", | |
658 | con, auth_meta->auth_method); | |
659 | // connection to mon? | |
660 | if (con->get_peer_type() == CEPH_ENTITY_TYPE_MON) { | |
661 | auto found = std::find_if(pending_conns.begin(), pending_conns.end(), | |
662 | [peer_addr = con->get_peer_addr()](auto& mc) { | |
663 | return mc->is_my_peer(peer_addr); | |
664 | }); | |
665 | if (found == pending_conns.end()) { | |
666 | throw crimson::auth::error{"unknown connection"}; | |
667 | } | |
668 | return (*found)->get_auth_request(entity_name, want_keys); | |
669 | } else { | |
670 | // generate authorizer | |
671 | if (!active_con) { | |
672 | logger().error(" but no auth handler is set up"); | |
673 | throw crimson::auth::error("no auth available"); | |
674 | } | |
675 | auto authorizer = active_con->get_authorizer(con->get_peer_type()); | |
676 | if (!authorizer) { | |
677 | logger().error("failed to build_authorizer for type {}", | |
678 | ceph_entity_type_name(con->get_peer_type())); | |
679 | throw crimson::auth::error("unable to build auth"); | |
680 | } | |
681 | auth_meta->authorizer.reset(authorizer); | |
682 | auth_meta->auth_method = authorizer->protocol; | |
683 | vector<uint32_t> modes; | |
684 | auth_registry.get_supported_modes(con->get_peer_type(), | |
685 | auth_meta->auth_method, | |
686 | &modes); | |
687 | return {authorizer->protocol, modes, authorizer->bl}; | |
688 | } | |
689 | } | |
690 | ||
691 | ceph::bufferlist Client::handle_auth_reply_more(crimson::net::ConnectionRef conn, | |
692 | AuthConnectionMetaRef auth_meta, | |
693 | const bufferlist& bl) | |
694 | { | |
695 | if (conn->get_peer_type() == CEPH_ENTITY_TYPE_MON) { | |
696 | auto found = std::find_if(pending_conns.begin(), pending_conns.end(), | |
697 | [peer_addr = conn->get_peer_addr()](auto& mc) { | |
698 | return mc->is_my_peer(peer_addr); | |
699 | }); | |
700 | if (found == pending_conns.end()) { | |
701 | throw crimson::auth::error{"unknown connection"}; | |
702 | } | |
703 | bufferlist reply; | |
704 | tie(auth_meta->session_key, auth_meta->connection_secret, reply) = | |
705 | (*found)->handle_auth_reply_more(bl); | |
706 | return reply; | |
707 | } else { | |
708 | // authorizer challenges | |
709 | if (!active_con || !auth_meta->authorizer) { | |
710 | logger().error("no authorizer?"); | |
711 | throw crimson::auth::error("no auth available"); | |
712 | } | |
713 | auth_meta->authorizer->add_challenge(&cct, bl); | |
714 | return auth_meta->authorizer->bl; | |
715 | } | |
716 | } | |
717 | ||
718 | int Client::handle_auth_done(crimson::net::ConnectionRef conn, | |
719 | AuthConnectionMetaRef auth_meta, | |
720 | uint64_t global_id, | |
721 | uint32_t con_mode, | |
722 | const bufferlist& bl) | |
723 | { | |
724 | if (conn->get_peer_type() == CEPH_ENTITY_TYPE_MON) { | |
725 | auto found = std::find_if(pending_conns.begin(), pending_conns.end(), | |
726 | [peer_addr = conn->get_peer_addr()](auto& mc) { | |
727 | return mc->is_my_peer(peer_addr); | |
728 | }); | |
729 | if (found == pending_conns.end()) { | |
730 | return -ENOENT; | |
731 | } | |
732 | int r = 0; | |
733 | tie(auth_meta->session_key, auth_meta->connection_secret, r) = | |
734 | (*found)->handle_auth_done(global_id, bl); | |
735 | return r; | |
736 | } else { | |
737 | // verify authorizer reply | |
738 | auto p = bl.begin(); | |
739 | if (!auth_meta->authorizer->verify_reply(p, &auth_meta->connection_secret)) { | |
740 | logger().error("failed verifying authorizer reply"); | |
741 | return -EACCES; | |
742 | } | |
743 | auth_meta->session_key = auth_meta->authorizer->session_key; | |
744 | return 0; | |
745 | } | |
746 | } | |
747 | ||
748 | // Handle server's indication that the previous auth attempt failed | |
749 | int Client::handle_auth_bad_method(crimson::net::ConnectionRef conn, | |
750 | AuthConnectionMetaRef auth_meta, | |
751 | uint32_t old_auth_method, | |
752 | int result, | |
753 | const std::vector<uint32_t>& allowed_methods, | |
754 | const std::vector<uint32_t>& allowed_modes) | |
755 | { | |
756 | if (conn->get_peer_type() == CEPH_ENTITY_TYPE_MON) { | |
757 | auto found = std::find_if(pending_conns.begin(), pending_conns.end(), | |
758 | [peer_addr = conn->get_peer_addr()](auto& mc) { | |
759 | return mc->is_my_peer(peer_addr); | |
760 | }); | |
761 | if (found != pending_conns.end()) { | |
762 | return (*found)->handle_auth_bad_method( | |
763 | old_auth_method, result, | |
764 | allowed_methods, allowed_modes); | |
765 | } else { | |
766 | return -ENOENT; | |
767 | } | |
768 | } else { | |
769 | // huh... | |
770 | logger().info("hmm, they didn't like {} result {}", | |
771 | old_auth_method, cpp_strerror(result)); | |
772 | return -EACCES; | |
773 | } | |
774 | } | |
775 | ||
776 | seastar::future<> Client::handle_monmap(crimson::net::Connection* conn, | |
11fdf7f2 TL |
777 | Ref<MMonMap> m) |
778 | { | |
779 | monmap.decode(m->monmapbl); | |
780 | const auto peer_addr = conn->get_peer_addr(); | |
781 | auto cur_mon = monmap.get_name(peer_addr); | |
782 | logger().info("got monmap {}, mon.{}, is now rank {}", | |
783 | monmap.epoch, cur_mon, monmap.get_rank(cur_mon)); | |
784 | sub.got("monmap", monmap.get_epoch()); | |
785 | ||
786 | if (monmap.get_addr_name(peer_addr, cur_mon)) { | |
9f95a23c TL |
787 | if (active_con) { |
788 | logger().info("handle_monmap: renewing tickets"); | |
789 | return seastar::when_all_succeed( | |
790 | active_con->renew_tickets(), | |
791 | active_con->renew_rotating_keyring()).then([](){ | |
792 | logger().info("handle_mon_map: renewed tickets"); | |
793 | }); | |
794 | } else { | |
795 | return seastar::now(); | |
796 | } | |
11fdf7f2 TL |
797 | } else { |
798 | logger().warn("mon.{} went away", cur_mon); | |
799 | return reopen_session(-1); | |
800 | } | |
801 | } | |
802 | ||
9f95a23c | 803 | seastar::future<> Client::handle_auth_reply(crimson::net::Connection* conn, |
11fdf7f2 TL |
804 | Ref<MAuthReply> m) |
805 | { | |
9f95a23c TL |
806 | logger().info( |
807 | "handle_auth_reply mon {} => {} returns {}: {}", | |
808 | conn->get_messenger()->get_myaddr(), | |
809 | conn->get_peer_addr(), *m, m->result); | |
11fdf7f2 TL |
810 | auto found = std::find_if(pending_conns.begin(), pending_conns.end(), |
811 | [peer_addr = conn->get_peer_addr()](auto& mc) { | |
9f95a23c | 812 | return mc->is_my_peer(peer_addr); |
11fdf7f2 TL |
813 | }); |
814 | if (found != pending_conns.end()) { | |
9f95a23c | 815 | return (*found)->handle_auth_reply(m); |
11fdf7f2 TL |
816 | } else if (active_con) { |
817 | return active_con->handle_auth_reply(m); | |
818 | } else { | |
819 | logger().error("unknown auth reply from {}", conn->get_peer_addr()); | |
820 | return seastar::now(); | |
821 | } | |
822 | } | |
823 | ||
824 | seastar::future<> Client::handle_subscribe_ack(Ref<MMonSubscribeAck> m) | |
825 | { | |
826 | sub.acked(m->interval); | |
827 | return seastar::now(); | |
828 | } | |
829 | ||
830 | Client::get_version_t Client::get_version(const std::string& map) | |
831 | { | |
832 | auto m = make_message<MMonGetVersion>(); | |
833 | auto tid = ++last_version_req_id; | |
834 | m->handle = tid; | |
835 | m->what = map; | |
836 | auto& req = version_reqs[tid]; | |
837 | return active_con->get_conn()->send(m).then([&req] { | |
838 | return req.get_future(); | |
839 | }); | |
840 | } | |
841 | ||
842 | seastar::future<> | |
843 | Client::handle_get_version_reply(Ref<MMonGetVersionReply> m) | |
844 | { | |
845 | if (auto found = version_reqs.find(m->handle); | |
846 | found != version_reqs.end()) { | |
847 | auto& result = found->second; | |
848 | logger().trace("{}: {} returns {}", | |
849 | __func__, m->handle, m->version); | |
850 | result.set_value(m->version, m->oldest_version); | |
851 | version_reqs.erase(found); | |
852 | } else { | |
853 | logger().warn("{}: version request with handle {} not found", | |
854 | __func__, m->handle); | |
855 | } | |
856 | return seastar::now(); | |
857 | } | |
858 | ||
859 | seastar::future<> Client::handle_mon_command_ack(Ref<MMonCommandAck> m) | |
860 | { | |
861 | const auto tid = m->get_tid(); | |
862 | if (auto found = mon_commands.find(tid); | |
863 | found != mon_commands.end()) { | |
864 | auto& result = found->second; | |
865 | logger().trace("{} {}", __func__, tid); | |
866 | result.set_value(m->r, m->rs, std::move(m->get_data())); | |
867 | mon_commands.erase(found); | |
868 | } else { | |
869 | logger().warn("{} {} not found", __func__, tid); | |
870 | } | |
871 | return seastar::now(); | |
872 | } | |
873 | ||
874 | seastar::future<> Client::handle_log_ack(Ref<MLogAck> m) | |
875 | { | |
876 | // XXX | |
877 | return seastar::now(); | |
878 | } | |
879 | ||
880 | seastar::future<> Client::handle_config(Ref<MConfig> m) | |
881 | { | |
9f95a23c | 882 | return crimson::common::local_conf().set_mon_vals(m->config); |
11fdf7f2 TL |
883 | } |
884 | ||
885 | std::vector<unsigned> Client::get_random_mons(unsigned n) const | |
886 | { | |
887 | uint16_t min_priority = std::numeric_limits<uint16_t>::max(); | |
888 | for (const auto& m : monmap.mon_info) { | |
889 | if (m.second.priority < min_priority) { | |
890 | min_priority = m.second.priority; | |
891 | } | |
892 | } | |
893 | vector<unsigned> ranks; | |
894 | for (auto [name, info] : monmap.mon_info) { | |
895 | // TODO: #msgr-v2 | |
896 | if (info.public_addrs.legacy_addr().is_blank_ip()) { | |
897 | continue; | |
898 | } | |
899 | if (info.priority == min_priority) { | |
900 | ranks.push_back(monmap.get_rank(name)); | |
901 | } | |
902 | } | |
903 | std::random_device rd; | |
904 | std::default_random_engine rng{rd()}; | |
905 | std::shuffle(ranks.begin(), ranks.end(), rng); | |
906 | if (n == 0 || n > ranks.size()) { | |
907 | return ranks; | |
908 | } else { | |
909 | return {ranks.begin(), ranks.begin() + n}; | |
910 | } | |
911 | } | |
912 | ||
913 | seastar::future<> Client::authenticate() | |
914 | { | |
915 | return reopen_session(-1); | |
916 | } | |
917 | ||
918 | seastar::future<> Client::stop() | |
919 | { | |
920 | return tick_gate.close().then([this] { | |
9f95a23c | 921 | timer.cancel(); |
11fdf7f2 TL |
922 | if (active_con) { |
923 | return active_con->close(); | |
924 | } else { | |
925 | return seastar::now(); | |
926 | } | |
927 | }); | |
928 | } | |
929 | ||
930 | seastar::future<> Client::reopen_session(int rank) | |
931 | { | |
9f95a23c | 932 | logger().info("{} to mon.{}", __func__, rank); |
11fdf7f2 TL |
933 | vector<unsigned> mons; |
934 | if (rank >= 0) { | |
935 | mons.push_back(rank); | |
936 | } else { | |
937 | const auto parallel = | |
9f95a23c | 938 | crimson::common::local_conf().get_val<uint64_t>("mon_client_hunt_parallel"); |
11fdf7f2 TL |
939 | mons = get_random_mons(parallel); |
940 | } | |
941 | pending_conns.reserve(mons.size()); | |
942 | return seastar::parallel_for_each(mons, [this](auto rank) { | |
943 | #warning fixme | |
9f95a23c | 944 | auto peer = monmap.get_addrs(rank).front(); |
11fdf7f2 | 945 | logger().info("connecting to mon.{}", rank); |
9f95a23c TL |
946 | return seastar::futurize_apply( |
947 | [peer, this] () -> seastar::future<Connection::AuthResult> { | |
948 | auto conn = msgr.connect(peer, CEPH_ENTITY_TYPE_MON); | |
949 | auto& mc = pending_conns.emplace_back( | |
950 | std::make_unique<Connection>(auth_registry, conn, &keyring)); | |
951 | if (conn->get_peer_addr().is_msgr2()) { | |
952 | return mc->authenticate_v2(); | |
953 | } else { | |
954 | return mc->authenticate_v1(monmap.get_epoch(), entity_name, want_keys) | |
955 | .handle_exception([conn](auto ep) { | |
956 | return conn->close().then([ep=std::move(ep)](){ | |
957 | return seastar::make_exception_future<Connection::AuthResult>(ep); | |
958 | }); | |
959 | }); | |
960 | } | |
961 | }).then([peer, this](auto result) { | |
962 | if (result == Connection::AuthResult::canceled) { | |
963 | return seastar::now(); | |
964 | } | |
965 | ||
11fdf7f2 TL |
966 | if (!is_hunting()) { |
967 | return seastar::now(); | |
968 | } | |
9f95a23c TL |
969 | logger().info("found mon.{}", monmap.get_name(peer)); |
970 | ||
971 | auto found = std::find_if( | |
972 | pending_conns.begin(), pending_conns.end(), | |
973 | [peer](auto& conn) { | |
974 | return conn->is_my_peer(peer); | |
975 | }); | |
976 | if (found == pending_conns.end()) { | |
977 | // Happens if another connection has won the race | |
978 | ceph_assert(active_con && pending_conns.empty()); | |
979 | logger().info( | |
980 | "no pending connection for mon.{}, peer {}", | |
981 | monmap.get_name(peer), | |
982 | peer); | |
983 | return seastar::now(); | |
984 | } | |
985 | ||
986 | ceph_assert(!active_con && !pending_conns.empty()); | |
987 | active_con = std::move(*found); | |
988 | found->reset(); | |
989 | auto ret = seastar::do_with( | |
990 | std::move(pending_conns), | |
991 | [](auto &pending_conns) { | |
992 | return seastar::parallel_for_each( | |
993 | pending_conns, | |
994 | [] (auto &conn) { | |
995 | if (!conn) { | |
996 | return seastar::now(); | |
997 | } else { | |
998 | return conn->close(); | |
999 | } | |
1000 | }); | |
1001 | }); | |
1002 | pending_conns.clear(); | |
1003 | return ret; | |
1004 | }).then([]() { | |
1005 | logger().debug("reopen_session mon connection attempts complete"); | |
1006 | }).handle_exception([](auto ep) { | |
1007 | logger().error("mon connections failed with ep {}", ep); | |
1008 | return seastar::make_exception_future(ep); | |
11fdf7f2 TL |
1009 | }); |
1010 | }).then([this] { | |
9f95a23c TL |
1011 | ceph_assert_always(active_con); |
1012 | return active_con->renew_rotating_keyring(); | |
11fdf7f2 TL |
1013 | }); |
1014 | } | |
1015 | ||
1016 | Client::command_result_t | |
1017 | Client::run_command(const std::vector<std::string>& cmd, | |
1018 | const bufferlist& bl) | |
1019 | { | |
1020 | auto m = make_message<MMonCommand>(monmap.fsid); | |
1021 | auto tid = ++last_mon_command_id; | |
1022 | m->set_tid(tid); | |
1023 | m->cmd = cmd; | |
1024 | m->set_data(bl); | |
1025 | auto& req = mon_commands[tid]; | |
1026 | return active_con->get_conn()->send(m).then([&req] { | |
1027 | return req.get_future(); | |
1028 | }); | |
1029 | } | |
1030 | ||
1031 | seastar::future<> Client::send_message(MessageRef m) | |
1032 | { | |
1033 | return active_con->get_conn()->send(m); | |
1034 | } | |
1035 | ||
1036 | bool Client::sub_want(const std::string& what, version_t start, unsigned flags) | |
1037 | { | |
1038 | return sub.want(what, start, flags); | |
1039 | } | |
1040 | ||
1041 | void Client::sub_got(const std::string& what, version_t have) | |
1042 | { | |
1043 | sub.got(what, have); | |
1044 | } | |
1045 | ||
1046 | void Client::sub_unwant(const std::string& what) | |
1047 | { | |
1048 | sub.unwant(what); | |
1049 | } | |
1050 | ||
1051 | bool Client::sub_want_increment(const std::string& what, | |
1052 | version_t start, | |
1053 | unsigned flags) | |
1054 | { | |
1055 | return sub.inc_want(what, start, flags); | |
1056 | } | |
1057 | ||
1058 | seastar::future<> Client::renew_subs() | |
1059 | { | |
1060 | if (!sub.have_new()) { | |
1061 | logger().warn("{} - empty", __func__); | |
1062 | return seastar::now(); | |
1063 | } | |
1064 | logger().trace("{}", __func__); | |
1065 | ||
1066 | auto m = make_message<MMonSubscribe>(); | |
1067 | m->what = sub.get_subs(); | |
1068 | m->hostname = ceph_get_short_hostname(); | |
1069 | return active_con->get_conn()->send(m).then([this] { | |
1070 | sub.renewed(); | |
1071 | }); | |
1072 | } | |
1073 | ||
9f95a23c | 1074 | } // namespace crimson::mon |