1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2015 Red Hat
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #include <string_view>
17 #include "common/debug.h"
18 #include "common/errno.h"
19 #include "common/likely.h"
20 #include "common/async/blocked_completion.h"
22 #include "messages/MClientRequestForward.h"
23 #include "messages/MMDSLoadTargets.h"
24 #include "messages/MMDSTableRequest.h"
25 #include "messages/MMDSMetrics.h"
27 #include "mgr/MgrClient.h"
29 #include "MDSDaemon.h"
31 #include "MetricAggregator.h"
32 #include "SnapClient.h"
33 #include "SnapServer.h"
34 #include "MDBalancer.h"
38 #include "mon/MonClient.h"
39 #include "common/HeartbeatMap.h"
40 #include "ScrubStack.h"
46 #define dout_context g_ceph_context
47 #define dout_subsys ceph_subsys_mds
49 #define dout_prefix *_dout << "mds." << whoami << '.' << incarnation << ' '
55 using TOPNSPC::common::cmd_getval
;
57 class C_Flush_Journal
: public MDSInternalContext
{
59 C_Flush_Journal(MDCache
*mdcache
, MDLog
*mdlog
, MDSRank
*mds
,
60 std::ostream
*ss
, Context
*on_finish
)
61 : MDSInternalContext(mds
),
62 mdcache(mdcache
), mdlog(mdlog
), ss(ss
), on_finish(on_finish
),
63 whoami(mds
->whoami
), incarnation(mds
->incarnation
) {
67 ceph_assert(ceph_mutex_is_locked(mds
->mds_lock
));
69 dout(20) << __func__
<< dendl
;
71 if (mdcache
->is_readonly()) {
72 dout(5) << __func__
<< ": read-only FS" << dendl
;
73 complete(-CEPHFS_EROFS
);
77 if (!mds
->is_active()) {
78 dout(5) << __func__
<< ": MDS not active, no-op" << dendl
;
89 dout(20) << __func__
<< dendl
;
91 // I need to seal off the current segment, and then mark all
92 // previous segments for expiry
93 mdlog
->start_new_segment();
95 Context
*ctx
= new LambdaContext([this](int r
) {
96 handle_flush_mdlog(r
);
99 // Flush initially so that all the segments older than our new one
100 // will be elegible for expiry
102 mdlog
->wait_for_safe(new MDSInternalContextWrapper(mds
, ctx
));
105 void handle_flush_mdlog(int r
) {
106 dout(20) << __func__
<< ": r=" << r
<< dendl
;
109 *ss
<< "Error " << r
<< " (" << cpp_strerror(r
) << ") while flushing journal";
118 dout(20) << __func__
<< dendl
;
120 Context
*ctx
= new LambdaContext([this](int r
) {
121 handle_clear_mdlog(r
);
124 // Because we may not be the last wait_for_safe context on MDLog,
125 // and subsequent contexts might wake up in the middle of our
126 // later trim_all and interfere with expiry (by e.g. marking
127 // dirs/dentries dirty on previous log segments), we run a second
128 // wait_for_safe here. See #10368
129 mdlog
->wait_for_safe(new MDSInternalContextWrapper(mds
, ctx
));
132 void handle_clear_mdlog(int r
) {
133 dout(20) << __func__
<< ": r=" << r
<< dendl
;
136 *ss
<< "Error " << r
<< " (" << cpp_strerror(r
) << ") while flushing journal";
145 // Put all the old log segments into expiring or expired state
146 dout(5) << __func__
<< ": beginning segment expiry" << dendl
;
148 int ret
= mdlog
->trim_all();
150 *ss
<< "Error " << ret
<< " (" << cpp_strerror(ret
) << ") while trimming log";
158 void expire_segments() {
159 dout(20) << __func__
<< dendl
;
161 // Attach contexts to wait for all expiring segments to expire
162 MDSGatherBuilder
expiry_gather(g_ceph_context
);
164 const auto &expiring_segments
= mdlog
->get_expiring_segments();
165 for (auto p
: expiring_segments
) {
166 p
->wait_for_expiry(expiry_gather
.new_sub());
168 dout(5) << __func__
<< ": waiting for " << expiry_gather
.num_subs_created()
169 << " segments to expire" << dendl
;
171 if (!expiry_gather
.has_subs()) {
176 Context
*ctx
= new LambdaContext([this](int r
) {
177 handle_expire_segments(r
);
179 expiry_gather
.set_finisher(new MDSInternalContextWrapper(mds
, ctx
));
180 expiry_gather
.activate();
183 void handle_expire_segments(int r
) {
184 dout(20) << __func__
<< ": r=" << r
<< dendl
;
186 ceph_assert(r
== 0); // MDLog is not allowed to raise errors via
191 void trim_segments() {
192 dout(20) << __func__
<< dendl
;
194 Context
*ctx
= new C_OnFinisher(new LambdaContext([this](int) {
195 std::lock_guard
locker(mds
->mds_lock
);
196 trim_expired_segments();
201 void trim_expired_segments() {
202 dout(5) << __func__
<< ": expiry complete, expire_pos/trim_pos is now "
203 << std::hex
<< mdlog
->get_journaler()->get_expire_pos() << "/"
204 << mdlog
->get_journaler()->get_trimmed_pos() << dendl
;
206 // Now everyone I'm interested in is expired
207 mdlog
->trim_expired_segments();
209 dout(5) << __func__
<< ": trim complete, expire_pos/trim_pos is now "
210 << std::hex
<< mdlog
->get_journaler()->get_expire_pos() << "/"
211 << mdlog
->get_journaler()->get_trimmed_pos() << dendl
;
213 write_journal_head();
216 void write_journal_head() {
217 dout(20) << __func__
<< dendl
;
219 Context
*ctx
= new LambdaContext([this](int r
) {
220 std::lock_guard
locker(mds
->mds_lock
);
221 handle_write_head(r
);
223 // Flush the journal header so that readers will start from after
224 // the flushed region
225 mdlog
->get_journaler()->write_head(ctx
);
228 void handle_write_head(int r
) {
230 *ss
<< "Error " << r
<< " (" << cpp_strerror(r
) << ") while writing header";
232 dout(5) << __func__
<< ": write_head complete, all done!" << dendl
;
238 void finish(int r
) override
{
239 dout(20) << __func__
<< ": r=" << r
<< dendl
;
240 on_finish
->complete(r
);
253 class C_Drop_Cache
: public MDSInternalContext
{
255 C_Drop_Cache(Server
*server
, MDCache
*mdcache
, MDLog
*mdlog
,
256 MDSRank
*mds
, uint64_t recall_timeout
,
257 Formatter
*f
, Context
*on_finish
)
258 : MDSInternalContext(mds
),
259 server(server
), mdcache(mdcache
), mdlog(mdlog
),
260 recall_timeout(recall_timeout
), recall_start(mono_clock::now()),
261 f(f
), on_finish(on_finish
),
262 whoami(mds
->whoami
), incarnation(mds
->incarnation
) {
266 // not really a hard requirement here, but lets ensure this in
267 // case we change the logic here.
268 ceph_assert(ceph_mutex_is_locked(mds
->mds_lock
));
270 dout(20) << __func__
<< dendl
;
271 f
->open_object_section("result");
272 recall_client_state();
276 // context which completes itself (with -CEPHFS_ETIMEDOUT) after a specified
277 // timeout or when explicitly completed, whichever comes first. Note
278 // that the context does not detroy itself after completion -- it
279 // needs to be explicitly freed.
280 class C_ContextTimeout
: public MDSInternalContext
{
282 C_ContextTimeout(MDSRank
*mds
, uint64_t timeout
, Context
*on_finish
)
283 : MDSInternalContext(mds
),
285 on_finish(on_finish
) {
287 ~C_ContextTimeout() {
288 ceph_assert(timer_task
== nullptr);
296 timer_task
= new LambdaContext([this](int) {
297 timer_task
= nullptr;
298 complete(-CEPHFS_ETIMEDOUT
);
300 mds
->timer
.add_event_after(timeout
, timer_task
);
303 void finish(int r
) override
{
304 Context
*ctx
= nullptr;
306 std::lock_guard
locker(lock
);
307 std::swap(on_finish
, ctx
);
309 if (ctx
!= nullptr) {
313 void complete(int r
) override
{
314 if (timer_task
!= nullptr) {
315 mds
->timer
.cancel_event(timer_task
);
322 ceph::mutex lock
= ceph::make_mutex("mds::context::timeout");
323 Context
*on_finish
= nullptr;
324 Context
*timer_task
= nullptr;
328 auto [throttled
, count
] = mdcache
->trim(UINT64_MAX
);
330 << (throttled
? " (throttled)" : "")
331 << " trimmed " << count
<< " caps" << dendl
;
332 dentries_trimmed
+= count
;
333 return std::make_pair(throttled
, count
);
336 void recall_client_state() {
337 dout(20) << __func__
<< dendl
;
338 auto now
= mono_clock::now();
339 auto duration
= std::chrono::duration
<double>(now
-recall_start
).count();
341 MDSGatherBuilder
gather(g_ceph_context
);
342 auto flags
= Server::RecallFlags::STEADY
|Server::RecallFlags::TRIM
;
343 auto [throttled
, count
] = server
->recall_client_state(&gather
, flags
);
345 << (throttled
? " (throttled)" : "")
346 << " recalled " << count
<< " caps" << dendl
;
348 caps_recalled
+= count
;
349 if ((throttled
|| count
> 0) && (recall_timeout
== 0 || duration
< recall_timeout
)) {
350 C_ContextTimeout
*ctx
= new C_ContextTimeout(
351 mds
, 1, new LambdaContext([this](int r
) {
352 recall_client_state();
355 gather
.set_finisher(new MDSInternalContextWrapper(mds
, ctx
));
357 mdlog
->flush(); /* use down-time to incrementally flush log */
358 do_trim(); /* use down-time to incrementally trim cache */
360 if (!gather
.has_subs()) {
361 return handle_recall_client_state(0);
362 } else if (recall_timeout
> 0 && duration
> recall_timeout
) {
363 gather
.set_finisher(new C_MDSInternalNoop
);
365 return handle_recall_client_state(-CEPHFS_ETIMEDOUT
);
367 uint64_t remaining
= (recall_timeout
== 0 ? 0 : recall_timeout
-duration
);
368 C_ContextTimeout
*ctx
= new C_ContextTimeout(
369 mds
, remaining
, new LambdaContext([this](int r
) {
370 handle_recall_client_state(r
);
374 gather
.set_finisher(new MDSInternalContextWrapper(mds
, ctx
));
380 void handle_recall_client_state(int r
) {
381 dout(20) << __func__
<< ": r=" << r
<< dendl
;
383 // client recall section
384 f
->open_object_section("client_recall");
385 f
->dump_int("return_code", r
);
386 f
->dump_string("message", cpp_strerror(r
));
387 f
->dump_int("recalled", caps_recalled
);
390 // we can still continue after recall timeout
394 void flush_journal() {
395 dout(20) << __func__
<< dendl
;
397 Context
*ctx
= new LambdaContext([this](int r
) {
398 handle_flush_journal(r
);
401 C_Flush_Journal
*flush_journal
= new C_Flush_Journal(mdcache
, mdlog
, mds
, &ss
, ctx
);
402 flush_journal
->send();
405 void handle_flush_journal(int r
) {
406 dout(20) << __func__
<< ": r=" << r
<< dendl
;
409 cmd_err(f
, ss
.str());
414 // journal flush section
415 f
->open_object_section("flush_journal");
416 f
->dump_int("return_code", r
);
417 f
->dump_string("message", ss
.str());
424 dout(20) << __func__
<< dendl
;
426 auto [throttled
, count
] = do_trim();
427 if (throttled
&& count
> 0) {
428 auto timer
= new LambdaContext([this](int) {
431 mds
->timer
.add_event_after(1.0, timer
);
437 void cache_status() {
438 dout(20) << __func__
<< dendl
;
440 f
->open_object_section("trim_cache");
441 f
->dump_int("trimmed", dentries_trimmed
);
444 // cache status section
445 mdcache
->cache_status(f
);
450 void finish(int r
) override
{
451 dout(20) << __func__
<< ": r=" << r
<< dendl
;
453 auto d
= std::chrono::duration
<double>(mono_clock::now()-recall_start
);
454 f
->dump_float("duration", d
.count());
457 on_finish
->complete(r
);
463 uint64_t recall_timeout
;
464 mono_time recall_start
;
469 std::stringstream ss
;
470 uint64_t caps_recalled
= 0;
471 uint64_t dentries_trimmed
= 0;
477 void cmd_err(Formatter
*f
, std::string_view err
) {
479 f
->open_object_section("result");
480 f
->dump_string("error", err
);
487 ceph::fair_mutex
&mds_lock_
,
488 LogChannelRef
&clog_
,
489 CommonSafeTimer
<ceph::fair_mutex
> &timer_
,
491 std::unique_ptr
<MDSMap
>& mdsmap_
,
495 Context
*respawn_hook_
,
496 Context
*suicide_hook_
,
497 boost::asio::io_context
& ioc
) :
498 cct(msgr
->cct
), mds_lock(mds_lock_
), clog(clog_
),
499 timer(timer_
), mdsmap(mdsmap_
),
500 objecter(new Objecter(g_ceph_context
, msgr
, monc_
, ioc
)),
501 damage_table(whoami_
), sessionmap(this),
502 op_tracker(g_ceph_context
, g_conf()->mds_enable_op_tracker
,
503 g_conf()->osd_num_op_tracker_shard
),
504 progress_thread(this), whoami(whoami_
),
505 purge_queue(g_ceph_context
, whoami_
,
506 mdsmap_
->get_metadata_pool(), objecter
,
507 new LambdaContext([this](int r
) {
508 std::lock_guard
l(mds_lock
);
509 handle_write_error(r
);
513 metrics_handler(cct
, this),
515 messenger(msgr
), monc(monc_
), mgrc(mgrc
),
516 respawn_hook(respawn_hook_
),
517 suicide_hook(suicide_hook_
),
518 inject_journal_corrupt_dentry_first(g_conf().get_val
<double>("mds_inject_journal_corrupt_dentry_first")),
519 starttime(mono_clock::now()),
522 hb
= g_ceph_context
->get_heartbeat_map()->add_worker("MDSRank", pthread_self());
524 // The metadata pool won't change in the whole life time
525 // of the fs, with this we can get rid of the mds_lock
526 // in many places too.
527 metadata_pool
= mdsmap
->get_metadata_pool();
529 purge_queue
.update_op_limit(*mdsmap
);
531 objecter
->unset_honor_pool_full();
533 finisher
= new Finisher(cct
, "MDSRank", "MR_Finisher");
535 mdcache
= new MDCache(this, purge_queue
);
536 mdlog
= new MDLog(this);
537 balancer
= new MDBalancer(this, messenger
, monc
);
539 scrubstack
= new ScrubStack(mdcache
, clog
, finisher
);
541 inotable
= new InoTable(this);
542 snapserver
= new SnapServer(this, monc
);
543 snapclient
= new SnapClient(this);
545 server
= new Server(this, &metrics_handler
);
546 locker
= new Locker(this, mdcache
);
548 _heartbeat_reset_grace
= g_conf().get_val
<uint64_t>("mds_heartbeat_reset_grace");
549 heartbeat_grace
= g_conf().get_val
<double>("mds_heartbeat_grace");
550 op_tracker
.set_complaint_and_threshold(cct
->_conf
->mds_op_complaint_time
,
551 cct
->_conf
->mds_op_log_threshold
);
552 op_tracker
.set_history_size_and_duration(cct
->_conf
->mds_op_history_size
,
553 cct
->_conf
->mds_op_history_duration
);
555 schedule_update_timer_task();
561 g_ceph_context
->get_heartbeat_map()->remove_worker(hb
);
565 if (scrubstack
) { delete scrubstack
; scrubstack
= NULL
; }
566 if (mdcache
) { delete mdcache
; mdcache
= NULL
; }
567 if (mdlog
) { delete mdlog
; mdlog
= NULL
; }
568 if (balancer
) { delete balancer
; balancer
= NULL
; }
569 if (inotable
) { delete inotable
; inotable
= NULL
; }
570 if (snapserver
) { delete snapserver
; snapserver
= NULL
; }
571 if (snapclient
) { delete snapclient
; snapclient
= NULL
; }
573 if (server
) { delete server
; server
= 0; }
574 if (locker
) { delete locker
; locker
= 0; }
577 g_ceph_context
->get_perfcounters_collection()->remove(logger
);
582 g_ceph_context
->get_perfcounters_collection()->remove(mlogger
);
600 void MDSRankDispatcher::init()
603 messenger
->add_dispatcher_head(objecter
);
610 // Expose the OSDMap (already populated during MDS::init) to anyone
611 // who is interested in it.
614 progress_thread
.create("mds_rank_progr");
621 void MDSRank::update_targets()
623 // get MonMap's idea of my export_targets
624 const set
<mds_rank_t
>& map_targets
= mdsmap
->get_mds_info(get_nodeid()).export_targets
;
626 dout(20) << "updating export targets, currently " << map_targets
.size() << " ranks are targets" << dendl
;
629 set
<mds_rank_t
> new_map_targets
;
631 auto it
= export_targets
.begin();
632 while (it
!= export_targets
.end()) {
633 mds_rank_t rank
= it
->first
;
634 auto &counter
= it
->second
;
635 dout(20) << "export target mds." << rank
<< " is " << counter
<< dendl
;
637 double val
= counter
.get();
639 dout(15) << "export target mds." << rank
<< " is no longer an export target" << dendl
;
640 export_targets
.erase(it
++);
644 if (!map_targets
.count(rank
)) {
645 dout(15) << "export target mds." << rank
<< " not in map's export_targets" << dendl
;
648 new_map_targets
.insert(rank
);
651 if (new_map_targets
.size() < map_targets
.size()) {
652 dout(15) << "export target map holds stale targets, sending update" << dendl
;
657 dout(15) << "updating export_targets, now " << new_map_targets
.size() << " ranks are targets" << dendl
;
658 auto m
= make_message
<MMDSLoadTargets
>(mds_gid_t(monc
->get_global_id()), new_map_targets
);
659 monc
->send_mon_message(m
.detach());
663 void MDSRank::hit_export_target(mds_rank_t rank
, double amount
)
665 double rate
= g_conf()->mds_bal_target_decay
;
667 amount
= 100.0/g_conf()->mds_bal_target_decay
; /* a good default for "i am trying to keep this export_target active" */
669 auto em
= export_targets
.emplace(std::piecewise_construct
, std::forward_as_tuple(rank
), std::forward_as_tuple(DecayRate(rate
)));
670 auto &counter
= em
.first
->second
;
673 dout(15) << "hit export target (new) is " << counter
<< dendl
;
675 dout(15) << "hit export target is " << counter
<< dendl
;
679 class C_MDS_MonCommand
: public MDSInternalContext
{
683 C_MDS_MonCommand(MDSRank
*m
, std::string_view c
)
684 : MDSInternalContext(m
), cmd(c
) {}
685 void finish(int r
) override
{
686 mds
->_mon_command_finish(r
, cmd
, outs
);
690 void MDSRank::_mon_command_finish(int r
, std::string_view cmd
, std::string_view outs
)
693 dout(0) << __func__
<< ": mon command " << cmd
<< " failed with errno " << r
694 << " (" << outs
<< ")" << dendl
;
696 dout(1) << __func__
<< ": mon command " << cmd
<< " succeed" << dendl
;
700 void MDSRank::set_mdsmap_multimds_snaps_allowed()
702 static bool already_sent
= false;
706 CachedStackStringStream css
;
707 *css
<< "{\"prefix\":\"fs set\", \"fs_name\":\"" << mdsmap
->get_fs_name() << "\", ";
708 *css
<< "\"var\":\"allow_multimds_snaps\", \"val\":\"true\", ";
709 *css
<< "\"confirm\":\"--yes-i-am-really-a-mds\"}";
710 std::vector
<std::string
> cmd
= {css
->str()};
712 dout(0) << __func__
<< ": sending mon command: " << cmd
[0] << dendl
;
714 C_MDS_MonCommand
*fin
= new C_MDS_MonCommand(this, cmd
[0]);
715 monc
->start_mon_command(cmd
, {}, nullptr, &fin
->outs
, new C_IO_Wrapper(this, fin
));
720 void MDSRankDispatcher::tick()
724 if (beacon
.is_laggy()) {
725 dout(1) << "skipping upkeep work because connection to Monitors appears laggy" << dendl
;
729 check_ops_in_flight();
731 // Wake up thread in case we use to be laggy and have waiting_for_nolaggy
732 // messages to progress.
733 progress_thread
.signal();
735 // make sure mds log flushes, trims periodically
738 // update average session uptime
739 sessionmap
.update_average_session_age();
741 if (is_active() || is_stopping()) {
742 mdlog
->trim(); // NOT during recovery!
746 if (is_clientreplay() || is_active() || is_stopping()) {
747 server
->find_idle_sessions();
748 server
->evict_cap_revoke_non_responders();
754 logger
->set(l_mds_subtrees
, mdcache
->num_subtrees());
759 server
->reconnect_tick();
763 mdcache
->find_stale_fragment_freeze();
764 mdcache
->migrator
->find_stale_export_freeze();
766 if (mdsmap
->get_tableserver() == whoami
) {
767 snapserver
->check_osd_map(false);
768 // Filesystem was created by pre-mimic mds. Allow multi-active mds after
769 // all old snapshots are deleted.
770 if (!mdsmap
->allows_multimds_snaps() &&
771 snapserver
->can_allow_multimds_snaps()) {
772 set_mdsmap_multimds_snaps_allowed();
777 scrubstack
->advance_scrub_status();
780 if (is_active() || is_stopping()) {
787 if (mdcache
->shutdown_pass()) {
788 uint64_t pq_progress
= 0 ;
789 uint64_t pq_total
= 0;
790 size_t pq_in_flight
= 0;
791 if (!purge_queue
.drain(&pq_progress
, &pq_total
, &pq_in_flight
)) {
792 dout(7) << "shutdown_pass=true, but still waiting for purge queue"
794 // This takes unbounded time, so we must indicate progress
795 // to the administrator: we do it in a slightly imperfect way
796 // by sending periodic (tick frequency) clog messages while
798 clog
->info() << "MDS rank " << whoami
<< " waiting for purge queue ("
799 << std::dec
<< pq_progress
<< "/" << pq_total
<< " " << pq_in_flight
800 << " files purging" << ")";
802 dout(7) << "shutdown_pass=true, finished w/ shutdown, moving to "
803 "down:stopped" << dendl
;
808 dout(7) << "shutdown_pass=false" << dendl
;
812 // Expose ourselves to Beacon to update health indicators
813 beacon
.notify_health(this);
816 void MDSRankDispatcher::shutdown()
818 // It should never be possible for shutdown to get called twice, because
819 // anyone picking up mds_lock checks if stopping is true and drops
821 ceph_assert(stopping
== false);
824 dout(1) << __func__
<< ": shutting down rank " << whoami
<< dendl
;
826 g_conf().remove_observer(this);
830 // MDLog has to shut down before the finisher, because some of its
831 // threads block on IOs that require finisher to complete.
837 purge_queue
.shutdown();
839 // shutdown metrics handler/updater -- this is ok even if it was not
841 metrics_handler
.shutdown();
843 // shutdown metric aggergator
844 if (metric_aggregator
!= nullptr) {
845 metric_aggregator
->shutdown();
849 finisher
->stop(); // no flushing
852 if (objecter
->initialized
)
853 objecter
->shutdown();
857 op_tracker
.on_shutdown();
859 progress_thread
.shutdown();
861 // release mds_lock for finisher/messenger threads (e.g.
862 // MDSDaemon::ms_handle_reset called from Messenger).
865 // shut down messenger
866 messenger
->shutdown();
870 // Workaround unclean shutdown: HeartbeatMap will assert if
871 // worker is not removed (as we do in ~MDS), but ~MDS is not
872 // always called after suicide.
874 g_ceph_context
->get_heartbeat_map()->remove_worker(hb
);
880 * Helper for simple callbacks that call a void fn with no args.
882 class C_MDS_VoidFn
: public MDSInternalContext
884 typedef void (MDSRank::*fn_ptr
)();
888 C_MDS_VoidFn(MDSRank
*mds_
, fn_ptr fn_
)
889 : MDSInternalContext(mds_
), fn(fn_
)
895 void finish(int r
) override
901 MDSTableClient
*MDSRank::get_table_client(int t
)
904 case TABLE_ANCHOR
: return NULL
;
905 case TABLE_SNAP
: return snapclient
;
906 default: ceph_abort();
910 MDSTableServer
*MDSRank::get_table_server(int t
)
913 case TABLE_ANCHOR
: return NULL
;
914 case TABLE_SNAP
: return snapserver
;
915 default: ceph_abort();
919 void MDSRank::suicide()
922 suicide_hook
->complete(0);
927 void MDSRank::respawn()
930 respawn_hook
->complete(0);
935 void MDSRank::abort(std::string_view msg
)
941 void MDSRank::damaged()
943 ceph_assert(whoami
!= MDS_RANK_NONE
);
944 ceph_assert(ceph_mutex_is_locked_by_me(mds_lock
));
946 beacon
.set_want_state(*mdsmap
, MDSMap::STATE_DAMAGED
);
947 monc
->flush_log(); // Flush any clog error from before we were called
948 beacon
.notify_health(this); // Include latest status in our swan song
949 beacon
.send_and_wait(g_conf()->mds_mon_shutdown_timeout
);
951 // It's okay if we timed out and the mon didn't get our beacon, because
952 // another daemon (or ourselves after respawn) will eventually take the
953 // rank and report DAMAGED again when it hits same problem we did.
955 respawn(); // Respawn into standby in case mon has other work for us
958 void MDSRank::damaged_unlocked()
960 std::lock_guard
l(mds_lock
);
964 void MDSRank::handle_write_error(int err
)
966 if (err
== -CEPHFS_EBLOCKLISTED
) {
967 derr
<< "we have been blocklisted (fenced), respawning..." << dendl
;
972 if (g_conf()->mds_action_on_write_error
>= 2) {
973 derr
<< "unhandled write error " << cpp_strerror(err
) << ", suicide..." << dendl
;
975 } else if (g_conf()->mds_action_on_write_error
== 1) {
976 derr
<< "unhandled write error " << cpp_strerror(err
) << ", force readonly..." << dendl
;
977 mdcache
->force_readonly();
980 derr
<< "unhandled write error " << cpp_strerror(err
) << ", ignore..." << dendl
;
984 void MDSRank::handle_write_error_with_lock(int err
)
986 std::scoped_lock
l(mds_lock
);
987 handle_write_error(err
);
990 void *MDSRank::ProgressThread::entry()
992 std::unique_lock
l(mds
->mds_lock
);
994 cond
.wait(l
, [this] {
995 return (mds
->stopping
||
996 !mds
->finished_queue
.empty() ||
997 (!mds
->waiting_for_nolaggy
.empty() && !mds
->beacon
.is_laggy()));
1000 if (mds
->stopping
) {
1004 mds
->_advance_queues();
1011 void MDSRank::ProgressThread::shutdown()
1013 ceph_assert(ceph_mutex_is_locked_by_me(mds
->mds_lock
));
1014 ceph_assert(mds
->stopping
);
1017 // Stopping is set, we will fall out of our main loop naturally
1019 // Kick the thread to notice mds->stopping, and join it
1021 mds
->mds_lock
.unlock();
1024 mds
->mds_lock
.lock();
1028 bool MDSRankDispatcher::ms_dispatch(const cref_t
<Message
> &m
)
1030 if (m
->get_source().is_mds()) {
1031 const Message
*msg
= m
.get();
1032 const MMDSOp
*op
= dynamic_cast<const MMDSOp
*>(msg
);
1034 dout(0) << typeid(*msg
).name() << " is not an MMDSOp type" << dendl
;
1037 else if (m
->get_source().is_client()) {
1038 Session
*session
= static_cast<Session
*>(m
->get_connection()->get_priv().get());
1040 session
->last_seen
= Session::clock::now();
1043 inc_dispatch_depth();
1044 bool ret
= _dispatch(m
, true);
1045 dec_dispatch_depth();
1049 bool MDSRank::_dispatch(const cref_t
<Message
> &m
, bool new_msg
)
1051 if (is_stale_message(m
)) {
1054 // do not proceed if this message cannot be handled
1055 if (!is_valid_message(m
)) {
1059 if (beacon
.is_laggy()) {
1060 dout(5) << " laggy, deferring " << *m
<< dendl
;
1061 waiting_for_nolaggy
.push_back(m
);
1062 } else if (new_msg
&& !waiting_for_nolaggy
.empty()) {
1063 dout(5) << " there are deferred messages, deferring " << *m
<< dendl
;
1064 waiting_for_nolaggy
.push_back(m
);
1070 if (dispatch_depth
> 1)
1073 // finish any triggered contexts
1076 if (beacon
.is_laggy()) {
1077 // We've gone laggy during dispatch, don't do any
1078 // more housekeeping
1082 // hack: thrash exports
1083 static utime_t start
;
1084 utime_t now
= ceph_clock_now();
1085 if (start
== utime_t())
1087 /*double el = now - start;
1090 for (int i
=0; i
<g_conf()->mds_thrash_exports
; i
++) {
1092 if (!is_active()) break;
1093 mdsmap
->get_mds_set(s
, MDSMap::STATE_ACTIVE
);
1094 if (s
.size() < 2 || CInode::count() < 10)
1095 break; // need peers for this to work.
1096 if (mdcache
->migrator
->get_num_exporting() > g_conf()->mds_thrash_exports
* 5 ||
1097 mdcache
->migrator
->get_export_queue_size() > g_conf()->mds_thrash_exports
* 10)
1100 dout(7) << "mds thrashing exports pass " << (i
+1) << "/" << g_conf()->mds_thrash_exports
<< dendl
;
1102 // pick a random dir inode
1103 CInode
*in
= mdcache
->hack_pick_random_inode();
1105 auto&& ls
= in
->get_dirfrags();
1106 if (!ls
.empty()) { // must be an open dir.
1107 const auto& dir
= ls
[rand() % ls
.size()];
1108 if (!dir
->get_parent_dir()) continue; // must be linked.
1109 if (!dir
->is_auth()) continue; // must be auth.
1113 int k
= rand() % s
.size();
1114 set
<mds_rank_t
>::iterator p
= s
.begin();
1117 } while (dest
== whoami
);
1118 mdcache
->migrator
->export_dir_nicely(dir
,dest
);
1121 // hack: thrash fragments
1122 for (int i
=0; i
<g_conf()->mds_thrash_fragments
; i
++) {
1123 if (!is_active()) break;
1124 if (mdcache
->get_num_fragmenting_dirs() > 5 * g_conf()->mds_thrash_fragments
) break;
1125 dout(7) << "mds thrashing fragments pass " << (i
+1) << "/" << g_conf()->mds_thrash_fragments
<< dendl
;
1127 // pick a random dir inode
1128 CInode
*in
= mdcache
->hack_pick_random_inode();
1130 auto&& ls
= in
->get_dirfrags();
1131 if (ls
.empty()) continue; // must be an open dir.
1132 CDir
*dir
= ls
.front();
1133 if (!dir
->get_parent_dir()) continue; // must be linked.
1134 if (!dir
->is_auth()) continue; // must be auth.
1135 frag_t fg
= dir
->get_frag();
1136 if ((fg
== frag_t() || (rand() % (1 << fg
.bits()) == 0))) {
1137 mdcache
->split_dir(dir
, 1);
1139 balancer
->queue_merge(dir
);
1143 // hack: force hash root?
1146 mdcache->get_root() &&
1147 mdcache->get_root()->dir &&
1148 !(mdcache->get_root()->dir->is_hashed() ||
1149 mdcache->get_root()->dir->is_hashing())) {
1150 dout(0) << "hashing root" << dendl;
1151 mdcache->migrator->hash_dir(mdcache->get_root()->dir);
1159 void MDSRank::update_mlogger()
1162 mlogger
->set(l_mdm_ino
, CInode::count());
1163 mlogger
->set(l_mdm_dir
, CDir::count());
1164 mlogger
->set(l_mdm_dn
, CDentry::count());
1165 mlogger
->set(l_mdm_cap
, Capability::count());
1166 mlogger
->set(l_mdm_inoa
, CInode::increments());
1167 mlogger
->set(l_mdm_inos
, CInode::decrements());
1168 mlogger
->set(l_mdm_dira
, CDir::increments());
1169 mlogger
->set(l_mdm_dirs
, CDir::decrements());
1170 mlogger
->set(l_mdm_dna
, CDentry::increments());
1171 mlogger
->set(l_mdm_dns
, CDentry::decrements());
1172 mlogger
->set(l_mdm_capa
, Capability::increments());
1173 mlogger
->set(l_mdm_caps
, Capability::decrements());
1177 // message types that the mds can handle
1178 bool MDSRank::is_valid_message(const cref_t
<Message
> &m
) {
1179 int port
= m
->get_type() & 0xff00;
1180 int type
= m
->get_type();
1182 if (port
== MDS_PORT_CACHE
||
1183 port
== MDS_PORT_MIGRATOR
||
1184 type
== CEPH_MSG_CLIENT_SESSION
||
1185 type
== CEPH_MSG_CLIENT_RECONNECT
||
1186 type
== CEPH_MSG_CLIENT_RECLAIM
||
1187 type
== CEPH_MSG_CLIENT_REQUEST
||
1188 type
== MSG_MDS_PEER_REQUEST
||
1189 type
== MSG_MDS_HEARTBEAT
||
1190 type
== MSG_MDS_TABLE_REQUEST
||
1191 type
== MSG_MDS_LOCK
||
1192 type
== MSG_MDS_INODEFILECAPS
||
1193 type
== MSG_MDS_SCRUB
||
1194 type
== MSG_MDS_SCRUB_STATS
||
1195 type
== CEPH_MSG_CLIENT_CAPS
||
1196 type
== CEPH_MSG_CLIENT_CAPRELEASE
||
1197 type
== CEPH_MSG_CLIENT_LEASE
) {
1205 * lower priority messages we defer if we seem laggy
1208 #define ALLOW_MESSAGES_FROM(peers) \
1210 if (m->get_connection() && (m->get_connection()->get_peer_type() & (peers)) == 0) { \
1211 dout(0) << __FILE__ << "." << __LINE__ << ": filtered out request, peer=" << m->get_connection()->get_peer_type() \
1212 << " allowing=" << #peers << " message=" << *m << dendl; \
1217 void MDSRank::handle_message(const cref_t
<Message
> &m
)
1219 int port
= m
->get_type() & 0xff00;
1222 case MDS_PORT_CACHE
:
1223 ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS
);
1224 mdcache
->dispatch(m
);
1227 case MDS_PORT_MIGRATOR
:
1228 ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS
);
1229 mdcache
->migrator
->dispatch(m
);
1233 switch (m
->get_type()) {
1235 case CEPH_MSG_CLIENT_SESSION
:
1236 case CEPH_MSG_CLIENT_RECONNECT
:
1237 case CEPH_MSG_CLIENT_RECLAIM
:
1238 ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_CLIENT
);
1240 case CEPH_MSG_CLIENT_REQUEST
:
1241 server
->dispatch(m
);
1243 case MSG_MDS_PEER_REQUEST
:
1244 ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS
);
1245 server
->dispatch(m
);
1248 case MSG_MDS_HEARTBEAT
:
1249 ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS
);
1250 balancer
->proc_message(m
);
1253 case MSG_MDS_TABLE_REQUEST
:
1254 ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS
);
1256 const cref_t
<MMDSTableRequest
> &req
= ref_cast
<MMDSTableRequest
>(m
);
1258 MDSTableClient
*client
= get_table_client(req
->table
);
1259 client
->handle_request(req
);
1261 MDSTableServer
*server
= get_table_server(req
->table
);
1262 server
->handle_request(req
);
1268 case MSG_MDS_INODEFILECAPS
:
1269 ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS
);
1270 locker
->dispatch(m
);
1273 case CEPH_MSG_CLIENT_CAPS
:
1274 case CEPH_MSG_CLIENT_CAPRELEASE
:
1275 case CEPH_MSG_CLIENT_LEASE
:
1276 ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_CLIENT
);
1277 locker
->dispatch(m
);
1281 case MSG_MDS_SCRUB_STATS
:
1282 ALLOW_MESSAGES_FROM(CEPH_ENTITY_TYPE_MDS
);
1283 scrubstack
->dispatch(m
);
1287 derr
<< "unrecognized message " << *m
<< dendl
;
1293 * Advance finished_queue and waiting_for_nolaggy.
1295 * Usually drain both queues, but may not drain waiting_for_nolaggy
1296 * if beacon is currently laggy.
1298 void MDSRank::_advance_queues()
1300 ceph_assert(ceph_mutex_is_locked_by_me(mds_lock
));
1302 if (!finished_queue
.empty()) {
1303 dout(7) << "mds has " << finished_queue
.size() << " queued contexts" << dendl
;
1304 while (!finished_queue
.empty()) {
1305 auto fin
= finished_queue
.front();
1306 finished_queue
.pop_front();
1308 dout(10) << " finish " << fin
<< dendl
;
1315 while (!waiting_for_nolaggy
.empty()) {
1316 // stop if we're laggy now!
1317 if (beacon
.is_laggy())
1320 cref_t
<Message
> old
= waiting_for_nolaggy
.front();
1321 waiting_for_nolaggy
.pop_front();
1323 if (!is_stale_message(old
)) {
1324 dout(7) << " processing laggy deferred " << *old
<< dendl
;
1325 ceph_assert(is_valid_message(old
));
1326 handle_message(old
);
1334 * Call this when you take mds_lock, or periodically if you're going to
1335 * hold the lock for a long time (e.g. iterating over clients/inodes)
1337 void MDSRank::heartbeat_reset()
1339 // Any thread might jump into mds_lock and call us immediately
1340 // after a call to suicide() completes, in which case MDSRank::hb
1341 // has been freed and we are a no-op.
1343 ceph_assert(stopping
);
1347 // NB not enabling suicide grace, because the mon takes care of killing us
1348 // (by blocklisting us) when we fail to send beacons, and it's simpler to
1349 // only have one way of dying.
1350 g_ceph_context
->get_heartbeat_map()->reset_timeout(hb
,
1351 ceph::make_timespan(heartbeat_grace
),
1352 ceph::timespan::zero());
1355 bool MDSRank::is_stale_message(const cref_t
<Message
> &m
) const
1358 if (m
->get_source().is_mds()) {
1359 mds_rank_t from
= mds_rank_t(m
->get_source().num());
1361 if (mdsmap
->is_down(from
)) {
1364 // FIXME: this is a convoluted check. we should be maintaining a nice
1365 // clean map of current ConnectionRefs for current mdses!!!
1366 auto c
= messenger
->connect_to(CEPH_ENTITY_TYPE_MDS
,
1367 mdsmap
->get_addrs(from
));
1368 if (c
!= m
->get_connection()) {
1370 dout(5) << " mds." << from
<< " should be " << c
<< " "
1371 << c
->get_peer_addrs() << " but this message is "
1372 << m
->get_connection() << " " << m
->get_source_addrs()
1378 if (m
->get_type() == CEPH_MSG_MDS_MAP
) {
1379 dout(5) << "got " << *m
<< " from old/bad/imposter mds " << m
->get_source()
1380 << ", but it's an mdsmap, looking at it" << dendl
;
1381 } else if (m
->get_type() == MSG_MDS_CACHEEXPIRE
&&
1382 mdsmap
->get_addrs(from
) == m
->get_source_addrs()) {
1383 dout(5) << "got " << *m
<< " from down mds " << m
->get_source()
1384 << ", but it's a cache_expire, looking at it" << dendl
;
1386 dout(5) << "got " << *m
<< " from down/old/bad/imposter mds " << m
->get_source()
1387 << ", dropping" << dendl
;
1395 Session
*MDSRank::get_session(const cref_t
<Message
> &m
)
1398 auto session
= static_cast<Session
*>(m
->get_connection()->get_priv().get());
1400 dout(20) << "get_session have " << session
<< " " << session
->info
.inst
1401 << " state " << session
->get_state_name() << dendl
;
1402 // Check if we've imported an open session since (new sessions start closed)
1403 if (session
->is_closed() && m
->get_type() == CEPH_MSG_CLIENT_SESSION
) {
1404 Session
*imported_session
= sessionmap
.get_session(session
->info
.inst
.name
);
1405 if (imported_session
&& imported_session
!= session
) {
1406 dout(10) << __func__
<< " replacing connection bootstrap session "
1407 << session
<< " with imported session " << imported_session
1409 imported_session
->info
.auth_name
= session
->info
.auth_name
;
1410 //assert(session->info.auth_name == imported_session->info.auth_name);
1411 ceph_assert(session
->info
.inst
== imported_session
->info
.inst
);
1412 imported_session
->set_connection(session
->get_connection().get());
1413 // send out any queued messages
1414 while (!session
->preopen_out_queue
.empty()) {
1415 imported_session
->get_connection()->send_message2(std::move(session
->preopen_out_queue
.front()));
1416 session
->preopen_out_queue
.pop_front();
1418 imported_session
->auth_caps
= session
->auth_caps
;
1419 imported_session
->last_seen
= session
->last_seen
;
1420 ceph_assert(session
->get_nref() == 1);
1421 imported_session
->get_connection()->set_priv(imported_session
->get());
1422 session
= imported_session
;
1426 dout(20) << "get_session dne for " << m
->get_source_inst() << dendl
;
1431 void MDSRank::send_message(const ref_t
<Message
>& m
, const ConnectionRef
& c
)
1434 c
->send_message2(m
);
1437 class C_MDS_RetrySendMessageMDS
: public MDSInternalContext
{
1439 C_MDS_RetrySendMessageMDS(MDSRank
* mds
, mds_rank_t who
, ref_t
<Message
> m
)
1440 : MDSInternalContext(mds
), who(who
), m(std::move(m
)) {}
1441 void finish(int r
) override
{
1442 mds
->send_message_mds(m
, who
);
1450 void MDSRank::send_message_mds(const ref_t
<Message
>& m
, mds_rank_t mds
)
1452 if (!mdsmap
->is_up(mds
)) {
1453 dout(10) << "send_message_mds mds." << mds
<< " not up, dropping " << *m
<< dendl
;
1455 } else if (mdsmap
->is_bootstrapping(mds
)) {
1456 dout(5) << __func__
<< "mds." << mds
<< " is bootstrapping, deferring " << *m
<< dendl
;
1457 wait_for_bootstrapped_peer(mds
, new C_MDS_RetrySendMessageMDS(this, mds
, m
));
1461 // send mdsmap first?
1462 auto addrs
= mdsmap
->get_addrs(mds
);
1463 if (mds
!= whoami
&& peer_mdsmap_epoch
[mds
] < mdsmap
->get_epoch()) {
1464 auto _m
= make_message
<MMDSMap
>(monc
->get_fsid(), *mdsmap
);
1465 send_message_mds(_m
, addrs
);
1466 peer_mdsmap_epoch
[mds
] = mdsmap
->get_epoch();
1470 send_message_mds(m
, addrs
);
1473 void MDSRank::send_message_mds(const ref_t
<Message
>& m
, const entity_addrvec_t
&addr
)
1475 messenger
->send_to_mds(ref_t
<Message
>(m
).detach(), addr
);
1478 void MDSRank::forward_message_mds(MDRequestRef
& mdr
, mds_rank_t mds
)
1480 ceph_assert(mds
!= whoami
);
1482 auto m
= mdr
->release_client_request();
1485 * don't actually forward if non-idempotent!
1486 * client has to do it. although the MDS will ignore duplicate requests,
1487 * the affected metadata may migrate, in which case the new authority
1488 * won't have the metareq_id in the completed request map.
1490 // NEW: always make the client resend!
1491 bool client_must_resend
= true; //!creq->can_forward();
1493 // tell the client where it should go
1494 auto session
= get_session(m
);
1496 dout(1) << "no session found, failed to forward client request " << mdr
<< dendl
;
1499 auto f
= make_message
<MClientRequestForward
>(m
->get_tid(), mds
, m
->get_num_fwd()+1, client_must_resend
);
1500 send_message_client(f
, session
);
1503 void MDSRank::send_message_client_counted(const ref_t
<Message
>& m
, client_t client
)
1505 Session
*session
= sessionmap
.get_session(entity_name_t::CLIENT(client
.v
));
1507 send_message_client_counted(m
, session
);
1509 dout(10) << "send_message_client_counted no session for client." << client
<< " " << *m
<< dendl
;
1513 void MDSRank::send_message_client_counted(const ref_t
<Message
>& m
, const ConnectionRef
& connection
)
1516 auto session
= static_cast<Session
*>(connection
->get_priv().get());
1518 send_message_client_counted(m
, session
);
1520 dout(10) << "send_message_client_counted has no session for " << m
->get_source_inst() << dendl
;
1521 // another Connection took over the Session
1525 void MDSRank::send_message_client_counted(const ref_t
<Message
>& m
, Session
* session
)
1527 version_t seq
= session
->inc_push_seq();
1528 dout(10) << "send_message_client_counted " << session
->info
.inst
.name
<< " seq "
1529 << seq
<< " " << *m
<< dendl
;
1530 if (session
->get_connection()) {
1531 session
->get_connection()->send_message2(m
);
1533 session
->preopen_out_queue
.push_back(m
);
1537 void MDSRank::send_message_client(const ref_t
<Message
>& m
, Session
* session
)
1539 dout(10) << "send_message_client " << session
->info
.inst
<< " " << *m
<< dendl
;
1540 if (session
->get_connection()) {
1541 session
->get_connection()->send_message2(m
);
1543 session
->preopen_out_queue
.push_back(m
);
1548 * This is used whenever a RADOS operation has been cancelled
1549 * or a RADOS client has been blocklisted, to cause the MDS and
1550 * any clients to wait for this OSD epoch before using any new caps.
1552 * See doc/cephfs/eviction
1554 void MDSRank::set_osd_epoch_barrier(epoch_t e
)
1556 dout(4) << __func__
<< ": epoch=" << e
<< dendl
;
1557 osd_epoch_barrier
= e
;
1560 void MDSRank::retry_dispatch(const cref_t
<Message
> &m
)
1562 inc_dispatch_depth();
1563 _dispatch(m
, false);
1564 dec_dispatch_depth();
1567 double MDSRank::get_dispatch_queue_max_age(utime_t now
) const
1569 return messenger
->get_dispatch_queue_max_age(now
);
1572 bool MDSRank::is_daemon_stopping() const
1577 void MDSRank::request_state(MDSMap::DaemonState s
)
1579 dout(3) << "request_state " << ceph_mds_state_name(s
) << dendl
;
1580 beacon
.set_want_state(*mdsmap
, s
);
1585 class C_MDS_BootStart
: public MDSInternalContext
{
1586 MDSRank::BootStep nextstep
;
1588 C_MDS_BootStart(MDSRank
*m
, MDSRank::BootStep n
)
1589 : MDSInternalContext(m
), nextstep(n
) {}
1590 void finish(int r
) override
{
1591 mds
->boot_start(nextstep
, r
);
1596 void MDSRank::boot_start(BootStep step
, int r
)
1598 // Handle errors from previous step
1600 if (is_standby_replay() && (r
== -CEPHFS_EAGAIN
)) {
1601 dout(0) << "boot_start encountered an error CEPHFS_EAGAIN"
1602 << ", respawning since we fell behind journal" << dendl
;
1604 } else if (r
== -CEPHFS_EINVAL
|| r
== -CEPHFS_ENOENT
) {
1605 // Invalid or absent data, indicates damaged on-disk structures
1606 clog
->error() << "Error loading MDS rank " << whoami
<< ": "
1609 ceph_assert(r
== 0); // Unreachable, damaged() calls respawn()
1610 } else if (r
== -CEPHFS_EROFS
) {
1611 dout(0) << "boot error forcing transition to read-only; MDS will try to continue" << dendl
;
1613 // Completely unexpected error, give up and die
1614 dout(0) << "boot_start encountered an error, failing" << dendl
;
1620 ceph_assert(is_starting() || is_any_replay());
1623 case MDS_BOOT_INITIAL
:
1625 mdcache
->init_layouts();
1627 MDSGatherBuilder
gather(g_ceph_context
,
1628 new C_MDS_BootStart(this, MDS_BOOT_OPEN_ROOT
));
1629 dout(2) << "Booting: " << step
<< ": opening inotable" << dendl
;
1630 inotable
->set_rank(whoami
);
1631 inotable
->load(gather
.new_sub());
1633 dout(2) << "Booting: " << step
<< ": opening sessionmap" << dendl
;
1634 sessionmap
.set_rank(whoami
);
1635 sessionmap
.load(gather
.new_sub());
1637 dout(2) << "Booting: " << step
<< ": opening mds log" << dendl
;
1638 mdlog
->open(gather
.new_sub());
1640 if (is_starting()) {
1641 dout(2) << "Booting: " << step
<< ": opening purge queue" << dendl
;
1642 purge_queue
.open(new C_IO_Wrapper(this, gather
.new_sub()));
1643 } else if (!standby_replaying
) {
1644 dout(2) << "Booting: " << step
<< ": opening purge queue (async)" << dendl
;
1645 purge_queue
.open(NULL
);
1646 dout(2) << "Booting: " << step
<< ": loading open file table (async)" << dendl
;
1647 mdcache
->open_file_table
.load(nullptr);
1650 if (mdsmap
->get_tableserver() == whoami
) {
1651 dout(2) << "Booting: " << step
<< ": opening snap table" << dendl
;
1652 snapserver
->set_rank(whoami
);
1653 snapserver
->load(gather
.new_sub());
1659 case MDS_BOOT_OPEN_ROOT
:
1661 dout(2) << "Booting: " << step
<< ": loading/discovering base inodes" << dendl
;
1663 MDSGatherBuilder
gather(g_ceph_context
,
1664 new C_MDS_BootStart(this, MDS_BOOT_PREPARE_LOG
));
1666 if (is_starting()) {
1667 // load mydir frag for the first log segment (creating subtree map)
1668 mdcache
->open_mydir_frag(gather
.new_sub());
1670 mdcache
->open_mydir_inode(gather
.new_sub());
1673 mdcache
->create_global_snaprealm();
1675 if (whoami
== mdsmap
->get_root()) { // load root inode off disk if we are auth
1676 mdcache
->open_root_inode(gather
.new_sub());
1677 } else if (is_any_replay()) {
1678 // replay. make up fake root inode to start with
1679 mdcache
->create_root_inode();
1684 case MDS_BOOT_PREPARE_LOG
:
1685 if (is_any_replay()) {
1686 dout(2) << "Booting: " << step
<< ": replaying mds log" << dendl
;
1687 MDSGatherBuilder
gather(g_ceph_context
,
1688 new C_MDS_BootStart(this, MDS_BOOT_REPLAY_DONE
));
1690 if (!standby_replaying
) {
1691 dout(2) << "Booting: " << step
<< ": waiting for purge queue recovered" << dendl
;
1692 purge_queue
.wait_for_recovery(new C_IO_Wrapper(this, gather
.new_sub()));
1695 mdlog
->replay(gather
.new_sub());
1698 dout(2) << "Booting: " << step
<< ": positioning at end of old mds log" << dendl
;
1703 case MDS_BOOT_REPLAY_DONE
:
1704 ceph_assert(is_any_replay());
1706 // Sessiontable and inotable should be in sync after replay, validate
1707 // that they are consistent.
1708 validate_sessions();
1715 void MDSRank::validate_sessions()
1717 ceph_assert(ceph_mutex_is_locked_by_me(mds_lock
));
1720 // Identify any sessions which have state inconsistent with other,
1721 // after they have been loaded from rados during startup.
1722 // Mitigate bugs like: http://tracker.ceph.com/issues/16842
1723 for (const auto &i
: sessionmap
.get_sessions()) {
1724 Session
*session
= i
.second
;
1725 ceph_assert(session
->info
.prealloc_inos
== session
->free_prealloc_inos
);
1727 interval_set
<inodeno_t
> badones
;
1728 if (inotable
->intersects_free(session
->info
.prealloc_inos
, &badones
)) {
1729 clog
->error() << "client " << *session
1730 << "loaded with preallocated inodes that are inconsistent with inotable";
1741 void MDSRank::starting_done()
1743 dout(3) << "starting_done" << dendl
;
1744 ceph_assert(is_starting());
1745 request_state(MDSMap::STATE_ACTIVE
);
1747 mdlog
->start_new_segment();
1749 // sync snaptable cache
1750 snapclient
->sync(new C_MDSInternalNoop
);
1754 void MDSRank::calc_recovery_set()
1756 // initialize gather sets
1758 mdsmap
->get_recovery_mds_set(rs
);
1760 mdcache
->set_recovery_set(rs
);
1762 dout(1) << " recovery set is " << rs
<< dendl
;
1765 void MDSRank::replay_start()
1767 dout(1) << "replay_start" << dendl
;
1769 if (is_standby_replay()) {
1770 standby_replaying
= true;
1771 if (unlikely(g_conf().get_val
<bool>("mds_standby_replay_damaged"))) {
1776 // Check if we need to wait for a newer OSD map before starting
1777 bool const ready
= objecter
->with_osdmap(
1778 [this](const OSDMap
& o
) {
1779 return o
.get_epoch() >= mdsmap
->get_last_failure_osd_epoch();
1785 dout(1) << " waiting for osdmap " << mdsmap
->get_last_failure_osd_epoch()
1786 << " (which blocklists prior instance)" << dendl
;
1787 Context
*fin
= new C_IO_Wrapper(this, new C_MDS_BootStart(this, MDS_BOOT_INITIAL
));
1788 objecter
->wait_for_map(
1789 mdsmap
->get_last_failure_osd_epoch(),
1795 class MDSRank::C_MDS_StandbyReplayRestartFinish
: public MDSIOContext
{
1796 uint64_t old_read_pos
;
1798 C_MDS_StandbyReplayRestartFinish(MDSRank
*mds_
, uint64_t old_read_pos_
) :
1799 MDSIOContext(mds_
), old_read_pos(old_read_pos_
) {}
1800 void finish(int r
) override
{
1801 mds
->_standby_replay_restart_finish(r
, old_read_pos
);
1803 void print(ostream
& out
) const override
{
1804 out
<< "standby_replay_restart";
1808 void MDSRank::_standby_replay_restart_finish(int r
, uint64_t old_read_pos
)
1810 if (old_read_pos
< mdlog
->get_journaler()->get_trimmed_pos()) {
1811 dout(0) << "standby MDS fell behind active MDS journal's expire_pos, restarting" << dendl
;
1812 respawn(); /* we're too far back, and this is easier than
1813 trying to reset everything in the cache, etc */
1815 mdlog
->standby_trim_segments();
1816 boot_start(MDS_BOOT_PREPARE_LOG
, r
);
1820 class MDSRank::C_MDS_StandbyReplayRestart
: public MDSInternalContext
{
1822 explicit C_MDS_StandbyReplayRestart(MDSRank
*m
) : MDSInternalContext(m
) {}
1823 void finish(int r
) override
{
1825 mds
->standby_replay_restart();
1829 void MDSRank::standby_replay_restart()
1831 if (standby_replaying
) {
1832 /* Go around for another pass of replaying in standby */
1833 dout(5) << "Restarting replay as standby-replay" << dendl
;
1834 mdlog
->get_journaler()->reread_head_and_probe(
1835 new C_MDS_StandbyReplayRestartFinish(
1837 mdlog
->get_journaler()->get_read_pos()));
1839 /* We are transitioning out of standby: wait for OSD map update
1840 before making final pass */
1841 dout(1) << "standby_replay_restart (final takeover pass)" << dendl
;
1842 bool ready
= objecter
->with_osdmap(
1843 [this](const OSDMap
& o
) {
1844 return o
.get_epoch() >= mdsmap
->get_last_failure_osd_epoch();
1847 mdlog
->get_journaler()->reread_head_and_probe(
1848 new C_MDS_StandbyReplayRestartFinish(
1850 mdlog
->get_journaler()->get_read_pos()));
1852 dout(1) << " opening purge_queue (async)" << dendl
;
1853 purge_queue
.open(NULL
);
1854 dout(1) << " opening open_file_table (async)" << dendl
;
1855 mdcache
->open_file_table
.load(nullptr);
1857 auto fin
= new C_IO_Wrapper(this, new C_MDS_StandbyReplayRestart(this));
1858 dout(1) << " waiting for osdmap " << mdsmap
->get_last_failure_osd_epoch()
1859 << " (which blocklists prior instance)" << dendl
;
1860 objecter
->wait_for_map(mdsmap
->get_last_failure_osd_epoch(),
1866 void MDSRank::replay_done()
1868 if (!standby_replaying
) {
1869 dout(1) << "Finished replaying journal" << dendl
;
1871 dout(5) << "Finished replaying journal as standby-replay" << dendl
;
1874 if (is_standby_replay()) {
1875 // The replay was done in standby state, and we are still in that state
1876 ceph_assert(standby_replaying
);
1877 dout(10) << "setting replay timer" << dendl
;
1878 timer
.add_event_after(g_conf()->mds_replay_interval
,
1879 new C_MDS_StandbyReplayRestart(this));
1881 } else if (standby_replaying
) {
1882 // The replay was done in standby state, we have now _left_ that state
1883 dout(10) << " last replay pass was as a standby; making final pass" << dendl
;
1884 standby_replaying
= false;
1885 standby_replay_restart();
1888 // Replay is complete, journal read should be up to date
1889 ceph_assert(mdlog
->get_journaler()->get_read_pos() == mdlog
->get_journaler()->get_write_pos());
1890 ceph_assert(!is_standby_replay());
1892 // Reformat and come back here
1893 if (mdlog
->get_journaler()->get_stream_format() < g_conf()->mds_journal_format
) {
1894 dout(4) << "reformatting journal on standby-replay->replay transition" << dendl
;
1895 mdlog
->reopen(new C_MDS_BootStart(this, MDS_BOOT_REPLAY_DONE
));
1900 dout(1) << "making mds journal writeable" << dendl
;
1901 mdlog
->get_journaler()->set_writeable();
1902 mdlog
->get_journaler()->trim_tail();
1904 if (mdsmap
->get_tableserver() == whoami
&&
1905 snapserver
->upgrade_format()) {
1906 dout(1) << "upgrading snaptable format" << dendl
;
1907 snapserver
->save(new C_MDSInternalNoop
);
1910 if (g_conf()->mds_wipe_sessions
) {
1911 dout(1) << "wiping out client sessions" << dendl
;
1913 sessionmap
.save(new C_MDSInternalNoop
);
1915 if (g_conf()->mds_wipe_ino_prealloc
) {
1916 dout(1) << "wiping out ino prealloc from sessions" << dendl
;
1917 sessionmap
.wipe_ino_prealloc();
1918 sessionmap
.save(new C_MDSInternalNoop
);
1920 if (g_conf()->mds_skip_ino
) {
1921 inodeno_t i
= g_conf()->mds_skip_ino
;
1922 dout(1) << "skipping " << i
<< " inodes" << dendl
;
1923 inotable
->skip_inos(i
);
1924 inotable
->save(new C_MDSInternalNoop
);
1927 if (mdsmap
->get_num_in_mds() == 1 &&
1928 mdsmap
->get_num_failed_mds() == 0) { // just me!
1929 dout(2) << "i am alone, moving to state reconnect" << dendl
;
1930 request_state(MDSMap::STATE_RECONNECT
);
1931 // sync snaptable cache
1932 snapclient
->sync(new C_MDSInternalNoop
);
1934 dout(2) << "i am not alone, moving to state resolve" << dendl
;
1935 request_state(MDSMap::STATE_RESOLVE
);
1939 void MDSRank::reopen_log()
1941 dout(1) << "reopen_log" << dendl
;
1942 mdcache
->rollback_uncommitted_fragments();
1945 void MDSRank::resolve_start()
1947 dout(1) << "resolve_start" << dendl
;
1951 calc_recovery_set();
1953 mdcache
->resolve_start(new C_MDS_VoidFn(this, &MDSRank::resolve_done
));
1954 finish_contexts(g_ceph_context
, waiting_for_resolve
);
1957 void MDSRank::resolve_done()
1959 dout(1) << "resolve_done" << dendl
;
1960 request_state(MDSMap::STATE_RECONNECT
);
1961 // sync snaptable cache
1962 snapclient
->sync(new C_MDSInternalNoop
);
1965 void MDSRank::apply_blocklist(const std::set
<entity_addr_t
> &addrs
, epoch_t epoch
) {
1966 auto victims
= server
->apply_blocklist();
1967 dout(4) << __func__
<< ": killed " << victims
<< ", blocklisted sessions ("
1968 << addrs
.size() << " blocklist entries, "
1969 << sessionmap
.get_sessions().size() << ")" << dendl
;
1971 set_osd_epoch_barrier(epoch
);
1976 void MDSRank::reconnect_start()
1978 dout(1) << "reconnect_start" << dendl
;
1980 if (last_state
== MDSMap::STATE_REPLAY
) {
1984 // Drop any blocklisted clients from the SessionMap before going
1985 // into reconnect, so that we don't wait for them.
1986 objecter
->enable_blocklist_events();
1987 std::set
<entity_addr_t
> blocklist
;
1988 std::set
<entity_addr_t
> range
;
1990 objecter
->with_osdmap([&blocklist
, &range
, &epoch
](const OSDMap
& o
) {
1991 o
.get_blocklist(&blocklist
, &range
);
1992 epoch
= o
.get_epoch();
1995 apply_blocklist(blocklist
, epoch
);
1997 server
->reconnect_clients(new C_MDS_VoidFn(this, &MDSRank::reconnect_done
));
1998 finish_contexts(g_ceph_context
, waiting_for_reconnect
);
2000 void MDSRank::reconnect_done()
2002 dout(1) << "reconnect_done" << dendl
;
2003 request_state(MDSMap::STATE_REJOIN
); // move to rejoin state
2006 void MDSRank::rejoin_joint_start()
2008 dout(1) << "rejoin_joint_start" << dendl
;
2009 mdcache
->rejoin_send_rejoins();
2011 void MDSRank::rejoin_start()
2013 dout(1) << "rejoin_start" << dendl
;
2014 mdcache
->rejoin_start(new C_MDS_VoidFn(this, &MDSRank::rejoin_done
));
2015 finish_contexts(g_ceph_context
, waiting_for_rejoin
);
2017 void MDSRank::rejoin_done()
2019 dout(1) << "rejoin_done" << dendl
;
2020 mdcache
->show_subtrees();
2021 mdcache
->show_cache();
2023 if (mdcache
->is_any_uncommitted_fragment()) {
2024 dout(1) << " waiting for uncommitted fragments" << dendl
;
2025 mdcache
->wait_for_uncommitted_fragments(new C_MDS_VoidFn(this, &MDSRank::rejoin_done
));
2029 // funny case: is our cache empty? no subtrees?
2030 if (!mdcache
->is_subtrees()) {
2032 // The root should always have a subtree!
2033 clog
->error() << "No subtrees found for root MDS rank!";
2035 ceph_assert(mdcache
->is_subtrees());
2037 dout(1) << " empty cache, no subtrees, leaving cluster" << dendl
;
2038 request_state(MDSMap::STATE_STOPPED
);
2043 if (replay_queue
.empty() && !server
->get_num_pending_reclaim()) {
2044 request_state(MDSMap::STATE_ACTIVE
);
2046 replaying_requests_done
= replay_queue
.empty();
2047 request_state(MDSMap::STATE_CLIENTREPLAY
);
2051 void MDSRank::clientreplay_start()
2053 dout(1) << "clientreplay_start" << dendl
;
2054 finish_contexts(g_ceph_context
, waiting_for_replay
); // kick waiters
2058 bool MDSRank::queue_one_replay()
2060 if (!replay_queue
.empty()) {
2061 queue_waiter(replay_queue
.front());
2062 replay_queue
.pop_front();
2065 if (!replaying_requests_done
) {
2066 replaying_requests_done
= true;
2069 maybe_clientreplay_done();
2073 void MDSRank::maybe_clientreplay_done()
2075 if (is_clientreplay() && get_want_state() == MDSMap::STATE_CLIENTREPLAY
) {
2077 // don't go to active if there are session waiting for being reclaimed
2078 if (replaying_requests_done
&& !server
->get_num_pending_reclaim()) {
2079 mdlog
->wait_for_safe(new C_MDS_VoidFn(this, &MDSRank::clientreplay_done
));
2083 dout(1) << " still have " << replay_queue
.size() + (int)!replaying_requests_done
2084 << " requests need to be replayed, " << server
->get_num_pending_reclaim()
2085 << " sessions need to be reclaimed" << dendl
;
2089 void MDSRank::clientreplay_done()
2091 dout(1) << "clientreplay_done" << dendl
;
2092 request_state(MDSMap::STATE_ACTIVE
);
2095 void MDSRank::active_start()
2097 dout(1) << "active_start" << dendl
;
2099 if (last_state
== MDSMap::STATE_CREATING
||
2100 last_state
== MDSMap::STATE_STARTING
) {
2101 mdcache
->open_root();
2104 dout(10) << __func__
<< ": initializing metrics handler" << dendl
;
2105 metrics_handler
.init();
2106 messenger
->add_dispatcher_tail(&metrics_handler
);
2108 // metric aggregation is solely done by rank 0
2110 dout(10) << __func__
<< ": initializing metric aggregator" << dendl
;
2111 ceph_assert(metric_aggregator
== nullptr);
2112 metric_aggregator
= std::make_unique
<MetricAggregator
>(cct
, this, mgrc
);
2113 metric_aggregator
->init();
2114 messenger
->add_dispatcher_tail(metric_aggregator
.get());
2117 mdcache
->clean_open_file_lists();
2118 mdcache
->export_remaining_imported_caps();
2119 finish_contexts(g_ceph_context
, waiting_for_replay
); // kick waiters
2121 mdcache
->reissue_all_caps();
2123 finish_contexts(g_ceph_context
, waiting_for_active
); // kick waiters
2126 void MDSRank::recovery_done(int oldstate
)
2128 dout(1) << "recovery_done -- successful recovery!" << dendl
;
2129 ceph_assert(is_clientreplay() || is_active());
2131 if (oldstate
== MDSMap::STATE_CREATING
)
2134 mdcache
->start_recovered_truncates();
2135 mdcache
->start_purge_inodes();
2136 mdcache
->start_files_to_recover();
2138 mdcache
->populate_mydir();
2141 void MDSRank::creating_done()
2143 dout(1)<< "creating_done" << dendl
;
2144 request_state(MDSMap::STATE_ACTIVE
);
2145 // sync snaptable cache
2146 snapclient
->sync(new C_MDSInternalNoop
);
2149 void MDSRank::boot_create()
2151 dout(3) << "boot_create" << dendl
;
2153 MDSGatherBuilder
fin(g_ceph_context
, new C_MDS_VoidFn(this, &MDSRank::creating_done
));
2155 mdcache
->init_layouts();
2157 inotable
->set_rank(whoami
);
2158 sessionmap
.set_rank(whoami
);
2160 // start with a fresh journal
2161 dout(10) << "boot_create creating fresh journal" << dendl
;
2162 mdlog
->create(fin
.new_sub());
2164 // open new journal segment, but do not journal subtree map (yet)
2165 mdlog
->prepare_new_segment();
2167 if (whoami
== mdsmap
->get_root()) {
2168 dout(3) << "boot_create creating fresh hierarchy" << dendl
;
2169 mdcache
->create_empty_hierarchy(fin
.get());
2172 dout(3) << "boot_create creating mydir hierarchy" << dendl
;
2173 mdcache
->create_mydir_hierarchy(fin
.get());
2175 dout(3) << "boot_create creating global snaprealm" << dendl
;
2176 mdcache
->create_global_snaprealm();
2178 // fixme: fake out inotable (reset, pretend loaded)
2179 dout(10) << "boot_create creating fresh inotable table" << dendl
;
2181 inotable
->save(fin
.new_sub());
2183 // write empty sessionmap
2184 sessionmap
.save(fin
.new_sub());
2186 // Create empty purge queue
2187 purge_queue
.create(new C_IO_Wrapper(this, fin
.new_sub()));
2189 // initialize tables
2190 if (mdsmap
->get_tableserver() == whoami
) {
2191 dout(10) << "boot_create creating fresh snaptable" << dendl
;
2192 snapserver
->set_rank(whoami
);
2193 snapserver
->reset();
2194 snapserver
->save(fin
.new_sub());
2197 ceph_assert(g_conf()->mds_kill_create_at
!= 1);
2199 // ok now journal it
2200 mdlog
->journal_segment_subtree_map(fin
.new_sub());
2203 // Usually we do this during reconnect, but creation skips that.
2204 objecter
->enable_blocklist_events();
2209 void MDSRank::stopping_start()
2211 dout(2) << "Stopping..." << dendl
;
2213 if (mdsmap
->get_num_in_mds() == 1 && !sessionmap
.empty()) {
2214 std::vector
<Session
*> victims
;
2215 const auto& sessions
= sessionmap
.get_sessions();
2216 for (const auto& p
: sessions
) {
2217 if (!p
.first
.is_client()) {
2221 Session
*s
= p
.second
;
2222 victims
.push_back(s
);
2225 dout(20) << __func__
<< " matched " << victims
.size() << " sessions" << dendl
;
2226 ceph_assert(!victims
.empty());
2228 C_GatherBuilder
gather(g_ceph_context
, new C_MDSInternalNoop
);
2229 for (const auto &s
: victims
) {
2230 CachedStackStringStream css
;
2231 evict_client(s
->get_client().v
, false,
2232 g_conf()->mds_session_blocklist_on_evict
, *css
, gather
.new_sub());
2237 mdcache
->shutdown_start();
2240 void MDSRank::stopping_done()
2242 dout(2) << "Finished stopping..." << dendl
;
2244 // tell monitor we shut down cleanly.
2245 request_state(MDSMap::STATE_STOPPED
);
2248 void MDSRankDispatcher::handle_mds_map(
2249 const cref_t
<MMDSMap
> &m
,
2250 const MDSMap
&oldmap
)
2252 // I am only to be passed MDSMaps in which I hold a rank
2253 ceph_assert(whoami
!= MDS_RANK_NONE
);
2255 mds_gid_t mds_gid
= mds_gid_t(monc
->get_global_id());
2256 MDSMap::DaemonState oldstate
= oldmap
.get_state_gid(mds_gid
);
2257 if (oldstate
== MDSMap::STATE_NULL
) {
2258 // monitor may skip sending me the STANDBY map (e.g. if paxos_propose_interval is high)
2259 // Assuming I have passed STANDBY state if I got a rank in the first map.
2260 oldstate
= MDSMap::STATE_STANDBY
;
2262 // I should not miss map update
2263 ceph_assert(state
== oldstate
);
2264 state
= mdsmap
->get_state_gid(mds_gid
);
2265 if (state
!= oldstate
) {
2266 last_state
= oldstate
;
2267 incarnation
= mdsmap
->get_inc_gid(mds_gid
);
2270 version_t epoch
= m
->get_epoch();
2272 // note source's map version
2273 if (m
->get_source().is_mds() &&
2274 peer_mdsmap_epoch
[mds_rank_t(m
->get_source().num())] < epoch
) {
2275 dout(15) << " peer " << m
->get_source()
2276 << " has mdsmap epoch >= " << epoch
2278 peer_mdsmap_epoch
[mds_rank_t(m
->get_source().num())] = epoch
;
2281 // Validate state transitions while I hold a rank
2282 if (!MDSMap::state_transition_valid(oldstate
, state
)) {
2283 derr
<< "Invalid state transition " << ceph_mds_state_name(oldstate
)
2284 << "->" << ceph_mds_state_name(state
) << dendl
;
2288 if (oldstate
!= state
) {
2289 // update messenger.
2290 auto sleep_rank_change
= g_conf().get_val
<double>("mds_sleep_rank_change");
2291 if (unlikely(sleep_rank_change
> 0)) {
2292 // This is to trigger a race where another rank tries to connect to this
2293 // MDS before an update to the messenger "myname" is processed. This race
2294 // should be closed by ranks holding messages until the rank is out of a
2295 // "bootstrapping" state.
2296 usleep(sleep_rank_change
);
2297 } if (state
== MDSMap::STATE_STANDBY_REPLAY
) {
2298 dout(1) << "handle_mds_map i am now mds." << mds_gid
<< "." << incarnation
2299 << " replaying mds." << whoami
<< "." << incarnation
<< dendl
;
2300 messenger
->set_myname(entity_name_t::MDS(mds_gid
));
2302 dout(1) << "handle_mds_map i am now mds." << whoami
<< "." << incarnation
<< dendl
;
2303 messenger
->set_myname(entity_name_t::MDS(whoami
));
2307 // tell objecter my incarnation
2308 if (objecter
->get_client_incarnation() != incarnation
)
2309 objecter
->set_client_incarnation(incarnation
);
2311 if (mdsmap
->get_required_client_features() != oldmap
.get_required_client_features())
2312 server
->update_required_client_features();
2315 if (g_conf()->mds_dump_cache_on_map
)
2316 mdcache
->dump_cache();
2318 cluster_degraded
= mdsmap
->is_degraded();
2320 // mdsmap and oldmap can be discontinuous. failover might happen in the missing mdsmap.
2321 // the 'restart' set tracks ranks that have restarted since the old mdsmap
2322 set
<mds_rank_t
> restart
;
2323 // replaying mds does not communicate with other ranks
2324 if (state
>= MDSMap::STATE_RESOLVE
) {
2325 // did someone fail?
2327 set
<mds_rank_t
> olddown
, down
;
2328 oldmap
.get_down_mds_set(&olddown
);
2329 mdsmap
->get_down_mds_set(&down
);
2330 for (const auto& r
: down
) {
2331 if (oldmap
.have_inst(r
) && olddown
.count(r
) == 0) {
2332 messenger
->mark_down_addrs(oldmap
.get_addrs(r
));
2333 handle_mds_failure(r
);
2337 // did someone fail?
2338 // did their addr/inst change?
2340 mdsmap
->get_up_mds_set(up
);
2341 for (const auto& r
: up
) {
2342 auto& info
= mdsmap
->get_info(r
);
2343 if (oldmap
.have_inst(r
)) {
2344 auto& oldinfo
= oldmap
.get_info(r
);
2345 if (info
.inc
!= oldinfo
.inc
) {
2346 messenger
->mark_down_addrs(oldinfo
.get_addrs());
2347 if (info
.state
== MDSMap::STATE_REPLAY
||
2348 info
.state
== MDSMap::STATE_RESOLVE
) {
2350 handle_mds_failure(r
);
2352 ceph_assert(info
.state
== MDSMap::STATE_STARTING
||
2353 info
.state
== MDSMap::STATE_ACTIVE
);
2354 // -> stopped (missing) -> starting -> active
2356 mdcache
->migrator
->handle_mds_failure_or_stop(r
);
2357 if (mdsmap
->get_tableserver() == whoami
)
2358 snapserver
->handle_mds_failure_or_stop(r
);
2362 if (info
.state
== MDSMap::STATE_REPLAY
||
2363 info
.state
== MDSMap::STATE_RESOLVE
) {
2364 // -> starting/creating (missing) -> active (missing) -> replay -> resolve
2366 handle_mds_failure(r
);
2368 ceph_assert(info
.state
== MDSMap::STATE_CREATING
||
2369 info
.state
== MDSMap::STATE_STARTING
||
2370 info
.state
== MDSMap::STATE_ACTIVE
);
2377 if (oldstate
!= state
) {
2378 dout(1) << "handle_mds_map state change "
2379 << ceph_mds_state_name(oldstate
) << " --> "
2380 << ceph_mds_state_name(state
) << dendl
;
2381 beacon
.set_want_state(*mdsmap
, state
);
2383 if (oldstate
== MDSMap::STATE_STANDBY_REPLAY
) {
2384 dout(10) << "Monitor activated us! Deactivating replay loop" << dendl
;
2385 ceph_assert (state
== MDSMap::STATE_REPLAY
);
2387 // did i just recover?
2388 if ((is_active() || is_clientreplay()) &&
2389 (oldstate
== MDSMap::STATE_CREATING
||
2390 oldstate
== MDSMap::STATE_REJOIN
||
2391 oldstate
== MDSMap::STATE_RECONNECT
))
2392 recovery_done(oldstate
);
2396 } else if (is_any_replay()) {
2398 } else if (is_resolve()) {
2400 } else if (is_reconnect()) {
2402 } else if (is_rejoin()) {
2404 } else if (is_clientreplay()) {
2405 clientreplay_start();
2406 } else if (is_creating()) {
2408 } else if (is_starting()) {
2410 } else if (is_stopping()) {
2411 ceph_assert(oldstate
== MDSMap::STATE_ACTIVE
);
2418 // is someone else newly resolving?
2419 if (state
>= MDSMap::STATE_RESOLVE
) {
2420 // recover snaptable
2421 if (mdsmap
->get_tableserver() == whoami
) {
2422 if (oldstate
< MDSMap::STATE_RESOLVE
) {
2424 mdsmap
->get_mds_set_lower_bound(s
, MDSMap::STATE_RESOLVE
);
2425 snapserver
->finish_recovery(s
);
2427 set
<mds_rank_t
> old_set
, new_set
;
2428 oldmap
.get_mds_set_lower_bound(old_set
, MDSMap::STATE_RESOLVE
);
2429 mdsmap
->get_mds_set_lower_bound(new_set
, MDSMap::STATE_RESOLVE
);
2430 for (const auto& r
: new_set
) {
2433 if (!old_set
.count(r
) || restart
.count(r
)) { // newly so?
2434 snapserver
->handle_mds_recovery(r
);
2440 if ((!oldmap
.is_resolving() || !restart
.empty()) && mdsmap
->is_resolving()) {
2441 set
<mds_rank_t
> resolve
;
2442 mdsmap
->get_mds_set(resolve
, MDSMap::STATE_RESOLVE
);
2443 dout(10) << " resolve set is " << resolve
<< dendl
;
2444 calc_recovery_set();
2445 mdcache
->send_resolves();
2450 // is everybody finally rejoining?
2451 if (state
>= MDSMap::STATE_REJOIN
) {
2453 if (!oldmap
.is_rejoining() && mdsmap
->is_rejoining())
2454 rejoin_joint_start();
2457 if (g_conf()->mds_dump_cache_after_rejoin
&&
2458 oldmap
.is_rejoining() && !mdsmap
->is_rejoining())
2459 mdcache
->dump_cache(); // for DEBUG only
2461 if (oldstate
>= MDSMap::STATE_REJOIN
||
2462 oldstate
== MDSMap::STATE_STARTING
) {
2463 // ACTIVE|CLIENTREPLAY|REJOIN => we can discover from them.
2464 set
<mds_rank_t
> olddis
, dis
;
2465 oldmap
.get_mds_set_lower_bound(olddis
, MDSMap::STATE_REJOIN
);
2466 mdsmap
->get_mds_set_lower_bound(dis
, MDSMap::STATE_REJOIN
);
2467 for (const auto& r
: dis
) {
2470 if (!olddis
.count(r
) || restart
.count(r
)) { // newly so?
2471 mdcache
->kick_discovers(r
);
2472 mdcache
->kick_open_ino_peers(r
);
2478 if (oldmap
.is_degraded() && !cluster_degraded
&& state
>= MDSMap::STATE_ACTIVE
) {
2479 dout(1) << "cluster recovered." << dendl
;
2480 auto it
= waiting_for_active_peer
.find(MDS_RANK_NONE
);
2481 if (it
!= waiting_for_active_peer
.end()) {
2482 queue_waiters(it
->second
);
2483 waiting_for_active_peer
.erase(it
);
2487 // did someone leave a "bootstrapping" state? We can't connect until then to
2488 // allow messenger "myname" updates.
2490 std::vector
<mds_rank_t
> erase
;
2491 for (auto& [rank
, queue
] : waiting_for_bootstrapping_peer
) {
2492 auto state
= mdsmap
->get_state(rank
);
2493 if (state
> MDSMap::STATE_REPLAY
) {
2494 queue_waiters(queue
);
2495 erase
.push_back(rank
);
2498 for (const auto& rank
: erase
) {
2499 waiting_for_bootstrapping_peer
.erase(rank
);
2503 if (unlikely(g_conf().get_val
<bool>("mds_connect_bootstrapping"))) {
2504 std::set
<mds_rank_t
> bootstrapping
;
2505 mdsmap
->get_mds_set(bootstrapping
, MDSMap::STATE_REPLAY
);
2506 mdsmap
->get_mds_set(bootstrapping
, MDSMap::STATE_CREATING
);
2507 mdsmap
->get_mds_set(bootstrapping
, MDSMap::STATE_STARTING
);
2508 for (const auto& rank
: bootstrapping
) {
2509 auto m
= make_message
<MMDSMap
>(monc
->get_fsid(), *mdsmap
);
2510 send_message_mds(std::move(m
), rank
);
2514 // did someone go active?
2515 if (state
>= MDSMap::STATE_CLIENTREPLAY
&&
2516 oldstate
>= MDSMap::STATE_CLIENTREPLAY
) {
2517 set
<mds_rank_t
> oldactive
, active
;
2518 oldmap
.get_mds_set_lower_bound(oldactive
, MDSMap::STATE_CLIENTREPLAY
);
2519 mdsmap
->get_mds_set_lower_bound(active
, MDSMap::STATE_CLIENTREPLAY
);
2520 for (const auto& r
: active
) {
2523 if (!oldactive
.count(r
) || restart
.count(r
)) // newly so?
2524 handle_mds_recovery(r
);
2528 if (is_clientreplay() || is_active() || is_stopping()) {
2530 set
<mds_rank_t
> oldstopped
, stopped
;
2531 oldmap
.get_stopped_mds_set(oldstopped
);
2532 mdsmap
->get_stopped_mds_set(stopped
);
2533 for (const auto& r
: stopped
)
2534 if (oldstopped
.count(r
) == 0) { // newly so?
2535 mdcache
->migrator
->handle_mds_failure_or_stop(r
);
2536 if (mdsmap
->get_tableserver() == whoami
)
2537 snapserver
->handle_mds_failure_or_stop(r
);
2542 map
<epoch_t
,MDSContext::vec
>::iterator p
= waiting_for_mdsmap
.begin();
2543 while (p
!= waiting_for_mdsmap
.end() && p
->first
<= mdsmap
->get_epoch()) {
2546 waiting_for_mdsmap
.erase(p
++);
2552 // Before going active, set OSD epoch barrier to latest (so that
2553 // we don't risk handing out caps to clients with old OSD maps that
2554 // might not include barriers from the previous incarnation of this MDS)
2555 set_osd_epoch_barrier(objecter
->with_osdmap(
2556 std::mem_fn(&OSDMap::get_epoch
)));
2558 /* Now check if we should hint to the OSD that a read may follow */
2559 if (mdsmap
->has_standby_replay(whoami
))
2560 mdlog
->set_write_iohint(0);
2562 mdlog
->set_write_iohint(CEPH_OSD_OP_FLAG_FADVISE_DONTNEED
);
2565 if (oldmap
.get_max_mds() != mdsmap
->get_max_mds()) {
2566 purge_queue
.update_op_limit(*mdsmap
);
2569 if (mdsmap
->get_inline_data_enabled() && !oldmap
.get_inline_data_enabled())
2570 dout(0) << "WARNING: inline_data support has been deprecated and will be removed in a future release" << dendl
;
2572 mdcache
->handle_mdsmap(*mdsmap
, oldmap
);
2574 if (metric_aggregator
!= nullptr) {
2575 metric_aggregator
->notify_mdsmap(*mdsmap
);
2577 metrics_handler
.notify_mdsmap(*mdsmap
);
2580 void MDSRank::handle_mds_recovery(mds_rank_t who
)
2582 dout(5) << "handle_mds_recovery mds." << who
<< dendl
;
2584 mdcache
->handle_mds_recovery(who
);
2586 queue_waiters(waiting_for_active_peer
[who
]);
2587 waiting_for_active_peer
.erase(who
);
2590 void MDSRank::handle_mds_failure(mds_rank_t who
)
2592 if (who
== whoami
) {
2593 dout(5) << "handle_mds_failure for myself; not doing anything" << dendl
;
2596 dout(5) << "handle_mds_failure mds." << who
<< dendl
;
2598 mdcache
->handle_mds_failure(who
);
2600 if (mdsmap
->get_tableserver() == whoami
)
2601 snapserver
->handle_mds_failure_or_stop(who
);
2603 snapclient
->handle_mds_failure(who
);
2605 scrubstack
->handle_mds_failure(who
);
2608 void MDSRankDispatcher::handle_asok_command(
2609 std::string_view command
,
2610 const cmdmap_t
& cmdmap
,
2612 const bufferlist
&inbl
,
2613 std::function
<void(int,const std::string
&,bufferlist
&)> on_finish
)
2616 CachedStackStringStream css
;
2618 dout(10) << __func__
<< ": " << command
<< dendl
;
2619 if (command
== "dump_ops_in_flight") {
2620 if (!op_tracker
.dump_ops_in_flight(f
)) {
2621 *css
<< "op_tracker disabled; set mds_enable_op_tracker=true to enable";
2623 } else if (command
== "ops") {
2624 vector
<string
> flags
;
2625 cmd_getval(cmdmap
, "flags", flags
);
2626 std::unique_lock
l(mds_lock
, std::defer_lock
);
2627 auto lambda
= OpTracker::default_dumper
;
2629 /* use std::function if we actually want to capture flags someday */
2630 lambda
= [](const TrackedOp
& op
, Formatter
* f
) {
2631 auto* req
= dynamic_cast<const MDRequestImpl
*>(&op
);
2633 req
->dump_with_mds_lock(f
);
2640 if (!op_tracker
.dump_ops_in_flight(f
, false, {""}, false, lambda
)) {
2641 *css
<< "op_tracker disabled; set mds_enable_op_tracker=true to enable";
2643 } else if (command
== "dump_blocked_ops") {
2644 if (!op_tracker
.dump_ops_in_flight(f
, true)) {
2645 *css
<< "op_tracker disabled; set mds_enable_op_tracker=true to enable";
2647 } else if (command
== "dump_blocked_ops_count") {
2648 if (!op_tracker
.dump_ops_in_flight(f
, true, {""}, true)) {
2649 *css
<< "op_tracker disabled; set mds_enable_op_tracker=true to enable";
2651 } else if (command
== "dump_historic_ops") {
2652 if (!op_tracker
.dump_historic_ops(f
)) {
2653 *css
<< "op_tracker disabled; set mds_enable_op_tracker=true to enable";
2655 } else if (command
== "dump_historic_ops_by_duration") {
2656 if (!op_tracker
.dump_historic_ops(f
, true)) {
2657 *css
<< "op_tracker disabled; set mds_enable_op_tracker=true to enable";
2659 } else if (command
== "osdmap barrier") {
2660 int64_t target_epoch
= 0;
2661 bool got_val
= cmd_getval(cmdmap
, "target_epoch", target_epoch
);
2664 *css
<< "no target epoch given";
2669 std::lock_guard
l(mds_lock
);
2670 set_osd_epoch_barrier(target_epoch
);
2672 boost::system::error_code ec
;
2673 dout(4) << __func__
<< ": possibly waiting for OSD epoch " << target_epoch
<< dendl
;
2674 objecter
->wait_for_map(target_epoch
, ceph::async::use_blocked
[ec
]);
2675 } else if (command
== "session ls" ||
2676 command
== "client ls") {
2677 std::lock_guard
l(mds_lock
);
2678 bool cap_dump
= false;
2679 std::vector
<std::string
> filter_args
;
2680 cmd_getval(cmdmap
, "cap_dump", cap_dump
);
2681 cmd_getval(cmdmap
, "filters", filter_args
);
2683 SessionFilter filter
;
2684 r
= filter
.parse(filter_args
, css
.get());
2688 dump_sessions(filter
, f
, cap_dump
);
2689 } else if (command
== "session evict" ||
2690 command
== "client evict") {
2691 std::lock_guard
l(mds_lock
);
2692 std::vector
<std::string
> filter_args
;
2693 cmd_getval(cmdmap
, "filters", filter_args
);
2695 SessionFilter filter
;
2696 r
= filter
.parse(filter_args
, css
.get());
2701 evict_clients(filter
, on_finish
);
2703 } else if (command
== "session kill") {
2704 std::string client_id
;
2705 if (!cmd_getval(cmdmap
, "client_id", client_id
)) {
2706 *css
<< "Invalid client_id specified";
2710 std::lock_guard
l(mds_lock
);
2711 bool evicted
= evict_client(strtol(client_id
.c_str(), 0, 10), true,
2712 g_conf()->mds_session_blocklist_on_evict
, *css
);
2714 dout(15) << css
->strv() << dendl
;
2717 } else if (command
== "session config" ||
2718 command
== "client config") {
2723 cmd_getval(cmdmap
, "client_id", client_id
);
2724 cmd_getval(cmdmap
, "option", option
);
2725 bool got_value
= cmd_getval(cmdmap
, "value", value
);
2727 std::lock_guard
l(mds_lock
);
2728 r
= config_client(client_id
, !got_value
, option
, value
, *css
);
2729 } else if (command
== "scrub start" ||
2730 command
== "scrub_start") {
2732 *css
<< "Not rank 0";
2739 vector
<string
> scrubop_vec
;
2740 cmd_getval(cmdmap
, "scrubops", scrubop_vec
);
2741 cmd_getval(cmdmap
, "path", path
);
2742 cmd_getval(cmdmap
, "tag", tag
);
2746 [this, on_finish
, f
, path
, tag
, scrubop_vec
](int r
) {
2747 command_scrub_start(
2748 f
, path
, tag
, scrubop_vec
,
2750 [on_finish
](int r
) {
2752 on_finish(r
, {}, outbl
);
2756 } else if (command
== "scrub abort") {
2758 *css
<< "Not rank 0";
2765 [this, on_finish
, f
](int r
) {
2766 command_scrub_abort(
2769 [on_finish
, f
](int r
) {
2771 f
->open_object_section("result");
2772 f
->dump_int("return_code", r
);
2774 on_finish(r
, {}, outbl
);
2778 } else if (command
== "scrub pause") {
2780 *css
<< "Not rank 0";
2787 [this, on_finish
, f
](int r
) {
2788 command_scrub_pause(
2791 [on_finish
, f
](int r
) {
2793 f
->open_object_section("result");
2794 f
->dump_int("return_code", r
);
2796 on_finish(r
, {}, outbl
);
2800 } else if (command
== "scrub resume") {
2802 *css
<< "Not rank 0";
2806 command_scrub_resume(f
);
2807 } else if (command
== "scrub status") {
2808 command_scrub_status(f
);
2809 } else if (command
== "tag path") {
2811 *css
<< "Not rank 0";
2816 cmd_getval(cmdmap
, "path", path
);
2818 cmd_getval(cmdmap
, "tag", tag
);
2819 command_tag_path(f
, path
, tag
);
2820 } else if (command
== "flush_path") {
2822 cmd_getval(cmdmap
, "path", path
);
2823 command_flush_path(f
, path
);
2824 } else if (command
== "flush journal") {
2825 command_flush_journal(f
);
2826 } else if (command
== "get subtrees") {
2827 command_get_subtrees(f
);
2828 } else if (command
== "export dir") {
2830 if(!cmd_getval(cmdmap
, "path", path
)) {
2831 *css
<< "malformed path";
2836 if(!cmd_getval(cmdmap
, "rank", rank
)) {
2837 *css
<< "malformed rank";
2841 command_export_dir(f
, path
, (mds_rank_t
)rank
);
2842 } else if (command
== "dump cache") {
2843 std::lock_guard
l(mds_lock
);
2844 int64_t timeout
= 0;
2845 cmd_getval(cmdmap
, "timeout", timeout
);
2846 auto mds_beacon_interval
= g_conf().get_val
<double>("mds_beacon_interval");
2848 timeout
= mds_beacon_interval
/ 2;
2849 else if (timeout
> mds_beacon_interval
)
2850 timeout
= mds_beacon_interval
;
2852 if (!cmd_getval(cmdmap
, "path", path
)) {
2853 r
= mdcache
->dump_cache(f
, timeout
);
2855 r
= mdcache
->dump_cache(path
, timeout
);
2857 } else if (command
== "cache drop") {
2858 int64_t timeout
= 0;
2859 cmd_getval(cmdmap
, "timeout", timeout
);
2862 [this, on_finish
, f
, timeout
](int r
) {
2866 [on_finish
](int r
) {
2868 on_finish(r
, {}, outbl
);
2872 } else if (command
== "cache status") {
2873 std::lock_guard
l(mds_lock
);
2874 mdcache
->cache_status(f
);
2875 } else if (command
== "dump tree") {
2876 command_dump_tree(cmdmap
, *css
, f
);
2877 } else if (command
== "dump loads") {
2878 std::lock_guard
l(mds_lock
);
2880 bool got
= cmd_getval(cmdmap
, "depth", depth
);
2881 if (!got
|| depth
< 0) {
2882 dout(10) << "no depth limit when dirfrags dump_load" << dendl
;
2884 r
= balancer
->dump_loads(f
, depth
);
2885 } else if (command
== "dump snaps") {
2886 std::lock_guard
l(mds_lock
);
2888 cmd_getval(cmdmap
, "server", server
);
2889 if (server
== "--server") {
2890 if (mdsmap
->get_tableserver() == whoami
) {
2891 snapserver
->dump(f
);
2894 *css
<< "Not snapserver";
2897 r
= snapclient
->dump_cache(f
);
2899 } else if (command
== "force_readonly") {
2900 std::lock_guard
l(mds_lock
);
2901 mdcache
->force_readonly();
2902 } else if (command
== "dirfrag split") {
2903 command_dirfrag_split(cmdmap
, *css
);
2904 } else if (command
== "dirfrag merge") {
2905 command_dirfrag_merge(cmdmap
, *css
);
2906 } else if (command
== "dirfrag ls") {
2907 command_dirfrag_ls(cmdmap
, *css
, f
);
2908 } else if (command
== "openfiles ls") {
2909 command_openfiles_ls(f
);
2910 } else if (command
== "dump inode") {
2911 command_dump_inode(f
, cmdmap
, *css
);
2912 } else if (command
== "damage ls") {
2913 std::lock_guard
l(mds_lock
);
2914 damage_table
.dump(f
);
2915 } else if (command
== "damage rm") {
2916 std::lock_guard
l(mds_lock
);
2917 damage_entry_id_t id
= 0;
2918 if (!cmd_getval(cmdmap
, "damage_id", (int64_t&)id
)) {
2922 damage_table
.erase(id
);
2927 on_finish(r
, css
->str(), outbl
);
2931 * This function drops the mds_lock, so don't do anything with
2932 * MDSRank after calling it (we could have gone into shutdown): just
2933 * send your result back to the calling client and finish.
2935 void MDSRankDispatcher::evict_clients(
2936 const SessionFilter
&filter
,
2937 std::function
<void(int,const std::string
&,bufferlist
&)> on_finish
)
2940 if (is_any_replay()) {
2941 on_finish(-CEPHFS_EAGAIN
, "MDS is replaying log", outbl
);
2945 std::vector
<Session
*> victims
;
2946 const auto& sessions
= sessionmap
.get_sessions();
2947 for (const auto& p
: sessions
) {
2948 if (!p
.first
.is_client()) {
2952 Session
*s
= p
.second
;
2954 if (filter
.match(*s
, std::bind(&Server::waiting_for_reconnect
, server
,
2955 std::placeholders::_1
))) {
2956 victims
.push_back(s
);
2960 dout(20) << __func__
<< " matched " << victims
.size() << " sessions" << dendl
;
2962 if (victims
.empty()) {
2963 on_finish(0, {}, outbl
);
2967 C_GatherBuilder
gather(g_ceph_context
,
2968 new LambdaContext([on_finish
](int r
) {
2970 on_finish(r
, {}, bl
);
2972 for (const auto s
: victims
) {
2973 CachedStackStringStream css
;
2974 evict_client(s
->get_client().v
, false,
2975 g_conf()->mds_session_blocklist_on_evict
, *css
, gather
.new_sub());
2980 void MDSRankDispatcher::dump_sessions(const SessionFilter
&filter
, Formatter
*f
, bool cap_dump
) const
2982 // Dump sessions, decorated with recovery/replay status
2983 f
->open_array_section("sessions");
2984 for (auto& [name
, s
] : sessionmap
.get_sessions()) {
2985 if (!name
.is_client()) {
2989 if (!filter
.match(*s
, std::bind(&Server::waiting_for_reconnect
, server
, std::placeholders::_1
))) {
2993 f
->open_object_section("session");
2994 s
->dump(f
, cap_dump
);
2997 f
->close_section(); // sessions
3000 void MDSRank::command_scrub_start(Formatter
*f
,
3001 std::string_view path
, std::string_view tag
,
3002 const vector
<string
>& scrubop_vec
, Context
*on_finish
)
3005 bool recursive
= false;
3006 bool repair
= false;
3007 bool scrub_mdsdir
= false;
3008 for (auto &op
: scrubop_vec
) {
3011 else if (op
== "recursive")
3013 else if (op
== "repair")
3015 else if (op
== "scrub_mdsdir" && path
== "/")
3016 scrub_mdsdir
= true;
3019 std::lock_guard
l(mds_lock
);
3020 mdcache
->enqueue_scrub(path
, tag
, force
, recursive
, repair
, scrub_mdsdir
,
3022 // scrub_dentry() finishers will dump the data for us; we're done!
3025 void MDSRank::command_tag_path(Formatter
*f
,
3026 std::string_view path
, std::string_view tag
)
3030 std::lock_guard
l(mds_lock
);
3031 mdcache
->enqueue_scrub(path
, tag
, true, true, false, false, f
, &scond
);
3036 void MDSRank::command_scrub_abort(Formatter
*f
, Context
*on_finish
) {
3037 std::lock_guard
l(mds_lock
);
3038 scrubstack
->scrub_abort(on_finish
);
3041 void MDSRank::command_scrub_pause(Formatter
*f
, Context
*on_finish
) {
3042 std::lock_guard
l(mds_lock
);
3043 scrubstack
->scrub_pause(on_finish
);
3046 void MDSRank::command_scrub_resume(Formatter
*f
) {
3047 std::lock_guard
l(mds_lock
);
3048 int r
= scrubstack
->scrub_resume();
3050 f
->open_object_section("result");
3051 f
->dump_int("return_code", r
);
3055 void MDSRank::command_scrub_status(Formatter
*f
) {
3056 std::lock_guard
l(mds_lock
);
3057 scrubstack
->scrub_status(f
);
3060 void MDSRank::command_flush_path(Formatter
*f
, std::string_view path
)
3064 std::lock_guard
l(mds_lock
);
3065 mdcache
->flush_dentry(path
, &scond
);
3067 int r
= scond
.wait();
3068 f
->open_object_section("results");
3069 f
->dump_int("return_code", r
);
3070 f
->close_section(); // results
3073 // synchronous wrapper around "journal flush" asynchronous context
3075 void MDSRank::command_flush_journal(Formatter
*f
) {
3076 ceph_assert(f
!= NULL
);
3079 CachedStackStringStream css
;
3081 std::lock_guard
locker(mds_lock
);
3082 C_Flush_Journal
*flush_journal
= new C_Flush_Journal(mdcache
, mdlog
, this, css
.get(), &cond
);
3083 flush_journal
->send();
3085 int r
= cond
.wait();
3087 f
->open_object_section("result");
3088 f
->dump_string("message", css
->strv());
3089 f
->dump_int("return_code", r
);
3093 void MDSRank::command_get_subtrees(Formatter
*f
)
3095 ceph_assert(f
!= NULL
);
3096 std::lock_guard
l(mds_lock
);
3098 std::vector
<CDir
*> subtrees
;
3099 mdcache
->get_subtrees(subtrees
);
3101 f
->open_array_section("subtrees");
3102 for (const auto& dir
: subtrees
) {
3103 f
->open_object_section("subtree");
3105 f
->dump_bool("is_auth", dir
->is_auth());
3106 f
->dump_int("auth_first", dir
->get_dir_auth().first
);
3107 f
->dump_int("auth_second", dir
->get_dir_auth().second
); {
3108 mds_rank_t export_pin
= dir
->inode
->get_export_pin(false);
3109 f
->dump_int("export_pin", export_pin
>= 0 ? export_pin
: -1);
3110 f
->dump_bool("distributed_ephemeral_pin", export_pin
== MDS_RANK_EPHEMERAL_DIST
);
3111 f
->dump_bool("random_ephemeral_pin", export_pin
== MDS_RANK_EPHEMERAL_RAND
);
3113 f
->dump_int("export_pin_target", dir
->get_export_pin(false));
3114 f
->open_object_section("dir");
3124 void MDSRank::command_export_dir(Formatter
*f
,
3125 std::string_view path
,
3128 int r
= _command_export_dir(path
, target
);
3129 f
->open_object_section("results");
3130 f
->dump_int("return_code", r
);
3131 f
->close_section(); // results
3134 int MDSRank::_command_export_dir(
3135 std::string_view path
,
3138 std::lock_guard
l(mds_lock
);
3141 if (target
== whoami
|| !mdsmap
->is_up(target
) || !mdsmap
->is_in(target
)) {
3142 derr
<< "bad MDS target " << target
<< dendl
;
3143 return -CEPHFS_ENOENT
;
3146 CInode
*in
= mdcache
->cache_traverse(fp
);
3148 derr
<< "bad path '" << path
<< "'" << dendl
;
3149 return -CEPHFS_ENOENT
;
3151 CDir
*dir
= in
->get_dirfrag(frag_t());
3152 if (!dir
|| !(dir
->is_auth())) {
3153 derr
<< "bad export_dir path dirfrag frag_t() or dir not auth" << dendl
;
3154 return -CEPHFS_EINVAL
;
3157 mdcache
->migrator
->export_dir(dir
, target
);
3161 void MDSRank::command_dump_tree(const cmdmap_t
&cmdmap
, std::ostream
&ss
, Formatter
*f
)
3165 cmd_getval(cmdmap
, "root", root
);
3169 if (!cmd_getval(cmdmap
, "depth", depth
))
3171 std::lock_guard
l(mds_lock
);
3172 CInode
*in
= mdcache
->cache_traverse(filepath(root
.c_str()));
3174 ss
<< "root inode is not in cache";
3177 f
->open_array_section("inodes");
3178 mdcache
->dump_tree(in
, 0, depth
, f
);
3182 CDir
*MDSRank::_command_dirfrag_get(
3183 const cmdmap_t
&cmdmap
,
3187 bool got
= cmd_getval(cmdmap
, "path", path
);
3189 ss
<< "missing path argument";
3193 std::string frag_str
;
3194 if (!cmd_getval(cmdmap
, "frag", frag_str
)) {
3195 ss
<< "missing frag argument";
3199 CInode
*in
= mdcache
->cache_traverse(filepath(path
.c_str()));
3201 // TODO really we should load something in if it's not in cache,
3202 // but the infrastructure is harder, and we might still be unable
3203 // to act on it if someone else is auth.
3204 ss
<< "directory '" << path
<< "' inode not in cache";
3210 if (!fg
.parse(frag_str
.c_str())) {
3211 ss
<< "frag " << frag_str
<< " failed to parse";
3215 CDir
*dir
= in
->get_dirfrag(fg
);
3217 ss
<< "frag " << in
->ino() << "/" << fg
<< " not in cache ("
3218 "use `dirfrag ls` to see if it should exist)";
3222 if (!dir
->is_auth()) {
3223 ss
<< "frag " << dir
->dirfrag() << " not auth (auth = "
3224 << dir
->authority() << ")";
3231 bool MDSRank::command_dirfrag_split(
3235 std::lock_guard
l(mds_lock
);
3237 if (!cmd_getval(cmdmap
, "bits", by
)) {
3238 ss
<< "missing bits argument";
3243 ss
<< "must split by >0 bits";
3247 CDir
*dir
= _command_dirfrag_get(cmdmap
, ss
);
3252 mdcache
->split_dir(dir
, by
);
3257 bool MDSRank::command_dirfrag_merge(
3261 std::lock_guard
l(mds_lock
);
3263 bool got
= cmd_getval(cmdmap
, "path", path
);
3265 ss
<< "missing path argument";
3269 std::string frag_str
;
3270 if (!cmd_getval(cmdmap
, "frag", frag_str
)) {
3271 ss
<< "missing frag argument";
3275 CInode
*in
= mdcache
->cache_traverse(filepath(path
.c_str()));
3277 ss
<< "directory '" << path
<< "' inode not in cache";
3282 if (!fg
.parse(frag_str
.c_str())) {
3283 ss
<< "frag " << frag_str
<< " failed to parse";
3287 mdcache
->merge_dir(in
, fg
);
3292 bool MDSRank::command_dirfrag_ls(
3297 std::lock_guard
l(mds_lock
);
3299 bool got
= cmd_getval(cmdmap
, "path", path
);
3301 ss
<< "missing path argument";
3305 CInode
*in
= mdcache
->cache_traverse(filepath(path
.c_str()));
3307 ss
<< "directory inode not in cache";
3311 f
->open_array_section("frags");
3313 // NB using get_leaves_under instead of get_dirfrags to give
3314 // you the list of what dirfrags may exist, not which are in cache
3315 in
->dirfragtree
.get_leaves_under(frag_t(), leaves
);
3316 for (const auto& leaf
: leaves
) {
3317 f
->open_object_section("frag");
3318 f
->dump_int("value", leaf
.value());
3319 f
->dump_int("bits", leaf
.bits());
3320 CachedStackStringStream css
;
3321 *css
<< std::hex
<< leaf
.value() << "/" << std::dec
<< leaf
.bits();
3322 f
->dump_string("str", css
->strv());
3330 void MDSRank::command_openfiles_ls(Formatter
*f
)
3332 std::lock_guard
l(mds_lock
);
3333 mdcache
->dump_openfiles(f
);
3336 void MDSRank::command_dump_inode(Formatter
*f
, const cmdmap_t
&cmdmap
, std::ostream
&ss
)
3338 std::lock_guard
l(mds_lock
);
3340 bool got
= cmd_getval(cmdmap
, "number", number
);
3342 ss
<< "missing inode number";
3346 bool success
= mdcache
->dump_inode(f
, number
);
3348 ss
<< "dump inode failed, wrong inode number or the inode is not cached";
3352 void MDSRank::dump_status(Formatter
*f
) const
3354 f
->dump_string("fs_name", std::string(mdsmap
->get_fs_name()));
3355 if (state
== MDSMap::STATE_REPLAY
||
3356 state
== MDSMap::STATE_STANDBY_REPLAY
) {
3357 mdlog
->dump_replay_status(f
);
3358 } else if (state
== MDSMap::STATE_RESOLVE
) {
3359 mdcache
->dump_resolve_status(f
);
3360 } else if (state
== MDSMap::STATE_RECONNECT
) {
3361 server
->dump_reconnect_status(f
);
3362 } else if (state
== MDSMap::STATE_REJOIN
) {
3363 mdcache
->dump_rejoin_status(f
);
3364 } else if (state
== MDSMap::STATE_CLIENTREPLAY
) {
3365 dump_clientreplay_status(f
);
3367 f
->dump_float("rank_uptime", get_uptime().count());
3370 void MDSRank::dump_clientreplay_status(Formatter
*f
) const
3372 f
->open_object_section("clientreplay_status");
3373 f
->dump_unsigned("clientreplay_queue", replay_queue
.size());
3374 f
->dump_unsigned("active_replay", mdcache
->get_num_client_requests());
3378 void MDSRankDispatcher::update_log_config()
3380 auto parsed_options
= clog
->parse_client_options(g_ceph_context
);
3381 dout(10) << __func__
<< " log_to_monitors " << parsed_options
.log_to_monitors
<< dendl
;
3384 void MDSRank::create_logger()
3386 dout(10) << "create_logger" << dendl
;
3388 PerfCountersBuilder
mds_plb(g_ceph_context
, "mds", l_mds_first
, l_mds_last
);
3390 // super useful (high prio) perf stats
3391 mds_plb
.add_u64_counter(l_mds_request
, "request", "Requests", "req",
3392 PerfCountersBuilder::PRIO_CRITICAL
);
3393 mds_plb
.add_time_avg(l_mds_reply_latency
, "reply_latency", "Reply latency", "rlat",
3394 PerfCountersBuilder::PRIO_CRITICAL
);
3395 mds_plb
.add_u64(l_mds_inodes
, "inodes", "Inodes", "inos",
3396 PerfCountersBuilder::PRIO_CRITICAL
);
3397 mds_plb
.add_u64_counter(l_mds_forward
, "forward", "Forwarding request", "fwd",
3398 PerfCountersBuilder::PRIO_INTERESTING
);
3399 mds_plb
.add_u64(l_mds_caps
, "caps", "Capabilities", "caps",
3400 PerfCountersBuilder::PRIO_INTERESTING
);
3401 mds_plb
.add_u64_counter(l_mds_exported_inodes
, "exported_inodes", "Exported inodes",
3402 "exi", PerfCountersBuilder::PRIO_INTERESTING
);
3403 mds_plb
.add_u64_counter(l_mds_imported_inodes
, "imported_inodes", "Imported inodes",
3404 "imi", PerfCountersBuilder::PRIO_INTERESTING
);
3405 mds_plb
.add_u64_counter(l_mds_slow_reply
, "slow_reply", "Slow replies", "slr",
3406 PerfCountersBuilder::PRIO_INTERESTING
);
3409 mds_plb
.add_u64_counter(l_mdss_handle_client_caps
, "handle_client_caps",
3410 "Client caps msg", "hcc", PerfCountersBuilder::PRIO_INTERESTING
);
3411 mds_plb
.add_u64_counter(l_mdss_handle_client_caps_dirty
, "handle_client_caps_dirty",
3412 "Client dirty caps msg", "hccd", PerfCountersBuilder::PRIO_INTERESTING
);
3413 mds_plb
.add_u64_counter(l_mdss_handle_client_cap_release
, "handle_client_cap_release",
3414 "Client cap release msg", "hccr", PerfCountersBuilder::PRIO_INTERESTING
);
3415 mds_plb
.add_u64_counter(l_mdss_process_request_cap_release
, "process_request_cap_release",
3416 "Process request cap release", "prcr", PerfCountersBuilder::PRIO_INTERESTING
);
3417 mds_plb
.add_u64_counter(l_mdss_ceph_cap_op_revoke
, "ceph_cap_op_revoke",
3418 "Revoke caps", "crev", PerfCountersBuilder::PRIO_INTERESTING
);
3419 mds_plb
.add_u64_counter(l_mdss_ceph_cap_op_grant
, "ceph_cap_op_grant",
3420 "Grant caps", "cgra", PerfCountersBuilder::PRIO_INTERESTING
);
3421 mds_plb
.add_u64_counter(l_mdss_ceph_cap_op_trunc
, "ceph_cap_op_trunc",
3422 "caps truncate notify", "ctru", PerfCountersBuilder::PRIO_INTERESTING
);
3423 mds_plb
.add_u64_counter(l_mdss_ceph_cap_op_flushsnap_ack
, "ceph_cap_op_flushsnap_ack",
3424 "caps truncate notify", "cfsa", PerfCountersBuilder::PRIO_INTERESTING
);
3425 mds_plb
.add_u64_counter(l_mdss_ceph_cap_op_flush_ack
, "ceph_cap_op_flush_ack",
3426 "caps truncate notify", "cfa", PerfCountersBuilder::PRIO_INTERESTING
);
3427 mds_plb
.add_u64_counter(l_mdss_handle_inode_file_caps
, "handle_inode_file_caps",
3428 "Inter mds caps msg", "hifc", PerfCountersBuilder::PRIO_INTERESTING
);
3430 // useful dir/inode/subtree stats
3431 mds_plb
.set_prio_default(PerfCountersBuilder::PRIO_USEFUL
);
3432 mds_plb
.add_u64(l_mds_root_rfiles
, "root_rfiles", "root inode rfiles");
3433 mds_plb
.add_u64(l_mds_root_rbytes
, "root_rbytes", "root inode rbytes");
3434 mds_plb
.add_u64(l_mds_root_rsnaps
, "root_rsnaps", "root inode rsnaps");
3435 mds_plb
.add_u64_counter(l_mds_dir_fetch_complete
,
3436 "dir_fetch_complete", "Fetch complete dirfrag");
3437 mds_plb
.add_u64_counter(l_mds_dir_fetch_keys
,
3438 "dir_fetch_keys", "Fetch keys from dirfrag");
3439 mds_plb
.add_u64_counter(l_mds_dir_commit
, "dir_commit", "Directory commit");
3440 mds_plb
.add_u64_counter(l_mds_dir_split
, "dir_split", "Directory split");
3441 mds_plb
.add_u64_counter(l_mds_dir_merge
, "dir_merge", "Directory merge");
3442 mds_plb
.add_u64(l_mds_inodes_pinned
, "inodes_pinned", "Inodes pinned");
3443 mds_plb
.add_u64(l_mds_inodes_expired
, "inodes_expired", "Inodes expired");
3444 mds_plb
.add_u64(l_mds_inodes_with_caps
, "inodes_with_caps",
3445 "Inodes with capabilities");
3446 mds_plb
.add_u64(l_mds_subtrees
, "subtrees", "Subtrees");
3447 mds_plb
.add_u64(l_mds_load_cent
, "load_cent", "Load per cent");
3448 mds_plb
.add_u64_counter(l_mds_openino_dir_fetch
, "openino_dir_fetch",
3449 "OpenIno incomplete directory fetchings");
3452 mds_plb
.set_prio_default(PerfCountersBuilder::PRIO_DEBUGONLY
);
3453 mds_plb
.add_u64_counter(l_mds_reply
, "reply", "Replies");
3454 mds_plb
.add_u64(l_mds_inodes_top
, "inodes_top", "Inodes on top");
3455 mds_plb
.add_u64(l_mds_inodes_bottom
, "inodes_bottom", "Inodes on bottom");
3457 l_mds_inodes_pin_tail
, "inodes_pin_tail", "Inodes on pin tail");
3458 mds_plb
.add_u64_counter(l_mds_traverse
, "traverse", "Traverses");
3459 mds_plb
.add_u64_counter(l_mds_traverse_hit
, "traverse_hit", "Traverse hits");
3460 mds_plb
.add_u64_counter(l_mds_traverse_forward
, "traverse_forward",
3461 "Traverse forwards");
3462 mds_plb
.add_u64_counter(l_mds_traverse_discover
, "traverse_discover",
3463 "Traverse directory discovers");
3464 mds_plb
.add_u64_counter(l_mds_traverse_dir_fetch
, "traverse_dir_fetch",
3465 "Traverse incomplete directory content fetchings");
3466 mds_plb
.add_u64_counter(l_mds_traverse_remote_ino
, "traverse_remote_ino",
3467 "Traverse remote dentries");
3468 mds_plb
.add_u64_counter(l_mds_traverse_lock
, "traverse_lock",
3470 mds_plb
.add_u64(l_mds_dispatch_queue_len
, "q", "Dispatch queue length");
3471 mds_plb
.add_u64_counter(l_mds_exported
, "exported", "Exports");
3472 mds_plb
.add_u64_counter(l_mds_imported
, "imported", "Imports");
3473 mds_plb
.add_u64_counter(l_mds_openino_backtrace_fetch
, "openino_backtrace_fetch",
3474 "OpenIno backtrace fetchings");
3475 mds_plb
.add_u64_counter(l_mds_openino_peer_discover
, "openino_peer_discover",
3476 "OpenIno peer inode discovers");
3479 mds_plb
.add_u64(l_mds_scrub_backtrace_fetch
, "scrub_backtrace_fetch",
3480 "Scrub backtrace fetchings");
3481 mds_plb
.add_u64(l_mds_scrub_set_tag
, "scrub_set_tag",
3483 mds_plb
.add_u64(l_mds_scrub_backtrace_repaired
, "scrub_backtrace_repaired",
3484 "Scrub backtraces repaired");
3485 mds_plb
.add_u64(l_mds_scrub_inotable_repaired
, "scrub_inotable_repaired",
3486 "Scrub inotable repaired");
3487 mds_plb
.add_u64(l_mds_scrub_dir_inodes
, "scrub_dir_inodes",
3488 "Scrub directory inodes");
3489 mds_plb
.add_u64(l_mds_scrub_dir_base_inodes
, "scrub_dir_base_inodes",
3490 "Scrub directory base inodes");
3491 mds_plb
.add_u64(l_mds_scrub_dirfrag_rstats
, "scrub_dirfrag_rstats",
3492 "Scrub dirfrags rstates");
3493 mds_plb
.add_u64(l_mds_scrub_file_inodes
, "scrub_file_inodes",
3494 "Scrub file inodes");
3496 logger
= mds_plb
.create_perf_counters();
3497 g_ceph_context
->get_perfcounters_collection()->add(logger
);
3501 PerfCountersBuilder
mdm_plb(g_ceph_context
, "mds_mem", l_mdm_first
, l_mdm_last
);
3502 mdm_plb
.add_u64(l_mdm_ino
, "ino", "Inodes", "ino",
3503 PerfCountersBuilder::PRIO_INTERESTING
);
3504 mdm_plb
.add_u64(l_mdm_dn
, "dn", "Dentries", "dn",
3505 PerfCountersBuilder::PRIO_INTERESTING
);
3507 mdm_plb
.set_prio_default(PerfCountersBuilder::PRIO_USEFUL
);
3508 mdm_plb
.add_u64_counter(l_mdm_inoa
, "ino+", "Inodes opened");
3509 mdm_plb
.add_u64_counter(l_mdm_inos
, "ino-", "Inodes closed");
3510 mdm_plb
.add_u64(l_mdm_dir
, "dir", "Directories");
3511 mdm_plb
.add_u64_counter(l_mdm_dira
, "dir+", "Directories opened");
3512 mdm_plb
.add_u64_counter(l_mdm_dirs
, "dir-", "Directories closed");
3513 mdm_plb
.add_u64_counter(l_mdm_dna
, "dn+", "Dentries opened");
3514 mdm_plb
.add_u64_counter(l_mdm_dns
, "dn-", "Dentries closed");
3515 mdm_plb
.add_u64(l_mdm_cap
, "cap", "Capabilities");
3516 mdm_plb
.add_u64_counter(l_mdm_capa
, "cap+", "Capabilities added");
3517 mdm_plb
.add_u64_counter(l_mdm_caps
, "cap-", "Capabilities removed");
3518 mdm_plb
.add_u64(l_mdm_heap
, "heap", "Heap size");
3520 mdm_plb
.set_prio_default(PerfCountersBuilder::PRIO_DEBUGONLY
);
3521 mdm_plb
.add_u64(l_mdm_rss
, "rss", "RSS");
3523 mlogger
= mdm_plb
.create_perf_counters();
3524 g_ceph_context
->get_perfcounters_collection()->add(mlogger
);
3527 mdlog
->create_logger();
3528 server
->create_logger();
3529 purge_queue
.create_logger();
3530 sessionmap
.register_perfcounters();
3531 mdcache
->register_perfcounters();
3534 void MDSRank::check_ops_in_flight()
3537 vector
<string
> warnings
;
3539 if (op_tracker
.check_ops_in_flight(&summary
, warnings
, &slow
)) {
3540 clog
->warn() << summary
;
3541 for (const auto& warning
: warnings
) {
3542 clog
->warn() << warning
;
3546 // set mds slow request count
3547 mds_slow_req_count
= slow
;
3551 void MDSRankDispatcher::handle_osd_map()
3554 mdsmap
->get_tableserver() == whoami
) {
3555 snapserver
->check_osd_map(true);
3558 server
->handle_osd_map();
3560 purge_queue
.update_op_limit(*mdsmap
);
3562 // it's ok if replay state is reached via standby-replay, the
3563 // reconnect state will journal blocklisted clients (journal
3564 // is opened for writing in `replay_done` before moving to
3566 if (!is_any_replay()) {
3567 std::set
<entity_addr_t
> newly_blocklisted
;
3568 objecter
->consume_blocklist_events(&newly_blocklisted
);
3569 auto epoch
= objecter
->with_osdmap([](const OSDMap
&o
){return o
.get_epoch();});
3570 apply_blocklist(newly_blocklisted
, epoch
);
3573 // By default the objecter only requests OSDMap updates on use,
3574 // we would like to always receive the latest maps in order to
3575 // apply policy based on the FULL flag.
3576 objecter
->maybe_request_map();
3579 int MDSRank::config_client(int64_t session_id
, bool remove
,
3580 const std::string
& option
, const std::string
& value
,
3583 Session
*session
= sessionmap
.get_session(entity_name_t(CEPH_ENTITY_TYPE_CLIENT
, session_id
));
3585 ss
<< "session " << session_id
<< " not in sessionmap!";
3586 return -CEPHFS_ENOENT
;
3589 if (option
== "timeout") {
3591 auto it
= session
->info
.client_metadata
.find("timeout");
3592 if (it
== session
->info
.client_metadata
.end()) {
3593 ss
<< "Nonexistent config: " << option
;
3594 return -CEPHFS_ENODATA
;
3596 session
->info
.client_metadata
.erase(it
);
3599 strtoul(value
.c_str(), &end
, 0);
3601 ss
<< "Invalid config for timeout: " << value
;
3602 return -CEPHFS_EINVAL
;
3604 session
->info
.client_metadata
[option
] = value
;
3606 //sessionmap._mark_dirty(session, true);
3608 ss
<< "Invalid config option: " << option
;
3609 return -CEPHFS_EINVAL
;
3615 bool MDSRank::evict_client(int64_t session_id
,
3616 bool wait
, bool blocklist
, std::ostream
& err_ss
,
3619 ceph_assert(ceph_mutex_is_locked_by_me(mds_lock
));
3621 // Mutually exclusive args
3622 ceph_assert(!(wait
&& on_killed
!= nullptr));
3624 if (is_any_replay()) {
3625 err_ss
<< "MDS is replaying log";
3629 Session
*session
= sessionmap
.get_session(
3630 entity_name_t(CEPH_ENTITY_TYPE_CLIENT
, session_id
));
3632 err_ss
<< "session " << session_id
<< " not in sessionmap!";
3636 auto& addr
= session
->info
.inst
.addr
;
3638 CachedStackStringStream css
;
3639 *css
<< "Evicting " << (blocklist
? "(and blocklisting) " : "")
3640 << "client session " << session_id
<< " (" << addr
<< ")";
3641 dout(1) << css
->strv() << dendl
;
3642 clog
->info() << css
->strv();
3645 dout(4) << "Preparing blocklist command... (wait=" << wait
<< ")" << dendl
;
3646 CachedStackStringStream css
;
3647 *css
<< "{\"prefix\":\"osd blocklist\", \"blocklistop\":\"add\",";
3648 *css
<< "\"addr\":\"";
3651 std::vector
<std::string
> cmd
= {css
->str()};
3653 auto kill_client_session
= [this, session_id
, wait
, on_killed
](){
3654 ceph_assert(ceph_mutex_is_locked_by_me(mds_lock
));
3655 Session
*session
= sessionmap
.get_session(
3656 entity_name_t(CEPH_ENTITY_TYPE_CLIENT
, session_id
));
3658 if (on_killed
|| !wait
) {
3659 server
->kill_session(session
, on_killed
);
3661 C_SaferCond on_safe
;
3662 server
->kill_session(session
, &on_safe
);
3669 dout(1) << "session " << session_id
<< " was removed while we waited "
3670 "for blocklist" << dendl
;
3672 // Even though it wasn't us that removed it, kick our completion
3673 // as the session has been removed.
3675 on_killed
->complete(0);
3680 auto apply_blocklist
= [this, cmd
](std::function
<void ()> fn
){
3681 ceph_assert(ceph_mutex_is_locked_by_me(mds_lock
));
3683 Context
*on_blocklist_done
= new LambdaContext([this, fn
](int r
) {
3684 objecter
->wait_for_latest_osdmap(
3685 lambdafy((new C_OnFinisher(
3686 new LambdaContext([this, fn
](int r
) {
3687 std::lock_guard
l(mds_lock
);
3688 auto epoch
= objecter
->with_osdmap([](const OSDMap
&o
){
3689 return o
.get_epoch();
3692 set_osd_epoch_barrier(epoch
);
3699 dout(4) << "Sending mon blocklist command: " << cmd
[0] << dendl
;
3700 monc
->start_mon_command(cmd
, {}, nullptr, nullptr, on_blocklist_done
);
3705 C_SaferCond inline_ctx
;
3706 apply_blocklist([&inline_ctx
](){inline_ctx
.complete(0);});
3712 // We dropped mds_lock, so check that session still exists
3713 session
= sessionmap
.get_session(entity_name_t(CEPH_ENTITY_TYPE_CLIENT
,
3716 dout(1) << "session " << session_id
<< " was removed while we waited "
3717 "for blocklist" << dendl
;
3718 client_eviction_dump
= true;
3721 kill_client_session();
3724 apply_blocklist(kill_client_session
);
3726 kill_client_session();
3730 client_eviction_dump
= true;
3734 MDSRankDispatcher::MDSRankDispatcher(
3736 ceph::fair_mutex
&mds_lock_
,
3737 LogChannelRef
&clog_
,
3738 CommonSafeTimer
<ceph::fair_mutex
> &timer_
,
3740 std::unique_ptr
<MDSMap
> &mdsmap_
,
3744 Context
*respawn_hook_
,
3745 Context
*suicide_hook_
,
3746 boost::asio::io_context
& ioc
)
3747 : MDSRank(whoami_
, mds_lock_
, clog_
, timer_
, beacon_
, mdsmap_
,
3748 msgr
, monc_
, mgrc
, respawn_hook_
, suicide_hook_
, ioc
)
3750 g_conf().add_observer(this);
3753 void MDSRank::command_cache_drop(uint64_t timeout
, Formatter
*f
, Context
*on_finish
) {
3754 dout(20) << __func__
<< dendl
;
3756 std::lock_guard
locker(mds_lock
);
3757 C_Drop_Cache
*request
= new C_Drop_Cache(server
, mdcache
, mdlog
, this,
3758 timeout
, f
, on_finish
);
3762 epoch_t
MDSRank::get_osd_epoch() const
3764 return objecter
->with_osdmap(std::mem_fn(&OSDMap::get_epoch
));
3767 const char** MDSRankDispatcher::get_tracked_conf_keys() const
3769 static const char* KEYS
[] = {
3771 "clog_to_graylog_host",
3772 "clog_to_graylog_port",
3775 "clog_to_syslog_facility",
3776 "clog_to_syslog_level",
3779 "mds_bal_fragment_dirs",
3780 "mds_bal_fragment_interval",
3781 "mds_bal_fragment_size_max",
3782 "mds_cache_memory_limit",
3784 "mds_cache_reservation",
3785 "mds_cache_trim_decay_rate",
3786 "mds_cap_revoke_eviction_timeout",
3787 "mds_dump_cache_threshold_file",
3788 "mds_dump_cache_threshold_formatter",
3789 "mds_enable_op_tracker",
3790 "mds_export_ephemeral_random",
3791 "mds_export_ephemeral_random_max",
3792 "mds_export_ephemeral_distributed",
3793 "mds_health_cache_threshold",
3794 "mds_inject_migrator_session_race",
3796 "mds_max_export_size",
3797 "mds_max_purge_files",
3798 "mds_forward_all_requests_to_auth",
3799 "mds_max_purge_ops",
3800 "mds_max_purge_ops_per_pg",
3801 "mds_max_snaps_per_dir",
3802 "mds_op_complaint_time",
3803 "mds_op_history_duration",
3804 "mds_op_history_size",
3805 "mds_op_log_threshold",
3806 "mds_recall_max_decay_rate",
3807 "mds_recall_warning_decay_rate",
3808 "mds_request_load_average_decay_rate",
3809 "mds_session_cache_liveness_decay_rate",
3810 "mds_heartbeat_reset_grace",
3811 "mds_heartbeat_grace",
3812 "mds_session_cap_acquisition_decay_rate",
3813 "mds_max_caps_per_client",
3814 "mds_session_cap_acquisition_throttle",
3815 "mds_session_max_caps_throttle_ratio",
3816 "mds_cap_acquisition_throttle_retry_request_time",
3817 "mds_alternate_name_max",
3818 "mds_dir_max_entries",
3819 "mds_symlink_recovery",
3820 "mds_extraordinary_events_dump_interval",
3821 "mds_inject_rename_corrupt_dentry_first",
3822 "mds_inject_journal_corrupt_dentry_first",
3823 "mds_session_metadata_threshold",
3829 void MDSRankDispatcher::handle_conf_change(const ConfigProxy
& conf
, const std::set
<std::string
>& changed
)
3831 // XXX with or without mds_lock!
3833 if (changed
.count("mds_heartbeat_reset_grace")) {
3834 _heartbeat_reset_grace
= conf
.get_val
<uint64_t>("mds_heartbeat_reset_grace");
3836 if (changed
.count("mds_heartbeat_grace")) {
3837 heartbeat_grace
= conf
.get_val
<double>("mds_heartbeat_grace");
3839 if (changed
.count("mds_op_complaint_time") || changed
.count("mds_op_log_threshold")) {
3840 op_tracker
.set_complaint_and_threshold(conf
->mds_op_complaint_time
, conf
->mds_op_log_threshold
);
3842 if (changed
.count("mds_op_history_size") || changed
.count("mds_op_history_duration")) {
3843 op_tracker
.set_history_size_and_duration(conf
->mds_op_history_size
, conf
->mds_op_history_duration
);
3845 if (changed
.count("mds_enable_op_tracker")) {
3846 op_tracker
.set_tracking(conf
->mds_enable_op_tracker
);
3848 if (changed
.count("mds_extraordinary_events_dump_interval")) {
3849 reset_event_flags();
3850 extraordinary_events_dump_interval
= conf
.get_val
<std::chrono::seconds
>
3851 ("mds_extraordinary_events_dump_interval").count();
3853 //Enable the logging only during low level debugging
3854 if (extraordinary_events_dump_interval
) {
3855 uint64_t log_level
, gather_level
;
3856 std::string debug_mds
= g_conf().get_val
<std::string
>("debug_mds");
3857 auto delim
= debug_mds
.find("/");
3858 std::istringstream(debug_mds
.substr(0, delim
)) >> log_level
;
3859 std::istringstream(debug_mds
.substr(delim
+ 1)) >> gather_level
;
3861 if (log_level
< 10 && gather_level
>= 10) {
3862 dout(0) << __func__
<< " Enabling in-memory log dump..." << dendl
;
3863 std::scoped_lock
lock(mds_lock
);
3864 schedule_inmemory_logger();
3867 dout(0) << __func__
<< " Enabling in-memory log dump failed. debug_mds=" << log_level
3868 << "/" << gather_level
<< dendl
;
3869 extraordinary_events_dump_interval
= 0;
3873 //The user set mds_extraordinary_events_dump_interval = 0
3874 dout(0) << __func__
<< " In-memory log dump disabled" << dendl
;
3877 if (changed
.count("clog_to_monitors") ||
3878 changed
.count("clog_to_syslog") ||
3879 changed
.count("clog_to_syslog_level") ||
3880 changed
.count("clog_to_syslog_facility") ||
3881 changed
.count("clog_to_graylog") ||
3882 changed
.count("clog_to_graylog_host") ||
3883 changed
.count("clog_to_graylog_port") ||
3884 changed
.count("host") ||
3885 changed
.count("fsid")) {
3886 update_log_config();
3888 if (changed
.count("mds_inject_journal_corrupt_dentry_first")) {
3889 inject_journal_corrupt_dentry_first
= g_conf().get_val
<double>("mds_inject_journal_corrupt_dentry_first");
3892 finisher
->queue(new LambdaContext([this, changed
](int) {
3893 std::scoped_lock
lock(mds_lock
);
3895 dout(10) << "flushing conf change to components: " << changed
<< dendl
;
3897 if (changed
.count("mds_log_pause") && !g_conf()->mds_log_pause
) {
3898 mdlog
->kick_submitter();
3900 sessionmap
.handle_conf_change(changed
);
3901 server
->handle_conf_change(changed
);
3902 mdcache
->handle_conf_change(changed
, *mdsmap
);
3903 purge_queue
.handle_conf_change(changed
, *mdsmap
);
3907 void MDSRank::get_task_status(std::map
<std::string
, std::string
> *status
) {
3908 dout(20) << __func__
<< dendl
;
3910 // scrub summary for now..
3911 std::string_view scrub_summary
= scrubstack
->scrub_summary();
3912 if (!ScrubStack::is_idle(scrub_summary
)) {
3914 status
->emplace(SCRUB_STATUS_KEY
, std::move(scrub_summary
));
3918 void MDSRank::schedule_update_timer_task() {
3919 dout(20) << __func__
<< dendl
;
3921 timer
.add_event_after(g_conf().get_val
<double>("mds_task_status_update_interval"),
3922 new LambdaContext([this](int) {
3927 void MDSRank::send_task_status() {
3928 std::map
<std::string
, std::string
> status
;
3929 get_task_status(&status
);
3932 if (status
.empty()) {
3933 send_status
= false;
3936 dout(20) << __func__
<< ": updating " << status
.size() << " status keys" << dendl
;
3937 int r
= mgrc
->service_daemon_update_task_status(std::move(status
));
3939 derr
<< ": failed to update service daemon status: " << cpp_strerror(r
) << dendl
;
3944 schedule_update_timer_task();
3947 void MDSRank::schedule_inmemory_logger() {
3948 dout(20) << __func__
<< dendl
;
3949 timer
.add_event_after(extraordinary_events_dump_interval
,
3950 new LambdaContext([this]() {
3955 void MDSRank::inmemory_logger() {
3956 if (client_eviction_dump
||
3957 beacon
.missed_beacon_ack_dump
||
3958 beacon
.missed_internal_heartbeat_dump
) {
3959 //dump the in-memory logs if any of these events occured recently
3960 dout(0) << __func__
<< " client_eviction_dump "<< client_eviction_dump
3961 << ", missed_beacon_ack_dump " << beacon
.missed_beacon_ack_dump
3962 << ", missed_internal_heartbeat_dump " << beacon
.missed_internal_heartbeat_dump
3964 reset_event_flags();
3965 g_ceph_context
->_log
->dump_recent();
3968 //reschedule if it's enabled
3969 if (extraordinary_events_dump_interval
) {
3970 schedule_inmemory_logger();
3974 void MDSRank::reset_event_flags() {
3975 client_eviction_dump
= false;
3976 beacon
.missed_beacon_ack_dump
= false;
3977 beacon
.missed_internal_heartbeat_dump
= false;