1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
7 * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
8 * Copyright (C) 2014 Red Hat <contact@redhat.com>
10 * Author: Loic Dachary <loic@dachary.org>
12 * This is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU Lesser General Public
14 * License version 2.1, as published by the Free Software
15 * Foundation. See file COPYING.
20 #include <boost/algorithm/string.hpp>
24 #include "mon/OSDMonitor.h"
25 #include "mon/Monitor.h"
26 #include "mon/MDSMonitor.h"
27 #include "mon/PGMonitor.h"
28 #include "mon/MgrStatMonitor.h"
29 #include "mon/AuthMonitor.h"
30 #include "mon/ConfigKeyService.h"
32 #include "mon/MonitorDBStore.h"
33 #include "mon/Session.h"
35 #include "crush/CrushWrapper.h"
36 #include "crush/CrushTester.h"
37 #include "crush/CrushTreeDumper.h"
39 #include "messages/MOSDBeacon.h"
40 #include "messages/MOSDFailure.h"
41 #include "messages/MOSDMarkMeDown.h"
42 #include "messages/MOSDFull.h"
43 #include "messages/MOSDMap.h"
44 #include "messages/MMonGetOSDMap.h"
45 #include "messages/MOSDBoot.h"
46 #include "messages/MOSDAlive.h"
47 #include "messages/MPoolOp.h"
48 #include "messages/MPoolOpReply.h"
49 #include "messages/MOSDPGCreate.h"
50 #include "messages/MOSDPGCreated.h"
51 #include "messages/MOSDPGTemp.h"
52 #include "messages/MMonCommand.h"
53 #include "messages/MRemoveSnaps.h"
54 #include "messages/MOSDScrub.h"
55 #include "messages/MRoute.h"
57 #include "common/TextTable.h"
58 #include "common/Timer.h"
59 #include "common/ceph_argparse.h"
60 #include "common/perf_counters.h"
61 #include "common/strtol.h"
63 #include "common/config.h"
64 #include "common/errno.h"
66 #include "erasure-code/ErasureCodePlugin.h"
67 #include "compressor/Compressor.h"
68 #include "common/Checksummer.h"
70 #include "include/compat.h"
71 #include "include/assert.h"
72 #include "include/stringify.h"
73 #include "include/util.h"
74 #include "common/cmdparse.h"
75 #include "include/str_list.h"
76 #include "include/str_map.h"
77 #include "include/scope_guard.h"
79 #include "json_spirit/json_spirit_reader.h"
81 #include <boost/algorithm/string/predicate.hpp>
83 #define dout_subsys ceph_subsys_mon
84 #define OSD_PG_CREATING_PREFIX "osd_pg_creating"
88 const uint32_t MAX_POOL_APPLICATIONS
= 4;
89 const uint32_t MAX_POOL_APPLICATION_KEYS
= 64;
90 const uint32_t MAX_POOL_APPLICATION_LENGTH
= 128;
92 } // anonymous namespace
94 void LastEpochClean::Lec::report(ps_t ps
, epoch_t last_epoch_clean
)
96 if (epoch_by_pg
.size() <= ps
) {
97 epoch_by_pg
.resize(ps
+ 1, 0);
99 const auto old_lec
= epoch_by_pg
[ps
];
100 if (old_lec
>= last_epoch_clean
) {
104 epoch_by_pg
[ps
] = last_epoch_clean
;
105 if (last_epoch_clean
< floor
) {
106 floor
= last_epoch_clean
;
107 } else if (last_epoch_clean
> floor
) {
108 if (old_lec
== floor
) {
109 // probably should increase floor?
110 auto new_floor
= std::min_element(std::begin(epoch_by_pg
),
111 std::end(epoch_by_pg
));
115 if (ps
!= next_missing
) {
118 for (; next_missing
< epoch_by_pg
.size(); next_missing
++) {
119 if (epoch_by_pg
[next_missing
] == 0) {
125 void LastEpochClean::remove_pool(uint64_t pool
)
127 report_by_pool
.erase(pool
);
130 void LastEpochClean::report(const pg_t
& pg
, epoch_t last_epoch_clean
)
132 auto& lec
= report_by_pool
[pg
.pool()];
133 return lec
.report(pg
.ps(), last_epoch_clean
);
136 epoch_t
LastEpochClean::get_lower_bound(const OSDMap
& latest
) const
138 auto floor
= latest
.get_epoch();
139 for (auto& pool
: latest
.get_pools()) {
140 auto reported
= report_by_pool
.find(pool
.first
);
141 if (reported
== report_by_pool
.end()) {
144 if (reported
->second
.next_missing
< pool
.second
.get_pg_num()) {
147 if (reported
->second
.floor
< floor
) {
148 floor
= reported
->second
.floor
;
155 struct C_UpdateCreatingPGs
: public Context
{
159 C_UpdateCreatingPGs(OSDMonitor
*osdmon
, epoch_t e
) :
160 osdmon(osdmon
), start(ceph_clock_now()), epoch(e
) {}
161 void finish(int r
) override
{
163 utime_t end
= ceph_clock_now();
164 dout(10) << "osdmap epoch " << epoch
<< " mapping took "
165 << (end
- start
) << " seconds" << dendl
;
166 osdmon
->update_creating_pgs();
167 osdmon
->check_pg_creates_subs();
173 #define dout_prefix _prefix(_dout, mon, osdmap)
174 static ostream
& _prefix(std::ostream
*_dout
, Monitor
*mon
, const OSDMap
& osdmap
) {
175 return *_dout
<< "mon." << mon
->name
<< "@" << mon
->rank
176 << "(" << mon
->get_state_name()
177 << ").osd e" << osdmap
.get_epoch() << " ";
180 OSDMonitor::OSDMonitor(
184 const string
& service_name
)
185 : PaxosService(mn
, p
, service_name
),
187 inc_osd_cache(g_conf
->mon_osd_cache_size
),
188 full_osd_cache(g_conf
->mon_osd_cache_size
),
189 last_attempted_minwait_time(utime_t()),
190 mapper(mn
->cct
, &mn
->cpu_tp
),
191 op_tracker(cct
, true, 1)
194 bool OSDMonitor::_have_pending_crush()
196 return pending_inc
.crush
.length() > 0;
199 CrushWrapper
&OSDMonitor::_get_stable_crush()
201 return *osdmap
.crush
;
204 void OSDMonitor::_get_pending_crush(CrushWrapper
& newcrush
)
207 if (pending_inc
.crush
.length())
208 bl
= pending_inc
.crush
;
210 osdmap
.crush
->encode(bl
, CEPH_FEATURES_SUPPORTED_DEFAULT
);
212 bufferlist::iterator p
= bl
.begin();
216 void OSDMonitor::create_initial()
218 dout(10) << "create_initial for " << mon
->monmap
->fsid
<< dendl
;
223 mon
->store
->get("mkfs", "osdmap", bl
);
227 newmap
.set_fsid(mon
->monmap
->fsid
);
229 newmap
.build_simple(g_ceph_context
, 0, mon
->monmap
->fsid
, 0);
232 newmap
.created
= newmap
.modified
= ceph_clock_now();
234 // new clusters should sort bitwise by default.
235 newmap
.set_flag(CEPH_OSDMAP_SORTBITWISE
);
237 // new cluster should require latest by default
238 if (g_conf
->mon_debug_no_require_luminous
) {
239 newmap
.require_osd_release
= CEPH_RELEASE_KRAKEN
;
240 derr
<< __func__
<< " mon_debug_no_require_luminous=true" << dendl
;
242 newmap
.require_osd_release
= CEPH_RELEASE_LUMINOUS
;
243 newmap
.flags
|= CEPH_OSDMAP_RECOVERY_DELETES
;
244 newmap
.full_ratio
= g_conf
->mon_osd_full_ratio
;
245 if (newmap
.full_ratio
> 1.0) newmap
.full_ratio
/= 100;
246 newmap
.backfillfull_ratio
= g_conf
->mon_osd_backfillfull_ratio
;
247 if (newmap
.backfillfull_ratio
> 1.0) newmap
.backfillfull_ratio
/= 100;
248 newmap
.nearfull_ratio
= g_conf
->mon_osd_nearfull_ratio
;
249 if (newmap
.nearfull_ratio
> 1.0) newmap
.nearfull_ratio
/= 100;
250 int r
= ceph_release_from_name(
251 g_conf
->mon_osd_initial_require_min_compat_client
.c_str());
253 assert(0 == "mon_osd_initial_require_min_compat_client is not valid");
255 newmap
.require_min_compat_client
= r
;
258 // encode into pending incremental
259 newmap
.encode(pending_inc
.fullmap
,
260 mon
->get_quorum_con_features() | CEPH_FEATURE_RESERVED
);
261 pending_inc
.full_crc
= newmap
.get_crc();
262 dout(20) << " full crc " << pending_inc
.full_crc
<< dendl
;
265 void OSDMonitor::get_store_prefixes(std::set
<string
>& s
)
267 s
.insert(service_name
);
268 s
.insert(OSD_PG_CREATING_PREFIX
);
271 void OSDMonitor::update_from_paxos(bool *need_bootstrap
)
273 version_t version
= get_last_committed();
274 if (version
== osdmap
.epoch
)
276 assert(version
> osdmap
.epoch
);
278 dout(15) << "update_from_paxos paxos e " << version
279 << ", my e " << osdmap
.epoch
<< dendl
;
282 if (!mapping_job
->is_done()) {
283 dout(1) << __func__
<< " mapping job "
284 << mapping_job
.get() << " did not complete, "
285 << mapping_job
->shards
<< " left, canceling" << dendl
;
286 mapping_job
->abort();
294 * We will possibly have a stashed latest that *we* wrote, and we will
295 * always be sure to have the oldest full map in the first..last range
296 * due to encode_trim_extra(), which includes the oldest full map in the trim
299 * encode_trim_extra() does not however write the full map's
300 * version to 'full_latest'. This is only done when we are building the
301 * full maps from the incremental versions. But don't panic! We make sure
302 * that the following conditions find whichever full map version is newer.
304 version_t latest_full
= get_version_latest_full();
305 if (latest_full
== 0 && get_first_committed() > 1)
306 latest_full
= get_first_committed();
308 if (get_first_committed() > 1 &&
309 latest_full
< get_first_committed()) {
310 // the monitor could be just sync'ed with its peer, and the latest_full key
311 // is not encoded in the paxos commits in encode_pending(), so we need to
312 // make sure we get it pointing to a proper version.
313 version_t lc
= get_last_committed();
314 version_t fc
= get_first_committed();
316 dout(10) << __func__
<< " looking for valid full map in interval"
317 << " [" << fc
<< ", " << lc
<< "]" << dendl
;
320 for (version_t v
= lc
; v
>= fc
; v
--) {
321 string full_key
= "full_" + stringify(v
);
322 if (mon
->store
->exists(get_service_name(), full_key
)) {
323 dout(10) << __func__
<< " found latest full map v " << v
<< dendl
;
329 assert(latest_full
> 0);
330 auto t(std::make_shared
<MonitorDBStore::Transaction
>());
331 put_version_latest_full(t
, latest_full
);
332 mon
->store
->apply_transaction(t
);
333 dout(10) << __func__
<< " updated the on-disk full map version to "
334 << latest_full
<< dendl
;
337 if ((latest_full
> 0) && (latest_full
> osdmap
.epoch
)) {
338 bufferlist latest_bl
;
339 get_version_full(latest_full
, latest_bl
);
340 assert(latest_bl
.length() != 0);
341 dout(7) << __func__
<< " loading latest full map e" << latest_full
<< dendl
;
342 osdmap
.decode(latest_bl
);
345 if (mon
->monmap
->get_required_features().contains_all(
346 ceph::features::mon::FEATURE_LUMINOUS
)) {
348 if (!mon
->store
->get(OSD_PG_CREATING_PREFIX
, "creating", bl
)) {
350 std::lock_guard
<std::mutex
> l(creating_pgs_lock
);
351 creating_pgs
.decode(p
);
352 dout(7) << __func__
<< " loading creating_pgs last_scan_epoch "
353 << creating_pgs
.last_scan_epoch
354 << " with " << creating_pgs
.pgs
.size() << " pgs" << dendl
;
356 dout(1) << __func__
<< " missing creating pgs; upgrade from post-kraken?"
361 // make sure we're using the right pg service.. remove me post-luminous!
362 if (osdmap
.require_osd_release
>= CEPH_RELEASE_LUMINOUS
) {
363 dout(10) << __func__
<< " pgservice is mgrstat" << dendl
;
364 mon
->pgservice
= mon
->mgrstatmon()->get_pg_stat_service();
366 dout(10) << __func__
<< " pgservice is pg" << dendl
;
367 mon
->pgservice
= mon
->pgmon()->get_pg_stat_service();
370 // walk through incrementals
371 MonitorDBStore::TransactionRef t
;
373 while (version
> osdmap
.epoch
) {
375 int err
= get_version(osdmap
.epoch
+1, inc_bl
);
377 assert(inc_bl
.length());
379 dout(7) << "update_from_paxos applying incremental " << osdmap
.epoch
+1
381 OSDMap::Incremental
inc(inc_bl
);
382 err
= osdmap
.apply_incremental(inc
);
386 t
.reset(new MonitorDBStore::Transaction
);
388 // Write out the full map for all past epochs. Encode the full
389 // map with the same features as the incremental. If we don't
390 // know, use the quorum features. If we don't know those either,
391 // encode with all features.
392 uint64_t f
= inc
.encode_features
;
394 f
= mon
->get_quorum_con_features();
398 osdmap
.encode(full_bl
, f
| CEPH_FEATURE_RESERVED
);
399 tx_size
+= full_bl
.length();
401 bufferlist orig_full_bl
;
402 get_version_full(osdmap
.epoch
, orig_full_bl
);
403 if (orig_full_bl
.length()) {
404 // the primary provided the full map
405 assert(inc
.have_crc
);
406 if (inc
.full_crc
!= osdmap
.crc
) {
407 // This will happen if the mons were running mixed versions in
408 // the past or some other circumstance made the full encoded
409 // maps divergent. Reloading here will bring us back into
410 // sync with the primary for this and all future maps. OSDs
411 // will also be brought back into sync when they discover the
412 // crc mismatch and request a full map from a mon.
413 derr
<< __func__
<< " full map CRC mismatch, resetting to canonical"
416 osdmap
.decode(orig_full_bl
);
419 assert(!inc
.have_crc
);
420 put_version_full(t
, osdmap
.epoch
, full_bl
);
422 put_version_latest_full(t
, osdmap
.epoch
);
425 dout(1) << osdmap
<< dendl
;
427 if (osdmap
.epoch
== 1) {
428 t
->erase("mkfs", "osdmap");
431 // make sure we're using the right pg service.. remove me post-luminous!
432 if (osdmap
.require_osd_release
>= CEPH_RELEASE_LUMINOUS
) {
433 dout(10) << __func__
<< " pgservice is mgrstat" << dendl
;
434 mon
->pgservice
= mon
->mgrstatmon()->get_pg_stat_service();
436 dout(10) << __func__
<< " pgservice is pg" << dendl
;
437 mon
->pgservice
= mon
->pgmon()->get_pg_stat_service();
440 if (tx_size
> g_conf
->mon_sync_max_payload_size
*2) {
441 mon
->store
->apply_transaction(t
);
442 t
= MonitorDBStore::TransactionRef();
445 if (mon
->monmap
->get_required_features().contains_all(
446 ceph::features::mon::FEATURE_LUMINOUS
)) {
447 for (const auto &osd_state
: inc
.new_state
) {
448 if (osd_state
.second
& CEPH_OSD_UP
) {
449 // could be marked up *or* down, but we're too lazy to check which
450 last_osd_report
.erase(osd_state
.first
);
452 if (osd_state
.second
& CEPH_OSD_EXISTS
) {
453 // could be created *or* destroyed, but we can safely drop it
454 osd_epochs
.erase(osd_state
.first
);
461 mon
->store
->apply_transaction(t
);
464 for (int o
= 0; o
< osdmap
.get_max_osd(); o
++) {
465 if (osdmap
.is_out(o
))
467 auto found
= down_pending_out
.find(o
);
468 if (osdmap
.is_down(o
)) {
469 // populate down -> out map
470 if (found
== down_pending_out
.end()) {
471 dout(10) << " adding osd." << o
<< " to down_pending_out map" << dendl
;
472 down_pending_out
[o
] = ceph_clock_now();
475 if (found
!= down_pending_out
.end()) {
476 dout(10) << " removing osd." << o
<< " from down_pending_out map" << dendl
;
477 down_pending_out
.erase(found
);
481 // XXX: need to trim MonSession connected with a osd whose id > max_osd?
483 if (mon
->is_leader()) {
484 // kick pgmon, make sure it's seen the latest map
485 mon
->pgmon()->check_osd_map(osdmap
.epoch
);
489 check_pg_creates_subs();
491 share_map_with_random_osd();
496 // make sure our feature bits reflect the latest map
497 update_msgr_features();
499 if (!mon
->is_leader()) {
500 // will be called by on_active() on the leader, avoid doing so twice
505 void OSDMonitor::start_mapping()
507 // initiate mapping job
509 dout(10) << __func__
<< " canceling previous mapping_job " << mapping_job
.get()
511 mapping_job
->abort();
513 if (!osdmap
.get_pools().empty()) {
514 auto fin
= new C_UpdateCreatingPGs(this, osdmap
.get_epoch());
515 mapping_job
= mapping
.start_update(osdmap
, mapper
,
516 g_conf
->mon_osd_mapping_pgs_per_chunk
);
517 dout(10) << __func__
<< " started mapping job " << mapping_job
.get()
518 << " at " << fin
->start
<< dendl
;
519 mapping_job
->set_finish_event(fin
);
521 dout(10) << __func__
<< " no pools, no mapping job" << dendl
;
522 mapping_job
= nullptr;
526 void OSDMonitor::update_msgr_features()
529 types
.insert((int)entity_name_t::TYPE_OSD
);
530 types
.insert((int)entity_name_t::TYPE_CLIENT
);
531 types
.insert((int)entity_name_t::TYPE_MDS
);
532 types
.insert((int)entity_name_t::TYPE_MON
);
533 for (set
<int>::iterator q
= types
.begin(); q
!= types
.end(); ++q
) {
535 uint64_t features
= osdmap
.get_features(*q
, &mask
);
536 if ((mon
->messenger
->get_policy(*q
).features_required
& mask
) != features
) {
537 dout(0) << "crush map has features " << features
<< ", adjusting msgr requires" << dendl
;
538 Messenger::Policy p
= mon
->messenger
->get_policy(*q
);
539 p
.features_required
= (p
.features_required
& ~mask
) | features
;
540 mon
->messenger
->set_policy(*q
, p
);
545 void OSDMonitor::on_active()
549 if (mon
->is_leader()) {
550 mon
->clog
->debug() << "osdmap " << osdmap
;
552 list
<MonOpRequestRef
> ls
;
553 take_all_failures(ls
);
554 while (!ls
.empty()) {
555 MonOpRequestRef op
= ls
.front();
556 op
->mark_osdmon_event(__func__
);
564 void OSDMonitor::on_restart()
566 last_osd_report
.clear();
568 if (mon
->is_leader()) {
569 // fix ruleset != ruleid
570 if (osdmap
.crush
->has_legacy_rulesets() &&
571 !osdmap
.crush
->has_multirule_rulesets()) {
572 CrushWrapper newcrush
;
573 _get_pending_crush(newcrush
);
574 int r
= newcrush
.renumber_rules_by_ruleset();
576 dout(1) << __func__
<< " crush map has ruleset != rule id; fixing" << dendl
;
577 pending_inc
.crush
.clear();
578 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
580 dout(10) << __func__
<< " unable to renumber rules by ruleset" << dendl
;
586 void OSDMonitor::on_shutdown()
588 dout(10) << __func__
<< dendl
;
590 dout(10) << __func__
<< " canceling previous mapping_job " << mapping_job
.get()
592 mapping_job
->abort();
595 // discard failure info, waiters
596 list
<MonOpRequestRef
> ls
;
597 take_all_failures(ls
);
601 void OSDMonitor::update_logger()
603 dout(10) << "update_logger" << dendl
;
605 mon
->cluster_logger
->set(l_cluster_num_osd
, osdmap
.get_num_osds());
606 mon
->cluster_logger
->set(l_cluster_num_osd_up
, osdmap
.get_num_up_osds());
607 mon
->cluster_logger
->set(l_cluster_num_osd_in
, osdmap
.get_num_in_osds());
608 mon
->cluster_logger
->set(l_cluster_osd_epoch
, osdmap
.get_epoch());
611 void OSDMonitor::create_pending()
613 pending_inc
= OSDMap::Incremental(osdmap
.epoch
+1);
614 pending_inc
.fsid
= mon
->monmap
->fsid
;
616 dout(10) << "create_pending e " << pending_inc
.epoch
<< dendl
;
618 // clean up pg_temp, primary_temp
619 OSDMap::clean_temps(g_ceph_context
, osdmap
, &pending_inc
);
620 dout(10) << "create_pending did clean_temps" << dendl
;
622 // On upgrade OSDMap has new field set by mon_osd_backfillfull_ratio config
623 // instead of osd_backfill_full_ratio config
624 if (osdmap
.backfillfull_ratio
<= 0) {
625 pending_inc
.new_backfillfull_ratio
= g_conf
->mon_osd_backfillfull_ratio
;
626 if (pending_inc
.new_backfillfull_ratio
> 1.0)
627 pending_inc
.new_backfillfull_ratio
/= 100;
628 dout(1) << __func__
<< " setting backfillfull_ratio = "
629 << pending_inc
.new_backfillfull_ratio
<< dendl
;
631 if (osdmap
.get_epoch() > 0 &&
632 osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
633 // transition full ratios from PGMap to OSDMap (on upgrade)
634 float full_ratio
= mon
->pgservice
->get_full_ratio();
635 float nearfull_ratio
= mon
->pgservice
->get_nearfull_ratio();
636 if (osdmap
.full_ratio
!= full_ratio
) {
637 dout(10) << __func__
<< " full_ratio " << osdmap
.full_ratio
638 << " -> " << full_ratio
<< " (from pgmap)" << dendl
;
639 pending_inc
.new_full_ratio
= full_ratio
;
641 if (osdmap
.nearfull_ratio
!= nearfull_ratio
) {
642 dout(10) << __func__
<< " nearfull_ratio " << osdmap
.nearfull_ratio
643 << " -> " << nearfull_ratio
<< " (from pgmap)" << dendl
;
644 pending_inc
.new_nearfull_ratio
= nearfull_ratio
;
647 // safety check (this shouldn't really happen)
648 if (osdmap
.full_ratio
<= 0) {
649 pending_inc
.new_full_ratio
= g_conf
->mon_osd_full_ratio
;
650 if (pending_inc
.new_full_ratio
> 1.0)
651 pending_inc
.new_full_ratio
/= 100;
652 dout(1) << __func__
<< " setting full_ratio = "
653 << pending_inc
.new_full_ratio
<< dendl
;
655 if (osdmap
.nearfull_ratio
<= 0) {
656 pending_inc
.new_nearfull_ratio
= g_conf
->mon_osd_nearfull_ratio
;
657 if (pending_inc
.new_nearfull_ratio
> 1.0)
658 pending_inc
.new_nearfull_ratio
/= 100;
659 dout(1) << __func__
<< " setting nearfull_ratio = "
660 << pending_inc
.new_nearfull_ratio
<< dendl
;
666 OSDMonitor::update_pending_pgs(const OSDMap::Incremental
& inc
)
668 dout(10) << __func__
<< dendl
;
669 creating_pgs_t pending_creatings
;
671 std::lock_guard
<std::mutex
> l(creating_pgs_lock
);
672 pending_creatings
= creating_pgs
;
674 // check for new or old pools
675 if (pending_creatings
.last_scan_epoch
< inc
.epoch
) {
676 if (osdmap
.get_epoch() &&
677 osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
679 mon
->pgservice
->maybe_add_creating_pgs(creating_pgs
.last_scan_epoch
,
682 dout(7) << __func__
<< " " << added
<< " pgs added from pgmap" << dendl
;
685 queued
+= scan_for_creating_pgs(osdmap
.get_pools(),
689 queued
+= scan_for_creating_pgs(inc
.new_pools
,
693 dout(10) << __func__
<< " " << queued
<< " pools queued" << dendl
;
694 for (auto deleted_pool
: inc
.old_pools
) {
695 auto removed
= pending_creatings
.remove_pool(deleted_pool
);
696 dout(10) << __func__
<< " " << removed
697 << " pg removed because containing pool deleted: "
698 << deleted_pool
<< dendl
;
699 last_epoch_clean
.remove_pool(deleted_pool
);
701 // pgmon updates its creating_pgs in check_osd_map() which is called by
702 // on_active() and check_osd_map() could be delayed if lease expires, so its
703 // creating_pgs could be stale in comparison with the one of osdmon. let's
704 // trim them here. otherwise, they will be added back after being erased.
705 unsigned removed
= 0;
706 for (auto& pg
: pending_created_pgs
) {
707 dout(20) << __func__
<< " noting created pg " << pg
<< dendl
;
708 pending_creatings
.created_pools
.insert(pg
.pool());
709 removed
+= pending_creatings
.pgs
.erase(pg
);
711 pending_created_pgs
.clear();
712 dout(10) << __func__
<< " " << removed
713 << " pgs removed because they're created" << dendl
;
714 pending_creatings
.last_scan_epoch
= osdmap
.get_epoch();
718 unsigned max
= MAX(1, g_conf
->mon_osd_max_creating_pgs
);
719 const auto total
= pending_creatings
.pgs
.size();
720 while (pending_creatings
.pgs
.size() < max
&&
721 !pending_creatings
.queue
.empty()) {
722 auto p
= pending_creatings
.queue
.begin();
723 int64_t poolid
= p
->first
;
724 dout(10) << __func__
<< " pool " << poolid
725 << " created " << p
->second
.created
726 << " modified " << p
->second
.modified
727 << " [" << p
->second
.start
<< "-" << p
->second
.end
<< ")"
729 int n
= MIN(max
- pending_creatings
.pgs
.size(),
730 p
->second
.end
- p
->second
.start
);
731 ps_t first
= p
->second
.start
;
732 ps_t end
= first
+ n
;
733 for (ps_t ps
= first
; ps
< end
; ++ps
) {
734 const pg_t pgid
{ps
, static_cast<uint64_t>(poolid
)};
735 // NOTE: use the *current* epoch as the PG creation epoch so that the
736 // OSD does not have to generate a long set of PastIntervals.
737 pending_creatings
.pgs
.emplace(pgid
, make_pair(inc
.epoch
,
738 p
->second
.modified
));
739 dout(10) << __func__
<< " adding " << pgid
<< dendl
;
741 p
->second
.start
= end
;
742 if (p
->second
.done()) {
743 dout(10) << __func__
<< " done with queue for " << poolid
<< dendl
;
744 pending_creatings
.queue
.erase(p
);
746 dout(10) << __func__
<< " pool " << poolid
747 << " now [" << p
->second
.start
<< "-" << p
->second
.end
<< ")"
751 dout(10) << __func__
<< " queue remaining: " << pending_creatings
.queue
.size()
752 << " pools" << dendl
;
754 << " " << (pending_creatings
.pgs
.size() - total
)
755 << "/" << pending_creatings
.pgs
.size()
756 << " pgs added from queued pools" << dendl
;
757 return pending_creatings
;
760 void OSDMonitor::maybe_prime_pg_temp()
763 if (pending_inc
.crush
.length()) {
764 dout(10) << __func__
<< " new crush map, all" << dendl
;
768 if (!pending_inc
.new_up_client
.empty()) {
769 dout(10) << __func__
<< " new up osds, all" << dendl
;
773 // check for interesting OSDs
775 for (auto p
= pending_inc
.new_state
.begin();
776 !all
&& p
!= pending_inc
.new_state
.end();
778 if ((p
->second
& CEPH_OSD_UP
) &&
779 osdmap
.is_up(p
->first
)) {
780 osds
.insert(p
->first
);
783 for (map
<int32_t,uint32_t>::iterator p
= pending_inc
.new_weight
.begin();
784 !all
&& p
!= pending_inc
.new_weight
.end();
786 if (p
->second
< osdmap
.get_weight(p
->first
)) {
788 osds
.insert(p
->first
);
790 dout(10) << __func__
<< " osd." << p
->first
<< " weight increase, all"
796 if (!all
&& osds
.empty())
801 mapping
.get_osd_acting_pgs(*osds
.begin()).size() * osds
.size();
802 if (estimate
> mapping
.get_num_pgs() *
803 g_conf
->mon_osd_prime_pg_temp_max_estimate
) {
804 dout(10) << __func__
<< " estimate " << estimate
<< " pgs on "
805 << osds
.size() << " osds >= "
806 << g_conf
->mon_osd_prime_pg_temp_max_estimate
<< " of total "
807 << mapping
.get_num_pgs() << " pgs, all"
811 dout(10) << __func__
<< " estimate " << estimate
<< " pgs on "
812 << osds
.size() << " osds" << dendl
;
817 next
.deepish_copy_from(osdmap
);
818 next
.apply_incremental(pending_inc
);
820 if (next
.get_pools().empty()) {
821 dout(10) << __func__
<< " no pools, no pg_temp priming" << dendl
;
823 PrimeTempJob
job(next
, this);
824 mapper
.queue(&job
, g_conf
->mon_osd_mapping_pgs_per_chunk
);
825 if (job
.wait_for(g_conf
->mon_osd_prime_pg_temp_max_time
)) {
826 dout(10) << __func__
<< " done in " << job
.get_duration() << dendl
;
828 dout(10) << __func__
<< " did not finish in "
829 << g_conf
->mon_osd_prime_pg_temp_max_time
830 << ", stopping" << dendl
;
834 dout(10) << __func__
<< " " << osds
.size() << " interesting osds" << dendl
;
835 utime_t stop
= ceph_clock_now();
836 stop
+= g_conf
->mon_osd_prime_pg_temp_max_time
;
837 const int chunk
= 1000;
839 std::unordered_set
<pg_t
> did_pgs
;
840 for (auto osd
: osds
) {
841 auto& pgs
= mapping
.get_osd_acting_pgs(osd
);
842 dout(20) << __func__
<< " osd." << osd
<< " " << pgs
<< dendl
;
843 for (auto pgid
: pgs
) {
844 if (!did_pgs
.insert(pgid
).second
) {
847 prime_pg_temp(next
, pgid
);
850 if (ceph_clock_now() > stop
) {
851 dout(10) << __func__
<< " consumed more than "
852 << g_conf
->mon_osd_prime_pg_temp_max_time
853 << " seconds, stopping"
863 void OSDMonitor::prime_pg_temp(
867 if (mon
->monmap
->get_required_features().contains_all(
868 ceph::features::mon::FEATURE_LUMINOUS
)) {
869 // TODO: remove this creating_pgs direct access?
870 if (creating_pgs
.pgs
.count(pgid
)) {
874 if (mon
->pgservice
->is_creating_pg(pgid
)) {
878 if (!osdmap
.pg_exists(pgid
)) {
882 vector
<int> up
, acting
;
883 mapping
.get(pgid
, &up
, nullptr, &acting
, nullptr);
885 vector
<int> next_up
, next_acting
;
886 int next_up_primary
, next_acting_primary
;
887 next
.pg_to_up_acting_osds(pgid
, &next_up
, &next_up_primary
,
888 &next_acting
, &next_acting_primary
);
889 if (acting
== next_acting
&& next_up
!= next_acting
)
890 return; // no change since last epoch
893 return; // if previously empty now we can be no worse off
894 const pg_pool_t
*pool
= next
.get_pg_pool(pgid
.pool());
895 if (pool
&& acting
.size() < pool
->min_size
)
896 return; // can be no worse off than before
898 if (next_up
== next_acting
) {
900 dout(20) << __func__
<< "next_up === next_acting now, clear pg_temp"
904 dout(20) << __func__
<< " " << pgid
<< " " << up
<< "/" << acting
905 << " -> " << next_up
<< "/" << next_acting
906 << ", priming " << acting
909 Mutex::Locker
l(prime_pg_temp_lock
);
910 // do not touch a mapping if a change is pending
911 pending_inc
.new_pg_temp
.emplace(
913 mempool::osdmap::vector
<int>(acting
.begin(), acting
.end()));
918 * @note receiving a transaction in this function gives a fair amount of
919 * freedom to the service implementation if it does need it. It shouldn't.
921 void OSDMonitor::encode_pending(MonitorDBStore::TransactionRef t
)
923 dout(10) << "encode_pending e " << pending_inc
.epoch
926 // finalize up pending_inc
927 pending_inc
.modified
= ceph_clock_now();
929 int r
= pending_inc
.propagate_snaps_to_tiers(g_ceph_context
, osdmap
);
933 if (!mapping_job
->is_done()) {
934 dout(1) << __func__
<< " skipping prime_pg_temp; mapping job "
935 << mapping_job
.get() << " did not complete, "
936 << mapping_job
->shards
<< " left" << dendl
;
937 mapping_job
->abort();
938 } else if (mapping
.get_epoch() < osdmap
.get_epoch()) {
939 dout(1) << __func__
<< " skipping prime_pg_temp; mapping job "
940 << mapping_job
.get() << " is prior epoch "
941 << mapping
.get_epoch() << dendl
;
943 if (g_conf
->mon_osd_prime_pg_temp
) {
944 maybe_prime_pg_temp();
947 } else if (g_conf
->mon_osd_prime_pg_temp
) {
948 dout(1) << __func__
<< " skipping prime_pg_temp; mapping job did not start"
953 // ensure we don't have blank new_state updates. these are interrpeted as
954 // CEPH_OSD_UP (and almost certainly not what we want!).
955 auto p
= pending_inc
.new_state
.begin();
956 while (p
!= pending_inc
.new_state
.end()) {
957 if (p
->second
== 0) {
958 dout(10) << "new_state for osd." << p
->first
<< " is 0, removing" << dendl
;
959 p
= pending_inc
.new_state
.erase(p
);
969 tmp
.deepish_copy_from(osdmap
);
970 tmp
.apply_incremental(pending_inc
);
972 if (tmp
.require_osd_release
>= CEPH_RELEASE_LUMINOUS
) {
973 // set or clear full/nearfull?
974 int full
, backfill
, nearfull
;
975 tmp
.count_full_nearfull_osds(&full
, &backfill
, &nearfull
);
977 if (!tmp
.test_flag(CEPH_OSDMAP_FULL
)) {
978 dout(10) << __func__
<< " setting full flag" << dendl
;
979 add_flag(CEPH_OSDMAP_FULL
);
980 remove_flag(CEPH_OSDMAP_NEARFULL
);
983 if (tmp
.test_flag(CEPH_OSDMAP_FULL
)) {
984 dout(10) << __func__
<< " clearing full flag" << dendl
;
985 remove_flag(CEPH_OSDMAP_FULL
);
988 if (!tmp
.test_flag(CEPH_OSDMAP_NEARFULL
)) {
989 dout(10) << __func__
<< " setting nearfull flag" << dendl
;
990 add_flag(CEPH_OSDMAP_NEARFULL
);
993 if (tmp
.test_flag(CEPH_OSDMAP_NEARFULL
)) {
994 dout(10) << __func__
<< " clearing nearfull flag" << dendl
;
995 remove_flag(CEPH_OSDMAP_NEARFULL
);
1000 // min_compat_client?
1001 if (tmp
.require_min_compat_client
== 0) {
1002 auto mv
= tmp
.get_min_compat_client();
1003 dout(1) << __func__
<< " setting require_min_compat_client to currently "
1004 << "required " << ceph_release_name(mv
) << dendl
;
1005 mon
->clog
->info() << "setting require_min_compat_client to currently "
1006 << "required " << ceph_release_name(mv
);
1007 pending_inc
.new_require_min_compat_client
= mv
;
1010 if (osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
1011 // convert ec profile ruleset-* -> crush-*
1012 for (auto& p
: tmp
.erasure_code_profiles
) {
1013 bool changed
= false;
1014 map
<string
,string
> newprofile
;
1015 for (auto& q
: p
.second
) {
1016 if (q
.first
.find("ruleset-") == 0) {
1017 string key
= "crush-";
1018 key
+= q
.first
.substr(8);
1019 newprofile
[key
] = q
.second
;
1021 dout(20) << " updating ec profile " << p
.first
1022 << " key " << q
.first
<< " -> " << key
<< dendl
;
1024 newprofile
[q
.first
] = q
.second
;
1028 dout(10) << " updated ec profile " << p
.first
<< ": "
1029 << newprofile
<< dendl
;
1030 pending_inc
.new_erasure_code_profiles
[p
.first
] = newprofile
;
1034 // auto-enable pool applications upon upgrade
1035 // NOTE: this can be removed post-Luminous assuming upgrades need to
1036 // proceed through Luminous
1037 for (auto &pool_pair
: tmp
.pools
) {
1038 int64_t pool_id
= pool_pair
.first
;
1039 pg_pool_t pg_pool
= pool_pair
.second
;
1040 if (pg_pool
.is_tier()) {
1044 std::string pool_name
= tmp
.get_pool_name(pool_id
);
1045 uint32_t match_count
= 0;
1048 FSMap
const &pending_fsmap
= mon
->mdsmon()->get_pending();
1049 if (pending_fsmap
.pool_in_use(pool_id
)) {
1050 dout(10) << __func__
<< " auto-enabling CephFS on pool '"
1051 << pool_name
<< "'" << dendl
;
1052 pg_pool
.application_metadata
.insert(
1053 {pg_pool_t::APPLICATION_NAME_CEPHFS
, {}});
1057 // RBD heuristics (default OpenStack pool names from docs and
1059 if (boost::algorithm::contains(pool_name
, "rbd") ||
1060 pool_name
== "images" || pool_name
== "volumes" ||
1061 pool_name
== "backups" || pool_name
== "vms") {
1062 dout(10) << __func__
<< " auto-enabling RBD on pool '"
1063 << pool_name
<< "'" << dendl
;
1064 pg_pool
.application_metadata
.insert(
1065 {pg_pool_t::APPLICATION_NAME_RBD
, {}});
1070 if (boost::algorithm::contains(pool_name
, ".rgw") ||
1071 boost::algorithm::contains(pool_name
, ".log") ||
1072 boost::algorithm::contains(pool_name
, ".intent-log") ||
1073 boost::algorithm::contains(pool_name
, ".usage") ||
1074 boost::algorithm::contains(pool_name
, ".users")) {
1075 dout(10) << __func__
<< " auto-enabling RGW on pool '"
1076 << pool_name
<< "'" << dendl
;
1077 pg_pool
.application_metadata
.insert(
1078 {pg_pool_t::APPLICATION_NAME_RGW
, {}});
1082 // OpenStack gnocchi (from ceph-ansible)
1083 if (pool_name
== "metrics" && match_count
== 0) {
1084 dout(10) << __func__
<< " auto-enabling OpenStack Gnocchi on pool '"
1085 << pool_name
<< "'" << dendl
;
1086 pg_pool
.application_metadata
.insert({"openstack_gnocchi", {}});
1090 if (match_count
== 1) {
1091 pg_pool
.last_change
= pending_inc
.epoch
;
1092 pending_inc
.new_pools
[pool_id
] = pg_pool
;
1093 } else if (match_count
> 1) {
1094 auto pstat
= mon
->pgservice
->get_pool_stat(pool_id
);
1095 if (pstat
!= nullptr && pstat
->stats
.sum
.num_objects
> 0) {
1096 mon
->clog
->info() << "unable to auto-enable application for pool "
1097 << "'" << pool_name
<< "'";
1106 for (auto i
= pending_inc
.new_state
.begin();
1107 i
!= pending_inc
.new_state
.end();
1109 int s
= i
->second
? i
->second
: CEPH_OSD_UP
;
1110 if (s
& CEPH_OSD_UP
)
1111 dout(2) << " osd." << i
->first
<< " DOWN" << dendl
;
1112 if (s
& CEPH_OSD_EXISTS
)
1113 dout(2) << " osd." << i
->first
<< " DNE" << dendl
;
1115 for (map
<int32_t,entity_addr_t
>::iterator i
= pending_inc
.new_up_client
.begin();
1116 i
!= pending_inc
.new_up_client
.end();
1118 //FIXME: insert cluster addresses too
1119 dout(2) << " osd." << i
->first
<< " UP " << i
->second
<< dendl
;
1121 for (map
<int32_t,uint32_t>::iterator i
= pending_inc
.new_weight
.begin();
1122 i
!= pending_inc
.new_weight
.end();
1124 if (i
->second
== CEPH_OSD_OUT
) {
1125 dout(2) << " osd." << i
->first
<< " OUT" << dendl
;
1126 } else if (i
->second
== CEPH_OSD_IN
) {
1127 dout(2) << " osd." << i
->first
<< " IN" << dendl
;
1129 dout(2) << " osd." << i
->first
<< " WEIGHT " << hex
<< i
->second
<< dec
<< dendl
;
1133 // features for osdmap and its incremental
1134 uint64_t features
= mon
->get_quorum_con_features();
1136 // encode full map and determine its crc
1139 tmp
.deepish_copy_from(osdmap
);
1140 tmp
.apply_incremental(pending_inc
);
1142 // determine appropriate features
1143 if (tmp
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
1144 dout(10) << __func__
<< " encoding without feature SERVER_LUMINOUS"
1146 features
&= ~CEPH_FEATURE_SERVER_LUMINOUS
;
1148 if (tmp
.require_osd_release
< CEPH_RELEASE_KRAKEN
) {
1149 dout(10) << __func__
<< " encoding without feature SERVER_KRAKEN | "
1150 << "MSG_ADDR2" << dendl
;
1151 features
&= ~(CEPH_FEATURE_SERVER_KRAKEN
|
1152 CEPH_FEATURE_MSG_ADDR2
);
1154 if (tmp
.require_osd_release
< CEPH_RELEASE_JEWEL
) {
1155 dout(10) << __func__
<< " encoding without feature SERVER_JEWEL" << dendl
;
1156 features
&= ~CEPH_FEATURE_SERVER_JEWEL
;
1158 dout(10) << __func__
<< " encoding full map with " << features
<< dendl
;
1161 ::encode(tmp
, fullbl
, features
| CEPH_FEATURE_RESERVED
);
1162 pending_inc
.full_crc
= tmp
.get_crc();
1164 // include full map in the txn. note that old monitors will
1165 // overwrite this. new ones will now skip the local full map
1166 // encode and reload from this.
1167 put_version_full(t
, pending_inc
.epoch
, fullbl
);
1171 assert(get_last_committed() + 1 == pending_inc
.epoch
);
1172 ::encode(pending_inc
, bl
, features
| CEPH_FEATURE_RESERVED
);
1174 dout(20) << " full_crc " << tmp
.get_crc()
1175 << " inc_crc " << pending_inc
.inc_crc
<< dendl
;
1177 /* put everything in the transaction */
1178 put_version(t
, pending_inc
.epoch
, bl
);
1179 put_last_committed(t
, pending_inc
.epoch
);
1182 for (map
<int,bufferlist
>::iterator p
= pending_metadata
.begin();
1183 p
!= pending_metadata
.end();
1185 t
->put(OSD_METADATA_PREFIX
, stringify(p
->first
), p
->second
);
1186 for (set
<int>::iterator p
= pending_metadata_rm
.begin();
1187 p
!= pending_metadata_rm
.end();
1189 t
->erase(OSD_METADATA_PREFIX
, stringify(*p
));
1190 pending_metadata
.clear();
1191 pending_metadata_rm
.clear();
1193 // and pg creating, also!
1194 if (mon
->monmap
->get_required_features().contains_all(
1195 ceph::features::mon::FEATURE_LUMINOUS
)) {
1196 auto pending_creatings
= update_pending_pgs(pending_inc
);
1197 if (osdmap
.get_epoch() &&
1198 osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
1199 dout(7) << __func__
<< " in the middle of upgrading, "
1200 << " trimming pending creating_pgs using pgmap" << dendl
;
1201 mon
->pgservice
->maybe_trim_creating_pgs(&pending_creatings
);
1203 bufferlist creatings_bl
;
1204 ::encode(pending_creatings
, creatings_bl
);
1205 t
->put(OSD_PG_CREATING_PREFIX
, "creating", creatings_bl
);
1209 health_check_map_t next
;
1210 tmp
.check_health(&next
);
1211 encode_health(next
, t
);
1214 void OSDMonitor::trim_creating_pgs(creating_pgs_t
* creating_pgs
,
1215 const ceph::unordered_map
<pg_t
,pg_stat_t
>& pg_stat
)
1217 auto p
= creating_pgs
->pgs
.begin();
1218 while (p
!= creating_pgs
->pgs
.end()) {
1219 auto q
= pg_stat
.find(p
->first
);
1220 if (q
!= pg_stat
.end() &&
1221 !(q
->second
.state
& PG_STATE_CREATING
)) {
1222 dout(20) << __func__
<< " pgmap shows " << p
->first
<< " is created"
1224 p
= creating_pgs
->pgs
.erase(p
);
1231 int OSDMonitor::load_metadata(int osd
, map
<string
, string
>& m
, ostream
*err
)
1234 int r
= mon
->store
->get(OSD_METADATA_PREFIX
, stringify(osd
), bl
);
1238 bufferlist::iterator p
= bl
.begin();
1241 catch (buffer::error
& e
) {
1243 *err
<< "osd." << osd
<< " metadata is corrupt";
1249 void OSDMonitor::count_metadata(const string
& field
, map
<string
,int> *out
)
1251 for (int osd
= 0; osd
< osdmap
.get_max_osd(); ++osd
) {
1252 if (osdmap
.is_up(osd
)) {
1253 map
<string
,string
> meta
;
1254 load_metadata(osd
, meta
, nullptr);
1255 auto p
= meta
.find(field
);
1256 if (p
== meta
.end()) {
1257 (*out
)["unknown"]++;
1259 (*out
)[p
->second
]++;
1265 void OSDMonitor::count_metadata(const string
& field
, Formatter
*f
)
1267 map
<string
,int> by_val
;
1268 count_metadata(field
, &by_val
);
1269 f
->open_object_section(field
.c_str());
1270 for (auto& p
: by_val
) {
1271 f
->dump_int(p
.first
.c_str(), p
.second
);
1276 int OSDMonitor::get_osd_objectstore_type(int osd
, string
*type
)
1278 map
<string
, string
> metadata
;
1279 int r
= load_metadata(osd
, metadata
, nullptr);
1283 auto it
= metadata
.find("osd_objectstore");
1284 if (it
== metadata
.end())
1290 bool OSDMonitor::is_pool_currently_all_bluestore(int64_t pool_id
,
1291 const pg_pool_t
&pool
,
1294 // just check a few pgs for efficiency - this can't give a guarantee anyway,
1295 // since filestore osds could always join the pool later
1296 set
<int> checked_osds
;
1297 for (unsigned ps
= 0; ps
< MIN(8, pool
.get_pg_num()); ++ps
) {
1298 vector
<int> up
, acting
;
1299 pg_t
pgid(ps
, pool_id
, -1);
1300 osdmap
.pg_to_up_acting_osds(pgid
, up
, acting
);
1301 for (int osd
: up
) {
1302 if (checked_osds
.find(osd
) != checked_osds
.end())
1304 string objectstore_type
;
1305 int r
= get_osd_objectstore_type(osd
, &objectstore_type
);
1306 // allow with missing metadata, e.g. due to an osd never booting yet
1307 if (r
< 0 || objectstore_type
== "bluestore") {
1308 checked_osds
.insert(osd
);
1311 *err
<< "osd." << osd
<< " uses " << objectstore_type
;
1318 int OSDMonitor::dump_osd_metadata(int osd
, Formatter
*f
, ostream
*err
)
1320 map
<string
,string
> m
;
1321 if (int r
= load_metadata(osd
, m
, err
))
1323 for (map
<string
,string
>::iterator p
= m
.begin(); p
!= m
.end(); ++p
)
1324 f
->dump_string(p
->first
.c_str(), p
->second
);
1328 void OSDMonitor::print_nodes(Formatter
*f
)
1330 // group OSDs by their hosts
1331 map
<string
, list
<int> > osds
; // hostname => osd
1332 for (int osd
= 0; osd
< osdmap
.get_max_osd(); osd
++) {
1333 map
<string
, string
> m
;
1334 if (load_metadata(osd
, m
, NULL
)) {
1337 map
<string
, string
>::iterator hostname
= m
.find("hostname");
1338 if (hostname
== m
.end()) {
1339 // not likely though
1342 osds
[hostname
->second
].push_back(osd
);
1345 dump_services(f
, osds
, "osd");
1348 void OSDMonitor::share_map_with_random_osd()
1350 if (osdmap
.get_num_up_osds() == 0) {
1351 dout(10) << __func__
<< " no up osds, don't share with anyone" << dendl
;
1355 MonSession
*s
= mon
->session_map
.get_random_osd_session(&osdmap
);
1357 dout(10) << __func__
<< " no up osd on our session map" << dendl
;
1361 dout(10) << "committed, telling random " << s
->inst
<< " all about it" << dendl
;
1362 // whatev, they'll request more if they need it
1363 MOSDMap
*m
= build_incremental(osdmap
.get_epoch() - 1, osdmap
.get_epoch());
1364 s
->con
->send_message(m
);
1365 // NOTE: do *not* record osd has up to this epoch (as we do
1366 // elsewhere) as they may still need to request older values.
1369 version_t
OSDMonitor::get_trim_to()
1371 if (mon
->get_quorum().empty()) {
1372 dout(10) << __func__
<< ": quorum not formed" << dendl
;
1377 if (mon
->monmap
->get_required_features().contains_all(
1378 ceph::features::mon::FEATURE_LUMINOUS
)) {
1380 // TODO: Get this hidden in PGStatService
1381 std::lock_guard
<std::mutex
> l(creating_pgs_lock
);
1382 if (!creating_pgs
.pgs
.empty()) {
1386 floor
= get_min_last_epoch_clean();
1388 if (!mon
->pgservice
->is_readable())
1390 if (mon
->pgservice
->have_creating_pgs()) {
1393 floor
= mon
->pgservice
->get_min_last_epoch_clean();
1396 dout(10) << " min_last_epoch_clean " << floor
<< dendl
;
1397 if (g_conf
->mon_osd_force_trim_to
> 0 &&
1398 g_conf
->mon_osd_force_trim_to
< (int)get_last_committed()) {
1399 floor
= g_conf
->mon_osd_force_trim_to
;
1400 dout(10) << " explicit mon_osd_force_trim_to = " << floor
<< dendl
;
1402 unsigned min
= g_conf
->mon_min_osdmap_epochs
;
1403 if (floor
+ min
> get_last_committed()) {
1404 if (min
< get_last_committed())
1405 floor
= get_last_committed() - min
;
1409 if (floor
> get_first_committed())
1415 epoch_t
OSDMonitor::get_min_last_epoch_clean() const
1417 auto floor
= last_epoch_clean
.get_lower_bound(osdmap
);
1418 // also scan osd epochs
1419 // don't trim past the oldest reported osd epoch
1420 for (auto& osd_epoch
: osd_epochs
) {
1421 if (osd_epoch
.second
< floor
) {
1422 floor
= osd_epoch
.second
;
1428 void OSDMonitor::encode_trim_extra(MonitorDBStore::TransactionRef tx
,
1431 dout(10) << __func__
<< " including full map for e " << first
<< dendl
;
1433 get_version_full(first
, bl
);
1434 put_version_full(tx
, first
, bl
);
1439 bool OSDMonitor::preprocess_query(MonOpRequestRef op
)
1441 op
->mark_osdmon_event(__func__
);
1442 Message
*m
= op
->get_req();
1443 dout(10) << "preprocess_query " << *m
<< " from " << m
->get_orig_source_inst() << dendl
;
1445 switch (m
->get_type()) {
1447 case MSG_MON_COMMAND
:
1448 return preprocess_command(op
);
1449 case CEPH_MSG_MON_GET_OSDMAP
:
1450 return preprocess_get_osdmap(op
);
1453 case MSG_OSD_MARK_ME_DOWN
:
1454 return preprocess_mark_me_down(op
);
1456 return preprocess_full(op
);
1457 case MSG_OSD_FAILURE
:
1458 return preprocess_failure(op
);
1460 return preprocess_boot(op
);
1462 return preprocess_alive(op
);
1463 case MSG_OSD_PG_CREATED
:
1464 return preprocess_pg_created(op
);
1465 case MSG_OSD_PGTEMP
:
1466 return preprocess_pgtemp(op
);
1467 case MSG_OSD_BEACON
:
1468 return preprocess_beacon(op
);
1470 case CEPH_MSG_POOLOP
:
1471 return preprocess_pool_op(op
);
1473 case MSG_REMOVE_SNAPS
:
1474 return preprocess_remove_snaps(op
);
1482 bool OSDMonitor::prepare_update(MonOpRequestRef op
)
1484 op
->mark_osdmon_event(__func__
);
1485 Message
*m
= op
->get_req();
1486 dout(7) << "prepare_update " << *m
<< " from " << m
->get_orig_source_inst() << dendl
;
1488 switch (m
->get_type()) {
1490 case MSG_OSD_MARK_ME_DOWN
:
1491 return prepare_mark_me_down(op
);
1493 return prepare_full(op
);
1494 case MSG_OSD_FAILURE
:
1495 return prepare_failure(op
);
1497 return prepare_boot(op
);
1499 return prepare_alive(op
);
1500 case MSG_OSD_PG_CREATED
:
1501 return prepare_pg_created(op
);
1502 case MSG_OSD_PGTEMP
:
1503 return prepare_pgtemp(op
);
1504 case MSG_OSD_BEACON
:
1505 return prepare_beacon(op
);
1507 case MSG_MON_COMMAND
:
1508 return prepare_command(op
);
1510 case CEPH_MSG_POOLOP
:
1511 return prepare_pool_op(op
);
1513 case MSG_REMOVE_SNAPS
:
1514 return prepare_remove_snaps(op
);
1524 bool OSDMonitor::should_propose(double& delay
)
1526 dout(10) << "should_propose" << dendl
;
1528 // if full map, propose immediately! any subsequent changes will be clobbered.
1529 if (pending_inc
.fullmap
.length())
1532 // adjust osd weights?
1533 if (!osd_weight
.empty() &&
1534 osd_weight
.size() == (unsigned)osdmap
.get_max_osd()) {
1535 dout(0) << " adjusting osd weights based on " << osd_weight
<< dendl
;
1536 osdmap
.adjust_osd_weights(osd_weight
, pending_inc
);
1542 // propose as fast as possible if updating up_thru or pg_temp
1543 // want to merge OSDMap changes as much as possible
1544 if ((pending_inc
.new_primary_temp
.size() == 1
1545 || pending_inc
.new_up_thru
.size() == 1)
1546 && pending_inc
.new_state
.size() < 2) {
1547 dout(15) << " propose as fast as possible for up_thru/pg_temp" << dendl
;
1549 utime_t now
= ceph_clock_now();
1550 if (now
- last_attempted_minwait_time
> g_conf
->paxos_propose_interval
1551 && now
- paxos
->get_last_commit_time() > g_conf
->paxos_min_wait
) {
1552 delay
= g_conf
->paxos_min_wait
;
1553 last_attempted_minwait_time
= now
;
1558 return PaxosService::should_propose(delay
);
1563 // ---------------------------
1566 bool OSDMonitor::preprocess_get_osdmap(MonOpRequestRef op
)
1568 op
->mark_osdmon_event(__func__
);
1569 MMonGetOSDMap
*m
= static_cast<MMonGetOSDMap
*>(op
->get_req());
1570 dout(10) << __func__
<< " " << *m
<< dendl
;
1571 MOSDMap
*reply
= new MOSDMap(mon
->monmap
->fsid
);
1572 epoch_t first
= get_first_committed();
1573 epoch_t last
= osdmap
.get_epoch();
1574 int max
= g_conf
->osd_map_message_max
;
1575 for (epoch_t e
= MAX(first
, m
->get_full_first());
1576 e
<= MIN(last
, m
->get_full_last()) && max
> 0;
1578 int r
= get_version_full(e
, reply
->maps
[e
]);
1581 for (epoch_t e
= MAX(first
, m
->get_inc_first());
1582 e
<= MIN(last
, m
->get_inc_last()) && max
> 0;
1584 int r
= get_version(e
, reply
->incremental_maps
[e
]);
1587 reply
->oldest_map
= first
;
1588 reply
->newest_map
= last
;
1589 mon
->send_reply(op
, reply
);
1594 // ---------------------------
1599 bool OSDMonitor::check_source(PaxosServiceMessage
*m
, uuid_d fsid
) {
1600 // check permissions
1601 MonSession
*session
= m
->get_session();
1604 if (!session
->is_capable("osd", MON_CAP_X
)) {
1605 dout(0) << "got MOSDFailure from entity with insufficient caps "
1606 << session
->caps
<< dendl
;
1609 if (fsid
!= mon
->monmap
->fsid
) {
1610 dout(0) << "check_source: on fsid " << fsid
1611 << " != " << mon
->monmap
->fsid
<< dendl
;
1618 bool OSDMonitor::preprocess_failure(MonOpRequestRef op
)
1620 op
->mark_osdmon_event(__func__
);
1621 MOSDFailure
*m
= static_cast<MOSDFailure
*>(op
->get_req());
1622 // who is target_osd
1623 int badboy
= m
->get_target().name
.num();
1625 // check permissions
1626 if (check_source(m
, m
->fsid
))
1629 // first, verify the reporting host is valid
1630 if (m
->get_orig_source().is_osd()) {
1631 int from
= m
->get_orig_source().num();
1632 if (!osdmap
.exists(from
) ||
1633 osdmap
.get_addr(from
) != m
->get_orig_source_inst().addr
||
1634 (osdmap
.is_down(from
) && m
->if_osd_failed())) {
1635 dout(5) << "preprocess_failure from dead osd." << from
<< ", ignoring" << dendl
;
1636 send_incremental(op
, m
->get_epoch()+1);
1643 if (osdmap
.is_down(badboy
)) {
1644 dout(5) << "preprocess_failure dne(/dup?): " << m
->get_target() << ", from " << m
->get_orig_source_inst() << dendl
;
1645 if (m
->get_epoch() < osdmap
.get_epoch())
1646 send_incremental(op
, m
->get_epoch()+1);
1649 if (osdmap
.get_inst(badboy
) != m
->get_target()) {
1650 dout(5) << "preprocess_failure wrong osd: report " << m
->get_target() << " != map's " << osdmap
.get_inst(badboy
)
1651 << ", from " << m
->get_orig_source_inst() << dendl
;
1652 if (m
->get_epoch() < osdmap
.get_epoch())
1653 send_incremental(op
, m
->get_epoch()+1);
1657 // already reported?
1658 if (osdmap
.is_down(badboy
) ||
1659 osdmap
.get_up_from(badboy
) > m
->get_epoch()) {
1660 dout(5) << "preprocess_failure dup/old: " << m
->get_target() << ", from " << m
->get_orig_source_inst() << dendl
;
1661 if (m
->get_epoch() < osdmap
.get_epoch())
1662 send_incremental(op
, m
->get_epoch()+1);
1666 if (!can_mark_down(badboy
)) {
1667 dout(5) << "preprocess_failure ignoring report of " << m
->get_target() << " from " << m
->get_orig_source_inst() << dendl
;
1671 dout(10) << "preprocess_failure new: " << m
->get_target() << ", from " << m
->get_orig_source_inst() << dendl
;
1678 class C_AckMarkedDown
: public C_MonOp
{
1684 : C_MonOp(op
), osdmon(osdmon
) {}
1686 void _finish(int) override
{
1687 MOSDMarkMeDown
*m
= static_cast<MOSDMarkMeDown
*>(op
->get_req());
1688 osdmon
->mon
->send_reply(
1694 false)); // ACK itself does not request an ack
1696 ~C_AckMarkedDown() override
{
1700 bool OSDMonitor::preprocess_mark_me_down(MonOpRequestRef op
)
1702 op
->mark_osdmon_event(__func__
);
1703 MOSDMarkMeDown
*m
= static_cast<MOSDMarkMeDown
*>(op
->get_req());
1704 int requesting_down
= m
->get_target().name
.num();
1705 int from
= m
->get_orig_source().num();
1707 // check permissions
1708 if (check_source(m
, m
->fsid
))
1711 // first, verify the reporting host is valid
1712 if (!m
->get_orig_source().is_osd())
1715 if (!osdmap
.exists(from
) ||
1716 osdmap
.is_down(from
) ||
1717 osdmap
.get_addr(from
) != m
->get_target().addr
) {
1718 dout(5) << "preprocess_mark_me_down from dead osd."
1719 << from
<< ", ignoring" << dendl
;
1720 send_incremental(op
, m
->get_epoch()+1);
1724 // no down might be set
1725 if (!can_mark_down(requesting_down
))
1728 dout(10) << "MOSDMarkMeDown for: " << m
->get_target() << dendl
;
1732 if (m
->request_ack
) {
1733 Context
*c(new C_AckMarkedDown(this, op
));
1739 bool OSDMonitor::prepare_mark_me_down(MonOpRequestRef op
)
1741 op
->mark_osdmon_event(__func__
);
1742 MOSDMarkMeDown
*m
= static_cast<MOSDMarkMeDown
*>(op
->get_req());
1743 int target_osd
= m
->get_target().name
.num();
1745 assert(osdmap
.is_up(target_osd
));
1746 assert(osdmap
.get_addr(target_osd
) == m
->get_target().addr
);
1748 mon
->clog
->info() << "osd." << target_osd
<< " marked itself down";
1749 pending_inc
.new_state
[target_osd
] = CEPH_OSD_UP
;
1751 wait_for_finished_proposal(op
, new C_AckMarkedDown(this, op
));
1755 bool OSDMonitor::can_mark_down(int i
)
1757 if (osdmap
.test_flag(CEPH_OSDMAP_NODOWN
)) {
1758 dout(5) << __func__
<< " NODOWN flag set, will not mark osd." << i
1759 << " down" << dendl
;
1763 if (osdmap
.is_nodown(i
)) {
1764 dout(5) << __func__
<< " osd." << i
<< " is marked as nodown, "
1765 << "will not mark it down" << dendl
;
1769 int num_osds
= osdmap
.get_num_osds();
1770 if (num_osds
== 0) {
1771 dout(5) << __func__
<< " no osds" << dendl
;
1774 int up
= osdmap
.get_num_up_osds() - pending_inc
.get_net_marked_down(&osdmap
);
1775 float up_ratio
= (float)up
/ (float)num_osds
;
1776 if (up_ratio
< g_conf
->mon_osd_min_up_ratio
) {
1777 dout(2) << __func__
<< " current up_ratio " << up_ratio
<< " < min "
1778 << g_conf
->mon_osd_min_up_ratio
1779 << ", will not mark osd." << i
<< " down" << dendl
;
1785 bool OSDMonitor::can_mark_up(int i
)
1787 if (osdmap
.test_flag(CEPH_OSDMAP_NOUP
)) {
1788 dout(5) << __func__
<< " NOUP flag set, will not mark osd." << i
1793 if (osdmap
.is_noup(i
)) {
1794 dout(5) << __func__
<< " osd." << i
<< " is marked as noup, "
1795 << "will not mark it up" << dendl
;
1803 * @note the parameter @p i apparently only exists here so we can output the
1804 * osd's id on messages.
1806 bool OSDMonitor::can_mark_out(int i
)
1808 if (osdmap
.test_flag(CEPH_OSDMAP_NOOUT
)) {
1809 dout(5) << __func__
<< " NOOUT flag set, will not mark osds out" << dendl
;
1813 if (osdmap
.is_noout(i
)) {
1814 dout(5) << __func__
<< " osd." << i
<< " is marked as noout, "
1815 << "will not mark it out" << dendl
;
1819 int num_osds
= osdmap
.get_num_osds();
1820 if (num_osds
== 0) {
1821 dout(5) << __func__
<< " no osds" << dendl
;
1824 int in
= osdmap
.get_num_in_osds() - pending_inc
.get_net_marked_out(&osdmap
);
1825 float in_ratio
= (float)in
/ (float)num_osds
;
1826 if (in_ratio
< g_conf
->mon_osd_min_in_ratio
) {
1828 dout(5) << __func__
<< " current in_ratio " << in_ratio
<< " < min "
1829 << g_conf
->mon_osd_min_in_ratio
1830 << ", will not mark osd." << i
<< " out" << dendl
;
1832 dout(5) << __func__
<< " current in_ratio " << in_ratio
<< " < min "
1833 << g_conf
->mon_osd_min_in_ratio
1834 << ", will not mark osds out" << dendl
;
1841 bool OSDMonitor::can_mark_in(int i
)
1843 if (osdmap
.test_flag(CEPH_OSDMAP_NOIN
)) {
1844 dout(5) << __func__
<< " NOIN flag set, will not mark osd." << i
1849 if (osdmap
.is_noin(i
)) {
1850 dout(5) << __func__
<< " osd." << i
<< " is marked as noin, "
1851 << "will not mark it in" << dendl
;
1858 bool OSDMonitor::check_failures(utime_t now
)
1860 bool found_failure
= false;
1861 for (map
<int,failure_info_t
>::iterator p
= failure_info
.begin();
1862 p
!= failure_info
.end();
1864 if (can_mark_down(p
->first
)) {
1865 found_failure
|= check_failure(now
, p
->first
, p
->second
);
1868 return found_failure
;
1871 bool OSDMonitor::check_failure(utime_t now
, int target_osd
, failure_info_t
& fi
)
1873 // already pending failure?
1874 if (pending_inc
.new_state
.count(target_osd
) &&
1875 pending_inc
.new_state
[target_osd
] & CEPH_OSD_UP
) {
1876 dout(10) << " already pending failure" << dendl
;
1880 set
<string
> reporters_by_subtree
;
1881 string reporter_subtree_level
= g_conf
->mon_osd_reporter_subtree_level
;
1882 utime_t
orig_grace(g_conf
->osd_heartbeat_grace
, 0);
1883 utime_t max_failed_since
= fi
.get_failed_since();
1884 utime_t failed_for
= now
- max_failed_since
;
1886 utime_t grace
= orig_grace
;
1887 double my_grace
= 0, peer_grace
= 0;
1889 if (g_conf
->mon_osd_adjust_heartbeat_grace
) {
1890 double halflife
= (double)g_conf
->mon_osd_laggy_halflife
;
1891 decay_k
= ::log(.5) / halflife
;
1893 // scale grace period based on historical probability of 'lagginess'
1894 // (false positive failures due to slowness).
1895 const osd_xinfo_t
& xi
= osdmap
.get_xinfo(target_osd
);
1896 double decay
= exp((double)failed_for
* decay_k
);
1897 dout(20) << " halflife " << halflife
<< " decay_k " << decay_k
1898 << " failed_for " << failed_for
<< " decay " << decay
<< dendl
;
1899 my_grace
= decay
* (double)xi
.laggy_interval
* xi
.laggy_probability
;
1903 // consider the peers reporting a failure a proxy for a potential
1904 // 'subcluster' over the overall cluster that is similarly
1905 // laggy. this is clearly not true in all cases, but will sometimes
1906 // help us localize the grace correction to a subset of the system
1907 // (say, a rack with a bad switch) that is unhappy.
1908 assert(fi
.reporters
.size());
1909 for (map
<int,failure_reporter_t
>::iterator p
= fi
.reporters
.begin();
1910 p
!= fi
.reporters
.end();
1912 // get the parent bucket whose type matches with "reporter_subtree_level".
1913 // fall back to OSD if the level doesn't exist.
1914 map
<string
, string
> reporter_loc
= osdmap
.crush
->get_full_location(p
->first
);
1915 map
<string
, string
>::iterator iter
= reporter_loc
.find(reporter_subtree_level
);
1916 if (iter
== reporter_loc
.end()) {
1917 reporters_by_subtree
.insert("osd." + to_string(p
->first
));
1919 reporters_by_subtree
.insert(iter
->second
);
1921 if (g_conf
->mon_osd_adjust_heartbeat_grace
) {
1922 const osd_xinfo_t
& xi
= osdmap
.get_xinfo(p
->first
);
1923 utime_t elapsed
= now
- xi
.down_stamp
;
1924 double decay
= exp((double)elapsed
* decay_k
);
1925 peer_grace
+= decay
* (double)xi
.laggy_interval
* xi
.laggy_probability
;
1929 if (g_conf
->mon_osd_adjust_heartbeat_grace
) {
1930 peer_grace
/= (double)fi
.reporters
.size();
1931 grace
+= peer_grace
;
1934 dout(10) << " osd." << target_osd
<< " has "
1935 << fi
.reporters
.size() << " reporters, "
1936 << grace
<< " grace (" << orig_grace
<< " + " << my_grace
1937 << " + " << peer_grace
<< "), max_failed_since " << max_failed_since
1940 if (failed_for
>= grace
&&
1941 (int)reporters_by_subtree
.size() >= g_conf
->mon_osd_min_down_reporters
) {
1942 dout(1) << " we have enough reporters to mark osd." << target_osd
1943 << " down" << dendl
;
1944 pending_inc
.new_state
[target_osd
] = CEPH_OSD_UP
;
1946 mon
->clog
->info() << "osd." << target_osd
<< " failed ("
1947 << osdmap
.crush
->get_full_location_ordered_string(
1950 << (int)reporters_by_subtree
.size()
1951 << " reporters from different "
1952 << reporter_subtree_level
<< " after "
1953 << failed_for
<< " >= grace " << grace
<< ")";
1959 void OSDMonitor::force_failure(int target_osd
, int by
)
1961 // already pending failure?
1962 if (pending_inc
.new_state
.count(target_osd
) &&
1963 pending_inc
.new_state
[target_osd
] & CEPH_OSD_UP
) {
1964 dout(10) << " already pending failure" << dendl
;
1968 dout(1) << " we're forcing failure of osd." << target_osd
<< dendl
;
1969 pending_inc
.new_state
[target_osd
] = CEPH_OSD_UP
;
1971 mon
->clog
->info() << "osd." << target_osd
<< " failed ("
1972 << osdmap
.crush
->get_full_location_ordered_string(target_osd
)
1973 << ") (connection refused reported by osd." << by
<< ")";
1977 bool OSDMonitor::prepare_failure(MonOpRequestRef op
)
1979 op
->mark_osdmon_event(__func__
);
1980 MOSDFailure
*m
= static_cast<MOSDFailure
*>(op
->get_req());
1981 dout(1) << "prepare_failure " << m
->get_target()
1982 << " from " << m
->get_orig_source_inst()
1983 << " is reporting failure:" << m
->if_osd_failed() << dendl
;
1985 int target_osd
= m
->get_target().name
.num();
1986 int reporter
= m
->get_orig_source().num();
1987 assert(osdmap
.is_up(target_osd
));
1988 assert(osdmap
.get_addr(target_osd
) == m
->get_target().addr
);
1990 if (m
->if_osd_failed()) {
1991 // calculate failure time
1992 utime_t now
= ceph_clock_now();
1993 utime_t failed_since
=
1994 m
->get_recv_stamp() - utime_t(m
->failed_for
, 0);
1997 if (m
->is_immediate()) {
1998 mon
->clog
->debug() << m
->get_target() << " reported immediately failed by "
1999 << m
->get_orig_source_inst();
2000 force_failure(target_osd
, reporter
);
2003 mon
->clog
->debug() << m
->get_target() << " reported failed by "
2004 << m
->get_orig_source_inst();
2006 failure_info_t
& fi
= failure_info
[target_osd
];
2007 MonOpRequestRef old_op
= fi
.add_report(reporter
, failed_since
, op
);
2009 mon
->no_reply(old_op
);
2012 return check_failure(now
, target_osd
, fi
);
2014 // remove the report
2015 mon
->clog
->debug() << m
->get_target() << " failure report canceled by "
2016 << m
->get_orig_source_inst();
2017 if (failure_info
.count(target_osd
)) {
2018 failure_info_t
& fi
= failure_info
[target_osd
];
2019 MonOpRequestRef report_op
= fi
.cancel_report(reporter
);
2021 mon
->no_reply(report_op
);
2023 if (fi
.reporters
.empty()) {
2024 dout(10) << " removing last failure_info for osd." << target_osd
2026 failure_info
.erase(target_osd
);
2028 dout(10) << " failure_info for osd." << target_osd
<< " now "
2029 << fi
.reporters
.size() << " reporters" << dendl
;
2032 dout(10) << " no failure_info for osd." << target_osd
<< dendl
;
2040 void OSDMonitor::process_failures()
2042 map
<int,failure_info_t
>::iterator p
= failure_info
.begin();
2043 while (p
!= failure_info
.end()) {
2044 if (osdmap
.is_up(p
->first
)) {
2047 dout(10) << "process_failures osd." << p
->first
<< dendl
;
2048 list
<MonOpRequestRef
> ls
;
2049 p
->second
.take_report_messages(ls
);
2050 failure_info
.erase(p
++);
2052 while (!ls
.empty()) {
2053 MonOpRequestRef o
= ls
.front();
2055 o
->mark_event(__func__
);
2056 MOSDFailure
*m
= o
->get_req
<MOSDFailure
>();
2057 send_latest(o
, m
->get_epoch());
2065 void OSDMonitor::take_all_failures(list
<MonOpRequestRef
>& ls
)
2067 dout(10) << __func__
<< " on " << failure_info
.size() << " osds" << dendl
;
2069 for (map
<int,failure_info_t
>::iterator p
= failure_info
.begin();
2070 p
!= failure_info
.end();
2072 p
->second
.take_report_messages(ls
);
2074 failure_info
.clear();
2080 bool OSDMonitor::preprocess_boot(MonOpRequestRef op
)
2082 op
->mark_osdmon_event(__func__
);
2083 MOSDBoot
*m
= static_cast<MOSDBoot
*>(op
->get_req());
2084 int from
= m
->get_orig_source_inst().name
.num();
2086 // check permissions, ignore if failed (no response expected)
2087 MonSession
*session
= m
->get_session();
2090 if (!session
->is_capable("osd", MON_CAP_X
)) {
2091 dout(0) << "got preprocess_boot message from entity with insufficient caps"
2092 << session
->caps
<< dendl
;
2096 if (m
->sb
.cluster_fsid
!= mon
->monmap
->fsid
) {
2097 dout(0) << "preprocess_boot on fsid " << m
->sb
.cluster_fsid
2098 << " != " << mon
->monmap
->fsid
<< dendl
;
2102 if (m
->get_orig_source_inst().addr
.is_blank_ip()) {
2103 dout(0) << "preprocess_boot got blank addr for " << m
->get_orig_source_inst() << dendl
;
2107 assert(m
->get_orig_source_inst().name
.is_osd());
2109 // check if osd has required features to boot
2110 if ((osdmap
.get_features(CEPH_ENTITY_TYPE_OSD
, NULL
) &
2111 CEPH_FEATURE_OSD_ERASURE_CODES
) &&
2112 !(m
->get_connection()->get_features() & CEPH_FEATURE_OSD_ERASURE_CODES
)) {
2113 dout(0) << __func__
<< " osdmap requires erasure code but osd at "
2114 << m
->get_orig_source_inst()
2115 << " doesn't announce support -- ignore" << dendl
;
2119 if ((osdmap
.get_features(CEPH_ENTITY_TYPE_OSD
, NULL
) &
2120 CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2
) &&
2121 !(m
->get_connection()->get_features() & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2
)) {
2122 dout(0) << __func__
<< " osdmap requires erasure code plugins v2 but osd at "
2123 << m
->get_orig_source_inst()
2124 << " doesn't announce support -- ignore" << dendl
;
2128 if ((osdmap
.get_features(CEPH_ENTITY_TYPE_OSD
, NULL
) &
2129 CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3
) &&
2130 !(m
->get_connection()->get_features() & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3
)) {
2131 dout(0) << __func__
<< " osdmap requires erasure code plugins v3 but osd at "
2132 << m
->get_orig_source_inst()
2133 << " doesn't announce support -- ignore" << dendl
;
2137 if (osdmap
.require_osd_release
>= CEPH_RELEASE_LUMINOUS
&&
2138 !HAVE_FEATURE(m
->osd_features
, SERVER_LUMINOUS
)) {
2139 mon
->clog
->info() << "disallowing boot of OSD "
2140 << m
->get_orig_source_inst()
2141 << " because the osdmap requires"
2142 << " CEPH_FEATURE_SERVER_LUMINOUS"
2143 << " but the osd lacks CEPH_FEATURE_SERVER_LUMINOUS";
2147 if (osdmap
.require_osd_release
>= CEPH_RELEASE_JEWEL
&&
2148 !(m
->osd_features
& CEPH_FEATURE_SERVER_JEWEL
)) {
2149 mon
->clog
->info() << "disallowing boot of OSD "
2150 << m
->get_orig_source_inst()
2151 << " because the osdmap requires"
2152 << " CEPH_FEATURE_SERVER_JEWEL"
2153 << " but the osd lacks CEPH_FEATURE_SERVER_JEWEL";
2157 if (osdmap
.require_osd_release
>= CEPH_RELEASE_KRAKEN
&&
2158 !HAVE_FEATURE(m
->osd_features
, SERVER_KRAKEN
)) {
2159 mon
->clog
->info() << "disallowing boot of OSD "
2160 << m
->get_orig_source_inst()
2161 << " because the osdmap requires"
2162 << " CEPH_FEATURE_SERVER_KRAKEN"
2163 << " but the osd lacks CEPH_FEATURE_SERVER_KRAKEN";
2167 if (osdmap
.test_flag(CEPH_OSDMAP_SORTBITWISE
) &&
2168 !(m
->osd_features
& CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT
)) {
2169 mon
->clog
->info() << "disallowing boot of OSD "
2170 << m
->get_orig_source_inst()
2171 << " because 'sortbitwise' osdmap flag is set and OSD lacks the OSD_BITWISE_HOBJ_SORT feature";
2175 if (osdmap
.test_flag(CEPH_OSDMAP_RECOVERY_DELETES
) &&
2176 !(m
->osd_features
& CEPH_FEATURE_OSD_RECOVERY_DELETES
)) {
2177 mon
->clog
->info() << "disallowing boot of OSD "
2178 << m
->get_orig_source_inst()
2179 << " because 'recovery_deletes' osdmap flag is set and OSD lacks the OSD_RECOVERY_DELETES feature";
2183 if (any_of(osdmap
.get_pools().begin(),
2184 osdmap
.get_pools().end(),
2185 [](const std::pair
<int64_t,pg_pool_t
>& pool
)
2186 { return pool
.second
.use_gmt_hitset
; })) {
2187 assert(osdmap
.get_num_up_osds() == 0 ||
2188 osdmap
.get_up_osd_features() & CEPH_FEATURE_OSD_HITSET_GMT
);
2189 if (!(m
->osd_features
& CEPH_FEATURE_OSD_HITSET_GMT
)) {
2190 dout(0) << __func__
<< " one or more pools uses GMT hitsets but osd at "
2191 << m
->get_orig_source_inst()
2192 << " doesn't announce support -- ignore" << dendl
;
2197 // make sure upgrades stop at luminous
2198 if (HAVE_FEATURE(m
->osd_features
, SERVER_M
) &&
2199 osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
2200 mon
->clog
->info() << "disallowing boot of post-luminous OSD "
2201 << m
->get_orig_source_inst()
2202 << " because require_osd_release < luminous";
2206 // make sure upgrades stop at jewel
2207 if (HAVE_FEATURE(m
->osd_features
, SERVER_KRAKEN
) &&
2208 osdmap
.require_osd_release
< CEPH_RELEASE_JEWEL
) {
2209 mon
->clog
->info() << "disallowing boot of post-jewel OSD "
2210 << m
->get_orig_source_inst()
2211 << " because require_osd_release < jewel";
2215 // make sure upgrades stop at hammer
2216 // * HAMMER_0_94_4 is the required hammer feature
2217 // * MON_METADATA is the first post-hammer feature
2218 if (osdmap
.get_num_up_osds() > 0) {
2219 if ((m
->osd_features
& CEPH_FEATURE_MON_METADATA
) &&
2220 !(osdmap
.get_up_osd_features() & CEPH_FEATURE_HAMMER_0_94_4
)) {
2221 mon
->clog
->info() << "disallowing boot of post-hammer OSD "
2222 << m
->get_orig_source_inst()
2223 << " because one or more up OSDs is pre-hammer v0.94.4";
2226 if (!(m
->osd_features
& CEPH_FEATURE_HAMMER_0_94_4
) &&
2227 (osdmap
.get_up_osd_features() & CEPH_FEATURE_MON_METADATA
)) {
2228 mon
->clog
->info() << "disallowing boot of pre-hammer v0.94.4 OSD "
2229 << m
->get_orig_source_inst()
2230 << " because all up OSDs are post-hammer";
2236 if (osdmap
.is_up(from
) &&
2237 osdmap
.get_inst(from
) == m
->get_orig_source_inst() &&
2238 osdmap
.get_cluster_addr(from
) == m
->cluster_addr
) {
2240 dout(7) << "preprocess_boot dup from " << m
->get_orig_source_inst()
2241 << " == " << osdmap
.get_inst(from
) << dendl
;
2246 if (osdmap
.exists(from
) &&
2247 !osdmap
.get_uuid(from
).is_zero() &&
2248 osdmap
.get_uuid(from
) != m
->sb
.osd_fsid
) {
2249 dout(7) << __func__
<< " from " << m
->get_orig_source_inst()
2250 << " clashes with existing osd: different fsid"
2251 << " (ours: " << osdmap
.get_uuid(from
)
2252 << " ; theirs: " << m
->sb
.osd_fsid
<< ")" << dendl
;
2256 if (osdmap
.exists(from
) &&
2257 osdmap
.get_info(from
).up_from
> m
->version
&&
2258 osdmap
.get_most_recent_inst(from
) == m
->get_orig_source_inst()) {
2259 dout(7) << "prepare_boot msg from before last up_from, ignoring" << dendl
;
2260 send_latest(op
, m
->sb
.current_epoch
+1);
2265 if (!can_mark_up(from
)) {
2266 dout(7) << "preprocess_boot ignoring boot from " << m
->get_orig_source_inst() << dendl
;
2267 send_latest(op
, m
->sb
.current_epoch
+1);
2271 dout(10) << "preprocess_boot from " << m
->get_orig_source_inst() << dendl
;
2278 bool OSDMonitor::prepare_boot(MonOpRequestRef op
)
2280 op
->mark_osdmon_event(__func__
);
2281 MOSDBoot
*m
= static_cast<MOSDBoot
*>(op
->get_req());
2282 dout(7) << __func__
<< " from " << m
->get_orig_source_inst() << " sb " << m
->sb
2283 << " cluster_addr " << m
->cluster_addr
2284 << " hb_back_addr " << m
->hb_back_addr
2285 << " hb_front_addr " << m
->hb_front_addr
2288 assert(m
->get_orig_source().is_osd());
2289 int from
= m
->get_orig_source().num();
2291 // does this osd exist?
2292 if (from
>= osdmap
.get_max_osd()) {
2293 dout(1) << "boot from osd." << from
<< " >= max_osd "
2294 << osdmap
.get_max_osd() << dendl
;
2298 int oldstate
= osdmap
.exists(from
) ? osdmap
.get_state(from
) : CEPH_OSD_NEW
;
2299 if (pending_inc
.new_state
.count(from
))
2300 oldstate
^= pending_inc
.new_state
[from
];
2302 // already up? mark down first?
2303 if (osdmap
.is_up(from
)) {
2304 dout(7) << __func__
<< " was up, first marking down "
2305 << osdmap
.get_inst(from
) << dendl
;
2306 // preprocess should have caught these; if not, assert.
2307 assert(osdmap
.get_inst(from
) != m
->get_orig_source_inst() ||
2308 osdmap
.get_cluster_addr(from
) != m
->cluster_addr
);
2309 assert(osdmap
.get_uuid(from
) == m
->sb
.osd_fsid
);
2311 if (pending_inc
.new_state
.count(from
) == 0 ||
2312 (pending_inc
.new_state
[from
] & CEPH_OSD_UP
) == 0) {
2313 // mark previous guy down
2314 pending_inc
.new_state
[from
] = CEPH_OSD_UP
;
2316 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
2317 } else if (pending_inc
.new_up_client
.count(from
)) {
2318 // already prepared, just wait
2319 dout(7) << __func__
<< " already prepared, waiting on "
2320 << m
->get_orig_source_addr() << dendl
;
2321 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
2324 pending_inc
.new_up_client
[from
] = m
->get_orig_source_addr();
2325 if (!m
->cluster_addr
.is_blank_ip())
2326 pending_inc
.new_up_cluster
[from
] = m
->cluster_addr
;
2327 pending_inc
.new_hb_back_up
[from
] = m
->hb_back_addr
;
2328 if (!m
->hb_front_addr
.is_blank_ip())
2329 pending_inc
.new_hb_front_up
[from
] = m
->hb_front_addr
;
2331 down_pending_out
.erase(from
); // if any
2334 osd_weight
[from
] = m
->sb
.weight
;
2337 dout(10) << " setting osd." << from
<< " uuid to " << m
->sb
.osd_fsid
2339 if (!osdmap
.exists(from
) || osdmap
.get_uuid(from
) != m
->sb
.osd_fsid
) {
2340 // preprocess should have caught this; if not, assert.
2341 assert(!osdmap
.exists(from
) || osdmap
.get_uuid(from
).is_zero());
2342 pending_inc
.new_uuid
[from
] = m
->sb
.osd_fsid
;
2346 if (m
->sb
.newest_map
== 0 && osdmap
.exists(from
)) {
2347 const osd_info_t
& i
= osdmap
.get_info(from
);
2348 if (i
.up_from
> i
.lost_at
) {
2349 dout(10) << " fresh osd; marking lost_at too" << dendl
;
2350 pending_inc
.new_lost
[from
] = osdmap
.get_epoch();
2355 bufferlist osd_metadata
;
2356 ::encode(m
->metadata
, osd_metadata
);
2357 pending_metadata
[from
] = osd_metadata
;
2358 pending_metadata_rm
.erase(from
);
2360 // adjust last clean unmount epoch?
2361 const osd_info_t
& info
= osdmap
.get_info(from
);
2362 dout(10) << " old osd_info: " << info
<< dendl
;
2363 if (m
->sb
.mounted
> info
.last_clean_begin
||
2364 (m
->sb
.mounted
== info
.last_clean_begin
&&
2365 m
->sb
.clean_thru
> info
.last_clean_end
)) {
2366 epoch_t begin
= m
->sb
.mounted
;
2367 epoch_t end
= m
->sb
.clean_thru
;
2369 dout(10) << __func__
<< " osd." << from
<< " last_clean_interval "
2370 << "[" << info
.last_clean_begin
<< "," << info
.last_clean_end
2371 << ") -> [" << begin
<< "-" << end
<< ")"
2373 pending_inc
.new_last_clean_interval
[from
] =
2374 pair
<epoch_t
,epoch_t
>(begin
, end
);
2377 osd_xinfo_t xi
= osdmap
.get_xinfo(from
);
2378 if (m
->boot_epoch
== 0) {
2379 xi
.laggy_probability
*= (1.0 - g_conf
->mon_osd_laggy_weight
);
2380 xi
.laggy_interval
*= (1.0 - g_conf
->mon_osd_laggy_weight
);
2381 dout(10) << " not laggy, new xi " << xi
<< dendl
;
2383 if (xi
.down_stamp
.sec()) {
2384 int interval
= ceph_clock_now().sec() -
2385 xi
.down_stamp
.sec();
2386 if (g_conf
->mon_osd_laggy_max_interval
&&
2387 (interval
> g_conf
->mon_osd_laggy_max_interval
)) {
2388 interval
= g_conf
->mon_osd_laggy_max_interval
;
2391 interval
* g_conf
->mon_osd_laggy_weight
+
2392 xi
.laggy_interval
* (1.0 - g_conf
->mon_osd_laggy_weight
);
2394 xi
.laggy_probability
=
2395 g_conf
->mon_osd_laggy_weight
+
2396 xi
.laggy_probability
* (1.0 - g_conf
->mon_osd_laggy_weight
);
2397 dout(10) << " laggy, now xi " << xi
<< dendl
;
2400 // set features shared by the osd
2401 if (m
->osd_features
)
2402 xi
.features
= m
->osd_features
;
2404 xi
.features
= m
->get_connection()->get_features();
2407 if ((g_conf
->mon_osd_auto_mark_auto_out_in
&&
2408 (oldstate
& CEPH_OSD_AUTOOUT
)) ||
2409 (g_conf
->mon_osd_auto_mark_new_in
&& (oldstate
& CEPH_OSD_NEW
)) ||
2410 (g_conf
->mon_osd_auto_mark_in
)) {
2411 if (can_mark_in(from
)) {
2412 if (osdmap
.osd_xinfo
[from
].old_weight
> 0) {
2413 pending_inc
.new_weight
[from
] = osdmap
.osd_xinfo
[from
].old_weight
;
2416 pending_inc
.new_weight
[from
] = CEPH_OSD_IN
;
2419 dout(7) << __func__
<< " NOIN set, will not mark in "
2420 << m
->get_orig_source_addr() << dendl
;
2424 pending_inc
.new_xinfo
[from
] = xi
;
2427 wait_for_finished_proposal(op
, new C_Booted(this, op
));
2432 void OSDMonitor::_booted(MonOpRequestRef op
, bool logit
)
2434 op
->mark_osdmon_event(__func__
);
2435 MOSDBoot
*m
= static_cast<MOSDBoot
*>(op
->get_req());
2436 dout(7) << "_booted " << m
->get_orig_source_inst()
2437 << " w " << m
->sb
.weight
<< " from " << m
->sb
.current_epoch
<< dendl
;
2440 mon
->clog
->info() << m
->get_orig_source_inst() << " boot";
2443 send_latest(op
, m
->sb
.current_epoch
+1);
2450 bool OSDMonitor::preprocess_full(MonOpRequestRef op
)
2452 op
->mark_osdmon_event(__func__
);
2453 MOSDFull
*m
= static_cast<MOSDFull
*>(op
->get_req());
2454 int from
= m
->get_orig_source().num();
2456 unsigned mask
= CEPH_OSD_NEARFULL
| CEPH_OSD_BACKFILLFULL
| CEPH_OSD_FULL
;
2458 // check permissions, ignore if failed
2459 MonSession
*session
= m
->get_session();
2462 if (!session
->is_capable("osd", MON_CAP_X
)) {
2463 dout(0) << "MOSDFull from entity with insufficient privileges:"
2464 << session
->caps
<< dendl
;
2468 // ignore a full message from the osd instance that already went down
2469 if (!osdmap
.exists(from
)) {
2470 dout(7) << __func__
<< " ignoring full message from nonexistent "
2471 << m
->get_orig_source_inst() << dendl
;
2474 if ((!osdmap
.is_up(from
) &&
2475 osdmap
.get_most_recent_inst(from
) == m
->get_orig_source_inst()) ||
2476 (osdmap
.is_up(from
) &&
2477 osdmap
.get_inst(from
) != m
->get_orig_source_inst())) {
2478 dout(7) << __func__
<< " ignoring full message from down "
2479 << m
->get_orig_source_inst() << dendl
;
2483 OSDMap::calc_state_set(osdmap
.get_state(from
), state
);
2485 if ((osdmap
.get_state(from
) & mask
) == m
->state
) {
2486 dout(7) << __func__
<< " state already " << state
<< " for osd." << from
2487 << " " << m
->get_orig_source_inst() << dendl
;
2488 _reply_map(op
, m
->version
);
2492 dout(10) << __func__
<< " want state " << state
<< " for osd." << from
2493 << " " << m
->get_orig_source_inst() << dendl
;
2500 bool OSDMonitor::prepare_full(MonOpRequestRef op
)
2502 op
->mark_osdmon_event(__func__
);
2503 const MOSDFull
*m
= static_cast<MOSDFull
*>(op
->get_req());
2504 const int from
= m
->get_orig_source().num();
2506 const unsigned mask
= CEPH_OSD_NEARFULL
| CEPH_OSD_BACKFILLFULL
| CEPH_OSD_FULL
;
2507 const unsigned want_state
= m
->state
& mask
; // safety first
2509 unsigned cur_state
= osdmap
.get_state(from
);
2510 auto p
= pending_inc
.new_state
.find(from
);
2511 if (p
!= pending_inc
.new_state
.end()) {
2512 cur_state
^= p
->second
;
2516 set
<string
> want_state_set
, cur_state_set
;
2517 OSDMap::calc_state_set(want_state
, want_state_set
);
2518 OSDMap::calc_state_set(cur_state
, cur_state_set
);
2520 if (cur_state
!= want_state
) {
2521 if (p
!= pending_inc
.new_state
.end()) {
2524 pending_inc
.new_state
[from
] = 0;
2526 pending_inc
.new_state
[from
] |= (osdmap
.get_state(from
) & mask
) ^ want_state
;
2527 dout(7) << __func__
<< " osd." << from
<< " " << cur_state_set
2528 << " -> " << want_state_set
<< dendl
;
2530 dout(7) << __func__
<< " osd." << from
<< " " << cur_state_set
2531 << " = wanted " << want_state_set
<< ", just waiting" << dendl
;
2534 wait_for_finished_proposal(op
, new C_ReplyMap(this, op
, m
->version
));
2541 bool OSDMonitor::preprocess_alive(MonOpRequestRef op
)
2543 op
->mark_osdmon_event(__func__
);
2544 MOSDAlive
*m
= static_cast<MOSDAlive
*>(op
->get_req());
2545 int from
= m
->get_orig_source().num();
2547 // check permissions, ignore if failed
2548 MonSession
*session
= m
->get_session();
2551 if (!session
->is_capable("osd", MON_CAP_X
)) {
2552 dout(0) << "attempt to send MOSDAlive from entity with insufficient privileges:"
2553 << session
->caps
<< dendl
;
2557 if (!osdmap
.is_up(from
) ||
2558 osdmap
.get_inst(from
) != m
->get_orig_source_inst()) {
2559 dout(7) << "preprocess_alive ignoring alive message from down " << m
->get_orig_source_inst() << dendl
;
2563 if (osdmap
.get_up_thru(from
) >= m
->want
) {
2565 dout(7) << "preprocess_alive want up_thru " << m
->want
<< " dup from " << m
->get_orig_source_inst() << dendl
;
2566 _reply_map(op
, m
->version
);
2570 dout(10) << "preprocess_alive want up_thru " << m
->want
2571 << " from " << m
->get_orig_source_inst() << dendl
;
2578 bool OSDMonitor::prepare_alive(MonOpRequestRef op
)
2580 op
->mark_osdmon_event(__func__
);
2581 MOSDAlive
*m
= static_cast<MOSDAlive
*>(op
->get_req());
2582 int from
= m
->get_orig_source().num();
2584 if (0) { // we probably don't care much about these
2585 mon
->clog
->debug() << m
->get_orig_source_inst() << " alive";
2588 dout(7) << "prepare_alive want up_thru " << m
->want
<< " have " << m
->version
2589 << " from " << m
->get_orig_source_inst() << dendl
;
2591 update_up_thru(from
, m
->version
); // set to the latest map the OSD has
2592 wait_for_finished_proposal(op
, new C_ReplyMap(this, op
, m
->version
));
2596 void OSDMonitor::_reply_map(MonOpRequestRef op
, epoch_t e
)
2598 op
->mark_osdmon_event(__func__
);
2599 dout(7) << "_reply_map " << e
2600 << " from " << op
->get_req()->get_orig_source_inst()
2606 bool OSDMonitor::preprocess_pg_created(MonOpRequestRef op
)
2608 op
->mark_osdmon_event(__func__
);
2609 auto m
= static_cast<MOSDPGCreated
*>(op
->get_req());
2610 dout(10) << __func__
<< " " << *m
<< dendl
;
2611 auto session
= m
->get_session();
2613 dout(10) << __func__
<< ": no monitor session!" << dendl
;
2616 if (!session
->is_capable("osd", MON_CAP_X
)) {
2617 derr
<< __func__
<< " received from entity "
2618 << "with insufficient privileges " << session
->caps
<< dendl
;
2621 // always forward the "created!" to the leader
2625 bool OSDMonitor::prepare_pg_created(MonOpRequestRef op
)
2627 op
->mark_osdmon_event(__func__
);
2628 auto m
= static_cast<MOSDPGCreated
*>(op
->get_req());
2629 dout(10) << __func__
<< " " << *m
<< dendl
;
2630 auto src
= m
->get_orig_source();
2631 auto from
= src
.num();
2632 if (!src
.is_osd() ||
2633 !mon
->osdmon()->osdmap
.is_up(from
) ||
2634 m
->get_orig_source_inst() != mon
->osdmon()->osdmap
.get_inst(from
)) {
2635 dout(1) << __func__
<< " ignoring stats from non-active osd." << dendl
;
2638 pending_created_pgs
.push_back(m
->pgid
);
2645 bool OSDMonitor::preprocess_pgtemp(MonOpRequestRef op
)
2647 MOSDPGTemp
*m
= static_cast<MOSDPGTemp
*>(op
->get_req());
2648 dout(10) << "preprocess_pgtemp " << *m
<< dendl
;
2649 mempool::osdmap::vector
<int> empty
;
2650 int from
= m
->get_orig_source().num();
2651 size_t ignore_cnt
= 0;
2654 MonSession
*session
= m
->get_session();
2657 if (!session
->is_capable("osd", MON_CAP_X
)) {
2658 dout(0) << "attempt to send MOSDPGTemp from entity with insufficient caps "
2659 << session
->caps
<< dendl
;
2663 if (!osdmap
.is_up(from
) ||
2664 osdmap
.get_inst(from
) != m
->get_orig_source_inst()) {
2665 dout(7) << "ignoring pgtemp message from down " << m
->get_orig_source_inst() << dendl
;
2669 for (auto p
= m
->pg_temp
.begin(); p
!= m
->pg_temp
.end(); ++p
) {
2670 dout(20) << " " << p
->first
2671 << (osdmap
.pg_temp
->count(p
->first
) ? osdmap
.pg_temp
->get(p
->first
) : empty
)
2672 << " -> " << p
->second
<< dendl
;
2674 // does the pool exist?
2675 if (!osdmap
.have_pg_pool(p
->first
.pool())) {
2677 * 1. If the osdmap does not have the pool, it means the pool has been
2678 * removed in-between the osd sending this message and us handling it.
2679 * 2. If osdmap doesn't have the pool, it is safe to assume the pool does
2680 * not exist in the pending either, as the osds would not send a
2681 * message about a pool they know nothing about (yet).
2682 * 3. However, if the pool does exist in the pending, then it must be a
2683 * new pool, and not relevant to this message (see 1).
2685 dout(10) << __func__
<< " ignore " << p
->first
<< " -> " << p
->second
2686 << ": pool has been removed" << dendl
;
2691 int acting_primary
= -1;
2692 osdmap
.pg_to_up_acting_osds(
2693 p
->first
, nullptr, nullptr, nullptr, &acting_primary
);
2694 if (acting_primary
!= from
) {
2695 /* If the source isn't the primary based on the current osdmap, we know
2696 * that the interval changed and that we can discard this message.
2697 * Indeed, we must do so to avoid 16127 since we can't otherwise determine
2698 * which of two pg temp mappings on the same pg is more recent.
2700 dout(10) << __func__
<< " ignore " << p
->first
<< " -> " << p
->second
2701 << ": primary has changed" << dendl
;
2707 if (p
->second
.empty() && (osdmap
.pg_temp
->count(p
->first
) ||
2708 osdmap
.primary_temp
->count(p
->first
)))
2711 // NOTE: we assume that this will clear pg_primary, so consider
2712 // an existing pg_primary field to imply a change
2713 if (p
->second
.size() &&
2714 (osdmap
.pg_temp
->count(p
->first
) == 0 ||
2715 !vectors_equal(osdmap
.pg_temp
->get(p
->first
), p
->second
) ||
2716 osdmap
.primary_temp
->count(p
->first
)))
2720 // should we ignore all the pgs?
2721 if (ignore_cnt
== m
->pg_temp
.size())
2724 dout(7) << "preprocess_pgtemp e" << m
->map_epoch
<< " no changes from " << m
->get_orig_source_inst() << dendl
;
2725 _reply_map(op
, m
->map_epoch
);
2732 void OSDMonitor::update_up_thru(int from
, epoch_t up_thru
)
2734 epoch_t old_up_thru
= osdmap
.get_up_thru(from
);
2735 auto ut
= pending_inc
.new_up_thru
.find(from
);
2736 if (ut
!= pending_inc
.new_up_thru
.end()) {
2737 old_up_thru
= ut
->second
;
2739 if (up_thru
> old_up_thru
) {
2740 // set up_thru too, so the osd doesn't have to ask again
2741 pending_inc
.new_up_thru
[from
] = up_thru
;
2745 bool OSDMonitor::prepare_pgtemp(MonOpRequestRef op
)
2747 op
->mark_osdmon_event(__func__
);
2748 MOSDPGTemp
*m
= static_cast<MOSDPGTemp
*>(op
->get_req());
2749 int from
= m
->get_orig_source().num();
2750 dout(7) << "prepare_pgtemp e" << m
->map_epoch
<< " from " << m
->get_orig_source_inst() << dendl
;
2751 for (map
<pg_t
,vector
<int32_t> >::iterator p
= m
->pg_temp
.begin(); p
!= m
->pg_temp
.end(); ++p
) {
2752 uint64_t pool
= p
->first
.pool();
2753 if (pending_inc
.old_pools
.count(pool
)) {
2754 dout(10) << __func__
<< " ignore " << p
->first
<< " -> " << p
->second
2755 << ": pool pending removal" << dendl
;
2758 if (!osdmap
.have_pg_pool(pool
)) {
2759 dout(10) << __func__
<< " ignore " << p
->first
<< " -> " << p
->second
2760 << ": pool has been removed" << dendl
;
2763 pending_inc
.new_pg_temp
[p
->first
] =
2764 mempool::osdmap::vector
<int>(p
->second
.begin(), p
->second
.end());
2766 // unconditionally clear pg_primary (until this message can encode
2767 // a change for that, too.. at which point we need to also fix
2768 // preprocess_pg_temp)
2769 if (osdmap
.primary_temp
->count(p
->first
) ||
2770 pending_inc
.new_primary_temp
.count(p
->first
))
2771 pending_inc
.new_primary_temp
[p
->first
] = -1;
2774 // set up_thru too, so the osd doesn't have to ask again
2775 update_up_thru(from
, m
->map_epoch
);
2777 wait_for_finished_proposal(op
, new C_ReplyMap(this, op
, m
->map_epoch
));
2784 bool OSDMonitor::preprocess_remove_snaps(MonOpRequestRef op
)
2786 op
->mark_osdmon_event(__func__
);
2787 MRemoveSnaps
*m
= static_cast<MRemoveSnaps
*>(op
->get_req());
2788 dout(7) << "preprocess_remove_snaps " << *m
<< dendl
;
2790 // check privilege, ignore if failed
2791 MonSession
*session
= m
->get_session();
2794 if (!session
->caps
.is_capable(
2796 CEPH_ENTITY_TYPE_MON
,
2797 session
->entity_name
,
2798 "osd", "osd pool rmsnap", {}, true, true, false)) {
2799 dout(0) << "got preprocess_remove_snaps from entity with insufficient caps "
2800 << session
->caps
<< dendl
;
2804 for (map
<int, vector
<snapid_t
> >::iterator q
= m
->snaps
.begin();
2805 q
!= m
->snaps
.end();
2807 if (!osdmap
.have_pg_pool(q
->first
)) {
2808 dout(10) << " ignoring removed_snaps " << q
->second
<< " on non-existent pool " << q
->first
<< dendl
;
2811 const pg_pool_t
*pi
= osdmap
.get_pg_pool(q
->first
);
2812 for (vector
<snapid_t
>::iterator p
= q
->second
.begin();
2813 p
!= q
->second
.end();
2815 if (*p
> pi
->get_snap_seq() ||
2816 !pi
->removed_snaps
.contains(*p
))
2825 bool OSDMonitor::prepare_remove_snaps(MonOpRequestRef op
)
2827 op
->mark_osdmon_event(__func__
);
2828 MRemoveSnaps
*m
= static_cast<MRemoveSnaps
*>(op
->get_req());
2829 dout(7) << "prepare_remove_snaps " << *m
<< dendl
;
2831 for (map
<int, vector
<snapid_t
> >::iterator p
= m
->snaps
.begin();
2832 p
!= m
->snaps
.end();
2835 if (!osdmap
.have_pg_pool(p
->first
)) {
2836 dout(10) << " ignoring removed_snaps " << p
->second
<< " on non-existent pool " << p
->first
<< dendl
;
2840 pg_pool_t
& pi
= osdmap
.pools
[p
->first
];
2841 for (vector
<snapid_t
>::iterator q
= p
->second
.begin();
2842 q
!= p
->second
.end();
2844 if (!pi
.removed_snaps
.contains(*q
) &&
2845 (!pending_inc
.new_pools
.count(p
->first
) ||
2846 !pending_inc
.new_pools
[p
->first
].removed_snaps
.contains(*q
))) {
2847 pg_pool_t
*newpi
= pending_inc
.get_new_pool(p
->first
, &pi
);
2848 newpi
->removed_snaps
.insert(*q
);
2849 dout(10) << " pool " << p
->first
<< " removed_snaps added " << *q
2850 << " (now " << newpi
->removed_snaps
<< ")" << dendl
;
2851 if (*q
> newpi
->get_snap_seq()) {
2852 dout(10) << " pool " << p
->first
<< " snap_seq " << newpi
->get_snap_seq() << " -> " << *q
<< dendl
;
2853 newpi
->set_snap_seq(*q
);
2855 newpi
->set_snap_epoch(pending_inc
.epoch
);
2863 bool OSDMonitor::preprocess_beacon(MonOpRequestRef op
)
2865 op
->mark_osdmon_event(__func__
);
2866 auto beacon
= static_cast<MOSDBeacon
*>(op
->get_req());
2868 auto session
= beacon
->get_session();
2870 dout(10) << __func__
<< " no monitor session!" << dendl
;
2873 if (!session
->is_capable("osd", MON_CAP_X
)) {
2874 derr
<< __func__
<< " received from entity "
2875 << "with insufficient privileges " << session
->caps
<< dendl
;
2878 // Always forward the beacon to the leader, even if they are the same as
2879 // the old one. The leader will mark as down osds that haven't sent
2880 // beacon for a few minutes.
2884 bool OSDMonitor::prepare_beacon(MonOpRequestRef op
)
2886 op
->mark_osdmon_event(__func__
);
2887 const auto beacon
= static_cast<MOSDBeacon
*>(op
->get_req());
2888 const auto src
= beacon
->get_orig_source();
2889 dout(10) << __func__
<< " " << *beacon
2890 << " from " << src
<< dendl
;
2891 int from
= src
.num();
2893 if (!src
.is_osd() ||
2894 !osdmap
.is_up(from
) ||
2895 beacon
->get_orig_source_inst() != osdmap
.get_inst(from
)) {
2896 dout(1) << " ignoring beacon from non-active osd." << dendl
;
2900 last_osd_report
[from
] = ceph_clock_now();
2901 osd_epochs
[from
] = beacon
->version
;
2903 for (const auto& pg
: beacon
->pgs
) {
2904 last_epoch_clean
.report(pg
, beacon
->min_last_epoch_clean
);
2912 void OSDMonitor::send_latest(MonOpRequestRef op
, epoch_t start
)
2914 op
->mark_osdmon_event(__func__
);
2915 dout(5) << "send_latest to " << op
->get_req()->get_orig_source_inst()
2916 << " start " << start
<< dendl
;
2920 send_incremental(op
, start
);
2924 MOSDMap
*OSDMonitor::build_latest_full()
2926 MOSDMap
*r
= new MOSDMap(mon
->monmap
->fsid
);
2927 get_version_full(osdmap
.get_epoch(), r
->maps
[osdmap
.get_epoch()]);
2928 r
->oldest_map
= get_first_committed();
2929 r
->newest_map
= osdmap
.get_epoch();
2933 MOSDMap
*OSDMonitor::build_incremental(epoch_t from
, epoch_t to
)
2935 dout(10) << "build_incremental [" << from
<< ".." << to
<< "]" << dendl
;
2936 MOSDMap
*m
= new MOSDMap(mon
->monmap
->fsid
);
2937 m
->oldest_map
= get_first_committed();
2938 m
->newest_map
= osdmap
.get_epoch();
2940 for (epoch_t e
= to
; e
>= from
&& e
> 0; e
--) {
2942 int err
= get_version(e
, bl
);
2944 assert(bl
.length());
2945 // if (get_version(e, bl) > 0) {
2946 dout(20) << "build_incremental inc " << e
<< " "
2947 << bl
.length() << " bytes" << dendl
;
2948 m
->incremental_maps
[e
] = bl
;
2950 assert(err
== -ENOENT
);
2951 assert(!bl
.length());
2952 get_version_full(e
, bl
);
2953 if (bl
.length() > 0) {
2954 //else if (get_version("full", e, bl) > 0) {
2955 dout(20) << "build_incremental full " << e
<< " "
2956 << bl
.length() << " bytes" << dendl
;
2959 ceph_abort(); // we should have all maps.
2966 void OSDMonitor::send_full(MonOpRequestRef op
)
2968 op
->mark_osdmon_event(__func__
);
2969 dout(5) << "send_full to " << op
->get_req()->get_orig_source_inst() << dendl
;
2970 mon
->send_reply(op
, build_latest_full());
2973 void OSDMonitor::send_incremental(MonOpRequestRef op
, epoch_t first
)
2975 op
->mark_osdmon_event(__func__
);
2977 MonSession
*s
= op
->get_session();
2981 s
->proxy_con
->has_feature(CEPH_FEATURE_MON_ROUTE_OSDMAP
)) {
2982 // oh, we can tell the other mon to do it
2983 dout(10) << __func__
<< " asking proxying mon to send_incremental from "
2985 MRoute
*r
= new MRoute(s
->proxy_tid
, NULL
);
2986 r
->send_osdmap_first
= first
;
2987 s
->proxy_con
->send_message(r
);
2988 op
->mark_event("reply: send routed send_osdmap_first reply");
2991 send_incremental(first
, s
, false, op
);
2995 void OSDMonitor::send_incremental(epoch_t first
,
2996 MonSession
*session
,
2998 MonOpRequestRef req
)
3000 dout(5) << "send_incremental [" << first
<< ".." << osdmap
.get_epoch() << "]"
3001 << " to " << session
->inst
<< dendl
;
3003 if (first
<= session
->osd_epoch
) {
3004 dout(10) << __func__
<< " " << session
->inst
<< " should already have epoch "
3005 << session
->osd_epoch
<< dendl
;
3006 first
= session
->osd_epoch
+ 1;
3009 if (first
< get_first_committed()) {
3010 first
= get_first_committed();
3012 int err
= get_version_full(first
, bl
);
3014 assert(bl
.length());
3016 dout(20) << "send_incremental starting with base full "
3017 << first
<< " " << bl
.length() << " bytes" << dendl
;
3019 MOSDMap
*m
= new MOSDMap(osdmap
.get_fsid());
3020 m
->oldest_map
= get_first_committed();
3021 m
->newest_map
= osdmap
.get_epoch();
3022 m
->maps
[first
] = bl
;
3025 mon
->send_reply(req
, m
);
3026 session
->osd_epoch
= first
;
3029 session
->con
->send_message(m
);
3030 session
->osd_epoch
= first
;
3035 while (first
<= osdmap
.get_epoch()) {
3036 epoch_t last
= MIN(first
+ g_conf
->osd_map_message_max
- 1,
3037 osdmap
.get_epoch());
3038 MOSDMap
*m
= build_incremental(first
, last
);
3041 // send some maps. it may not be all of them, but it will get them
3043 mon
->send_reply(req
, m
);
3045 session
->con
->send_message(m
);
3048 session
->osd_epoch
= last
;
3054 int OSDMonitor::get_version(version_t ver
, bufferlist
& bl
)
3056 if (inc_osd_cache
.lookup(ver
, &bl
)) {
3059 int ret
= PaxosService::get_version(ver
, bl
);
3061 inc_osd_cache
.add(ver
, bl
);
3066 int OSDMonitor::get_version_full(version_t ver
, bufferlist
& bl
)
3068 if (full_osd_cache
.lookup(ver
, &bl
)) {
3071 int ret
= PaxosService::get_version_full(ver
, bl
);
3073 full_osd_cache
.add(ver
, bl
);
3078 epoch_t
OSDMonitor::blacklist(const entity_addr_t
& a
, utime_t until
)
3080 dout(10) << "blacklist " << a
<< " until " << until
<< dendl
;
3081 pending_inc
.new_blacklist
[a
] = until
;
3082 return pending_inc
.epoch
;
3086 void OSDMonitor::check_osdmap_subs()
3088 dout(10) << __func__
<< dendl
;
3089 if (!osdmap
.get_epoch()) {
3092 auto osdmap_subs
= mon
->session_map
.subs
.find("osdmap");
3093 if (osdmap_subs
== mon
->session_map
.subs
.end()) {
3096 auto p
= osdmap_subs
->second
->begin();
3100 check_osdmap_sub(sub
);
3104 void OSDMonitor::check_osdmap_sub(Subscription
*sub
)
3106 dout(10) << __func__
<< " " << sub
<< " next " << sub
->next
3107 << (sub
->onetime
? " (onetime)":" (ongoing)") << dendl
;
3108 if (sub
->next
<= osdmap
.get_epoch()) {
3110 send_incremental(sub
->next
, sub
->session
, sub
->incremental_onetime
);
3112 sub
->session
->con
->send_message(build_latest_full());
3114 mon
->session_map
.remove_sub(sub
);
3116 sub
->next
= osdmap
.get_epoch() + 1;
3120 void OSDMonitor::check_pg_creates_subs()
3122 if (!mon
->monmap
->get_required_features().contains_all(
3123 ceph::features::mon::FEATURE_LUMINOUS
)) {
3124 // PGMonitor takes care of this in pre-luminous era.
3127 if (!osdmap
.get_num_up_osds()) {
3130 assert(osdmap
.get_up_osd_features() & CEPH_FEATURE_MON_STATEFUL_SUB
);
3131 mon
->with_session_map([this](const MonSessionMap
& session_map
) {
3132 auto pg_creates_subs
= session_map
.subs
.find("osd_pg_creates");
3133 if (pg_creates_subs
== session_map
.subs
.end()) {
3136 for (auto sub
: *pg_creates_subs
->second
) {
3137 check_pg_creates_sub(sub
);
3142 void OSDMonitor::check_pg_creates_sub(Subscription
*sub
)
3144 dout(20) << __func__
<< " .. " << sub
->session
->inst
<< dendl
;
3145 assert(sub
->type
== "osd_pg_creates");
3146 // only send these if the OSD is up. we will check_subs() when they do
3147 // come up so they will get the creates then.
3148 if (sub
->session
->inst
.name
.is_osd() &&
3149 mon
->osdmon()->osdmap
.is_up(sub
->session
->inst
.name
.num())) {
3150 sub
->next
= send_pg_creates(sub
->session
->inst
.name
.num(),
3151 sub
->session
->con
.get(),
3156 void OSDMonitor::do_application_enable(int64_t pool_id
,
3157 const std::string
&app_name
)
3159 assert(paxos
->is_plugged());
3161 dout(20) << __func__
<< ": pool_id=" << pool_id
<< ", app_name=" << app_name
3164 auto pp
= osdmap
.get_pg_pool(pool_id
);
3165 assert(pp
!= nullptr);
3168 if (pending_inc
.new_pools
.count(pool_id
)) {
3169 p
= pending_inc
.new_pools
[pool_id
];
3172 p
.application_metadata
.insert({app_name
, {}});
3173 p
.last_change
= pending_inc
.epoch
;
3174 pending_inc
.new_pools
[pool_id
] = p
;
3177 unsigned OSDMonitor::scan_for_creating_pgs(
3178 const mempool::osdmap::map
<int64_t,pg_pool_t
>& pools
,
3179 const mempool::osdmap::set
<int64_t>& removed_pools
,
3181 creating_pgs_t
* creating_pgs
) const
3183 unsigned queued
= 0;
3184 for (auto& p
: pools
) {
3185 int64_t poolid
= p
.first
;
3186 const pg_pool_t
& pool
= p
.second
;
3187 int ruleno
= osdmap
.crush
->find_rule(pool
.get_crush_rule(),
3188 pool
.get_type(), pool
.get_size());
3189 if (ruleno
< 0 || !osdmap
.crush
->rule_exists(ruleno
))
3192 const auto last_scan_epoch
= creating_pgs
->last_scan_epoch
;
3193 const auto created
= pool
.get_last_change();
3194 if (last_scan_epoch
&& created
<= last_scan_epoch
) {
3195 dout(10) << __func__
<< " no change in pool " << poolid
3196 << " " << pool
<< dendl
;
3199 if (removed_pools
.count(poolid
)) {
3200 dout(10) << __func__
<< " pool is being removed: " << poolid
3201 << " " << pool
<< dendl
;
3204 dout(10) << __func__
<< " queueing pool create for " << poolid
3205 << " " << pool
<< dendl
;
3206 if (creating_pgs
->create_pool(poolid
, pool
.get_pg_num(),
3207 created
, modified
)) {
3214 void OSDMonitor::update_creating_pgs()
3216 dout(10) << __func__
<< " " << creating_pgs
.pgs
.size() << " pgs creating, "
3217 << creating_pgs
.queue
.size() << " pools in queue" << dendl
;
3218 decltype(creating_pgs_by_osd_epoch
) new_pgs_by_osd_epoch
;
3219 std::lock_guard
<std::mutex
> l(creating_pgs_lock
);
3220 for (const auto& pg
: creating_pgs
.pgs
) {
3221 int acting_primary
= -1;
3222 auto pgid
= pg
.first
;
3223 auto mapped
= pg
.second
.first
;
3224 dout(20) << __func__
<< " looking up " << pgid
<< "@" << mapped
<< dendl
;
3225 mapping
.get(pgid
, nullptr, nullptr, nullptr, &acting_primary
);
3226 // check the previous creating_pgs, look for the target to whom the pg was
3227 // previously mapped
3228 for (const auto& pgs_by_epoch
: creating_pgs_by_osd_epoch
) {
3229 const auto last_acting_primary
= pgs_by_epoch
.first
;
3230 for (auto& pgs
: pgs_by_epoch
.second
) {
3231 if (pgs
.second
.count(pgid
)) {
3232 if (last_acting_primary
== acting_primary
) {
3235 dout(20) << __func__
<< " " << pgid
<< " "
3236 << " acting_primary:" << last_acting_primary
3237 << " -> " << acting_primary
<< dendl
;
3238 // note epoch if the target of the create message changed.
3239 mapped
= mapping
.get_epoch();
3244 mapped
= mapping
.get_epoch();
3248 dout(10) << __func__
<< " will instruct osd." << acting_primary
3249 << " to create " << pgid
<< "@" << mapped
<< dendl
;
3250 new_pgs_by_osd_epoch
[acting_primary
][mapped
].insert(pgid
);
3252 creating_pgs_by_osd_epoch
= std::move(new_pgs_by_osd_epoch
);
3253 creating_pgs_epoch
= mapping
.get_epoch();
3256 epoch_t
OSDMonitor::send_pg_creates(int osd
, Connection
*con
, epoch_t next
) const
3258 dout(30) << __func__
<< " osd." << osd
<< " next=" << next
3259 << " " << creating_pgs_by_osd_epoch
<< dendl
;
3260 std::lock_guard
<std::mutex
> l(creating_pgs_lock
);
3261 auto creating_pgs_by_epoch
= creating_pgs_by_osd_epoch
.find(osd
);
3262 if (creating_pgs_by_epoch
== creating_pgs_by_osd_epoch
.end())
3264 assert(!creating_pgs_by_epoch
->second
.empty());
3266 MOSDPGCreate
*m
= nullptr;
3268 for (auto epoch_pgs
= creating_pgs_by_epoch
->second
.lower_bound(next
);
3269 epoch_pgs
!= creating_pgs_by_epoch
->second
.end(); ++epoch_pgs
) {
3270 auto epoch
= epoch_pgs
->first
;
3271 auto& pgs
= epoch_pgs
->second
;
3272 dout(20) << __func__
<< " osd." << osd
<< " from " << next
3273 << " : epoch " << epoch
<< " " << pgs
.size() << " pgs" << dendl
;
3275 for (auto& pg
: pgs
) {
3277 m
= new MOSDPGCreate(creating_pgs_epoch
);
3278 // Need the create time from the monitor using its clock to set
3279 // last_scrub_stamp upon pg creation.
3280 auto create
= creating_pgs
.pgs
.find(pg
);
3281 assert(create
!= creating_pgs
.pgs
.end());
3282 m
->mkpg
.emplace(pg
, pg_create_t
{create
->second
.first
, pg
, 0});
3283 m
->ctimes
.emplace(pg
, create
->second
.second
);
3284 dout(20) << __func__
<< " will create " << pg
3285 << " at " << create
->second
.first
<< dendl
;
3289 dout(20) << __func__
<< " osd." << osd
<< " from " << next
3290 << " has nothing to send" << dendl
;
3293 con
->send_message(m
);
3294 // sub is current through last + 1
3301 void OSDMonitor::tick()
3303 if (!is_active()) return;
3305 dout(10) << osdmap
<< dendl
;
3307 if (!mon
->is_leader()) return;
3309 bool do_propose
= false;
3310 utime_t now
= ceph_clock_now();
3312 if (osdmap
.require_osd_release
>= CEPH_RELEASE_LUMINOUS
&&
3313 mon
->monmap
->get_required_features().contains_all(
3314 ceph::features::mon::FEATURE_LUMINOUS
)) {
3315 if (handle_osd_timeouts(now
, last_osd_report
)) {
3321 if (check_failures(now
))
3324 // mark down osds out?
3326 /* can_mark_out() checks if we can mark osds as being out. The -1 has no
3327 * influence at all. The decision is made based on the ratio of "in" osds,
3328 * and the function returns false if this ratio is lower that the minimum
3329 * ratio set by g_conf->mon_osd_min_in_ratio. So it's not really up to us.
3331 if (can_mark_out(-1)) {
3332 set
<int> down_cache
; // quick cache of down subtrees
3334 map
<int,utime_t
>::iterator i
= down_pending_out
.begin();
3335 while (i
!= down_pending_out
.end()) {
3341 if (osdmap
.is_down(o
) &&
3344 utime_t
orig_grace(g_conf
->mon_osd_down_out_interval
, 0);
3345 utime_t grace
= orig_grace
;
3346 double my_grace
= 0.0;
3348 if (g_conf
->mon_osd_adjust_down_out_interval
) {
3349 // scale grace period the same way we do the heartbeat grace.
3350 const osd_xinfo_t
& xi
= osdmap
.get_xinfo(o
);
3351 double halflife
= (double)g_conf
->mon_osd_laggy_halflife
;
3352 double decay_k
= ::log(.5) / halflife
;
3353 double decay
= exp((double)down
* decay_k
);
3354 dout(20) << "osd." << o
<< " laggy halflife " << halflife
<< " decay_k " << decay_k
3355 << " down for " << down
<< " decay " << decay
<< dendl
;
3356 my_grace
= decay
* (double)xi
.laggy_interval
* xi
.laggy_probability
;
3360 // is this an entire large subtree down?
3361 if (g_conf
->mon_osd_down_out_subtree_limit
.length()) {
3362 int type
= osdmap
.crush
->get_type_id(g_conf
->mon_osd_down_out_subtree_limit
);
3364 if (osdmap
.containing_subtree_is_down(g_ceph_context
, o
, type
, &down_cache
)) {
3365 dout(10) << "tick entire containing " << g_conf
->mon_osd_down_out_subtree_limit
3366 << " subtree for osd." << o
<< " is down; resetting timer" << dendl
;
3367 // reset timer, too.
3368 down_pending_out
[o
] = now
;
3374 bool down_out
= !osdmap
.is_destroyed(o
) &&
3375 g_conf
->mon_osd_down_out_interval
> 0 && down
.sec() >= grace
;
3376 bool destroyed_out
= osdmap
.is_destroyed(o
) &&
3377 g_conf
->mon_osd_destroyed_out_interval
> 0 &&
3378 // this is not precise enough as we did not make a note when this osd
3379 // was marked as destroyed, but let's not bother with that
3380 // complexity for now.
3381 down
.sec() >= g_conf
->mon_osd_destroyed_out_interval
;
3382 if (down_out
|| destroyed_out
) {
3383 dout(10) << "tick marking osd." << o
<< " OUT after " << down
3384 << " sec (target " << grace
<< " = " << orig_grace
<< " + " << my_grace
<< ")" << dendl
;
3385 pending_inc
.new_weight
[o
] = CEPH_OSD_OUT
;
3387 // set the AUTOOUT bit.
3388 if (pending_inc
.new_state
.count(o
) == 0)
3389 pending_inc
.new_state
[o
] = 0;
3390 pending_inc
.new_state
[o
] |= CEPH_OSD_AUTOOUT
;
3392 // remember previous weight
3393 if (pending_inc
.new_xinfo
.count(o
) == 0)
3394 pending_inc
.new_xinfo
[o
] = osdmap
.osd_xinfo
[o
];
3395 pending_inc
.new_xinfo
[o
].old_weight
= osdmap
.osd_weight
[o
];
3399 mon
->clog
->info() << "Marking osd." << o
<< " out (has been down for "
3400 << int(down
.sec()) << " seconds)";
3405 down_pending_out
.erase(o
);
3408 dout(10) << "tick NOOUT flag set, not checking down osds" << dendl
;
3411 // expire blacklisted items?
3412 for (ceph::unordered_map
<entity_addr_t
,utime_t
>::iterator p
= osdmap
.blacklist
.begin();
3413 p
!= osdmap
.blacklist
.end();
3415 if (p
->second
< now
) {
3416 dout(10) << "expiring blacklist item " << p
->first
<< " expired " << p
->second
<< " < now " << now
<< dendl
;
3417 pending_inc
.old_blacklist
.push_back(p
->first
);
3422 // if map full setting has changed, get that info out there!
3423 if (osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
&&
3424 mon
->pgservice
->is_readable()) {
3425 // for pre-luminous compat only!
3426 if (mon
->pgservice
->have_full_osds()) {
3427 dout(5) << "There are full osds, setting full flag" << dendl
;
3428 add_flag(CEPH_OSDMAP_FULL
);
3429 } else if (osdmap
.test_flag(CEPH_OSDMAP_FULL
)){
3430 dout(10) << "No full osds, removing full flag" << dendl
;
3431 remove_flag(CEPH_OSDMAP_FULL
);
3434 if (mon
->pgservice
->have_nearfull_osds()) {
3435 dout(5) << "There are near full osds, setting nearfull flag" << dendl
;
3436 add_flag(CEPH_OSDMAP_NEARFULL
);
3437 } else if (osdmap
.test_flag(CEPH_OSDMAP_NEARFULL
)){
3438 dout(10) << "No near full osds, removing nearfull flag" << dendl
;
3439 remove_flag(CEPH_OSDMAP_NEARFULL
);
3441 if (pending_inc
.new_flags
!= -1 &&
3442 (pending_inc
.new_flags
^ osdmap
.flags
) & (CEPH_OSDMAP_FULL
| CEPH_OSDMAP_NEARFULL
)) {
3443 dout(1) << "New setting for" <<
3444 (pending_inc
.new_flags
& CEPH_OSDMAP_FULL
? " CEPH_OSDMAP_FULL" : "") <<
3445 (pending_inc
.new_flags
& CEPH_OSDMAP_NEARFULL
? " CEPH_OSDMAP_NEARFULL" : "")
3446 << " -- doing propose" << dendl
;
3451 if (update_pools_status())
3455 !pending_inc
.new_pg_temp
.empty()) // also propose if we adjusted pg_temp
3459 bool OSDMonitor::handle_osd_timeouts(const utime_t
&now
,
3460 std::map
<int,utime_t
> &last_osd_report
)
3462 utime_t
timeo(g_conf
->mon_osd_report_timeout
, 0);
3463 if (now
- mon
->get_leader_since() < timeo
) {
3464 // We haven't been the leader for long enough to consider OSD timeouts
3468 int max_osd
= osdmap
.get_max_osd();
3469 bool new_down
= false;
3471 for (int i
=0; i
< max_osd
; ++i
) {
3472 dout(30) << __func__
<< ": checking up on osd " << i
<< dendl
;
3473 if (!osdmap
.exists(i
)) {
3474 last_osd_report
.erase(i
); // if any
3477 if (!osdmap
.is_up(i
))
3479 const std::map
<int,utime_t
>::const_iterator t
= last_osd_report
.find(i
);
3480 if (t
== last_osd_report
.end()) {
3481 // it wasn't in the map; start the timer.
3482 last_osd_report
[i
] = now
;
3483 } else if (can_mark_down(i
)) {
3484 utime_t diff
= now
- t
->second
;
3486 mon
->clog
->info() << "osd." << i
<< " marked down after no beacon for "
3487 << diff
<< " seconds";
3488 derr
<< "no beacon from osd." << i
<< " since " << t
->second
3489 << ", " << diff
<< " seconds ago. marking down" << dendl
;
3490 pending_inc
.new_state
[i
] = CEPH_OSD_UP
;
3498 void OSDMonitor::get_health(list
<pair
<health_status_t
,string
> >& summary
,
3499 list
<pair
<health_status_t
,string
> > *detail
,
3500 CephContext
*cct
) const
3502 int num_osds
= osdmap
.get_num_osds();
3504 if (num_osds
== 0) {
3505 summary
.push_back(make_pair(HEALTH_ERR
, "no osds"));
3507 int num_in_osds
= 0;
3508 int num_down_in_osds
= 0;
3510 set
<int> down_in_osds
;
3511 set
<int> up_in_osds
;
3512 set
<int> subtree_up
;
3513 unordered_map
<int, set
<int> > subtree_type_down
;
3514 unordered_map
<int, int> num_osds_subtree
;
3515 int max_type
= osdmap
.crush
->get_max_type_id();
3517 for (int i
= 0; i
< osdmap
.get_max_osd(); i
++) {
3518 if (!osdmap
.exists(i
)) {
3519 if (osdmap
.crush
->item_exists(i
)) {
3524 if (osdmap
.is_out(i
))
3527 if (down_in_osds
.count(i
) || up_in_osds
.count(i
))
3529 if (!osdmap
.is_up(i
)) {
3530 down_in_osds
.insert(i
);
3533 for (int type
= 0; type
<= max_type
; type
++) {
3534 if (!osdmap
.crush
->get_type_name(type
))
3536 int r
= osdmap
.crush
->get_immediate_parent_id(current
, &parent_id
);
3539 // break early if this parent is already marked as up
3540 if (subtree_up
.count(parent_id
))
3542 type
= osdmap
.crush
->get_bucket_type(parent_id
);
3543 if (!osdmap
.subtree_type_is_down(
3544 g_ceph_context
, parent_id
, type
,
3545 &down_in_osds
, &up_in_osds
, &subtree_up
, &subtree_type_down
))
3547 current
= parent_id
;
3552 // calculate the number of down osds in each down subtree and
3553 // store it in num_osds_subtree
3554 for (int type
= 1; type
<= max_type
; type
++) {
3555 if (!osdmap
.crush
->get_type_name(type
))
3557 for (auto j
= subtree_type_down
[type
].begin();
3558 j
!= subtree_type_down
[type
].end();
3562 int num
= osdmap
.crush
->get_children(*j
, &children
);
3563 num_osds_subtree
[*j
] = num
;
3567 int num_children
= osdmap
.crush
->get_children(*j
, &children
);
3568 if (num_children
== 0)
3570 for (auto l
= children
.begin(); l
!= children
.end(); ++l
) {
3571 if (num_osds_subtree
[*l
] > 0) {
3572 num
= num
+ num_osds_subtree
[*l
];
3575 num_osds_subtree
[*j
] = num
;
3579 num_down_in_osds
= down_in_osds
.size();
3580 assert(num_down_in_osds
<= num_in_osds
);
3581 if (num_down_in_osds
> 0) {
3582 // summary of down subtree types and osds
3583 for (int type
= max_type
; type
> 0; type
--) {
3584 if (!osdmap
.crush
->get_type_name(type
))
3586 if (subtree_type_down
[type
].size() > 0) {
3588 ss
<< subtree_type_down
[type
].size() << " "
3589 << osdmap
.crush
->get_type_name(type
);
3590 if (subtree_type_down
[type
].size() > 1) {
3593 int sum_down_osds
= 0;
3594 for (auto j
= subtree_type_down
[type
].begin();
3595 j
!= subtree_type_down
[type
].end();
3597 sum_down_osds
= sum_down_osds
+ num_osds_subtree
[*j
];
3599 ss
<< " (" << sum_down_osds
<< " osds) down";
3600 summary
.push_back(make_pair(HEALTH_WARN
, ss
.str()));
3604 ss
<< down_in_osds
.size() << " osds down";
3605 summary
.push_back(make_pair(HEALTH_WARN
, ss
.str()));
3608 // details of down subtree types
3609 for (int type
= max_type
; type
> 0; type
--) {
3610 if (!osdmap
.crush
->get_type_name(type
))
3612 for (auto j
= subtree_type_down
[type
].rbegin();
3613 j
!= subtree_type_down
[type
].rend();
3616 ss
<< osdmap
.crush
->get_type_name(type
);
3618 ss
<< osdmap
.crush
->get_item_name(*j
);
3619 // at the top level, do not print location
3620 if (type
!= max_type
) {
3622 ss
<< osdmap
.crush
->get_full_location_ordered_string(*j
);
3625 int num
= num_osds_subtree
[*j
];
3626 ss
<< " (" << num
<< " osds)";
3628 detail
->push_back(make_pair(HEALTH_WARN
, ss
.str()));
3631 // details of down osds
3632 for (auto it
= down_in_osds
.begin(); it
!= down_in_osds
.end(); ++it
) {
3634 ss
<< "osd." << *it
<< " (";
3635 ss
<< osdmap
.crush
->get_full_location_ordered_string(*it
);
3637 detail
->push_back(make_pair(HEALTH_WARN
, ss
.str()));
3642 if (!osds
.empty()) {
3644 ss
<< osds
.size() << " osds exist in the crush map but not in the osdmap";
3645 summary
.push_back(make_pair(HEALTH_WARN
, ss
.str()));
3647 ss
<< " (osds: " << osds
<< ")";
3648 detail
->push_back(make_pair(HEALTH_WARN
, ss
.str()));
3652 // note: we leave it to ceph-mgr to generate details health warnings
3653 // with actual osd utilizations
3656 uint64_t warn_flags
=
3658 CEPH_OSDMAP_PAUSERD
|
3659 CEPH_OSDMAP_PAUSEWR
|
3660 CEPH_OSDMAP_PAUSEREC
|
3662 CEPH_OSDMAP_NODOWN
|
3665 CEPH_OSDMAP_NOBACKFILL
|
3666 CEPH_OSDMAP_NORECOVER
|
3667 CEPH_OSDMAP_NOSCRUB
|
3668 CEPH_OSDMAP_NODEEP_SCRUB
|
3669 CEPH_OSDMAP_NOTIERAGENT
|
3670 CEPH_OSDMAP_NOREBALANCE
;
3671 if (osdmap
.test_flag(warn_flags
)) {
3673 ss
<< osdmap
.get_flag_string(osdmap
.get_flags() & warn_flags
)
3675 summary
.push_back(make_pair(HEALTH_WARN
, ss
.str()));
3677 detail
->push_back(make_pair(HEALTH_WARN
, ss
.str()));
3680 // old crush tunables?
3681 if (g_conf
->mon_warn_on_legacy_crush_tunables
) {
3682 string min
= osdmap
.crush
->get_min_required_version();
3683 if (min
< g_conf
->mon_crush_min_required_version
) {
3685 ss
<< "crush map has legacy tunables (require " << min
3686 << ", min is " << g_conf
->mon_crush_min_required_version
<< ")";
3687 summary
.push_back(make_pair(HEALTH_WARN
, ss
.str()));
3689 ss
<< "; see http://docs.ceph.com/docs/master/rados/operations/crush-map/#tunables";
3690 detail
->push_back(make_pair(HEALTH_WARN
, ss
.str()));
3694 if (g_conf
->mon_warn_on_crush_straw_calc_version_zero
) {
3695 if (osdmap
.crush
->get_straw_calc_version() == 0) {
3697 ss
<< "crush map has straw_calc_version=0";
3698 summary
.push_back(make_pair(HEALTH_WARN
, ss
.str()));
3700 ss
<< "; see http://docs.ceph.com/docs/master/rados/operations/crush-map/#tunables";
3701 detail
->push_back(make_pair(HEALTH_WARN
, ss
.str()));
3706 // hit_set-less cache_mode?
3707 if (g_conf
->mon_warn_on_cache_pools_without_hit_sets
) {
3708 int problem_cache_pools
= 0;
3709 for (map
<int64_t, pg_pool_t
>::const_iterator p
= osdmap
.pools
.begin();
3710 p
!= osdmap
.pools
.end();
3712 const pg_pool_t
& info
= p
->second
;
3713 if (info
.cache_mode_requires_hit_set() &&
3714 info
.hit_set_params
.get_type() == HitSet::TYPE_NONE
) {
3715 ++problem_cache_pools
;
3718 ss
<< "pool '" << osdmap
.get_pool_name(p
->first
)
3719 << "' with cache_mode " << info
.get_cache_mode_name()
3720 << " needs hit_set_type to be set but it is not";
3721 detail
->push_back(make_pair(HEALTH_WARN
, ss
.str()));
3725 if (problem_cache_pools
) {
3727 ss
<< problem_cache_pools
<< " cache pools are missing hit_sets";
3728 summary
.push_back(make_pair(HEALTH_WARN
, ss
.str()));
3732 if (osdmap
.crush
->has_multirule_rulesets()) {
3734 ss
<< "CRUSH map contains multirule rulesets";
3735 summary
.push_back(make_pair(HEALTH_WARN
, ss
.str()));
3737 ss
<< "; please manually fix the map";
3738 detail
->push_back(make_pair(HEALTH_WARN
, ss
.str()));
3742 // Not using 'sortbitwise' and should be?
3743 if (!osdmap
.test_flag(CEPH_OSDMAP_SORTBITWISE
) &&
3744 (osdmap
.get_up_osd_features() &
3745 CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT
)) {
3747 ss
<< "no legacy OSD present but 'sortbitwise' flag is not set";
3748 summary
.push_back(make_pair(HEALTH_WARN
, ss
.str()));
3751 // Warn if 'mon_osd_down_out_interval' is set to zero.
3752 // Having this option set to zero on the leader acts much like the
3753 // 'noout' flag. It's hard to figure out what's going wrong with clusters
3754 // without the 'noout' flag set but acting like that just the same, so
3755 // we report a HEALTH_WARN in case this option is set to zero.
3756 // This is an ugly hack to get the warning out, but until we find a way
3757 // to spread global options throughout the mon cluster and have all mons
3758 // using a base set of the same options, we need to work around this sort
3760 // There's also the obvious drawback that if this is set on a single
3761 // monitor on a 3-monitor cluster, this warning will only be shown every
3762 // third monitor connection.
3763 if (g_conf
->mon_warn_on_osd_down_out_interval_zero
&&
3764 g_conf
->mon_osd_down_out_interval
== 0) {
3766 ss
<< "mon." << mon
->name
<< " has mon_osd_down_out_interval set to 0";
3767 summary
.push_back(make_pair(HEALTH_WARN
, ss
.str()));
3769 ss
<< "; this has the same effect as the 'noout' flag";
3770 detail
->push_back(make_pair(HEALTH_WARN
, ss
.str()));
3774 // warn about upgrade flags that can be set but are not.
3775 if (g_conf
->mon_debug_no_require_luminous
) {
3776 // ignore these checks
3777 } else if (HAVE_FEATURE(osdmap
.get_up_osd_features(), SERVER_LUMINOUS
) &&
3778 osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
3779 string msg
= "all OSDs are running luminous or later but"
3780 " require_osd_release < luminous";
3781 summary
.push_back(make_pair(HEALTH_WARN
, msg
));
3783 detail
->push_back(make_pair(HEALTH_WARN
, msg
));
3785 } else if (HAVE_FEATURE(osdmap
.get_up_osd_features(), SERVER_KRAKEN
) &&
3786 osdmap
.require_osd_release
< CEPH_RELEASE_KRAKEN
) {
3787 string msg
= "all OSDs are running kraken or later but"
3788 " require_osd_release < kraken";
3789 summary
.push_back(make_pair(HEALTH_WARN
, msg
));
3791 detail
->push_back(make_pair(HEALTH_WARN
, msg
));
3793 } else if (HAVE_FEATURE(osdmap
.get_up_osd_features(), SERVER_JEWEL
) &&
3794 osdmap
.require_osd_release
< CEPH_RELEASE_JEWEL
) {
3795 string msg
= "all OSDs are running jewel or later but"
3796 " require_osd_release < jewel";
3797 summary
.push_back(make_pair(HEALTH_WARN
, msg
));
3799 detail
->push_back(make_pair(HEALTH_WARN
, msg
));
3803 for (auto it
: osdmap
.get_pools()) {
3804 const pg_pool_t
&pool
= it
.second
;
3805 if (pool
.has_flag(pg_pool_t::FLAG_FULL
)) {
3806 const string
& pool_name
= osdmap
.get_pool_name(it
.first
);
3808 ss
<< "pool '" << pool_name
<< "' is full";
3809 summary
.push_back(make_pair(HEALTH_WARN
, ss
.str()));
3811 detail
->push_back(make_pair(HEALTH_WARN
, ss
.str()));
3817 void OSDMonitor::dump_info(Formatter
*f
)
3819 f
->open_object_section("osdmap");
3823 f
->open_array_section("osd_metadata");
3824 for (int i
=0; i
<osdmap
.get_max_osd(); ++i
) {
3825 if (osdmap
.exists(i
)) {
3826 f
->open_object_section("osd");
3827 f
->dump_unsigned("id", i
);
3828 dump_osd_metadata(i
, f
, NULL
);
3834 f
->dump_unsigned("osdmap_first_committed", get_first_committed());
3835 f
->dump_unsigned("osdmap_last_committed", get_last_committed());
3837 f
->open_object_section("crushmap");
3838 osdmap
.crush
->dump(f
);
3843 enum osd_pool_get_choices
{
3844 SIZE
, MIN_SIZE
, CRASH_REPLAY_INTERVAL
,
3845 PG_NUM
, PGP_NUM
, CRUSH_RULE
, HASHPSPOOL
,
3846 NODELETE
, NOPGCHANGE
, NOSIZECHANGE
,
3847 WRITE_FADVISE_DONTNEED
, NOSCRUB
, NODEEP_SCRUB
,
3848 HIT_SET_TYPE
, HIT_SET_PERIOD
, HIT_SET_COUNT
, HIT_SET_FPP
,
3849 USE_GMT_HITSET
, AUID
, TARGET_MAX_OBJECTS
, TARGET_MAX_BYTES
,
3850 CACHE_TARGET_DIRTY_RATIO
, CACHE_TARGET_DIRTY_HIGH_RATIO
,
3851 CACHE_TARGET_FULL_RATIO
,
3852 CACHE_MIN_FLUSH_AGE
, CACHE_MIN_EVICT_AGE
,
3853 ERASURE_CODE_PROFILE
, MIN_READ_RECENCY_FOR_PROMOTE
,
3854 MIN_WRITE_RECENCY_FOR_PROMOTE
, FAST_READ
,
3855 HIT_SET_GRADE_DECAY_RATE
, HIT_SET_SEARCH_LAST_N
,
3856 SCRUB_MIN_INTERVAL
, SCRUB_MAX_INTERVAL
, DEEP_SCRUB_INTERVAL
,
3857 RECOVERY_PRIORITY
, RECOVERY_OP_PRIORITY
, SCRUB_PRIORITY
,
3858 COMPRESSION_MODE
, COMPRESSION_ALGORITHM
, COMPRESSION_REQUIRED_RATIO
,
3859 COMPRESSION_MAX_BLOB_SIZE
, COMPRESSION_MIN_BLOB_SIZE
,
3860 CSUM_TYPE
, CSUM_MAX_BLOCK
, CSUM_MIN_BLOCK
};
3862 std::set
<osd_pool_get_choices
>
3863 subtract_second_from_first(const std::set
<osd_pool_get_choices
>& first
,
3864 const std::set
<osd_pool_get_choices
>& second
)
3866 std::set
<osd_pool_get_choices
> result
;
3867 std::set_difference(first
.begin(), first
.end(),
3868 second
.begin(), second
.end(),
3869 std::inserter(result
, result
.end()));
3875 bool OSDMonitor::preprocess_command(MonOpRequestRef op
)
3877 op
->mark_osdmon_event(__func__
);
3878 MMonCommand
*m
= static_cast<MMonCommand
*>(op
->get_req());
3881 stringstream ss
, ds
;
3883 map
<string
, cmd_vartype
> cmdmap
;
3884 if (!cmdmap_from_json(m
->cmd
, &cmdmap
, ss
)) {
3885 string rs
= ss
.str();
3886 mon
->reply_command(op
, -EINVAL
, rs
, get_last_committed());
3890 MonSession
*session
= m
->get_session();
3892 mon
->reply_command(op
, -EACCES
, "access denied", get_last_committed());
3897 cmd_getval(g_ceph_context
, cmdmap
, "prefix", prefix
);
3900 cmd_getval(g_ceph_context
, cmdmap
, "format", format
, string("plain"));
3901 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
));
3903 if (prefix
== "osd stat") {
3904 osdmap
.print_summary(f
.get(), ds
, "");
3910 else if (prefix
== "osd perf" ||
3911 prefix
== "osd blocked-by") {
3912 r
= mon
->pgservice
->process_pg_command(prefix
, cmdmap
,
3913 osdmap
, f
.get(), &ss
, &rdata
);
3915 else if (prefix
== "osd dump" ||
3916 prefix
== "osd tree" ||
3917 prefix
== "osd ls" ||
3918 prefix
== "osd getmap" ||
3919 prefix
== "osd getcrushmap" ||
3920 prefix
== "osd ls-tree") {
3925 cmd_getval(g_ceph_context
, cmdmap
, "epoch", epochnum
, (int64_t)osdmap
.get_epoch());
3928 bufferlist osdmap_bl
;
3929 int err
= get_version_full(epoch
, osdmap_bl
);
3930 if (err
== -ENOENT
) {
3932 ss
<< "there is no map for epoch " << epoch
;
3936 assert(osdmap_bl
.length());
3939 if (epoch
== osdmap
.get_epoch()) {
3943 p
->decode(osdmap_bl
);
3946 auto sg
= make_scope_guard([&] {
3952 if (prefix
== "osd dump") {
3955 f
->open_object_section("osdmap");
3965 } else if (prefix
== "osd ls") {
3967 f
->open_array_section("osds");
3968 for (int i
= 0; i
< osdmap
.get_max_osd(); i
++) {
3969 if (osdmap
.exists(i
)) {
3970 f
->dump_int("osd", i
);
3977 for (int i
= 0; i
< osdmap
.get_max_osd(); i
++) {
3978 if (osdmap
.exists(i
)) {
3987 } else if (prefix
== "osd tree") {
3988 vector
<string
> states
;
3989 cmd_getval(g_ceph_context
, cmdmap
, "states", states
);
3990 unsigned filter
= 0;
3991 for (auto& s
: states
) {
3993 filter
|= OSDMap::DUMP_UP
;
3994 } else if (s
== "down") {
3995 filter
|= OSDMap::DUMP_DOWN
;
3996 } else if (s
== "in") {
3997 filter
|= OSDMap::DUMP_IN
;
3998 } else if (s
== "out") {
3999 filter
|= OSDMap::DUMP_OUT
;
4000 } else if (s
== "destroyed") {
4001 filter
|= OSDMap::DUMP_DESTROYED
;
4003 ss
<< "unrecognized state '" << s
<< "'";
4008 if ((filter
& (OSDMap::DUMP_IN
|OSDMap::DUMP_OUT
)) ==
4009 (OSDMap::DUMP_IN
|OSDMap::DUMP_OUT
)) {
4010 ss
<< "cannot specify both 'in' and 'out'";
4014 if (((filter
& (OSDMap::DUMP_UP
|OSDMap::DUMP_DOWN
)) ==
4015 (OSDMap::DUMP_UP
|OSDMap::DUMP_DOWN
)) ||
4016 ((filter
& (OSDMap::DUMP_UP
|OSDMap::DUMP_DESTROYED
)) ==
4017 (OSDMap::DUMP_UP
|OSDMap::DUMP_DESTROYED
)) ||
4018 ((filter
& (OSDMap::DUMP_DOWN
|OSDMap::DUMP_DESTROYED
)) ==
4019 (OSDMap::DUMP_DOWN
|OSDMap::DUMP_DESTROYED
))) {
4020 ss
<< "can specify only one of 'up', 'down' and 'destroyed'";
4025 f
->open_object_section("tree");
4026 p
->print_tree(f
.get(), NULL
, filter
);
4030 p
->print_tree(NULL
, &ds
, filter
);
4033 } else if (prefix
== "osd getmap") {
4034 rdata
.append(osdmap_bl
);
4035 ss
<< "got osdmap epoch " << p
->get_epoch();
4036 } else if (prefix
== "osd getcrushmap") {
4037 p
->crush
->encode(rdata
, mon
->get_quorum_con_features());
4038 ss
<< p
->get_crush_version();
4039 } else if (prefix
== "osd ls-tree") {
4041 cmd_getval(g_ceph_context
, cmdmap
, "name", bucket_name
);
4043 r
= p
->get_osds_by_bucket_name(bucket_name
, &osds
);
4045 ss
<< "\"" << bucket_name
<< "\" does not exist";
4048 ss
<< "can not parse bucket name:\"" << bucket_name
<< "\"";
4053 f
->open_array_section("osds");
4054 for (auto &i
: osds
) {
4055 if (osdmap
.exists(i
)) {
4056 f
->dump_int("osd", i
);
4063 for (auto &i
: osds
) {
4064 if (osdmap
.exists(i
)) {
4075 } else if (prefix
== "osd df") {
4077 cmd_getval(g_ceph_context
, cmdmap
, "output_method", method
);
4078 print_osd_utilization(osdmap
, mon
->pgservice
, ds
,
4079 f
.get(), method
== "tree");
4081 } else if (prefix
== "osd getmaxosd") {
4083 f
->open_object_section("getmaxosd");
4084 f
->dump_unsigned("epoch", osdmap
.get_epoch());
4085 f
->dump_int("max_osd", osdmap
.get_max_osd());
4089 ds
<< "max_osd = " << osdmap
.get_max_osd() << " in epoch " << osdmap
.get_epoch();
4092 } else if (prefix
== "osd utilization") {
4094 osdmap
.summarize_mapping_stats(NULL
, NULL
, &out
, f
.get());
4101 } else if (prefix
== "osd find") {
4103 if (!cmd_getval(g_ceph_context
, cmdmap
, "id", osd
)) {
4104 ss
<< "unable to parse osd id value '"
4105 << cmd_vartype_stringify(cmdmap
["id"]) << "'";
4109 if (!osdmap
.exists(osd
)) {
4110 ss
<< "osd." << osd
<< " does not exist";
4115 cmd_getval(g_ceph_context
, cmdmap
, "format", format
);
4116 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
, "json-pretty", "json-pretty"));
4117 f
->open_object_section("osd_location");
4118 f
->dump_int("osd", osd
);
4119 f
->dump_stream("ip") << osdmap
.get_addr(osd
);
4120 f
->open_object_section("crush_location");
4121 map
<string
,string
> loc
= osdmap
.crush
->get_full_location(osd
);
4122 for (map
<string
,string
>::iterator p
= loc
.begin(); p
!= loc
.end(); ++p
)
4123 f
->dump_string(p
->first
.c_str(), p
->second
);
4127 } else if (prefix
== "osd metadata") {
4129 if (cmd_vartype_stringify(cmdmap
["id"]).size() &&
4130 !cmd_getval(g_ceph_context
, cmdmap
, "id", osd
)) {
4131 ss
<< "unable to parse osd id value '"
4132 << cmd_vartype_stringify(cmdmap
["id"]) << "'";
4136 if (osd
>= 0 && !osdmap
.exists(osd
)) {
4137 ss
<< "osd." << osd
<< " does not exist";
4142 cmd_getval(g_ceph_context
, cmdmap
, "format", format
);
4143 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
, "json-pretty", "json-pretty"));
4145 f
->open_object_section("osd_metadata");
4146 f
->dump_unsigned("id", osd
);
4147 r
= dump_osd_metadata(osd
, f
.get(), &ss
);
4153 f
->open_array_section("osd_metadata");
4154 for (int i
=0; i
<osdmap
.get_max_osd(); ++i
) {
4155 if (osdmap
.exists(i
)) {
4156 f
->open_object_section("osd");
4157 f
->dump_unsigned("id", i
);
4158 r
= dump_osd_metadata(i
, f
.get(), NULL
);
4159 if (r
== -EINVAL
|| r
== -ENOENT
) {
4160 // Drop error, continue to get other daemons' metadata
4161 dout(4) << "No metadata for osd." << i
<< dendl
;
4173 } else if (prefix
== "osd versions") {
4175 f
.reset(Formatter::create("json-pretty"));
4176 count_metadata("ceph_version", f
.get());
4179 } else if (prefix
== "osd count-metadata") {
4181 f
.reset(Formatter::create("json-pretty"));
4183 cmd_getval(g_ceph_context
, cmdmap
, "property", field
);
4184 count_metadata(field
, f
.get());
4187 } else if (prefix
== "osd map") {
4188 string poolstr
, objstr
, namespacestr
;
4189 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
4190 cmd_getval(g_ceph_context
, cmdmap
, "object", objstr
);
4191 cmd_getval(g_ceph_context
, cmdmap
, "nspace", namespacestr
);
4193 int64_t pool
= osdmap
.lookup_pg_pool_name(poolstr
.c_str());
4195 ss
<< "pool " << poolstr
<< " does not exist";
4199 object_locator_t
oloc(pool
, namespacestr
);
4200 object_t
oid(objstr
);
4201 pg_t pgid
= osdmap
.object_locator_to_pg(oid
, oloc
);
4202 pg_t mpgid
= osdmap
.raw_pg_to_pg(pgid
);
4203 vector
<int> up
, acting
;
4205 osdmap
.pg_to_up_acting_osds(mpgid
, &up
, &up_p
, &acting
, &acting_p
);
4208 if (!namespacestr
.empty())
4209 fullobjname
= namespacestr
+ string("/") + oid
.name
;
4211 fullobjname
= oid
.name
;
4213 f
->open_object_section("osd_map");
4214 f
->dump_unsigned("epoch", osdmap
.get_epoch());
4215 f
->dump_string("pool", poolstr
);
4216 f
->dump_int("pool_id", pool
);
4217 f
->dump_stream("objname") << fullobjname
;
4218 f
->dump_stream("raw_pgid") << pgid
;
4219 f
->dump_stream("pgid") << mpgid
;
4220 f
->open_array_section("up");
4221 for (vector
<int>::iterator p
= up
.begin(); p
!= up
.end(); ++p
)
4222 f
->dump_int("osd", *p
);
4224 f
->dump_int("up_primary", up_p
);
4225 f
->open_array_section("acting");
4226 for (vector
<int>::iterator p
= acting
.begin(); p
!= acting
.end(); ++p
)
4227 f
->dump_int("osd", *p
);
4229 f
->dump_int("acting_primary", acting_p
);
4230 f
->close_section(); // osd_map
4233 ds
<< "osdmap e" << osdmap
.get_epoch()
4234 << " pool '" << poolstr
<< "' (" << pool
<< ")"
4235 << " object '" << fullobjname
<< "' ->"
4236 << " pg " << pgid
<< " (" << mpgid
<< ")"
4237 << " -> up (" << pg_vector_string(up
) << ", p" << up_p
<< ") acting ("
4238 << pg_vector_string(acting
) << ", p" << acting_p
<< ")";
4242 } else if (prefix
== "pg map") {
4245 cmd_getval(g_ceph_context
, cmdmap
, "pgid", pgidstr
);
4246 if (!pgid
.parse(pgidstr
.c_str())) {
4247 ss
<< "invalid pgid '" << pgidstr
<< "'";
4251 vector
<int> up
, acting
;
4252 if (!osdmap
.have_pg_pool(pgid
.pool())) {
4253 ss
<< "pg '" << pgidstr
<< "' does not exist";
4257 pg_t mpgid
= osdmap
.raw_pg_to_pg(pgid
);
4258 osdmap
.pg_to_up_acting_osds(pgid
, up
, acting
);
4260 f
->open_object_section("pg_map");
4261 f
->dump_unsigned("epoch", osdmap
.get_epoch());
4262 f
->dump_stream("raw_pgid") << pgid
;
4263 f
->dump_stream("pgid") << mpgid
;
4264 f
->open_array_section("up");
4265 for (auto osd
: up
) {
4266 f
->dump_int("up_osd", osd
);
4269 f
->open_array_section("acting");
4270 for (auto osd
: acting
) {
4271 f
->dump_int("acting_osd", osd
);
4277 ds
<< "osdmap e" << osdmap
.get_epoch()
4278 << " pg " << pgid
<< " (" << mpgid
<< ")"
4279 << " -> up " << up
<< " acting " << acting
;
4284 } else if (prefix
== "osd scrub" ||
4285 prefix
== "osd deep-scrub" ||
4286 prefix
== "osd repair") {
4288 cmd_getval(g_ceph_context
, cmdmap
, "who", whostr
);
4289 vector
<string
> pvec
;
4290 get_str_vec(prefix
, pvec
);
4292 if (whostr
== "*" || whostr
== "all" || whostr
== "any") {
4295 for (int i
= 0; i
< osdmap
.get_max_osd(); i
++)
4296 if (osdmap
.is_up(i
)) {
4297 ss
<< (c
++ ? "," : "") << i
;
4298 mon
->try_send_message(new MOSDScrub(osdmap
.get_fsid(),
4299 pvec
.back() == "repair",
4300 pvec
.back() == "deep-scrub"),
4301 osdmap
.get_inst(i
));
4304 ss
<< " instructed to " << pvec
.back();
4306 long osd
= parse_osd_id(whostr
.c_str(), &ss
);
4309 } else if (osdmap
.is_up(osd
)) {
4310 mon
->try_send_message(new MOSDScrub(osdmap
.get_fsid(),
4311 pvec
.back() == "repair",
4312 pvec
.back() == "deep-scrub"),
4313 osdmap
.get_inst(osd
));
4314 ss
<< "osd." << osd
<< " instructed to " << pvec
.back();
4316 ss
<< "osd." << osd
<< " is not up";
4320 } else if (prefix
== "osd lspools") {
4322 cmd_getval(g_ceph_context
, cmdmap
, "auid", auid
, int64_t(0));
4324 f
->open_array_section("pools");
4325 for (map
<int64_t, pg_pool_t
>::iterator p
= osdmap
.pools
.begin();
4326 p
!= osdmap
.pools
.end();
4328 if (!auid
|| p
->second
.auid
== (uint64_t)auid
) {
4330 f
->open_object_section("pool");
4331 f
->dump_int("poolnum", p
->first
);
4332 f
->dump_string("poolname", osdmap
.pool_name
[p
->first
]);
4335 ds
<< p
->first
<< ' ' << osdmap
.pool_name
[p
->first
] << ',';
4344 } else if (prefix
== "osd blacklist ls") {
4346 f
->open_array_section("blacklist");
4348 for (ceph::unordered_map
<entity_addr_t
,utime_t
>::iterator p
= osdmap
.blacklist
.begin();
4349 p
!= osdmap
.blacklist
.end();
4352 f
->open_object_section("entry");
4353 f
->dump_stream("addr") << p
->first
;
4354 f
->dump_stream("until") << p
->second
;
4359 ss
<< p
->first
<< " " << p
->second
;
4369 ss
<< "listed " << osdmap
.blacklist
.size() << " entries";
4371 } else if (prefix
== "osd pool ls") {
4373 cmd_getval(g_ceph_context
, cmdmap
, "detail", detail
);
4374 if (!f
&& detail
== "detail") {
4376 osdmap
.print_pools(ss
);
4377 rdata
.append(ss
.str());
4380 f
->open_array_section("pools");
4381 for (map
<int64_t,pg_pool_t
>::const_iterator it
= osdmap
.get_pools().begin();
4382 it
!= osdmap
.get_pools().end();
4385 if (detail
== "detail") {
4386 f
->open_object_section("pool");
4387 f
->dump_string("pool_name", osdmap
.get_pool_name(it
->first
));
4388 it
->second
.dump(f
.get());
4391 f
->dump_string("pool_name", osdmap
.get_pool_name(it
->first
));
4394 rdata
.append(osdmap
.get_pool_name(it
->first
) + "\n");
4403 } else if (prefix
== "osd crush get-tunable") {
4405 cmd_getval(g_ceph_context
, cmdmap
, "tunable", tunable
);
4408 f
->open_object_section("tunable");
4409 if (tunable
== "straw_calc_version") {
4411 f
->dump_int(tunable
.c_str(), osdmap
.crush
->get_straw_calc_version());
4413 rss
<< osdmap
.crush
->get_straw_calc_version() << "\n";
4422 rdata
.append(rss
.str());
4426 } else if (prefix
== "osd pool get") {
4428 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
4429 int64_t pool
= osdmap
.lookup_pg_pool_name(poolstr
.c_str());
4431 ss
<< "unrecognized pool '" << poolstr
<< "'";
4436 const pg_pool_t
*p
= osdmap
.get_pg_pool(pool
);
4438 cmd_getval(g_ceph_context
, cmdmap
, "var", var
);
4440 typedef std::map
<std::string
, osd_pool_get_choices
> choices_map_t
;
4441 const choices_map_t ALL_CHOICES
= {
4443 {"min_size", MIN_SIZE
},
4444 {"crash_replay_interval", CRASH_REPLAY_INTERVAL
},
4445 {"pg_num", PG_NUM
}, {"pgp_num", PGP_NUM
},
4446 {"crush_rule", CRUSH_RULE
},
4447 {"hashpspool", HASHPSPOOL
}, {"nodelete", NODELETE
},
4448 {"nopgchange", NOPGCHANGE
}, {"nosizechange", NOSIZECHANGE
},
4449 {"noscrub", NOSCRUB
}, {"nodeep-scrub", NODEEP_SCRUB
},
4450 {"write_fadvise_dontneed", WRITE_FADVISE_DONTNEED
},
4451 {"hit_set_type", HIT_SET_TYPE
}, {"hit_set_period", HIT_SET_PERIOD
},
4452 {"hit_set_count", HIT_SET_COUNT
}, {"hit_set_fpp", HIT_SET_FPP
},
4453 {"use_gmt_hitset", USE_GMT_HITSET
},
4454 {"auid", AUID
}, {"target_max_objects", TARGET_MAX_OBJECTS
},
4455 {"target_max_bytes", TARGET_MAX_BYTES
},
4456 {"cache_target_dirty_ratio", CACHE_TARGET_DIRTY_RATIO
},
4457 {"cache_target_dirty_high_ratio", CACHE_TARGET_DIRTY_HIGH_RATIO
},
4458 {"cache_target_full_ratio", CACHE_TARGET_FULL_RATIO
},
4459 {"cache_min_flush_age", CACHE_MIN_FLUSH_AGE
},
4460 {"cache_min_evict_age", CACHE_MIN_EVICT_AGE
},
4461 {"erasure_code_profile", ERASURE_CODE_PROFILE
},
4462 {"min_read_recency_for_promote", MIN_READ_RECENCY_FOR_PROMOTE
},
4463 {"min_write_recency_for_promote", MIN_WRITE_RECENCY_FOR_PROMOTE
},
4464 {"fast_read", FAST_READ
},
4465 {"hit_set_grade_decay_rate", HIT_SET_GRADE_DECAY_RATE
},
4466 {"hit_set_search_last_n", HIT_SET_SEARCH_LAST_N
},
4467 {"scrub_min_interval", SCRUB_MIN_INTERVAL
},
4468 {"scrub_max_interval", SCRUB_MAX_INTERVAL
},
4469 {"deep_scrub_interval", DEEP_SCRUB_INTERVAL
},
4470 {"recovery_priority", RECOVERY_PRIORITY
},
4471 {"recovery_op_priority", RECOVERY_OP_PRIORITY
},
4472 {"scrub_priority", SCRUB_PRIORITY
},
4473 {"compression_mode", COMPRESSION_MODE
},
4474 {"compression_algorithm", COMPRESSION_ALGORITHM
},
4475 {"compression_required_ratio", COMPRESSION_REQUIRED_RATIO
},
4476 {"compression_max_blob_size", COMPRESSION_MAX_BLOB_SIZE
},
4477 {"compression_min_blob_size", COMPRESSION_MIN_BLOB_SIZE
},
4478 {"csum_type", CSUM_TYPE
},
4479 {"csum_max_block", CSUM_MAX_BLOCK
},
4480 {"csum_min_block", CSUM_MIN_BLOCK
},
4483 typedef std::set
<osd_pool_get_choices
> choices_set_t
;
4485 const choices_set_t ONLY_TIER_CHOICES
= {
4486 HIT_SET_TYPE
, HIT_SET_PERIOD
, HIT_SET_COUNT
, HIT_SET_FPP
,
4487 TARGET_MAX_OBJECTS
, TARGET_MAX_BYTES
, CACHE_TARGET_FULL_RATIO
,
4488 CACHE_TARGET_DIRTY_RATIO
, CACHE_TARGET_DIRTY_HIGH_RATIO
,
4489 CACHE_MIN_FLUSH_AGE
, CACHE_MIN_EVICT_AGE
,
4490 MIN_READ_RECENCY_FOR_PROMOTE
,
4491 MIN_WRITE_RECENCY_FOR_PROMOTE
,
4492 HIT_SET_GRADE_DECAY_RATE
, HIT_SET_SEARCH_LAST_N
4494 const choices_set_t ONLY_ERASURE_CHOICES
= {
4495 ERASURE_CODE_PROFILE
4498 choices_set_t selected_choices
;
4500 for(choices_map_t::const_iterator it
= ALL_CHOICES
.begin();
4501 it
!= ALL_CHOICES
.end(); ++it
) {
4502 selected_choices
.insert(it
->second
);
4506 selected_choices
= subtract_second_from_first(selected_choices
,
4510 if(!p
->is_erasure()) {
4511 selected_choices
= subtract_second_from_first(selected_choices
,
4512 ONLY_ERASURE_CHOICES
);
4514 } else /* var != "all" */ {
4515 choices_map_t::const_iterator found
= ALL_CHOICES
.find(var
);
4516 osd_pool_get_choices selected
= found
->second
;
4518 if (!p
->is_tier() &&
4519 ONLY_TIER_CHOICES
.find(selected
) != ONLY_TIER_CHOICES
.end()) {
4520 ss
<< "pool '" << poolstr
4521 << "' is not a tier pool: variable not applicable";
4526 if (!p
->is_erasure() &&
4527 ONLY_ERASURE_CHOICES
.find(selected
)
4528 != ONLY_ERASURE_CHOICES
.end()) {
4529 ss
<< "pool '" << poolstr
4530 << "' is not a erasure pool: variable not applicable";
4535 selected_choices
.insert(selected
);
4539 for(choices_set_t::const_iterator it
= selected_choices
.begin();
4540 it
!= selected_choices
.end(); ++it
) {
4541 choices_map_t::const_iterator i
;
4542 for (i
= ALL_CHOICES
.begin(); i
!= ALL_CHOICES
.end(); ++i
) {
4543 if (i
->second
== *it
) {
4547 assert(i
!= ALL_CHOICES
.end());
4548 bool pool_opt
= pool_opts_t::is_opt_name(i
->first
);
4550 f
->open_object_section("pool");
4551 f
->dump_string("pool", poolstr
);
4552 f
->dump_int("pool_id", pool
);
4556 f
->dump_int("pg_num", p
->get_pg_num());
4559 f
->dump_int("pgp_num", p
->get_pgp_num());
4562 f
->dump_int("auid", p
->get_auid());
4565 f
->dump_int("size", p
->get_size());
4568 f
->dump_int("min_size", p
->get_min_size());
4570 case CRASH_REPLAY_INTERVAL
:
4571 f
->dump_int("crash_replay_interval",
4572 p
->get_crash_replay_interval());
4575 if (osdmap
.crush
->rule_exists(p
->get_crush_rule())) {
4576 f
->dump_string("crush_rule", osdmap
.crush
->get_rule_name(
4577 p
->get_crush_rule()));
4579 f
->dump_string("crush_rule", stringify(p
->get_crush_rule()));
4586 case WRITE_FADVISE_DONTNEED
:
4589 f
->dump_string(i
->first
.c_str(),
4590 p
->has_flag(pg_pool_t::get_flag_by_name(i
->first
)) ?
4593 case HIT_SET_PERIOD
:
4594 f
->dump_int("hit_set_period", p
->hit_set_period
);
4597 f
->dump_int("hit_set_count", p
->hit_set_count
);
4600 f
->dump_string("hit_set_type",
4601 HitSet::get_type_name(p
->hit_set_params
.get_type()));
4605 if (p
->hit_set_params
.get_type() == HitSet::TYPE_BLOOM
) {
4606 BloomHitSet::Params
*bloomp
=
4607 static_cast<BloomHitSet::Params
*>(p
->hit_set_params
.impl
.get());
4608 f
->dump_float("hit_set_fpp", bloomp
->get_fpp());
4609 } else if(var
!= "all") {
4611 ss
<< "hit set is not of type Bloom; " <<
4612 "invalid to get a false positive rate!";
4618 case USE_GMT_HITSET
:
4619 f
->dump_bool("use_gmt_hitset", p
->use_gmt_hitset
);
4621 case TARGET_MAX_OBJECTS
:
4622 f
->dump_unsigned("target_max_objects", p
->target_max_objects
);
4624 case TARGET_MAX_BYTES
:
4625 f
->dump_unsigned("target_max_bytes", p
->target_max_bytes
);
4627 case CACHE_TARGET_DIRTY_RATIO
:
4628 f
->dump_unsigned("cache_target_dirty_ratio_micro",
4629 p
->cache_target_dirty_ratio_micro
);
4630 f
->dump_float("cache_target_dirty_ratio",
4631 ((float)p
->cache_target_dirty_ratio_micro
/1000000));
4633 case CACHE_TARGET_DIRTY_HIGH_RATIO
:
4634 f
->dump_unsigned("cache_target_dirty_high_ratio_micro",
4635 p
->cache_target_dirty_high_ratio_micro
);
4636 f
->dump_float("cache_target_dirty_high_ratio",
4637 ((float)p
->cache_target_dirty_high_ratio_micro
/1000000));
4639 case CACHE_TARGET_FULL_RATIO
:
4640 f
->dump_unsigned("cache_target_full_ratio_micro",
4641 p
->cache_target_full_ratio_micro
);
4642 f
->dump_float("cache_target_full_ratio",
4643 ((float)p
->cache_target_full_ratio_micro
/1000000));
4645 case CACHE_MIN_FLUSH_AGE
:
4646 f
->dump_unsigned("cache_min_flush_age", p
->cache_min_flush_age
);
4648 case CACHE_MIN_EVICT_AGE
:
4649 f
->dump_unsigned("cache_min_evict_age", p
->cache_min_evict_age
);
4651 case ERASURE_CODE_PROFILE
:
4652 f
->dump_string("erasure_code_profile", p
->erasure_code_profile
);
4654 case MIN_READ_RECENCY_FOR_PROMOTE
:
4655 f
->dump_int("min_read_recency_for_promote",
4656 p
->min_read_recency_for_promote
);
4658 case MIN_WRITE_RECENCY_FOR_PROMOTE
:
4659 f
->dump_int("min_write_recency_for_promote",
4660 p
->min_write_recency_for_promote
);
4663 f
->dump_int("fast_read", p
->fast_read
);
4665 case HIT_SET_GRADE_DECAY_RATE
:
4666 f
->dump_int("hit_set_grade_decay_rate",
4667 p
->hit_set_grade_decay_rate
);
4669 case HIT_SET_SEARCH_LAST_N
:
4670 f
->dump_int("hit_set_search_last_n",
4671 p
->hit_set_search_last_n
);
4673 case SCRUB_MIN_INTERVAL
:
4674 case SCRUB_MAX_INTERVAL
:
4675 case DEEP_SCRUB_INTERVAL
:
4676 case RECOVERY_PRIORITY
:
4677 case RECOVERY_OP_PRIORITY
:
4678 case SCRUB_PRIORITY
:
4679 case COMPRESSION_MODE
:
4680 case COMPRESSION_ALGORITHM
:
4681 case COMPRESSION_REQUIRED_RATIO
:
4682 case COMPRESSION_MAX_BLOB_SIZE
:
4683 case COMPRESSION_MIN_BLOB_SIZE
:
4685 case CSUM_MAX_BLOCK
:
4686 case CSUM_MIN_BLOCK
:
4687 pool_opts_t::key_t key
= pool_opts_t::get_opt_desc(i
->first
).key
;
4688 if (p
->opts
.is_set(key
)) {
4689 f
->open_object_section("pool");
4690 f
->dump_string("pool", poolstr
);
4691 f
->dump_int("pool_id", pool
);
4692 if(*it
== CSUM_TYPE
) {
4694 p
->opts
.get(pool_opts_t::CSUM_TYPE
, &val
);
4695 f
->dump_string(i
->first
.c_str(), Checksummer::get_csum_type_string(val
));
4697 p
->opts
.dump(i
->first
, f
.get());
4711 for(choices_set_t::const_iterator it
= selected_choices
.begin();
4712 it
!= selected_choices
.end(); ++it
) {
4713 choices_map_t::const_iterator i
;
4716 ss
<< "pg_num: " << p
->get_pg_num() << "\n";
4719 ss
<< "pgp_num: " << p
->get_pgp_num() << "\n";
4722 ss
<< "auid: " << p
->get_auid() << "\n";
4725 ss
<< "size: " << p
->get_size() << "\n";
4728 ss
<< "min_size: " << p
->get_min_size() << "\n";
4730 case CRASH_REPLAY_INTERVAL
:
4731 ss
<< "crash_replay_interval: " <<
4732 p
->get_crash_replay_interval() << "\n";
4735 if (osdmap
.crush
->rule_exists(p
->get_crush_rule())) {
4736 ss
<< "crush_rule: " << osdmap
.crush
->get_rule_name(
4737 p
->get_crush_rule()) << "\n";
4739 ss
<< "crush_rule: " << p
->get_crush_rule() << "\n";
4742 case HIT_SET_PERIOD
:
4743 ss
<< "hit_set_period: " << p
->hit_set_period
<< "\n";
4746 ss
<< "hit_set_count: " << p
->hit_set_count
<< "\n";
4749 ss
<< "hit_set_type: " <<
4750 HitSet::get_type_name(p
->hit_set_params
.get_type()) << "\n";
4754 if (p
->hit_set_params
.get_type() == HitSet::TYPE_BLOOM
) {
4755 BloomHitSet::Params
*bloomp
=
4756 static_cast<BloomHitSet::Params
*>(p
->hit_set_params
.impl
.get());
4757 ss
<< "hit_set_fpp: " << bloomp
->get_fpp() << "\n";
4758 } else if(var
!= "all") {
4759 ss
<< "hit set is not of type Bloom; " <<
4760 "invalid to get a false positive rate!";
4766 case USE_GMT_HITSET
:
4767 ss
<< "use_gmt_hitset: " << p
->use_gmt_hitset
<< "\n";
4769 case TARGET_MAX_OBJECTS
:
4770 ss
<< "target_max_objects: " << p
->target_max_objects
<< "\n";
4772 case TARGET_MAX_BYTES
:
4773 ss
<< "target_max_bytes: " << p
->target_max_bytes
<< "\n";
4775 case CACHE_TARGET_DIRTY_RATIO
:
4776 ss
<< "cache_target_dirty_ratio: "
4777 << ((float)p
->cache_target_dirty_ratio_micro
/1000000) << "\n";
4779 case CACHE_TARGET_DIRTY_HIGH_RATIO
:
4780 ss
<< "cache_target_dirty_high_ratio: "
4781 << ((float)p
->cache_target_dirty_high_ratio_micro
/1000000) << "\n";
4783 case CACHE_TARGET_FULL_RATIO
:
4784 ss
<< "cache_target_full_ratio: "
4785 << ((float)p
->cache_target_full_ratio_micro
/1000000) << "\n";
4787 case CACHE_MIN_FLUSH_AGE
:
4788 ss
<< "cache_min_flush_age: " << p
->cache_min_flush_age
<< "\n";
4790 case CACHE_MIN_EVICT_AGE
:
4791 ss
<< "cache_min_evict_age: " << p
->cache_min_evict_age
<< "\n";
4793 case ERASURE_CODE_PROFILE
:
4794 ss
<< "erasure_code_profile: " << p
->erasure_code_profile
<< "\n";
4796 case MIN_READ_RECENCY_FOR_PROMOTE
:
4797 ss
<< "min_read_recency_for_promote: " <<
4798 p
->min_read_recency_for_promote
<< "\n";
4800 case HIT_SET_GRADE_DECAY_RATE
:
4801 ss
<< "hit_set_grade_decay_rate: " <<
4802 p
->hit_set_grade_decay_rate
<< "\n";
4804 case HIT_SET_SEARCH_LAST_N
:
4805 ss
<< "hit_set_search_last_n: " <<
4806 p
->hit_set_search_last_n
<< "\n";
4812 case WRITE_FADVISE_DONTNEED
:
4815 for (i
= ALL_CHOICES
.begin(); i
!= ALL_CHOICES
.end(); ++i
) {
4816 if (i
->second
== *it
)
4819 assert(i
!= ALL_CHOICES
.end());
4820 ss
<< i
->first
<< ": " <<
4821 (p
->has_flag(pg_pool_t::get_flag_by_name(i
->first
)) ?
4822 "true" : "false") << "\n";
4824 case MIN_WRITE_RECENCY_FOR_PROMOTE
:
4825 ss
<< "min_write_recency_for_promote: " <<
4826 p
->min_write_recency_for_promote
<< "\n";
4829 ss
<< "fast_read: " << p
->fast_read
<< "\n";
4831 case SCRUB_MIN_INTERVAL
:
4832 case SCRUB_MAX_INTERVAL
:
4833 case DEEP_SCRUB_INTERVAL
:
4834 case RECOVERY_PRIORITY
:
4835 case RECOVERY_OP_PRIORITY
:
4836 case SCRUB_PRIORITY
:
4837 case COMPRESSION_MODE
:
4838 case COMPRESSION_ALGORITHM
:
4839 case COMPRESSION_REQUIRED_RATIO
:
4840 case COMPRESSION_MAX_BLOB_SIZE
:
4841 case COMPRESSION_MIN_BLOB_SIZE
:
4843 case CSUM_MAX_BLOCK
:
4844 case CSUM_MIN_BLOCK
:
4845 for (i
= ALL_CHOICES
.begin(); i
!= ALL_CHOICES
.end(); ++i
) {
4846 if (i
->second
== *it
)
4849 assert(i
!= ALL_CHOICES
.end());
4851 pool_opts_t::key_t key
= pool_opts_t::get_opt_desc(i
->first
).key
;
4852 if (p
->opts
.is_set(key
)) {
4853 if(key
== pool_opts_t::CSUM_TYPE
) {
4855 p
->opts
.get(key
, &val
);
4856 ss
<< i
->first
<< ": " << Checksummer::get_csum_type_string(val
) << "\n";
4858 ss
<< i
->first
<< ": " << p
->opts
.get(key
) << "\n";
4864 rdata
.append(ss
.str());
4869 } else if (prefix
== "osd pool stats") {
4870 r
= mon
->pgservice
->process_pg_command(prefix
, cmdmap
,
4871 osdmap
, f
.get(), &ss
, &rdata
);
4872 } else if (prefix
== "osd pool get-quota") {
4874 cmd_getval(g_ceph_context
, cmdmap
, "pool", pool_name
);
4876 int64_t poolid
= osdmap
.lookup_pg_pool_name(pool_name
);
4878 assert(poolid
== -ENOENT
);
4879 ss
<< "unrecognized pool '" << pool_name
<< "'";
4883 const pg_pool_t
*p
= osdmap
.get_pg_pool(poolid
);
4886 f
->open_object_section("pool_quotas");
4887 f
->dump_string("pool_name", pool_name
);
4888 f
->dump_unsigned("pool_id", poolid
);
4889 f
->dump_unsigned("quota_max_objects", p
->quota_max_objects
);
4890 f
->dump_unsigned("quota_max_bytes", p
->quota_max_bytes
);
4895 rs
<< "quotas for pool '" << pool_name
<< "':\n"
4896 << " max objects: ";
4897 if (p
->quota_max_objects
== 0)
4900 rs
<< si_t(p
->quota_max_objects
) << " objects";
4903 if (p
->quota_max_bytes
== 0)
4906 rs
<< si_t(p
->quota_max_bytes
) << "B";
4907 rdata
.append(rs
.str());
4911 } else if (prefix
== "osd crush rule list" ||
4912 prefix
== "osd crush rule ls") {
4914 f
->open_array_section("rules");
4915 osdmap
.crush
->list_rules(f
.get());
4920 osdmap
.crush
->list_rules(&ss
);
4921 rdata
.append(ss
.str());
4923 } else if (prefix
== "osd crush rule dump") {
4925 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
4927 cmd_getval(g_ceph_context
, cmdmap
, "format", format
);
4928 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
, "json-pretty", "json-pretty"));
4930 f
->open_array_section("rules");
4931 osdmap
.crush
->dump_rules(f
.get());
4934 int ruleno
= osdmap
.crush
->get_rule_id(name
);
4936 ss
<< "unknown crush rule '" << name
<< "'";
4940 osdmap
.crush
->dump_rule(ruleno
, f
.get());
4945 rdata
.append(rs
.str());
4946 } else if (prefix
== "osd crush dump") {
4948 cmd_getval(g_ceph_context
, cmdmap
, "format", format
);
4949 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
, "json-pretty", "json-pretty"));
4950 f
->open_object_section("crush_map");
4951 osdmap
.crush
->dump(f
.get());
4956 rdata
.append(rs
.str());
4957 } else if (prefix
== "osd crush show-tunables") {
4959 cmd_getval(g_ceph_context
, cmdmap
, "format", format
);
4960 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
, "json-pretty", "json-pretty"));
4961 f
->open_object_section("crush_map_tunables");
4962 osdmap
.crush
->dump_tunables(f
.get());
4967 rdata
.append(rs
.str());
4968 } else if (prefix
== "osd crush tree") {
4970 cmd_getval(g_ceph_context
, cmdmap
, "shadow", shadow
);
4971 bool show_shadow
= shadow
== "--show-shadow";
4972 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
));
4974 osdmap
.crush
->dump_tree(nullptr,
4976 osdmap
.get_pool_names(),
4981 osdmap
.crush
->dump_tree(&ss
,
4983 osdmap
.get_pool_names(),
4985 rdata
.append(ss
.str());
4987 } else if (prefix
== "osd crush class ls") {
4988 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
, "json-pretty", "json-pretty"));
4989 f
->open_array_section("crush_classes");
4990 for (auto i
: osdmap
.crush
->class_name
)
4991 f
->dump_string("class", i
.second
);
4994 } else if (prefix
== "osd crush class ls-osd") {
4996 cmd_getval(g_ceph_context
, cmdmap
, "class", name
);
4997 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
, "json-pretty", "json-pretty"));
4999 osdmap
.crush
->get_devices_by_class(name
, &osds
);
5000 f
->open_array_section("osds");
5001 for (auto& osd
: osds
)
5002 f
->dump_int("osd", osd
);
5005 } else if (prefix
== "osd erasure-code-profile ls") {
5006 const auto &profiles
= osdmap
.get_erasure_code_profiles();
5008 f
->open_array_section("erasure-code-profiles");
5009 for (auto i
= profiles
.begin(); i
!= profiles
.end(); ++i
) {
5011 f
->dump_string("profile", i
->first
.c_str());
5013 rdata
.append(i
->first
+ "\n");
5020 rdata
.append(rs
.str());
5022 } else if (prefix
== "osd crush weight-set ls") {
5023 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
));
5025 f
->open_array_section("weight_sets");
5026 if (osdmap
.crush
->have_choose_args(CrushWrapper::DEFAULT_CHOOSE_ARGS
)) {
5027 f
->dump_string("pool", "(compat)");
5029 for (auto& i
: osdmap
.crush
->choose_args
) {
5031 f
->dump_string("pool", osdmap
.get_pool_name(i
.first
));
5038 if (osdmap
.crush
->have_choose_args(CrushWrapper::DEFAULT_CHOOSE_ARGS
)) {
5041 for (auto& i
: osdmap
.crush
->choose_args
) {
5043 rs
<< osdmap
.get_pool_name(i
.first
) << "\n";
5046 rdata
.append(rs
.str());
5048 } else if (prefix
== "osd crush weight-set dump") {
5049 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
, "json-pretty",
5051 osdmap
.crush
->dump_choose_args(f
.get());
5053 } else if (prefix
== "osd erasure-code-profile get") {
5055 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
5056 if (!osdmap
.has_erasure_code_profile(name
)) {
5057 ss
<< "unknown erasure code profile '" << name
<< "'";
5061 const map
<string
,string
> &profile
= osdmap
.get_erasure_code_profile(name
);
5063 f
->open_object_section("profile");
5064 for (map
<string
,string
>::const_iterator i
= profile
.begin();
5068 f
->dump_string(i
->first
.c_str(), i
->second
.c_str());
5070 rdata
.append(i
->first
+ "=" + i
->second
+ "\n");
5077 rdata
.append(rs
.str());
5080 // try prepare update
5087 mon
->reply_command(op
, r
, rs
, rdata
, get_last_committed());
5091 void OSDMonitor::update_pool_flags(int64_t pool_id
, uint64_t flags
)
5093 const pg_pool_t
*pool
= osdmap
.get_pg_pool(pool_id
);
5094 pending_inc
.get_new_pool(pool_id
, pool
)->flags
= flags
;
5097 bool OSDMonitor::update_pools_status()
5099 if (!mon
->pgservice
->is_readable())
5104 auto& pools
= osdmap
.get_pools();
5105 for (auto it
= pools
.begin(); it
!= pools
.end(); ++it
) {
5106 const pool_stat_t
*pstat
= mon
->pgservice
->get_pool_stat(it
->first
);
5109 const object_stat_sum_t
& sum
= pstat
->stats
.sum
;
5110 const pg_pool_t
&pool
= it
->second
;
5111 const string
& pool_name
= osdmap
.get_pool_name(it
->first
);
5114 (pool
.quota_max_bytes
> 0 && (uint64_t)sum
.num_bytes
>= pool
.quota_max_bytes
) ||
5115 (pool
.quota_max_objects
> 0 && (uint64_t)sum
.num_objects
>= pool
.quota_max_objects
);
5117 if (pool
.has_flag(pg_pool_t::FLAG_FULL
)) {
5121 mon
->clog
->info() << "pool '" << pool_name
5122 << "' no longer full; removing FULL flag";
5124 update_pool_flags(it
->first
, pool
.get_flags() & ~pg_pool_t::FLAG_FULL
);
5130 if (pool
.quota_max_bytes
> 0 &&
5131 (uint64_t)sum
.num_bytes
>= pool
.quota_max_bytes
) {
5132 mon
->clog
->warn() << "pool '" << pool_name
<< "' is full"
5133 << " (reached quota's max_bytes: "
5134 << si_t(pool
.quota_max_bytes
) << ")";
5136 if (pool
.quota_max_objects
> 0 &&
5137 (uint64_t)sum
.num_objects
>= pool
.quota_max_objects
) {
5138 mon
->clog
->warn() << "pool '" << pool_name
<< "' is full"
5139 << " (reached quota's max_objects: "
5140 << pool
.quota_max_objects
<< ")";
5142 update_pool_flags(it
->first
, pool
.get_flags() | pg_pool_t::FLAG_FULL
);
5149 int OSDMonitor::prepare_new_pool(MonOpRequestRef op
)
5151 op
->mark_osdmon_event(__func__
);
5152 MPoolOp
*m
= static_cast<MPoolOp
*>(op
->get_req());
5153 dout(10) << "prepare_new_pool from " << m
->get_connection() << dendl
;
5154 MonSession
*session
= m
->get_session();
5157 string erasure_code_profile
;
5161 return prepare_new_pool(m
->name
, m
->auid
, m
->crush_rule
, rule_name
,
5163 erasure_code_profile
,
5164 pg_pool_t::TYPE_REPLICATED
, 0, FAST_READ_OFF
, &ss
);
5166 return prepare_new_pool(m
->name
, session
->auid
, m
->crush_rule
, rule_name
,
5168 erasure_code_profile
,
5169 pg_pool_t::TYPE_REPLICATED
, 0, FAST_READ_OFF
, &ss
);
5172 int OSDMonitor::crush_rename_bucket(const string
& srcname
,
5173 const string
& dstname
,
5178 // Avoid creating a pending crush if it does not already exists and
5179 // the rename would fail.
5181 if (!_have_pending_crush()) {
5182 ret
= _get_stable_crush().can_rename_bucket(srcname
,
5189 CrushWrapper newcrush
;
5190 _get_pending_crush(newcrush
);
5192 ret
= newcrush
.rename_bucket(srcname
,
5198 pending_inc
.crush
.clear();
5199 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
5200 *ss
<< "renamed bucket " << srcname
<< " into " << dstname
;
5204 void OSDMonitor::check_legacy_ec_plugin(const string
& plugin
, const string
& profile
) const
5206 string replacement
= "";
5208 if (plugin
== "jerasure_generic" ||
5209 plugin
== "jerasure_sse3" ||
5210 plugin
== "jerasure_sse4" ||
5211 plugin
== "jerasure_neon") {
5212 replacement
= "jerasure";
5213 } else if (plugin
== "shec_generic" ||
5214 plugin
== "shec_sse3" ||
5215 plugin
== "shec_sse4" ||
5216 plugin
== "shec_neon") {
5217 replacement
= "shec";
5220 if (replacement
!= "") {
5221 dout(0) << "WARNING: erasure coding profile " << profile
<< " uses plugin "
5222 << plugin
<< " that has been deprecated. Please use "
5223 << replacement
<< " instead." << dendl
;
5227 int OSDMonitor::normalize_profile(const string
& profilename
,
5228 ErasureCodeProfile
&profile
,
5232 ErasureCodeInterfaceRef erasure_code
;
5233 ErasureCodePluginRegistry
&instance
= ErasureCodePluginRegistry::instance();
5234 ErasureCodeProfile::const_iterator plugin
= profile
.find("plugin");
5235 check_legacy_ec_plugin(plugin
->second
, profilename
);
5236 int err
= instance
.factory(plugin
->second
,
5237 g_conf
->get_val
<std::string
>("erasure_code_dir"),
5238 profile
, &erasure_code
, ss
);
5243 err
= erasure_code
->init(profile
, ss
);
5248 auto it
= profile
.find("stripe_unit");
5249 if (it
!= profile
.end()) {
5251 uint32_t stripe_unit
= strict_si_cast
<uint32_t>(it
->second
.c_str(), &err_str
);
5252 if (!err_str
.empty()) {
5253 *ss
<< "could not parse stripe_unit '" << it
->second
5254 << "': " << err_str
<< std::endl
;
5257 uint32_t data_chunks
= erasure_code
->get_data_chunk_count();
5258 uint32_t chunk_size
= erasure_code
->get_chunk_size(stripe_unit
* data_chunks
);
5259 if (chunk_size
!= stripe_unit
) {
5260 *ss
<< "stripe_unit " << stripe_unit
<< " does not match ec profile "
5261 << "alignment. Would be padded to " << chunk_size
5265 if ((stripe_unit
% 4096) != 0 && !force
) {
5266 *ss
<< "stripe_unit should be a multiple of 4096 bytes for best performance."
5267 << "use --force to override this check" << std::endl
;
5274 int OSDMonitor::crush_rule_create_erasure(const string
&name
,
5275 const string
&profile
,
5279 int ruleid
= osdmap
.crush
->get_rule_id(name
);
5280 if (ruleid
!= -ENOENT
) {
5281 *rule
= osdmap
.crush
->get_rule_mask_ruleset(ruleid
);
5285 CrushWrapper newcrush
;
5286 _get_pending_crush(newcrush
);
5288 ruleid
= newcrush
.get_rule_id(name
);
5289 if (ruleid
!= -ENOENT
) {
5290 *rule
= newcrush
.get_rule_mask_ruleset(ruleid
);
5293 ErasureCodeInterfaceRef erasure_code
;
5294 int err
= get_erasure_code(profile
, &erasure_code
, ss
);
5296 *ss
<< "failed to load plugin using profile " << profile
<< std::endl
;
5300 err
= erasure_code
->create_rule(name
, newcrush
, ss
);
5301 erasure_code
.reset();
5305 pending_inc
.crush
.clear();
5306 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
5311 int OSDMonitor::get_erasure_code(const string
&erasure_code_profile
,
5312 ErasureCodeInterfaceRef
*erasure_code
,
5315 if (pending_inc
.has_erasure_code_profile(erasure_code_profile
))
5317 ErasureCodeProfile profile
=
5318 osdmap
.get_erasure_code_profile(erasure_code_profile
);
5319 ErasureCodeProfile::const_iterator plugin
=
5320 profile
.find("plugin");
5321 if (plugin
== profile
.end()) {
5322 *ss
<< "cannot determine the erasure code plugin"
5323 << " because there is no 'plugin' entry in the erasure_code_profile "
5324 << profile
<< std::endl
;
5327 check_legacy_ec_plugin(plugin
->second
, erasure_code_profile
);
5328 ErasureCodePluginRegistry
&instance
= ErasureCodePluginRegistry::instance();
5329 return instance
.factory(plugin
->second
,
5330 g_conf
->get_val
<std::string
>("erasure_code_dir"),
5331 profile
, erasure_code
, ss
);
5334 int OSDMonitor::check_cluster_features(uint64_t features
,
5337 stringstream unsupported_ss
;
5338 int unsupported_count
= 0;
5339 if ((mon
->get_quorum_con_features() & features
) != features
) {
5340 unsupported_ss
<< "the monitor cluster";
5341 ++unsupported_count
;
5344 set
<int32_t> up_osds
;
5345 osdmap
.get_up_osds(up_osds
);
5346 for (set
<int32_t>::iterator it
= up_osds
.begin();
5347 it
!= up_osds
.end(); ++it
) {
5348 const osd_xinfo_t
&xi
= osdmap
.get_xinfo(*it
);
5349 if ((xi
.features
& features
) != features
) {
5350 if (unsupported_count
> 0)
5351 unsupported_ss
<< ", ";
5352 unsupported_ss
<< "osd." << *it
;
5353 unsupported_count
++;
5357 if (unsupported_count
> 0) {
5358 ss
<< "features " << features
<< " unsupported by: "
5359 << unsupported_ss
.str();
5363 // check pending osd state, too!
5364 for (map
<int32_t,osd_xinfo_t
>::const_iterator p
=
5365 pending_inc
.new_xinfo
.begin();
5366 p
!= pending_inc
.new_xinfo
.end(); ++p
) {
5367 const osd_xinfo_t
&xi
= p
->second
;
5368 if ((xi
.features
& features
) != features
) {
5369 dout(10) << __func__
<< " pending osd." << p
->first
5370 << " features are insufficient; retry" << dendl
;
5378 bool OSDMonitor::validate_crush_against_features(const CrushWrapper
*newcrush
,
5381 OSDMap::Incremental new_pending
= pending_inc
;
5382 ::encode(*newcrush
, new_pending
.crush
, mon
->get_quorum_con_features());
5384 newmap
.deepish_copy_from(osdmap
);
5385 newmap
.apply_incremental(new_pending
);
5388 if (newmap
.require_min_compat_client
> 0) {
5389 auto mv
= newmap
.get_min_compat_client();
5390 if (mv
> newmap
.require_min_compat_client
) {
5391 ss
<< "new crush map requires client version " << ceph_release_name(mv
)
5392 << " but require_min_compat_client is "
5393 << ceph_release_name(newmap
.require_min_compat_client
);
5400 newmap
.get_features(CEPH_ENTITY_TYPE_MON
, NULL
) |
5401 newmap
.get_features(CEPH_ENTITY_TYPE_OSD
, NULL
);
5402 stringstream features_ss
;
5403 int r
= check_cluster_features(features
, features_ss
);
5405 ss
<< "Could not change CRUSH: " << features_ss
.str();
5412 bool OSDMonitor::erasure_code_profile_in_use(
5413 const mempool::osdmap::map
<int64_t, pg_pool_t
> &pools
,
5414 const string
&profile
,
5418 for (map
<int64_t, pg_pool_t
>::const_iterator p
= pools
.begin();
5421 if (p
->second
.erasure_code_profile
== profile
) {
5422 *ss
<< osdmap
.pool_name
[p
->first
] << " ";
5427 *ss
<< "pool(s) are using the erasure code profile '" << profile
<< "'";
5432 int OSDMonitor::parse_erasure_code_profile(const vector
<string
> &erasure_code_profile
,
5433 map
<string
,string
> *erasure_code_profile_map
,
5436 int r
= get_json_str_map(g_conf
->osd_pool_default_erasure_code_profile
,
5438 erasure_code_profile_map
);
5441 assert((*erasure_code_profile_map
).count("plugin"));
5442 string default_plugin
= (*erasure_code_profile_map
)["plugin"];
5443 map
<string
,string
> user_map
;
5444 for (vector
<string
>::const_iterator i
= erasure_code_profile
.begin();
5445 i
!= erasure_code_profile
.end();
5447 size_t equal
= i
->find('=');
5448 if (equal
== string::npos
) {
5449 user_map
[*i
] = string();
5450 (*erasure_code_profile_map
)[*i
] = string();
5452 const string key
= i
->substr(0, equal
);
5454 const string value
= i
->substr(equal
);
5455 user_map
[key
] = value
;
5456 (*erasure_code_profile_map
)[key
] = value
;
5460 if (user_map
.count("plugin") && user_map
["plugin"] != default_plugin
)
5461 (*erasure_code_profile_map
) = user_map
;
5466 int OSDMonitor::prepare_pool_size(const unsigned pool_type
,
5467 const string
&erasure_code_profile
,
5468 unsigned *size
, unsigned *min_size
,
5472 switch (pool_type
) {
5473 case pg_pool_t::TYPE_REPLICATED
:
5474 *size
= g_conf
->osd_pool_default_size
;
5475 *min_size
= g_conf
->get_osd_pool_default_min_size();
5477 case pg_pool_t::TYPE_ERASURE
:
5479 ErasureCodeInterfaceRef erasure_code
;
5480 err
= get_erasure_code(erasure_code_profile
, &erasure_code
, ss
);
5482 *size
= erasure_code
->get_chunk_count();
5483 *min_size
= MIN(erasure_code
->get_data_chunk_count() + 1, *size
);
5488 *ss
<< "prepare_pool_size: " << pool_type
<< " is not a known pool type";
5495 int OSDMonitor::prepare_pool_stripe_width(const unsigned pool_type
,
5496 const string
&erasure_code_profile
,
5497 uint32_t *stripe_width
,
5501 switch (pool_type
) {
5502 case pg_pool_t::TYPE_REPLICATED
:
5505 case pg_pool_t::TYPE_ERASURE
:
5507 ErasureCodeProfile profile
=
5508 osdmap
.get_erasure_code_profile(erasure_code_profile
);
5509 ErasureCodeInterfaceRef erasure_code
;
5510 err
= get_erasure_code(erasure_code_profile
, &erasure_code
, ss
);
5513 uint32_t data_chunks
= erasure_code
->get_data_chunk_count();
5514 uint32_t stripe_unit
= g_conf
->osd_pool_erasure_code_stripe_unit
;
5515 auto it
= profile
.find("stripe_unit");
5516 if (it
!= profile
.end()) {
5518 stripe_unit
= strict_si_cast
<uint32_t>(it
->second
.c_str(), &err_str
);
5519 assert(err_str
.empty());
5521 *stripe_width
= data_chunks
*
5522 erasure_code
->get_chunk_size(stripe_unit
* data_chunks
);
5526 *ss
<< "prepare_pool_stripe_width: "
5527 << pool_type
<< " is not a known pool type";
5534 int OSDMonitor::prepare_pool_crush_rule(const unsigned pool_type
,
5535 const string
&erasure_code_profile
,
5536 const string
&rule_name
,
5541 if (*crush_rule
< 0) {
5542 switch (pool_type
) {
5543 case pg_pool_t::TYPE_REPLICATED
:
5545 if (rule_name
== "") {
5547 *crush_rule
= osdmap
.crush
->get_osd_pool_default_crush_replicated_ruleset(g_ceph_context
);
5548 if (*crush_rule
< 0) {
5549 // Errors may happen e.g. if no valid rule is available
5550 *ss
<< "No suitable CRUSH rule exists, check "
5551 << "'osd pool default crush *' config options";
5555 return get_crush_rule(rule_name
, crush_rule
, ss
);
5559 case pg_pool_t::TYPE_ERASURE
:
5561 int err
= crush_rule_create_erasure(rule_name
,
5562 erasure_code_profile
,
5566 dout(20) << "prepare_pool_crush_rule: rule "
5567 << rule_name
<< " try again" << dendl
;
5570 // need to wait for the crush rule to be proposed before proceeding
5581 *ss
<< "prepare_pool_crush_rule: " << pool_type
5582 << " is not a known pool type";
5587 if (!osdmap
.crush
->ruleset_exists(*crush_rule
)) {
5588 *ss
<< "CRUSH rule " << *crush_rule
<< " not found";
5596 int OSDMonitor::get_crush_rule(const string
&rule_name
,
5601 ret
= osdmap
.crush
->get_rule_id(rule_name
);
5602 if (ret
!= -ENOENT
) {
5606 CrushWrapper newcrush
;
5607 _get_pending_crush(newcrush
);
5609 ret
= newcrush
.get_rule_id(rule_name
);
5610 if (ret
!= -ENOENT
) {
5611 // found it, wait for it to be proposed
5612 dout(20) << __func__
<< ": rule " << rule_name
5613 << " try again" << dendl
;
5616 // Cannot find it , return error
5617 *ss
<< "specified rule " << rule_name
<< " doesn't exist";
5625 * @param name The name of the new pool
5626 * @param auid The auid of the pool owner. Can be -1
5627 * @param crush_rule The crush rule to use. If <0, will use the system default
5628 * @param crush_rule_name The crush rule to use, if crush_rulset <0
5629 * @param pg_num The pg_num to use. If set to 0, will use the system default
5630 * @param pgp_num The pgp_num to use. If set to 0, will use the system default
5631 * @param erasure_code_profile The profile name in OSDMap to be used for erasure code
5632 * @param pool_type TYPE_ERASURE, or TYPE_REP
5633 * @param expected_num_objects expected number of objects on the pool
5634 * @param fast_read fast read type.
5635 * @param ss human readable error message, if any.
5637 * @return 0 on success, negative errno on failure.
5639 int OSDMonitor::prepare_new_pool(string
& name
, uint64_t auid
,
5641 const string
&crush_rule_name
,
5642 unsigned pg_num
, unsigned pgp_num
,
5643 const string
&erasure_code_profile
,
5644 const unsigned pool_type
,
5645 const uint64_t expected_num_objects
,
5646 FastReadType fast_read
,
5649 if (name
.length() == 0)
5652 pg_num
= g_conf
->osd_pool_default_pg_num
;
5654 pgp_num
= g_conf
->osd_pool_default_pgp_num
;
5655 if (pg_num
> (unsigned)g_conf
->mon_max_pool_pg_num
) {
5656 *ss
<< "'pg_num' must be greater than 0 and less than or equal to "
5657 << g_conf
->mon_max_pool_pg_num
5658 << " (you may adjust 'mon max pool pg num' for higher values)";
5661 if (pgp_num
> pg_num
) {
5662 *ss
<< "'pgp_num' must be greater than 0 and lower or equal than 'pg_num'"
5663 << ", which in this case is " << pg_num
;
5666 if (pool_type
== pg_pool_t::TYPE_REPLICATED
&& fast_read
== FAST_READ_ON
) {
5667 *ss
<< "'fast_read' can only apply to erasure coding pool";
5671 r
= prepare_pool_crush_rule(pool_type
, erasure_code_profile
,
5672 crush_rule_name
, &crush_rule
, ss
);
5674 dout(10) << " prepare_pool_crush_rule returns " << r
<< dendl
;
5677 if (g_conf
->mon_osd_crush_smoke_test
) {
5678 CrushWrapper newcrush
;
5679 _get_pending_crush(newcrush
);
5681 CrushTester
tester(newcrush
, err
);
5682 tester
.set_max_x(50);
5683 tester
.set_rule(crush_rule
);
5684 r
= tester
.test_with_fork(g_conf
->mon_lease
);
5686 dout(10) << " tester.test_with_fork returns " << r
5687 << ": " << err
.str() << dendl
;
5688 *ss
<< "crush test failed with " << r
<< ": " << err
.str();
5692 unsigned size
, min_size
;
5693 r
= prepare_pool_size(pool_type
, erasure_code_profile
, &size
, &min_size
, ss
);
5695 dout(10) << " prepare_pool_size returns " << r
<< dendl
;
5699 if (!osdmap
.crush
->check_crush_rule(crush_rule
, pool_type
, size
, *ss
)) {
5703 uint32_t stripe_width
= 0;
5704 r
= prepare_pool_stripe_width(pool_type
, erasure_code_profile
, &stripe_width
, ss
);
5706 dout(10) << " prepare_pool_stripe_width returns " << r
<< dendl
;
5711 if (pool_type
== pg_pool_t::TYPE_ERASURE
) {
5712 switch (fast_read
) {
5719 case FAST_READ_DEFAULT
:
5720 fread
= g_conf
->mon_osd_pool_ec_fast_read
;
5723 *ss
<< "invalid fast_read setting: " << fast_read
;
5728 for (map
<int64_t,string
>::iterator p
= pending_inc
.new_pool_names
.begin();
5729 p
!= pending_inc
.new_pool_names
.end();
5731 if (p
->second
== name
)
5735 if (-1 == pending_inc
.new_pool_max
)
5736 pending_inc
.new_pool_max
= osdmap
.pool_max
;
5737 int64_t pool
= ++pending_inc
.new_pool_max
;
5739 pg_pool_t
*pi
= pending_inc
.get_new_pool(pool
, &empty
);
5740 pi
->type
= pool_type
;
5741 pi
->fast_read
= fread
;
5742 pi
->flags
= g_conf
->osd_pool_default_flags
;
5743 if (g_conf
->osd_pool_default_flag_hashpspool
)
5744 pi
->set_flag(pg_pool_t::FLAG_HASHPSPOOL
);
5745 if (g_conf
->osd_pool_default_flag_nodelete
)
5746 pi
->set_flag(pg_pool_t::FLAG_NODELETE
);
5747 if (g_conf
->osd_pool_default_flag_nopgchange
)
5748 pi
->set_flag(pg_pool_t::FLAG_NOPGCHANGE
);
5749 if (g_conf
->osd_pool_default_flag_nosizechange
)
5750 pi
->set_flag(pg_pool_t::FLAG_NOSIZECHANGE
);
5751 if (g_conf
->osd_pool_use_gmt_hitset
&&
5752 (osdmap
.get_up_osd_features() & CEPH_FEATURE_OSD_HITSET_GMT
))
5753 pi
->use_gmt_hitset
= true;
5755 pi
->use_gmt_hitset
= false;
5758 pi
->min_size
= min_size
;
5759 pi
->crush_rule
= crush_rule
;
5760 pi
->expected_num_objects
= expected_num_objects
;
5761 pi
->object_hash
= CEPH_STR_HASH_RJENKINS
;
5762 pi
->set_pg_num(pg_num
);
5763 pi
->set_pgp_num(pgp_num
);
5764 pi
->last_change
= pending_inc
.epoch
;
5766 pi
->erasure_code_profile
= erasure_code_profile
;
5767 pi
->stripe_width
= stripe_width
;
5768 pi
->cache_target_dirty_ratio_micro
=
5769 g_conf
->osd_pool_default_cache_target_dirty_ratio
* 1000000;
5770 pi
->cache_target_dirty_high_ratio_micro
=
5771 g_conf
->osd_pool_default_cache_target_dirty_high_ratio
* 1000000;
5772 pi
->cache_target_full_ratio_micro
=
5773 g_conf
->osd_pool_default_cache_target_full_ratio
* 1000000;
5774 pi
->cache_min_flush_age
= g_conf
->osd_pool_default_cache_min_flush_age
;
5775 pi
->cache_min_evict_age
= g_conf
->osd_pool_default_cache_min_evict_age
;
5776 pending_inc
.new_pool_names
[pool
] = name
;
5780 bool OSDMonitor::prepare_set_flag(MonOpRequestRef op
, int flag
)
5782 op
->mark_osdmon_event(__func__
);
5784 if (pending_inc
.new_flags
< 0)
5785 pending_inc
.new_flags
= osdmap
.get_flags();
5786 pending_inc
.new_flags
|= flag
;
5787 ss
<< OSDMap::get_flag_string(flag
) << " is set";
5788 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, ss
.str(),
5789 get_last_committed() + 1));
5793 bool OSDMonitor::prepare_unset_flag(MonOpRequestRef op
, int flag
)
5795 op
->mark_osdmon_event(__func__
);
5797 if (pending_inc
.new_flags
< 0)
5798 pending_inc
.new_flags
= osdmap
.get_flags();
5799 pending_inc
.new_flags
&= ~flag
;
5800 ss
<< OSDMap::get_flag_string(flag
) << " is unset";
5801 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, ss
.str(),
5802 get_last_committed() + 1));
5806 int OSDMonitor::prepare_command_pool_set(map
<string
,cmd_vartype
> &cmdmap
,
5810 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
5811 int64_t pool
= osdmap
.lookup_pg_pool_name(poolstr
.c_str());
5813 ss
<< "unrecognized pool '" << poolstr
<< "'";
5817 cmd_getval(g_ceph_context
, cmdmap
, "var", var
);
5819 pg_pool_t p
= *osdmap
.get_pg_pool(pool
);
5820 if (pending_inc
.new_pools
.count(pool
))
5821 p
= pending_inc
.new_pools
[pool
];
5823 // accept val as a json string in the normal case (current
5824 // generation monitor). parse out int or float values from the
5825 // string as needed. however, if it is not a string, try to pull
5826 // out an int, in case an older monitor with an older json schema is
5827 // forwarding a request.
5829 string interr
, floaterr
;
5832 int64_t uf
= 0; // micro-f
5833 if (!cmd_getval(g_ceph_context
, cmdmap
, "val", val
)) {
5834 // wasn't a string; maybe an older mon forwarded json with an int?
5835 if (!cmd_getval(g_ceph_context
, cmdmap
, "val", n
))
5836 return -EINVAL
; // no value!
5838 // we got a string. see if it contains an int.
5839 n
= strict_strtoll(val
.c_str(), 10, &interr
);
5841 f
= strict_strtod(val
.c_str(), &floaterr
);
5842 uf
= llrintl(f
* (double)1000000.0);
5846 (var
== "hit_set_type" || var
== "hit_set_period" ||
5847 var
== "hit_set_count" || var
== "hit_set_fpp" ||
5848 var
== "target_max_objects" || var
== "target_max_bytes" ||
5849 var
== "cache_target_full_ratio" || var
== "cache_target_dirty_ratio" ||
5850 var
== "cache_target_dirty_high_ratio" || var
== "use_gmt_hitset" ||
5851 var
== "cache_min_flush_age" || var
== "cache_min_evict_age" ||
5852 var
== "hit_set_grade_decay_rate" || var
== "hit_set_search_last_n" ||
5853 var
== "min_read_recency_for_promote" || var
== "min_write_recency_for_promote")) {
5857 if (var
== "size") {
5858 if (p
.has_flag(pg_pool_t::FLAG_NOSIZECHANGE
)) {
5859 ss
<< "pool size change is disabled; you must unset nosizechange flag for the pool first";
5862 if (p
.type
== pg_pool_t::TYPE_ERASURE
) {
5863 ss
<< "can not change the size of an erasure-coded pool";
5866 if (interr
.length()) {
5867 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
5870 if (n
<= 0 || n
> 10) {
5871 ss
<< "pool size must be between 1 and 10";
5877 } else if (var
== "min_size") {
5878 if (p
.has_flag(pg_pool_t::FLAG_NOSIZECHANGE
)) {
5879 ss
<< "pool min size change is disabled; you must unset nosizechange flag for the pool first";
5882 if (interr
.length()) {
5883 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
5887 if (p
.type
!= pg_pool_t::TYPE_ERASURE
) {
5888 if (n
< 1 || n
> p
.size
) {
5889 ss
<< "pool min_size must be between 1 and " << (int)p
.size
;
5893 ErasureCodeInterfaceRef erasure_code
;
5896 int err
= get_erasure_code(p
.erasure_code_profile
, &erasure_code
, &tmp
);
5898 k
= erasure_code
->get_data_chunk_count();
5900 ss
<< __func__
<< " get_erasure_code failed: " << tmp
.rdbuf();
5904 if (n
< k
|| n
> p
.size
) {
5905 ss
<< "pool min_size must be between " << k
<< " and " << (int)p
.size
;
5910 } else if (var
== "auid") {
5911 if (interr
.length()) {
5912 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
5916 } else if (var
== "crash_replay_interval") {
5917 if (interr
.length()) {
5918 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
5921 p
.crash_replay_interval
= n
;
5922 } else if (var
== "pg_num") {
5923 if (p
.has_flag(pg_pool_t::FLAG_NOPGCHANGE
)) {
5924 ss
<< "pool pg_num change is disabled; you must unset nopgchange flag for the pool first";
5927 if (interr
.length()) {
5928 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
5931 if (n
<= (int)p
.get_pg_num()) {
5932 ss
<< "specified pg_num " << n
<< " <= current " << p
.get_pg_num();
5933 if (n
< (int)p
.get_pg_num())
5937 if (n
> (unsigned)g_conf
->mon_max_pool_pg_num
) {
5938 ss
<< "'pg_num' must be greater than 0 and less than or equal to "
5939 << g_conf
->mon_max_pool_pg_num
5940 << " (you may adjust 'mon max pool pg num' for higher values)";
5944 cmd_getval(g_ceph_context
,cmdmap
, "force", force
);
5945 if (p
.cache_mode
!= pg_pool_t::CACHEMODE_NONE
&&
5946 force
!= "--yes-i-really-mean-it") {
5947 ss
<< "splits in cache pools must be followed by scrubs and leave sufficient free space to avoid overfilling. use --yes-i-really-mean-it to force.";
5950 int expected_osds
= MIN(p
.get_pg_num(), osdmap
.get_num_osds());
5951 int64_t new_pgs
= n
- p
.get_pg_num();
5952 if (new_pgs
> g_conf
->mon_osd_max_split_count
* expected_osds
) {
5953 ss
<< "specified pg_num " << n
<< " is too large (creating "
5954 << new_pgs
<< " new PGs on ~" << expected_osds
5955 << " OSDs exceeds per-OSD max of " << g_conf
->mon_osd_max_split_count
5960 // force pre-luminous clients to resend their ops, since they
5961 // don't understand that split PGs now form a new interval.
5962 p
.last_force_op_resend_preluminous
= pending_inc
.epoch
;
5963 } else if (var
== "pgp_num") {
5964 if (p
.has_flag(pg_pool_t::FLAG_NOPGCHANGE
)) {
5965 ss
<< "pool pgp_num change is disabled; you must unset nopgchange flag for the pool first";
5968 if (interr
.length()) {
5969 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
5973 ss
<< "specified pgp_num must > 0, but you set to " << n
;
5976 if (n
> (int)p
.get_pg_num()) {
5977 ss
<< "specified pgp_num " << n
<< " > pg_num " << p
.get_pg_num();
5981 } else if (var
== "crush_rule") {
5982 int id
= osdmap
.crush
->get_rule_id(val
);
5983 if (id
== -ENOENT
) {
5984 ss
<< "crush rule " << val
<< " does not exist";
5988 ss
<< cpp_strerror(id
);
5991 if (!osdmap
.crush
->check_crush_rule(id
, p
.get_type(), p
.get_size(), ss
)) {
5995 } else if (var
== "nodelete" || var
== "nopgchange" ||
5996 var
== "nosizechange" || var
== "write_fadvise_dontneed" ||
5997 var
== "noscrub" || var
== "nodeep-scrub") {
5998 uint64_t flag
= pg_pool_t::get_flag_by_name(var
);
5999 // make sure we only compare against 'n' if we didn't receive a string
6000 if (val
== "true" || (interr
.empty() && n
== 1)) {
6002 } else if (val
== "false" || (interr
.empty() && n
== 0)) {
6005 ss
<< "expecting value 'true', 'false', '0', or '1'";
6008 } else if (var
== "hashpspool") {
6009 uint64_t flag
= pg_pool_t::get_flag_by_name(var
);
6011 cmd_getval(g_ceph_context
, cmdmap
, "force", force
);
6012 if (force
!= "--yes-i-really-mean-it") {
6013 ss
<< "are you SURE? this will remap all placement groups in this pool,"
6014 " this triggers large data movement,"
6015 " pass --yes-i-really-mean-it if you really do.";
6018 // make sure we only compare against 'n' if we didn't receive a string
6019 if (val
== "true" || (interr
.empty() && n
== 1)) {
6021 } else if (val
== "false" || (interr
.empty() && n
== 0)) {
6024 ss
<< "expecting value 'true', 'false', '0', or '1'";
6027 } else if (var
== "hit_set_type") {
6029 p
.hit_set_params
= HitSet::Params();
6031 int err
= check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL
, ss
);
6034 if (val
== "bloom") {
6035 BloomHitSet::Params
*bsp
= new BloomHitSet::Params
;
6036 bsp
->set_fpp(g_conf
->osd_pool_default_hit_set_bloom_fpp
);
6037 p
.hit_set_params
= HitSet::Params(bsp
);
6038 } else if (val
== "explicit_hash")
6039 p
.hit_set_params
= HitSet::Params(new ExplicitHashHitSet::Params
);
6040 else if (val
== "explicit_object")
6041 p
.hit_set_params
= HitSet::Params(new ExplicitObjectHitSet::Params
);
6043 ss
<< "unrecognized hit_set type '" << val
<< "'";
6047 } else if (var
== "hit_set_period") {
6048 if (interr
.length()) {
6049 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
6052 p
.hit_set_period
= n
;
6053 } else if (var
== "hit_set_count") {
6054 if (interr
.length()) {
6055 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
6058 p
.hit_set_count
= n
;
6059 } else if (var
== "hit_set_fpp") {
6060 if (floaterr
.length()) {
6061 ss
<< "error parsing floating point value '" << val
<< "': " << floaterr
;
6064 if (p
.hit_set_params
.get_type() != HitSet::TYPE_BLOOM
) {
6065 ss
<< "hit set is not of type Bloom; invalid to set a false positive rate!";
6068 BloomHitSet::Params
*bloomp
= static_cast<BloomHitSet::Params
*>(p
.hit_set_params
.impl
.get());
6070 } else if (var
== "use_gmt_hitset") {
6071 if (val
== "true" || (interr
.empty() && n
== 1)) {
6072 if (!(osdmap
.get_up_osd_features() & CEPH_FEATURE_OSD_HITSET_GMT
)) {
6073 ss
<< "not all OSDs support GMT hit set.";
6076 p
.use_gmt_hitset
= true;
6078 ss
<< "expecting value 'true' or '1'";
6081 } else if (var
== "allow_ec_overwrites") {
6082 if (!p
.is_erasure()) {
6083 ss
<< "ec overwrites can only be enabled for an erasure coded pool";
6087 if (!g_conf
->mon_debug_no_require_bluestore_for_ec_overwrites
&&
6088 !is_pool_currently_all_bluestore(pool
, p
, &err
)) {
6089 ss
<< "pool must only be stored on bluestore for scrubbing to work: " << err
.str();
6092 if (val
== "true" || (interr
.empty() && n
== 1)) {
6093 p
.flags
|= pg_pool_t::FLAG_EC_OVERWRITES
;
6094 } else if (val
== "false" || (interr
.empty() && n
== 0)) {
6095 ss
<< "ec overwrites cannot be disabled once enabled";
6098 ss
<< "expecting value 'true', 'false', '0', or '1'";
6101 } else if (var
== "target_max_objects") {
6102 if (interr
.length()) {
6103 ss
<< "error parsing int '" << val
<< "': " << interr
;
6106 p
.target_max_objects
= n
;
6107 } else if (var
== "target_max_bytes") {
6108 if (interr
.length()) {
6109 ss
<< "error parsing int '" << val
<< "': " << interr
;
6112 p
.target_max_bytes
= n
;
6113 } else if (var
== "cache_target_dirty_ratio") {
6114 if (floaterr
.length()) {
6115 ss
<< "error parsing float '" << val
<< "': " << floaterr
;
6118 if (f
< 0 || f
> 1.0) {
6119 ss
<< "value must be in the range 0..1";
6122 p
.cache_target_dirty_ratio_micro
= uf
;
6123 } else if (var
== "cache_target_dirty_high_ratio") {
6124 if (floaterr
.length()) {
6125 ss
<< "error parsing float '" << val
<< "': " << floaterr
;
6128 if (f
< 0 || f
> 1.0) {
6129 ss
<< "value must be in the range 0..1";
6132 p
.cache_target_dirty_high_ratio_micro
= uf
;
6133 } else if (var
== "cache_target_full_ratio") {
6134 if (floaterr
.length()) {
6135 ss
<< "error parsing float '" << val
<< "': " << floaterr
;
6138 if (f
< 0 || f
> 1.0) {
6139 ss
<< "value must be in the range 0..1";
6142 p
.cache_target_full_ratio_micro
= uf
;
6143 } else if (var
== "cache_min_flush_age") {
6144 if (interr
.length()) {
6145 ss
<< "error parsing int '" << val
<< "': " << interr
;
6148 p
.cache_min_flush_age
= n
;
6149 } else if (var
== "cache_min_evict_age") {
6150 if (interr
.length()) {
6151 ss
<< "error parsing int '" << val
<< "': " << interr
;
6154 p
.cache_min_evict_age
= n
;
6155 } else if (var
== "min_read_recency_for_promote") {
6156 if (interr
.length()) {
6157 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
6160 p
.min_read_recency_for_promote
= n
;
6161 } else if (var
== "hit_set_grade_decay_rate") {
6162 if (interr
.length()) {
6163 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
6166 if (n
> 100 || n
< 0) {
6167 ss
<< "value out of range,valid range is 0 - 100";
6170 p
.hit_set_grade_decay_rate
= n
;
6171 } else if (var
== "hit_set_search_last_n") {
6172 if (interr
.length()) {
6173 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
6176 if (n
> p
.hit_set_count
|| n
< 0) {
6177 ss
<< "value out of range,valid range is 0 - hit_set_count";
6180 p
.hit_set_search_last_n
= n
;
6181 } else if (var
== "min_write_recency_for_promote") {
6182 if (interr
.length()) {
6183 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
6186 p
.min_write_recency_for_promote
= n
;
6187 } else if (var
== "fast_read") {
6188 if (p
.is_replicated()) {
6189 ss
<< "fast read is not supported in replication pool";
6192 if (val
== "true" || (interr
.empty() && n
== 1)) {
6194 } else if (val
== "false" || (interr
.empty() && n
== 0)) {
6195 p
.fast_read
= false;
6197 ss
<< "expecting value 'true', 'false', '0', or '1'";
6200 } else if (pool_opts_t::is_opt_name(var
)) {
6201 bool unset
= val
== "unset";
6202 if (var
== "compression_mode") {
6204 auto cmode
= Compressor::get_comp_mode_type(val
);
6206 ss
<< "unrecognized compression mode '" << val
<< "'";
6210 } else if (var
== "compression_algorithm") {
6212 auto alg
= Compressor::get_comp_alg_type(val
);
6214 ss
<< "unrecognized compression_algorithm '" << val
<< "'";
6218 } else if (var
== "compression_required_ratio") {
6219 if (floaterr
.length()) {
6220 ss
<< "error parsing float value '" << val
<< "': " << floaterr
;
6223 if (f
< 0 || f
> 1) {
6224 ss
<< "compression_required_ratio is out of range (0-1): '" << val
<< "'";
6227 } else if (var
== "csum_type") {
6228 auto t
= unset
? 0 : Checksummer::get_csum_string_type(val
);
6230 ss
<< "unrecognized csum_type '" << val
<< "'";
6233 //preserve csum_type numeric value
6236 } else if (var
== "compression_max_blob_size" ||
6237 var
== "compression_min_blob_size" ||
6238 var
== "csum_max_block" ||
6239 var
== "csum_min_block") {
6240 if (interr
.length()) {
6241 ss
<< "error parsing int value '" << val
<< "': " << interr
;
6246 pool_opts_t::opt_desc_t desc
= pool_opts_t::get_opt_desc(var
);
6247 switch (desc
.type
) {
6248 case pool_opts_t::STR
:
6250 p
.opts
.unset(desc
.key
);
6252 p
.opts
.set(desc
.key
, static_cast<std::string
>(val
));
6255 case pool_opts_t::INT
:
6256 if (interr
.length()) {
6257 ss
<< "error parsing integer value '" << val
<< "': " << interr
;
6261 p
.opts
.unset(desc
.key
);
6263 p
.opts
.set(desc
.key
, static_cast<int>(n
));
6266 case pool_opts_t::DOUBLE
:
6267 if (floaterr
.length()) {
6268 ss
<< "error parsing floating point value '" << val
<< "': " << floaterr
;
6272 p
.opts
.unset(desc
.key
);
6274 p
.opts
.set(desc
.key
, static_cast<double>(f
));
6278 assert(!"unknown type");
6281 ss
<< "unrecognized variable '" << var
<< "'";
6284 if (val
!= "unset") {
6285 ss
<< "set pool " << pool
<< " " << var
<< " to " << val
;
6287 ss
<< "unset pool " << pool
<< " " << var
;
6289 p
.last_change
= pending_inc
.epoch
;
6290 pending_inc
.new_pools
[pool
] = p
;
6294 int OSDMonitor::prepare_command_pool_application(const string
&prefix
,
6295 map
<string
,cmd_vartype
> &cmdmap
,
6299 cmd_getval(g_ceph_context
, cmdmap
, "pool", pool_name
);
6300 int64_t pool
= osdmap
.lookup_pg_pool_name(pool_name
.c_str());
6302 ss
<< "unrecognized pool '" << pool_name
<< "'";
6306 pg_pool_t p
= *osdmap
.get_pg_pool(pool
);
6307 if (pending_inc
.new_pools
.count(pool
)) {
6308 p
= pending_inc
.new_pools
[pool
];
6312 cmd_getval(g_ceph_context
, cmdmap
, "app", app
);
6313 bool app_exists
= (p
.application_metadata
.count(app
) > 0);
6315 if (boost::algorithm::ends_with(prefix
, "enable")) {
6317 ss
<< "application name must be provided";
6322 ss
<< "application must be enabled on base tier";
6327 cmd_getval(g_ceph_context
, cmdmap
, "force", force
);
6329 if (!app_exists
&& !p
.application_metadata
.empty() &&
6330 force
!= "--yes-i-really-mean-it") {
6331 ss
<< "Are you SURE? Pool '" << pool_name
<< "' already has an enabled "
6332 << "application; pass --yes-i-really-mean-it to proceed anyway";
6336 if (!app_exists
&& p
.application_metadata
.size() >= MAX_POOL_APPLICATIONS
) {
6337 ss
<< "too many enabled applications on pool '" << pool_name
<< "'; "
6338 << "max " << MAX_POOL_APPLICATIONS
;
6342 if (app
.length() > MAX_POOL_APPLICATION_LENGTH
) {
6343 ss
<< "application name '" << app
<< "' too long; max length "
6344 << MAX_POOL_APPLICATION_LENGTH
;
6349 p
.application_metadata
[app
] = {};
6351 ss
<< "enabled application '" << app
<< "' on pool '" << pool_name
<< "'";
6353 } else if (boost::algorithm::ends_with(prefix
, "disable")) {
6355 cmd_getval(g_ceph_context
, cmdmap
, "force", force
);
6357 if (force
!= "--yes-i-really-mean-it") {
6358 ss
<< "Are you SURE? Disabling an application within a pool might result "
6359 << "in loss of application functionality; pass "
6360 << "--yes-i-really-mean-it to proceed anyway";
6365 ss
<< "application '" << app
<< "' is not enabled on pool '" << pool_name
6367 return 0; // idempotent
6370 p
.application_metadata
.erase(app
);
6371 ss
<< "disable application '" << app
<< "' on pool '" << pool_name
<< "'";
6373 } else if (boost::algorithm::ends_with(prefix
, "set")) {
6375 ss
<< "application metadata must be set on base tier";
6380 ss
<< "application '" << app
<< "' is not enabled on pool '" << pool_name
6386 cmd_getval(g_ceph_context
, cmdmap
, "key", key
);
6389 ss
<< "key must be provided";
6393 auto &app_keys
= p
.application_metadata
[app
];
6394 if (app_keys
.count(key
) == 0 &&
6395 app_keys
.size() >= MAX_POOL_APPLICATION_KEYS
) {
6396 ss
<< "too many keys set for application '" << app
<< "' on pool '"
6397 << pool_name
<< "'; max " << MAX_POOL_APPLICATION_KEYS
;
6401 if (key
.length() > MAX_POOL_APPLICATION_LENGTH
) {
6402 ss
<< "key '" << app
<< "' too long; max length "
6403 << MAX_POOL_APPLICATION_LENGTH
;
6408 cmd_getval(g_ceph_context
, cmdmap
, "value", value
);
6409 if (value
.length() > MAX_POOL_APPLICATION_LENGTH
) {
6410 ss
<< "value '" << value
<< "' too long; max length "
6411 << MAX_POOL_APPLICATION_LENGTH
;
6415 p
.application_metadata
[app
][key
] = value
;
6416 ss
<< "set application '" << app
<< "' key '" << key
<< "' to '"
6417 << value
<< "' on pool '" << pool_name
<< "'";
6418 } else if (boost::algorithm::ends_with(prefix
, "rm")) {
6420 ss
<< "application '" << app
<< "' is not enabled on pool '" << pool_name
6426 cmd_getval(g_ceph_context
, cmdmap
, "key", key
);
6427 auto it
= p
.application_metadata
[app
].find(key
);
6428 if (it
== p
.application_metadata
[app
].end()) {
6429 ss
<< "application '" << app
<< "' on pool '" << pool_name
6430 << "' does not have key '" << key
<< "'";
6431 return 0; // idempotent
6434 p
.application_metadata
[app
].erase(it
);
6435 ss
<< "removed application '" << app
<< "' key '" << key
<< "' on pool '"
6436 << pool_name
<< "'";
6441 p
.last_change
= pending_inc
.epoch
;
6442 pending_inc
.new_pools
[pool
] = p
;
6446 int OSDMonitor::_prepare_command_osd_crush_remove(
6447 CrushWrapper
&newcrush
,
6456 err
= newcrush
.remove_item_under(g_ceph_context
, id
, ancestor
,
6459 err
= newcrush
.remove_item(g_ceph_context
, id
, unlink_only
);
6464 void OSDMonitor::do_osd_crush_remove(CrushWrapper
& newcrush
)
6466 pending_inc
.crush
.clear();
6467 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
6470 int OSDMonitor::prepare_command_osd_crush_remove(
6471 CrushWrapper
&newcrush
,
6477 int err
= _prepare_command_osd_crush_remove(
6478 newcrush
, id
, ancestor
,
6479 has_ancestor
, unlink_only
);
6485 do_osd_crush_remove(newcrush
);
6490 int OSDMonitor::prepare_command_osd_remove(int32_t id
)
6492 if (osdmap
.is_up(id
)) {
6496 pending_inc
.new_state
[id
] = osdmap
.get_state(id
);
6497 pending_inc
.new_uuid
[id
] = uuid_d();
6498 pending_metadata_rm
.insert(id
);
6499 pending_metadata
.erase(id
);
6504 int32_t OSDMonitor::_allocate_osd_id(int32_t* existing_id
)
6506 assert(existing_id
);
6509 for (int32_t i
= 0; i
< osdmap
.get_max_osd(); ++i
) {
6510 if (!osdmap
.exists(i
) &&
6511 pending_inc
.new_up_client
.count(i
) == 0 &&
6512 (pending_inc
.new_state
.count(i
) == 0 ||
6513 (pending_inc
.new_state
[i
] & CEPH_OSD_EXISTS
) == 0)) {
6519 if (pending_inc
.new_max_osd
< 0) {
6520 return osdmap
.get_max_osd();
6522 return pending_inc
.new_max_osd
;
6525 void OSDMonitor::do_osd_create(
6530 dout(10) << __func__
<< " uuid " << uuid
<< dendl
;
6533 // We presume validation has been performed prior to calling this
6534 // function. We assert with prejudice.
6536 int32_t allocated_id
= -1; // declare here so we can jump
6537 int32_t existing_id
= -1;
6538 if (!uuid
.is_zero()) {
6539 existing_id
= osdmap
.identify_osd(uuid
);
6540 if (existing_id
>= 0) {
6541 assert(id
< 0 || id
== existing_id
);
6542 *new_id
= existing_id
;
6544 } else if (id
>= 0) {
6545 // uuid does not exist, and id has been provided, so just create
6552 // allocate a new id
6553 allocated_id
= _allocate_osd_id(&existing_id
);
6554 dout(10) << __func__
<< " allocated id " << allocated_id
6555 << " existing id " << existing_id
<< dendl
;
6556 if (existing_id
>= 0) {
6557 assert(existing_id
< osdmap
.get_max_osd());
6558 assert(allocated_id
< 0);
6559 pending_inc
.new_weight
[existing_id
] = CEPH_OSD_OUT
;
6560 *new_id
= existing_id
;
6562 } else if (allocated_id
>= 0) {
6563 assert(existing_id
< 0);
6565 if (pending_inc
.new_max_osd
< 0) {
6566 pending_inc
.new_max_osd
= osdmap
.get_max_osd() + 1;
6568 ++pending_inc
.new_max_osd
;
6570 *new_id
= pending_inc
.new_max_osd
- 1;
6571 assert(*new_id
== allocated_id
);
6573 assert(0 == "unexpected condition");
6577 dout(10) << __func__
<< " using id " << *new_id
<< dendl
;
6578 if (osdmap
.get_max_osd() <= *new_id
&& pending_inc
.new_max_osd
<= *new_id
) {
6579 pending_inc
.new_max_osd
= *new_id
+ 1;
6582 pending_inc
.new_state
[*new_id
] |= CEPH_OSD_EXISTS
| CEPH_OSD_NEW
;
6583 if (!uuid
.is_zero())
6584 pending_inc
.new_uuid
[*new_id
] = uuid
;
6587 int OSDMonitor::validate_osd_create(
6590 const bool check_osd_exists
,
6591 int32_t* existing_id
,
6595 dout(10) << __func__
<< " id " << id
<< " uuid " << uuid
6596 << " check_osd_exists " << check_osd_exists
<< dendl
;
6598 assert(existing_id
);
6600 if (id
< 0 && uuid
.is_zero()) {
6601 // we have nothing to validate
6604 } else if (uuid
.is_zero()) {
6605 // we have an id but we will ignore it - because that's what
6606 // `osd create` does.
6611 * This function will be used to validate whether we are able to
6612 * create a new osd when the `uuid` is specified.
6614 * It will be used by both `osd create` and `osd new`, as the checks
6615 * are basically the same when it pertains to osd id and uuid validation.
6616 * However, `osd create` presumes an `uuid` is optional, for legacy
6617 * reasons, while `osd new` requires the `uuid` to be provided. This
6618 * means that `osd create` will not be idempotent if an `uuid` is not
6619 * provided, but we will always guarantee the idempotency of `osd new`.
6622 assert(!uuid
.is_zero());
6623 if (pending_inc
.identify_osd(uuid
) >= 0) {
6624 // osd is about to exist
6628 int32_t i
= osdmap
.identify_osd(uuid
);
6630 // osd already exists
6631 if (id
>= 0 && i
!= id
) {
6632 ss
<< "uuid " << uuid
<< " already in use for different id " << i
;
6635 // return a positive errno to distinguish between a blocking error
6636 // and an error we consider to not be a problem (i.e., this would be
6637 // an idempotent operation).
6643 if (pending_inc
.new_state
.count(id
)) {
6644 // osd is about to exist
6647 // we may not care if an osd exists if we are recreating a previously
6649 if (check_osd_exists
&& osdmap
.exists(id
)) {
6650 ss
<< "id " << id
<< " already in use and does not match uuid "
6658 int OSDMonitor::prepare_command_osd_create(
6661 int32_t* existing_id
,
6664 dout(10) << __func__
<< " id " << id
<< " uuid " << uuid
<< dendl
;
6665 assert(existing_id
);
6667 if (uuid
.is_zero()) {
6668 dout(10) << __func__
<< " no uuid; assuming legacy `osd create`" << dendl
;
6671 return validate_osd_create(id
, uuid
, true, existing_id
, ss
);
6674 int OSDMonitor::prepare_command_osd_new(
6676 const map
<string
,cmd_vartype
>& cmdmap
,
6677 const map
<string
,string
>& secrets
,
6685 assert(paxos
->is_plugged());
6687 dout(10) << __func__
<< " " << op
<< dendl
;
6689 /* validate command. abort now if something's wrong. */
6691 /* `osd new` will expect a `uuid` to be supplied; `id` is optional.
6693 * If `id` is not specified, we will identify any existing osd based
6694 * on `uuid`. Operation will be idempotent iff secrets match.
6696 * If `id` is specified, we will identify any existing osd based on
6697 * `uuid` and match against `id`. If they match, operation will be
6698 * idempotent iff secrets match.
6700 * `-i secrets.json` will be optional. If supplied, will be used
6701 * to check for idempotency when `id` and `uuid` match.
6703 * If `id` is not specified, and `uuid` does not exist, an id will
6704 * be found or allocated for the osd.
6706 * If `id` is specified, and the osd has been previously marked
6707 * as destroyed, then the `id` will be reused.
6709 if (!cmd_getval(g_ceph_context
, cmdmap
, "uuid", uuidstr
)) {
6710 ss
<< "requires the OSD's UUID to be specified.";
6712 } else if (!uuid
.parse(uuidstr
.c_str())) {
6713 ss
<< "invalid UUID value '" << uuidstr
<< "'.";
6717 if (cmd_getval(g_ceph_context
, cmdmap
, "id", id
) &&
6719 ss
<< "invalid OSD id; must be greater or equal than zero.";
6723 // are we running an `osd create`-like command, or recreating
6724 // a previously destroyed osd?
6726 bool is_recreate_destroyed
= (id
>= 0 && osdmap
.is_destroyed(id
));
6728 // we will care about `id` to assess whether osd is `destroyed`, or
6729 // to create a new osd.
6730 // we will need an `id` by the time we reach auth.
6732 int32_t existing_id
= -1;
6733 int err
= validate_osd_create(id
, uuid
, !is_recreate_destroyed
,
6736 bool may_be_idempotent
= false;
6737 if (err
== EEXIST
) {
6738 // this is idempotent from the osdmon's point-of-view
6739 may_be_idempotent
= true;
6740 assert(existing_id
>= 0);
6742 } else if (err
< 0) {
6746 if (!may_be_idempotent
) {
6747 // idempotency is out of the window. We are either creating a new
6748 // osd or recreating a destroyed osd.
6750 // We now need to figure out if we have an `id` (and if it's valid),
6751 // of find an `id` if we don't have one.
6753 // NOTE: we need to consider the case where the `id` is specified for
6754 // `osd create`, and we must honor it. So this means checking if
6755 // the `id` is destroyed, and if so assume the destroy; otherwise,
6756 // check if it `exists` - in which case we complain about not being
6757 // `destroyed`. In the end, if nothing fails, we must allow the
6758 // creation, so that we are compatible with `create`.
6759 if (id
>= 0 && osdmap
.exists(id
) && !osdmap
.is_destroyed(id
)) {
6760 dout(10) << __func__
<< " osd." << id
<< " isn't destroyed" << dendl
;
6761 ss
<< "OSD " << id
<< " has not yet been destroyed";
6763 } else if (id
< 0) {
6765 id
= _allocate_osd_id(&existing_id
);
6767 assert(existing_id
>= 0);
6770 dout(10) << __func__
<< " found id " << id
<< " to use" << dendl
;
6771 } else if (id
>= 0 && osdmap
.is_destroyed(id
)) {
6772 dout(10) << __func__
<< " recreating osd." << id
<< dendl
;
6774 dout(10) << __func__
<< " creating new osd." << id
<< dendl
;
6778 assert(osdmap
.exists(id
));
6781 // we are now able to either create a brand new osd or reuse an existing
6782 // osd that has been previously destroyed.
6784 dout(10) << __func__
<< " id " << id
<< " uuid " << uuid
<< dendl
;
6786 if (may_be_idempotent
&& secrets
.empty()) {
6787 // nothing to do, really.
6788 dout(10) << __func__
<< " idempotent and no secrets -- no op." << dendl
;
6791 f
->open_object_section("created_osd");
6792 f
->dump_int("osdid", id
);
6800 string cephx_secret
, lockbox_secret
, dmcrypt_key
;
6801 bool has_lockbox
= false;
6802 bool has_secrets
= (!secrets
.empty());
6804 ConfigKeyService
*svc
= nullptr;
6805 AuthMonitor::auth_entity_t cephx_entity
, lockbox_entity
;
6808 if (secrets
.count("cephx_secret") == 0) {
6809 ss
<< "requires a cephx secret.";
6812 cephx_secret
= secrets
.at("cephx_secret");
6814 bool has_lockbox_secret
= (secrets
.count("cephx_lockbox_secret") > 0);
6815 bool has_dmcrypt_key
= (secrets
.count("dmcrypt_key") > 0);
6817 dout(10) << __func__
<< " has lockbox " << has_lockbox_secret
6818 << " dmcrypt " << has_dmcrypt_key
<< dendl
;
6820 if (has_lockbox_secret
&& has_dmcrypt_key
) {
6822 lockbox_secret
= secrets
.at("cephx_lockbox_secret");
6823 dmcrypt_key
= secrets
.at("dmcrypt_key");
6824 } else if (!has_lockbox_secret
!= !has_dmcrypt_key
) {
6825 ss
<< "requires both a cephx lockbox secret and a dm-crypt key.";
6829 dout(10) << __func__
<< " validate secrets using osd id " << id
<< dendl
;
6831 err
= mon
->authmon()->validate_osd_new(id
, uuid
,
6839 } else if (may_be_idempotent
&& err
!= EEXIST
) {
6840 // for this to be idempotent, `id` should already be >= 0; no need
6841 // to use validate_id.
6843 ss
<< "osd." << id
<< " exists but secrets do not match";
6848 svc
= (ConfigKeyService
*)mon
->config_key_service
;
6849 err
= svc
->validate_osd_new(uuid
, dmcrypt_key
, ss
);
6852 } else if (may_be_idempotent
&& err
!= EEXIST
) {
6854 ss
<< "osd." << id
<< " exists but dm-crypt key does not match.";
6859 assert(!has_secrets
|| !cephx_secret
.empty());
6860 assert(!has_lockbox
|| !lockbox_secret
.empty());
6862 if (may_be_idempotent
) {
6863 // we have nothing to do for either the osdmon or the authmon,
6864 // and we have no lockbox - so the config key service will not be
6865 // touched. This is therefore an idempotent operation, and we can
6866 // just return right away.
6867 dout(10) << __func__
<< " idempotent -- no op." << dendl
;
6870 f
->open_object_section("created_osd");
6871 f
->dump_int("osdid", id
);
6878 assert(!may_be_idempotent
);
6882 assert(!cephx_secret
.empty());
6883 assert((lockbox_secret
.empty() && dmcrypt_key
.empty()) ||
6884 (!lockbox_secret
.empty() && !dmcrypt_key
.empty()));
6886 err
= mon
->authmon()->do_osd_new(cephx_entity
,
6892 assert(nullptr != svc
);
6893 svc
->do_osd_new(uuid
, dmcrypt_key
);
6897 if (is_recreate_destroyed
) {
6899 assert(osdmap
.is_destroyed(id
));
6900 pending_inc
.new_weight
[id
] = CEPH_OSD_OUT
;
6901 pending_inc
.new_state
[id
] |= CEPH_OSD_DESTROYED
| CEPH_OSD_NEW
;
6902 if (osdmap
.get_state(id
) & CEPH_OSD_UP
) {
6903 // due to http://tracker.ceph.com/issues/20751 some clusters may
6904 // have UP set for non-existent OSDs; make sure it is cleared
6905 // for a newly created osd.
6906 pending_inc
.new_state
[id
] |= CEPH_OSD_UP
;
6908 pending_inc
.new_uuid
[id
] = uuid
;
6911 int32_t new_id
= -1;
6912 do_osd_create(id
, uuid
, &new_id
);
6913 assert(new_id
>= 0);
6914 assert(id
== new_id
);
6918 f
->open_object_section("created_osd");
6919 f
->dump_int("osdid", id
);
6928 bool OSDMonitor::prepare_command(MonOpRequestRef op
)
6930 op
->mark_osdmon_event(__func__
);
6931 MMonCommand
*m
= static_cast<MMonCommand
*>(op
->get_req());
6933 map
<string
, cmd_vartype
> cmdmap
;
6934 if (!cmdmap_from_json(m
->cmd
, &cmdmap
, ss
)) {
6935 string rs
= ss
.str();
6936 mon
->reply_command(op
, -EINVAL
, rs
, get_last_committed());
6940 MonSession
*session
= m
->get_session();
6942 mon
->reply_command(op
, -EACCES
, "access denied", get_last_committed());
6946 return prepare_command_impl(op
, cmdmap
);
6949 static int parse_reweights(CephContext
*cct
,
6950 const map
<string
,cmd_vartype
> &cmdmap
,
6951 const OSDMap
& osdmap
,
6952 map
<int32_t, uint32_t>* weights
)
6955 if (!cmd_getval(g_ceph_context
, cmdmap
, "weights", weights_str
)) {
6958 std::replace(begin(weights_str
), end(weights_str
), '\'', '"');
6959 json_spirit::mValue json_value
;
6960 if (!json_spirit::read(weights_str
, json_value
)) {
6963 if (json_value
.type() != json_spirit::obj_type
) {
6966 const auto obj
= json_value
.get_obj();
6968 for (auto& osd_weight
: obj
) {
6969 auto osd_id
= std::stoi(osd_weight
.first
);
6970 if (!osdmap
.exists(osd_id
)) {
6973 if (osd_weight
.second
.type() != json_spirit::str_type
) {
6976 auto weight
= std::stoul(osd_weight
.second
.get_str());
6977 weights
->insert({osd_id
, weight
});
6979 } catch (const std::logic_error
& e
) {
6985 int OSDMonitor::prepare_command_osd_destroy(
6989 assert(paxos
->is_plugged());
6991 // we check if the osd exists for the benefit of `osd purge`, which may
6992 // have previously removed the osd. If the osd does not exist, return
6993 // -ENOENT to convey this, and let the caller deal with it.
6995 // we presume that all auth secrets and config keys were removed prior
6996 // to this command being called. if they exist by now, we also assume
6997 // they must have been created by some other command and do not pertain
6998 // to this non-existent osd.
6999 if (!osdmap
.exists(id
)) {
7000 dout(10) << __func__
<< " osd." << id
<< " does not exist." << dendl
;
7004 uuid_d uuid
= osdmap
.get_uuid(id
);
7005 dout(10) << __func__
<< " destroying osd." << id
7006 << " uuid " << uuid
<< dendl
;
7008 // if it has been destroyed, we assume our work here is done.
7009 if (osdmap
.is_destroyed(id
)) {
7010 ss
<< "destroyed osd." << id
;
7014 EntityName cephx_entity
, lockbox_entity
;
7015 bool idempotent_auth
= false, idempotent_cks
= false;
7017 int err
= mon
->authmon()->validate_osd_destroy(id
, uuid
,
7022 if (err
== -ENOENT
) {
7023 idempotent_auth
= true;
7029 ConfigKeyService
*svc
= (ConfigKeyService
*)mon
->config_key_service
;
7030 err
= svc
->validate_osd_destroy(id
, uuid
);
7032 assert(err
== -ENOENT
);
7034 idempotent_cks
= true;
7037 if (!idempotent_auth
) {
7038 err
= mon
->authmon()->do_osd_destroy(cephx_entity
, lockbox_entity
);
7042 if (!idempotent_cks
) {
7043 svc
->do_osd_destroy(id
, uuid
);
7046 pending_inc
.new_state
[id
] = CEPH_OSD_DESTROYED
;
7047 pending_inc
.new_uuid
[id
] = uuid_d();
7049 // we can only propose_pending() once per service, otherwise we'll be
7050 // defying PaxosService and all laws of nature. Therefore, as we may
7051 // be used during 'osd purge', let's keep the caller responsible for
7057 int OSDMonitor::prepare_command_osd_purge(
7061 assert(paxos
->is_plugged());
7062 dout(10) << __func__
<< " purging osd." << id
<< dendl
;
7064 assert(!osdmap
.is_up(id
));
7067 * This may look a bit weird, but this is what's going to happen:
7069 * 1. we make sure that removing from crush works
7070 * 2. we call `prepare_command_osd_destroy()`. If it returns an
7071 * error, then we abort the whole operation, as no updates
7072 * have been made. However, we this function will have
7073 * side-effects, thus we need to make sure that all operations
7074 * performed henceforth will *always* succeed.
7075 * 3. we call `prepare_command_osd_remove()`. Although this
7076 * function can return an error, it currently only checks if the
7077 * osd is up - and we have made sure that it is not so, so there
7078 * is no conflict, and it is effectively an update.
7079 * 4. finally, we call `do_osd_crush_remove()`, which will perform
7080 * the crush update we delayed from before.
7083 CrushWrapper newcrush
;
7084 _get_pending_crush(newcrush
);
7086 bool may_be_idempotent
= false;
7088 int err
= _prepare_command_osd_crush_remove(newcrush
, id
, 0, false, false);
7089 if (err
== -ENOENT
) {
7091 may_be_idempotent
= true;
7092 } else if (err
< 0) {
7093 ss
<< "error removing osd." << id
<< " from crush";
7097 // no point destroying the osd again if it has already been marked destroyed
7098 if (!osdmap
.is_destroyed(id
)) {
7099 err
= prepare_command_osd_destroy(id
, ss
);
7101 if (err
== -ENOENT
) {
7107 may_be_idempotent
= false;
7112 if (may_be_idempotent
&& !osdmap
.exists(id
)) {
7113 dout(10) << __func__
<< " osd." << id
<< " does not exist and "
7114 << "we are idempotent." << dendl
;
7118 err
= prepare_command_osd_remove(id
);
7119 // we should not be busy, as we should have made sure this id is not up.
7122 do_osd_crush_remove(newcrush
);
7126 bool OSDMonitor::prepare_command_impl(MonOpRequestRef op
,
7127 map
<string
,cmd_vartype
> &cmdmap
)
7129 op
->mark_osdmon_event(__func__
);
7130 MMonCommand
*m
= static_cast<MMonCommand
*>(op
->get_req());
7138 cmd_getval(g_ceph_context
, cmdmap
, "format", format
, string("plain"));
7139 boost::scoped_ptr
<Formatter
> f(Formatter::create(format
));
7142 cmd_getval(g_ceph_context
, cmdmap
, "prefix", prefix
);
7146 bool osdid_present
= cmd_getval(g_ceph_context
, cmdmap
, "id", osdid
);
7147 if (osdid_present
) {
7149 oss
<< "osd." << osdid
;
7153 // Even if there's a pending state with changes that could affect
7154 // a command, considering that said state isn't yet committed, we
7155 // just don't care about those changes if the command currently being
7156 // handled acts as a no-op against the current committed state.
7157 // In a nutshell, we assume this command happens *before*.
7159 // Let me make this clearer:
7161 // - If we have only one client, and that client issues some
7162 // operation that would conflict with this operation but is
7163 // still on the pending state, then we would be sure that said
7164 // operation wouldn't have returned yet, so the client wouldn't
7165 // issue this operation (unless the client didn't wait for the
7166 // operation to finish, and that would be the client's own fault).
7168 // - If we have more than one client, each client will observe
7169 // whatever is the state at the moment of the commit. So, if we
7170 // have two clients, one issuing an unlink and another issuing a
7171 // link, and if the link happens while the unlink is still on the
7172 // pending state, from the link's point-of-view this is a no-op.
7173 // If different clients are issuing conflicting operations and
7174 // they care about that, then the clients should make sure they
7175 // enforce some kind of concurrency mechanism -- from our
7176 // perspective that's what Douglas Adams would call an SEP.
7178 // This should be used as a general guideline for most commands handled
7179 // in this function. Adapt as you see fit, but please bear in mind that
7180 // this is the expected behavior.
7183 if (prefix
== "osd setcrushmap" ||
7184 (prefix
== "osd crush set" && !osdid_present
)) {
7185 if (pending_inc
.crush
.length()) {
7186 dout(10) << __func__
<< " waiting for pending crush update " << dendl
;
7187 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
7190 dout(10) << "prepare_command setting new crush map" << dendl
;
7191 bufferlist
data(m
->get_data());
7194 bufferlist::iterator
bl(data
.begin());
7197 catch (const std::exception
&e
) {
7199 ss
<< "Failed to parse crushmap: " << e
.what();
7203 int64_t prior_version
= 0;
7204 if (cmd_getval(g_ceph_context
, cmdmap
, "prior_version", prior_version
)) {
7205 if (prior_version
== osdmap
.get_crush_version() - 1) {
7206 // see if we are a resend of the last update. this is imperfect
7207 // (multiple racing updaters may not both get reliable success)
7208 // but we expect crush updaters (via this interface) to be rare-ish.
7209 bufferlist current
, proposed
;
7210 osdmap
.crush
->encode(current
, mon
->get_quorum_con_features());
7211 crush
.encode(proposed
, mon
->get_quorum_con_features());
7212 if (current
.contents_equal(proposed
)) {
7213 dout(10) << __func__
7214 << " proposed matches current and version equals previous"
7217 ss
<< osdmap
.get_crush_version();
7221 if (prior_version
!= osdmap
.get_crush_version()) {
7223 ss
<< "prior_version " << prior_version
<< " != crush version "
7224 << osdmap
.get_crush_version();
7229 if (crush
.has_legacy_rulesets()) {
7231 ss
<< "crush maps with ruleset != ruleid are no longer allowed";
7234 if (!validate_crush_against_features(&crush
, ss
)) {
7239 const auto& osdmap_pools
= osdmap
.get_pools();
7240 for (auto pit
= osdmap_pools
.begin(); pit
!= osdmap_pools
.end(); ++pit
) {
7241 const int64_t pool_id
= pit
->first
;
7242 const pg_pool_t
&pool
= pit
->second
;
7243 int ruleno
= pool
.get_crush_rule();
7244 if (!crush
.rule_exists(ruleno
)) {
7245 ss
<< " the crush rule no "<< ruleno
<< " for pool id " << pool_id
<< " is in use";
7251 if (g_conf
->mon_osd_crush_smoke_test
) {
7252 // sanity check: test some inputs to make sure this map isn't
7254 dout(10) << " testing map" << dendl
;
7256 CrushTester
tester(crush
, ess
);
7257 tester
.set_max_x(50);
7258 int r
= tester
.test_with_fork(g_conf
->mon_lease
);
7260 dout(10) << " tester.test_with_fork returns " << r
7261 << ": " << ess
.str() << dendl
;
7262 ss
<< "crush smoke test failed with " << r
<< ": " << ess
.str();
7266 dout(10) << " crush test result " << ess
.str() << dendl
;
7269 pending_inc
.crush
= data
;
7270 ss
<< osdmap
.get_crush_version() + 1;
7273 } else if (prefix
== "osd crush set-device-class") {
7274 if (osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
7275 ss
<< "you must complete the upgrade and 'ceph osd require-osd-release "
7276 << "luminous' before using crush device classes";
7281 string device_class
;
7282 if (!cmd_getval(g_ceph_context
, cmdmap
, "class", device_class
)) {
7283 err
= -EINVAL
; // no value!
7288 vector
<string
> idvec
;
7289 cmd_getval(g_ceph_context
, cmdmap
, "ids", idvec
);
7290 CrushWrapper newcrush
;
7291 _get_pending_crush(newcrush
);
7293 for (unsigned j
= 0; j
< idvec
.size() && !stop
; j
++) {
7297 (idvec
[0] == "any" || idvec
[0] == "all" || idvec
[0] == "*")) {
7298 osdmap
.get_all_osds(osds
);
7301 // try traditional single osd way
7302 long osd
= parse_osd_id(idvec
[j
].c_str(), &ss
);
7304 // ss has reason for failure
7305 ss
<< ", unable to parse osd id:\"" << idvec
[j
] << "\". ";
7312 for (auto &osd
: osds
) {
7313 if (!osdmap
.exists(osd
)) {
7314 ss
<< "osd." << osd
<< " does not exist. ";
7319 oss
<< "osd." << osd
;
7320 string name
= oss
.str();
7323 if (newcrush
.item_exists(osd
)) {
7324 action
= "updating";
7326 action
= "creating";
7327 newcrush
.set_item_name(osd
, name
);
7330 dout(5) << action
<< " crush item id " << osd
<< " name '" << name
7331 << "' device_class '" << device_class
<< "'"
7333 err
= newcrush
.update_device_class(osd
, device_class
, name
, &ss
);
7337 if (err
== 0 && !_have_pending_crush()) {
7339 // for single osd only, wildcard makes too much noise
7340 ss
<< "set-device-class item id " << osd
<< " name '" << name
7341 << "' device_class '" << device_class
<< "': no change";
7344 updated
.insert(osd
);
7349 if (!updated
.empty()) {
7350 pending_inc
.crush
.clear();
7351 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7352 ss
<< "set osd(s) " << updated
<< " to class '" << device_class
<< "'";
7354 wait_for_finished_proposal(op
,
7355 new Monitor::C_Command(mon
,op
, 0, rs
, get_last_committed() + 1));
7359 } else if (prefix
== "osd crush rm-device-class") {
7361 vector
<string
> idvec
;
7362 cmd_getval(g_ceph_context
, cmdmap
, "ids", idvec
);
7363 CrushWrapper newcrush
;
7364 _get_pending_crush(newcrush
);
7367 for (unsigned j
= 0; j
< idvec
.size() && !stop
; j
++) {
7372 (idvec
[0] == "any" || idvec
[0] == "all" || idvec
[0] == "*")) {
7373 osdmap
.get_all_osds(osds
);
7376 // try traditional single osd way
7377 long osd
= parse_osd_id(idvec
[j
].c_str(), &ss
);
7379 // ss has reason for failure
7380 ss
<< ", unable to parse osd id:\"" << idvec
[j
] << "\". ";
7387 for (auto &osd
: osds
) {
7388 if (!osdmap
.exists(osd
)) {
7389 ss
<< "osd." << osd
<< " does not exist. ";
7393 auto class_name
= newcrush
.get_item_class(osd
);
7396 ss
<< "osd." << osd
<< " belongs to no class, ";
7399 // note that we do not verify if class_is_in_use here
7400 // in case the device is misclassified and user wants
7401 // to overridely reset...
7403 err
= newcrush
.remove_device_class(g_ceph_context
, osd
, &ss
);
7405 // ss has reason for failure
7408 updated
.insert(osd
);
7412 if (!updated
.empty()) {
7413 pending_inc
.crush
.clear();
7414 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7415 ss
<< "done removing class of osd(s): " << updated
;
7417 wait_for_finished_proposal(op
,
7418 new Monitor::C_Command(mon
,op
, 0, rs
, get_last_committed() + 1));
7422 } else if (prefix
== "osd crush add-bucket") {
7423 // os crush add-bucket <name> <type>
7424 string name
, typestr
;
7425 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
7426 cmd_getval(g_ceph_context
, cmdmap
, "type", typestr
);
7428 if (!_have_pending_crush() &&
7429 _get_stable_crush().name_exists(name
)) {
7430 ss
<< "bucket '" << name
<< "' already exists";
7434 CrushWrapper newcrush
;
7435 _get_pending_crush(newcrush
);
7437 if (newcrush
.name_exists(name
)) {
7438 ss
<< "bucket '" << name
<< "' already exists";
7441 int type
= newcrush
.get_type_id(typestr
);
7443 ss
<< "type '" << typestr
<< "' does not exist";
7448 ss
<< "type '" << typestr
<< "' is for devices, not buckets";
7453 err
= newcrush
.add_bucket(0, 0,
7454 CRUSH_HASH_DEFAULT
, type
, 0, NULL
,
7457 ss
<< "add_bucket error: '" << cpp_strerror(err
) << "'";
7460 err
= newcrush
.set_item_name(bucketno
, name
);
7462 ss
<< "error setting bucket name to '" << name
<< "'";
7466 pending_inc
.crush
.clear();
7467 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7468 ss
<< "added bucket " << name
<< " type " << typestr
7471 } else if (prefix
== "osd crush rename-bucket") {
7472 string srcname
, dstname
;
7473 cmd_getval(g_ceph_context
, cmdmap
, "srcname", srcname
);
7474 cmd_getval(g_ceph_context
, cmdmap
, "dstname", dstname
);
7476 err
= crush_rename_bucket(srcname
, dstname
, &ss
);
7477 if (err
== -EALREADY
) // equivalent to success for idempotency
7483 } else if (prefix
== "osd crush class rm") {
7484 string device_class
;
7485 if (!cmd_getval(g_ceph_context
, cmdmap
, "class", device_class
)) {
7486 err
= -EINVAL
; // no value!
7489 if (osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
7490 ss
<< "you must complete the upgrade and 'ceph osd require-osd-release "
7491 << "luminous' before using crush device classes";
7496 CrushWrapper newcrush
;
7497 _get_pending_crush(newcrush
);
7499 if (!newcrush
.class_exists(device_class
)) {
7501 ss
<< "class '" << device_class
<< "' does not exist";
7505 int class_id
= newcrush
.get_class_id(device_class
);
7508 if (newcrush
.class_is_in_use(class_id
, &ts
)) {
7510 ss
<< "class '" << device_class
<< "' " << ts
.str();
7515 newcrush
.get_devices_by_class(device_class
, &osds
);
7516 for (auto& p
: osds
) {
7517 err
= newcrush
.remove_device_class(g_ceph_context
, p
, &ss
);
7519 // ss has reason for failure
7525 // empty class, remove directly
7526 err
= newcrush
.remove_class_name(device_class
);
7528 ss
<< "class '" << device_class
<< "' cannot be removed '"
7529 << cpp_strerror(err
) << "'";
7534 pending_inc
.crush
.clear();
7535 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7536 ss
<< "removed class " << device_class
<< " with id " << class_id
7537 << " from crush map";
7540 } else if (prefix
== "osd crush weight-set create" ||
7541 prefix
== "osd crush weight-set create-compat") {
7542 CrushWrapper newcrush
;
7543 _get_pending_crush(newcrush
);
7546 if (newcrush
.has_non_straw2_buckets()) {
7547 ss
<< "crush map contains one or more bucket(s) that are not straw2";
7551 if (prefix
== "osd crush weight-set create") {
7552 if (osdmap
.require_min_compat_client
> 0 &&
7553 osdmap
.require_min_compat_client
< CEPH_RELEASE_LUMINOUS
) {
7554 ss
<< "require_min_compat_client "
7555 << ceph_release_name(osdmap
.require_min_compat_client
)
7556 << " < luminous, which is required for per-pool weight-sets. "
7557 << "Try 'ceph osd set-require-min-compat-client luminous' "
7558 << "before using the new interface";
7562 string poolname
, mode
;
7563 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolname
);
7564 pool
= osdmap
.lookup_pg_pool_name(poolname
.c_str());
7566 ss
<< "pool '" << poolname
<< "' not found";
7570 cmd_getval(g_ceph_context
, cmdmap
, "mode", mode
);
7571 if (mode
!= "flat" && mode
!= "positional") {
7572 ss
<< "unrecognized weight-set mode '" << mode
<< "'";
7576 positions
= mode
== "flat" ? 1 : osdmap
.get_pg_pool(pool
)->get_size();
7578 pool
= CrushWrapper::DEFAULT_CHOOSE_ARGS
;
7581 newcrush
.create_choose_args(pool
, positions
);
7582 pending_inc
.crush
.clear();
7583 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7586 } else if (prefix
== "osd crush weight-set rm" ||
7587 prefix
== "osd crush weight-set rm-compat") {
7588 CrushWrapper newcrush
;
7589 _get_pending_crush(newcrush
);
7591 if (prefix
== "osd crush weight-set rm") {
7593 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolname
);
7594 pool
= osdmap
.lookup_pg_pool_name(poolname
.c_str());
7596 ss
<< "pool '" << poolname
<< "' not found";
7601 pool
= CrushWrapper::DEFAULT_CHOOSE_ARGS
;
7603 newcrush
.rm_choose_args(pool
);
7604 pending_inc
.crush
.clear();
7605 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7608 } else if (prefix
== "osd crush weight-set reweight" ||
7609 prefix
== "osd crush weight-set reweight-compat") {
7610 string poolname
, item
;
7611 vector
<double> weight
;
7612 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolname
);
7613 cmd_getval(g_ceph_context
, cmdmap
, "item", item
);
7614 cmd_getval(g_ceph_context
, cmdmap
, "weight", weight
);
7615 CrushWrapper newcrush
;
7616 _get_pending_crush(newcrush
);
7618 if (prefix
== "osd crush weight-set reweight") {
7619 pool
= osdmap
.lookup_pg_pool_name(poolname
.c_str());
7621 ss
<< "pool '" << poolname
<< "' not found";
7625 if (!newcrush
.have_choose_args(pool
)) {
7626 ss
<< "no weight-set for pool '" << poolname
<< "'";
7630 auto arg_map
= newcrush
.choose_args_get(pool
);
7631 int positions
= newcrush
.get_choose_args_positions(arg_map
);
7632 if (weight
.size() != (size_t)positions
) {
7633 ss
<< "must specify exact " << positions
<< " weight values";
7638 pool
= CrushWrapper::DEFAULT_CHOOSE_ARGS
;
7639 if (!newcrush
.have_choose_args(pool
)) {
7640 ss
<< "no backward-compatible weight-set";
7645 if (!newcrush
.name_exists(item
)) {
7646 ss
<< "item '" << item
<< "' does not exist";
7650 err
= newcrush
.choose_args_adjust_item_weightf(
7652 newcrush
.choose_args_get(pool
),
7653 newcrush
.get_item_id(item
),
7660 pending_inc
.crush
.clear();
7661 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7663 } else if (osdid_present
&&
7664 (prefix
== "osd crush set" || prefix
== "osd crush add")) {
7665 // <OsdName> is 'osd.<id>' or '<id>', passed as int64_t id
7666 // osd crush set <OsdName> <weight> <loc1> [<loc2> ...]
7667 // osd crush add <OsdName> <weight> <loc1> [<loc2> ...]
7669 if (!osdmap
.exists(osdid
)) {
7671 ss
<< name
<< " does not exist. Create it before updating the crush map";
7676 if (!cmd_getval(g_ceph_context
, cmdmap
, "weight", weight
)) {
7677 ss
<< "unable to parse weight value '"
7678 << cmd_vartype_stringify(cmdmap
["weight"]) << "'";
7684 vector
<string
> argvec
;
7685 cmd_getval(g_ceph_context
, cmdmap
, "args", argvec
);
7686 map
<string
,string
> loc
;
7687 CrushWrapper::parse_loc_map(argvec
, &loc
);
7689 if (prefix
== "osd crush set"
7690 && !_get_stable_crush().item_exists(osdid
)) {
7692 ss
<< "unable to set item id " << osdid
<< " name '" << name
7693 << "' weight " << weight
<< " at location " << loc
7694 << ": does not exist";
7698 dout(5) << "adding/updating crush item id " << osdid
<< " name '"
7699 << name
<< "' weight " << weight
<< " at location "
7701 CrushWrapper newcrush
;
7702 _get_pending_crush(newcrush
);
7705 if (prefix
== "osd crush set" ||
7706 newcrush
.check_item_loc(g_ceph_context
, osdid
, loc
, (int *)NULL
)) {
7708 err
= newcrush
.update_item(g_ceph_context
, osdid
, weight
, name
, loc
);
7711 err
= newcrush
.insert_item(g_ceph_context
, osdid
, weight
, name
, loc
);
7719 if (err
== 0 && !_have_pending_crush()) {
7720 ss
<< action
<< " item id " << osdid
<< " name '" << name
<< "' weight "
7721 << weight
<< " at location " << loc
<< ": no change";
7725 pending_inc
.crush
.clear();
7726 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7727 ss
<< action
<< " item id " << osdid
<< " name '" << name
<< "' weight "
7728 << weight
<< " at location " << loc
<< " to crush map";
7730 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
7731 get_last_committed() + 1));
7734 } else if (prefix
== "osd crush create-or-move") {
7736 // osd crush create-or-move <OsdName> <initial_weight> <loc1> [<loc2> ...]
7737 if (!osdmap
.exists(osdid
)) {
7739 ss
<< name
<< " does not exist. create it before updating the crush map";
7744 if (!cmd_getval(g_ceph_context
, cmdmap
, "weight", weight
)) {
7745 ss
<< "unable to parse weight value '"
7746 << cmd_vartype_stringify(cmdmap
["weight"]) << "'";
7752 vector
<string
> argvec
;
7753 cmd_getval(g_ceph_context
, cmdmap
, "args", argvec
);
7754 map
<string
,string
> loc
;
7755 CrushWrapper::parse_loc_map(argvec
, &loc
);
7757 dout(0) << "create-or-move crush item name '" << name
<< "' initial_weight " << weight
7758 << " at location " << loc
<< dendl
;
7760 CrushWrapper newcrush
;
7761 _get_pending_crush(newcrush
);
7763 err
= newcrush
.create_or_move_item(g_ceph_context
, osdid
, weight
, name
, loc
);
7765 ss
<< "create-or-move updated item name '" << name
<< "' weight " << weight
7766 << " at location " << loc
<< " to crush map";
7770 pending_inc
.crush
.clear();
7771 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7772 ss
<< "create-or-move updating item name '" << name
<< "' weight " << weight
7773 << " at location " << loc
<< " to crush map";
7775 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
7776 get_last_committed() + 1));
7781 } else if (prefix
== "osd crush move") {
7783 // osd crush move <name> <loc1> [<loc2> ...]
7786 vector
<string
> argvec
;
7787 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
7788 cmd_getval(g_ceph_context
, cmdmap
, "args", argvec
);
7789 map
<string
,string
> loc
;
7790 CrushWrapper::parse_loc_map(argvec
, &loc
);
7792 dout(0) << "moving crush item name '" << name
<< "' to location " << loc
<< dendl
;
7793 CrushWrapper newcrush
;
7794 _get_pending_crush(newcrush
);
7796 if (!newcrush
.name_exists(name
)) {
7798 ss
<< "item " << name
<< " does not exist";
7801 int id
= newcrush
.get_item_id(name
);
7803 if (!newcrush
.check_item_loc(g_ceph_context
, id
, loc
, (int *)NULL
)) {
7805 err
= newcrush
.create_or_move_item(g_ceph_context
, id
, 0, name
, loc
);
7807 err
= newcrush
.move_bucket(g_ceph_context
, id
, loc
);
7810 ss
<< "moved item id " << id
<< " name '" << name
<< "' to location " << loc
<< " in crush map";
7811 pending_inc
.crush
.clear();
7812 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7814 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
7815 get_last_committed() + 1));
7819 ss
<< "no need to move item id " << id
<< " name '" << name
<< "' to location " << loc
<< " in crush map";
7823 } else if (prefix
== "osd crush swap-bucket") {
7824 string source
, dest
, force
;
7825 cmd_getval(g_ceph_context
, cmdmap
, "source", source
);
7826 cmd_getval(g_ceph_context
, cmdmap
, "dest", dest
);
7827 cmd_getval(g_ceph_context
, cmdmap
, "force", force
);
7828 CrushWrapper newcrush
;
7829 _get_pending_crush(newcrush
);
7830 if (!newcrush
.name_exists(source
)) {
7831 ss
<< "source item " << source
<< " does not exist";
7835 if (!newcrush
.name_exists(dest
)) {
7836 ss
<< "dest item " << dest
<< " does not exist";
7840 int sid
= newcrush
.get_item_id(source
);
7841 int did
= newcrush
.get_item_id(dest
);
7843 if (newcrush
.get_immediate_parent_id(sid
, &sparent
) == 0 &&
7844 force
!= "--yes-i-really-mean-it") {
7845 ss
<< "source item " << source
<< " is not an orphan bucket; pass --yes-i-really-mean-it to proceed anyway";
7849 if (newcrush
.get_bucket_alg(sid
) != newcrush
.get_bucket_alg(did
) &&
7850 force
!= "--yes-i-really-mean-it") {
7851 ss
<< "source bucket alg " << crush_alg_name(newcrush
.get_bucket_alg(sid
)) << " != "
7852 << "dest bucket alg " << crush_alg_name(newcrush
.get_bucket_alg(did
))
7853 << "; pass --yes-i-really-mean-it to proceed anyway";
7857 int r
= newcrush
.swap_bucket(g_ceph_context
, sid
, did
);
7859 ss
<< "failed to swap bucket contents: " << cpp_strerror(r
);
7863 ss
<< "swapped bucket of " << source
<< " to " << dest
;
7864 pending_inc
.crush
.clear();
7865 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7866 wait_for_finished_proposal(op
,
7867 new Monitor::C_Command(mon
, op
, err
, ss
.str(),
7868 get_last_committed() + 1));
7870 } else if (prefix
== "osd crush link") {
7871 // osd crush link <name> <loc1> [<loc2> ...]
7873 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
7874 vector
<string
> argvec
;
7875 cmd_getval(g_ceph_context
, cmdmap
, "args", argvec
);
7876 map
<string
,string
> loc
;
7877 CrushWrapper::parse_loc_map(argvec
, &loc
);
7879 // Need an explicit check for name_exists because get_item_id returns
7881 int id
= osdmap
.crush
->get_item_id(name
);
7882 if (!osdmap
.crush
->name_exists(name
)) {
7884 ss
<< "item " << name
<< " does not exist";
7887 dout(5) << "resolved crush name '" << name
<< "' to id " << id
<< dendl
;
7889 if (osdmap
.crush
->check_item_loc(g_ceph_context
, id
, loc
, (int*) NULL
)) {
7890 ss
<< "no need to move item id " << id
<< " name '" << name
7891 << "' to location " << loc
<< " in crush map";
7896 dout(5) << "linking crush item name '" << name
<< "' at location " << loc
<< dendl
;
7897 CrushWrapper newcrush
;
7898 _get_pending_crush(newcrush
);
7900 if (!newcrush
.name_exists(name
)) {
7902 ss
<< "item " << name
<< " does not exist";
7905 int id
= newcrush
.get_item_id(name
);
7906 if (!newcrush
.check_item_loc(g_ceph_context
, id
, loc
, (int *)NULL
)) {
7907 err
= newcrush
.link_bucket(g_ceph_context
, id
, loc
);
7909 ss
<< "linked item id " << id
<< " name '" << name
7910 << "' to location " << loc
<< " in crush map";
7911 pending_inc
.crush
.clear();
7912 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7914 ss
<< "cannot link item id " << id
<< " name '" << name
7915 << "' to location " << loc
;
7919 ss
<< "no need to move item id " << id
<< " name '" << name
7920 << "' to location " << loc
<< " in crush map";
7924 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, err
, ss
.str(),
7925 get_last_committed() + 1));
7927 } else if (prefix
== "osd crush rm" ||
7928 prefix
== "osd crush remove" ||
7929 prefix
== "osd crush unlink") {
7931 // osd crush rm <id> [ancestor]
7932 CrushWrapper newcrush
;
7933 _get_pending_crush(newcrush
);
7936 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
7938 if (!osdmap
.crush
->name_exists(name
)) {
7940 ss
<< "device '" << name
<< "' does not appear in the crush map";
7943 if (!newcrush
.name_exists(name
)) {
7945 ss
<< "device '" << name
<< "' does not appear in the crush map";
7947 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
7948 get_last_committed() + 1));
7951 int id
= newcrush
.get_item_id(name
);
7954 bool unlink_only
= prefix
== "osd crush unlink";
7955 string ancestor_str
;
7956 if (cmd_getval(g_ceph_context
, cmdmap
, "ancestor", ancestor_str
)) {
7957 if (!newcrush
.name_exists(ancestor_str
)) {
7959 ss
<< "ancestor item '" << ancestor_str
7960 << "' does not appear in the crush map";
7963 ancestor
= newcrush
.get_item_id(ancestor_str
);
7966 err
= prepare_command_osd_crush_remove(
7969 (ancestor
< 0), unlink_only
);
7971 if (err
== -ENOENT
) {
7972 ss
<< "item " << id
<< " does not appear in that position";
7977 ss
<< "removed item id " << id
<< " name '" << name
<< "' from crush map";
7979 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
7980 get_last_committed() + 1));
7985 } else if (prefix
== "osd crush reweight-all") {
7986 CrushWrapper newcrush
;
7987 _get_pending_crush(newcrush
);
7989 newcrush
.reweight(g_ceph_context
);
7990 pending_inc
.crush
.clear();
7991 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
7992 ss
<< "reweighted crush hierarchy";
7994 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
7995 get_last_committed() + 1));
7997 } else if (prefix
== "osd crush reweight") {
7998 // osd crush reweight <name> <weight>
7999 CrushWrapper newcrush
;
8000 _get_pending_crush(newcrush
);
8003 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
8004 if (!newcrush
.name_exists(name
)) {
8006 ss
<< "device '" << name
<< "' does not appear in the crush map";
8010 int id
= newcrush
.get_item_id(name
);
8012 ss
<< "device '" << name
<< "' is not a leaf in the crush map";
8017 if (!cmd_getval(g_ceph_context
, cmdmap
, "weight", w
)) {
8018 ss
<< "unable to parse weight value '"
8019 << cmd_vartype_stringify(cmdmap
["weight"]) << "'";
8024 err
= newcrush
.adjust_item_weightf(g_ceph_context
, id
, w
);
8027 pending_inc
.crush
.clear();
8028 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
8029 ss
<< "reweighted item id " << id
<< " name '" << name
<< "' to " << w
8032 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8033 get_last_committed() + 1));
8035 } else if (prefix
== "osd crush reweight-subtree") {
8036 // osd crush reweight <name> <weight>
8037 CrushWrapper newcrush
;
8038 _get_pending_crush(newcrush
);
8041 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
8042 if (!newcrush
.name_exists(name
)) {
8044 ss
<< "device '" << name
<< "' does not appear in the crush map";
8048 int id
= newcrush
.get_item_id(name
);
8050 ss
<< "device '" << name
<< "' is not a subtree in the crush map";
8055 if (!cmd_getval(g_ceph_context
, cmdmap
, "weight", w
)) {
8056 ss
<< "unable to parse weight value '"
8057 << cmd_vartype_stringify(cmdmap
["weight"]) << "'";
8062 err
= newcrush
.adjust_subtree_weightf(g_ceph_context
, id
, w
);
8065 pending_inc
.crush
.clear();
8066 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
8067 ss
<< "reweighted subtree id " << id
<< " name '" << name
<< "' to " << w
8070 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8071 get_last_committed() + 1));
8073 } else if (prefix
== "osd crush tunables") {
8074 CrushWrapper newcrush
;
8075 _get_pending_crush(newcrush
);
8079 cmd_getval(g_ceph_context
, cmdmap
, "profile", profile
);
8080 if (profile
== "legacy" || profile
== "argonaut") {
8081 newcrush
.set_tunables_legacy();
8082 } else if (profile
== "bobtail") {
8083 newcrush
.set_tunables_bobtail();
8084 } else if (profile
== "firefly") {
8085 newcrush
.set_tunables_firefly();
8086 } else if (profile
== "hammer") {
8087 newcrush
.set_tunables_hammer();
8088 } else if (profile
== "jewel") {
8089 newcrush
.set_tunables_jewel();
8090 } else if (profile
== "optimal") {
8091 newcrush
.set_tunables_optimal();
8092 } else if (profile
== "default") {
8093 newcrush
.set_tunables_default();
8095 ss
<< "unrecognized profile '" << profile
<< "'";
8100 if (!validate_crush_against_features(&newcrush
, ss
)) {
8105 pending_inc
.crush
.clear();
8106 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
8107 ss
<< "adjusted tunables profile to " << profile
;
8109 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8110 get_last_committed() + 1));
8112 } else if (prefix
== "osd crush set-tunable") {
8113 CrushWrapper newcrush
;
8114 _get_pending_crush(newcrush
);
8118 cmd_getval(g_ceph_context
, cmdmap
, "tunable", tunable
);
8121 if (!cmd_getval(g_ceph_context
, cmdmap
, "value", value
)) {
8123 ss
<< "failed to parse integer value " << cmd_vartype_stringify(cmdmap
["value"]);
8127 if (tunable
== "straw_calc_version") {
8128 if (value
!= 0 && value
!= 1) {
8129 ss
<< "value must be 0 or 1; got " << value
;
8133 newcrush
.set_straw_calc_version(value
);
8135 ss
<< "unrecognized tunable '" << tunable
<< "'";
8140 if (!validate_crush_against_features(&newcrush
, ss
)) {
8145 pending_inc
.crush
.clear();
8146 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
8147 ss
<< "adjusted tunable " << tunable
<< " to " << value
;
8149 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8150 get_last_committed() + 1));
8153 } else if (prefix
== "osd crush rule create-simple") {
8154 string name
, root
, type
, mode
;
8155 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
8156 cmd_getval(g_ceph_context
, cmdmap
, "root", root
);
8157 cmd_getval(g_ceph_context
, cmdmap
, "type", type
);
8158 cmd_getval(g_ceph_context
, cmdmap
, "mode", mode
);
8162 if (osdmap
.crush
->rule_exists(name
)) {
8163 // The name is uniquely associated to a ruleid and the rule it contains
8164 // From the user point of view, the rule is more meaningfull.
8165 ss
<< "rule " << name
<< " already exists";
8170 CrushWrapper newcrush
;
8171 _get_pending_crush(newcrush
);
8173 if (newcrush
.rule_exists(name
)) {
8174 // The name is uniquely associated to a ruleid and the rule it contains
8175 // From the user point of view, the rule is more meaningfull.
8176 ss
<< "rule " << name
<< " already exists";
8179 int ruleno
= newcrush
.add_simple_rule(name
, root
, type
, "", mode
,
8180 pg_pool_t::TYPE_REPLICATED
, &ss
);
8186 pending_inc
.crush
.clear();
8187 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
8190 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8191 get_last_committed() + 1));
8194 } else if (prefix
== "osd crush rule create-replicated") {
8195 string name
, root
, type
, device_class
;
8196 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
8197 cmd_getval(g_ceph_context
, cmdmap
, "root", root
);
8198 cmd_getval(g_ceph_context
, cmdmap
, "type", type
);
8199 cmd_getval(g_ceph_context
, cmdmap
, "class", device_class
);
8201 if (!device_class
.empty()) {
8202 if (osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
8203 ss
<< "you must complete the upgrade and 'ceph osd require-osd-release "
8204 << "luminous' before using crush device classes";
8210 if (osdmap
.crush
->rule_exists(name
)) {
8211 // The name is uniquely associated to a ruleid and the rule it contains
8212 // From the user point of view, the rule is more meaningfull.
8213 ss
<< "rule " << name
<< " already exists";
8218 CrushWrapper newcrush
;
8219 _get_pending_crush(newcrush
);
8221 if (newcrush
.rule_exists(name
)) {
8222 // The name is uniquely associated to a ruleid and the rule it contains
8223 // From the user point of view, the rule is more meaningfull.
8224 ss
<< "rule " << name
<< " already exists";
8227 int ruleno
= newcrush
.add_simple_rule(
8228 name
, root
, type
, device_class
,
8229 "firstn", pg_pool_t::TYPE_REPLICATED
, &ss
);
8235 pending_inc
.crush
.clear();
8236 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
8239 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8240 get_last_committed() + 1));
8243 } else if (prefix
== "osd erasure-code-profile rm") {
8245 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
8247 if (erasure_code_profile_in_use(pending_inc
.new_pools
, name
, &ss
))
8250 if (erasure_code_profile_in_use(osdmap
.pools
, name
, &ss
)) {
8255 if (osdmap
.has_erasure_code_profile(name
) ||
8256 pending_inc
.new_erasure_code_profiles
.count(name
)) {
8257 if (osdmap
.has_erasure_code_profile(name
)) {
8258 pending_inc
.old_erasure_code_profiles
.push_back(name
);
8260 dout(20) << "erasure code profile rm " << name
<< ": creation canceled" << dendl
;
8261 pending_inc
.new_erasure_code_profiles
.erase(name
);
8265 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8266 get_last_committed() + 1));
8269 ss
<< "erasure-code-profile " << name
<< " does not exist";
8274 } else if (prefix
== "osd erasure-code-profile set") {
8276 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
8277 vector
<string
> profile
;
8278 cmd_getval(g_ceph_context
, cmdmap
, "profile", profile
);
8280 if (profile
.size() > 0 && profile
.back() == "--force") {
8286 map
<string
,string
> profile_map
;
8287 err
= parse_erasure_code_profile(profile
, &profile_map
, &ss
);
8290 if (profile_map
.find("plugin") == profile_map
.end()) {
8291 ss
<< "erasure-code-profile " << profile_map
8292 << " must contain a plugin entry" << std::endl
;
8296 string plugin
= profile_map
["plugin"];
8298 if (pending_inc
.has_erasure_code_profile(name
)) {
8299 dout(20) << "erasure code profile " << name
<< " try again" << dendl
;
8302 if (plugin
== "isa" || plugin
== "lrc") {
8303 err
= check_cluster_features(CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2
, ss
);
8308 } else if (plugin
== "shec") {
8309 err
= check_cluster_features(CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3
, ss
);
8315 err
= normalize_profile(name
, profile_map
, force
, &ss
);
8319 if (osdmap
.has_erasure_code_profile(name
)) {
8320 ErasureCodeProfile existing_profile_map
=
8321 osdmap
.get_erasure_code_profile(name
);
8322 err
= normalize_profile(name
, existing_profile_map
, force
, &ss
);
8326 if (existing_profile_map
== profile_map
) {
8332 ss
<< "will not override erasure code profile " << name
8333 << " because the existing profile "
8334 << existing_profile_map
8335 << " is different from the proposed profile "
8341 dout(20) << "erasure code profile set " << name
<< "="
8342 << profile_map
<< dendl
;
8343 pending_inc
.set_erasure_code_profile(name
, profile_map
);
8347 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8348 get_last_committed() + 1));
8351 } else if (prefix
== "osd crush rule create-erasure") {
8352 err
= check_cluster_features(CEPH_FEATURE_CRUSH_V2
, ss
);
8357 string name
, poolstr
;
8358 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
8360 cmd_getval(g_ceph_context
, cmdmap
, "profile", profile
);
8362 profile
= "default";
8363 if (profile
== "default") {
8364 if (!osdmap
.has_erasure_code_profile(profile
)) {
8365 if (pending_inc
.has_erasure_code_profile(profile
)) {
8366 dout(20) << "erasure code profile " << profile
<< " already pending" << dendl
;
8370 map
<string
,string
> profile_map
;
8371 err
= osdmap
.get_erasure_code_profile_default(g_ceph_context
,
8376 err
= normalize_profile(name
, profile_map
, true, &ss
);
8379 dout(20) << "erasure code profile set " << profile
<< "="
8380 << profile_map
<< dendl
;
8381 pending_inc
.set_erasure_code_profile(profile
, profile_map
);
8387 err
= crush_rule_create_erasure(name
, profile
, &rule
, &ss
);
8390 case -EEXIST
: // return immediately
8391 ss
<< "rule " << name
<< " already exists";
8395 case -EALREADY
: // wait for pending to be proposed
8396 ss
<< "rule " << name
<< " already exists";
8399 default: // non recoverable error
8404 ss
<< "created rule " << name
<< " at " << rule
;
8408 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8409 get_last_committed() + 1));
8412 } else if (prefix
== "osd crush rule rm") {
8414 cmd_getval(g_ceph_context
, cmdmap
, "name", name
);
8416 if (!osdmap
.crush
->rule_exists(name
)) {
8417 ss
<< "rule " << name
<< " does not exist";
8422 CrushWrapper newcrush
;
8423 _get_pending_crush(newcrush
);
8425 if (!newcrush
.rule_exists(name
)) {
8426 ss
<< "rule " << name
<< " does not exist";
8429 int ruleno
= newcrush
.get_rule_id(name
);
8430 assert(ruleno
>= 0);
8432 // make sure it is not in use.
8433 // FIXME: this is ok in some situations, but let's not bother with that
8435 int ruleset
= newcrush
.get_rule_mask_ruleset(ruleno
);
8436 if (osdmap
.crush_ruleset_in_use(ruleset
)) {
8437 ss
<< "crush ruleset " << name
<< " " << ruleset
<< " is in use";
8442 err
= newcrush
.remove_rule(ruleno
);
8447 pending_inc
.crush
.clear();
8448 newcrush
.encode(pending_inc
.crush
, mon
->get_quorum_con_features());
8451 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8452 get_last_committed() + 1));
8455 } else if (prefix
== "osd setmaxosd") {
8457 if (!cmd_getval(g_ceph_context
, cmdmap
, "newmax", newmax
)) {
8458 ss
<< "unable to parse 'newmax' value '"
8459 << cmd_vartype_stringify(cmdmap
["newmax"]) << "'";
8464 if (newmax
> g_conf
->mon_max_osd
) {
8466 ss
<< "cannot set max_osd to " << newmax
<< " which is > conf.mon_max_osd ("
8467 << g_conf
->mon_max_osd
<< ")";
8471 // Don't allow shrinking OSD number as this will cause data loss
8472 // and may cause kernel crashes.
8473 // Note: setmaxosd sets the maximum OSD number and not the number of OSDs
8474 if (newmax
< osdmap
.get_max_osd()) {
8475 // Check if the OSDs exist between current max and new value.
8476 // If there are any OSDs exist, then don't allow shrinking number
8478 for (int i
= newmax
; i
< osdmap
.get_max_osd(); i
++) {
8479 if (osdmap
.exists(i
)) {
8481 ss
<< "cannot shrink max_osd to " << newmax
8482 << " because osd." << i
<< " (and possibly others) still in use";
8488 pending_inc
.new_max_osd
= newmax
;
8489 ss
<< "set new max_osd = " << pending_inc
.new_max_osd
;
8491 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8492 get_last_committed() + 1));
8495 } else if (prefix
== "osd set-full-ratio" ||
8496 prefix
== "osd set-backfillfull-ratio" ||
8497 prefix
== "osd set-nearfull-ratio") {
8498 if (osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
8499 ss
<< "you must complete the upgrade and 'ceph osd require-osd-release "
8500 << "luminous' before using the new interface";
8505 if (!cmd_getval(g_ceph_context
, cmdmap
, "ratio", n
)) {
8506 ss
<< "unable to parse 'ratio' value '"
8507 << cmd_vartype_stringify(cmdmap
["ratio"]) << "'";
8511 if (prefix
== "osd set-full-ratio")
8512 pending_inc
.new_full_ratio
= n
;
8513 else if (prefix
== "osd set-backfillfull-ratio")
8514 pending_inc
.new_backfillfull_ratio
= n
;
8515 else if (prefix
== "osd set-nearfull-ratio")
8516 pending_inc
.new_nearfull_ratio
= n
;
8517 ss
<< prefix
<< " " << n
;
8519 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8520 get_last_committed() + 1));
8522 } else if (prefix
== "osd set-require-min-compat-client") {
8523 if (osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
8524 ss
<< "you must complete the upgrade and 'ceph osd require-osd-release "
8525 << "luminous' before using the new interface";
8530 cmd_getval(g_ceph_context
, cmdmap
, "version", v
);
8531 int vno
= ceph_release_from_name(v
.c_str());
8533 ss
<< "version " << v
<< " is not recognized";
8538 newmap
.deepish_copy_from(osdmap
);
8539 newmap
.apply_incremental(pending_inc
);
8540 newmap
.require_min_compat_client
= vno
;
8541 auto mvno
= newmap
.get_min_compat_client();
8543 ss
<< "osdmap current utilizes features that require "
8544 << ceph_release_name(mvno
)
8545 << "; cannot set require_min_compat_client below that to "
8546 << ceph_release_name(vno
);
8551 cmd_getval(g_ceph_context
, cmdmap
, "sure", sure
);
8552 if (sure
!= "--yes-i-really-mean-it") {
8554 mon
->get_combined_feature_map(&m
);
8555 uint64_t features
= ceph_release_features(vno
);
8559 CEPH_ENTITY_TYPE_CLIENT
,
8560 CEPH_ENTITY_TYPE_MDS
,
8561 CEPH_ENTITY_TYPE_MGR
}) {
8562 auto p
= m
.m
.find(type
);
8563 if (p
== m
.m
.end()) {
8566 for (auto& q
: p
->second
) {
8567 uint64_t missing
= ~q
.first
& features
;
8570 ss
<< "cannot set require_min_compat_client to " << v
<< ": ";
8575 ss
<< q
.second
<< " connected " << ceph_entity_type_name(type
)
8576 << "(s) look like " << ceph_release_name(
8577 ceph_release_from_features(q
.first
))
8578 << " (missing 0x" << std::hex
<< missing
<< std::dec
<< ")";
8584 ss
<< "; add --yes-i-really-mean-it to do it anyway";
8589 ss
<< "set require_min_compat_client to " << ceph_release_name(vno
);
8590 pending_inc
.new_require_min_compat_client
= vno
;
8592 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
8593 get_last_committed() + 1));
8595 } else if (prefix
== "osd pause") {
8596 return prepare_set_flag(op
, CEPH_OSDMAP_PAUSERD
| CEPH_OSDMAP_PAUSEWR
);
8598 } else if (prefix
== "osd unpause") {
8599 return prepare_unset_flag(op
, CEPH_OSDMAP_PAUSERD
| CEPH_OSDMAP_PAUSEWR
);
8601 } else if (prefix
== "osd set") {
8603 cmd_getval(g_ceph_context
, cmdmap
, "key", key
);
8605 return prepare_set_flag(op
, CEPH_OSDMAP_FULL
);
8606 else if (key
== "pause")
8607 return prepare_set_flag(op
, CEPH_OSDMAP_PAUSERD
| CEPH_OSDMAP_PAUSEWR
);
8608 else if (key
== "noup")
8609 return prepare_set_flag(op
, CEPH_OSDMAP_NOUP
);
8610 else if (key
== "nodown")
8611 return prepare_set_flag(op
, CEPH_OSDMAP_NODOWN
);
8612 else if (key
== "noout")
8613 return prepare_set_flag(op
, CEPH_OSDMAP_NOOUT
);
8614 else if (key
== "noin")
8615 return prepare_set_flag(op
, CEPH_OSDMAP_NOIN
);
8616 else if (key
== "nobackfill")
8617 return prepare_set_flag(op
, CEPH_OSDMAP_NOBACKFILL
);
8618 else if (key
== "norebalance")
8619 return prepare_set_flag(op
, CEPH_OSDMAP_NOREBALANCE
);
8620 else if (key
== "norecover")
8621 return prepare_set_flag(op
, CEPH_OSDMAP_NORECOVER
);
8622 else if (key
== "noscrub")
8623 return prepare_set_flag(op
, CEPH_OSDMAP_NOSCRUB
);
8624 else if (key
== "nodeep-scrub")
8625 return prepare_set_flag(op
, CEPH_OSDMAP_NODEEP_SCRUB
);
8626 else if (key
== "notieragent")
8627 return prepare_set_flag(op
, CEPH_OSDMAP_NOTIERAGENT
);
8628 else if (key
== "sortbitwise") {
8629 if (osdmap
.get_up_osd_features() & CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT
) {
8630 return prepare_set_flag(op
, CEPH_OSDMAP_SORTBITWISE
);
8632 ss
<< "not all up OSDs have OSD_BITWISE_HOBJ_SORT feature";
8636 } else if (key
== "recovery_deletes") {
8637 if (HAVE_FEATURE(osdmap
.get_up_osd_features(), OSD_RECOVERY_DELETES
)) {
8638 return prepare_set_flag(op
, CEPH_OSDMAP_RECOVERY_DELETES
);
8640 ss
<< "not all up OSDs have OSD_RECOVERY_DELETES feature";
8644 } else if (key
== "require_jewel_osds") {
8645 if (!osdmap
.test_flag(CEPH_OSDMAP_SORTBITWISE
)) {
8646 ss
<< "the sortbitwise flag must be set before require_jewel_osds";
8649 } else if (osdmap
.require_osd_release
>= CEPH_RELEASE_JEWEL
) {
8650 ss
<< "require_osd_release is already >= jewel";
8653 } else if (HAVE_FEATURE(osdmap
.get_up_osd_features(), SERVER_JEWEL
)) {
8654 return prepare_set_flag(op
, CEPH_OSDMAP_REQUIRE_JEWEL
);
8656 ss
<< "not all up OSDs have CEPH_FEATURE_SERVER_JEWEL feature";
8659 } else if (key
== "require_kraken_osds") {
8660 if (!osdmap
.test_flag(CEPH_OSDMAP_SORTBITWISE
)) {
8661 ss
<< "the sortbitwise flag must be set before require_kraken_osds";
8664 } else if (osdmap
.require_osd_release
>= CEPH_RELEASE_KRAKEN
) {
8665 ss
<< "require_osd_release is already >= kraken";
8668 } else if (HAVE_FEATURE(osdmap
.get_up_osd_features(), SERVER_KRAKEN
)) {
8669 bool r
= prepare_set_flag(op
, CEPH_OSDMAP_REQUIRE_KRAKEN
);
8670 // ensure JEWEL is also set
8671 pending_inc
.new_flags
|= CEPH_OSDMAP_REQUIRE_JEWEL
;
8674 ss
<< "not all up OSDs have CEPH_FEATURE_SERVER_KRAKEN feature";
8678 ss
<< "unrecognized flag '" << key
<< "'";
8682 } else if (prefix
== "osd unset") {
8684 cmd_getval(g_ceph_context
, cmdmap
, "key", key
);
8686 return prepare_unset_flag(op
, CEPH_OSDMAP_FULL
);
8687 else if (key
== "pause")
8688 return prepare_unset_flag(op
, CEPH_OSDMAP_PAUSERD
| CEPH_OSDMAP_PAUSEWR
);
8689 else if (key
== "noup")
8690 return prepare_unset_flag(op
, CEPH_OSDMAP_NOUP
);
8691 else if (key
== "nodown")
8692 return prepare_unset_flag(op
, CEPH_OSDMAP_NODOWN
);
8693 else if (key
== "noout")
8694 return prepare_unset_flag(op
, CEPH_OSDMAP_NOOUT
);
8695 else if (key
== "noin")
8696 return prepare_unset_flag(op
, CEPH_OSDMAP_NOIN
);
8697 else if (key
== "nobackfill")
8698 return prepare_unset_flag(op
, CEPH_OSDMAP_NOBACKFILL
);
8699 else if (key
== "norebalance")
8700 return prepare_unset_flag(op
, CEPH_OSDMAP_NOREBALANCE
);
8701 else if (key
== "norecover")
8702 return prepare_unset_flag(op
, CEPH_OSDMAP_NORECOVER
);
8703 else if (key
== "noscrub")
8704 return prepare_unset_flag(op
, CEPH_OSDMAP_NOSCRUB
);
8705 else if (key
== "nodeep-scrub")
8706 return prepare_unset_flag(op
, CEPH_OSDMAP_NODEEP_SCRUB
);
8707 else if (key
== "notieragent")
8708 return prepare_unset_flag(op
, CEPH_OSDMAP_NOTIERAGENT
);
8710 ss
<< "unrecognized flag '" << key
<< "'";
8714 } else if (prefix
== "osd require-osd-release") {
8716 cmd_getval(g_ceph_context
, cmdmap
, "release", release
);
8717 if (!osdmap
.test_flag(CEPH_OSDMAP_SORTBITWISE
)) {
8718 ss
<< "the sortbitwise flag must be set first";
8722 int rel
= ceph_release_from_name(release
.c_str());
8724 ss
<< "unrecognized release " << release
;
8728 if (rel
< CEPH_RELEASE_LUMINOUS
) {
8729 ss
<< "use this command only for luminous and later";
8733 if (rel
== CEPH_RELEASE_LUMINOUS
) {
8734 if (!HAVE_FEATURE(osdmap
.get_up_osd_features(), SERVER_LUMINOUS
)) {
8735 ss
<< "not all up OSDs have CEPH_FEATURE_SERVER_LUMINOUS feature";
8740 ss
<< "not supported for this release yet";
8744 if (rel
< osdmap
.require_osd_release
) {
8745 ss
<< "require_osd_release cannot be lowered once it has been set";
8749 pending_inc
.new_require_osd_release
= rel
;
8750 if (rel
>= CEPH_RELEASE_LUMINOUS
&&
8751 !osdmap
.test_flag(CEPH_OSDMAP_RECOVERY_DELETES
)) {
8752 return prepare_set_flag(op
, CEPH_OSDMAP_RECOVERY_DELETES
);
8755 } else if (prefix
== "osd cluster_snap") {
8756 // ** DISABLE THIS FOR NOW **
8757 ss
<< "cluster snapshot currently disabled (broken implementation)";
8758 // ** DISABLE THIS FOR NOW **
8760 } else if (prefix
== "osd down" ||
8761 prefix
== "osd out" ||
8762 prefix
== "osd in" ||
8763 prefix
== "osd rm") {
8767 bool verbose
= true;
8769 vector
<string
> idvec
;
8770 cmd_getval(g_ceph_context
, cmdmap
, "ids", idvec
);
8771 for (unsigned j
= 0; j
< idvec
.size() && !stop
; j
++) {
8776 (idvec
[0] == "any" || idvec
[0] == "all" || idvec
[0] == "*")) {
8777 if (prefix
== "osd in") {
8778 // touch out osds only
8779 osdmap
.get_out_osds(osds
);
8781 osdmap
.get_all_osds(osds
);
8784 verbose
= false; // so the output is less noisy.
8786 long osd
= parse_osd_id(idvec
[j
].c_str(), &ss
);
8788 ss
<< "invalid osd id" << osd
;
8791 } else if (!osdmap
.exists(osd
)) {
8792 ss
<< "osd." << osd
<< " does not exist. ";
8799 for (auto &osd
: osds
) {
8800 if (prefix
== "osd down") {
8801 if (osdmap
.is_down(osd
)) {
8803 ss
<< "osd." << osd
<< " is already down. ";
8805 pending_inc
.pending_osd_state_set(osd
, CEPH_OSD_UP
);
8806 ss
<< "marked down osd." << osd
<< ". ";
8809 } else if (prefix
== "osd out") {
8810 if (osdmap
.is_out(osd
)) {
8812 ss
<< "osd." << osd
<< " is already out. ";
8814 pending_inc
.new_weight
[osd
] = CEPH_OSD_OUT
;
8815 if (osdmap
.osd_weight
[osd
]) {
8816 if (pending_inc
.new_xinfo
.count(osd
) == 0) {
8817 pending_inc
.new_xinfo
[osd
] = osdmap
.osd_xinfo
[osd
];
8819 pending_inc
.new_xinfo
[osd
].old_weight
= osdmap
.osd_weight
[osd
];
8821 ss
<< "marked out osd." << osd
<< ". ";
8822 std::ostringstream msg
;
8823 msg
<< "Client " << op
->get_session()->entity_name
8824 << " marked osd." << osd
<< " out";
8825 if (osdmap
.is_up(osd
)) {
8826 msg
<< ", while it was still marked up";
8828 msg
<< ", after it was down for " << int(down_pending_out
[osd
].sec())
8832 mon
->clog
->info() << msg
.str();
8835 } else if (prefix
== "osd in") {
8836 if (osdmap
.is_in(osd
)) {
8838 ss
<< "osd." << osd
<< " is already in. ";
8840 if (osdmap
.osd_xinfo
[osd
].old_weight
> 0) {
8841 pending_inc
.new_weight
[osd
] = osdmap
.osd_xinfo
[osd
].old_weight
;
8842 if (pending_inc
.new_xinfo
.count(osd
) == 0) {
8843 pending_inc
.new_xinfo
[osd
] = osdmap
.osd_xinfo
[osd
];
8845 pending_inc
.new_xinfo
[osd
].old_weight
= 0;
8847 pending_inc
.new_weight
[osd
] = CEPH_OSD_IN
;
8849 ss
<< "marked in osd." << osd
<< ". ";
8852 } else if (prefix
== "osd rm") {
8853 err
= prepare_command_osd_remove(osd
);
8855 if (err
== -EBUSY
) {
8858 ss
<< "osd." << osd
<< " is still up; must be down before removal. ";
8862 ss
<< ", osd." << osd
;
8864 ss
<< "removed osd." << osd
;
8873 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, err
, rs
,
8874 get_last_committed() + 1));
8877 } else if (prefix
== "osd add-noup" ||
8878 prefix
== "osd add-nodown" ||
8879 prefix
== "osd add-noin" ||
8880 prefix
== "osd add-noout") {
8889 if (prefix
== "osd add-noup") {
8891 } else if (prefix
== "osd add-nodown") {
8893 } else if (prefix
== "osd add-noin") {
8902 vector
<string
> idvec
;
8903 cmd_getval(g_ceph_context
, cmdmap
, "ids", idvec
);
8904 for (unsigned j
= 0; j
< idvec
.size() && !stop
; j
++) {
8910 (idvec
[0] == "any" || idvec
[0] == "all" || idvec
[0] == "*")) {
8911 osdmap
.get_all_osds(osds
);
8914 // try traditional single osd way
8916 long osd
= parse_osd_id(idvec
[j
].c_str(), &ss
);
8918 // ss has reason for failure
8919 ss
<< ", unable to parse osd id:\"" << idvec
[j
] << "\". ";
8927 for (auto &osd
: osds
) {
8929 if (!osdmap
.exists(osd
)) {
8930 ss
<< "osd." << osd
<< " does not exist. ";
8936 if (osdmap
.is_up(osd
)) {
8937 ss
<< "osd." << osd
<< " is already up. ";
8941 if (osdmap
.is_noup(osd
)) {
8942 if (pending_inc
.pending_osd_state_clear(osd
, CEPH_OSD_NOUP
))
8945 pending_inc
.pending_osd_state_set(osd
, CEPH_OSD_NOUP
);
8952 if (osdmap
.is_down(osd
)) {
8953 ss
<< "osd." << osd
<< " is already down. ";
8957 if (osdmap
.is_nodown(osd
)) {
8958 if (pending_inc
.pending_osd_state_clear(osd
, CEPH_OSD_NODOWN
))
8961 pending_inc
.pending_osd_state_set(osd
, CEPH_OSD_NODOWN
);
8968 if (osdmap
.is_in(osd
)) {
8969 ss
<< "osd." << osd
<< " is already in. ";
8973 if (osdmap
.is_noin(osd
)) {
8974 if (pending_inc
.pending_osd_state_clear(osd
, CEPH_OSD_NOIN
))
8977 pending_inc
.pending_osd_state_set(osd
, CEPH_OSD_NOIN
);
8984 if (osdmap
.is_out(osd
)) {
8985 ss
<< "osd." << osd
<< " is already out. ";
8989 if (osdmap
.is_noout(osd
)) {
8990 if (pending_inc
.pending_osd_state_clear(osd
, CEPH_OSD_NOOUT
))
8993 pending_inc
.pending_osd_state_set(osd
, CEPH_OSD_NOOUT
);
9000 assert(0 == "invalid option");
9007 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, err
, rs
,
9008 get_last_committed() + 1));
9011 } else if (prefix
== "osd rm-noup" ||
9012 prefix
== "osd rm-nodown" ||
9013 prefix
== "osd rm-noin" ||
9014 prefix
== "osd rm-noout") {
9023 if (prefix
== "osd rm-noup") {
9025 } else if (prefix
== "osd rm-nodown") {
9027 } else if (prefix
== "osd rm-noin") {
9036 vector
<string
> idvec
;
9037 cmd_getval(g_ceph_context
, cmdmap
, "ids", idvec
);
9039 for (unsigned j
= 0; j
< idvec
.size() && !stop
; j
++) {
9045 (idvec
[0] == "any" || idvec
[0] == "all" || idvec
[0] == "*")) {
9047 // touch previous noup/nodown/noin/noout osds only
9050 osdmap
.get_noup_osds(&osds
);
9053 osdmap
.get_nodown_osds(&osds
);
9056 osdmap
.get_noin_osds(&osds
);
9059 osdmap
.get_noout_osds(&osds
);
9062 assert(0 == "invalid option");
9065 // cancel any pending noup/nodown/noin/noout requests too
9066 vector
<int> pending_state_osds
;
9067 (void) pending_inc
.get_pending_state_osds(&pending_state_osds
);
9068 for (auto &p
: pending_state_osds
) {
9072 if (!osdmap
.is_noup(p
) &&
9073 pending_inc
.pending_osd_state_clear(p
, CEPH_OSD_NOUP
)) {
9079 if (!osdmap
.is_nodown(p
) &&
9080 pending_inc
.pending_osd_state_clear(p
, CEPH_OSD_NODOWN
)) {
9086 if (!osdmap
.is_noin(p
) &&
9087 pending_inc
.pending_osd_state_clear(p
, CEPH_OSD_NOIN
)) {
9093 if (!osdmap
.is_noout(p
) &&
9094 pending_inc
.pending_osd_state_clear(p
, CEPH_OSD_NOOUT
)) {
9100 assert(0 == "invalid option");
9106 // try traditional single osd way
9108 long osd
= parse_osd_id(idvec
[j
].c_str(), &ss
);
9110 // ss has reason for failure
9111 ss
<< ", unable to parse osd id:\"" << idvec
[j
] << "\". ";
9116 osds
.push_back(osd
);
9119 for (auto &osd
: osds
) {
9121 if (!osdmap
.exists(osd
)) {
9122 ss
<< "osd." << osd
<< " does not exist. ";
9128 if (osdmap
.is_noup(osd
)) {
9129 pending_inc
.pending_osd_state_set(osd
, CEPH_OSD_NOUP
);
9131 } else if (pending_inc
.pending_osd_state_clear(
9132 osd
, CEPH_OSD_NOUP
)) {
9138 if (osdmap
.is_nodown(osd
)) {
9139 pending_inc
.pending_osd_state_set(osd
, CEPH_OSD_NODOWN
);
9141 } else if (pending_inc
.pending_osd_state_clear(
9142 osd
, CEPH_OSD_NODOWN
)) {
9148 if (osdmap
.is_noin(osd
)) {
9149 pending_inc
.pending_osd_state_set(osd
, CEPH_OSD_NOIN
);
9151 } else if (pending_inc
.pending_osd_state_clear(
9152 osd
, CEPH_OSD_NOIN
)) {
9158 if (osdmap
.is_noout(osd
)) {
9159 pending_inc
.pending_osd_state_set(osd
, CEPH_OSD_NOOUT
);
9161 } else if (pending_inc
.pending_osd_state_clear(
9162 osd
, CEPH_OSD_NOOUT
)) {
9168 assert(0 == "invalid option");
9175 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, err
, rs
,
9176 get_last_committed() + 1));
9179 } else if (prefix
== "osd pg-temp") {
9181 if (!cmd_getval(g_ceph_context
, cmdmap
, "pgid", pgidstr
)) {
9182 ss
<< "unable to parse 'pgid' value '"
9183 << cmd_vartype_stringify(cmdmap
["pgid"]) << "'";
9188 if (!pgid
.parse(pgidstr
.c_str())) {
9189 ss
<< "invalid pgid '" << pgidstr
<< "'";
9193 if (!osdmap
.pg_exists(pgid
)) {
9194 ss
<< "pg " << pgid
<< " does not exist";
9198 if (pending_inc
.new_pg_temp
.count(pgid
)) {
9199 dout(10) << __func__
<< " waiting for pending update on " << pgid
<< dendl
;
9200 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
9204 vector
<int64_t> id_vec
;
9205 vector
<int32_t> new_pg_temp
;
9206 if (!cmd_getval(g_ceph_context
, cmdmap
, "id", id_vec
)) {
9207 ss
<< "unable to parse 'id' value(s) '"
9208 << cmd_vartype_stringify(cmdmap
["id"]) << "'";
9212 for (auto osd
: id_vec
) {
9213 if (!osdmap
.exists(osd
)) {
9214 ss
<< "osd." << osd
<< " does not exist";
9218 new_pg_temp
.push_back(osd
);
9221 int pool_min_size
= osdmap
.get_pg_pool_min_size(pgid
);
9222 if ((int)new_pg_temp
.size() < pool_min_size
) {
9223 ss
<< "num of osds (" << new_pg_temp
.size() <<") < pool min size ("
9224 << pool_min_size
<< ")";
9229 int pool_size
= osdmap
.get_pg_pool_size(pgid
);
9230 if ((int)new_pg_temp
.size() > pool_size
) {
9231 ss
<< "num of osds (" << new_pg_temp
.size() <<") > pool size ("
9232 << pool_size
<< ")";
9237 pending_inc
.new_pg_temp
[pgid
] = mempool::osdmap::vector
<int>(
9238 new_pg_temp
.begin(), new_pg_temp
.end());
9239 ss
<< "set " << pgid
<< " pg_temp mapping to " << new_pg_temp
;
9241 } else if (prefix
== "osd primary-temp") {
9243 if (!cmd_getval(g_ceph_context
, cmdmap
, "pgid", pgidstr
)) {
9244 ss
<< "unable to parse 'pgid' value '"
9245 << cmd_vartype_stringify(cmdmap
["pgid"]) << "'";
9250 if (!pgid
.parse(pgidstr
.c_str())) {
9251 ss
<< "invalid pgid '" << pgidstr
<< "'";
9255 if (!osdmap
.pg_exists(pgid
)) {
9256 ss
<< "pg " << pgid
<< " does not exist";
9262 if (!cmd_getval(g_ceph_context
, cmdmap
, "id", osd
)) {
9263 ss
<< "unable to parse 'id' value '"
9264 << cmd_vartype_stringify(cmdmap
["id"]) << "'";
9268 if (osd
!= -1 && !osdmap
.exists(osd
)) {
9269 ss
<< "osd." << osd
<< " does not exist";
9274 if (osdmap
.require_min_compat_client
> 0 &&
9275 osdmap
.require_min_compat_client
< CEPH_RELEASE_FIREFLY
) {
9276 ss
<< "require_min_compat_client "
9277 << ceph_release_name(osdmap
.require_min_compat_client
)
9278 << " < firefly, which is required for primary-temp";
9281 } else if (!g_conf
->mon_osd_allow_primary_temp
) {
9282 ss
<< "you must enable 'mon osd allow primary temp = true' on the mons before you can set primary_temp mappings. note that this is for developers only: older clients/OSDs will break and there is no feature bit infrastructure in place.";
9287 pending_inc
.new_primary_temp
[pgid
] = osd
;
9288 ss
<< "set " << pgid
<< " primary_temp mapping to " << osd
;
9290 } else if (prefix
== "osd pg-upmap" ||
9291 prefix
== "osd rm-pg-upmap" ||
9292 prefix
== "osd pg-upmap-items" ||
9293 prefix
== "osd rm-pg-upmap-items") {
9294 if (osdmap
.require_osd_release
< CEPH_RELEASE_LUMINOUS
) {
9295 ss
<< "you must complete the upgrade and 'ceph osd require-osd-release "
9296 << "luminous' before using the new interface";
9300 if (osdmap
.require_min_compat_client
< CEPH_RELEASE_LUMINOUS
) {
9301 ss
<< "min_compat_client "
9302 << ceph_release_name(osdmap
.require_min_compat_client
)
9303 << " < luminous, which is required for pg-upmap. "
9304 << "Try 'ceph osd set-require-min-compat-client luminous' "
9305 << "before using the new interface";
9309 err
= check_cluster_features(CEPH_FEATUREMASK_OSDMAP_PG_UPMAP
, ss
);
9315 if (!cmd_getval(g_ceph_context
, cmdmap
, "pgid", pgidstr
)) {
9316 ss
<< "unable to parse 'pgid' value '"
9317 << cmd_vartype_stringify(cmdmap
["pgid"]) << "'";
9322 if (!pgid
.parse(pgidstr
.c_str())) {
9323 ss
<< "invalid pgid '" << pgidstr
<< "'";
9327 if (!osdmap
.pg_exists(pgid
)) {
9328 ss
<< "pg " << pgid
<< " does not exist";
9337 OP_RM_PG_UPMAP_ITEMS
,
9340 if (prefix
== "osd pg-upmap") {
9341 option
= OP_PG_UPMAP
;
9342 } else if (prefix
== "osd rm-pg-upmap") {
9343 option
= OP_RM_PG_UPMAP
;
9344 } else if (prefix
== "osd pg-upmap-items") {
9345 option
= OP_PG_UPMAP_ITEMS
;
9347 option
= OP_RM_PG_UPMAP_ITEMS
;
9350 // check pending upmap changes
9352 case OP_PG_UPMAP
: // fall through
9353 case OP_RM_PG_UPMAP
:
9354 if (pending_inc
.new_pg_upmap
.count(pgid
) ||
9355 pending_inc
.old_pg_upmap
.count(pgid
)) {
9356 dout(10) << __func__
<< " waiting for pending update on "
9358 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
9363 case OP_PG_UPMAP_ITEMS
: // fall through
9364 case OP_RM_PG_UPMAP_ITEMS
:
9365 if (pending_inc
.new_pg_upmap_items
.count(pgid
) ||
9366 pending_inc
.old_pg_upmap_items
.count(pgid
)) {
9367 dout(10) << __func__
<< " waiting for pending update on "
9369 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
9375 assert(0 == "invalid option");
9381 vector
<int64_t> id_vec
;
9382 if (!cmd_getval(g_ceph_context
, cmdmap
, "id", id_vec
)) {
9383 ss
<< "unable to parse 'id' value(s) '"
9384 << cmd_vartype_stringify(cmdmap
["id"]) << "'";
9389 int pool_min_size
= osdmap
.get_pg_pool_min_size(pgid
);
9390 if ((int)id_vec
.size() < pool_min_size
) {
9391 ss
<< "num of osds (" << id_vec
.size() <<") < pool min size ("
9392 << pool_min_size
<< ")";
9397 int pool_size
= osdmap
.get_pg_pool_size(pgid
);
9398 if ((int)id_vec
.size() > pool_size
) {
9399 ss
<< "num of osds (" << id_vec
.size() <<") > pool size ("
9400 << pool_size
<< ")";
9405 vector
<int32_t> new_pg_upmap
;
9406 for (auto osd
: id_vec
) {
9407 if (osd
!= CRUSH_ITEM_NONE
&& !osdmap
.exists(osd
)) {
9408 ss
<< "osd." << osd
<< " does not exist";
9412 auto it
= std::find(new_pg_upmap
.begin(), new_pg_upmap
.end(), osd
);
9413 if (it
!= new_pg_upmap
.end()) {
9414 ss
<< "osd." << osd
<< " already exists, ";
9417 new_pg_upmap
.push_back(osd
);
9420 if (new_pg_upmap
.empty()) {
9421 ss
<< "no valid upmap items(pairs) is specified";
9426 pending_inc
.new_pg_upmap
[pgid
] = mempool::osdmap::vector
<int32_t>(
9427 new_pg_upmap
.begin(), new_pg_upmap
.end());
9428 ss
<< "set " << pgid
<< " pg_upmap mapping to " << new_pg_upmap
;
9432 case OP_RM_PG_UPMAP
:
9434 pending_inc
.old_pg_upmap
.insert(pgid
);
9435 ss
<< "clear " << pgid
<< " pg_upmap mapping";
9439 case OP_PG_UPMAP_ITEMS
:
9441 vector
<int64_t> id_vec
;
9442 if (!cmd_getval(g_ceph_context
, cmdmap
, "id", id_vec
)) {
9443 ss
<< "unable to parse 'id' value(s) '"
9444 << cmd_vartype_stringify(cmdmap
["id"]) << "'";
9449 if (id_vec
.size() % 2) {
9450 ss
<< "you must specify pairs of osd ids to be remapped";
9455 int pool_size
= osdmap
.get_pg_pool_size(pgid
);
9456 if ((int)(id_vec
.size() / 2) > pool_size
) {
9457 ss
<< "num of osd pairs (" << id_vec
.size() / 2 <<") > pool size ("
9458 << pool_size
<< ")";
9463 vector
<pair
<int32_t,int32_t>> new_pg_upmap_items
;
9464 ostringstream items
;
9466 for (auto p
= id_vec
.begin(); p
!= id_vec
.end(); ++p
) {
9470 ss
<< "from osd." << from
<< " == to osd." << to
<< ", ";
9473 if (!osdmap
.exists(from
)) {
9474 ss
<< "osd." << from
<< " does not exist";
9478 if (to
!= CRUSH_ITEM_NONE
&& !osdmap
.exists(to
)) {
9479 ss
<< "osd." << to
<< " does not exist";
9483 pair
<int32_t,int32_t> entry
= make_pair(from
, to
);
9484 auto it
= std::find(new_pg_upmap_items
.begin(),
9485 new_pg_upmap_items
.end(), entry
);
9486 if (it
!= new_pg_upmap_items
.end()) {
9487 ss
<< "osd." << from
<< " -> osd." << to
<< " already exists, ";
9490 new_pg_upmap_items
.push_back(entry
);
9491 items
<< from
<< "->" << to
<< ",";
9493 string
out(items
.str());
9494 out
.resize(out
.size() - 1); // drop last ','
9497 if (new_pg_upmap_items
.empty()) {
9498 ss
<< "no valid upmap items(pairs) is specified";
9503 pending_inc
.new_pg_upmap_items
[pgid
] =
9504 mempool::osdmap::vector
<pair
<int32_t,int32_t>>(
9505 new_pg_upmap_items
.begin(), new_pg_upmap_items
.end());
9506 ss
<< "set " << pgid
<< " pg_upmap_items mapping to " << out
;
9510 case OP_RM_PG_UPMAP_ITEMS
:
9512 pending_inc
.old_pg_upmap_items
.insert(pgid
);
9513 ss
<< "clear " << pgid
<< " pg_upmap_items mapping";
9518 assert(0 == "invalid option");
9522 } else if (prefix
== "osd primary-affinity") {
9524 if (!cmd_getval(g_ceph_context
, cmdmap
, "id", id
)) {
9525 ss
<< "invalid osd id value '"
9526 << cmd_vartype_stringify(cmdmap
["id"]) << "'";
9531 if (!cmd_getval(g_ceph_context
, cmdmap
, "weight", w
)) {
9532 ss
<< "unable to parse 'weight' value '"
9533 << cmd_vartype_stringify(cmdmap
["weight"]) << "'";
9537 long ww
= (int)((double)CEPH_OSD_MAX_PRIMARY_AFFINITY
*w
);
9539 ss
<< "weight must be >= 0";
9543 if (osdmap
.require_min_compat_client
> 0 &&
9544 osdmap
.require_min_compat_client
< CEPH_RELEASE_FIREFLY
) {
9545 ss
<< "require_min_compat_client "
9546 << ceph_release_name(osdmap
.require_min_compat_client
)
9547 << " < firefly, which is required for primary-affinity";
9550 } else if (!g_conf
->mon_osd_allow_primary_affinity
) {
9551 ss
<< "you must enable 'mon osd allow primary affinity = true' on the mons before you can adjust primary-affinity. note that older clients will no longer be able to communicate with the cluster.";
9555 err
= check_cluster_features(CEPH_FEATURE_OSD_PRIMARY_AFFINITY
, ss
);
9560 if (osdmap
.exists(id
)) {
9561 pending_inc
.new_primary_affinity
[id
] = ww
;
9562 ss
<< "set osd." << id
<< " primary-affinity to " << w
<< " (" << ios::hex
<< ww
<< ios::dec
<< ")";
9564 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
9565 get_last_committed() + 1));
9568 ss
<< "osd." << id
<< " does not exist";
9572 } else if (prefix
== "osd reweight") {
9574 if (!cmd_getval(g_ceph_context
, cmdmap
, "id", id
)) {
9575 ss
<< "unable to parse osd id value '"
9576 << cmd_vartype_stringify(cmdmap
["id"]) << "'";
9581 if (!cmd_getval(g_ceph_context
, cmdmap
, "weight", w
)) {
9582 ss
<< "unable to parse weight value '"
9583 << cmd_vartype_stringify(cmdmap
["weight"]) << "'";
9587 long ww
= (int)((double)CEPH_OSD_IN
*w
);
9589 ss
<< "weight must be >= 0";
9593 if (osdmap
.exists(id
)) {
9594 pending_inc
.new_weight
[id
] = ww
;
9595 ss
<< "reweighted osd." << id
<< " to " << w
<< " (" << std::hex
<< ww
<< std::dec
<< ")";
9597 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
9598 get_last_committed() + 1));
9601 ss
<< "osd." << id
<< " does not exist";
9605 } else if (prefix
== "osd reweightn") {
9606 map
<int32_t, uint32_t> weights
;
9607 err
= parse_reweights(g_ceph_context
, cmdmap
, osdmap
, &weights
);
9609 ss
<< "unable to parse 'weights' value '"
9610 << cmd_vartype_stringify(cmdmap
["weights"]) << "'";
9613 pending_inc
.new_weight
.insert(weights
.begin(), weights
.end());
9614 wait_for_finished_proposal(
9616 new Monitor::C_Command(mon
, op
, 0, rs
, rdata
, get_last_committed() + 1));
9618 } else if (prefix
== "osd lost") {
9620 if (!cmd_getval(g_ceph_context
, cmdmap
, "id", id
)) {
9621 ss
<< "unable to parse osd id value '"
9622 << cmd_vartype_stringify(cmdmap
["id"]) << "'";
9627 if (!cmd_getval(g_ceph_context
, cmdmap
, "sure", sure
) || sure
!= "--yes-i-really-mean-it") {
9628 ss
<< "are you SURE? this might mean real, permanent data loss. pass "
9629 "--yes-i-really-mean-it if you really do.";
9632 } else if (!osdmap
.exists(id
)) {
9633 ss
<< "osd." << id
<< " does not exist";
9636 } else if (!osdmap
.is_down(id
)) {
9637 ss
<< "osd." << id
<< " is not down";
9641 epoch_t e
= osdmap
.get_info(id
).down_at
;
9642 pending_inc
.new_lost
[id
] = e
;
9643 ss
<< "marked osd lost in epoch " << e
;
9645 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
9646 get_last_committed() + 1));
9650 } else if (prefix
== "osd destroy" || prefix
== "osd purge") {
9651 /* Destroying an OSD means that we don't expect to further make use of
9652 * the OSDs data (which may even become unreadable after this operation),
9653 * and that we are okay with scrubbing all its cephx keys and config-key
9654 * data (which may include lockbox keys, thus rendering the osd's data
9657 * The OSD will not be removed. Instead, we will mark it as destroyed,
9658 * such that a subsequent call to `create` will not reuse the osd id.
9659 * This will play into being able to recreate the OSD, at the same
9660 * crush location, with minimal data movement.
9663 // make sure authmon is writeable.
9664 if (!mon
->authmon()->is_writeable()) {
9665 dout(10) << __func__
<< " waiting for auth mon to be writeable for "
9666 << "osd destroy" << dendl
;
9667 mon
->authmon()->wait_for_writeable(op
, new C_RetryMessage(this, op
));
9672 if (!cmd_getval(g_ceph_context
, cmdmap
, "id", id
)) {
9673 ss
<< "unable to parse osd id value '"
9674 << cmd_vartype_stringify(cmdmap
["id"]) << "";
9679 bool is_destroy
= (prefix
== "osd destroy");
9681 assert("osd purge" == prefix
);
9685 if (!cmd_getval(g_ceph_context
, cmdmap
, "sure", sure
) ||
9686 sure
!= "--yes-i-really-mean-it") {
9687 ss
<< "Are you SURE? This will mean real, permanent data loss, as well "
9688 << "as cephx and lockbox keys. Pass --yes-i-really-mean-it if you "
9692 } else if (is_destroy
&& !osdmap
.exists(id
)) {
9693 ss
<< "osd." << id
<< " does not exist";
9696 } else if (osdmap
.is_up(id
)) {
9697 ss
<< "osd." << id
<< " is not `down`.";
9700 } else if (is_destroy
&& osdmap
.is_destroyed(id
)) {
9701 ss
<< "destroyed osd." << id
;
9706 bool goto_reply
= false;
9710 err
= prepare_command_osd_destroy(id
, ss
);
9711 // we checked above that it should exist.
9712 assert(err
!= -ENOENT
);
9714 err
= prepare_command_osd_purge(id
, ss
);
9715 if (err
== -ENOENT
) {
9717 ss
<< "osd." << id
<< " does not exist.";
9723 if (err
< 0 || goto_reply
) {
9728 ss
<< "destroyed osd." << id
;
9730 ss
<< "purged osd." << id
;
9734 wait_for_finished_proposal(op
,
9735 new Monitor::C_Command(mon
, op
, 0, rs
, get_last_committed() + 1));
9736 force_immediate_propose();
9739 } else if (prefix
== "osd new") {
9741 // make sure authmon is writeable.
9742 if (!mon
->authmon()->is_writeable()) {
9743 dout(10) << __func__
<< " waiting for auth mon to be writeable for "
9744 << "osd new" << dendl
;
9745 mon
->authmon()->wait_for_writeable(op
, new C_RetryMessage(this, op
));
9749 map
<string
,string
> secrets_map
;
9751 bufferlist bl
= m
->get_data();
9752 string secrets_json
= bl
.to_str();
9753 dout(20) << __func__
<< " osd new json = " << secrets_json
<< dendl
;
9755 err
= get_json_str_map(secrets_json
, ss
, &secrets_map
);
9759 dout(20) << __func__
<< " osd new secrets " << secrets_map
<< dendl
;
9762 err
= prepare_command_osd_new(op
, cmdmap
, secrets_map
, ss
, f
.get());
9775 if (err
== EEXIST
) {
9776 // idempotent operation
9781 wait_for_finished_proposal(op
,
9782 new Monitor::C_Command(mon
, op
, 0, rs
, rdata
,
9783 get_last_committed() + 1));
9784 force_immediate_propose();
9787 } else if (prefix
== "osd create") {
9789 // optional id provided?
9790 int64_t id
= -1, cmd_id
= -1;
9791 if (cmd_getval(g_ceph_context
, cmdmap
, "id", cmd_id
)) {
9793 ss
<< "invalid osd id value '" << cmd_id
<< "'";
9797 dout(10) << " osd create got id " << cmd_id
<< dendl
;
9802 if (cmd_getval(g_ceph_context
, cmdmap
, "uuid", uuidstr
)) {
9803 if (!uuid
.parse(uuidstr
.c_str())) {
9804 ss
<< "invalid uuid value '" << uuidstr
<< "'";
9808 // we only care about the id if we also have the uuid, to
9809 // ensure the operation's idempotency.
9813 int32_t new_id
= -1;
9814 err
= prepare_command_osd_create(id
, uuid
, &new_id
, ss
);
9816 if (err
== -EAGAIN
) {
9817 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
9820 // a check has failed; reply to the user.
9823 } else if (err
== EEXIST
) {
9824 // this is an idempotent operation; we can go ahead and reply.
9826 f
->open_object_section("created_osd");
9827 f
->dump_int("osdid", new_id
);
9838 do_osd_create(id
, uuid
, &new_id
);
9841 f
->open_object_section("created_osd");
9842 f
->dump_int("osdid", new_id
);
9849 wait_for_finished_proposal(op
,
9850 new Monitor::C_Command(mon
, op
, 0, rs
, rdata
,
9851 get_last_committed() + 1));
9854 } else if (prefix
== "osd blacklist clear") {
9855 pending_inc
.new_blacklist
.clear();
9856 std::list
<std::pair
<entity_addr_t
,utime_t
> > blacklist
;
9857 osdmap
.get_blacklist(&blacklist
);
9858 for (const auto &entry
: blacklist
) {
9859 pending_inc
.old_blacklist
.push_back(entry
.first
);
9861 ss
<< " removed all blacklist entries";
9863 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
9864 get_last_committed() + 1));
9866 } else if (prefix
== "osd blacklist") {
9868 cmd_getval(g_ceph_context
, cmdmap
, "addr", addrstr
);
9870 if (!addr
.parse(addrstr
.c_str(), 0)) {
9871 ss
<< "unable to parse address " << addrstr
;
9877 cmd_getval(g_ceph_context
, cmdmap
, "blacklistop", blacklistop
);
9878 if (blacklistop
== "add") {
9879 utime_t expires
= ceph_clock_now();
9882 cmd_getval(g_ceph_context
, cmdmap
, "expire", d
,
9883 g_conf
->mon_osd_blacklist_default_expire
);
9886 pending_inc
.new_blacklist
[addr
] = expires
;
9889 // cancel any pending un-blacklisting request too
9890 auto it
= std::find(pending_inc
.old_blacklist
.begin(),
9891 pending_inc
.old_blacklist
.end(), addr
);
9892 if (it
!= pending_inc
.old_blacklist
.end()) {
9893 pending_inc
.old_blacklist
.erase(it
);
9897 ss
<< "blacklisting " << addr
<< " until " << expires
<< " (" << d
<< " sec)";
9899 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
9900 get_last_committed() + 1));
9902 } else if (blacklistop
== "rm") {
9903 if (osdmap
.is_blacklisted(addr
) ||
9904 pending_inc
.new_blacklist
.count(addr
)) {
9905 if (osdmap
.is_blacklisted(addr
))
9906 pending_inc
.old_blacklist
.push_back(addr
);
9908 pending_inc
.new_blacklist
.erase(addr
);
9909 ss
<< "un-blacklisting " << addr
;
9911 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
9912 get_last_committed() + 1));
9915 ss
<< addr
<< " isn't blacklisted";
9920 } else if (prefix
== "osd pool mksnap") {
9922 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
9923 int64_t pool
= osdmap
.lookup_pg_pool_name(poolstr
.c_str());
9925 ss
<< "unrecognized pool '" << poolstr
<< "'";
9930 cmd_getval(g_ceph_context
, cmdmap
, "snap", snapname
);
9931 const pg_pool_t
*p
= osdmap
.get_pg_pool(pool
);
9932 if (p
->is_unmanaged_snaps_mode()) {
9933 ss
<< "pool " << poolstr
<< " is in unmanaged snaps mode";
9936 } else if (p
->snap_exists(snapname
.c_str())) {
9937 ss
<< "pool " << poolstr
<< " snap " << snapname
<< " already exists";
9940 } else if (p
->is_tier()) {
9941 ss
<< "pool " << poolstr
<< " is a cache tier";
9946 if (pending_inc
.new_pools
.count(pool
))
9947 pp
= &pending_inc
.new_pools
[pool
];
9949 pp
= &pending_inc
.new_pools
[pool
];
9952 if (pp
->snap_exists(snapname
.c_str())) {
9953 ss
<< "pool " << poolstr
<< " snap " << snapname
<< " already exists";
9955 pp
->add_snap(snapname
.c_str(), ceph_clock_now());
9956 pp
->set_snap_epoch(pending_inc
.epoch
);
9957 ss
<< "created pool " << poolstr
<< " snap " << snapname
;
9960 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
9961 get_last_committed() + 1));
9963 } else if (prefix
== "osd pool rmsnap") {
9965 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
9966 int64_t pool
= osdmap
.lookup_pg_pool_name(poolstr
.c_str());
9968 ss
<< "unrecognized pool '" << poolstr
<< "'";
9973 cmd_getval(g_ceph_context
, cmdmap
, "snap", snapname
);
9974 const pg_pool_t
*p
= osdmap
.get_pg_pool(pool
);
9975 if (p
->is_unmanaged_snaps_mode()) {
9976 ss
<< "pool " << poolstr
<< " is in unmanaged snaps mode";
9979 } else if (!p
->snap_exists(snapname
.c_str())) {
9980 ss
<< "pool " << poolstr
<< " snap " << snapname
<< " does not exist";
9985 if (pending_inc
.new_pools
.count(pool
))
9986 pp
= &pending_inc
.new_pools
[pool
];
9988 pp
= &pending_inc
.new_pools
[pool
];
9991 snapid_t sn
= pp
->snap_exists(snapname
.c_str());
9993 pp
->remove_snap(sn
);
9994 pp
->set_snap_epoch(pending_inc
.epoch
);
9995 ss
<< "removed pool " << poolstr
<< " snap " << snapname
;
9997 ss
<< "already removed pool " << poolstr
<< " snap " << snapname
;
10000 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
10001 get_last_committed() + 1));
10003 } else if (prefix
== "osd pool create") {
10006 cmd_getval(g_ceph_context
, cmdmap
, "pg_num", pg_num
, int64_t(0));
10007 cmd_getval(g_ceph_context
, cmdmap
, "pgp_num", pgp_num
, pg_num
);
10009 string pool_type_str
;
10010 cmd_getval(g_ceph_context
, cmdmap
, "pool_type", pool_type_str
);
10011 if (pool_type_str
.empty())
10012 pool_type_str
= g_conf
->osd_pool_default_type
;
10015 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
10016 int64_t pool_id
= osdmap
.lookup_pg_pool_name(poolstr
);
10017 if (pool_id
>= 0) {
10018 const pg_pool_t
*p
= osdmap
.get_pg_pool(pool_id
);
10019 if (pool_type_str
!= p
->get_type_name()) {
10020 ss
<< "pool '" << poolstr
<< "' cannot change to type " << pool_type_str
;
10023 ss
<< "pool '" << poolstr
<< "' already exists";
10030 if (pool_type_str
== "replicated") {
10031 pool_type
= pg_pool_t::TYPE_REPLICATED
;
10032 } else if (pool_type_str
== "erasure") {
10033 err
= check_cluster_features(CEPH_FEATURE_CRUSH_V2
|
10034 CEPH_FEATURE_OSD_ERASURE_CODES
,
10036 if (err
== -EAGAIN
)
10040 pool_type
= pg_pool_t::TYPE_ERASURE
;
10042 ss
<< "unknown pool type '" << pool_type_str
<< "'";
10047 bool implicit_rule_creation
= false;
10049 cmd_getval(g_ceph_context
, cmdmap
, "rule", rule_name
);
10050 string erasure_code_profile
;
10051 cmd_getval(g_ceph_context
, cmdmap
, "erasure_code_profile", erasure_code_profile
);
10053 if (pool_type
== pg_pool_t::TYPE_ERASURE
) {
10054 if (erasure_code_profile
== "")
10055 erasure_code_profile
= "default";
10056 //handle the erasure code profile
10057 if (erasure_code_profile
== "default") {
10058 if (!osdmap
.has_erasure_code_profile(erasure_code_profile
)) {
10059 if (pending_inc
.has_erasure_code_profile(erasure_code_profile
)) {
10060 dout(20) << "erasure code profile " << erasure_code_profile
<< " already pending" << dendl
;
10064 map
<string
,string
> profile_map
;
10065 err
= osdmap
.get_erasure_code_profile_default(g_ceph_context
,
10070 dout(20) << "erasure code profile " << erasure_code_profile
<< " set" << dendl
;
10071 pending_inc
.set_erasure_code_profile(erasure_code_profile
, profile_map
);
10075 if (rule_name
== "") {
10076 implicit_rule_creation
= true;
10077 if (erasure_code_profile
== "default") {
10078 rule_name
= "erasure-code";
10080 dout(1) << "implicitly use rule named after the pool: "
10081 << poolstr
<< dendl
;
10082 rule_name
= poolstr
;
10086 //NOTE:for replicated pool,cmd_map will put rule_name to erasure_code_profile field
10087 rule_name
= erasure_code_profile
;
10090 if (!implicit_rule_creation
&& rule_name
!= "") {
10092 err
= get_crush_rule(rule_name
, &rule
, &ss
);
10093 if (err
== -EAGAIN
) {
10094 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
10101 int64_t expected_num_objects
;
10102 cmd_getval(g_ceph_context
, cmdmap
, "expected_num_objects", expected_num_objects
, int64_t(0));
10103 if (expected_num_objects
< 0) {
10104 ss
<< "'expected_num_objects' must be non-negative";
10109 int64_t fast_read_param
;
10110 cmd_getval(g_ceph_context
, cmdmap
, "fast_read", fast_read_param
, int64_t(-1));
10111 FastReadType fast_read
= FAST_READ_DEFAULT
;
10112 if (fast_read_param
== 0)
10113 fast_read
= FAST_READ_OFF
;
10114 else if (fast_read_param
> 0)
10115 fast_read
= FAST_READ_ON
;
10117 err
= prepare_new_pool(poolstr
, 0, // auid=0 for admin created pool
10118 -1, // default crush rule
10121 erasure_code_profile
, pool_type
,
10122 (uint64_t)expected_num_objects
,
10128 ss
<< "pool '" << poolstr
<< "' already exists";
10131 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
10140 ss
<< "pool '" << poolstr
<< "' created";
10143 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
10144 get_last_committed() + 1));
10147 } else if (prefix
== "osd pool delete" ||
10148 prefix
== "osd pool rm") {
10149 // osd pool delete/rm <poolname> <poolname again> --yes-i-really-really-mean-it
10150 string poolstr
, poolstr2
, sure
;
10151 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
10152 cmd_getval(g_ceph_context
, cmdmap
, "pool2", poolstr2
);
10153 cmd_getval(g_ceph_context
, cmdmap
, "sure", sure
);
10154 int64_t pool
= osdmap
.lookup_pg_pool_name(poolstr
.c_str());
10156 ss
<< "pool '" << poolstr
<< "' does not exist";
10161 bool force_no_fake
= sure
== "--yes-i-really-really-mean-it-not-faking";
10162 if (poolstr2
!= poolstr
||
10163 (sure
!= "--yes-i-really-really-mean-it" && !force_no_fake
)) {
10164 ss
<< "WARNING: this will *PERMANENTLY DESTROY* all data stored in pool " << poolstr
10165 << ". If you are *ABSOLUTELY CERTAIN* that is what you want, pass the pool name *twice*, "
10166 << "followed by --yes-i-really-really-mean-it.";
10170 err
= _prepare_remove_pool(pool
, &ss
, force_no_fake
);
10171 if (err
== -EAGAIN
) {
10172 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
10178 } else if (prefix
== "osd pool rename") {
10179 string srcpoolstr
, destpoolstr
;
10180 cmd_getval(g_ceph_context
, cmdmap
, "srcpool", srcpoolstr
);
10181 cmd_getval(g_ceph_context
, cmdmap
, "destpool", destpoolstr
);
10182 int64_t pool_src
= osdmap
.lookup_pg_pool_name(srcpoolstr
.c_str());
10183 int64_t pool_dst
= osdmap
.lookup_pg_pool_name(destpoolstr
.c_str());
10185 if (pool_src
< 0) {
10186 if (pool_dst
>= 0) {
10187 // src pool doesn't exist, dst pool does exist: to ensure idempotency
10188 // of operations, assume this rename succeeded, as it is not changing
10189 // the current state. Make sure we output something understandable
10190 // for whoever is issuing the command, if they are paying attention,
10191 // in case it was not intentional; or to avoid a "wtf?" and a bug
10192 // report in case it was intentional, while expecting a failure.
10193 ss
<< "pool '" << srcpoolstr
<< "' does not exist; pool '"
10194 << destpoolstr
<< "' does -- assuming successful rename";
10197 ss
<< "unrecognized pool '" << srcpoolstr
<< "'";
10201 } else if (pool_dst
>= 0) {
10202 // source pool exists and so does the destination pool
10203 ss
<< "pool '" << destpoolstr
<< "' already exists";
10208 int ret
= _prepare_rename_pool(pool_src
, destpoolstr
);
10210 ss
<< "pool '" << srcpoolstr
<< "' renamed to '" << destpoolstr
<< "'";
10212 ss
<< "failed to rename pool '" << srcpoolstr
<< "' to '" << destpoolstr
<< "': "
10213 << cpp_strerror(ret
);
10216 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, ret
, rs
,
10217 get_last_committed() + 1));
10220 } else if (prefix
== "osd pool set") {
10221 err
= prepare_command_pool_set(cmdmap
, ss
);
10222 if (err
== -EAGAIN
)
10228 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
10229 get_last_committed() + 1));
10231 } else if (prefix
== "osd tier add") {
10232 err
= check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL
, ss
);
10233 if (err
== -EAGAIN
)
10238 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
10239 int64_t pool_id
= osdmap
.lookup_pg_pool_name(poolstr
);
10241 ss
<< "unrecognized pool '" << poolstr
<< "'";
10245 string tierpoolstr
;
10246 cmd_getval(g_ceph_context
, cmdmap
, "tierpool", tierpoolstr
);
10247 int64_t tierpool_id
= osdmap
.lookup_pg_pool_name(tierpoolstr
);
10248 if (tierpool_id
< 0) {
10249 ss
<< "unrecognized pool '" << tierpoolstr
<< "'";
10253 const pg_pool_t
*p
= osdmap
.get_pg_pool(pool_id
);
10255 const pg_pool_t
*tp
= osdmap
.get_pg_pool(tierpool_id
);
10258 if (!_check_become_tier(tierpool_id
, tp
, pool_id
, p
, &err
, &ss
)) {
10262 // make sure new tier is empty
10263 string force_nonempty
;
10264 cmd_getval(g_ceph_context
, cmdmap
, "force_nonempty", force_nonempty
);
10265 const pool_stat_t
*pstats
= mon
->pgservice
->get_pool_stat(tierpool_id
);
10266 if (pstats
&& pstats
->stats
.sum
.num_objects
!= 0 &&
10267 force_nonempty
!= "--force-nonempty") {
10268 ss
<< "tier pool '" << tierpoolstr
<< "' is not empty; --force-nonempty to force";
10272 if (tp
->ec_pool()) {
10273 ss
<< "tier pool '" << tierpoolstr
10274 << "' is an ec pool, which cannot be a tier";
10278 if ((!tp
->removed_snaps
.empty() || !tp
->snaps
.empty()) &&
10279 ((force_nonempty
!= "--force-nonempty") ||
10280 (!g_conf
->mon_debug_unsafe_allow_tier_with_nonempty_snaps
))) {
10281 ss
<< "tier pool '" << tierpoolstr
<< "' has snapshot state; it cannot be added as a tier without breaking the pool";
10286 pg_pool_t
*np
= pending_inc
.get_new_pool(pool_id
, p
);
10287 pg_pool_t
*ntp
= pending_inc
.get_new_pool(tierpool_id
, tp
);
10288 if (np
->tiers
.count(tierpool_id
) || ntp
->is_tier()) {
10289 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
10292 np
->tiers
.insert(tierpool_id
);
10293 np
->set_snap_epoch(pending_inc
.epoch
); // tier will update to our snap info
10294 ntp
->tier_of
= pool_id
;
10295 ss
<< "pool '" << tierpoolstr
<< "' is now (or already was) a tier of '" << poolstr
<< "'";
10296 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, ss
.str(),
10297 get_last_committed() + 1));
10299 } else if (prefix
== "osd tier remove" ||
10300 prefix
== "osd tier rm") {
10302 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
10303 int64_t pool_id
= osdmap
.lookup_pg_pool_name(poolstr
);
10305 ss
<< "unrecognized pool '" << poolstr
<< "'";
10309 string tierpoolstr
;
10310 cmd_getval(g_ceph_context
, cmdmap
, "tierpool", tierpoolstr
);
10311 int64_t tierpool_id
= osdmap
.lookup_pg_pool_name(tierpoolstr
);
10312 if (tierpool_id
< 0) {
10313 ss
<< "unrecognized pool '" << tierpoolstr
<< "'";
10317 const pg_pool_t
*p
= osdmap
.get_pg_pool(pool_id
);
10319 const pg_pool_t
*tp
= osdmap
.get_pg_pool(tierpool_id
);
10322 if (!_check_remove_tier(pool_id
, p
, tp
, &err
, &ss
)) {
10326 if (p
->tiers
.count(tierpool_id
) == 0) {
10327 ss
<< "pool '" << tierpoolstr
<< "' is now (or already was) not a tier of '" << poolstr
<< "'";
10331 if (tp
->tier_of
!= pool_id
) {
10332 ss
<< "tier pool '" << tierpoolstr
<< "' is a tier of '"
10333 << osdmap
.get_pool_name(tp
->tier_of
) << "': "
10334 // be scary about it; this is an inconsistency and bells must go off
10335 << "THIS SHOULD NOT HAVE HAPPENED AT ALL";
10339 if (p
->read_tier
== tierpool_id
) {
10340 ss
<< "tier pool '" << tierpoolstr
<< "' is the overlay for '" << poolstr
<< "'; please remove-overlay first";
10345 pg_pool_t
*np
= pending_inc
.get_new_pool(pool_id
, p
);
10346 pg_pool_t
*ntp
= pending_inc
.get_new_pool(tierpool_id
, tp
);
10347 if (np
->tiers
.count(tierpool_id
) == 0 ||
10348 ntp
->tier_of
!= pool_id
||
10349 np
->read_tier
== tierpool_id
) {
10350 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
10353 np
->tiers
.erase(tierpool_id
);
10355 ss
<< "pool '" << tierpoolstr
<< "' is now (or already was) not a tier of '" << poolstr
<< "'";
10356 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, ss
.str(),
10357 get_last_committed() + 1));
10359 } else if (prefix
== "osd tier set-overlay") {
10360 err
= check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL
, ss
);
10361 if (err
== -EAGAIN
)
10366 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
10367 int64_t pool_id
= osdmap
.lookup_pg_pool_name(poolstr
);
10369 ss
<< "unrecognized pool '" << poolstr
<< "'";
10373 string overlaypoolstr
;
10374 cmd_getval(g_ceph_context
, cmdmap
, "overlaypool", overlaypoolstr
);
10375 int64_t overlaypool_id
= osdmap
.lookup_pg_pool_name(overlaypoolstr
);
10376 if (overlaypool_id
< 0) {
10377 ss
<< "unrecognized pool '" << overlaypoolstr
<< "'";
10381 const pg_pool_t
*p
= osdmap
.get_pg_pool(pool_id
);
10383 const pg_pool_t
*overlay_p
= osdmap
.get_pg_pool(overlaypool_id
);
10385 if (p
->tiers
.count(overlaypool_id
) == 0) {
10386 ss
<< "tier pool '" << overlaypoolstr
<< "' is not a tier of '" << poolstr
<< "'";
10390 if (p
->read_tier
== overlaypool_id
) {
10392 ss
<< "overlay for '" << poolstr
<< "' is now (or already was) '" << overlaypoolstr
<< "'";
10395 if (p
->has_read_tier()) {
10396 ss
<< "pool '" << poolstr
<< "' has overlay '"
10397 << osdmap
.get_pool_name(p
->read_tier
)
10398 << "'; please remove-overlay first";
10404 pg_pool_t
*np
= pending_inc
.get_new_pool(pool_id
, p
);
10405 np
->read_tier
= overlaypool_id
;
10406 np
->write_tier
= overlaypool_id
;
10407 np
->set_last_force_op_resend(pending_inc
.epoch
);
10408 pg_pool_t
*noverlay_p
= pending_inc
.get_new_pool(overlaypool_id
, overlay_p
);
10409 noverlay_p
->set_last_force_op_resend(pending_inc
.epoch
);
10410 ss
<< "overlay for '" << poolstr
<< "' is now (or already was) '" << overlaypoolstr
<< "'";
10411 if (overlay_p
->cache_mode
== pg_pool_t::CACHEMODE_NONE
)
10412 ss
<<" (WARNING: overlay pool cache_mode is still NONE)";
10413 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, ss
.str(),
10414 get_last_committed() + 1));
10416 } else if (prefix
== "osd tier remove-overlay" ||
10417 prefix
== "osd tier rm-overlay") {
10419 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
10420 int64_t pool_id
= osdmap
.lookup_pg_pool_name(poolstr
);
10422 ss
<< "unrecognized pool '" << poolstr
<< "'";
10426 const pg_pool_t
*p
= osdmap
.get_pg_pool(pool_id
);
10428 if (!p
->has_read_tier()) {
10430 ss
<< "there is now (or already was) no overlay for '" << poolstr
<< "'";
10434 if (!_check_remove_tier(pool_id
, p
, NULL
, &err
, &ss
)) {
10439 pg_pool_t
*np
= pending_inc
.get_new_pool(pool_id
, p
);
10440 if (np
->has_read_tier()) {
10441 const pg_pool_t
*op
= osdmap
.get_pg_pool(np
->read_tier
);
10442 pg_pool_t
*nop
= pending_inc
.get_new_pool(np
->read_tier
,op
);
10443 nop
->set_last_force_op_resend(pending_inc
.epoch
);
10445 if (np
->has_write_tier()) {
10446 const pg_pool_t
*op
= osdmap
.get_pg_pool(np
->write_tier
);
10447 pg_pool_t
*nop
= pending_inc
.get_new_pool(np
->write_tier
, op
);
10448 nop
->set_last_force_op_resend(pending_inc
.epoch
);
10450 np
->clear_read_tier();
10451 np
->clear_write_tier();
10452 np
->set_last_force_op_resend(pending_inc
.epoch
);
10453 ss
<< "there is now (or already was) no overlay for '" << poolstr
<< "'";
10454 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, ss
.str(),
10455 get_last_committed() + 1));
10457 } else if (prefix
== "osd tier cache-mode") {
10458 err
= check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL
, ss
);
10459 if (err
== -EAGAIN
)
10464 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
10465 int64_t pool_id
= osdmap
.lookup_pg_pool_name(poolstr
);
10467 ss
<< "unrecognized pool '" << poolstr
<< "'";
10471 const pg_pool_t
*p
= osdmap
.get_pg_pool(pool_id
);
10473 if (!p
->is_tier()) {
10474 ss
<< "pool '" << poolstr
<< "' is not a tier";
10479 cmd_getval(g_ceph_context
, cmdmap
, "mode", modestr
);
10480 pg_pool_t::cache_mode_t mode
= pg_pool_t::get_cache_mode_from_str(modestr
);
10482 ss
<< "'" << modestr
<< "' is not a valid cache mode";
10488 cmd_getval(g_ceph_context
, cmdmap
, "sure", sure
);
10489 if ((mode
!= pg_pool_t::CACHEMODE_WRITEBACK
&&
10490 mode
!= pg_pool_t::CACHEMODE_NONE
&&
10491 mode
!= pg_pool_t::CACHEMODE_PROXY
&&
10492 mode
!= pg_pool_t::CACHEMODE_READPROXY
) &&
10493 sure
!= "--yes-i-really-mean-it") {
10494 ss
<< "'" << modestr
<< "' is not a well-supported cache mode and may "
10495 << "corrupt your data. pass --yes-i-really-mean-it to force.";
10500 // pool already has this cache-mode set and there are no pending changes
10501 if (p
->cache_mode
== mode
&&
10502 (pending_inc
.new_pools
.count(pool_id
) == 0 ||
10503 pending_inc
.new_pools
[pool_id
].cache_mode
== p
->cache_mode
)) {
10504 ss
<< "set cache-mode for pool '" << poolstr
<< "'"
10505 << " to " << pg_pool_t::get_cache_mode_name(mode
);
10510 /* Mode description:
10512 * none: No cache-mode defined
10513 * forward: Forward all reads and writes to base pool
10514 * writeback: Cache writes, promote reads from base pool
10515 * readonly: Forward writes to base pool
10516 * readforward: Writes are in writeback mode, Reads are in forward mode
10517 * proxy: Proxy all reads and writes to base pool
10518 * readproxy: Writes are in writeback mode, Reads are in proxy mode
10520 * Hence, these are the allowed transitions:
10523 * forward -> proxy || readforward || readproxy || writeback || any IF num_objects_dirty == 0
10524 * proxy -> forward || readforward || readproxy || writeback || any IF num_objects_dirty == 0
10525 * readforward -> forward || proxy || readproxy || writeback || any IF num_objects_dirty == 0
10526 * readproxy -> forward || proxy || readforward || writeback || any IF num_objects_dirty == 0
10527 * writeback -> readforward || readproxy || forward || proxy
10531 // We check if the transition is valid against the current pool mode, as
10532 // it is the only committed state thus far. We will blantly squash
10533 // whatever mode is on the pending state.
10535 if (p
->cache_mode
== pg_pool_t::CACHEMODE_WRITEBACK
&&
10536 (mode
!= pg_pool_t::CACHEMODE_FORWARD
&&
10537 mode
!= pg_pool_t::CACHEMODE_PROXY
&&
10538 mode
!= pg_pool_t::CACHEMODE_READFORWARD
&&
10539 mode
!= pg_pool_t::CACHEMODE_READPROXY
)) {
10540 ss
<< "unable to set cache-mode '" << pg_pool_t::get_cache_mode_name(mode
)
10541 << "' on a '" << pg_pool_t::get_cache_mode_name(p
->cache_mode
)
10542 << "' pool; only '"
10543 << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_FORWARD
)
10545 << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_PROXY
)
10547 << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_READFORWARD
)
10549 << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_READPROXY
)
10554 if ((p
->cache_mode
== pg_pool_t::CACHEMODE_READFORWARD
&&
10555 (mode
!= pg_pool_t::CACHEMODE_WRITEBACK
&&
10556 mode
!= pg_pool_t::CACHEMODE_FORWARD
&&
10557 mode
!= pg_pool_t::CACHEMODE_PROXY
&&
10558 mode
!= pg_pool_t::CACHEMODE_READPROXY
)) ||
10560 (p
->cache_mode
== pg_pool_t::CACHEMODE_READPROXY
&&
10561 (mode
!= pg_pool_t::CACHEMODE_WRITEBACK
&&
10562 mode
!= pg_pool_t::CACHEMODE_FORWARD
&&
10563 mode
!= pg_pool_t::CACHEMODE_READFORWARD
&&
10564 mode
!= pg_pool_t::CACHEMODE_PROXY
)) ||
10566 (p
->cache_mode
== pg_pool_t::CACHEMODE_PROXY
&&
10567 (mode
!= pg_pool_t::CACHEMODE_WRITEBACK
&&
10568 mode
!= pg_pool_t::CACHEMODE_FORWARD
&&
10569 mode
!= pg_pool_t::CACHEMODE_READFORWARD
&&
10570 mode
!= pg_pool_t::CACHEMODE_READPROXY
)) ||
10572 (p
->cache_mode
== pg_pool_t::CACHEMODE_FORWARD
&&
10573 (mode
!= pg_pool_t::CACHEMODE_WRITEBACK
&&
10574 mode
!= pg_pool_t::CACHEMODE_READFORWARD
&&
10575 mode
!= pg_pool_t::CACHEMODE_PROXY
&&
10576 mode
!= pg_pool_t::CACHEMODE_READPROXY
))) {
10578 const pool_stat_t
* pstats
=
10579 mon
->pgservice
->get_pool_stat(pool_id
);
10581 if (pstats
&& pstats
->stats
.sum
.num_objects_dirty
> 0) {
10582 ss
<< "unable to set cache-mode '"
10583 << pg_pool_t::get_cache_mode_name(mode
) << "' on pool '" << poolstr
10584 << "': dirty objects found";
10590 pg_pool_t
*np
= pending_inc
.get_new_pool(pool_id
, p
);
10591 np
->cache_mode
= mode
;
10592 // set this both when moving to and from cache_mode NONE. this is to
10593 // capture legacy pools that were set up before this flag existed.
10594 np
->flags
|= pg_pool_t::FLAG_INCOMPLETE_CLONES
;
10595 ss
<< "set cache-mode for pool '" << poolstr
10596 << "' to " << pg_pool_t::get_cache_mode_name(mode
);
10597 if (mode
== pg_pool_t::CACHEMODE_NONE
) {
10598 const pg_pool_t
*base_pool
= osdmap
.get_pg_pool(np
->tier_of
);
10600 if (base_pool
->read_tier
== pool_id
||
10601 base_pool
->write_tier
== pool_id
)
10602 ss
<<" (WARNING: pool is still configured as read or write tier)";
10604 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, ss
.str(),
10605 get_last_committed() + 1));
10607 } else if (prefix
== "osd tier add-cache") {
10608 err
= check_cluster_features(CEPH_FEATURE_OSD_CACHEPOOL
, ss
);
10609 if (err
== -EAGAIN
)
10614 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
10615 int64_t pool_id
= osdmap
.lookup_pg_pool_name(poolstr
);
10617 ss
<< "unrecognized pool '" << poolstr
<< "'";
10621 string tierpoolstr
;
10622 cmd_getval(g_ceph_context
, cmdmap
, "tierpool", tierpoolstr
);
10623 int64_t tierpool_id
= osdmap
.lookup_pg_pool_name(tierpoolstr
);
10624 if (tierpool_id
< 0) {
10625 ss
<< "unrecognized pool '" << tierpoolstr
<< "'";
10629 const pg_pool_t
*p
= osdmap
.get_pg_pool(pool_id
);
10631 const pg_pool_t
*tp
= osdmap
.get_pg_pool(tierpool_id
);
10634 if (!_check_become_tier(tierpool_id
, tp
, pool_id
, p
, &err
, &ss
)) {
10639 if (!cmd_getval(g_ceph_context
, cmdmap
, "size", size
)) {
10640 ss
<< "unable to parse 'size' value '"
10641 << cmd_vartype_stringify(cmdmap
["size"]) << "'";
10645 // make sure new tier is empty
10646 const pool_stat_t
*pstats
=
10647 mon
->pgservice
->get_pool_stat(tierpool_id
);
10648 if (pstats
&& pstats
->stats
.sum
.num_objects
!= 0) {
10649 ss
<< "tier pool '" << tierpoolstr
<< "' is not empty";
10653 string modestr
= g_conf
->osd_tier_default_cache_mode
;
10654 pg_pool_t::cache_mode_t mode
= pg_pool_t::get_cache_mode_from_str(modestr
);
10656 ss
<< "osd tier cache default mode '" << modestr
<< "' is not a valid cache mode";
10660 HitSet::Params hsp
;
10661 if (g_conf
->osd_tier_default_cache_hit_set_type
== "bloom") {
10662 BloomHitSet::Params
*bsp
= new BloomHitSet::Params
;
10663 bsp
->set_fpp(g_conf
->osd_pool_default_hit_set_bloom_fpp
);
10664 hsp
= HitSet::Params(bsp
);
10665 } else if (g_conf
->osd_tier_default_cache_hit_set_type
== "explicit_hash") {
10666 hsp
= HitSet::Params(new ExplicitHashHitSet::Params
);
10668 else if (g_conf
->osd_tier_default_cache_hit_set_type
== "explicit_object") {
10669 hsp
= HitSet::Params(new ExplicitObjectHitSet::Params
);
10671 ss
<< "osd tier cache default hit set type '" <<
10672 g_conf
->osd_tier_default_cache_hit_set_type
<< "' is not a known type";
10677 pg_pool_t
*np
= pending_inc
.get_new_pool(pool_id
, p
);
10678 pg_pool_t
*ntp
= pending_inc
.get_new_pool(tierpool_id
, tp
);
10679 if (np
->tiers
.count(tierpool_id
) || ntp
->is_tier()) {
10680 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
10683 np
->tiers
.insert(tierpool_id
);
10684 np
->read_tier
= np
->write_tier
= tierpool_id
;
10685 np
->set_snap_epoch(pending_inc
.epoch
); // tier will update to our snap info
10686 np
->set_last_force_op_resend(pending_inc
.epoch
);
10687 ntp
->set_last_force_op_resend(pending_inc
.epoch
);
10688 ntp
->tier_of
= pool_id
;
10689 ntp
->cache_mode
= mode
;
10690 ntp
->hit_set_count
= g_conf
->osd_tier_default_cache_hit_set_count
;
10691 ntp
->hit_set_period
= g_conf
->osd_tier_default_cache_hit_set_period
;
10692 ntp
->min_read_recency_for_promote
= g_conf
->osd_tier_default_cache_min_read_recency_for_promote
;
10693 ntp
->min_write_recency_for_promote
= g_conf
->osd_tier_default_cache_min_write_recency_for_promote
;
10694 ntp
->hit_set_grade_decay_rate
= g_conf
->osd_tier_default_cache_hit_set_grade_decay_rate
;
10695 ntp
->hit_set_search_last_n
= g_conf
->osd_tier_default_cache_hit_set_search_last_n
;
10696 ntp
->hit_set_params
= hsp
;
10697 ntp
->target_max_bytes
= size
;
10698 ss
<< "pool '" << tierpoolstr
<< "' is now (or already was) a cache tier of '" << poolstr
<< "'";
10699 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, ss
.str(),
10700 get_last_committed() + 1));
10702 } else if (prefix
== "osd pool set-quota") {
10704 cmd_getval(g_ceph_context
, cmdmap
, "pool", poolstr
);
10705 int64_t pool_id
= osdmap
.lookup_pg_pool_name(poolstr
);
10707 ss
<< "unrecognized pool '" << poolstr
<< "'";
10713 cmd_getval(g_ceph_context
, cmdmap
, "field", field
);
10714 if (field
!= "max_objects" && field
!= "max_bytes") {
10715 ss
<< "unrecognized field '" << field
<< "'; should be 'max_bytes' or 'max_objects'";
10720 // val could contain unit designations, so we treat as a string
10722 cmd_getval(g_ceph_context
, cmdmap
, "val", val
);
10724 int64_t value
= unit_to_bytesize(val
, &tss
);
10726 ss
<< "error parsing value '" << value
<< "': " << tss
.str();
10731 pg_pool_t
*pi
= pending_inc
.get_new_pool(pool_id
, osdmap
.get_pg_pool(pool_id
));
10732 if (field
== "max_objects") {
10733 pi
->quota_max_objects
= value
;
10734 } else if (field
== "max_bytes") {
10735 pi
->quota_max_bytes
= value
;
10737 assert(0 == "unrecognized option");
10739 ss
<< "set-quota " << field
<< " = " << value
<< " for pool " << poolstr
;
10741 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
10742 get_last_committed() + 1));
10744 } else if (prefix
== "osd pool application enable" ||
10745 prefix
== "osd pool application disable" ||
10746 prefix
== "osd pool application set" ||
10747 prefix
== "osd pool application rm") {
10748 err
= prepare_command_pool_application(prefix
, cmdmap
, ss
);
10749 if (err
== -EAGAIN
)
10755 wait_for_finished_proposal(
10756 op
, new Monitor::C_Command(mon
, op
, 0, rs
, get_last_committed() + 1));
10758 } else if (prefix
== "osd reweight-by-pg" ||
10759 prefix
== "osd reweight-by-utilization" ||
10760 prefix
== "osd test-reweight-by-pg" ||
10761 prefix
== "osd test-reweight-by-utilization") {
10763 prefix
== "osd reweight-by-pg" || prefix
== "osd test-reweight-by-pg";
10765 prefix
== "osd test-reweight-by-pg" ||
10766 prefix
== "osd test-reweight-by-utilization";
10768 cmd_getval(g_ceph_context
, cmdmap
, "oload", oload
, int64_t(120));
10769 set
<int64_t> pools
;
10770 vector
<string
> poolnamevec
;
10771 cmd_getval(g_ceph_context
, cmdmap
, "pools", poolnamevec
);
10772 for (unsigned j
= 0; j
< poolnamevec
.size(); j
++) {
10773 int64_t pool
= osdmap
.lookup_pg_pool_name(poolnamevec
[j
]);
10775 ss
<< "pool '" << poolnamevec
[j
] << "' does not exist";
10779 pools
.insert(pool
);
10781 double max_change
= g_conf
->mon_reweight_max_change
;
10782 cmd_getval(g_ceph_context
, cmdmap
, "max_change", max_change
);
10783 if (max_change
<= 0.0) {
10784 ss
<< "max_change " << max_change
<< " must be positive";
10788 int64_t max_osds
= g_conf
->mon_reweight_max_osds
;
10789 cmd_getval(g_ceph_context
, cmdmap
, "max_osds", max_osds
);
10790 if (max_osds
<= 0) {
10791 ss
<< "max_osds " << max_osds
<< " must be positive";
10795 string no_increasing
;
10796 cmd_getval(g_ceph_context
, cmdmap
, "no_increasing", no_increasing
);
10798 mempool::osdmap::map
<int32_t, uint32_t> new_weights
;
10799 err
= mon
->pgservice
->reweight_by_utilization(osdmap
,
10804 pools
.empty() ? NULL
: &pools
,
10805 no_increasing
== "--no-increasing",
10807 &ss
, &out_str
, f
.get());
10809 dout(10) << "reweight::by_utilization: finished with " << out_str
<< dendl
;
10814 rdata
.append(out_str
);
10816 ss
<< "FAILED reweight-by-pg";
10817 } else if (err
== 0 || dry_run
) {
10820 ss
<< "SUCCESSFUL reweight-by-pg";
10821 pending_inc
.new_weight
= std::move(new_weights
);
10822 wait_for_finished_proposal(
10824 new Monitor::C_Command(mon
, op
, 0, rs
, rdata
, get_last_committed() + 1));
10827 } else if (prefix
== "osd force-create-pg") {
10830 cmd_getval(g_ceph_context
, cmdmap
, "pgid", pgidstr
);
10831 if (!pgid
.parse(pgidstr
.c_str())) {
10832 ss
<< "invalid pgid '" << pgidstr
<< "'";
10838 std::lock_guard
<std::mutex
> l(creating_pgs_lock
);
10839 auto emplaced
= creating_pgs
.pgs
.emplace(pgid
,
10840 make_pair(osdmap
.get_epoch(),
10841 ceph_clock_now()));
10842 creating_now
= emplaced
.second
;
10844 if (creating_now
) {
10845 ss
<< "pg " << pgidstr
<< " now creating, ok";
10849 ss
<< "pg " << pgid
<< " already creating";
10859 if (err
< 0 && rs
.length() == 0)
10860 rs
= cpp_strerror(err
);
10861 mon
->reply_command(op
, err
, rs
, rdata
, get_last_committed());
10866 wait_for_finished_proposal(op
, new Monitor::C_Command(mon
, op
, 0, rs
,
10867 get_last_committed() + 1));
10871 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
10875 bool OSDMonitor::preprocess_pool_op(MonOpRequestRef op
)
10877 op
->mark_osdmon_event(__func__
);
10878 MPoolOp
*m
= static_cast<MPoolOp
*>(op
->get_req());
10880 if (m
->fsid
!= mon
->monmap
->fsid
) {
10881 dout(0) << __func__
<< " drop message on fsid " << m
->fsid
10882 << " != " << mon
->monmap
->fsid
<< " for " << *m
<< dendl
;
10883 _pool_op_reply(op
, -EINVAL
, osdmap
.get_epoch());
10887 if (m
->op
== POOL_OP_CREATE
)
10888 return preprocess_pool_op_create(op
);
10890 if (!osdmap
.get_pg_pool(m
->pool
)) {
10891 dout(10) << "attempt to operate on non-existent pool id " << m
->pool
<< dendl
;
10892 _pool_op_reply(op
, 0, osdmap
.get_epoch());
10896 // check if the snap and snapname exist
10897 bool snap_exists
= false;
10898 const pg_pool_t
*p
= osdmap
.get_pg_pool(m
->pool
);
10899 if (p
->snap_exists(m
->name
.c_str()))
10900 snap_exists
= true;
10903 case POOL_OP_CREATE_SNAP
:
10904 if (p
->is_unmanaged_snaps_mode() || p
->is_tier()) {
10905 _pool_op_reply(op
, -EINVAL
, osdmap
.get_epoch());
10909 _pool_op_reply(op
, 0, osdmap
.get_epoch());
10913 case POOL_OP_CREATE_UNMANAGED_SNAP
:
10914 if (p
->is_pool_snaps_mode()) {
10915 _pool_op_reply(op
, -EINVAL
, osdmap
.get_epoch());
10919 case POOL_OP_DELETE_SNAP
:
10920 if (p
->is_unmanaged_snaps_mode()) {
10921 _pool_op_reply(op
, -EINVAL
, osdmap
.get_epoch());
10924 if (!snap_exists
) {
10925 _pool_op_reply(op
, 0, osdmap
.get_epoch());
10929 case POOL_OP_DELETE_UNMANAGED_SNAP
:
10930 if (p
->is_pool_snaps_mode()) {
10931 _pool_op_reply(op
, -EINVAL
, osdmap
.get_epoch());
10934 if (p
->is_removed_snap(m
->snapid
)) {
10935 _pool_op_reply(op
, 0, osdmap
.get_epoch());
10939 case POOL_OP_DELETE
:
10940 if (osdmap
.lookup_pg_pool_name(m
->name
.c_str()) >= 0) {
10941 _pool_op_reply(op
, 0, osdmap
.get_epoch());
10945 case POOL_OP_AUID_CHANGE
:
10955 bool OSDMonitor::preprocess_pool_op_create(MonOpRequestRef op
)
10957 op
->mark_osdmon_event(__func__
);
10958 MPoolOp
*m
= static_cast<MPoolOp
*>(op
->get_req());
10959 MonSession
*session
= m
->get_session();
10961 _pool_op_reply(op
, -EPERM
, osdmap
.get_epoch());
10964 if (!session
->is_capable("osd", MON_CAP_W
)) {
10965 dout(5) << "attempt to create new pool without sufficient auid privileges!"
10966 << "message: " << *m
<< std::endl
10967 << "caps: " << session
->caps
<< dendl
;
10968 _pool_op_reply(op
, -EPERM
, osdmap
.get_epoch());
10972 int64_t pool
= osdmap
.lookup_pg_pool_name(m
->name
.c_str());
10974 _pool_op_reply(op
, 0, osdmap
.get_epoch());
10981 bool OSDMonitor::prepare_pool_op(MonOpRequestRef op
)
10983 op
->mark_osdmon_event(__func__
);
10984 MPoolOp
*m
= static_cast<MPoolOp
*>(op
->get_req());
10985 dout(10) << "prepare_pool_op " << *m
<< dendl
;
10986 if (m
->op
== POOL_OP_CREATE
) {
10987 return prepare_pool_op_create(op
);
10988 } else if (m
->op
== POOL_OP_DELETE
) {
10989 return prepare_pool_op_delete(op
);
10993 bool changed
= false;
10995 if (!osdmap
.have_pg_pool(m
->pool
)) {
10996 _pool_op_reply(op
, -ENOENT
, osdmap
.get_epoch());
11000 const pg_pool_t
*pool
= osdmap
.get_pg_pool(m
->pool
);
11003 case POOL_OP_CREATE_SNAP
:
11004 if (pool
->is_tier()) {
11006 _pool_op_reply(op
, ret
, osdmap
.get_epoch());
11008 } // else, fall through
11009 case POOL_OP_DELETE_SNAP
:
11010 if (!pool
->is_unmanaged_snaps_mode()) {
11011 bool snap_exists
= pool
->snap_exists(m
->name
.c_str());
11012 if ((m
->op
== POOL_OP_CREATE_SNAP
&& snap_exists
)
11013 || (m
->op
== POOL_OP_DELETE_SNAP
&& !snap_exists
)) {
11021 _pool_op_reply(op
, ret
, osdmap
.get_epoch());
11024 case POOL_OP_DELETE_UNMANAGED_SNAP
:
11025 // we won't allow removal of an unmanaged snapshot from a pool
11026 // not in unmanaged snaps mode.
11027 if (!pool
->is_unmanaged_snaps_mode()) {
11028 _pool_op_reply(op
, -ENOTSUP
, osdmap
.get_epoch());
11032 case POOL_OP_CREATE_UNMANAGED_SNAP
:
11033 // but we will allow creating an unmanaged snapshot on any pool
11034 // as long as it is not in 'pool' snaps mode.
11035 if (pool
->is_pool_snaps_mode()) {
11036 _pool_op_reply(op
, -EINVAL
, osdmap
.get_epoch());
11041 // projected pool info
11043 if (pending_inc
.new_pools
.count(m
->pool
))
11044 pp
= pending_inc
.new_pools
[m
->pool
];
11046 pp
= *osdmap
.get_pg_pool(m
->pool
);
11048 bufferlist reply_data
;
11050 // pool snaps vs unmanaged snaps are mutually exclusive
11052 case POOL_OP_CREATE_SNAP
:
11053 case POOL_OP_DELETE_SNAP
:
11054 if (pp
.is_unmanaged_snaps_mode()) {
11060 case POOL_OP_CREATE_UNMANAGED_SNAP
:
11061 case POOL_OP_DELETE_UNMANAGED_SNAP
:
11062 if (pp
.is_pool_snaps_mode()) {
11069 case POOL_OP_CREATE_SNAP
:
11070 if (!pp
.snap_exists(m
->name
.c_str())) {
11071 pp
.add_snap(m
->name
.c_str(), ceph_clock_now());
11072 dout(10) << "create snap in pool " << m
->pool
<< " " << m
->name
<< " seq " << pp
.get_snap_epoch() << dendl
;
11077 case POOL_OP_DELETE_SNAP
:
11079 snapid_t s
= pp
.snap_exists(m
->name
.c_str());
11087 case POOL_OP_CREATE_UNMANAGED_SNAP
:
11090 pp
.add_unmanaged_snap(snapid
);
11091 ::encode(snapid
, reply_data
);
11096 case POOL_OP_DELETE_UNMANAGED_SNAP
:
11097 if (!pp
.is_removed_snap(m
->snapid
)) {
11098 pp
.remove_unmanaged_snap(m
->snapid
);
11103 case POOL_OP_AUID_CHANGE
:
11104 if (pp
.auid
!= m
->auid
) {
11116 pp
.set_snap_epoch(pending_inc
.epoch
);
11117 pending_inc
.new_pools
[m
->pool
] = pp
;
11121 wait_for_finished_proposal(op
, new OSDMonitor::C_PoolOp(this, op
, ret
, pending_inc
.epoch
, &reply_data
));
11125 bool OSDMonitor::prepare_pool_op_create(MonOpRequestRef op
)
11127 op
->mark_osdmon_event(__func__
);
11128 int err
= prepare_new_pool(op
);
11129 wait_for_finished_proposal(op
, new OSDMonitor::C_PoolOp(this, op
, err
, pending_inc
.epoch
));
11133 int OSDMonitor::_check_remove_pool(int64_t pool_id
, const pg_pool_t
& pool
,
11136 const string
& poolstr
= osdmap
.get_pool_name(pool_id
);
11138 // If the Pool is in use by CephFS, refuse to delete it
11139 FSMap
const &pending_fsmap
= mon
->mdsmon()->get_pending();
11140 if (pending_fsmap
.pool_in_use(pool_id
)) {
11141 *ss
<< "pool '" << poolstr
<< "' is in use by CephFS";
11145 if (pool
.tier_of
>= 0) {
11146 *ss
<< "pool '" << poolstr
<< "' is a tier of '"
11147 << osdmap
.get_pool_name(pool
.tier_of
) << "'";
11150 if (!pool
.tiers
.empty()) {
11151 *ss
<< "pool '" << poolstr
<< "' has tiers";
11152 for(auto tier
: pool
.tiers
) {
11153 *ss
<< " " << osdmap
.get_pool_name(tier
);
11158 if (!g_conf
->mon_allow_pool_delete
) {
11159 *ss
<< "pool deletion is disabled; you must first set the mon_allow_pool_delete config option to true before you can destroy a pool";
11163 if (pool
.has_flag(pg_pool_t::FLAG_NODELETE
)) {
11164 *ss
<< "pool deletion is disabled; you must unset nodelete flag for the pool first";
11168 *ss
<< "pool '" << poolstr
<< "' removed";
11173 * Check if it is safe to add a tier to a base pool
11176 * True if the operation should proceed, false if we should abort here
11177 * (abort doesn't necessarily mean error, could be idempotency)
11179 bool OSDMonitor::_check_become_tier(
11180 const int64_t tier_pool_id
, const pg_pool_t
*tier_pool
,
11181 const int64_t base_pool_id
, const pg_pool_t
*base_pool
,
11185 const std::string
&tier_pool_name
= osdmap
.get_pool_name(tier_pool_id
);
11186 const std::string
&base_pool_name
= osdmap
.get_pool_name(base_pool_id
);
11188 const FSMap
&pending_fsmap
= mon
->mdsmon()->get_pending();
11189 if (pending_fsmap
.pool_in_use(tier_pool_id
)) {
11190 *ss
<< "pool '" << tier_pool_name
<< "' is in use by CephFS";
11195 if (base_pool
->tiers
.count(tier_pool_id
)) {
11196 assert(tier_pool
->tier_of
== base_pool_id
);
11198 *ss
<< "pool '" << tier_pool_name
<< "' is now (or already was) a tier of '"
11199 << base_pool_name
<< "'";
11203 if (base_pool
->is_tier()) {
11204 *ss
<< "pool '" << base_pool_name
<< "' is already a tier of '"
11205 << osdmap
.get_pool_name(base_pool
->tier_of
) << "', "
11206 << "multiple tiers are not yet supported.";
11211 if (tier_pool
->has_tiers()) {
11212 *ss
<< "pool '" << tier_pool_name
<< "' has following tier(s) already:";
11213 for (set
<uint64_t>::iterator it
= tier_pool
->tiers
.begin();
11214 it
!= tier_pool
->tiers
.end(); ++it
)
11215 *ss
<< "'" << osdmap
.get_pool_name(*it
) << "',";
11216 *ss
<< " multiple tiers are not yet supported.";
11221 if (tier_pool
->is_tier()) {
11222 *ss
<< "tier pool '" << tier_pool_name
<< "' is already a tier of '"
11223 << osdmap
.get_pool_name(tier_pool
->tier_of
) << "'";
11234 * Check if it is safe to remove a tier from this base pool
11237 * True if the operation should proceed, false if we should abort here
11238 * (abort doesn't necessarily mean error, could be idempotency)
11240 bool OSDMonitor::_check_remove_tier(
11241 const int64_t base_pool_id
, const pg_pool_t
*base_pool
,
11242 const pg_pool_t
*tier_pool
,
11243 int *err
, ostream
*ss
) const
11245 const std::string
&base_pool_name
= osdmap
.get_pool_name(base_pool_id
);
11247 // Apply CephFS-specific checks
11248 const FSMap
&pending_fsmap
= mon
->mdsmon()->get_pending();
11249 if (pending_fsmap
.pool_in_use(base_pool_id
)) {
11250 if (base_pool
->type
!= pg_pool_t::TYPE_REPLICATED
) {
11251 // If the underlying pool is erasure coded, we can't permit the
11252 // removal of the replicated tier that CephFS relies on to access it
11253 *ss
<< "pool '" << base_pool_name
<< "' is in use by CephFS via its tier";
11258 if (tier_pool
&& tier_pool
->cache_mode
== pg_pool_t::CACHEMODE_WRITEBACK
) {
11259 *ss
<< "pool '" << base_pool_name
<< "' is in use by CephFS, and this "
11260 "tier is still in use as a writeback cache. Change the cache "
11261 "mode and flush the cache before removing it";
11271 int OSDMonitor::_prepare_remove_pool(
11272 int64_t pool
, ostream
*ss
, bool no_fake
)
11274 dout(10) << __func__
<< " " << pool
<< dendl
;
11275 const pg_pool_t
*p
= osdmap
.get_pg_pool(pool
);
11276 int r
= _check_remove_pool(pool
, *p
, ss
);
11280 auto new_pool
= pending_inc
.new_pools
.find(pool
);
11281 if (new_pool
!= pending_inc
.new_pools
.end()) {
11282 // if there is a problem with the pending info, wait and retry
11284 const auto& p
= new_pool
->second
;
11285 int r
= _check_remove_pool(pool
, p
, ss
);
11290 if (pending_inc
.old_pools
.count(pool
)) {
11291 dout(10) << __func__
<< " " << pool
<< " already pending removal"
11296 if (g_conf
->mon_fake_pool_delete
&& !no_fake
) {
11297 string old_name
= osdmap
.get_pool_name(pool
);
11298 string new_name
= old_name
+ "." + stringify(pool
) + ".DELETED";
11299 dout(1) << __func__
<< " faking pool deletion: renaming " << pool
<< " "
11300 << old_name
<< " -> " << new_name
<< dendl
;
11301 pending_inc
.new_pool_names
[pool
] = new_name
;
11306 pending_inc
.old_pools
.insert(pool
);
11308 // remove any pg_temp mappings for this pool
11309 for (auto p
= osdmap
.pg_temp
->begin();
11310 p
!= osdmap
.pg_temp
->end();
11312 if (p
->first
.pool() == (uint64_t)pool
) {
11313 dout(10) << __func__
<< " " << pool
<< " removing obsolete pg_temp "
11314 << p
->first
<< dendl
;
11315 pending_inc
.new_pg_temp
[p
->first
].clear();
11318 // remove any primary_temp mappings for this pool
11319 for (auto p
= osdmap
.primary_temp
->begin();
11320 p
!= osdmap
.primary_temp
->end();
11322 if (p
->first
.pool() == (uint64_t)pool
) {
11323 dout(10) << __func__
<< " " << pool
11324 << " removing obsolete primary_temp" << p
->first
<< dendl
;
11325 pending_inc
.new_primary_temp
[p
->first
] = -1;
11328 // remove any pg_upmap mappings for this pool
11329 for (auto& p
: osdmap
.pg_upmap
) {
11330 if (p
.first
.pool() == (uint64_t)pool
) {
11331 dout(10) << __func__
<< " " << pool
11332 << " removing obsolete pg_upmap "
11333 << p
.first
<< dendl
;
11334 pending_inc
.old_pg_upmap
.insert(p
.first
);
11337 // remove any pg_upmap_items mappings for this pool
11338 for (auto& p
: osdmap
.pg_upmap_items
) {
11339 if (p
.first
.pool() == (uint64_t)pool
) {
11340 dout(10) << __func__
<< " " << pool
11341 << " removing obsolete pg_upmap_items " << p
.first
11343 pending_inc
.old_pg_upmap_items
.insert(p
.first
);
11349 int OSDMonitor::_prepare_rename_pool(int64_t pool
, string newname
)
11351 dout(10) << "_prepare_rename_pool " << pool
<< dendl
;
11352 if (pending_inc
.old_pools
.count(pool
)) {
11353 dout(10) << "_prepare_rename_pool " << pool
<< " pending removal" << dendl
;
11356 for (map
<int64_t,string
>::iterator p
= pending_inc
.new_pool_names
.begin();
11357 p
!= pending_inc
.new_pool_names
.end();
11359 if (p
->second
== newname
&& p
->first
!= pool
) {
11364 pending_inc
.new_pool_names
[pool
] = newname
;
11368 bool OSDMonitor::prepare_pool_op_delete(MonOpRequestRef op
)
11370 op
->mark_osdmon_event(__func__
);
11371 MPoolOp
*m
= static_cast<MPoolOp
*>(op
->get_req());
11373 int ret
= _prepare_remove_pool(m
->pool
, &ss
, false);
11374 if (ret
== -EAGAIN
) {
11375 wait_for_finished_proposal(op
, new C_RetryMessage(this, op
));
11379 dout(10) << __func__
<< " got " << ret
<< " " << ss
.str() << dendl
;
11380 wait_for_finished_proposal(op
, new OSDMonitor::C_PoolOp(this, op
, ret
,
11381 pending_inc
.epoch
));
11385 void OSDMonitor::_pool_op_reply(MonOpRequestRef op
,
11386 int ret
, epoch_t epoch
, bufferlist
*blp
)
11388 op
->mark_osdmon_event(__func__
);
11389 MPoolOp
*m
= static_cast<MPoolOp
*>(op
->get_req());
11390 dout(20) << "_pool_op_reply " << ret
<< dendl
;
11391 MPoolOpReply
*reply
= new MPoolOpReply(m
->fsid
, m
->get_tid(),
11392 ret
, epoch
, get_last_committed(), blp
);
11393 mon
->send_reply(op
, reply
);