* Foundation. See file COPYING.
*/
+#include "PyModuleRegistry.h"
+
+#if __has_include(<filesystem>)
+#include <filesystem>
+namespace fs = std::filesystem;
+#elif __has_include(<experimental/filesystem>)
+#include <experimental/filesystem>
+namespace fs = std::experimental::filesystem;
+#else
+#error std::filesystem not available!
+#endif
#include "include/stringify.h"
#include "common/errno.h"
-#include "common/backport14.h"
+#include "common/split.h"
#include "BaseMgrModule.h"
#include "PyOSDMap.h"
#include "BaseMgrStandbyModule.h"
#include "Gil.h"
+#include "MgrContext.h"
+#include "mgr/mgr_commands.h"
#include "ActivePyModules.h"
-#include "PyModuleRegistry.h"
-
-// definition for non-const static member
-std::string PyModuleRegistry::config_prefix;
-
-
-
#define dout_context g_ceph_context
#define dout_subsys ceph_subsys_mgr
#undef dout_prefix
#define dout_prefix *_dout << "mgr[py] "
-namespace {
- PyObject* log_write(PyObject*, PyObject* args) {
- char* m = nullptr;
- if (PyArg_ParseTuple(args, "s", &m)) {
- auto len = strlen(m);
- if (len && m[len-1] == '\n') {
- m[len-1] = '\0';
- }
- dout(4) << m << dendl;
- }
- Py_RETURN_NONE;
- }
-
- PyObject* log_flush(PyObject*, PyObject*){
- Py_RETURN_NONE;
- }
-
- static PyMethodDef log_methods[] = {
- {"write", log_write, METH_VARARGS, "write stdout and stderr"},
- {"flush", log_flush, METH_VARARGS, "flush"},
- {nullptr, nullptr, 0, nullptr}
- };
-}
-
-#undef dout_prefix
-#define dout_prefix *_dout << "mgr " << __func__ << " "
-
-
-
-std::string PyModule::get_site_packages()
-{
- std::stringstream site_packages;
-
- // CPython doesn't auto-add site-packages dirs to sys.path for us,
- // but it does provide a module that we can ask for them.
- auto site_module = PyImport_ImportModule("site");
- assert(site_module);
-
- auto site_packages_fn = PyObject_GetAttrString(site_module, "getsitepackages");
- if (site_packages_fn != nullptr) {
- auto site_packages_list = PyObject_CallObject(site_packages_fn, nullptr);
- assert(site_packages_list);
-
- auto n = PyList_Size(site_packages_list);
- for (Py_ssize_t i = 0; i < n; ++i) {
- if (i != 0) {
- site_packages << ":";
- }
- site_packages << PyString_AsString(PyList_GetItem(site_packages_list, i));
- }
-
- Py_DECREF(site_packages_list);
- Py_DECREF(site_packages_fn);
- } else {
- // Fall back to generating our own site-packages paths by imitating
- // what the standard site.py does. This is annoying but it lets us
- // run inside virtualenvs :-/
-
- auto site_packages_fn = PyObject_GetAttrString(site_module, "addsitepackages");
- assert(site_packages_fn);
-
- auto known_paths = PySet_New(nullptr);
- auto pArgs = PyTuple_Pack(1, known_paths);
- PyObject_CallObject(site_packages_fn, pArgs);
- Py_DECREF(pArgs);
- Py_DECREF(known_paths);
- Py_DECREF(site_packages_fn);
-
- auto sys_module = PyImport_ImportModule("sys");
- assert(sys_module);
- auto sys_path = PyObject_GetAttrString(sys_module, "path");
- assert(sys_path);
-
- dout(1) << "sys.path:" << dendl;
- auto n = PyList_Size(sys_path);
- bool first = true;
- for (Py_ssize_t i = 0; i < n; ++i) {
- dout(1) << " " << PyString_AsString(PyList_GetItem(sys_path, i)) << dendl;
- if (first) {
- first = false;
- } else {
- site_packages << ":";
- }
- site_packages << PyString_AsString(PyList_GetItem(sys_path, i));
- }
-
- Py_DECREF(sys_path);
- Py_DECREF(sys_module);
- }
- Py_DECREF(site_module);
- return site_packages.str();
-}
-
-int PyModuleRegistry::init(const MgrMap &map)
+void PyModuleRegistry::init()
{
- Mutex::Locker locker(lock);
-
- // Don't try and init me if you don't really have a map
- assert(map.epoch > 0);
-
- mgr_map = map;
-
- // namespace in config-key prefixed by "mgr/"
- config_prefix = std::string(g_conf->name.get_type_str()) + "/";
+ std::lock_guard locker(lock);
// Set up global python interpreter
- Py_SetProgramName(const_cast<char*>(PYTHON_EXECUTABLE));
+#if PY_MAJOR_VERSION >= 3
+#define WCHAR(s) L ## #s
+ Py_SetProgramName(const_cast<wchar_t*>(WCHAR(MGR_PYTHON_EXECUTABLE)));
+#undef WCHAR
+#else
+ Py_SetProgramName(const_cast<char*>(MGR_PYTHON_EXECUTABLE));
+#endif
+ // Add more modules
+ if (g_conf().get_val<bool>("daemonize")) {
+ PyImport_AppendInittab("ceph_logger", PyModule::init_ceph_logger);
+ }
+ PyImport_AppendInittab("ceph_module", PyModule::init_ceph_module);
Py_InitializeEx(0);
// Let CPython know that we will be calling it back from other
// Drop the GIL and remember the main thread state (current
// thread state becomes NULL)
pMainThreadState = PyEval_SaveThread();
- assert(pMainThreadState != nullptr);
+ ceph_assert(pMainThreadState != nullptr);
std::list<std::string> failed_modules;
+ const std::string module_path = g_conf().get_val<std::string>("mgr_module_path");
+ std::set<std::string> module_names = probe_modules(module_path);
// Load python code
- for (const auto& module_name : mgr_map.modules) {
+ for (const auto& module_name : module_names) {
dout(1) << "Loading python module '" << module_name << "'" << dendl;
- auto mod = ceph::make_unique<PyModule>(module_name);
+
+ // Everything starts disabled, set enabled flag on module
+ // when we see first MgrMap
+ auto mod = std::make_shared<PyModule>(module_name);
int r = mod->load(pMainThreadState);
if (r != 0) {
// Don't use handle_pyerror() here; we don't have the GIL
<< cpp_strerror(r) << dendl;
failed_modules.push_back(module_name);
// Don't drop out here, load the other modules
- } else {
- // Success!
- modules[module_name] = std::move(mod);
}
- }
+ // Record the module even if the load failed, so that we can
+ // report its loading error
+ modules[module_name] = std::move(mod);
+ }
+ if (module_names.empty()) {
+ clog->error() << "No ceph-mgr modules found in " << module_path;
+ }
if (!failed_modules.empty()) {
clog->error() << "Failed to load ceph-mgr modules: " << joinify(
failed_modules.begin(), failed_modules.end(), std::string(", "));
}
-
- return 0;
}
-
-int PyModule::load(PyThreadState *pMainThreadState)
+bool PyModuleRegistry::handle_mgr_map(const MgrMap &mgr_map_)
{
- assert(pMainThreadState != nullptr);
-
- // Configure sub-interpreter and construct C++-generated python classes
- {
- SafeThreadState sts(pMainThreadState);
- Gil gil(sts);
-
- auto thread_state = Py_NewInterpreter();
- if (thread_state == nullptr) {
- derr << "Failed to create python sub-interpreter for '" << module_name << '"' << dendl;
- return -EINVAL;
- } else {
- pMyThreadState.set(thread_state);
- // Some python modules do not cope with an unpopulated argv, so lets
- // fake one. This step also picks up site-packages into sys.path.
- const char *argv[] = {"ceph-mgr"};
- PySys_SetArgv(1, (char**)argv);
-
- if (g_conf->daemonize) {
- auto py_logger = Py_InitModule("ceph_logger", log_methods);
-#if PY_MAJOR_VERSION >= 3
- PySys_SetObject("stderr", py_logger);
- PySys_SetObject("stdout", py_logger);
-#else
- PySys_SetObject(const_cast<char*>("stderr"), py_logger);
- PySys_SetObject(const_cast<char*>("stdout"), py_logger);
-#endif
- }
-
- // Configure sys.path to include mgr_module_path
- std::string sys_path = std::string(Py_GetPath()) + ":" + get_site_packages()
- + ":" + g_conf->get_val<std::string>("mgr_module_path");
- dout(10) << "Computed sys.path '" << sys_path << "'" << dendl;
-
- PySys_SetPath(const_cast<char*>(sys_path.c_str()));
- }
-
- PyMethodDef ModuleMethods[] = {
- {nullptr}
- };
-
- // Initialize module
- PyObject *ceph_module = Py_InitModule("ceph_module", ModuleMethods);
- assert(ceph_module != nullptr);
-
- auto load_class = [ceph_module](const char *name, PyTypeObject *type)
- {
- type->tp_new = PyType_GenericNew;
- if (PyType_Ready(type) < 0) {
- assert(0);
- }
- Py_INCREF(type);
-
- PyModule_AddObject(ceph_module, name, (PyObject *)type);
- };
-
- load_class("BaseMgrModule", &BaseMgrModuleType);
- load_class("BaseMgrStandbyModule", &BaseMgrStandbyModuleType);
- load_class("BasePyOSDMap", &BasePyOSDMapType);
- load_class("BasePyOSDMapIncremental", &BasePyOSDMapIncrementalType);
- load_class("BasePyCRUSH", &BasePyCRUSHType);
- }
-
- // Environment is all good, import the external module
- {
- Gil gil(pMyThreadState);
-
- // Load the module
- PyObject *pName = PyString_FromString(module_name.c_str());
- auto pModule = PyImport_Import(pName);
- Py_DECREF(pName);
- if (pModule == nullptr) {
- derr << "Module not found: '" << module_name << "'" << dendl;
- derr << handle_pyerror() << dendl;
- return -ENOENT;
+ std::lock_guard l(lock);
+
+ if (mgr_map.epoch == 0) {
+ mgr_map = mgr_map_;
+
+ // First time we see MgrMap, set the enabled flags on modules
+ // This should always happen before someone calls standby_start
+ // or active_start
+ for (const auto &[module_name, module] : modules) {
+ const bool enabled = (mgr_map.modules.count(module_name) > 0);
+ module->set_enabled(enabled);
+ const bool always_on = (mgr_map.get_always_on_modules().count(module_name) > 0);
+ module->set_always_on(always_on);
}
- // Find the class
- // TODO: let them call it what they want instead of just 'Module'
- pClass = PyObject_GetAttrString(pModule, (const char*)"Module");
- if (pClass == nullptr) {
- derr << "Class not found in module '" << module_name << "'" << dendl;
- derr << handle_pyerror() << dendl;
- return -EINVAL;
- }
+ return false;
+ } else {
+ bool modules_changed = mgr_map_.modules != mgr_map.modules ||
+ mgr_map_.always_on_modules != mgr_map.always_on_modules;
+ mgr_map = mgr_map_;
- pStandbyClass = PyObject_GetAttrString(pModule,
- (const char*)"StandbyModule");
- if (pStandbyClass) {
- dout(4) << "Standby mode available in module '" << module_name
- << "'" << dendl;
- } else {
- dout(4) << "Standby mode not provided by module '" << module_name
- << "'" << dendl;
- PyErr_Clear();
+ if (standby_modules != nullptr) {
+ standby_modules->handle_mgr_map(mgr_map_);
}
- Py_DECREF(pModule);
+ return modules_changed;
}
+}
- return 0;
-}
-PyModule::~PyModule()
-{
- if (pMyThreadState.ts != nullptr) {
- Gil gil(pMyThreadState, true);
- Py_XDECREF(pClass);
- Py_XDECREF(pStandbyClass);
- }
-}
-void PyModuleRegistry::standby_start(MonClient *monc)
+void PyModuleRegistry::standby_start(MonClient &mc, Finisher &f)
{
- Mutex::Locker l(lock);
- assert(active_modules == nullptr);
- assert(standby_modules == nullptr);
- assert(is_initialized());
+ std::lock_guard l(lock);
+ ceph_assert(active_modules == nullptr);
+ ceph_assert(standby_modules == nullptr);
+
+ // Must have seen a MgrMap by this point, in order to know
+ // which modules should be enabled
+ ceph_assert(mgr_map.epoch > 0);
dout(4) << "Starting modules in standby mode" << dendl;
- standby_modules.reset(new StandbyPyModules(monc, mgr_map, clog));
+ standby_modules.reset(new StandbyPyModules(
+ mgr_map, module_config, clog, mc, f));
std::set<std::string> failed_modules;
for (const auto &i : modules) {
- if (i.second->pStandbyClass) {
- dout(4) << "starting module " << i.second->get_name() << dendl;
- int r = standby_modules->start_one(i.first,
- i.second->pStandbyClass,
- i.second->pMyThreadState);
- if (r != 0) {
- derr << "failed to start module '" << i.second->get_name()
- << "'" << dendl;;
+ if (!(i.second->is_enabled() && i.second->get_can_run())) {
+ // report always_on modules with a standby mode that won't run
+ if (i.second->is_always_on() && i.second->pStandbyClass) {
failed_modules.insert(i.second->get_name());
- // Continue trying to load any other modules
}
+ continue;
+ }
+
+ if (i.second->pStandbyClass) {
+ dout(4) << "starting module " << i.second->get_name() << dendl;
+ standby_modules->start_one(i.second);
} else {
dout(4) << "skipping module '" << i.second->get_name() << "' because "
"it does not implement a standby mode" << dendl;
}
void PyModuleRegistry::active_start(
- PyModuleConfig &config_,
- DaemonStateIndex &ds, ClusterState &cs, MonClient &mc,
- LogChannelRef clog_, Objecter &objecter_, Client &client_,
- Finisher &f)
+ DaemonStateIndex &ds, ClusterState &cs,
+ const std::map<std::string, std::string> &kv_store,
+ MonClient &mc, LogChannelRef clog_, LogChannelRef audit_clog_,
+ Objecter &objecter_, Client &client_, Finisher &f,
+ DaemonServer &server)
{
- Mutex::Locker locker(lock);
+ std::lock_guard locker(lock);
dout(4) << "Starting modules in active mode" << dendl;
- assert(active_modules == nullptr);
- assert(is_initialized());
+ ceph_assert(active_modules == nullptr);
+
+ // Must have seen a MgrMap by this point, in order to know
+ // which modules should be enabled
+ ceph_assert(mgr_map.epoch > 0);
if (standby_modules != nullptr) {
standby_modules->shutdown();
}
active_modules.reset(new ActivePyModules(
- config_, ds, cs, mc, clog_, objecter_, client_, f));
+ module_config, kv_store, ds, cs, mc,
+ clog_, audit_clog_, objecter_, client_, f, server,
+ *this));
for (const auto &i : modules) {
- dout(4) << "Starting " << i.first << dendl;
- int r = active_modules->start_one(i.first,
- i.second->pClass,
- i.second->pMyThreadState);
- if (r != 0) {
- derr << "Failed to run module in active mode ('" << i.first << "')"
- << dendl;
+ // Anything we're skipping because of !can_run will be flagged
+ // to the user separately via get_health_checks
+ if (!(i.second->is_enabled() && i.second->is_loaded())) {
+ continue;
}
+
+ dout(4) << "Starting " << i.first << dendl;
+ active_modules->start_one(i.second);
}
}
void PyModuleRegistry::active_shutdown()
{
- Mutex::Locker locker(lock);
+ std::lock_guard locker(lock);
if (active_modules != nullptr) {
active_modules->shutdown();
void PyModuleRegistry::shutdown()
{
- Mutex::Locker locker(lock);
+ std::lock_guard locker(lock);
if (standby_modules != nullptr) {
standby_modules->shutdown();
Py_Finalize();
}
-static void _list_modules(
- const std::string path,
- std::set<std::string> *modules)
+std::set<std::string> PyModuleRegistry::probe_modules(const std::string &path) const
+{
+ const auto opt = g_conf().get_val<std::string>("mgr_disabled_modules");
+ const auto disabled_modules = ceph::split(opt);
+
+ std::set<std::string> modules;
+ for (const auto& entry: fs::directory_iterator(path)) {
+ if (!fs::is_directory(entry)) {
+ continue;
+ }
+ const std::string name = entry.path().filename();
+ if (std::count(disabled_modules.begin(), disabled_modules.end(), name)) {
+ dout(10) << "ignoring disabled module " << name << dendl;
+ continue;
+ }
+ auto module_path = entry.path() / "module.py";
+ if (fs::exists(module_path)) {
+ modules.emplace(name);
+ }
+ }
+ return modules;
+}
+
+int PyModuleRegistry::handle_command(
+ const ModuleCommand& module_command,
+ const MgrSession& session,
+ const cmdmap_t &cmdmap,
+ const bufferlist &inbuf,
+ std::stringstream *ds,
+ std::stringstream *ss)
+{
+ if (active_modules) {
+ return active_modules->handle_command(module_command, session, cmdmap,
+ inbuf, ds, ss);
+ } else {
+ // We do not expect to be called before active modules is up, but
+ // it's straightfoward to handle this case so let's do it.
+ return -EAGAIN;
+ }
+}
+
+std::vector<ModuleCommand> PyModuleRegistry::get_py_commands() const
+{
+ std::lock_guard l(lock);
+
+ std::vector<ModuleCommand> result;
+ for (const auto& i : modules) {
+ i.second->get_commands(&result);
+ }
+
+ return result;
+}
+
+std::vector<MonCommand> PyModuleRegistry::get_commands() const
{
- DIR *dir = opendir(path.c_str());
- if (!dir) {
- return;
+ std::vector<ModuleCommand> commands = get_py_commands();
+ std::vector<MonCommand> result;
+ for (auto &pyc: commands) {
+ uint64_t flags = MonCommand::FLAG_MGR;
+ if (pyc.polling) {
+ flags |= MonCommand::FLAG_POLL;
+ }
+ result.push_back({pyc.cmdstring, pyc.helpstring, "mgr",
+ pyc.perm, flags});
}
- struct dirent *entry = NULL;
- while ((entry = readdir(dir)) != NULL) {
- string n(entry->d_name);
- string fn = path + "/" + n;
- struct stat st;
- int r = ::stat(fn.c_str(), &st);
- if (r == 0 && S_ISDIR(st.st_mode)) {
- string initfn = fn + "/module.py";
- r = ::stat(initfn.c_str(), &st);
- if (r == 0) {
- modules->insert(n);
+ return result;
+}
+
+void PyModuleRegistry::get_health_checks(health_check_map_t *checks)
+{
+ std::lock_guard l(lock);
+
+ // Only the active mgr reports module issues
+ if (active_modules) {
+ active_modules->get_health_checks(checks);
+
+ std::map<std::string, std::string> dependency_modules;
+ std::map<std::string, std::string> failed_modules;
+
+ /*
+ * Break up broken modules into two categories:
+ * - can_run=false: the module is working fine but explicitly
+ * telling you that a dependency is missing. Advise the user to
+ * read the message from the module and install what's missing.
+ * - failed=true or loaded=false: something unexpected is broken,
+ * either at runtime (from serve()) or at load time. This indicates
+ * a bug and the user should be guided to inspect the mgr log
+ * to investigate and gather evidence.
+ */
+
+ for (const auto &i : modules) {
+ auto module = i.second;
+ if (module->is_enabled() && !module->get_can_run()) {
+ dependency_modules[module->get_name()] = module->get_error_string();
+ } else if ((module->is_enabled() && !module->is_loaded())
+ || (module->is_failed() && module->get_can_run())) {
+ // - Unloadable modules are only reported if they're enabled,
+ // to avoid spamming users about modules they don't have the
+ // dependencies installed for because they don't use it.
+ // - Failed modules are only reported if they passed the can_run
+ // checks (to avoid outputting two health messages about a
+ // module that said can_run=false but we tried running it anyway)
+ failed_modules[module->get_name()] = module->get_error_string();
+ }
+ }
+
+ // report failed always_on modules as health errors
+ for (const auto& name : mgr_map.get_always_on_modules()) {
+ if (active_modules->is_pending(name)) {
+ continue;
+ }
+ if (!active_modules->module_exists(name)) {
+ if (failed_modules.find(name) == failed_modules.end() &&
+ dependency_modules.find(name) == dependency_modules.end()) {
+ failed_modules[name] = "Not found or unloadable";
+ }
+ }
+ }
+
+ if (!dependency_modules.empty()) {
+ std::ostringstream ss;
+ if (dependency_modules.size() == 1) {
+ auto iter = dependency_modules.begin();
+ ss << "Module '" << iter->first << "' has failed dependency: "
+ << iter->second;
+ } else if (dependency_modules.size() > 1) {
+ ss << dependency_modules.size()
+ << " mgr modules have failed dependencies";
+ }
+ auto& d = checks->add("MGR_MODULE_DEPENDENCY", HEALTH_WARN, ss.str(),
+ dependency_modules.size());
+ for (auto& i : dependency_modules) {
+ std::ostringstream ss;
+ ss << "Module '" << i.first << "' has failed dependency: " << i.second;
+ d.detail.push_back(ss.str());
+ }
+ }
+
+ if (!failed_modules.empty()) {
+ std::ostringstream ss;
+ if (failed_modules.size() == 1) {
+ auto iter = failed_modules.begin();
+ ss << "Module '" << iter->first << "' has failed: " << iter->second;
+ } else if (failed_modules.size() > 1) {
+ ss << failed_modules.size() << " mgr modules have failed";
+ }
+ auto& d = checks->add("MGR_MODULE_ERROR", HEALTH_ERR, ss.str(),
+ failed_modules.size());
+ for (auto& i : failed_modules) {
+ std::ostringstream ss;
+ ss << "Module '" << i.first << "' has failed: " << i.second;
+ d.detail.push_back(ss.str());
}
}
}
- closedir(dir);
}
-void PyModuleRegistry::list_modules(std::set<std::string> *modules)
+void PyModuleRegistry::handle_config(const std::string &k, const std::string &v)
{
- _list_modules(g_conf->get_val<std::string>("mgr_module_path"), modules);
+ std::lock_guard l(module_config.lock);
+
+ if (!v.empty()) {
+ // removing value to hide sensitive data going into mgr logs
+ // leaving this for debugging purposes
+ // dout(10) << "Loaded module_config entry " << k << ":" << v << dendl;
+ dout(10) << "Loaded module_config entry " << k << ":" << dendl;
+ module_config.config[k] = v;
+ } else {
+ module_config.config.erase(k);
+ }
+}
+
+void PyModuleRegistry::handle_config_notify()
+{
+ std::lock_guard l(lock);
+ if (active_modules) {
+ active_modules->config_notify();
+ }
+}
+
+void PyModuleRegistry::upgrade_config(
+ MonClient *monc,
+ const std::map<std::string, std::string> &old_config)
+{
+ // Only bother doing anything if we didn't already have
+ // some new-style config.
+ if (module_config.config.empty()) {
+ dout(1) << "Upgrading module configuration for Mimic" << dendl;
+ // Upgrade luminous->mimic: migrate config-key configuration
+ // into main configuration store
+ for (auto &i : old_config) {
+ auto last_slash = i.first.rfind('/');
+ const std::string module_name = i.first.substr(4, i.first.substr(4).find('/'));
+ const std::string key = i.first.substr(last_slash + 1);
+
+ const auto &value = i.second;
+
+ // Heuristic to skip things that look more like stores
+ // than configs.
+ bool is_config = true;
+ for (const auto &c : value) {
+ if (c == '\n' || c == '\r' || c < 0x20) {
+ is_config = false;
+ break;
+ }
+ }
+
+ if (value.size() > 256) {
+ is_config = false;
+ }
+
+ if (!is_config) {
+ dout(1) << "Not migrating config module:key "
+ << module_name << " : " << key << dendl;
+ continue;
+ }
+
+ // Check that the named module exists
+ auto module_iter = modules.find(module_name);
+ if (module_iter == modules.end()) {
+ dout(1) << "KV store contains data for unknown module '"
+ << module_name << "'" << dendl;
+ continue;
+ }
+ PyModuleRef module = module_iter->second;
+
+ // Parse option name out of key
+ std::string option_name;
+ auto slash_loc = key.find("/");
+ if (slash_loc != std::string::npos) {
+ if (key.size() > slash_loc + 1) {
+ // Localized option
+ option_name = key.substr(slash_loc + 1);
+ } else {
+ // Trailing slash: garbage.
+ derr << "Invalid mgr store key: '" << key << "'" << dendl;
+ continue;
+ }
+ } else {
+ option_name = key;
+ }
+
+ // Consult module schema to see if this is really
+ // a configuration value
+ if (!option_name.empty() && module->is_option(option_name)) {
+ module_config.set_config(monc, module_name, key, i.second);
+ dout(4) << "Rewrote configuration module:key "
+ << module_name << ":" << key << dendl;
+ } else {
+ dout(4) << "Leaving store module:key " << module_name
+ << ":" << key << " in store, not config" << dendl;
+ }
+ }
+ } else {
+ dout(10) << "Module configuration contains "
+ << module_config.config.size() << " keys" << dendl;
+ }
}
projects / ceph.git / blobdiff