X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=ceph%2Fsrc%2Fcommon%2Fconfig.cc;h=c8101587b7190527088481a4b98ab1037738a308;hb=20effc670b57271cb089376d6d0800990e5218d5;hp=5cbbd85e97be0a0ed7b53bdb8bccdb6009498833;hpb=224ce89bce8186937e77bdbda572a650953f8c23;p=ceph.git diff --git a/ceph/src/common/config.cc b/ceph/src/common/config.cc index 5cbbd85e9..c8101587b 100644 --- a/ceph/src/common/config.cc +++ b/ceph/src/common/config.cc @@ -12,55 +12,84 @@ * */ +#include #include "common/ceph_argparse.h" #include "common/common_init.h" #include "common/config.h" -#include "common/config_validators.h" +#include "common/config_obs.h" #include "include/str_list.h" #include "include/stringify.h" #include "osd/osd_types.h" #include "common/errno.h" #include "common/hostname.h" - -#include +#include "common/dout.h" /* Don't use standard Ceph logging in this file. * We can't use logging until it's initialized, and a lot of the necessary * initialization happens here. */ #undef dout -#undef ldout #undef pdout #undef derr -#undef lderr #undef generic_dout -#undef dendl +// set set_mon_vals() +#define dout_subsys ceph_subsys_monc + +namespace fs = std::filesystem; + +using std::cerr; +using std::cout; using std::map; +using std::less; using std::list; +using std::ostream; using std::ostringstream; using std::pair; using std::string; +using std::string_view; +using std::vector; + +using ceph::bufferlist; +using ceph::decode; +using ceph::encode; +using ceph::Formatter; -const char *CEPH_CONF_FILE_DEFAULT = "$data_dir/config, /etc/ceph/$cluster.conf, ~/.ceph/$cluster.conf, $cluster.conf" +static const char *CEPH_CONF_FILE_DEFAULT = "$data_dir/config,/etc/ceph/$cluster.conf,$home/.ceph/$cluster.conf,$cluster.conf" #if defined(__FreeBSD__) - ", /usr/local/etc/ceph/$cluster.conf" + ",/usr/local/etc/ceph/$cluster.conf" +#elif defined(_WIN32) + ",$programdata/ceph/$cluster.conf" #endif ; #define _STR(x) #x #define STRINGIFY(x) _STR(x) +const char *ceph_conf_level_name(int level) +{ + switch (level) { + case CONF_DEFAULT: return "default"; // built-in default + case CONF_MON: return "mon"; // monitor config database + case CONF_ENV: return "env"; // process environment (CEPH_ARGS) + case CONF_FILE: return "file"; // ceph.conf file + case CONF_CMDLINE: return "cmdline"; // process command line args + case CONF_OVERRIDE: return "override"; // injectargs or 'config set' at runtime + case CONF_FINAL: return "final"; + default: return "???"; + } +} + int ceph_resolve_file_search(const std::string& filename_list, std::string& result) { list ls; - get_str_list(filename_list, ls); + get_str_list(filename_list, ";,", ls); int ret = -ENOENT; list::iterator iter; for (iter = ls.begin(); iter != ls.end(); ++iter) { - int fd = ::open(iter->c_str(), O_RDONLY); + int fd = ::open(iter->c_str(), O_RDONLY|O_CLOEXEC); if (fd < 0) { ret = -errno; continue; @@ -73,371 +102,543 @@ int ceph_resolve_file_search(const std::string& filename_list, return ret; } -#define OPTION(name, type, def_val) -#define OPTION_VALIDATOR(name) \ -struct md_config_t::option_##name##_t { \ - typedef decltype(md_config_t::name) type; \ -}; -#define SAFE_OPTION(name, type, def_val) -#define SUBSYS(name, log, gather) -#define DEFAULT_SUBSYS(log, gather) -#include "common/config_opts.h" -#undef OPTION -#undef OPTION_VALIDATOR -#undef SAFE_OPTION -#undef SUBSYS -#undef DEFAULT_SUBSYS - -namespace { - -template -typename std::enable_if::value, - md_config_t::validator_t>::type create_validator() { - return md_config_t::validator_t(); +static int conf_stringify(const Option::value_t& v, string *out) +{ + if (v == Option::value_t{}) { + return -ENOENT; + } + *out = Option::to_str(v); + return 0; } -template -typename std::enable_if::value, - md_config_t::validator_t>::type create_validator() { - // if T is defined (and not just forward declared), it implies - // that a validator function exists. use a dummy typed pointer to - // pick the correct validator function - return [](std::string *value, std::string *error_message) { - return ::validate(reinterpret_cast(0), value, error_message); - }; -} +md_config_t::md_config_t(ConfigValues& values, + const ConfigTracker& tracker, + bool is_daemon) + : is_daemon(is_daemon) +{ + // Load the compile-time list of Option into + // a map so that we can resolve keys quickly. + for (const auto &i : ceph_options) { + if (schema.count(i.name)) { + // We may be instantiated pre-logging so send + std::cerr << "Duplicate config key in schema: '" << i.name << "'" + << std::endl; + ceph_abort(); + } + schema.emplace(i.name, i); + } -} // anonymous namespace + // Define the debug_* options as well. + subsys_options.reserve(values.subsys.get_num()); + for (unsigned i = 0; i < values.subsys.get_num(); ++i) { + string name = string("debug_") + values.subsys.get_name(i); + subsys_options.push_back( + Option(name, Option::TYPE_STR, Option::LEVEL_ADVANCED)); + Option& opt = subsys_options.back(); + opt.set_default(stringify(values.subsys.get_log_level(i)) + "/" + + stringify(values.subsys.get_gather_level(i))); + string desc = string("Debug level for ") + values.subsys.get_name(i); + opt.set_description(desc.c_str()); + opt.set_flag(Option::FLAG_RUNTIME); + opt.set_long_description("The value takes the form 'N' or 'N/M' where N and M are values between 0 and 99. N is the debug level to log (all values below this are included), and M is the level to gather and buffer in memory. In the event of a crash, the most recent items <= M are dumped to the log file."); + opt.set_subsys(i); + opt.set_validator([](std::string *value, std::string *error_message) { + int m, n; + int r = sscanf(value->c_str(), "%d/%d", &m, &n); + if (r >= 1) { + if (m < 0 || m > 99) { + *error_message = "value must be in range [0, 99]"; + return -ERANGE; + } + if (r == 2) { + if (n < 0 || n > 99) { + *error_message = "value must be in range [0, 99]"; + return -ERANGE; + } + } else { + // normalize to M/N + n = m; + *value = stringify(m) + "/" + stringify(n); + } + } else { + *error_message = "value must take the form N or N/M, where N and M are integers"; + return -EINVAL; + } + return 0; + }); + } + for (auto& opt : subsys_options) { + schema.emplace(opt.name, opt); + } -md_config_t::md_config_t() - : cluster(""), - -#define OPTION_OPT_INT(name, def_val) name(def_val), -#define OPTION_OPT_LONGLONG(name, def_val) name((1LL) * def_val), -#define OPTION_OPT_STR(name, def_val) name(def_val), -#define OPTION_OPT_DOUBLE(name, def_val) name(def_val), -#define OPTION_OPT_FLOAT(name, def_val) name(def_val), -#define OPTION_OPT_BOOL(name, def_val) name(def_val), -#define OPTION_OPT_ADDR(name, def_val) name(def_val), -#define OPTION_OPT_U32(name, def_val) name(def_val), -#define OPTION_OPT_U64(name, def_val) name(((uint64_t)1) * def_val), -#define OPTION_OPT_UUID(name, def_val) name(def_val), -#define OPTION(name, type, def_val) OPTION_##type(name, def_val) -#define OPTION_VALIDATOR(name) -#define SAFE_OPTION(name, type, def_val) OPTION(name, type, def_val) -#define SUBSYS(name, log, gather) -#define DEFAULT_SUBSYS(log, gather) -#include "common/config_opts.h" -#undef OPTION_OPT_INT -#undef OPTION_OPT_LONGLONG -#undef OPTION_OPT_STR -#undef OPTION_OPT_DOUBLE -#undef OPTION_OPT_FLOAT -#undef OPTION_OPT_BOOL -#undef OPTION_OPT_ADDR -#undef OPTION_OPT_U32 -#undef OPTION_OPT_U64 -#undef OPTION_OPT_UUID -#undef OPTION -#undef OPTION_VALIDATOR -#undef SAFE_OPTION -#undef SUBSYS -#undef DEFAULT_SUBSYS - lock("md_config_t", true, false) -{ - static const std::vector s_config_options = { -#define OPTION4(name, type, def_val, safe) \ - config_option{ STRINGIFY(name), type, &md_config_t::name, safe, \ - create_validator() }, -#define OPTION(name, type, def_val) OPTION4(name, type, def_val, false) -#define OPTION_VALIDATOR(name) -#define SAFE_OPTION(name, type, def_val) OPTION4(name, type, def_val, true) -#define SUBSYS(name, log, gather) -#define DEFAULT_SUBSYS(log, gather) -#include "common/config_opts.h" -#undef OPTION4 + // Populate list of legacy_values according to the OPTION() definitions + // Note that this is just setting up our map of name->member ptr. The + // default values etc will get loaded in along with new-style data, + // as all loads write to both the values map, and the legacy + // members if present. + legacy_values = { +#define OPTION(name, type) \ + {STRINGIFY(name), &ConfigValues::name}, +#define SAFE_OPTION(name, type) OPTION(name, type) +#include "options/legacy_config_opts.h" #undef OPTION -#undef OPTION_VALIDATOR #undef SAFE_OPTION -#undef SUBSYS -#undef DEFAULT_SUBSYS }; - static std::shared_ptr - s_tbl(new std::vector(std::move(s_config_options))); - config_options = s_tbl; - validate_default_settings(); - init_subsys(); -} + validate_schema(); -void md_config_t::init_subsys() -{ -#define SUBSYS(name, log, gather) \ - subsys.add(ceph_subsys_##name, STRINGIFY(name), log, gather); -#define DEFAULT_SUBSYS(log, gather) \ - subsys.add(ceph_subsys_, "none", log, gather); -#define OPTION(a, b, c) -#define OPTION_VALIDATOR(a) -#define SAFE_OPTION(a, b, c) -#include "common/config_opts.h" -#undef OPTION -#undef OPTION_VALIDATOR -#undef SAFE_OPTION -#undef SUBSYS -#undef DEFAULT_SUBSYS + // Validate default values from the schema + for (const auto &i : schema) { + const Option &opt = i.second; + if (opt.type == Option::TYPE_STR) { + bool has_daemon_default = (opt.daemon_value != Option::value_t{}); + Option::value_t default_val; + if (is_daemon && has_daemon_default) { + default_val = opt.daemon_value; + } else { + default_val = opt.value; + } + // We call pre_validate as a sanity check, but also to get any + // side effect (value modification) from the validator. + auto* def_str = std::get_if(&default_val); + std::string val = *def_str; + std::string err; + if (opt.pre_validate(&val, &err) != 0) { + std::cerr << "Default value " << opt.name << "=" << *def_str << " is " + "invalid: " << err << std::endl; + + // This is the compiled-in default that is failing its own option's + // validation, so this is super-invalid and should never make it + // past a pull request: crash out. + ceph_abort(); + } + if (val != *def_str) { + // if the validator normalizes the string into a different form than + // what was compiled in, use that. + set_val_default(values, tracker, opt.name, val); + } + } + } + + // Copy out values (defaults) into any legacy (C struct member) fields + update_legacy_vals(values); } md_config_t::~md_config_t() { } -void md_config_t::add_observer(md_config_obs_t* observer_) +/** + * Sanity check schema. Assert out on failures, to ensure any bad changes + * cannot possibly pass any testing and make it into a release. + */ +void md_config_t::validate_schema() { - Mutex::Locker l(lock); - const char **keys = observer_->get_tracked_conf_keys(); - for (const char ** k = keys; *k; ++k) { - obs_map_t::value_type val(*k, observer_); - observers.insert(val); + for (const auto &i : schema) { + const auto &opt = i.second; + for (const auto &see_also_key : opt.see_also) { + if (schema.count(see_also_key) == 0) { + std::cerr << "Non-existent see-also key '" << see_also_key + << "' on option '" << opt.name << "'" << std::endl; + ceph_abort(); + } + } + } + + for (const auto &i : legacy_values) { + if (schema.count(i.first) == 0) { + std::cerr << "Schema is missing legacy field '" << i.first << "'" + << std::endl; + ceph_abort(); + } } } -void md_config_t::remove_observer(md_config_obs_t* observer_) +const Option *md_config_t::find_option(const std::string_view name) const { - Mutex::Locker l(lock); - bool found_obs = false; - for (obs_map_t::iterator o = observers.begin(); o != observers.end(); ) { - if (o->second == observer_) { - observers.erase(o++); - found_obs = true; - } - else { - ++o; - } + auto p = schema.find(name); + if (p != schema.end()) { + return &p->second; } - assert(found_obs); + return nullptr; } -int md_config_t::parse_config_files(const char *conf_files, - std::ostream *warnings, - int flags) +void md_config_t::set_val_default(ConfigValues& values, + const ConfigTracker& tracker, + const string_view name, const std::string& val) { - Mutex::Locker l(lock); + const Option *o = find_option(name); + ceph_assert(o); + string err; + int r = _set_val(values, tracker, val, *o, CONF_DEFAULT, &err); + ceph_assert(r >= 0); +} - if (internal_safe_to_start_threads) - return -ENOSYS; +int md_config_t::set_mon_vals(CephContext *cct, + ConfigValues& values, + const ConfigTracker& tracker, + const map>& kv, + config_callback config_cb) +{ + ignored_mon_values.clear(); - if (!cluster.size() && !conf_files) { - /* - * set the cluster name to 'ceph' when neither cluster name nor - * configuration file are specified. - */ - cluster = "ceph"; + if (!config_cb) { + ldout(cct, 4) << __func__ << " no callback set" << dendl; } - if (!conf_files) { - const char *c = getenv("CEPH_CONF"); - if (c) { - conf_files = c; + for (auto& i : kv) { + if (config_cb) { + if (config_cb(i.first, i.second)) { + ldout(cct, 4) << __func__ << " callback consumed " << i.first << dendl; + continue; + } + ldout(cct, 4) << __func__ << " callback ignored " << i.first << dendl; } - else { - if (flags & CINIT_FLAG_NO_DEFAULT_CONFIG_FILE) - return 0; - conf_files = CEPH_CONF_FILE_DEFAULT; + const Option *o = find_option(i.first); + if (!o) { + ldout(cct,10) << __func__ << " " << i.first << " = " << i.second + << " (unrecognized option)" << dendl; + continue; } - } - - std::list cfl; - get_str_list(conf_files, cfl); - - auto p = cfl.begin(); - while (p != cfl.end()) { - // expand $data_dir? - string &s = *p; - if (s.find("$data_dir") != string::npos) { - if (data_dir_option.length()) { - list stack; - expand_meta(s, NULL, stack, warnings); - p++; - } else { - cfl.erase(p++); // ignore this item - } + if (o->has_flag(Option::FLAG_NO_MON_UPDATE)) { + ignored_mon_values.emplace(i); + continue; + } + std::string err; + int r = _set_val(values, tracker, i.second, *o, CONF_MON, &err); + if (r < 0) { + ldout(cct, 4) << __func__ << " failed to set " << i.first << " = " + << i.second << ": " << err << dendl; + ignored_mon_values.emplace(i); + } else if (r == ConfigValues::SET_NO_CHANGE || + r == ConfigValues::SET_NO_EFFECT) { + ldout(cct,20) << __func__ << " " << i.first << " = " << i.second + << " (no change)" << dendl; + } else if (r == ConfigValues::SET_HAVE_EFFECT) { + ldout(cct,10) << __func__ << " " << i.first << " = " << i.second << dendl; } else { - ++p; + ceph_abort(); } } - return parse_config_files_impl(cfl, warnings); + values.for_each([&] (auto name, auto configs) { + auto config = configs.find(CONF_MON); + if (config == configs.end()) { + return; + } + if (kv.find(name) != kv.end()) { + return; + } + ldout(cct,10) << __func__ << " " << name + << " cleared (was " << Option::to_str(config->second) << ")" + << dendl; + values.rm_val(name, CONF_MON); + // if this is a debug option, it needs to propagate to teh subsys; + // this isn't covered by update_legacy_vals() below. similarly, + // we want to trigger a config notification for these items. + const Option *o = find_option(name); + _refresh(values, *o); + }); + values_bl.clear(); + update_legacy_vals(values); + return 0; } -int md_config_t::parse_config_files_impl(const std::list &conf_files, - std::ostream *warnings) +int md_config_t::parse_config_files(ConfigValues& values, + const ConfigTracker& tracker, + const char *conf_files_str, + std::ostream *warnings, + int flags) { - assert(lock.is_locked()); + if (safe_to_start_threads) + return -ENOSYS; + if (values.cluster.empty() && !conf_files_str) { + values.cluster = get_cluster_name(nullptr); + } // open new conf - list::const_iterator c; - for (c = conf_files.begin(); c != conf_files.end(); ++c) { - cf.clear(); - string fn = *c; - expand_meta(fn, warnings); - int ret = cf.parse_file(fn.c_str(), &parse_errors, warnings); - if (ret == 0) + for (auto& fn : get_conffile_paths(values, conf_files_str, warnings, flags)) { + bufferlist bl; + std::string error; + if (bl.read_file(fn.c_str(), &error)) { + parse_error = error; + continue; + } + ostringstream oss; + int ret = parse_buffer(values, tracker, bl.c_str(), bl.length(), &oss); + if (ret == 0) { + parse_error.clear(); + conf_path = fn; break; - else if (ret != -ENOENT) + } + parse_error = oss.str(); + if (ret != -ENOENT) { return ret; + } } // it must have been all ENOENTs, that's the only way we got here - if (c == conf_files.end()) + if (conf_path.empty()) { return -ENOENT; - - if (cluster.size() == 0) { - /* - * If cluster name is not set yet, use the prefix of the - * basename of configuration file as cluster name. - */ - auto start = c->rfind('/') + 1; - auto end = c->find(".conf", start); - if (end == c->npos) { - /* - * If the configuration file does not follow $cluster.conf - * convention, we do the last try and assign the cluster to - * 'ceph'. - */ - cluster = "ceph"; - } else { - cluster = c->substr(start, end - start); - } } + if (values.cluster.empty()) { + values.cluster = get_cluster_name(conf_path.c_str()); + } + update_legacy_vals(values); + return 0; +} - std::vector my_sections; - _get_my_sections(my_sections); - for (auto& opt: *config_options) { +int +md_config_t::parse_buffer(ConfigValues& values, + const ConfigTracker& tracker, + const char* buf, size_t len, + std::ostream* warnings) +{ + if (!cf.parse_buffer(string_view{buf, len}, warnings)) { + return -EINVAL; + } + const auto my_sections = get_my_sections(values); + for (const auto &i : schema) { + const auto &opt = i.second; std::string val; - int ret = _get_val_from_conf_file(my_sections, opt.name, val, false); - if (ret == 0) { - std::string error_message; - int r = set_val_impl(val, &opt, &error_message); - if (warnings != nullptr && (r != 0 || !error_message.empty())) { - *warnings << "parse error setting '" << opt.name << "' to '" << val - << "'"; + if (_get_val_from_conf_file(my_sections, opt.name, val)) { + continue; + } + std::string error_message; + if (_set_val(values, tracker, val, opt, CONF_FILE, &error_message) < 0) { + if (warnings != nullptr) { + *warnings << "parse error setting " << std::quoted(opt.name) + << " to " << std::quoted(val); if (!error_message.empty()) { *warnings << " (" << error_message << ")"; } - *warnings << std::endl; + *warnings << '\n'; } } } + cf.check_old_style_section_names({"mds", "mon", "osd"}, cerr); + return 0; +} - // subsystems? - for (int o = 0; o < subsys.get_num(); o++) { - std::string as_option("debug_"); - as_option += subsys.get_name(o); - std::string val; - int ret = _get_val_from_conf_file(my_sections, as_option.c_str(), val, false); - if (ret == 0) { - int log, gather; - int r = sscanf(val.c_str(), "%d/%d", &log, &gather); - if (r >= 1) { - if (r < 2) - gather = log; - // cout << "config subsys " << subsys.get_name(o) << " log " << log << " gather " << gather << std::endl; - subsys.set_log_level(o, log); - subsys.set_gather_level(o, gather); - } - } +std::list +md_config_t::get_conffile_paths(const ConfigValues& values, + const char *conf_files_str, + std::ostream *warnings, + int flags) const +{ + if (!conf_files_str) { + const char *c = getenv("CEPH_CONF"); + if (c) { + conf_files_str = c; + } else { + if (flags & CINIT_FLAG_NO_DEFAULT_CONFIG_FILE) + return {}; + conf_files_str = CEPH_CONF_FILE_DEFAULT; + } } - // Warn about section names that look like old-style section names - std::deque < std::string > old_style_section_names; - for (ConfFile::const_section_iter_t s = cf.sections_begin(); - s != cf.sections_end(); ++s) { - const string &str(s->first); - if (((str.find("mds") == 0) || (str.find("mon") == 0) || - (str.find("osd") == 0)) && (str.size() > 3) && (str[3] != '.')) { - old_style_section_names.push_back(str); + std::list paths; + get_str_list(conf_files_str, ";,", paths); + for (auto i = paths.begin(); i != paths.end(); ) { + string& path = *i; + if (path.find("$data_dir") != path.npos && + data_dir_option.empty()) { + // useless $data_dir item, skip + i = paths.erase(i); + } else { + early_expand_meta(values, path, warnings); + ++i; } } - if (!old_style_section_names.empty()) { - ostringstream oss; - cerr << "ERROR! old-style section name(s) found: "; - string sep; - for (std::deque < std::string >::const_iterator os = old_style_section_names.begin(); - os != old_style_section_names.end(); ++os) { - cerr << sep << *os; - sep = ", "; + return paths; +} + +std::string md_config_t::get_cluster_name(const char* conffile) +{ + if (conffile) { + // If cluster name is not set yet, use the prefix of the + // basename of configuration file as cluster name. + if (fs::path path{conffile}; path.extension() == ".conf") { + return path.stem().string(); + } else { + // If the configuration file does not follow $cluster.conf + // convention, we do the last try and assign the cluster to + // 'ceph'. + return "ceph"; } - cerr << ". Please use the new style section names that include a period."; + } else { + // set the cluster name to 'ceph' when configuration file is not specified. + return "ceph"; } - return 0; } -void md_config_t::parse_env() +void md_config_t::parse_env(unsigned entity_type, + ConfigValues& values, + const ConfigTracker& tracker, + const char *args_var) { - Mutex::Locker l(lock); - if (internal_safe_to_start_threads) + if (safe_to_start_threads) return; - if (getenv("CEPH_KEYRING")) { - set_val_or_die("keyring", getenv("CEPH_KEYRING")); + if (!args_var) { + args_var = "CEPH_ARGS"; + } + if (auto s = getenv("CEPH_KEYRING"); s) { + string err; + _set_val(values, tracker, s, *find_option("keyring"), CONF_ENV, &err); + } + if (auto dir = getenv("CEPH_LIB"); dir) { + for (auto name : { "erasure_code_dir", "plugin_dir", "osd_class_dir" }) { + std::string err; + const Option *o = find_option(name); + ceph_assert(o); + _set_val(values, tracker, dir, *o, CONF_ENV, &err); + } + } + + // Apply pod memory limits: + // + // There are two types of resource requests: `limits` and `requests`. + // + // - Requests: Used by the K8s scheduler to determine on which nodes to + // schedule the pods. This helps spread the pods to different nodes. This + // value should be conservative in order to make sure all the pods are + // schedulable. This corresponds to POD_MEMORY_REQUEST (set by the Rook + // CRD) and is the target memory utilization we try to maintain for daemons + // that respect it. + // + // If POD_MEMORY_REQUEST is present, we use it as the target. + // + // - Limits: At runtime, the container runtime (and Linux) will use the + // limits to see if the pod is using too many resources. In that case, the + // pod will be killed/restarted automatically if the pod goes over the limit. + // This should be higher than what is specified for requests (potentially + // much higher). This corresponds to the cgroup memory limit that will + // trigger the Linux OOM killer. + // + // If POD_MEMORY_LIMIT is present, we use it as the /default/ value for + // the target, which means it will only apply if the *_memory_target option + // isn't set via some other path (e.g., POD_MEMORY_REQUEST, or the cluster + // config, or whatever.) + // + // Here are the documented best practices: + // https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#motivation-for-cpu-requests-and-limits + // + // When the operator creates the CephCluster CR, it will need to generate the + // desired requests and limits. As long as we are conservative in our choice + // for requests and generous with the limits we should be in a good place to + // get started. + // + // The support in Rook is already there for applying the limits as seen in + // these links. + // + // Rook docs on the resource requests and limits: + // https://rook.io/docs/rook/v1.0/ceph-cluster-crd.html#cluster-wide-resources-configuration-settings + // Example CR settings: + // https://github.com/rook/rook/blob/6d2ef936698593036185aabcb00d1d74f9c7bfc1/cluster/examples/kubernetes/ceph/cluster.yaml#L90 + // + uint64_t pod_limit = 0, pod_request = 0; + if (auto pod_lim = getenv("POD_MEMORY_LIMIT"); pod_lim) { + string err; + uint64_t v = atoll(pod_lim); + if (v) { + switch (entity_type) { + case CEPH_ENTITY_TYPE_OSD: + { + double cgroup_ratio = get_val( + values, "osd_memory_target_cgroup_limit_ratio"); + if (cgroup_ratio > 0.0) { + pod_limit = v * cgroup_ratio; + // set osd_memory_target *default* based on cgroup limit, so that + // it can be overridden by any explicit settings elsewhere. + set_val_default(values, tracker, + "osd_memory_target", stringify(pod_limit)); + } + } + } + } + } + if (auto pod_req = getenv("POD_MEMORY_REQUEST"); pod_req) { + if (uint64_t v = atoll(pod_req); v) { + pod_request = v; + } + } + if (pod_request && pod_limit) { + // If both LIMIT and REQUEST are set, ensure that we use the + // min of request and limit*ratio. This is important + // because k8s set set LIMIT == REQUEST if only LIMIT is + // specified, and we want to apply the ratio in that case, + // even though REQUEST is present. + pod_request = std::min(pod_request, pod_limit); } + if (pod_request) { + string err; + switch (entity_type) { + case CEPH_ENTITY_TYPE_OSD: + _set_val(values, tracker, stringify(pod_request), + *find_option("osd_memory_target"), + CONF_ENV, &err); + break; + } + } + + if (getenv(args_var)) { + vector env_args; + env_to_vec(env_args, args_var); + parse_argv(values, tracker, env_args, CONF_ENV); + } +} + +void md_config_t::show_config(const ConfigValues& values, + std::ostream& out) const +{ + _show_config(values, &out, nullptr); } -void md_config_t::show_config(std::ostream& out) +void md_config_t::show_config(const ConfigValues& values, + Formatter *f) const { - Mutex::Locker l(lock); - _show_config(&out, NULL); + _show_config(values, nullptr, f); } -void md_config_t::show_config(Formatter *f) +void md_config_t::config_options(Formatter *f) const { - Mutex::Locker l(lock); - _show_config(NULL, f); + f->open_array_section("options"); + for (const auto& i: schema) { + f->dump_object("option", i.second); + } + f->close_section(); } -void md_config_t::_show_config(std::ostream *out, Formatter *f) +void md_config_t::_show_config(const ConfigValues& values, + std::ostream *out, Formatter *f) const { if (out) { - *out << "name = " << name << std::endl; - *out << "cluster = " << cluster << std::endl; + *out << "name = " << values.name << std::endl; + *out << "cluster = " << values.cluster << std::endl; } if (f) { - f->dump_string("name", stringify(name)); - f->dump_string("cluster", cluster); - } - for (int o = 0; o < subsys.get_num(); o++) { - if (out) - *out << "debug_" << subsys.get_name(o) - << " = " << subsys.get_log_level(o) - << "/" << subsys.get_gather_level(o) << std::endl; + f->dump_string("name", stringify(values.name)); + f->dump_string("cluster", values.cluster); + } + for (const auto& i: schema) { + const Option &opt = i.second; + string val; + conf_stringify(_get_val(values, opt), &val); + if (out) { + *out << opt.name << " = " << val << std::endl; + } if (f) { - ostringstream ss; - std::string debug_name = "debug_"; - debug_name += subsys.get_name(o); - ss << subsys.get_log_level(o) - << "/" << subsys.get_gather_level(o); - f->dump_string(debug_name.c_str(), ss.str()); + f->dump_string(opt.name.c_str(), val); } } - for (auto& opt: *config_options) { - char *buf; - _get_val(opt.name, &buf, -1); - if (out) - *out << opt.name << " = " << buf << std::endl; - if (f) - f->dump_string(opt.name, buf); - free(buf); - } } -int md_config_t::parse_argv(std::vector& args) +int md_config_t::parse_argv(ConfigValues& values, + const ConfigTracker& tracker, + std::vector& args, int level) { - Mutex::Locker l(lock); - if (internal_safe_to_start_threads) { + if (safe_to_start_threads) { return -ENOSYS; } - bool show_config = false; - bool show_config_value = false; - string show_config_value_arg; - // In this function, don't change any parts of the configuration directly. // Instead, use set_val to set them. This will allow us to send the proper // observer notifications later. @@ -454,133 +655,145 @@ int md_config_t::parse_argv(std::vector& args) _exit(0); } else if (ceph_argparse_flag(args, i, "--show_config", (char*)NULL)) { - show_config = true; + do_show_config = true; } else if (ceph_argparse_witharg(args, i, &val, "--show_config_value", (char*)NULL)) { - show_config_value = true; - show_config_value_arg = val; + do_show_config_value = val; + } + else if (ceph_argparse_flag(args, i, "--no-mon-config", (char*)NULL)) { + values.no_mon_config = true; + } + else if (ceph_argparse_flag(args, i, "--mon-config", (char*)NULL)) { + values.no_mon_config = false; } else if (ceph_argparse_flag(args, i, "--foreground", "-f", (char*)NULL)) { - set_val_or_die("daemonize", "false"); + set_val_or_die(values, tracker, "daemonize", "false"); } else if (ceph_argparse_flag(args, i, "-d", (char*)NULL)) { - set_val_or_die("daemonize", "false"); - set_val_or_die("log_file", ""); - set_val_or_die("log_to_stderr", "true"); - set_val_or_die("err_to_stderr", "true"); - set_val_or_die("log_to_syslog", "false"); + set_val_or_die(values, tracker, "fuse_debug", "true"); + set_val_or_die(values, tracker, "daemonize", "false"); + set_val_or_die(values, tracker, "log_file", ""); + set_val_or_die(values, tracker, "log_to_stderr", "true"); + set_val_or_die(values, tracker, "err_to_stderr", "true"); + set_val_or_die(values, tracker, "log_to_syslog", "false"); } // Some stuff that we wanted to give universal single-character options for // Careful: you can burn through the alphabet pretty quickly by adding // to this list. else if (ceph_argparse_witharg(args, i, &val, "--monmap", "-M", (char*)NULL)) { - set_val_or_die("monmap", val.c_str()); + set_val_or_die(values, tracker, "monmap", val.c_str()); } else if (ceph_argparse_witharg(args, i, &val, "--mon_host", "-m", (char*)NULL)) { - set_val_or_die("mon_host", val.c_str()); + set_val_or_die(values, tracker, "mon_host", val.c_str()); } else if (ceph_argparse_witharg(args, i, &val, "--bind", (char*)NULL)) { - set_val_or_die("public_addr", val.c_str()); + set_val_or_die(values, tracker, "public_addr", val.c_str()); } else if (ceph_argparse_witharg(args, i, &val, "--keyfile", "-K", (char*)NULL)) { - set_val_or_die("keyfile", val.c_str()); + bufferlist bl; + string err; + int r; + if (val == "-") { + r = bl.read_fd(STDIN_FILENO, 1024); + } else { + r = bl.read_file(val.c_str(), &err); + } + if (r >= 0) { + string k(bl.c_str(), bl.length()); + set_val_or_die(values, tracker, "key", k.c_str()); + } } else if (ceph_argparse_witharg(args, i, &val, "--keyring", "-k", (char*)NULL)) { - set_val_or_die("keyring", val.c_str()); + set_val_or_die(values, tracker, "keyring", val.c_str()); } else if (ceph_argparse_witharg(args, i, &val, "--client_mountpoint", "-r", (char*)NULL)) { - set_val_or_die("client_mountpoint", val.c_str()); + set_val_or_die(values, tracker, "client_mountpoint", val.c_str()); } else { - parse_option(args, i, NULL); + int r = parse_option(values, tracker, args, i, NULL, level); + if (r < 0) { + return r; + } } } + // meta expands could have modified anything. Copy it all out again. + update_legacy_vals(values); + return 0; +} - if (show_config) { - expand_all_meta(); - _show_config(&cout, NULL); +void md_config_t::do_argv_commands(const ConfigValues& values) const +{ + + if (do_show_config) { + _show_config(values, &cout, NULL); _exit(0); } - if (show_config_value) { - char *buf = 0; - int r = _get_val(show_config_value_arg.c_str(), &buf, -1); + if (do_show_config_value.size()) { + string val; + int r = conf_stringify(_get_val(values, do_show_config_value, 0, &cerr), + &val); if (r < 0) { if (r == -ENOENT) - std::cerr << "failed to get config option '" << - show_config_value_arg << "': option not found" << std::endl; + std::cerr << "failed to get config option '" + << do_show_config_value << "': option not found" << std::endl; else - std::cerr << "failed to get config option '" << - show_config_value_arg << "': " << cpp_strerror(r) << std::endl; + std::cerr << "failed to get config option '" + << do_show_config_value << "': " << cpp_strerror(r) + << std::endl; _exit(1); } - string s = buf; - expand_meta(s, &std::cerr); - std::cout << s << std::endl; + std::cout << val << std::endl; _exit(0); } - - return 0; } -int md_config_t::parse_option(std::vector& args, - std::vector::iterator& i, - ostream *oss) +int md_config_t::parse_option(ConfigValues& values, + const ConfigTracker& tracker, + std::vector& args, + std::vector::iterator& i, + ostream *oss, + int level) { int ret = 0; - int o; + size_t o = 0; std::string val; - // subsystems? - for (o = 0; o < subsys.get_num(); o++) { - std::string as_option("--"); - as_option += "debug_"; - as_option += subsys.get_name(o); - if (ceph_argparse_witharg(args, i, &val, - as_option.c_str(), (char*)NULL)) { - int log, gather; - int r = sscanf(val.c_str(), "%d/%d", &log, &gather); - if (r >= 1) { - if (r < 2) - gather = log; - // cout << "subsys " << subsys.get_name(o) << " log " << log << " gather " << gather << std::endl; - subsys.set_log_level(o, log); - subsys.set_gather_level(o, gather); - if (oss) - *oss << "debug_" << subsys.get_name(o) << "=" << log << "/" << gather << " "; - } - break; - } - } - if (o < subsys.get_num()) { - return ret; - } - - const char *option_name = nullptr; + std::string option_name; std::string error_message; o = 0; - for (auto& opt_ref: *config_options) { + for (const auto& opt_iter: schema) { + const Option &opt = opt_iter.second; ostringstream err; - config_option const *opt = &opt_ref; std::string as_option("--"); - as_option += opt->name; - option_name = opt->name; - if (opt->type == OPT_BOOL) { + as_option += opt.name; + option_name = opt.name; + if (ceph_argparse_witharg( + args, i, &val, err, + string(string("--default-") + opt.name).c_str(), (char*)NULL)) { + if (!err.str().empty()) { + error_message = err.str(); + ret = -EINVAL; + break; + } + ret = _set_val(values, tracker, val, opt, CONF_DEFAULT, &error_message); + break; + } else if (opt.type == Option::TYPE_BOOL) { int res; if (ceph_argparse_binary_flag(args, i, &res, oss, as_option.c_str(), (char*)NULL)) { if (res == 0) - ret = set_val_impl("false", opt, &error_message); + ret = _set_val(values, tracker, "false", opt, level, &error_message); else if (res == 1) - ret = set_val_impl("true", opt, &error_message); + ret = _set_val(values, tracker, "true", opt, level, &error_message); else ret = res; break; } else { std::string no("--no-"); - no += opt->name; + no += opt.name; if (ceph_argparse_flag(args, i, no.c_str(), (char*)NULL)) { - ret = set_val_impl("false", opt, &error_message); + ret = _set_val(values, tracker, "false", opt, level, &error_message); break; } } @@ -591,19 +804,14 @@ int md_config_t::parse_option(std::vector& args, ret = -EINVAL; break; } - if (oss && ((!opt->is_safe()) && - (observers.find(opt->name) == observers.end()))) { - *oss << "You cannot change " << opt->name << " using injectargs.\n"; - return -ENOSYS; - } - ret = set_val_impl(val, opt, &error_message); + ret = _set_val(values, tracker, val, opt, level, &error_message); break; } ++o; } - if (ret != 0 || !error_message.empty()) { - assert(option_name); + if (ret < 0 || !error_message.empty()) { + ceph_assert(!option_name.empty()); if (oss) { *oss << "Parse error setting " << option_name << " to '" << val << "' using injectargs"; @@ -621,111 +829,42 @@ int md_config_t::parse_option(std::vector& args, } } - if (o == (int)config_options->size()) { + if (o == schema.size()) { // ignore ++i; } - return ret; + return ret >= 0 ? 0 : ret; } -int md_config_t::parse_injectargs(std::vector& args, +int md_config_t::parse_injectargs(ConfigValues& values, + const ConfigTracker& tracker, + std::vector& args, std::ostream *oss) { - assert(lock.is_locked()); int ret = 0; for (std::vector::iterator i = args.begin(); i != args.end(); ) { - int r = parse_option(args, i, oss); + int r = parse_option(values, tracker, args, i, oss, CONF_OVERRIDE); if (r < 0) ret = r; } return ret; } -void md_config_t::apply_changes(std::ostream *oss) +void md_config_t::set_safe_to_start_threads() { - Mutex::Locker l(lock); - /* - * apply changes until the cluster name is assigned - */ - if (cluster.size()) - _apply_changes(oss); + safe_to_start_threads = true; } -bool md_config_t::_internal_field(const string& s) +void md_config_t::_clear_safe_to_start_threads() { - if (s == "internal_safe_to_start_threads") - return true; - return false; + safe_to_start_threads = false; } -void md_config_t::_apply_changes(std::ostream *oss) -{ - /* Maps observers to the configuration options that they care about which - * have changed. */ - typedef std::map < md_config_obs_t*, std::set > rev_obs_map_t; - - expand_all_meta(); - - // create the reverse observer mapping, mapping observers to the set of - // changed keys that they'll get. - rev_obs_map_t robs; - std::set empty_set; - char buf[128]; - char *bufptr = (char*)buf; - for (changed_set_t::const_iterator c = changed.begin(); - c != changed.end(); ++c) { - const std::string &key(*c); - pair < obs_map_t::iterator, obs_map_t::iterator > - range(observers.equal_range(key)); - if ((oss) && - (!_get_val(key.c_str(), &bufptr, sizeof(buf))) && - !_internal_field(key)) { - (*oss) << key << " = '" << buf << "' "; - if (range.first == range.second) { - (*oss) << "(not observed, change may require restart) "; - } - } - for (obs_map_t::iterator r = range.first; r != range.second; ++r) { - rev_obs_map_t::value_type robs_val(r->second, empty_set); - pair < rev_obs_map_t::iterator, bool > robs_ret(robs.insert(robs_val)); - std::set &keys(robs_ret.first->second); - keys.insert(key); - } - } - - changed.clear(); - - // Make any pending observer callbacks - for (rev_obs_map_t::const_iterator r = robs.begin(); r != robs.end(); ++r) { - md_config_obs_t *obs = r->first; - obs->handle_conf_change(this, r->second); - } - -} - -void md_config_t::call_all_observers() -{ - std::map > obs; - { - Mutex::Locker l(lock); - - expand_all_meta(); - - for (auto r = observers.begin(); r != observers.end(); ++r) { - obs[r->second].insert(r->first); - } - } - for (auto p = obs.begin(); - p != obs.end(); - ++p) { - p->first->handle_conf_change(this, p->second); - } -} - -int md_config_t::injectargs(const std::string& s, std::ostream *oss) +int md_config_t::injectargs(ConfigValues& values, + const ConfigTracker& tracker, + const std::string& s, std::ostream *oss) { int ret; - Mutex::Locker l(lock); char b[s.length()+1]; strcpy(b, s.c_str()); std::vector nargs; @@ -738,7 +877,7 @@ int md_config_t::injectargs(const std::string& s, std::ostream *oss) *p++ = 0; while (*p && *p == ' ') p++; } - ret = parse_injectargs(nargs, oss); + ret = parse_injectargs(values, tracker, nargs, oss); if (!nargs.empty()) { *oss << " failed to parse arguments: "; std::string prefix; @@ -750,184 +889,411 @@ int md_config_t::injectargs(const std::string& s, std::ostream *oss) *oss << "\n"; ret = -EINVAL; } - _apply_changes(oss); + update_legacy_vals(values); return ret; } -void md_config_t::set_val_or_die(const char *key, const char *val) +void md_config_t::set_val_or_die(ConfigValues& values, + const ConfigTracker& tracker, + const std::string_view key, + const std::string &val) { - int ret = set_val(key, val); - assert(ret == 0); + std::stringstream err; + int ret = set_val(values, tracker, key, val, &err); + if (ret != 0) { + std::cerr << "set_val_or_die(" << key << "): " << err.str(); + } + ceph_assert(ret == 0); } -struct is_integer_member : public boost::static_visitor { - template, int>::type = 0> - bool operator()(const T md_config_t::* /* member_ptr */) const { - return true; +int md_config_t::set_val(ConfigValues& values, + const ConfigTracker& tracker, + const std::string_view key, const char *val, + std::stringstream *err_ss) +{ + if (key.empty()) { + if (err_ss) *err_ss << "No key specified"; + return -EINVAL; } - template::value, int>::type = 0> - bool operator()(const T md_config_t::* /* member_ptr */) const { - return false; + if (!val) { + return -EINVAL; } -}; -struct is_float_member : public boost::static_visitor { - template, int>::type = 0> - bool operator()(const T md_config_t::* /* member_ptr */) const { - return true; - } - template::value, int>::type = 0> - bool operator()(const T md_config_t::* /* member_ptr */) const { - return false; + std::string v(val); + + string k(ConfFile::normalize_key_name(key)); + + const auto &opt_iter = schema.find(k); + if (opt_iter != schema.end()) { + const Option &opt = opt_iter->second; + std::string error_message; + int r = _set_val(values, tracker, v, opt, CONF_OVERRIDE, &error_message); + if (r >= 0) { + if (err_ss) *err_ss << "Set " << opt.name << " to " << v; + r = 0; + } else { + if (err_ss) *err_ss << error_message; + } + return r; } -}; -bool md_config_t::config_option::is_safe() const { - // for now integer and floating point options considered thread safe - return safe || - boost::apply_visitor(is_integer_member(), md_member_ptr) || - boost::apply_visitor(is_float_member(), md_member_ptr); + if (err_ss) *err_ss << "Configuration option not found: '" << key << "'"; + return -ENOENT; } -md_config_t::config_option const *md_config_t::find_config_option(const std::string &normalized_key) const +int md_config_t::rm_val(ConfigValues& values, const std::string_view key) { - auto opt_it = std::find_if(config_options->begin(), - config_options->end(), - [normalized_key](const config_option &opt) -> bool { - return strcmp(normalized_key.c_str(), opt.name) == 0; - }); - return config_options->end() == opt_it ? nullptr : &(*opt_it); + return _rm_val(values, key, CONF_OVERRIDE); } -int md_config_t::set_val(const char *key, const char *val, bool meta) +void md_config_t::get_defaults_bl(const ConfigValues& values, + bufferlist *bl) { - Mutex::Locker l(lock); - if (!key) - return -EINVAL; - if (!val) - return -EINVAL; - - std::string v(val); - if (meta) - expand_meta(v, &std::cerr); - - string k(ConfFile::normalize_key_name(key)); - - // subsystems? - if (strncmp(k.c_str(), "debug_", 6) == 0) { - for (int o = 0; o < subsys.get_num(); o++) { - std::string as_option = "debug_" + subsys.get_name(o); - if (k == as_option) { - int log, gather; - int r = sscanf(v.c_str(), "%d/%d", &log, &gather); - if (r >= 1) { - if (r < 2) - gather = log; - // cout << "subsys " << subsys.get_name(o) << " log " << log << " gather " << gather << std::endl; - subsys.set_log_level(o, log); - subsys.set_gather_level(o, gather); - return 0; - } - return -EINVAL; + if (defaults_bl.length() == 0) { + uint32_t n = 0; + bufferlist bl; + for (const auto &i : schema) { + ++n; + encode(i.second.name, bl); + auto [value, found] = values.get_value(i.second.name, CONF_DEFAULT); + if (found) { + encode(Option::to_str(value), bl); + } else { + string val; + conf_stringify(_get_val_default(i.second), &val); + encode(val, bl); } - } + } + encode(n, defaults_bl); + defaults_bl.claim_append(bl); } + *bl = defaults_bl; +} - config_option const *opt = find_config_option(k); - if (opt) { - if ((!opt->is_safe()) && internal_safe_to_start_threads) { - // If threads have been started and the option is not thread safe - if (observers.find(opt->name) == observers.end()) { - // And there is no observer to safely change it... - // You lose. - return -ENOSYS; +void md_config_t::get_config_bl( + const ConfigValues& values, + uint64_t have_version, + bufferlist *bl, + uint64_t *got_version) +{ + if (values_bl.length() == 0) { + uint32_t n = 0; + bufferlist bl; + values.for_each([&](auto& name, auto& configs) { + if (name == "fsid" || + name == "host") { + return; + } + ++n; + encode(name, bl); + encode((uint32_t)configs.size(), bl); + for (auto& j : configs) { + encode(j.first, bl); + encode(Option::to_str(j.second), bl); + } + }); + // make sure overridden items appear, and include the default value + for (auto& i : ignored_mon_values) { + if (values.contains(i.first)) { + continue; } + if (i.first == "fsid" || + i.first == "host") { + continue; + } + const Option *opt = find_option(i.first); + if (!opt) { + continue; + } + ++n; + encode(i.first, bl); + encode((uint32_t)1, bl); + encode((int32_t)CONF_DEFAULT, bl); + string val; + conf_stringify(_get_val_default(*opt), &val); + encode(val, bl); } + encode(n, values_bl); + values_bl.claim_append(bl); + encode(ignored_mon_values, values_bl); + ++values_bl_version; + } + if (have_version != values_bl_version) { + *bl = values_bl; + *got_version = values_bl_version; + } +} - std::string error_message; - int r = set_val_impl(v, opt, &error_message); - return r; +std::optional md_config_t::get_val_default(std::string_view key) +{ + std::string val; + const Option *opt = find_option(key); + if (opt && (conf_stringify(_get_val_default(*opt), &val) == 0)) { + return std::make_optional(std::move(val)); } + return std::nullopt; +} - // couldn't find a configuration option with key 'key' - return -ENOENT; +int md_config_t::get_val(const ConfigValues& values, + const std::string_view key, char **buf, int len) const +{ + string k(ConfFile::normalize_key_name(key)); + return _get_val_cstr(values, k, buf, len); } +int md_config_t::get_val( + const ConfigValues& values, + const std::string_view key, + std::string *val) const +{ + return conf_stringify(get_val_generic(values, key), val); +} -int md_config_t::get_val(const char *key, char **buf, int len) const +Option::value_t md_config_t::get_val_generic( + const ConfigValues& values, + const std::string_view key) const { - Mutex::Locker l(lock); - return _get_val(key, buf,len); + return _get_val(values, key); } -md_config_t::config_value_t md_config_t::get_val_generic(const char *key) const +Option::value_t md_config_t::_get_val( + const ConfigValues& values, + const std::string_view key, + expand_stack_t *stack, + std::ostream *err) const { - Mutex::Locker l(lock); - return _get_val(key); + if (key.empty()) { + return {}; + } + + // In key names, leading and trailing whitespace are not significant. + string k(ConfFile::normalize_key_name(key)); + + const Option *o = find_option(k); + if (!o) { + // not a valid config option + return {}; + } + + return _get_val(values, *o, stack, err); } -class get_value_generic_visitor : public boost::static_visitor { - md_config_t const *conf; -public: - explicit get_value_generic_visitor(md_config_t const *conf_) : conf(conf_) { } - template md_config_t::config_value_t operator()(const T md_config_t::* member_ptr) { - return md_config_t::config_value_t(conf->*member_ptr); +Option::value_t md_config_t::_get_val( + const ConfigValues& values, + const Option& o, + expand_stack_t *stack, + std::ostream *err) const +{ + expand_stack_t a_stack; + if (!stack) { + stack = &a_stack; } -}; + return _expand_meta(values, + _get_val_nometa(values, o), + &o, stack, err); +} -md_config_t::config_value_t md_config_t::_get_val(const char *key) const +Option::value_t md_config_t::_get_val_nometa(const ConfigValues& values, + const Option& o) const { - assert(lock.is_locked()); + if (auto [value, found] = values.get_value(o.name, -1); found) { + return value; + } else { + return _get_val_default(o); + } +} - if (!key) - return config_value_t(invalid_config_value_t()); +const Option::value_t& md_config_t::_get_val_default(const Option& o) const +{ + bool has_daemon_default = (o.daemon_value != Option::value_t{}); + if (is_daemon && has_daemon_default) { + return o.daemon_value; + } else { + return o.value; + } +} - // In key names, leading and trailing whitespace are not significant. - string k(ConfFile::normalize_key_name(key)); +void md_config_t::early_expand_meta( + const ConfigValues& values, + std::string &val, + std::ostream *err) const +{ + expand_stack_t stack; + Option::value_t v = _expand_meta(values, + Option::value_t(val), + nullptr, &stack, err); + conf_stringify(v, &val); +} - config_option const *opt = find_config_option(k); - if (!opt) { - return config_value_t(invalid_config_value_t()); +bool md_config_t::finalize_reexpand_meta(ConfigValues& values, + const ConfigTracker& tracker) +{ + std::vector reexpands; + reexpands.swap(may_reexpand_meta); + for (auto& name : reexpands) { + // always refresh the options if they are in the may_reexpand_meta + // map, because the options may have already been expanded with old + // meta. + const auto &opt_iter = schema.find(name); + ceph_assert(opt_iter != schema.end()); + const Option &opt = opt_iter->second; + _refresh(values, opt); } - get_value_generic_visitor gvv(this); - return boost::apply_visitor(gvv, opt->md_member_ptr); + + return !may_reexpand_meta.empty(); } -int md_config_t::_get_val(const char *key, std::string *value) const { - assert(lock.is_locked()); +Option::value_t md_config_t::_expand_meta( + const ConfigValues& values, + const Option::value_t& in, + const Option *o, + expand_stack_t *stack, + std::ostream *err) const +{ + //cout << __func__ << " in '" << in << "' stack " << stack << std::endl; + if (!stack) { + return in; + } + const auto str = std::get_if(&in); + if (!str) { + // strings only! + return in; + } - std::string normalized_key(ConfFile::normalize_key_name(key)); - config_value_t config_value = _get_val(normalized_key.c_str()); - if (!boost::get(&config_value)) { - ostringstream oss; - if (bool *flag = boost::get(&config_value)) { - oss << (*flag ? "true" : "false"); - } else if (float *fp = boost::get(&config_value)) { - oss << std::fixed << *fp ; - } else if (double *dp = boost::get(&config_value)) { - oss << std::fixed << *dp ; + auto pos = str->find('$'); + if (pos == std::string::npos) { + // no substitutions! + return in; + } + + if (o) { + stack->push_back(make_pair(o, &in)); + } + string out; + decltype(pos) last_pos = 0; + while (pos != std::string::npos) { + ceph_assert((*str)[pos] == '$'); + if (pos > last_pos) { + out += str->substr(last_pos, pos - last_pos); + } + + // try to parse the variable name into var, either \$\{(.+)\} or + // \$[a-z\_]+ + const char *valid_chars = "abcdefghijklmnopqrstuvwxyz_"; + string var; + size_t endpos = 0; + if ((*str)[pos+1] == '{') { + // ...${foo_bar}... + endpos = str->find_first_not_of(valid_chars, pos + 2); + if (endpos != std::string::npos && + (*str)[endpos] == '}') { + var = str->substr(pos + 2, endpos - pos - 2); + endpos++; + } } else { - oss << config_value; + // ...$foo... + endpos = str->find_first_not_of(valid_chars, pos + 1); + if (endpos != std::string::npos) + var = str->substr(pos + 1, endpos - pos - 1); + else + var = str->substr(pos + 1); } - *value = oss.str(); - return 0; + last_pos = endpos; + + if (!var.size()) { + out += '$'; + } else { + //cout << " found var " << var << std::endl; + // special metavariable? + if (var == "type") { + out += values.name.get_type_name(); + } else if (var == "cluster") { + out += values.cluster; + } else if (var == "name") { + out += values.name.to_cstr(); + } else if (var == "host") { + if (values.host == "") { + out += ceph_get_short_hostname(); + } else { + out += values.host; + } + } else if (var == "num") { + out += values.name.get_id().c_str(); + } else if (var == "id") { + out += values.name.get_id(); + } else if (var == "pid") { + char *_pid = getenv("PID"); + if (_pid) { + out += _pid; + } else { + out += stringify(getpid()); + } + if (o) { + may_reexpand_meta.push_back(o->name); + } + } else if (var == "cctid") { + out += stringify((unsigned long long)this); + } else if (var == "home") { + const char *home = getenv("HOME"); + out = home ? std::string(home) : std::string(); + } else if (var == "programdata") { + const char *home = getenv("ProgramData"); + out = home ? std::string(home) : std::string(); + }else { + if (var == "data_dir") { + var = data_dir_option; + } + const Option *o = find_option(var); + if (!o) { + out += str->substr(pos, endpos - pos); + } else { + auto match = std::find_if( + stack->begin(), stack->end(), + [o](pair& item) { + return item.first == o; + }); + if (match != stack->end()) { + // substitution loop; break the cycle + if (err) { + *err << "variable expansion loop at " << var << "=" + << Option::to_str(*match->second) << "\n" + << "expansion stack:\n"; + for (auto i = stack->rbegin(); i != stack->rend(); ++i) { + *err << i->first->name << "=" + << Option::to_str(*i->second) << "\n"; + } + } + return Option::value_t(std::string("$") + o->name); + } else { + // recursively evaluate! + string n; + conf_stringify(_get_val(values, *o, stack, err), &n); + out += n; + } + } + } + } + pos = str->find('$', last_pos); } - return -ENOENT; + if (last_pos != std::string::npos) { + out += str->substr(last_pos); + } + if (o) { + stack->pop_back(); + } + + return Option::value_t(out); } -int md_config_t::_get_val(const char *key, char **buf, int len) const +int md_config_t::_get_val_cstr( + const ConfigValues& values, + const std::string& key, char **buf, int len) const { - assert(lock.is_locked()); - - if (!key) + if (key.empty()) return -EINVAL; - string val ; - if (!_get_val(key, &val)) { + string val; + if (conf_stringify(_get_val(values, key), &val) == 0) { int l = val.length() + 1; if (len == -1) { *buf = (char*)malloc(l); @@ -940,20 +1306,6 @@ int md_config_t::_get_val(const char *key, char **buf, int len) const return (l > len) ? -ENAMETOOLONG : 0; } - string k(ConfFile::normalize_key_name(key)); - // subsys? - for (int o = 0; o < subsys.get_num(); o++) { - std::string as_option = "debug_" + subsys.get_name(o); - if (k == as_option) { - if (len == -1) { - *buf = (char*)malloc(20); - len = 20; - } - int l = snprintf(*buf, len, "%d/%d", subsys.get_log_level(o), subsys.get_gather_level(o)); - return (l == len) ? -ENAMETOOLONG : 0; - } - } - // couldn't find a configuration option with key 'k' return -ENOENT; } @@ -962,16 +1314,14 @@ void md_config_t::get_all_keys(std::vector *keys) const { const std::string negative_flag_prefix("no_"); keys->clear(); - keys->reserve(config_options->size()); - for (auto& opt: *config_options) { + keys->reserve(schema.size()); + for (const auto &i: schema) { + const Option &opt = i.second; keys->push_back(opt.name); - if (opt.type == OPT_BOOL) { + if (opt.type == Option::TYPE_BOOL) { keys->push_back(negative_flag_prefix + opt.name); } } - for (int i = 0; i < subsys.get_num(); ++i) { - keys->push_back("debug_" + subsys.get_name(i)); - } } /* The order of the sections here is important. The first section in the @@ -979,417 +1329,249 @@ void md_config_t::get_all_keys(std::vector *keys) const { * looking. The lowest priority section is the one we look in only if all * others had nothing. This should always be the global section. */ -void md_config_t::get_my_sections(std::vector §ions) const +std::vector +md_config_t::get_my_sections(const ConfigValues& values) const { - Mutex::Locker l(lock); - _get_my_sections(sections); -} - -void md_config_t::_get_my_sections(std::vector §ions) const -{ - assert(lock.is_locked()); - sections.push_back(name.to_str()); - - sections.push_back(name.get_type_name()); - - sections.push_back("global"); + return {values.name.to_str(), + values.name.get_type_name().data(), + "global"}; } // Return a list of all sections int md_config_t::get_all_sections(std::vector §ions) const { - Mutex::Locker l(lock); - for (ConfFile::const_section_iter_t s = cf.sections_begin(); - s != cf.sections_end(); ++s) { - sections.push_back(s->first); + for (auto [section_name, section] : cf) { + sections.push_back(section_name); + std::ignore = section; } return 0; } -int md_config_t::get_val_from_conf_file(const std::vector §ions, - const char *key, std::string &out, bool emeta) const +int md_config_t::get_val_from_conf_file( + const ConfigValues& values, + const std::vector §ions, + const std::string_view key, + std::string &out, + bool emeta) const { - Mutex::Locker l(lock); - return _get_val_from_conf_file(sections, key, out, emeta); + int r = _get_val_from_conf_file(sections, key, out); + if (r < 0) { + return r; + } + if (emeta) { + expand_stack_t stack; + auto v = _expand_meta(values, Option::value_t(out), nullptr, &stack, nullptr); + conf_stringify(v, &out); + } + return 0; } -int md_config_t::_get_val_from_conf_file(const std::vector §ions, - const char *key, std::string &out, bool emeta) const +int md_config_t::_get_val_from_conf_file( + const std::vector §ions, + const std::string_view key, + std::string &out) const { - assert(lock.is_locked()); - std::vector ::const_iterator s = sections.begin(); - std::vector ::const_iterator s_end = sections.end(); - for (; s != s_end; ++s) { - int ret = cf.read(s->c_str(), key, out); + for (auto &s : sections) { + int ret = cf.read(s, key, out); if (ret == 0) { - if (emeta) - expand_meta(out, &std::cerr); return 0; - } - else if (ret != -ENOENT) + } else if (ret != -ENOENT) { return ret; + } } return -ENOENT; } -int md_config_t::set_val_impl(const std::string &val, config_option const *opt, - std::string *error_message) +int md_config_t::_set_val( + ConfigValues& values, + const ConfigTracker& observers, + const std::string &raw_val, + const Option &opt, + int level, + std::string *error_message) { - assert(lock.is_locked()); - std::string value(val); - if (opt->validator) { - int r = opt->validator(&value, error_message); - if (r < 0) { - return r; + Option::value_t new_value; + ceph_assert(error_message); + int r = opt.parse_value(raw_val, &new_value, error_message); + if (r < 0) { + return r; + } + + // unsafe runtime change? + if (!opt.can_update_at_runtime() && + safe_to_start_threads && + !observers.is_tracking(opt.name)) { + // accept value if it is not actually a change + if (new_value != _get_val_nometa(values, opt)) { + *error_message = string("Configuration option '") + opt.name + + "' may not be modified at runtime"; + return -EPERM; } } - int ret = set_val_raw(value.c_str(), opt); - if (ret) - return ret; - changed.insert(opt->name); - return 0; + // Apply the value to its entry in the `values` map + auto result = values.set_value(opt.name, std::move(new_value), level); + switch (result) { + case ConfigValues::SET_NO_CHANGE: + break; + case ConfigValues::SET_NO_EFFECT: + values_bl.clear(); + break; + case ConfigValues::SET_HAVE_EFFECT: + values_bl.clear(); + _refresh(values, opt); + break; + } + return result; } -template struct strtox_helper; - -template<> struct strtox_helper { - static inline void apply(const char *val, float &x, std::string &err) { - x = strict_strtof(val, &err); +void md_config_t::_refresh(ConfigValues& values, const Option& opt) +{ + // Apply the value to its legacy field, if it has one + auto legacy_ptr_iter = legacy_values.find(std::string(opt.name)); + if (legacy_ptr_iter != legacy_values.end()) { + update_legacy_val(values, opt, legacy_ptr_iter->second); } -}; - -template<> struct strtox_helper { - static inline void apply(const char *val, double &x, std::string &err) { - x = strict_strtod(val, &err); + // Was this a debug_* option update? + if (opt.subsys >= 0) { + string actual_val; + conf_stringify(_get_val(values, opt), &actual_val); + values.set_logging(opt.subsys, actual_val.c_str()); + } else { + // normal option, advertise the change. + values.changed.insert(opt.name); } -}; - -template static inline int strict_strtox(const char *val, T &x) { - std::string err; - strtox_helper::apply(val, x, err); - return err.empty() ? 0 : -EINVAL; } -class set_value_visitor : public boost::static_visitor { - md_config_t const *conf; - const char *val; -public: - explicit set_value_visitor(md_config_t const *conf_, const char *val_) : - conf(conf_), val(val_) { } - - int operator()(const std::string md_config_t::* member_ptr) { - auto *ptr = const_cast(&(conf->*member_ptr)); - *ptr = val ? val : ""; - return 0; +int md_config_t::_rm_val(ConfigValues& values, + const std::string_view key, + int level) +{ + if (schema.count(key) == 0) { + return -EINVAL; } - - int operator()(const bool md_config_t::* member_ptr) { - bool *ptr = const_cast(&(conf->*member_ptr)); - if (strcasecmp(val, "false") == 0) { - *ptr = false; - } else if (strcasecmp(val, "true") == 0) { - *ptr = true; - } else { - std::string err; - int b = strict_strtol(val, 10, &err); - if (!err.empty()) { - return -EINVAL; - } - *ptr = !!b; - } - return 0; + auto ret = values.rm_val(std::string{key}, level); + if (ret < 0) { + return ret; } - - // type has parse() member function - template, int>::type = 0> - int operator()(const T md_config_t::* member_ptr) { - T *obj = const_cast(&(conf->*member_ptr)); - if (!obj->parse(val)) { - return -EINVAL; - } - return 0; - } + if (ret == ConfigValues::SET_HAVE_EFFECT) { + _refresh(values, *find_option(key)); + } + values_bl.clear(); + return 0; +} - // float, double - template, int>::type = 0> - int operator()(const T md_config_t::* member_ptr) { - T* ptr = const_cast(&(conf->*member_ptr)); - return strict_strtox(val, *ptr); - } +namespace { +template +struct get_size_visitor +{ + get_size_visitor() {} - // integers - template::value && - !boost::is_same::value, int>::type = 0> - int operator()(const T md_config_t::* member_ptr) { - std::string err; - T f = strict_si_cast(val, &err); - if (!err.empty()) { - return -EINVAL; - } - T *ptr = const_cast(&(conf->*member_ptr)); - *ptr = f; - return 0; + template + Size operator()(const T&) const { + return -1; + } + Size operator()(const Option::size_t& sz) const { + return static_cast(sz.value); + } + Size operator()(const Size& v) const { + return v; } }; -int md_config_t::set_val_raw(const char *val, config_option const *opt) +/** + * Handles assigning from a variant-of-types to a variant-of-pointers-to-types + */ +class assign_visitor { - assert(lock.is_locked()); - set_value_visitor svv(this, val); - return boost::apply_visitor(svv, opt->md_member_ptr); -} + ConfigValues *conf; + Option::value_t val; + public: + + assign_visitor(ConfigValues *conf_, Option::value_t val_) + : conf(conf_), val(val_) + {} -static const char *CONF_METAVARIABLES[] = { - "data_dir", // put this first: it may contain some of the others - "cluster", "type", "name", "host", "num", "id", "pid", "cctid" + template + void operator()(T ConfigValues::* ptr) const + { + T *member = const_cast(&(conf->*(ptr))); + + *member = std::get(val); + } + void operator()(uint64_t ConfigValues::* ptr) const + { + using T = uint64_t; + auto member = const_cast(&(conf->*(ptr))); + *member = std::visit(get_size_visitor{}, val); + } + void operator()(int64_t ConfigValues::* ptr) const + { + using T = int64_t; + auto member = const_cast(&(conf->*(ptr))); + *member = std::visit(get_size_visitor{}, val); + } }; -static const int NUM_CONF_METAVARIABLES = - (sizeof(CONF_METAVARIABLES) / sizeof(CONF_METAVARIABLES[0])); +} // anonymous namespace -void md_config_t::expand_all_meta() +void md_config_t::update_legacy_vals(ConfigValues& values) { - // Expand all metavariables - ostringstream oss; - for (auto& opt: *config_options) { - std::string *str; - opt.conf_ptr(str, this); - if (str) { - list stack; - expand_meta(*str, &opt, stack, &oss); - } + for (const auto &i : legacy_values) { + const auto &name = i.first; + const auto &option = schema.at(name); + auto ptr = i.second; + update_legacy_val(values, option, ptr); } - cerr << oss.str(); } -bool md_config_t::expand_meta(std::string &origval, - std::ostream *oss) const +void md_config_t::update_legacy_val(ConfigValues& values, + const Option &opt, + md_config_t::member_ptr_t member_ptr) { - list stack; - return expand_meta(origval, NULL, stack, oss); + Option::value_t v = _get_val(values, opt); + std::visit(assign_visitor(&values, v), member_ptr); } -bool md_config_t::expand_meta(std::string &origval, - config_option const *opt, - std::list stack, - std::ostream *oss) const +static void dump(Formatter *f, int level, Option::value_t in) { - assert(lock.is_locked()); - - // no $ means no variable expansion is necessary - if (origval.find("$") == string::npos) - return false; - - // ignore an expansion loop and create a human readable - // message about it - if (opt) { - for (list::iterator i = stack.begin(); - i != stack.end(); - ++i) { - if (strcmp(opt->name, (*i)->name) == 0) { - *oss << "variable expansion loop at " - << opt->name << "=" << origval << std::endl; - *oss << "expansion stack: " << std::endl; - for (list::iterator j = stack.begin(); - j != stack.end(); - ++j) { - *oss << (*j)->name << "=" << *((*j)->conf_ptr(this)) << std::endl; - } - return false; - } - } - } - - if (opt) - stack.push_front(opt); - - bool found_meta = false; - string out; - string val = origval; - for (string::size_type s = 0; s < val.size(); ) { - if (val[s] != '$') { - out += val[s++]; - continue; - } - - // try to parse the variable name into var, either \$\{(.+)\} or - // \$[a-z\_]+ - const char *valid_chars = "abcdefghijklmnopqrstuvwxyz_"; - string var; - size_t endpos = 0; - if (val[s+1] == '{') { - // ...${foo_bar}... - endpos = val.find_first_not_of(valid_chars, s+2); - if (endpos != std::string::npos && - val[endpos] == '}') { - var = val.substr(s+2, endpos-s-2); - endpos++; - } - } else { - // ...$foo... - endpos = val.find_first_not_of(valid_chars, s+1); - if (endpos != std::string::npos) - var = val.substr(s+1, endpos-s-1); - else - var = val.substr(s+1); - } - - bool expanded = false; - if (var.length()) { - // special metavariable? - for (int i = 0; i < NUM_CONF_METAVARIABLES; ++i) { - if (var != CONF_METAVARIABLES[i]) - continue; - //cout << " meta match of " << var << " " << CONF_METAVARIABLES[i] << std::endl; - if (var == "type") - out += name.get_type_name(); - else if (var == "cluster") - out += cluster; - else if (var == "name") - out += name.to_cstr(); - else if (var == "host") - { - if (host == "") - out += ceph_get_short_hostname(); - else - out += host; - } - else if (var == "num") - out += name.get_id().c_str(); - else if (var == "id") - out += name.get_id().c_str(); - else if (var == "pid") - out += stringify(getpid()); - else if (var == "cctid") - out += stringify((unsigned long long)this); - else if (var == "data_dir") { - if (data_dir_option.length()) { - char *vv = NULL; - _get_val(data_dir_option.c_str(), &vv, -1); - string tmp = vv; - free(vv); - expand_meta(tmp, NULL, stack, oss); - out += tmp; - } else { - // this isn't really right, but it'll result in a mangled - // non-existent path that will fail any search list - out += "$data_dir"; - } - } else - ceph_abort(); // unreachable - expanded = true; - } - - if (!expanded) { - // config option? - for (auto& opt: *config_options) { - if (var == opt.name) { - string *origval; - opt.conf_ptr(origval, const_cast(this)); - if (origval) { - expand_meta(*origval, &opt, stack, oss); - out += *origval; - } else { - char *vv = NULL; - _get_val(opt.name, &vv, -1); - out += vv; - free(vv); - } - expanded = true; - break; - } - } - } - } - - if (expanded) { - found_meta = true; - s = endpos; - } else { - out += val[s++]; - } + if (const auto v = std::get_if(&in)) { + f->dump_bool(ceph_conf_level_name(level), *v); + } else if (const auto v = std::get_if(&in)) { + f->dump_int(ceph_conf_level_name(level), *v); + } else if (const auto v = std::get_if(&in)) { + f->dump_unsigned(ceph_conf_level_name(level), *v); + } else if (const auto v = std::get_if(&in)) { + f->dump_float(ceph_conf_level_name(level), *v); + } else { + f->dump_stream(ceph_conf_level_name(level)) << Option::to_str(in); } - // override the original value with the expanded value - origval = out; - return found_meta; } void md_config_t::diff( - const md_config_t *other, - map > *diff, - set *unknown) -{ - diff_helper(other, diff, unknown); -} -void md_config_t::diff( - const md_config_t *other, - map > *diff, - set *unknown, const string& setting) -{ - diff_helper(other, diff, unknown, setting); -} - -void md_config_t::diff_helper( - const md_config_t *other, - map > *diff, - set *unknown, const string& setting) + const ConfigValues& values, + Formatter *f, + string name) const { - Mutex::Locker l(lock); - - char local_buf[4096]; - char other_buf[4096]; - for (auto& opt : *config_options) { - if (!setting.empty()) { - if (setting != opt.name) { - continue; - } + values.for_each([this, f, &values] (auto& name, auto& configs) { + if (configs.empty()) { + return; } - memset(local_buf, 0, sizeof(local_buf)); - memset(other_buf, 0, sizeof(other_buf)); - - char *other_val = other_buf; - int err = other->get_val(opt.name, &other_val, sizeof(other_buf)); - if (err < 0) { - if (err == -ENOENT) { - unknown->insert(opt.name); - } - continue; + f->open_object_section(std::string{name}.c_str()); + const Option *o = find_option(name); + if (configs.size() && + configs.begin()->first != CONF_DEFAULT) { + // show compiled-in default only if an override default wasn't provided + dump(f, CONF_DEFAULT, _get_val_default(*o)); } - - char *local_val = local_buf; - err = _get_val(opt.name, &local_val, sizeof(local_buf)); - if (err != 0) - continue; - - if (strcmp(local_val, other_val)) - diff->insert(make_pair(opt.name, make_pair(local_val, other_val))); - else if (!setting.empty()) { - diff->insert(make_pair(opt.name, make_pair(local_val, other_val))); - break; + for (auto& j : configs) { + dump(f, j.first, j.second); } - } + dump(f, CONF_FINAL, _get_val(values, *o)); + f->close_section(); + }); } -void md_config_t::complain_about_parse_errors(CephContext *cct) +void md_config_t::complain_about_parse_error(CephContext *cct) { - ::complain_about_parse_errors(cct, &parse_errors); -} - -void md_config_t::validate_default_settings() { - Mutex::Locker l(lock); - for (auto &opt : *config_options) { - // normalize config defaults using their validator - if (opt.validator) { - std::string value; - int r = _get_val(opt.name, &value); - assert(r == 0); - - std::string error_message; - r = set_val_impl(value.c_str(), &opt, &error_message); - assert(r == 0); - } - } + ::complain_about_parse_error(cct, parse_error); }