ceph/src/common/options.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "acconfig.h"
   5 #include "options.h"
   6 #include "common/Formatter.h"
   7
   8 // Helpers for validators
   9 #include "include/stringify.h"
  10 #include "include/common_fwd.h"
  11 #include <boost/algorithm/string.hpp>
  12 #include <boost/lexical_cast.hpp>
  13 #include <regex>
  14
  15 // Definitions for enums
  16 #include "common/perf_counters.h"
  17
  18 // rbd feature validation
  19 #include "librbd/Features.h"
  20
  21 using std::ostream;
  22 using std::ostringstream;
  23
  24 using ceph::Formatter;
  25 using ceph::parse_timespan;
  26
  27 namespace {
  28 class printer : public boost::static_visitor<> {
  29   ostream& out;
  30 public:
  31   explicit printer(ostream& os)
  32     : out(os) {}
  33   template<typename T>
  34   void operator()(const T& v) const {
  35     out << v;
  36   }
  37   void operator()(boost::blank blank) const {
  38     return;
  39   }
  40   void operator()(bool v) const {
  41     out << (v ? "true" : "false");
  42   }
  43   void operator()(double v) const {
  44     out << std::fixed << v << std::defaultfloat;
  45   }
  46   void operator()(const Option::size_t& v) const {
  47     out << v.value;
  48   }
  49   void operator()(const std::chrono::seconds v) const {
  50     out << v.count();
  51   }
  52   void operator()(const std::chrono::milliseconds v) const {
  53     out << v.count();
  54   }
  55 };
  56 }
  57
  58 ostream& operator<<(ostream& os, const Option::value_t& v) {
  59   printer p{os};
  60   v.apply_visitor(p);
  61   return os;
  62 }
  63
  64 void Option::dump_value(const char *field_name,
  65     const Option::value_t &v, Formatter *f) const
  66 {
  67   if (boost::get<boost::blank>(&v)) {
  68     // This should be nil but Formatter doesn't allow it.
  69     f->dump_string(field_name, "");
  70     return;
  71   }
  72   switch (type) {
  73   case TYPE_INT:
  74     f->dump_int(field_name, boost::get<int64_t>(v)); break;
  75   case TYPE_UINT:
  76     f->dump_unsigned(field_name, boost::get<uint64_t>(v)); break;
  77   case TYPE_STR:
  78     f->dump_string(field_name, boost::get<std::string>(v)); break;
  79   case TYPE_FLOAT:
  80     f->dump_float(field_name, boost::get<double>(v)); break;
  81   case TYPE_BOOL:
  82     f->dump_bool(field_name, boost::get<bool>(v)); break;
  83   default:
  84     f->dump_stream(field_name) << v; break;
  85   }
  86 }
  87
  88 int Option::pre_validate(std::string *new_value, std::string *err) const
  89 {
  90   if (validator) {
  91     return validator(new_value, err);
  92   } else {
  93     return 0;
  94   }
  95 }
  96
  97 int Option::validate(const Option::value_t &new_value, std::string *err) const
  98 {
  99   // Generic validation: min
 100   if (!boost::get<boost::blank>(&(min))) {
 101     if (new_value < min) {
 102       std::ostringstream oss;
 103       oss << "Value '" << new_value << "' is below minimum " << min;
 104       *err = oss.str();
 105       return -EINVAL;
 106     }
 107   }
 108
 109   // Generic validation: max
 110   if (!boost::get<boost::blank>(&(max))) {
 111     if (new_value > max) {
 112       std::ostringstream oss;
 113       oss << "Value '" << new_value << "' exceeds maximum " << max;
 114       *err = oss.str();
 115       return -EINVAL;
 116     }
 117   }
 118
 119   // Generic validation: enum
 120   if (!enum_allowed.empty() && type == Option::TYPE_STR) {
 121     auto found = std::find(enum_allowed.begin(), enum_allowed.end(),
 122                            boost::get<std::string>(new_value));
 123     if (found == enum_allowed.end()) {
 124       std::ostringstream oss;
 125       oss << "'" << new_value << "' is not one of the permitted "
 126                  "values: " << joinify(enum_allowed.begin(),
 127                                        enum_allowed.end(),
 128                                        std::string(", "));
 129       *err = oss.str();
 130       return -EINVAL;
 131     }
 132   }
 133
 134   return 0;
 135 }
 136
 137 int Option::parse_value(
 138   const std::string& raw_val,
 139   value_t *out,
 140   std::string *error_message,
 141   std::string *normalized_value) const
 142 {
 143   std::string val = raw_val;
 144
 145   int r = pre_validate(&val, error_message);
 146   if (r != 0) {
 147     return r;
 148   }
 149
 150   if (type == Option::TYPE_INT) {
 151     int64_t f = strict_si_cast<int64_t>(val.c_str(), error_message);
 152     if (!error_message->empty()) {
 153       return -EINVAL;
 154     }
 155     *out = f;
 156   } else if (type == Option::TYPE_UINT) {
 157     uint64_t f = strict_si_cast<uint64_t>(val.c_str(), error_message);
 158     if (!error_message->empty()) {
 159       return -EINVAL;
 160     }
 161     *out = f;
 162   } else if (type == Option::TYPE_STR) {
 163     *out = val;
 164   } else if (type == Option::TYPE_FLOAT) {
 165     double f = strict_strtod(val.c_str(), error_message);
 166     if (!error_message->empty()) {
 167       return -EINVAL;
 168     } else {
 169       *out = f;
 170     }
 171   } else if (type == Option::TYPE_BOOL) {
 172     bool b = strict_strtob(val.c_str(), error_message);
 173     if (!error_message->empty()) {
 174       return -EINVAL;
 175     } else {
 176       *out = b;
 177     }
 178   } else if (type == Option::TYPE_ADDR) {
 179     entity_addr_t addr;
 180     if (!addr.parse(val.c_str())){
 181       return -EINVAL;
 182     }
 183     *out = addr;
 184   } else if (type == Option::TYPE_ADDRVEC) {
 185     entity_addrvec_t addr;
 186     if (!addr.parse(val.c_str())){
 187       return -EINVAL;
 188     }
 189     *out = addr;
 190   } else if (type == Option::TYPE_UUID) {
 191     uuid_d uuid;
 192     if (!uuid.parse(val.c_str())) {
 193       return -EINVAL;
 194     }
 195     *out = uuid;
 196   } else if (type == Option::TYPE_SIZE) {
 197     Option::size_t sz{strict_iecstrtoll(val.c_str(), error_message)};
 198     if (!error_message->empty()) {
 199       return -EINVAL;
 200     }
 201     *out = sz;
 202   } else if (type == Option::TYPE_SECS) {
 203     try {
 204       *out = parse_timespan(val);
 205     } catch (const std::invalid_argument& e) {
 206       *error_message = e.what();
 207       return -EINVAL;
 208     }
 209   } else if (type == Option::TYPE_MILLISECS) {
 210     try {
 211       *out = boost::lexical_cast<uint64_t>(val);
 212     } catch (const boost::bad_lexical_cast& e) {
 213       *error_message = e.what();
 214       return -EINVAL;
 215     }
 216   } else {
 217     ceph_abort();
 218   }
 219
 220   r = validate(*out, error_message);
 221   if (r != 0) {
 222     return r;
 223   }
 224
 225   if (normalized_value) {
 226     *normalized_value = to_str(*out);
 227   }
 228   return 0;
 229 }
 230
 231 void Option::dump(Formatter *f) const
 232 {
 233   f->dump_string("name", name);
 234
 235   f->dump_string("type", type_to_str(type));
 236
 237   f->dump_string("level", level_to_str(level));
 238
 239   f->dump_string("desc", desc);
 240   f->dump_string("long_desc", long_desc);
 241
 242   dump_value("default", value, f);
 243   dump_value("daemon_default", daemon_value, f);
 244
 245   f->open_array_section("tags");
 246   for (const auto t : tags) {
 247     f->dump_string("tag", t);
 248   }
 249   f->close_section();
 250
 251   f->open_array_section("services");
 252   for (const auto s : services) {
 253     f->dump_string("service", s);
 254   }
 255   f->close_section();
 256
 257   f->open_array_section("see_also");
 258   for (const auto sa : see_also) {
 259     f->dump_string("see_also", sa);
 260   }
 261   f->close_section();
 262
 263   if (type == TYPE_STR) {
 264     f->open_array_section("enum_values");
 265     for (const auto &ea : enum_allowed) {
 266       f->dump_string("enum_value", ea);
 267     }
 268     f->close_section();
 269   }
 270
 271   dump_value("min", min, f);
 272   dump_value("max", max, f);
 273
 274   f->dump_bool("can_update_at_runtime", can_update_at_runtime());
 275
 276   f->open_array_section("flags");
 277   if (has_flag(FLAG_RUNTIME)) {
 278     f->dump_string("option", "runtime");
 279   }
 280   if (has_flag(FLAG_NO_MON_UPDATE)) {
 281     f->dump_string("option", "no_mon_update");
 282   }
 283   if (has_flag(FLAG_STARTUP)) {
 284     f->dump_string("option", "startup");
 285   }
 286   if (has_flag(FLAG_CLUSTER_CREATE)) {
 287     f->dump_string("option", "cluster_create");
 288   }
 289   if (has_flag(FLAG_CREATE)) {
 290     f->dump_string("option", "create");
 291   }
 292   f->close_section();
 293 }
 294
 295 std::string Option::to_str(const Option::value_t& v)
 296 {
 297   return stringify(v);
 298 }
 299
 300 void Option::print(ostream *out) const
 301 {
 302   *out << name << " - " << desc << "\n";
 303   *out << "  (" << type_to_str(type) << ", " << level_to_str(level) << ")\n";
 304   if (!boost::get<boost::blank>(&daemon_value)) {
 305     *out << "  Default (non-daemon): " << stringify(value) << "\n";
 306     *out << "  Default (daemon): " << stringify(daemon_value) << "\n";
 307   } else {
 308     *out << "  Default: " << stringify(value) << "\n";
 309   }
 310   if (!enum_allowed.empty()) {
 311     *out << "  Possible values: ";
 312     for (auto& i : enum_allowed) {
 313       *out << " " << stringify(i);
 314     }
 315     *out << "\n";
 316   }
 317   if (!boost::get<boost::blank>(&min)) {
 318     *out << "  Minimum: " << stringify(min) << "\n"
 319          << "  Maximum: " << stringify(max) << "\n";
 320   }
 321   *out << "  Can update at runtime: "
 322        << (can_update_at_runtime() ? "true" : "false") << "\n";
 323   if (!services.empty()) {
 324     *out << "  Services: " << services << "\n";
 325   }
 326   if (!tags.empty()) {
 327     *out << "  Tags: " << tags << "\n";
 328   }
 329   if (!see_also.empty()) {
 330     *out << "  See also: " << see_also << "\n";
 331   }
 332
 333   if (long_desc.size()) {
 334     *out << "\n" << long_desc << "\n";
 335   }
 336 }
 337
 338 constexpr unsigned long long operator"" _min (unsigned long long min) {
 339   return min * 60;
 340 }
 341 constexpr unsigned long long operator"" _hr (unsigned long long hr) {
 342   return hr * 60 * 60;
 343 }
 344 constexpr unsigned long long operator"" _day (unsigned long long day) {
 345   return day * 24 * 60 * 60;
 346 }
 347 constexpr unsigned long long operator"" _K (unsigned long long n) {
 348   return n << 10;
 349 }
 350 constexpr unsigned long long operator"" _M (unsigned long long n) {
 351   return n << 20;
 352 }
 353 constexpr unsigned long long operator"" _G (unsigned long long n) {
 354   return n << 30;
 355 }
 356 constexpr unsigned long long operator"" _T (unsigned long long n) {
 357   return n << 40;
 358 }
 359
 360 std::vector<Option> get_global_options() {
 361   return std::vector<Option>({
 362     Option("host", Option::TYPE_STR, Option::LEVEL_BASIC)
 363     .set_description("local hostname")
 364     .set_long_description("if blank, ceph assumes the short hostname (hostname -s)")
 365     .set_flag(Option::FLAG_NO_MON_UPDATE)
 366     .add_service("common")
 367     .add_tag("network"),
 368
 369     Option("fsid", Option::TYPE_UUID, Option::LEVEL_BASIC)
 370     .set_description("cluster fsid (uuid)")
 371     .set_flag(Option::FLAG_NO_MON_UPDATE)
 372     .set_flag(Option::FLAG_STARTUP)
 373     .add_service("common")
 374     .add_tag("service"),
 375
 376     Option("public_addr", Option::TYPE_ADDR, Option::LEVEL_BASIC)
 377     .set_description("public-facing address to bind to")
 378     .set_flag(Option::FLAG_STARTUP)
 379     .add_service({"mon", "mds", "osd", "mgr"}),
 380
 381     Option("public_addrv", Option::TYPE_ADDRVEC, Option::LEVEL_BASIC)
 382     .set_description("public-facing address to bind to")
 383     .set_flag(Option::FLAG_STARTUP)
 384     .add_service({"mon", "mds", "osd", "mgr"}),
 385
 386     Option("public_bind_addr", Option::TYPE_ADDR, Option::LEVEL_ADVANCED)
 387     .set_default(entity_addr_t())
 388     .set_flag(Option::FLAG_STARTUP)
 389     .add_service("mon")
 390     .set_description(""),
 391
 392     Option("cluster_addr", Option::TYPE_ADDR, Option::LEVEL_BASIC)
 393     .set_description("cluster-facing address to bind to")
 394     .add_service("osd")
 395     .set_flag(Option::FLAG_STARTUP)
 396     .add_tag("network"),
 397
 398     Option("public_network", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 399     .add_service({"mon", "mds", "osd", "mgr"})
 400     .set_flag(Option::FLAG_STARTUP)
 401     .add_tag("network")
 402     .set_description("Network(s) from which to choose a public address to bind to"),
 403
 404     Option("public_network_interface", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 405     .add_service({"mon", "mds", "osd", "mgr"})
 406     .add_tag("network")
 407     .set_flag(Option::FLAG_STARTUP)
 408     .set_description("Interface name(s) from which to choose an address from a public_network to bind to; public_network must also be specified.")
 409     .add_see_also("public_network"),
 410
 411     Option("cluster_network", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 412     .add_service("osd")
 413     .set_flag(Option::FLAG_STARTUP)
 414     .add_tag("network")
 415     .set_description("Network(s) from which to choose a cluster address to bind to"),
 416
 417     Option("cluster_network_interface", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 418     .add_service({"mon", "mds", "osd", "mgr"})
 419     .set_flag(Option::FLAG_STARTUP)
 420     .add_tag("network")
 421     .set_description("Interface name(s) from which to choose an address from a cluster_network to bind to; cluster_network must also be specified.")
 422     .add_see_also("cluster_network"),
 423
 424     Option("monmap", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 425     .set_description("path to MonMap file")
 426     .set_long_description("This option is normally used during mkfs, but can also "
 427                         "be used to identify which monitors to connect to.")
 428     .set_flag(Option::FLAG_NO_MON_UPDATE)
 429     .add_service("mon")
 430     .set_flag(Option::FLAG_CREATE),
 431
 432     Option("mon_host", Option::TYPE_STR, Option::LEVEL_BASIC)
 433     .set_description("list of hosts or addresses to search for a monitor")
 434     .set_long_description("This is a comma, whitespace, or semicolon separated "
 435                         "list of IP addresses or hostnames. Hostnames are "
 436                         "resolved via DNS and all A or AAAA records are "
 437                         "included in the search list.")
 438     .set_flag(Option::FLAG_NO_MON_UPDATE)
 439     .set_flag(Option::FLAG_STARTUP)
 440     .add_service("common"),
 441
 442     Option("mon_host_override", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 443     .set_description("monitor(s) to use overriding the MonMap")
 444     .set_flag(Option::FLAG_NO_MON_UPDATE)
 445     .set_flag(Option::FLAG_STARTUP)
 446     .add_service("common"),
 447
 448     Option("mon_dns_srv_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 449     .set_default("ceph-mon")
 450     .set_description("name of DNS SRV record to check for monitor addresses")
 451     .set_flag(Option::FLAG_STARTUP)
 452     .add_service("common")
 453     .add_tag("network")
 454     .add_see_also("mon_host"),
 455
 456     Option("container_image", Option::TYPE_STR, Option::LEVEL_BASIC)
 457     .set_description("container image (used by cephadm orchestrator)")
 458     .set_flag(Option::FLAG_STARTUP)
 459     .set_default("docker.io/ceph/daemon-base:latest-pacific-devel"),
 460
 461     Option("no_config_file", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 462     .set_default(false)
 463     .set_flag(Option::FLAG_NO_MON_UPDATE)
 464     .set_flag(Option::FLAG_STARTUP)
 465     .add_service("common")
 466     .add_tag("config")
 467     .set_description("signal that we don't require a config file to be present")
 468     .set_long_description("When specified, we won't be looking for a "
 469                           "configuration file, and will instead expect that "
 470                           "whatever options or values are required for us to "
 471                           "work will be passed as arguments."),
 472
 473     // lockdep
 474     Option("lockdep", Option::TYPE_BOOL, Option::LEVEL_DEV)
 475     .set_description("enable lockdep lock dependency analyzer")
 476     .set_flag(Option::FLAG_NO_MON_UPDATE)
 477     .set_flag(Option::FLAG_STARTUP)
 478     .add_service("common"),
 479
 480     Option("lockdep_force_backtrace", Option::TYPE_BOOL, Option::LEVEL_DEV)
 481     .set_description("always gather current backtrace at every lock")
 482     .set_flag(Option::FLAG_STARTUP)
 483     .add_service("common")
 484     .add_see_also("lockdep"),
 485
 486     Option("run_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 487     .set_default("/var/run/ceph")
 488     .set_flag(Option::FLAG_STARTUP)
 489     .set_description("path for the 'run' directory for storing pid and socket files")
 490     .add_service("common")
 491     .add_see_also("admin_socket"),
 492
 493     Option("admin_socket", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 494     .set_default("")
 495     .set_daemon_default("$run_dir/$cluster-$name.asok")
 496     .set_flag(Option::FLAG_STARTUP)
 497     .set_description("path for the runtime control socket file, used by the 'ceph daemon' command")
 498     .add_service("common"),
 499
 500     Option("admin_socket_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 501     .set_description("file mode to set for the admin socket file, e.g, '0755'")
 502     .set_flag(Option::FLAG_STARTUP)
 503     .add_service("common")
 504     .add_see_also("admin_socket"),
 505
 506     // daemon
 507     Option("daemonize", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 508     .set_default(false)
 509     .set_daemon_default(true)
 510     .set_description("whether to daemonize (background) after startup")
 511     .set_flag(Option::FLAG_STARTUP)
 512     .set_flag(Option::FLAG_NO_MON_UPDATE)
 513     .add_service({"mon", "mgr", "osd", "mds"})
 514     .add_tag("service")
 515     .add_see_also({"pid_file", "chdir"}),
 516
 517     Option("setuser", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 518     .set_flag(Option::FLAG_STARTUP)
 519     .set_description("uid or user name to switch to on startup")
 520     .set_long_description("This is normally specified by the systemd unit file.")
 521     .add_service({"mon", "mgr", "osd", "mds"})
 522     .add_tag("service")
 523     .add_see_also("setgroup"),
 524
 525     Option("setgroup", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 526     .set_flag(Option::FLAG_STARTUP)
 527     .set_description("gid or group name to switch to on startup")
 528     .set_long_description("This is normally specified by the systemd unit file.")
 529     .add_service({"mon", "mgr", "osd", "mds"})
 530     .add_tag("service")
 531     .add_see_also("setuser"),
 532
 533     Option("setuser_match_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 534     .set_flag(Option::FLAG_STARTUP)
 535     .set_description("if set, setuser/setgroup is condition on this path matching ownership")
 536     .set_long_description("If setuser or setgroup are specified, and this option is non-empty, then the uid/gid of the daemon will only be changed if the file or directory specified by this option has a matching uid and/or gid.  This exists primarily to allow switching to user ceph for OSDs to be conditional on whether the osd data contents have also been chowned after an upgrade.  This is normally specified by the systemd unit file.")
 537     .add_service({"mon", "mgr", "osd", "mds"})
 538     .add_tag("service")
 539     .add_see_also({"setuser", "setgroup"}),
 540
 541     Option("pid_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 542     .set_flag(Option::FLAG_STARTUP)
 543     .set_description("path to write a pid file (if any)")
 544     .add_service({"mon", "mgr", "osd", "mds"})
 545     .add_tag("service"),
 546
 547     Option("chdir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 548     .set_description("path to chdir(2) to after daemonizing")
 549     .set_flag(Option::FLAG_STARTUP)
 550     .set_flag(Option::FLAG_NO_MON_UPDATE)
 551     .add_service({"mon", "mgr", "osd", "mds"})
 552     .add_tag("service")
 553     .add_see_also("daemonize"),
 554
 555     Option("fatal_signal_handlers", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 556     .set_default(true)
 557     .set_flag(Option::FLAG_STARTUP)
 558     .set_description("whether to register signal handlers for SIGABRT etc that dump a stack trace")
 559     .set_long_description("This is normally true for daemons and values for libraries.")
 560     .add_service({"mon", "mgr", "osd", "mds"})
 561     .add_tag("service"),
 562
 563     Option("crash_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 564     .set_flag(Option::FLAG_STARTUP)
 565     .set_default("/var/lib/ceph/crash")
 566     .set_description("Directory where crash reports are archived"),
 567
 568     // restapi
 569     Option("restapi_log_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 570     .set_description("default set by python code"),
 571
 572     Option("restapi_base_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 573     .set_description("default set by python code"),
 574
 575     Option("erasure_code_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 576     .set_default(CEPH_PKGLIBDIR"/erasure-code")
 577     .set_flag(Option::FLAG_STARTUP)
 578     .set_description("directory where erasure-code plugins can be found")
 579     .add_service({"mon", "osd"}),
 580
 581     // logging
 582     Option("log_file", Option::TYPE_STR, Option::LEVEL_BASIC)
 583     .set_default("")
 584     .set_daemon_default("/var/log/ceph/$cluster-$name.log")
 585     .set_description("path to log file")
 586     .add_see_also({"log_to_file",
 587                    "log_to_stderr",
 588                    "err_to_stderr",
 589                    "log_to_syslog",
 590                    "err_to_syslog"}),
 591
 592     Option("log_max_new", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 593     .set_default(1000)
 594     .set_description("max unwritten log entries to allow before waiting to flush to the log")
 595     .add_see_also("log_max_recent"),
 596
 597     Option("log_max_recent", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 598     .set_default(500)
 599     .set_daemon_default(10000)
 600     .set_description("recent log entries to keep in memory to dump in the event of a crash")
 601     .set_long_description("The purpose of this option is to log at a higher debug level only to the in-memory buffer, and write out the detailed log messages only if there is a crash.  Only log entries below the lower log level will be written unconditionally to the log.  For example, debug_osd=1/5 will write everything <= 1 to the log unconditionally but keep entries at levels 2-5 in memory.  If there is a seg fault or assertion failure, all entries will be dumped to the log."),
 602
 603     Option("log_to_file", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 604     .set_default(true)
 605     .set_description("send log lines to a file")
 606     .add_see_also("log_file"),
 607
 608     Option("log_to_stderr", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 609     .set_default(true)
 610     .set_daemon_default(false)
 611     .set_description("send log lines to stderr"),
 612
 613     Option("err_to_stderr", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 614     .set_default(false)
 615     .set_daemon_default(true)
 616     .set_description("send critical error log lines to stderr"),
 617
 618     Option("log_stderr_prefix", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 619     .set_description("String to prefix log messages with when sent to stderr")
 620     .set_long_description("This is useful in container environments when combined with mon_cluster_log_to_stderr.  The mon log prefixes each line with the channel name (e.g., 'default', 'audit'), while log_stderr_prefix can be set to 'debug '.")
 621     .add_see_also("mon_cluster_log_to_stderr"),
 622
 623     Option("log_to_syslog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 624     .set_default(false)
 625     .set_description("send log lines to syslog facility"),
 626
 627     Option("err_to_syslog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 628     .set_default(false)
 629     .set_description("send critical error log lines to syslog facility"),
 630
 631     Option("log_flush_on_exit", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 632     .set_default(false)
 633     .set_description("set a process exit handler to ensure the log is flushed on exit"),
 634
 635     Option("log_stop_at_utilization", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
 636     .set_default(.97)
 637     .set_min_max(0.0, 1.0)
 638     .set_description("stop writing to the log file when device utilization reaches this ratio")
 639     .add_see_also("log_file"),
 640
 641     Option("log_to_graylog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 642     .set_default(false)
 643     .set_description("send log lines to remote graylog server")
 644     .add_see_also({"err_to_graylog",
 645                    "log_graylog_host",
 646                    "log_graylog_port"}),
 647
 648     Option("err_to_graylog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 649     .set_default(false)
 650     .set_description("send critical error log lines to remote graylog server")
 651     .add_see_also({"log_to_graylog",
 652                    "log_graylog_host",
 653                    "log_graylog_port"}),
 654
 655     Option("log_graylog_host", Option::TYPE_STR, Option::LEVEL_BASIC)
 656     .set_default("127.0.0.1")
 657     .set_description("address or hostname of graylog server to log to")
 658     .add_see_also({"log_to_graylog",
 659                    "err_to_graylog",
 660                    "log_graylog_port"}),
 661
 662     Option("log_graylog_port", Option::TYPE_INT, Option::LEVEL_BASIC)
 663     .set_default(12201)
 664     .set_description("port number for the remote graylog server")
 665     .add_see_also("log_graylog_host"),
 666
 667     Option("log_coarse_timestamps", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 668     .set_default(true)
 669     .set_description("timestamp log entries from coarse system clock "
 670                      "to improve performance")
 671     .add_service("common")
 672     .add_tag("performance")
 673     .add_tag("service"),
 674
 675
 676     // unmodified
 677     Option("clog_to_monitors", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 678     .set_default("default=true")
 679     .set_flag(Option::FLAG_RUNTIME)
 680     .set_description("Make daemons send cluster log messages to monitors"),
 681
 682     Option("clog_to_syslog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 683     .set_default("false")
 684     .set_flag(Option::FLAG_RUNTIME)
 685     .set_description("Make daemons send cluster log messages to syslog"),
 686
 687     Option("clog_to_syslog_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 688     .set_default("info")
 689     .set_flag(Option::FLAG_RUNTIME)
 690     .set_description("Syslog level for cluster log messages")
 691     .add_see_also("clog_to_syslog"),
 692
 693     Option("clog_to_syslog_facility", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 694     .set_default("default=daemon audit=local0")
 695     .set_flag(Option::FLAG_RUNTIME)
 696     .set_description("Syslog facility for cluster log messages")
 697     .add_see_also("clog_to_syslog"),
 698
 699     Option("clog_to_graylog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 700     .set_default("false")
 701     .set_flag(Option::FLAG_RUNTIME)
 702     .set_description("Make daemons send cluster log to graylog"),
 703
 704     Option("clog_to_graylog_host", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 705     .set_default("127.0.0.1")
 706     .set_flag(Option::FLAG_RUNTIME)
 707     .set_description("Graylog host to cluster log messages")
 708     .add_see_also("clog_to_graylog"),
 709
 710     Option("clog_to_graylog_port", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 711     .set_default("12201")
 712     .set_flag(Option::FLAG_RUNTIME)
 713     .set_description("Graylog port number for cluster log messages")
 714     .add_see_also("clog_to_graylog"),
 715
 716     Option("mon_cluster_log_to_stderr", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 717     .set_default(false)
 718     .add_service("mon")
 719     .set_flag(Option::FLAG_RUNTIME)
 720     .set_description("Make monitor send cluster log messages to stderr (prefixed by channel)")
 721     .add_see_also("log_stderr_prefix"),
 722
 723     Option("mon_cluster_log_to_syslog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 724     .set_default("default=false")
 725     .set_flag(Option::FLAG_RUNTIME)
 726     .add_service("mon")
 727     .set_description("Make monitor send cluster log messages to syslog"),
 728
 729     Option("mon_cluster_log_to_syslog_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 730     .set_default("info")
 731     .add_service("mon")
 732     .set_flag(Option::FLAG_RUNTIME)
 733     .set_description("Syslog level for cluster log messages")
 734     .add_see_also("mon_cluster_log_to_syslog"),
 735
 736     Option("mon_cluster_log_to_syslog_facility", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 737     .set_default("daemon")
 738     .add_service("mon")
 739     .set_flag(Option::FLAG_RUNTIME)
 740     .set_description("Syslog facility for cluster log messages")
 741     .add_see_also("mon_cluster_log_to_syslog"),
 742
 743     Option("mon_cluster_log_to_file", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 744     .set_default(true)
 745     .set_flag(Option::FLAG_RUNTIME)
 746     .add_service("mon")
 747     .set_description("Make monitor send cluster log messages to file")
 748     .add_see_also("mon_cluster_log_file"),
 749
 750     Option("mon_cluster_log_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 751     .set_default("default=/var/log/ceph/$cluster.$channel.log cluster=/var/log/ceph/$cluster.log")
 752     .set_flag(Option::FLAG_RUNTIME)
 753     .add_service("mon")
 754     .set_description("File(s) to write cluster log to")
 755     .set_long_description("This can either be a simple file name to receive all messages, or a list of key/value pairs where the key is the log channel and the value is the filename, which may include $cluster and $channel metavariables")
 756     .add_see_also("mon_cluster_log_to_file"),
 757
 758     Option("mon_cluster_log_file_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 759     .set_default("debug")
 760     .set_flag(Option::FLAG_RUNTIME)
 761     .add_service("mon")
 762     .set_description("Lowest level to include is cluster log file")
 763     .add_see_also("mon_cluster_log_file"),
 764
 765     Option("mon_cluster_log_to_graylog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 766     .set_default("false")
 767     .set_flag(Option::FLAG_RUNTIME)
 768     .add_service("mon")
 769     .set_description("Make monitor send cluster log to graylog"),
 770
 771     Option("mon_cluster_log_to_graylog_host", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 772     .set_default("127.0.0.1")
 773     .set_flag(Option::FLAG_RUNTIME)
 774     .add_service("mon")
 775     .set_description("Graylog host for cluster log messages")
 776     .add_see_also("mon_cluster_log_to_graylog"),
 777
 778     Option("mon_cluster_log_to_graylog_port", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 779     .set_default("12201")
 780     .set_flag(Option::FLAG_RUNTIME)
 781     .add_service("mon")
 782     .set_description("Graylog port for cluster log messages")
 783     .add_see_also("mon_cluster_log_to_graylog"),
 784
 785     Option("enable_experimental_unrecoverable_data_corrupting_features", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 786     .set_flag(Option::FLAG_RUNTIME)
 787     .set_default("")
 788     .set_description("Enable named (or all with '*') experimental features that may be untested, dangerous, and/or cause permanent data loss"),
 789
 790     Option("plugin_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 791     .set_default(CEPH_PKGLIBDIR)
 792     .set_flag(Option::FLAG_STARTUP)
 793     .add_service({"mon", "osd"})
 794     .set_description("Base directory for dynamically loaded plugins"),
 795
 796     // Compressor
 797     Option("compressor_zlib_isal", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 798     .set_default(false)
 799     .set_description("Use Intel ISA-L accelerated zlib implementation if available"),
 800
 801     Option("compressor_zlib_level", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 802     .set_default(5)
 803     .set_description("Zlib compression level to use"),
 804
 805     Option("compressor_zlib_winsize", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 806     .set_default(-15)
 807     .set_min_max(-15,32)
 808     .set_description("Zlib compression winsize to use"),
 809
 810     Option("compressor_zstd_level", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 811     .set_default(1)
 812     .set_description("Zstd compression level to use"),
 813
 814     Option("qat_compressor_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 815     .set_default(false)
 816     .set_description("Enable Intel QAT acceleration support for compression if available"),
 817
 818     Option("plugin_crypto_accelerator", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 819     .set_default("crypto_isal")
 820     .set_description("Crypto accelerator library to use"),
 821
 822     Option("openssl_engine_opts", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 823     .set_default("")
 824     .set_flag(Option::FLAG_STARTUP)
 825     .set_description("Use engine for specific openssl algorithm")
 826     .set_long_description("Pass opts in this way: engine_id=engine1,dynamic_path=/some/path/engine1.so,default_algorithms=DIGESTS:engine_id=engine2,dynamic_path=/some/path/engine2.so,default_algorithms=CIPHERS,other_ctrl=other_value"),
 827
 828     Option("mempool_debug", Option::TYPE_BOOL, Option::LEVEL_DEV)
 829     .set_default(false)
 830     .set_flag(Option::FLAG_NO_MON_UPDATE)
 831     .set_description(""),
 832
 833     Option("thp", Option::TYPE_BOOL, Option::LEVEL_DEV)
 834     .set_default(false)
 835     .set_flag(Option::FLAG_STARTUP)
 836     .set_description("enable transparent huge page (THP) support")
 837     .set_long_description("Ceph is known to suffer from memory fragmentation due to THP use. This is indicated by RSS usage above configured memory targets. Enabling THP is currently discouraged until selective use of THP by Ceph is implemented."),
 838
 839     Option("key", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 840     .set_default("")
 841     .set_description("Authentication key")
 842     .set_long_description("A CephX authentication key, base64 encoded.  It normally looks something like 'AQAtut9ZdMbNJBAAHz6yBAWyJyz2yYRyeMWDag=='.")
 843     .set_flag(Option::FLAG_STARTUP)
 844     .set_flag(Option::FLAG_NO_MON_UPDATE)
 845     .add_see_also("keyfile")
 846     .add_see_also("keyring"),
 847
 848     Option("keyfile", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 849     .set_default("")
 850     .set_description("Path to a file containing a key")
 851     .set_long_description("The file should contain a CephX authentication key and optionally a trailing newline, but nothing else.")
 852     .set_flag(Option::FLAG_STARTUP)
 853     .set_flag(Option::FLAG_NO_MON_UPDATE)
 854     .add_see_also("key"),
 855
 856     Option("keyring", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 857     .set_default(
 858       "/etc/ceph/$cluster.$name.keyring,/etc/ceph/$cluster.keyring,"
 859       "/etc/ceph/keyring,/etc/ceph/keyring.bin,"
 860   #if defined(__FreeBSD)
 861       "/usr/local/etc/ceph/$cluster.$name.keyring,"
 862       "/usr/local/etc/ceph/$cluster.keyring,"
 863       "/usr/local/etc/ceph/keyring,/usr/local/etc/ceph/keyring.bin,"
 864   #endif
 865     )
 866     .set_description("Path to a keyring file.")
 867     .set_long_description("A keyring file is an INI-style formatted file where the section names are client or daemon names (e.g., 'osd.0') and each section contains a 'key' property with CephX authentication key as the value.")
 868     .set_flag(Option::FLAG_STARTUP)
 869     .set_flag(Option::FLAG_NO_MON_UPDATE)
 870     .add_see_also("key")
 871     .add_see_also("keyfile"),
 872
 873     Option("heartbeat_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 874     .set_default(5)
 875     .set_flag(Option::FLAG_STARTUP)
 876     .set_description("Frequency of internal heartbeat checks (seconds)"),
 877
 878     Option("heartbeat_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 879     .set_default("")
 880     .set_flag(Option::FLAG_STARTUP)
 881     .set_description("File to touch on successful internal heartbeat")
 882     .set_long_description("If set, this file will be touched every time an internal heartbeat check succeeds.")
 883     .add_see_also("heartbeat_interval"),
 884
 885     Option("heartbeat_inject_failure", Option::TYPE_INT, Option::LEVEL_DEV)
 886     .set_default(0)
 887     .set_description(""),
 888
 889     Option("perf", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 890     .set_default(true)
 891     .set_description("Enable internal performance metrics")
 892     .set_long_description("If enabled, collect and expose internal health metrics"),
 893
 894     Option("ms_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 895     .set_flag(Option::FLAG_STARTUP)
 896     .set_default("async+posix")
 897     .set_description("Messenger implementation to use for network communication"),
 898
 899     Option("ms_public_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 900     .set_default("")
 901     .set_flag(Option::FLAG_STARTUP)
 902     .set_description("Messenger implementation to use for the public network")
 903     .set_long_description("If not specified, use ms_type")
 904     .add_see_also("ms_type"),
 905
 906     Option("ms_cluster_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 907     .set_default("")
 908     .set_flag(Option::FLAG_STARTUP)
 909     .set_description("Messenger implementation to use for the internal cluster network")
 910     .set_long_description("If not specified, use ms_type")
 911     .add_see_also("ms_type"),
 912
 913     Option("ms_mon_cluster_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 914     .set_default("secure crc")
 915     .set_flag(Option::FLAG_STARTUP)
 916     .set_description("Connection modes (crc, secure) for intra-mon connections in order of preference")
 917     .add_see_also("ms_mon_service_mode")
 918     .add_see_also("ms_mon_client_mode")
 919     .add_see_also("ms_service_mode")
 920     .add_see_also("ms_cluster_mode")
 921     .add_see_also("ms_client_mode"),
 922
 923     Option("ms_mon_service_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 924     .set_default("secure crc")
 925     .set_flag(Option::FLAG_STARTUP)
 926     .set_description("Allowed connection modes (crc, secure) for connections to mons")
 927     .add_see_also("ms_service_mode")
 928     .add_see_also("ms_mon_cluster_mode")
 929     .add_see_also("ms_mon_client_mode")
 930     .add_see_also("ms_cluster_mode")
 931     .add_see_also("ms_client_mode"),
 932
 933     Option("ms_mon_client_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 934     .set_default("secure crc")
 935     .set_flag(Option::FLAG_STARTUP)
 936     .set_description("Connection modes (crc, secure) for connections from clients to monitors in order of preference")
 937     .add_see_also("ms_mon_service_mode")
 938     .add_see_also("ms_mon_cluster_mode")
 939     .add_see_also("ms_service_mode")
 940     .add_see_also("ms_cluster_mode")
 941     .add_see_also("ms_client_mode"),
 942
 943     Option("ms_cluster_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 944     .set_default("crc secure")
 945     .set_flag(Option::FLAG_STARTUP)
 946     .set_description("Connection modes (crc, secure) for intra-cluster connections in order of preference")
 947     .add_see_also("ms_service_mode")
 948     .add_see_also("ms_client_mode"),
 949
 950     Option("ms_service_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 951     .set_default("crc secure")
 952     .set_flag(Option::FLAG_STARTUP)
 953     .set_description("Allowed connection modes (crc, secure) for connections to daemons")
 954     .add_see_also("ms_cluster_mode")
 955     .add_see_also("ms_client_mode"),
 956
 957     Option("ms_client_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 958     .set_default("crc secure")
 959     .set_flag(Option::FLAG_STARTUP)
 960     .set_description("Connection modes (crc, secure) for connections from clients in order of preference")
 961     .add_see_also("ms_cluster_mode")
 962     .add_see_also("ms_service_mode"),
 963
 964     Option("ms_learn_addr_from_peer", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 965     .set_default(true)
 966     .set_description("Learn address from what IP our first peer thinks we connect from")
 967     .set_long_description("Use the IP address our first peer (usually a monitor) sees that we are connecting from.  This is useful if a client is behind some sort of NAT and we want to see it identified by its local (not NATed) address."),
 968
 969     Option("ms_tcp_nodelay", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 970     .set_default(true)
 971     .set_description("Disable Nagle's algorithm and send queued network traffic immediately"),
 972
 973     Option("ms_tcp_rcvbuf", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 974     .set_default(0)
 975     .set_description("Size of TCP socket receive buffer"),
 976
 977     Option("ms_tcp_prefetch_max_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 978     .set_default(4_K)
 979     .set_description("Maximum amount of data to prefetch out of the socket receive buffer"),
 980
 981     Option("ms_initial_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
 982     .set_default(.2)
 983     .set_description("Initial backoff after a network error is detected (seconds)"),
 984
 985     Option("ms_max_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
 986     .set_default(15.0)
 987     .set_description("Maximum backoff after a network error before retrying (seconds)")
 988     .add_see_also("ms_initial_backoff"),
 989
 990     Option("ms_crc_data", Option::TYPE_BOOL, Option::LEVEL_DEV)
 991     .set_default(true)
 992     .set_description("Set and/or verify crc32c checksum on data payload sent over network"),
 993
 994     Option("ms_crc_header", Option::TYPE_BOOL, Option::LEVEL_DEV)
 995     .set_default(true)
 996     .set_description("Set and/or verify crc32c checksum on header payload sent over network"),
 997
 998     Option("ms_die_on_bad_msg", Option::TYPE_BOOL, Option::LEVEL_DEV)
 999     .set_default(false)
1000     .set_description("Induce a daemon crash/exit when a bad network message is received"),
1001
1002     Option("ms_die_on_unhandled_msg", Option::TYPE_BOOL, Option::LEVEL_DEV)
1003     .set_default(false)
1004     .set_description("Induce a daemon crash/exit when an unrecognized message is received"),
1005
1006     Option("ms_die_on_old_message", Option::TYPE_BOOL, Option::LEVEL_DEV)
1007     .set_default(false)
1008     .set_description("Induce a daemon crash/exit when a old, undecodable message is received"),
1009
1010     Option("ms_die_on_skipped_message", Option::TYPE_BOOL, Option::LEVEL_DEV)
1011     .set_default(false)
1012     .set_description("Induce a daemon crash/exit if sender skips a message sequence number"),
1013
1014     Option("ms_die_on_bug", Option::TYPE_BOOL, Option::LEVEL_DEV)
1015     .set_default(false)
1016     .set_description("Induce a crash/exit on various bugs (for testing purposes)"),
1017
1018     Option("ms_dispatch_throttle_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1019     .set_default(100_M)
1020     .set_description("Limit messages that are read off the network but still being processed"),
1021
1022     Option("ms_bind_ipv4", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1023     .set_default(true)
1024     .set_description("Bind servers to IPv4 address(es)")
1025     .add_see_also("ms_bind_ipv6"),
1026
1027     Option("ms_bind_ipv6", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1028     .set_default(false)
1029     .set_description("Bind servers to IPv6 address(es)")
1030     .add_see_also("ms_bind_ipv4"),
1031
1032     Option("ms_bind_prefer_ipv4", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1033     .set_default(false)
1034     .set_description("Prefer IPV4 over IPV6 address(es)"),
1035
1036     Option("ms_bind_msgr1", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1037     .set_default(true)
1038     .set_description("Bind servers to msgr1 (legacy) protocol address(es)")
1039     .add_see_also("ms_bind_msgr2"),
1040
1041     Option("ms_bind_msgr2", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1042     .set_default(true)
1043     .set_description("Bind servers to msgr2 (nautilus+) protocol address(es)")
1044     .add_see_also("ms_bind_msgr1"),
1045
1046     Option("ms_bind_port_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1047     .set_default(6800)
1048     .set_description("Lowest port number to bind daemon(s) to"),
1049
1050     Option("ms_bind_port_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1051     .set_default(7300)
1052     .set_description("Highest port number to bind daemon(s) to"),
1053
1054     Option("ms_bind_retry_count", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1055   #if !defined(__FreeBSD__)
1056     .set_default(3)
1057   #else
1058     // FreeBSD does not use SO_REAUSEADDR so allow for a bit more time per default
1059     .set_default(6)
1060   #endif
1061     .set_description("Number of attempts to make while bind(2)ing to a port"),
1062
1063     Option("ms_bind_retry_delay", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1064   #if !defined(__FreeBSD__)
1065     .set_default(5)
1066   #else
1067     // FreeBSD does not use SO_REAUSEADDR so allow for a bit more time per default
1068     .set_default(6)
1069   #endif
1070     .set_description("Delay between bind(2) attempts (seconds)"),
1071
1072     Option("ms_bind_before_connect", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1073     .set_default(false)
1074     .set_description("Call bind(2) on client sockets"),
1075
1076     Option("ms_tcp_listen_backlog", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1077     .set_default(512)
1078     .set_description("Size of queue of incoming connections for accept(2)"),
1079
1080
1081     Option("ms_connection_ready_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1082     .set_default(10)
1083     .set_description("Time before we declare a not yet ready connection as dead (seconds)"),
1084
1085     Option("ms_connection_idle_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1086     .set_default(900)
1087     .set_description("Time before an idle connection is closed (seconds)"),
1088
1089     Option("ms_pq_max_tokens_per_priority", Option::TYPE_UINT, Option::LEVEL_DEV)
1090     .set_default(16777216)
1091     .set_description(""),
1092
1093     Option("ms_pq_min_cost", Option::TYPE_SIZE, Option::LEVEL_DEV)
1094     .set_default(65536)
1095     .set_description(""),
1096
1097     Option("ms_inject_socket_failures", Option::TYPE_UINT, Option::LEVEL_DEV)
1098     .set_default(0)
1099     .set_description("Inject a socket failure every Nth socket operation"),
1100
1101     Option("ms_inject_delay_type", Option::TYPE_STR, Option::LEVEL_DEV)
1102     .set_default("")
1103     .set_description("Entity type to inject delays for")
1104     .set_flag(Option::FLAG_RUNTIME),
1105
1106     Option("ms_inject_delay_max", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1107     .set_default(1)
1108     .set_description("Max delay to inject"),
1109
1110     Option("ms_inject_delay_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1111     .set_default(0)
1112     .set_description(""),
1113
1114     Option("ms_inject_internal_delays", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1115     .set_default(0)
1116     .set_description("Inject various internal delays to induce races (seconds)"),
1117
1118     Option("ms_blackhole_osd", Option::TYPE_BOOL, Option::LEVEL_DEV)
1119     .set_default(false)
1120     .set_description(""),
1121
1122     Option("ms_blackhole_mon", Option::TYPE_BOOL, Option::LEVEL_DEV)
1123     .set_default(false)
1124     .set_description(""),
1125
1126     Option("ms_blackhole_mds", Option::TYPE_BOOL, Option::LEVEL_DEV)
1127     .set_default(false)
1128     .set_description(""),
1129
1130     Option("ms_blackhole_mgr", Option::TYPE_BOOL, Option::LEVEL_DEV)
1131     .set_default(false)
1132     .set_description(""),
1133
1134     Option("ms_blackhole_client", Option::TYPE_BOOL, Option::LEVEL_DEV)
1135     .set_default(false)
1136     .set_description(""),
1137
1138     Option("ms_dump_on_send", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1139     .set_default(false)
1140     .set_description("Hexdump message to debug log on message send"),
1141
1142     Option("ms_dump_corrupt_message_level", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1143     .set_default(1)
1144     .set_description("Log level at which to hexdump corrupt messages we receive"),
1145
1146     Option("ms_async_op_threads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1147     .set_default(3)
1148     .set_min_max(1, 24)
1149     .set_description("Threadpool size for AsyncMessenger (ms_type=async)"),
1150
1151     Option("ms_async_max_op_threads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1152     .set_default(5)
1153     .set_description("Maximum threadpool size of AsyncMessenger")
1154     .add_see_also("ms_async_op_threads"),
1155
1156     Option("ms_async_rdma_device_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1157     .set_default("")
1158     .set_description(""),
1159
1160     Option("ms_async_rdma_enable_hugepage", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1161     .set_default(false)
1162     .set_description(""),
1163
1164     Option("ms_async_rdma_buffer_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1165     .set_default(128_K)
1166     .set_description(""),
1167
1168     Option("ms_async_rdma_send_buffers", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1169     .set_default(1_K)
1170     .set_description(""),
1171
1172     Option("ms_async_rdma_receive_buffers", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1173     .set_default(32768)
1174     .set_description(""),
1175
1176     Option("ms_async_rdma_receive_queue_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1177     .set_default(4096)
1178     .set_description(""),
1179
1180     Option("ms_async_rdma_support_srq", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1181     .set_default(true)
1182     .set_description(""),
1183
1184     Option("ms_async_rdma_port_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1185     .set_default(1)
1186     .set_description(""),
1187
1188     Option("ms_async_rdma_polling_us", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1189     .set_default(1000)
1190     .set_description(""),
1191
1192     Option("ms_async_rdma_gid_idx", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1193     .set_default(0)
1194     .set_description("use gid_idx to select GID for choosing RoCEv1 or RoCEv2"),
1195
1196     Option("ms_async_rdma_local_gid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1197     .set_default("")
1198     .set_description(""),
1199
1200     Option("ms_async_rdma_roce_ver", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1201     .set_default(1)
1202     .set_description(""),
1203
1204     Option("ms_async_rdma_sl", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1205     .set_default(3)
1206     .set_description(""),
1207
1208     Option("ms_async_rdma_dscp", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1209     .set_default(96)
1210     .set_description(""),
1211
1212     Option("ms_max_accept_failures", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1213     .set_default(4)
1214     .set_description("The maximum number of consecutive failed accept() calls before "
1215                      "considering the daemon is misconfigured and abort it."),
1216
1217     Option("ms_async_rdma_cm", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1218     .set_default(false)
1219     .set_description(""),
1220
1221     Option("ms_async_rdma_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1222     .set_default("ib")
1223     .set_description(""),
1224
1225     Option("ms_dpdk_port_id", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1226     .set_default(0)
1227     .set_description(""),
1228
1229     Option("ms_dpdk_coremask", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1230     .set_default("0xF")         //begin with 0x for the string
1231     .set_description("")
1232     .add_see_also("ms_async_op_threads"),
1233
1234     Option("ms_dpdk_memory_channel", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1235     .set_default("4")
1236     .set_description(""),
1237
1238     Option("ms_dpdk_hugepages", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1239     .set_default("")
1240     .set_description(""),
1241
1242     Option("ms_dpdk_pmd", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1243     .set_default("")
1244     .set_description(""),
1245
1246     Option("ms_dpdk_host_ipv4_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1247     .set_default("")
1248     .set_description(""),
1249
1250     Option("ms_dpdk_gateway_ipv4_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1251     .set_default("")
1252     .set_description(""),
1253
1254     Option("ms_dpdk_netmask_ipv4_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1255     .set_default("")
1256     .set_description(""),
1257
1258     Option("ms_dpdk_lro", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1259     .set_default(true)
1260     .set_description(""),
1261
1262     Option("ms_dpdk_hw_flow_control", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1263     .set_default(true)
1264     .set_description(""),
1265
1266     Option("ms_dpdk_hw_queue_weight", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1267     .set_default(1)
1268     .set_description(""),
1269
1270     Option("ms_dpdk_debug_allow_loopback", Option::TYPE_BOOL, Option::LEVEL_DEV)
1271     .set_default(false)
1272     .set_description(""),
1273
1274     Option("ms_dpdk_rx_buffer_count_per_core", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1275     .set_default(8192)
1276     .set_description(""),
1277
1278     Option("inject_early_sigterm", Option::TYPE_BOOL, Option::LEVEL_DEV)
1279     .set_default(false)
1280     .set_description("send ourselves a SIGTERM early during startup"),
1281
1282     // MON
1283     Option("mon_enable_op_tracker", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1284     .set_default(true)
1285     .add_service("mon")
1286     .set_description("enable/disable MON op tracking"),
1287
1288     Option("mon_op_complaint_time", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
1289     .set_default(30)
1290     .add_service("mon")
1291     .set_description("time after which to consider a monitor operation blocked "
1292                      "after no updates"),
1293
1294     Option("mon_op_log_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1295     .set_default(5)
1296     .add_service("mon")
1297     .set_description("max number of slow ops to display"),
1298
1299     Option("mon_op_history_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1300     .set_default(20)
1301     .add_service("mon")
1302     .set_description("max number of completed ops to track"),
1303
1304     Option("mon_op_history_duration", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
1305     .set_default(600)
1306     .add_service("mon")
1307     .set_description("expiration time in seconds of historical MON OPS"),
1308
1309     Option("mon_op_history_slow_op_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1310     .set_default(20)
1311     .add_service("mon")
1312     .set_description("max number of slow historical MON OPS to keep"),
1313
1314     Option("mon_op_history_slow_op_threshold", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
1315     .set_default(10)
1316     .add_service("mon")
1317     .set_description("duration of an op to be considered as a historical slow op"),
1318
1319     Option("mon_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1320     .set_flag(Option::FLAG_NO_MON_UPDATE)
1321     .set_default("/var/lib/ceph/mon/$cluster-$id")
1322     .add_service("mon")
1323     .set_description("path to mon database"),
1324
1325     Option("mon_initial_members", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1326     .set_default("")
1327     .add_service("mon")
1328     .set_flag(Option::FLAG_NO_MON_UPDATE)
1329     .set_flag(Option::FLAG_CLUSTER_CREATE)
1330     .set_description(""),
1331
1332     Option("mon_compact_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1333     .set_default(false)
1334     .add_service("mon")
1335     .set_description(""),
1336
1337     Option("mon_compact_on_bootstrap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1338     .set_default(false)
1339     .add_service("mon")
1340     .set_description(""),
1341
1342     Option("mon_compact_on_trim", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1343     .set_default(true)
1344     .add_service("mon")
1345     .set_description(""),
1346
1347     /* -- mon: osdmap prune (begin) -- */
1348     Option("mon_osdmap_full_prune_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1349     .set_default(true)
1350     .add_service("mon")
1351     .set_description("enables pruning full osdmap versions when we go over a given number of maps")
1352     .add_see_also("mon_osdmap_full_prune_min")
1353     .add_see_also("mon_osdmap_full_prune_interval")
1354     .add_see_also("mon_osdmap_full_prune_txsize"),
1355
1356     Option("mon_osdmap_full_prune_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1357     .set_default(10000)
1358     .add_service("mon")
1359     .set_description("minimum number of versions in the store to trigger full map pruning")
1360     .add_see_also("mon_osdmap_full_prune_enabled")
1361     .add_see_also("mon_osdmap_full_prune_interval")
1362     .add_see_also("mon_osdmap_full_prune_txsize"),
1363
1364     Option("mon_osdmap_full_prune_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1365     .set_default(10)
1366     .add_service("mon")
1367     .set_description("interval between maps that will not be pruned; maps in the middle will be pruned.")
1368     .add_see_also("mon_osdmap_full_prune_enabled")
1369     .add_see_also("mon_osdmap_full_prune_interval")
1370     .add_see_also("mon_osdmap_full_prune_txsize"),
1371
1372     Option("mon_osdmap_full_prune_txsize", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1373     .set_default(100)
1374     .add_service("mon")
1375     .set_description("number of maps we will prune per iteration")
1376     .add_see_also("mon_osdmap_full_prune_enabled")
1377     .add_see_also("mon_osdmap_full_prune_interval")
1378     .add_see_also("mon_osdmap_full_prune_txsize"),
1379     /* -- mon: osdmap prune (end) -- */
1380
1381     Option("mon_osd_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1382     .set_default(500)
1383     .add_service("mon")
1384     .set_description("maximum number of OSDMaps to cache in memory"),
1385
1386     Option("mon_osd_cache_size_min", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1387     .set_default(128_M)
1388     .add_service("mon")
1389     .set_description("The minimum amount of bytes to be kept mapped in memory for osd monitor caches."),
1390
1391     Option("mon_memory_target", Option::TYPE_SIZE, Option::LEVEL_BASIC)
1392     .set_default(2_G)
1393     .set_flag(Option::FLAG_RUNTIME)
1394     .add_service("mon")
1395     .set_description("The amount of bytes pertaining to osd monitor caches and kv cache to be kept mapped in memory with cache auto-tuning enabled"),
1396
1397     Option("mon_memory_autotune", Option::TYPE_BOOL, Option::LEVEL_BASIC)
1398     .set_default(true)
1399     .set_flag(Option::FLAG_RUNTIME)
1400     .add_service("mon")
1401     .set_description("Autotune the cache memory being used for osd monitors and kv database"),
1402
1403     Option("mon_cpu_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1404     .set_default(4)
1405     .add_service("mon")
1406     .set_description("worker threads for CPU intensive background work"),
1407
1408     Option("mon_osd_mapping_pgs_per_chunk", Option::TYPE_INT, Option::LEVEL_DEV)
1409     .set_default(4096)
1410     .add_service("mon")
1411     .set_description("granularity of PG placement calculation background work"),
1412
1413     Option("mon_clean_pg_upmaps_per_chunk", Option::TYPE_UINT, Option::LEVEL_DEV)
1414     .set_default(256)
1415     .add_service("mon")
1416     .set_description("granularity of PG upmap validation background work"),
1417
1418     Option("mon_osd_max_creating_pgs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1419     .set_default(1024)
1420     .add_service("mon")
1421     .set_description("maximum number of PGs the mon will create at once"),
1422
1423     Option("mon_osd_max_initial_pgs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1424     .set_default(1024)
1425     .add_service("mon")
1426     .set_description("maximum number of PGs a pool will created with")
1427     .set_long_description("If the user specifies more PGs than this, the cluster will subsequently split PGs after the pool is created in order to reach the target."),
1428
1429     Option("mon_tick_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1430     .set_default(5)
1431     .add_service("mon")
1432     .set_description("interval for internal mon background checks"),
1433
1434     Option("mon_session_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1435     .set_default(300)
1436     .add_service("mon")
1437     .set_description("close inactive mon client connections after this many seconds"),
1438
1439     Option("mon_subscribe_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1440     .set_default(1_day)
1441     .add_service("mon")
1442     .set_description("subscribe interval for pre-jewel clients"),
1443
1444     Option("mon_delta_reset_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1445     .set_default(10)
1446     .add_service("mon")
1447     .add_service("mon")
1448     .set_description("window duration for rate calculations in 'ceph status'"),
1449
1450     Option("mon_osd_laggy_halflife", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1451     .set_default(1_hr)
1452     .add_service("mon")
1453     .set_description("halflife of OSD 'lagginess' factor"),
1454
1455     Option("mon_osd_laggy_weight", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1456     .set_default(.3)
1457     .set_min_max(0.0, 1.0)
1458     .add_service("mon")
1459     .set_description("how heavily to weight OSD marking itself back up in overall laggy_probability")
1460     .set_long_description("1.0 means that an OSD marking itself back up (because it was marked down but not actually dead) means a 100% laggy_probability; 0.0 effectively disables tracking of laggy_probability."),
1461
1462     Option("mon_osd_laggy_max_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1463     .set_default(300)
1464     .add_service("mon")
1465     .set_description("cap value for period for OSD to be marked for laggy_interval calculation"),
1466
1467     Option("mon_osd_adjust_heartbeat_grace", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1468     .set_default(true)
1469     .add_service("mon")
1470     .set_description("increase OSD heartbeat grace if peers appear to be laggy")
1471     .set_long_description("If an OSD is marked down but then marks itself back up, it implies it wasn't actually down but was unable to respond to heartbeats.  If this option is true, we can use the laggy_probability and laggy_interval values calculated to model this situation to increase the heartbeat grace period for this OSD so that it isn't marked down again.  laggy_probability is an estimated probability that the given OSD is down because it is laggy (not actually down), and laggy_interval is an estiate on how long it stays down when it is laggy.")
1472     .add_see_also("mon_osd_laggy_halflife")
1473     .add_see_also("mon_osd_laggy_weight")
1474     .add_see_also("mon_osd_laggy_max_interval"),
1475
1476     Option("mon_osd_adjust_down_out_interval", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1477     .set_default(true)
1478     .add_service("mon")
1479     .set_description("increase the mon_osd_down_out_interval if an OSD appears to be laggy")
1480     .add_see_also("mon_osd_adjust_heartbeat_grace"),
1481
1482     Option("mon_osd_auto_mark_in", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1483     .set_default(false)
1484     .add_service("mon")
1485     .set_description("mark any OSD that comes up 'in'"),
1486
1487     Option("mon_osd_auto_mark_auto_out_in", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1488     .set_default(true)
1489     .add_service("mon")
1490     .set_description("mark any OSD that comes up that was automatically marked 'out' back 'in'")
1491     .add_see_also("mon_osd_down_out_interval"),
1492
1493     Option("mon_osd_auto_mark_new_in", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1494     .set_default(true)
1495     .add_service("mon")
1496     .set_description("mark any new OSD that comes up 'in'"),
1497
1498     Option("mon_osd_destroyed_out_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1499     .set_default(600)
1500     .add_service("mon")
1501     .set_description("mark any OSD 'out' that has been 'destroy'ed for this long (seconds)"),
1502
1503     Option("mon_osd_down_out_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1504     .set_default(600)
1505     .add_service("mon")
1506     .set_description("mark any OSD 'out' that has been 'down' for this long (seconds)"),
1507
1508     Option("mon_osd_down_out_subtree_limit", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1509     .set_default("rack")
1510     .set_flag(Option::FLAG_RUNTIME)
1511     .add_service("mon")
1512     .set_description("do not automatically mark OSDs 'out' if an entire subtree of this size is down")
1513     .add_see_also("mon_osd_down_out_interval"),
1514
1515     Option("mon_osd_min_up_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1516     .set_default(.3)
1517     .add_service("mon")
1518     .set_description("do not automatically mark OSDs 'out' if fewer than this many OSDs are 'up'")
1519     .add_see_also("mon_osd_down_out_interval"),
1520
1521     Option("mon_osd_min_in_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1522     .set_default(.75)
1523     .add_service("mon")
1524     .set_description("do not automatically mark OSDs 'out' if fewer than this many OSDs are 'in'")
1525     .add_see_also("mon_osd_down_out_interval"),
1526
1527     Option("mon_osd_warn_op_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1528     .set_default(32)
1529     .add_service("mgr")
1530     .set_description("issue REQUEST_SLOW health warning if OSD ops are slower than this age (seconds)"),
1531
1532     Option("mon_osd_warn_num_repaired", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1533     .set_default(10)
1534     .add_service("mon")
1535     .set_description("issue OSD_TOO_MANY_REPAIRS health warning if an OSD has more than this many read repairs"),
1536
1537     Option("mon_osd_err_op_age_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1538     .set_default(128)
1539     .add_service("mgr")
1540     .set_description("issue REQUEST_STUCK health error if OSD ops are slower than is age (seconds)"),
1541
1542     Option("mon_osd_prime_pg_temp", Option::TYPE_BOOL, Option::LEVEL_DEV)
1543     .set_default(true)
1544     .add_service("mon")
1545     .set_description("minimize peering work by priming pg_temp values after a map change"),
1546
1547     Option("mon_osd_prime_pg_temp_max_time", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1548     .set_default(.5)
1549     .add_service("mon")
1550     .set_description("maximum time to spend precalculating PG mappings on map change (seconds)"),
1551
1552     Option("mon_osd_prime_pg_temp_max_estimate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1553     .set_default(.25)
1554     .add_service("mon")
1555     .set_description("calculate all PG mappings if estimated fraction of PGs that change is above this amount"),
1556
1557     Option("mon_stat_smooth_intervals", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1558     .set_default(6)
1559     .set_min(1)
1560     .add_service("mgr")
1561     .set_description("number of PGMaps stats over which we calc the average read/write throughput of the whole cluster"),
1562
1563     Option("mon_election_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1564     .set_default(5)
1565     .add_service("mon")
1566     .set_description("maximum time for a mon election (seconds)"),
1567
1568     Option("mon_election_default_strategy", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1569     .set_default(1)
1570     .set_min_max(1, 3)
1571     .set_description("The election strategy to set when constructing the first monmap."),
1572
1573     Option("mon_lease", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1574     .set_default(5)
1575     .add_service("mon")
1576     .set_description("lease interval between quorum monitors (seconds)")
1577     .set_long_description("This setting controls how sensitive your mon quorum is to intermittent network issues or other failures."),
1578
1579     Option("mon_lease_renew_interval_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1580     .set_default(.6)
1581     .set_min_max((double)0.0, (double).9999999)
1582     .add_service("mon")
1583     .set_description("multiple of mon_lease for the lease renewal interval")
1584     .set_long_description("Leases must be renewed before they time out.  A smaller value means frequent renewals, while a value close to 1 makes a lease expiration more likely.")
1585     .add_see_also("mon_lease"),
1586
1587     Option("mon_lease_ack_timeout_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1588     .set_default(2.0)
1589     .set_min_max(1.0001, 100.0)
1590     .add_service("mon")
1591     .set_description("multiple of mon_lease for the lease ack interval before calling new election")
1592     .add_see_also("mon_lease"),
1593
1594     Option("mon_accept_timeout_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1595     .set_default(2.0)
1596     .add_service("mon")
1597     .set_description("multiple of mon_lease for follower mons to accept proposed state changes before calling a new election")
1598     .add_see_also("mon_lease"),
1599
1600     Option("mon_elector_ping_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1601     .set_default(2.0)
1602     .add_service("mon")
1603     .set_description("The time after which a ping 'times out' and a connection is considered down")
1604     .add_see_also("mon_elector_ping_divisor"),
1605
1606     Option("mon_elector_ping_divisor", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1607     .set_default(2)
1608     .add_service("mon")
1609     .set_description("We will send a ping up to this many times per timeout per")
1610     .add_see_also("mon_elector_ping_timeout"),
1611
1612     Option("mon_con_tracker_persist_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1613     .set_default(10)
1614     .set_min_max(1, 100000)
1615     .add_service("mon")
1616     .set_description("how many updates the ConnectionTracker takes before it persists to disk"),
1617
1618     Option("mon_con_tracker_score_halflife", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1619     .set_default(12*60*60)
1620     .set_min(60)
1621     .add_service("mon")
1622     .set_description("The 'halflife' used when updating/calculating peer connection scores"),
1623
1624     Option("mon_elector_ignore_propose_margin", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1625     .set_default(0.0005)
1626     .add_service("mon")
1627     .set_description("The difference in connection score allowed before a peon stops ignoring out-of-quorum PROPOSEs"),
1628
1629     Option("mon_warn_on_degraded_stretch_mode", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1630     .set_default(true)
1631     .add_service("mon")
1632     .set_description("Issue a health warning if we are in degraded stretch mode"),
1633
1634     Option("mon_stretch_cluster_recovery_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1635     .set_default(0.6)
1636     .add_service("mon")
1637     .set_description("the ratio of up OSDs at which a degraded stretch cluster enters recovery")
1638     .set_min_max(0.51, 1.0),
1639
1640     Option("mon_stretch_recovery_min_wait", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1641     .set_default(15.0)
1642     .add_service("mon")
1643     .set_description("how long the monitors wait before considering fully-healthy PGs as evidence the stretch mode is repaired")
1644     .set_min(1.0),
1645
1646     Option("mon_stretch_pool_size", Option::TYPE_UINT, Option::LEVEL_DEV)
1647     .set_default(4)
1648     .add_service("mon")
1649     .set_description("")
1650     .set_min_max(3, 6),
1651
1652     Option("mon_stretch_pool_min_size", Option::TYPE_UINT, Option::LEVEL_DEV)
1653     .set_default(2)
1654     .add_service("mon")
1655     .set_description("")
1656     .set_min_max(2, 4),
1657
1658     Option("mon_clock_drift_allowed", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1659     .set_default(.050)
1660     .add_service("mon")
1661     .set_description("allowed clock drift (in seconds) between mons before issuing a health warning"),
1662
1663     Option("mon_clock_drift_warn_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1664     .set_default(5)
1665     .add_service("mon")
1666     .set_description("exponential backoff factor for logging clock drift warnings in the cluster log"),
1667
1668     Option("mon_timecheck_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1669     .set_default(300.0)
1670     .add_service("mon")
1671     .set_description("frequency of clock synchronization checks between monitors (seconds)"),
1672
1673     Option("mon_timecheck_skew_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1674     .set_default(30.0)
1675     .add_service("mon")
1676     .set_description("frequency of clock synchronization (re)checks between monitors while clocks are believed to be skewed (seconds)")
1677     .add_see_also("mon_timecheck_interval"),
1678
1679     Option("mon_pg_stuck_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1680     .set_default(60)
1681     .set_description("number of seconds after which pgs can be considered stuck inactive, unclean, etc")
1682     .set_long_description("see doc/control.rst under dump_stuck for more info")
1683     .add_service("mgr"),
1684
1685     Option("mon_pg_warn_min_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1686     .set_default(0)
1687     .add_service("mgr")
1688     .set_description("minimal number PGs per (in) osd before we warn the admin"),
1689
1690     Option("mon_max_pg_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1691     .set_min(1)
1692     .set_default(250)
1693     .add_service("mgr")
1694     .set_description("Max number of PGs per OSD the cluster will allow")
1695     .set_long_description("If the number of PGs per OSD exceeds this, a "
1696         "health warning will be visible in `ceph status`.  This is also used "
1697         "in automated PG management, as the threshold at which some pools' "
1698         "pg_num may be shrunk in order to enable increasing the pg_num of "
1699         "others."),
1700
1701     Option("mon_target_pg_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1702     .set_min(1)
1703     .set_default(100)
1704     .set_description("Automated PG management creates this many PGs per OSD")
1705     .set_long_description("When creating pools, the automated PG management "
1706         "logic will attempt to reach this target.  In some circumstances, it "
1707         "may exceed this target, up to the ``mon_max_pg_per_osd`` limit. "
1708         "Conversely, a lower number of PGs per OSD may be created if the "
1709         "cluster is not yet fully utilised"),
1710
1711     Option("mon_pg_warn_max_object_skew", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1712     .set_default(10.0)
1713     .set_description("max skew few average in objects per pg")
1714     .add_service("mgr"),
1715
1716     Option("mon_pg_warn_min_objects", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1717     .set_default(10000)
1718     .set_description("do not warn below this object #")
1719     .add_service("mgr"),
1720
1721     Option("mon_pg_warn_min_pool_objects", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1722     .set_default(1000)
1723     .set_description("do not warn on pools below this object #")
1724     .add_service("mgr"),
1725
1726     Option("mon_pg_check_down_all_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1727     .set_default(.5)
1728     .set_description("threshold of down osds after which we check all pgs")
1729     .add_service("mgr"),
1730
1731     Option("mon_cache_target_full_warn_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1732     .set_default(.66)
1733     .add_service("mgr")
1734     .set_flag(Option::FLAG_NO_MON_UPDATE)
1735     .set_flag(Option::FLAG_CLUSTER_CREATE)
1736     .set_description("issue CACHE_POOL_NEAR_FULL health warning when cache pool utilization exceeds this ratio of usable space"),
1737
1738     Option("mon_osd_full_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1739     .set_default(.95)
1740     .set_flag(Option::FLAG_NO_MON_UPDATE)
1741     .set_flag(Option::FLAG_CLUSTER_CREATE)
1742     .set_description("full ratio of OSDs to be set during initial creation of the cluster"),
1743
1744     Option("mon_osd_backfillfull_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1745     .set_default(.90)
1746     .set_flag(Option::FLAG_NO_MON_UPDATE)
1747     .set_flag(Option::FLAG_CLUSTER_CREATE)
1748     .set_description(""),
1749
1750     Option("mon_osd_nearfull_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1751     .set_default(.85)
1752     .set_flag(Option::FLAG_NO_MON_UPDATE)
1753     .set_flag(Option::FLAG_CLUSTER_CREATE)
1754     .set_description("nearfull ratio for OSDs to be set during initial creation of cluster"),
1755
1756     Option("mon_osd_initial_require_min_compat_client", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1757     .set_default("luminous")
1758     .set_flag(Option::FLAG_NO_MON_UPDATE)
1759     .set_flag(Option::FLAG_CLUSTER_CREATE)
1760     .set_description(""),
1761
1762     Option("mon_allow_pool_delete", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1763     .set_default(false)
1764     .add_service("mon")
1765     .set_description("allow pool deletions"),
1766
1767     Option("mon_fake_pool_delete", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1768     .set_default(false)
1769     .add_service("mon")
1770     .set_description("fake pool deletions by renaming the rados pool"),
1771
1772     Option("mon_globalid_prealloc", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1773     .set_default(10000)
1774     .add_service("mon")
1775     .set_description("number of globalid values to preallocate")
1776     .set_long_description("This setting caps how many new clients can authenticate with the cluster before the monitors have to perform a write to preallocate more.  Large values burn through the 64-bit ID space more quickly."),
1777
1778     Option("mon_osd_report_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1779     .set_default(900)
1780     .add_service("mon")
1781     .set_description("time before OSDs who do not report to the mons are marked down (seconds)"),
1782
1783     Option("mon_warn_on_insecure_global_id_reclaim", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1784     .set_default(true)
1785     .add_service("mon")
1786     .set_description("issue AUTH_INSECURE_GLOBAL_ID_RECLAIM health warning if any connected clients are insecurely reclaiming global_id")
1787     .add_see_also("mon_warn_on_insecure_global_id_reclaim_allowed")
1788     .add_see_also("auth_allow_insecure_global_id_reclaim")
1789     .add_see_also("auth_expose_insecure_global_id_reclaim"),
1790
1791     Option("mon_warn_on_insecure_global_id_reclaim_allowed", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1792     .set_default(true)
1793     .add_service("mon")
1794     .set_description("issue AUTH_INSECURE_GLOBAL_ID_RECLAIM_ALLOWED health warning if insecure global_id reclaim is allowed")
1795     .add_see_also("mon_warn_on_insecure_global_id_reclaim")
1796     .add_see_also("auth_allow_insecure_global_id_reclaim")
1797     .add_see_also("auth_expose_insecure_global_id_reclaim"),
1798
1799     Option("mon_warn_on_msgr2_not_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1800     .set_default(true)
1801     .add_service("mon")
1802     .set_description("issue MON_MSGR2_NOT_ENABLED health warning if monitors are all running Nautilus but not all binding to a msgr2 port")
1803     .add_see_also("ms_bind_msgr2"),
1804
1805     Option("mon_warn_on_legacy_crush_tunables", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1806     .set_default(true)
1807     .add_service("mgr")
1808     .set_description("issue OLD_CRUSH_TUNABLES health warning if CRUSH tunables are older than mon_crush_min_required_version")
1809     .add_see_also("mon_crush_min_required_version"),
1810
1811     Option("mon_crush_min_required_version", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1812     .set_default("hammer")
1813     .add_service("mgr")
1814     .set_description("minimum ceph release to use for mon_warn_on_legacy_crush_tunables")
1815     .add_see_also("mon_warn_on_legacy_crush_tunables"),
1816
1817     Option("mon_warn_on_crush_straw_calc_version_zero", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1818     .set_default(true)
1819     .add_service("mgr")
1820     .set_description("issue OLD_CRUSH_STRAW_CALC_VERSION health warning if the CRUSH map's straw_calc_version is zero"),
1821
1822     Option("mon_warn_on_osd_down_out_interval_zero", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1823     .set_default(true)
1824     .add_service("mgr")
1825     .set_description("issue OSD_NO_DOWN_OUT_INTERVAL health warning if mon_osd_down_out_interval is zero")
1826     .set_long_description("Having mon_osd_down_out_interval set to 0 means that down OSDs are not marked out automatically and the cluster does not heal itself without administrator intervention.")
1827     .add_see_also("mon_osd_down_out_interval"),
1828
1829     Option("mon_warn_on_cache_pools_without_hit_sets", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1830     .set_default(true)
1831     .add_service("mgr")
1832     .set_description("issue CACHE_POOL_NO_HIT_SET health warning for cache pools that do not have hit sets configured"),
1833
1834     Option("mon_warn_on_pool_no_app", Option::TYPE_BOOL, Option::LEVEL_DEV)
1835     .set_default(true)
1836     .add_service("mgr")
1837     .set_description("issue POOL_APP_NOT_ENABLED health warning if pool has not application enabled"),
1838
1839     Option("mon_warn_on_pool_pg_num_not_power_of_two", Option::TYPE_BOOL, Option::LEVEL_DEV)
1840     .set_default(true)
1841     .add_service("mon")
1842     .set_description("issue POOL_PG_NUM_NOT_POWER_OF_TWO warning if pool has a non-power-of-two pg_num value"),
1843
1844     Option("mon_warn_on_pool_no_redundancy", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1845     .set_default(true)
1846     .add_service("mon")
1847     .set_description("Issue a health warning if any pool is configured with no replicas")
1848     .add_see_also("osd_pool_default_size")
1849     .add_see_also("osd_pool_default_min_size"),
1850
1851     Option("mon_allow_pool_size_one", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1852     .set_default(false)
1853     .add_service("mon")
1854     .set_description("allow configuring pool with no replicas"),
1855
1856     Option("mon_warn_on_misplaced", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1857     .set_default(false)
1858     .add_service("mgr")
1859     .set_description("Issue a health warning if there are misplaced objects"),
1860
1861     Option("mon_warn_on_too_few_osds", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1862     .set_default(true)
1863     .add_service("mgr")
1864     .set_description("Issue a health warning if there are fewer OSDs than osd_pool_default_size"),
1865
1866     Option("mon_warn_on_slow_ping_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1867     .set_default(0)
1868     .add_service("mgr")
1869     .set_description("Override mon_warn_on_slow_ping_ratio with specified threshold in milliseconds")
1870     .add_see_also("mon_warn_on_slow_ping_ratio"),
1871
1872     Option("mon_warn_on_slow_ping_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1873     .set_default(.05)
1874     .add_service("mgr")
1875     .set_description("Issue a health warning if heartbeat ping longer than percentage of osd_heartbeat_grace")
1876     .add_see_also("osd_heartbeat_grace")
1877     .add_see_also("mon_warn_on_slow_ping_time"),
1878
1879     Option("mon_max_snap_prune_per_epoch", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1880     .set_default(100)
1881     .add_service("mon")
1882     .set_description("max number of pruned snaps we will process in a single OSDMap epoch"),
1883
1884     Option("mon_min_osdmap_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1885     .set_default(500)
1886     .add_service("mon")
1887     .set_description("min number of OSDMaps to store"),
1888
1889     Option("mon_max_log_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1890     .set_default(500)
1891     .add_service("mon")
1892     .set_description("max number of past cluster log epochs to store"),
1893
1894     Option("mon_max_mdsmap_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1895     .set_default(500)
1896     .add_service("mon")
1897     .set_description("max number of FSMaps/MDSMaps to store"),
1898
1899     Option("mon_max_mgrmap_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1900     .set_default(500)
1901     .add_service("mon")
1902     .set_description("max number of MgrMaps to store"),
1903
1904     Option("mon_max_osd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1905     .set_default(10000)
1906     .add_service("mon")
1907     .set_description("max number of OSDs in a cluster"),
1908
1909     Option("mon_probe_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1910     .set_default(2.0)
1911     .add_service("mon")
1912     .set_description("timeout for querying other mons during bootstrap pre-election phase (seconds)"),
1913
1914     Option("mon_client_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1915     .set_default(100ul << 20)
1916     .add_service("mon")
1917     .set_description("max bytes of outstanding client messages mon will read off the network"),
1918
1919     Option("mon_daemon_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1920     .set_default(400ul << 20)
1921     .add_service("mon")
1922     .set_description("max bytes of outstanding mon messages mon will read off the network"),
1923
1924     Option("mon_mgr_proxy_client_bytes_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1925     .set_default(.3)
1926     .add_service("mon")
1927     .set_description("ratio of mon_client_bytes that can be consumed by "
1928                      "proxied mgr commands before we error out to client"),
1929
1930     Option("mon_log_max_summary", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1931     .set_default(50)
1932     .add_service("mon")
1933     .set_description("number of recent cluster log messages to retain"),
1934
1935     Option("mon_max_log_entries_per_event", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1936     .set_default(4096)
1937     .add_service("mon")
1938     .set_description("max cluster log entries per paxos event"),
1939
1940     Option("mon_reweight_min_pgs_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1941     .set_default(10)
1942     .add_service("mgr")
1943     .set_description(""),
1944
1945     Option("mon_reweight_min_bytes_per_osd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1946     .set_default(100_M)
1947     .add_service("mgr")
1948     .set_description(""),
1949
1950     Option("mon_reweight_max_osds", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1951     .set_default(4)
1952     .add_service("mgr")
1953     .set_description(""),
1954
1955     Option("mon_reweight_max_change", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1956     .set_default(0.05)
1957     .add_service("mgr")
1958     .set_description(""),
1959
1960     Option("mon_health_to_clog", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1961     .set_default(true)
1962     .add_service("mon")
1963     .set_description("log monitor health to cluster log"),
1964
1965     Option("mon_health_to_clog_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1966     .set_default(10_min)
1967     .add_service("mon")
1968     .set_description("frequency to log monitor health to cluster log")
1969     .add_see_also("mon_health_to_clog"),
1970
1971     Option("mon_health_to_clog_tick_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1972     .set_default(60.0)
1973     .add_service("mon")
1974     .set_description(""),
1975
1976     Option("mon_health_detail_to_clog", Option::TYPE_BOOL, Option::LEVEL_DEV)
1977     .set_default(true)
1978     .set_description("log health detail to cluster log"),
1979
1980     Option("mon_health_max_detail", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1981     .set_default(50)
1982     .add_service("mon")
1983     .set_description("max detailed pgs to report in health detail"),
1984
1985     Option("mon_health_log_update_period", Option::TYPE_INT, Option::LEVEL_DEV)
1986     .set_default(5)
1987     .add_service("mon")
1988     .set_description("minimum time in seconds between log messages about "
1989                      "each health check")
1990     .set_min(0),
1991
1992     Option("mon_data_avail_crit", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1993     .set_default(5)
1994     .add_service("mon")
1995     .set_description("issue MON_DISK_CRIT health error when mon available space below this percentage"),
1996
1997     Option("mon_data_avail_warn", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1998     .set_default(30)
1999     .add_service("mon")
2000     .set_description("issue MON_DISK_LOW health warning when mon available space below this percentage"),
2001
2002     Option("mon_data_size_warn", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2003     .set_default(15_G)
2004     .add_service("mon")
2005     .set_description("issue MON_DISK_BIG health warning when mon database is above this size"),
2006
2007     Option("mon_warn_pg_not_scrubbed_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2008     .set_default(0.5)
2009     .set_min(0)
2010     .set_description("Percentage of the scrub max interval past the scrub max interval to warn")
2011     .set_long_description("")
2012     .add_see_also("osd_scrub_max_interval"),
2013
2014     Option("mon_warn_pg_not_deep_scrubbed_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2015     .set_default(0.75)
2016     .set_min(0)
2017     .set_description("Percentage of the deep scrub interval past the deep scrub interval to warn")
2018     .set_long_description("")
2019     .add_see_also("osd_deep_scrub_interval"),
2020
2021     Option("mon_scrub_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
2022     .set_default(1_day)
2023     .add_service("mon")
2024     .set_description("frequency for scrubbing mon database"),
2025
2026     Option("mon_scrub_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2027     .set_default(5_min)
2028     .add_service("mon")
2029     .set_description("timeout to restart scrub of mon quorum participant does not respond for the latest chunk"),
2030
2031     Option("mon_scrub_max_keys", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2032     .set_default(100)
2033     .add_service("mon")
2034     .set_description("max keys per on scrub chunk/step"),
2035
2036     Option("mon_scrub_inject_crc_mismatch", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2037     .set_default(0.0)
2038     .add_service("mon")
2039     .set_description("probability for injecting crc mismatches into mon scrub"),
2040
2041     Option("mon_scrub_inject_missing_keys", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2042     .set_default(0.0)
2043     .add_service("mon")
2044     .set_description("probability for injecting missing keys into mon scrub"),
2045
2046     Option("mon_config_key_max_entry_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2047     .set_default(64_K)
2048     .add_service("mon")
2049     .set_description("Defines the number of bytes allowed to be held in a "
2050                      "single config-key entry"),
2051
2052     Option("mon_sync_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2053     .set_default(60.0)
2054     .add_service("mon")
2055     .set_description("timeout before canceling sync if syncing mon does not respond"),
2056
2057     Option("mon_sync_max_payload_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2058     .set_default(1_M)
2059     .add_service("mon")
2060     .set_description("target max message payload for mon sync"),
2061
2062     Option("mon_sync_max_payload_keys", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2063     .set_default(2000)
2064     .add_service("mon")
2065     .set_description("target max keys in message payload for mon sync"),
2066
2067     Option("mon_sync_debug", Option::TYPE_BOOL, Option::LEVEL_DEV)
2068     .set_default(false)
2069     .add_service("mon")
2070     .set_description("enable extra debugging during mon sync"),
2071
2072     Option("mon_inject_sync_get_chunk_delay", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2073     .set_default(0)
2074     .add_service("mon")
2075     .set_description("inject delay during sync (seconds)"),
2076
2077     Option("mon_osd_min_down_reporters", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2078     .set_default(2)
2079     .add_service("mon")
2080     .set_description("number of OSDs from different subtrees who need to report a down OSD for it to count")
2081     .add_see_also("mon_osd_reporter_subtree_level"),
2082
2083     Option("mon_osd_reporter_subtree_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2084     .set_default("host")
2085     .add_service("mon")
2086     .set_flag(Option::FLAG_RUNTIME)
2087     .set_description("in which level of parent bucket the reporters are counted"),
2088
2089     Option("mon_osd_snap_trim_queue_warn_on", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2090     .set_default(32768)
2091     .add_service("mon")
2092     .set_description("Warn when snap trim queue is that large (or larger).")
2093     .set_long_description("Warn when snap trim queue length for at least one PG crosses this value, as this is indicator of snap trimmer not keeping up, wasting disk space"),
2094
2095     Option("mon_osd_force_trim_to", Option::TYPE_INT, Option::LEVEL_DEV)
2096     .set_default(0)
2097     .add_service("mon")
2098     .set_description("force mons to trim osdmaps through this epoch"),
2099
2100     Option("mon_mds_force_trim_to", Option::TYPE_INT, Option::LEVEL_DEV)
2101     .set_default(0)
2102     .add_service("mon")
2103     .set_description("force mons to trim mdsmaps/fsmaps through this epoch"),
2104
2105     Option("mon_mds_skip_sanity", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2106     .set_default(false)
2107     .add_service("mon")
2108     .set_description("skip sanity checks on fsmap/mdsmap"),
2109
2110     Option("mon_debug_extra_checks", Option::TYPE_BOOL, Option::LEVEL_DEV)
2111     .set_default(false)
2112     .add_service("mon")
2113     .set_description("Enable some additional monitor checks")
2114     .set_long_description(
2115         "Enable some additional monitor checks that would be too expensive "
2116         "to run on production systems, or would only be relevant while "
2117         "testing or debugging."),
2118
2119     Option("mon_debug_block_osdmap_trim", Option::TYPE_BOOL, Option::LEVEL_DEV)
2120     .set_default(false)
2121     .add_service("mon")
2122     .set_description("Block OSDMap trimming while the option is enabled.")
2123     .set_long_description(
2124         "Blocking OSDMap trimming may be quite helpful to easily reproduce "
2125         "states in which the monitor keeps (hundreds of) thousands of "
2126         "osdmaps."),
2127
2128     Option("mon_debug_deprecated_as_obsolete", Option::TYPE_BOOL, Option::LEVEL_DEV)
2129     .set_default(false)
2130     .add_service("mon")
2131     .set_description("treat deprecated mon commands as obsolete"),
2132
2133     Option("mon_debug_dump_transactions", Option::TYPE_BOOL, Option::LEVEL_DEV)
2134     .set_default(false)
2135     .add_service("mon")
2136     .set_description("dump paxos transactions to log")
2137     .add_see_also("mon_debug_dump_location"),
2138
2139     Option("mon_debug_dump_json", Option::TYPE_BOOL, Option::LEVEL_DEV)
2140     .set_default(false)
2141     .add_service("mon")
2142     .set_description("dump paxos transasctions to log as json")
2143     .add_see_also("mon_debug_dump_transactions"),
2144
2145     Option("mon_debug_dump_location", Option::TYPE_STR, Option::LEVEL_DEV)
2146     .set_default("/var/log/ceph/$cluster-$name.tdump")
2147     .add_service("mon")
2148     .set_description("file to dump paxos transactions to")
2149     .add_see_also("mon_debug_dump_transactions"),
2150
2151     Option("mon_debug_no_require_octopus", Option::TYPE_BOOL, Option::LEVEL_DEV)
2152     .set_default(false)
2153     .add_service("mon")
2154     .set_flag(Option::FLAG_CLUSTER_CREATE)
2155     .set_description("do not set octopus feature for new mon clusters"),
2156
2157     Option("mon_debug_no_require_pacific", Option::TYPE_BOOL, Option::LEVEL_DEV)
2158     .set_default(false)
2159     .add_service("mon")
2160     .set_flag(Option::FLAG_CLUSTER_CREATE)
2161     .set_description("do not set pacific feature for new mon clusters"),
2162
2163     Option("mon_debug_no_require_bluestore_for_ec_overwrites", Option::TYPE_BOOL, Option::LEVEL_DEV)
2164     .set_default(false)
2165     .add_service("mon")
2166     .set_description("do not require bluestore OSDs to enable EC overwrites on a rados pool"),
2167
2168     Option("mon_debug_no_initial_persistent_features", Option::TYPE_BOOL, Option::LEVEL_DEV)
2169     .set_default(false)
2170     .add_service("mon")
2171     .set_flag(Option::FLAG_CLUSTER_CREATE)
2172     .set_description("do not set any monmap features for new mon clusters"),
2173
2174     Option("mon_inject_transaction_delay_max", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2175     .set_default(10.0)
2176     .add_service("mon")
2177     .set_description("max duration of injected delay in paxos"),
2178
2179     Option("mon_inject_transaction_delay_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2180     .set_default(0)
2181     .add_service("mon")
2182     .set_description("probability of injecting a delay in paxos"),
2183
2184     Option("mon_inject_pg_merge_bounce_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2185     .set_default(0)
2186     .add_service("mon")
2187     .set_description("probability of failing and reverting a pg_num decrement"),
2188
2189     Option("mon_sync_provider_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
2190     .set_default(0)
2191     .add_service("mon")
2192     .set_description("kill mon sync requester at specific point"),
2193
2194     Option("mon_sync_requester_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
2195     .set_default(0)
2196     .add_service("mon")
2197     .set_description("kill mon sync requestor at specific point"),
2198
2199     Option("mon_force_quorum_join", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2200     .set_default(false)
2201     .add_service("mon")
2202     .set_description("force mon to rejoin quorum even though it was just removed"),
2203
2204     Option("mon_keyvaluedb", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2205     .set_default("rocksdb")
2206     .set_enum_allowed({"leveldb", "rocksdb"})
2207     .set_flag(Option::FLAG_CREATE)
2208     .add_service("mon")
2209     .set_description("database backend to use for the mon database"),
2210
2211     Option("mon_debug_unsafe_allow_tier_with_nonempty_snaps", Option::TYPE_BOOL, Option::LEVEL_DEV)
2212     .set_default(false)
2213     .add_service("mon")
2214     .set_description(""),
2215
2216     Option("mon_osd_blocklist_default_expire", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2217     .set_default(1_hr)
2218     .add_service("mon")
2219     .set_description("Duration in seconds that blocklist entries for clients "
2220                      "remain in the OSD map"),
2221
2222     Option("mon_mds_blocklist_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2223     .set_default(1_day)
2224     .set_min(1_hr)
2225     .add_service("mon")
2226     .set_description("Duration in seconds that blocklist entries for MDS "
2227                      "daemons remain in the OSD map")
2228     .set_flag(Option::FLAG_RUNTIME),
2229
2230     Option("mon_mgr_blocklist_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2231     .set_default(1_day)
2232     .set_min(1_hr)
2233     .add_service("mon")
2234     .set_description("Duration in seconds that blocklist entries for mgr "
2235                      "daemons remain in the OSD map")
2236     .set_flag(Option::FLAG_RUNTIME),
2237
2238     Option("mon_osd_crush_smoke_test", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2239     .set_default(true)
2240     .add_service("mon")
2241     .set_description("perform a smoke test on any new CRUSH map before accepting changes"),
2242
2243     Option("mon_smart_report_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2244     .set_default(5)
2245     .add_service("mon")
2246     .set_description("Timeout (in seconds) for smarctl to run, default is set to 5"),
2247
2248     Option("mon_auth_validate_all_caps", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2249     .set_default(true)
2250     .add_service("mon")
2251     .set_description("Whether to parse non-monitor capabilities set by the "
2252                      "'ceph auth ...' commands. Disabling this saves CPU on the "
2253                      "monitor, but allows invalid capabilities to be set, and "
2254                      "only be rejected later, when they are used.")
2255     .set_flag(Option::FLAG_RUNTIME),
2256
2257     Option("mon_warn_on_older_version", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2258     .set_default(true)
2259     .add_service("mon")
2260     .set_description("issue DAEMON_OLD_VERSION health warning if daemons are not all running the same version"),
2261
2262     Option("mon_warn_older_version_delay", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
2263     .set_default(7_day)
2264     .add_service("mon")
2265     .set_description("issue DAEMON_OLD_VERSION health warning after this amount of time has elapsed"),
2266
2267     // PAXOS
2268
2269     Option("paxos_stash_full_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2270     .set_default(25)
2271     .add_service("mon")
2272     .set_description(""),
2273
2274     Option("paxos_max_join_drift", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2275     .set_default(10)
2276     .add_service("mon")
2277     .set_description(""),
2278
2279     Option("paxos_propose_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2280     .set_default(1.0)
2281     .add_service("mon")
2282     .set_description(""),
2283
2284     Option("paxos_min_wait", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2285     .set_default(0.05)
2286     .add_service("mon")
2287     .set_description(""),
2288
2289     Option("paxos_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2290     .set_default(500)
2291     .add_service("mon")
2292     .set_description(""),
2293
2294     Option("paxos_trim_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2295     .set_default(250)
2296     .add_service("mon")
2297     .set_description(""),
2298
2299     Option("paxos_trim_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2300     .set_default(500)
2301     .add_service("mon")
2302     .set_description(""),
2303
2304     Option("paxos_service_trim_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2305     .set_default(250)
2306     .add_service("mon")
2307     .set_description(""),
2308
2309     Option("paxos_service_trim_max", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2310     .set_default(500)
2311     .add_service("mon")
2312     .set_description(""),
2313
2314     Option("paxos_service_trim_max_multiplier", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2315     .set_default(20)
2316     .set_min(0)
2317     .add_service("mon")
2318     .set_description("factor by which paxos_service_trim_max will be multiplied to get a new upper bound when trim sizes are high  (0 disables it)")
2319     .set_flag(Option::FLAG_RUNTIME),
2320
2321     Option("paxos_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
2322     .set_default(0)
2323     .add_service("mon")
2324     .set_description(""),
2325
2326
2327     // AUTH
2328
2329     Option("auth_cluster_required", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2330     .set_default("cephx")
2331     .set_description("authentication methods required by the cluster"),
2332
2333     Option("auth_service_required", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2334     .set_default("cephx")
2335     .set_description("authentication methods required by service daemons"),
2336
2337     Option("auth_client_required", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2338     .set_default("cephx, none")
2339     .set_flag(Option::FLAG_MINIMAL_CONF)
2340     .set_description("authentication methods allowed by clients"),
2341
2342     Option("auth_supported", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2343     .set_default("")
2344     .set_description("authentication methods required (deprecated)"),
2345
2346     Option("max_rotating_auth_attempts", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2347     .set_default(10)
2348     .set_description("number of attempts to initialize rotating keys before giving up"),
2349
2350     Option("rotating_keys_bootstrap_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2351     .set_default(30)
2352     .set_description("timeout for obtaining rotating keys during bootstrap phase (seconds)"),
2353
2354     Option("rotating_keys_renewal_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2355     .set_default(10)
2356     .set_description("timeout for updating rotating keys (seconds)"),
2357
2358     Option("cephx_require_signatures", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2359     .set_default(false)
2360     .set_description(""),
2361
2362     Option("cephx_require_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2363     .set_default(2)
2364     .set_description("Cephx version required (1 = pre-mimic, 2 = mimic+)"),
2365
2366     Option("cephx_cluster_require_signatures", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2367     .set_default(false)
2368     .set_description(""),
2369
2370     Option("cephx_cluster_require_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2371     .set_default(2)
2372     .set_description("Cephx version required by the cluster from clients (1 = pre-mimic, 2 = mimic+)"),
2373
2374     Option("cephx_service_require_signatures", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2375     .set_default(false)
2376     .set_description(""),
2377
2378     Option("cephx_service_require_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2379     .set_default(2)
2380     .set_description("Cephx version required from ceph services (1 = pre-mimic, 2 = mimic+)"),
2381
2382     Option("cephx_sign_messages", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2383     .set_default(true)
2384     .set_description(""),
2385
2386     Option("auth_mon_ticket_ttl", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2387     .set_default(72_hr)
2388     .set_description(""),
2389
2390     Option("auth_service_ticket_ttl", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2391     .set_default(1_hr)
2392     .set_description(""),
2393
2394     Option("auth_allow_insecure_global_id_reclaim", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2395     .set_default(true)
2396     .set_description("Allow reclaiming global_id without presenting a valid ticket proving previous possession of that global_id")
2397     .set_long_description("Allowing unauthorized global_id (re)use poses a security risk.  Unfortunately, older clients may omit their ticket on reconnects and therefore rely on this being allowed for preserving their global_id for the lifetime of the client instance.  Setting this value to false would immediately prevent new connections from those clients (assuming auth_expose_insecure_global_id_reclaim set to true) and eventually break existing sessions as well (regardless of auth_expose_insecure_global_id_reclaim setting).")
2398     .add_see_also("mon_warn_on_insecure_global_id_reclaim")
2399     .add_see_also("mon_warn_on_insecure_global_id_reclaim_allowed")
2400     .add_see_also("auth_expose_insecure_global_id_reclaim"),
2401
2402     Option("auth_expose_insecure_global_id_reclaim", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2403     .set_default(true)
2404     .set_description("Force older clients that may omit their ticket on reconnects to reconnect as part of establishing a session")
2405     .set_long_description("In permissive mode (auth_allow_insecure_global_id_reclaim set to true), this helps with identifying clients that are not patched.  In enforcing mode (auth_allow_insecure_global_id_reclaim set to false), this is a fail-fast mechanism: don't establish a session that will almost inevitably be broken later.")
2406     .add_see_also("mon_warn_on_insecure_global_id_reclaim")
2407     .add_see_also("mon_warn_on_insecure_global_id_reclaim_allowed")
2408     .add_see_also("auth_allow_insecure_global_id_reclaim"),
2409
2410     Option("auth_debug", Option::TYPE_BOOL, Option::LEVEL_DEV)
2411     .set_default(false)
2412     .set_description(""),
2413
2414     Option("mon_client_hunt_parallel", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2415     .set_default(3)
2416     .set_description(""),
2417
2418     Option("mon_client_hunt_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2419     .set_default(3.0)
2420     .set_description(""),
2421
2422     Option("mon_client_log_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2423     .set_default(1.0)
2424     .set_description("How frequently we send queued cluster log messages to mon"),
2425
2426     Option("mon_client_ping_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2427     .set_default(10.0)
2428     .set_description(""),
2429
2430     Option("mon_client_ping_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2431     .set_default(30.0)
2432     .set_description(""),
2433
2434     Option("mon_client_hunt_interval_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2435     .set_default(1.5)
2436     .set_description(""),
2437
2438     Option("mon_client_hunt_interval_min_multiple", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2439     .set_default(1.0)
2440     .set_description(""),
2441
2442     Option("mon_client_hunt_interval_max_multiple", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2443     .set_default(10.0)
2444     .set_description(""),
2445
2446     Option("mon_client_max_log_entries_per_message", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2447     .set_default(1000)
2448     .set_description(""),
2449
2450     Option("mon_client_directed_command_retry", Option::TYPE_INT, Option::LEVEL_DEV)
2451     .set_default(2)
2452     .set_description("Number of times to try sending a command directed at a specific monitor"),
2453
2454     Option("mon_max_pool_pg_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2455     .set_default(65536)
2456     .set_description(""),
2457
2458     Option("mon_pool_quota_warn_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2459     .set_default(0)
2460     .set_description("percent of quota at which to issue warnings")
2461     .add_service("mgr"),
2462
2463     Option("mon_pool_quota_crit_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2464     .set_default(0)
2465     .set_description("percent of quota at which to issue errors")
2466     .add_service("mgr"),
2467
2468     Option("crush_location", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2469     .set_default("")
2470     .set_description(""),
2471
2472     Option("crush_location_hook", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2473     .set_default("")
2474     .set_description(""),
2475
2476     Option("crush_location_hook_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2477     .set_default(10)
2478     .set_description(""),
2479
2480     Option("objecter_tick_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2481     .set_default(5.0)
2482     .set_description(""),
2483
2484     Option("objecter_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2485     .set_default(10.0)
2486     .set_description("Seconds before in-flight op is considered 'laggy' and we query mon for the latest OSDMap"),
2487
2488     Option("objecter_inflight_op_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2489     .set_default(100_M)
2490     .set_description("Max in-flight data in bytes (both directions)"),
2491
2492     Option("objecter_inflight_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2493     .set_default(1024)
2494     .set_description("Max in-flight operations"),
2495
2496     Option("objecter_completion_locks_per_session", Option::TYPE_UINT, Option::LEVEL_DEV)
2497     .set_default(32)
2498     .set_description(""),
2499
2500     Option("objecter_inject_no_watch_ping", Option::TYPE_BOOL, Option::LEVEL_DEV)
2501     .set_default(false)
2502     .set_description(""),
2503
2504     Option("objecter_retry_writes_after_first_reply", Option::TYPE_BOOL, Option::LEVEL_DEV)
2505     .set_default(false)
2506     .set_description(""),
2507
2508     Option("objecter_debug_inject_relock_delay", Option::TYPE_BOOL, Option::LEVEL_DEV)
2509     .set_default(false)
2510     .set_description(""),
2511
2512     Option("filer_max_purge_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2513     .set_default(10)
2514     .set_description("Max in-flight operations for purging a striped range (e.g., MDS journal)"),
2515
2516     Option("filer_max_truncate_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2517     .set_default(128)
2518     .set_description("Max in-flight operations for truncating/deleting a striped sequence (e.g., MDS journal)"),
2519
2520     Option("journaler_write_head_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2521     .set_default(15)
2522     .set_description("Interval in seconds between journal header updates (to help bound replay time)"),
2523
2524     // * journal object size
2525     Option("journaler_prefetch_periods", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2526     .set_default(10)
2527     .set_min(2)                 // we need at least 2 periods to make progress.
2528     .set_description("Number of striping periods to prefetch while reading MDS journal"),
2529
2530     // * journal object size
2531     Option("journaler_prezero_periods", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2532     .set_default(5)
2533     // we need to zero at least two periods, minimum, to ensure that we
2534     // have a full empty object/period in front of us.
2535     .set_min(2)
2536     .set_description("Number of striping periods to zero head of MDS journal write position"),
2537
2538     // -- OSD --
2539     Option("osd_calc_pg_upmaps_aggressively", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2540     .set_default(true)
2541     .set_flag(Option::FLAG_RUNTIME)
2542     .set_description("try to calculate PG upmaps more aggressively, e.g., "
2543                      "by doing a fairly exhaustive search of existing PGs "
2544                      "that can be unmapped or upmapped"),
2545
2546     Option("osd_calc_pg_upmaps_local_fallback_retries", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2547     .set_default(100)
2548     .set_flag(Option::FLAG_RUNTIME)
2549     .set_description("Maximum number of PGs we can attempt to unmap or upmap "
2550                      "for a specific overfull or underfull osd per iteration "),
2551
2552     Option("osd_numa_prefer_iface", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2553     .set_default(true)
2554     .set_flag(Option::FLAG_STARTUP)
2555     .set_description("prefer IP on network interface on same numa node as storage")
2556     .add_see_also("osd_numa_auto_affinity"),
2557
2558     Option("osd_numa_auto_affinity", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2559     .set_default(true)
2560     .set_flag(Option::FLAG_STARTUP)
2561     .set_description("automatically set affinity to numa node when storage and network match"),
2562
2563     Option("osd_numa_node", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2564     .set_default(-1)
2565     .set_flag(Option::FLAG_STARTUP)
2566     .set_description("set affinity to a numa node (-1 for none)")
2567     .add_see_also("osd_numa_auto_affinity"),
2568
2569     Option("osd_smart_report_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2570     .set_default(5)
2571     .set_description("Timeout (in seconds) for smarctl to run, default is set to 5"),
2572
2573     Option("osd_check_max_object_name_len_on_startup", Option::TYPE_BOOL, Option::LEVEL_DEV)
2574     .set_default(true)
2575     .set_description(""),
2576
2577     Option("osd_max_backfills", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2578     .set_default(1)
2579     .set_flag(Option::FLAG_RUNTIME)
2580     .set_description("Maximum number of concurrent local and remote backfills or recoveries per OSD ")
2581     .set_long_description("There can be osd_max_backfills local reservations AND the same remote reservations per OSD. So a value of 1 lets this OSD participate as 1 PG primary in recovery and 1 shard of another recovering PG."),
2582
2583     Option("osd_min_recovery_priority", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2584     .set_default(0)
2585     .set_description("Minimum priority below which recovery is not performed")
2586     .set_long_description("The purpose here is to prevent the cluster from doing *any* lower priority work (e.g., rebalancing) below this threshold and focus solely on higher priority work (e.g., replicating degraded objects)."),
2587
2588     Option("osd_backfill_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2589     .set_default(30.0)
2590     .set_description("how frequently to retry backfill reservations after being denied (e.g., due to a full OSD)"),
2591
2592     Option("osd_recovery_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2593     .set_default(30.0)
2594     .set_description("how frequently to retry recovery reservations after being denied (e.g., due to a full OSD)"),
2595
2596     Option("osd_agent_max_ops", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2597     .set_default(4)
2598     .set_description("maximum concurrent tiering operations for tiering agent"),
2599
2600     Option("osd_agent_max_low_ops", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2601     .set_default(2)
2602     .set_description("maximum concurrent low-priority tiering operations for tiering agent"),
2603
2604     Option("osd_agent_min_evict_effort", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2605     .set_default(.1)
2606     .set_min_max(0.0, .99)
2607     .set_description("minimum effort to expend evicting clean objects"),
2608
2609     Option("osd_agent_quantize_effort", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2610     .set_default(.1)
2611     .set_description("size of quantize unit for eviction effort"),
2612
2613     Option("osd_agent_delay_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2614     .set_default(5.0)
2615     .set_description("how long agent should sleep if it has no work to do"),
2616
2617     Option("osd_find_best_info_ignore_history_les", Option::TYPE_BOOL, Option::LEVEL_DEV)
2618     .set_default(false)
2619     .set_description("ignore last_epoch_started value when peering AND PROBABLY LOSE DATA")
2620     .set_long_description("THIS IS AN EXTREMELY DANGEROUS OPTION THAT SHOULD ONLY BE USED AT THE DIRECTION OF A DEVELOPER.  It makes peering ignore the last_epoch_started value when peering, which can allow the OSD to believe an OSD has an authoritative view of a PG's contents even when it is in fact old and stale, typically leading to data loss (by believing a stale PG is up to date)."),
2621
2622     Option("osd_agent_hist_halflife", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2623     .set_default(1000)
2624     .set_description("halflife of agent atime and temp histograms"),
2625
2626     Option("osd_agent_slop", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2627     .set_default(.02)
2628     .set_description("slop factor to avoid switching tiering flush and eviction mode"),
2629
2630     Option("osd_uuid", Option::TYPE_UUID, Option::LEVEL_ADVANCED)
2631     .set_default(uuid_d())
2632     .set_flag(Option::FLAG_CREATE)
2633     .set_description("uuid label for a new OSD"),
2634
2635     Option("osd_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2636     .set_default("/var/lib/ceph/osd/$cluster-$id")
2637     .set_flag(Option::FLAG_NO_MON_UPDATE)
2638     .set_description("path to OSD data"),
2639
2640     Option("osd_journal", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2641     .set_default("/var/lib/ceph/osd/$cluster-$id/journal")
2642     .set_flag(Option::FLAG_NO_MON_UPDATE)
2643     .set_description("path to OSD journal (when FileStore backend is in use)"),
2644
2645     Option("osd_journal_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2646     .set_default(5120)
2647     .set_flag(Option::FLAG_CREATE)
2648     .set_description("size of FileStore journal (in MiB)"),
2649
2650     Option("osd_journal_flush_on_shutdown", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2651     .set_default(true)
2652     .set_description("flush FileStore journal contents during clean OSD shutdown"),
2653
2654     Option("osd_compact_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2655     .set_default(false)
2656     .set_description("compact OSD's object store's OMAP on start"),
2657
2658     Option("osd_os_flags", Option::TYPE_UINT, Option::LEVEL_DEV)
2659     .set_default(0)
2660     .set_description("flags to skip filestore omap or journal initialization"),
2661
2662     Option("osd_max_write_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2663     .set_min(4)
2664     .set_default(90)
2665     .set_description("Maximum size of a RADOS write operation in megabytes")
2666     .set_long_description("This setting prevents clients from doing "
2667         "very large writes to RADOS.  If you set this to a value "
2668         "below what clients expect, they will receive an error "
2669         "when attempting to write to the cluster."),
2670
2671     Option("osd_max_pgls", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2672     .set_default(1024)
2673     .set_description("maximum number of results when listing objects in a pool"),
2674
2675     Option("osd_client_message_size_cap", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2676     .set_default(500_M)
2677     .set_description("maximum memory to devote to in-flight client requests")
2678     .set_long_description("If this value is exceeded, the OSD will not read any new client data off of the network until memory is freed."),
2679
2680     Option("osd_client_message_cap", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2681     .set_default(0)
2682     .set_description("maximum number of in-flight client requests"),
2683
2684     Option("osd_crush_update_weight_set", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2685     .set_default(true)
2686     .set_description("update CRUSH weight-set weights when updating weights")
2687     .set_long_description("If this setting is true, we will update the weight-set weights when adjusting an item's weight, effectively making changes take effect immediately, and discarding any previous optimization in the weight-set value.  Setting this value to false will leave it to the balancer to (slowly, presumably) adjust weights to approach the new target value."),
2688
2689     Option("osd_crush_chooseleaf_type", Option::TYPE_INT, Option::LEVEL_DEV)
2690     .set_default(1)
2691     .set_flag(Option::FLAG_CLUSTER_CREATE)
2692     .set_description("default chooseleaf type for osdmaptool --create"),
2693
2694     Option("osd_pool_use_gmt_hitset", Option::TYPE_BOOL, Option::LEVEL_DEV)
2695     .set_default(true)
2696     .set_description("use UTC for hitset timestamps")
2697     .set_long_description("This setting only exists for compatibility with hammer (and older) clusters."),
2698
2699     Option("osd_crush_update_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2700     .set_default(true)
2701     .set_description("update OSD CRUSH location on startup"),
2702
2703     Option("osd_class_update_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2704     .set_default(true)
2705     .set_description("set OSD device class on startup"),
2706
2707     Option("osd_crush_initial_weight", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2708     .set_default(-1)
2709     .set_description("if >= 0, initial CRUSH weight for newly created OSDs")
2710     .set_long_description("If this value is negative, the size of the OSD in TiB is used."),
2711
2712     Option("osd_pool_default_ec_fast_read", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2713     .set_default(false)
2714     .set_description("set ec_fast_read for new erasure-coded pools")
2715     .add_service("mon"),
2716
2717     Option("osd_pool_default_crush_rule", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2718     .set_default(-1)
2719     .set_description("CRUSH rule for newly created pools")
2720     .add_service("mon"),
2721
2722     Option("osd_pool_erasure_code_stripe_unit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2723     .set_default(4_K)
2724     .set_description("the amount of data (in bytes) in a data chunk, per stripe")
2725     .add_service("mon"),
2726
2727     Option("osd_pool_default_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2728     .set_default(3)
2729     .set_min_max(0, 10)
2730     .set_flag(Option::FLAG_RUNTIME)
2731     .set_description("the number of copies of an object for new replicated pools")
2732     .add_service("mon"),
2733
2734     Option("osd_pool_default_min_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2735     .set_default(0)
2736     .set_min_max(0, 255)
2737     .set_flag(Option::FLAG_RUNTIME)
2738     .set_description("the minimal number of copies allowed to write to a degraded pool for new replicated pools")
2739     .set_long_description("0 means no specific default; ceph will use size-size/2")
2740     .add_see_also("osd_pool_default_size")
2741     .add_service("mon"),
2742
2743     Option("osd_pool_default_pg_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2744     .set_default(32)
2745     .set_description("number of PGs for new pools")
2746     .set_flag(Option::FLAG_RUNTIME)
2747     .add_service("mon"),
2748
2749     Option("osd_pool_default_pgp_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2750     .set_default(0)
2751     .set_description("number of PGs for placement purposes (0 to match pg_num)")
2752     .add_see_also("osd_pool_default_pg_num")
2753     .set_flag(Option::FLAG_RUNTIME)
2754     .add_service("mon"),
2755
2756     Option("osd_pool_default_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2757     .set_default("replicated")
2758     .set_enum_allowed({"replicated", "erasure"})
2759     .set_flag(Option::FLAG_RUNTIME)
2760     .set_description("default type of pool to create")
2761     .add_service("mon"),
2762
2763     Option("osd_pool_default_erasure_code_profile", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2764     .set_default("plugin=jerasure technique=reed_sol_van k=2 m=2")
2765     .set_flag(Option::FLAG_RUNTIME)
2766     .set_description("default erasure code profile for new erasure-coded pools")
2767     .add_service("mon"),
2768
2769     Option("osd_erasure_code_plugins", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2770     .set_default("jerasure lrc"
2771   #if defined(HAVE_NASM_X64_AVX2) || defined(HAVE_ARMV8_SIMD)
2772          " isa"
2773   #endif
2774         )
2775     .set_flag(Option::FLAG_STARTUP)
2776     .set_description("erasure code plugins to load")
2777     .add_service("mon")
2778     .add_service("osd"),
2779
2780     Option("osd_allow_recovery_below_min_size", Option::TYPE_BOOL, Option::LEVEL_DEV)
2781     .set_default(true)
2782     .set_description("allow replicated pools to recover with < min_size active members")
2783     .add_service("osd"),
2784
2785     Option("osd_pool_default_flags", Option::TYPE_INT, Option::LEVEL_DEV)
2786     .set_default(0)
2787     .set_description("(integer) flags to set on new pools")
2788     .add_service("mon"),
2789
2790     Option("osd_pool_default_flag_hashpspool", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2791     .set_default(true)
2792     .set_description("set hashpspool (better hashing scheme) flag on new pools")
2793     .add_service("mon"),
2794
2795     Option("osd_pool_default_flag_nodelete", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2796     .set_default(false)
2797     .set_description("set nodelete flag on new pools")
2798     .add_service("mon"),
2799
2800     Option("osd_pool_default_flag_nopgchange", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2801     .set_default(false)
2802     .set_description("set nopgchange flag on new pools")
2803     .add_service("mon"),
2804
2805     Option("osd_pool_default_flag_nosizechange", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2806     .set_default(false)
2807     .set_description("set nosizechange flag on new pools")
2808     .add_service("mon"),
2809
2810     Option("osd_pool_default_hit_set_bloom_fpp", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2811     .set_default(.05)
2812     .set_description("")
2813     .add_see_also("osd_tier_default_cache_hit_set_type")
2814     .add_service("mon"),
2815
2816     Option("osd_pool_default_cache_target_dirty_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2817     .set_default(.4)
2818     .set_description(""),
2819
2820     Option("osd_pool_default_cache_target_dirty_high_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2821     .set_default(.6)
2822     .set_description(""),
2823
2824     Option("osd_pool_default_cache_target_full_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2825     .set_default(.8)
2826     .set_description(""),
2827
2828     Option("osd_pool_default_cache_min_flush_age", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2829     .set_default(0)
2830     .set_description(""),
2831
2832     Option("osd_pool_default_cache_min_evict_age", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2833     .set_default(0)
2834     .set_description(""),
2835
2836     Option("osd_pool_default_cache_max_evict_check_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2837     .set_default(10)
2838     .set_description(""),
2839
2840     Option("osd_pool_default_pg_autoscale_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2841     .set_default("on")
2842     .set_flag(Option::FLAG_RUNTIME)
2843     .set_enum_allowed({"off", "warn", "on"})
2844     .set_description("Default PG autoscaling behavior for new pools"),
2845
2846     Option("osd_pool_default_read_lease_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2847     .set_default(.8)
2848     .set_flag(Option::FLAG_RUNTIME)
2849     .set_description("Default read_lease_ratio for a pool, as a multiple of osd_heartbeat_grace")
2850     .set_long_description("This should be <= 1.0 so that the read lease will have expired by the time we decide to mark a peer OSD down.")
2851     .add_see_also("osd_heartbeat_grace"),
2852
2853     Option("osd_hit_set_min_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2854     .set_default(1000)
2855     .set_description(""),
2856
2857     Option("osd_hit_set_max_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2858     .set_default(100000)
2859     .set_description(""),
2860
2861     Option("osd_hit_set_namespace", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2862     .set_default(".ceph-internal")
2863     .set_description(""),
2864
2865     Option("osd_tier_promote_max_objects_sec", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2866     .set_default(25)
2867     .set_description(""),
2868
2869     Option("osd_tier_promote_max_bytes_sec", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2870     .set_default(5_M)
2871     .set_description(""),
2872
2873     Option("osd_tier_default_cache_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2874     .set_default("writeback")
2875     .set_enum_allowed({"none", "writeback", "forward",
2876                        "readonly", "readforward", "readproxy", "proxy"})
2877     .set_flag(Option::FLAG_RUNTIME)
2878     .set_description(""),
2879
2880     Option("osd_tier_default_cache_hit_set_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2881     .set_default(4)
2882     .set_description(""),
2883
2884     Option("osd_tier_default_cache_hit_set_period", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2885     .set_default(1200)
2886     .set_description(""),
2887
2888     Option("osd_tier_default_cache_hit_set_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2889     .set_default("bloom")
2890     .set_enum_allowed({"bloom", "explicit_hash", "explicit_object"})
2891     .set_flag(Option::FLAG_RUNTIME)
2892     .set_description(""),
2893
2894     Option("osd_tier_default_cache_min_read_recency_for_promote", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2895     .set_default(1)
2896     .set_description("number of recent HitSets the object must appear in to be promoted (on read)"),
2897
2898     Option("osd_tier_default_cache_min_write_recency_for_promote", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2899     .set_default(1)
2900     .set_description("number of recent HitSets the object must appear in to be promoted (on write)"),
2901
2902     Option("osd_tier_default_cache_hit_set_grade_decay_rate", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2903     .set_default(20)
2904     .set_description(""),
2905
2906     Option("osd_tier_default_cache_hit_set_search_last_n", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2907     .set_default(1)
2908     .set_description(""),
2909
2910     Option("osd_objecter_finishers", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2911     .set_default(1)
2912     .set_flag(Option::FLAG_STARTUP)
2913     .set_description(""),
2914
2915     Option("osd_map_dedup", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2916     .set_default(true)
2917     .set_description(""),
2918
2919     Option("osd_map_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2920     .set_default(50)
2921     .set_description(""),
2922
2923     Option("osd_map_message_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2924     .set_default(40)
2925     .set_description("maximum number of OSDMaps to include in a single message"),
2926
2927     Option("osd_map_message_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2928     .set_default(10_M)
2929     .set_description("maximum number of bytes worth of OSDMaps to include in a single message"),
2930
2931     Option("osd_map_share_max_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2932     .set_default(40)
2933     .set_description(""),
2934
2935     Option("osd_pg_epoch_max_lag_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2936     .set_default(2.0)
2937     .set_description("Max multiple of the map cache that PGs can lag before we throttle map injest")
2938     .add_see_also("osd_map_cache_size"),
2939
2940     Option("osd_inject_bad_map_crc_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2941     .set_default(0)
2942     .set_description(""),
2943
2944     Option("osd_inject_failure_on_pg_removal", Option::TYPE_BOOL, Option::LEVEL_DEV)
2945     .set_default(false)
2946     .set_description(""),
2947
2948     Option("osd_max_markdown_period", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2949     .set_default(600)
2950     .set_description(""),
2951
2952     Option("osd_max_markdown_count", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2953     .set_default(5)
2954     .set_description(""),
2955
2956     Option("osd_op_pq_max_tokens_per_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2957     .set_default(4194304)
2958     .set_description(""),
2959
2960     Option("osd_op_pq_min_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2961     .set_default(65536)
2962     .set_description(""),
2963
2964     Option("osd_recover_clone_overlap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2965     .set_default(true)
2966     .set_description(""),
2967
2968     Option("osd_num_cache_shards", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2969     .set_default(32)
2970     .set_flag(Option::FLAG_STARTUP)
2971     .set_description("The number of cache shards to use in the object store."),
2972
2973     Option("osd_op_num_threads_per_shard", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2974     .set_default(0)
2975     .set_flag(Option::FLAG_STARTUP)
2976     .set_description(""),
2977
2978     Option("osd_op_num_threads_per_shard_hdd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2979     .set_default(1)
2980     .set_flag(Option::FLAG_STARTUP)
2981     .set_description("")
2982     .add_see_also("osd_op_num_threads_per_shard"),
2983
2984     Option("osd_op_num_threads_per_shard_ssd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2985     .set_default(2)
2986     .set_flag(Option::FLAG_STARTUP)
2987     .set_description("")
2988     .add_see_also("osd_op_num_threads_per_shard"),
2989
2990     Option("osd_op_num_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2991     .set_default(0)
2992     .set_flag(Option::FLAG_STARTUP)
2993     .set_description(""),
2994
2995     Option("osd_op_num_shards_hdd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2996     .set_default(5)
2997     .set_flag(Option::FLAG_STARTUP)
2998     .set_description("")
2999     .add_see_also("osd_op_num_shards"),
3000
3001     Option("osd_op_num_shards_ssd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3002     .set_default(8)
3003     .set_flag(Option::FLAG_STARTUP)
3004     .set_description("")
3005     .add_see_also("osd_op_num_shards"),
3006
3007     Option("osd_skip_data_digest", Option::TYPE_BOOL, Option::LEVEL_DEV)
3008     .set_default(false)
3009     .set_description("Do not store full-object checksums if the backend (bluestore) does its own checksums.  Only usable with all BlueStore OSDs."),
3010
3011     Option("osd_op_queue", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3012     .set_default("wpq")
3013     .set_enum_allowed( { "wpq", "mclock_scheduler", "debug_random" } )
3014     .set_description("which operation priority queue algorithm to use")
3015     .set_long_description("which operation priority queue algorithm to use; "
3016                           "mclock_scheduler is currently experimental")
3017     .add_see_also("osd_op_queue_cut_off"),
3018
3019     Option("osd_op_queue_cut_off", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3020     .set_default("high")
3021     .set_enum_allowed( { "low", "high", "debug_random" } )
3022     .set_description("the threshold between high priority ops and low priority ops")
3023     .set_long_description("the threshold between high priority ops that use strict priority ordering and low priority ops that use a fairness algorithm that may or may not incorporate priority")
3024     .add_see_also("osd_op_queue"),
3025
3026     Option("osd_mclock_scheduler_client_res", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3027     .set_default(1)
3028     .set_description("IO proportion reserved for each client (default)")
3029     .set_long_description("Only considered for osd_op_queue = mclock_scheduler")
3030     .add_see_also("osd_op_queue"),
3031
3032     Option("osd_mclock_scheduler_client_wgt", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3033     .set_default(1)
3034     .set_description("IO share for each client (default) over reservation")
3035     .set_long_description("Only considered for osd_op_queue = mclock_scheduler")
3036     .add_see_also("osd_op_queue"),
3037
3038     Option("osd_mclock_scheduler_client_lim", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3039     .set_default(999999)
3040     .set_description("IO limit for each client (default) over reservation")
3041     .set_long_description("Only considered for osd_op_queue = mclock_scheduler")
3042     .add_see_also("osd_op_queue"),
3043
3044     Option("osd_mclock_scheduler_background_recovery_res", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3045     .set_default(1)
3046     .set_description("IO proportion reserved for background recovery (default)")
3047     .set_long_description("Only considered for osd_op_queue = mclock_scheduler")
3048     .add_see_also("osd_op_queue"),
3049
3050     Option("osd_mclock_scheduler_background_recovery_wgt", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3051     .set_default(1)
3052     .set_description("IO share for each background recovery over reservation")
3053     .set_long_description("Only considered for osd_op_queue = mclock_scheduler")
3054     .add_see_also("osd_op_queue"),
3055
3056     Option("osd_mclock_scheduler_background_recovery_lim", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3057     .set_default(999999)
3058     .set_description("IO limit for background recovery over reservation")
3059     .set_long_description("Only considered for osd_op_queue = mclock_scheduler")
3060     .add_see_also("osd_op_queue"),
3061
3062     Option("osd_mclock_scheduler_background_best_effort_res", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3063     .set_default(1)
3064     .set_description("IO proportion reserved for background best_effort (default)")
3065     .set_long_description("Only considered for osd_op_queue = mclock_scheduler")
3066     .add_see_also("osd_op_queue"),
3067
3068     Option("osd_mclock_scheduler_background_best_effort_wgt", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3069     .set_default(1)
3070     .set_description("IO share for each background best_effort over reservation")
3071     .set_long_description("Only considered for osd_op_queue = mclock_scheduler")
3072     .add_see_also("osd_op_queue"),
3073
3074     Option("osd_mclock_scheduler_background_best_effort_lim", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3075     .set_default(999999)
3076     .set_description("IO limit for background best_effort over reservation")
3077     .set_long_description("Only considered for osd_op_queue = mclock_scheduler")
3078     .add_see_also("osd_op_queue"),
3079
3080     Option("osd_mclock_scheduler_anticipation_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3081     .set_default(0.0)
3082     .set_description("mclock anticipation timeout in seconds")
3083     .set_long_description("the amount of time that mclock waits until the unused resource is forfeited"),
3084
3085     Option("osd_mclock_cost_per_io_usec", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3086     .set_default(0.0)
3087     .set_description("Cost per IO in microseconds to consider per OSD (overrides _ssd and _hdd if non-zero)")
3088     .set_long_description("This option specifies the cost factor to consider in usec per OSD. This is considered by the mclock scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
3089     .set_flag(Option::FLAG_RUNTIME),
3090
3091     Option("osd_mclock_cost_per_io_usec_hdd", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3092     .set_default(25000.0)
3093     .set_description("Cost per IO in microseconds to consider per OSD (for rotational media)")
3094     .set_long_description("This option specifies the cost factor to consider in usec per OSD for rotational device type. This is considered by the mclock_scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
3095     .set_flag(Option::FLAG_RUNTIME),
3096
3097     Option("osd_mclock_cost_per_io_usec_ssd", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3098     .set_default(50.0)
3099     .set_description("Cost per IO in microseconds to consider per OSD (for solid state media)")
3100     .set_long_description("This option specifies the cost factor to consider in usec per OSD for solid state device type. This is considered by the mclock_scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
3101     .set_flag(Option::FLAG_RUNTIME),
3102
3103     Option("osd_mclock_cost_per_byte_usec", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3104     .set_default(0.0)
3105     .set_description("Cost per byte in microseconds to consider per OSD (overrides _ssd and _hdd if non-zero)")
3106     .set_long_description("This option specifies the cost per byte to consider in microseconds per OSD. This is considered by the mclock scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
3107     .set_flag(Option::FLAG_RUNTIME),
3108
3109     Option("osd_mclock_cost_per_byte_usec_hdd", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3110     .set_default(5.2)
3111     .set_description("Cost per byte in microseconds to consider per OSD (for rotational media)")
3112     .set_long_description("This option specifies the cost per byte to consider in microseconds per OSD for rotational device type. This is considered by the mclock_scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
3113     .set_flag(Option::FLAG_RUNTIME),
3114
3115     Option("osd_mclock_cost_per_byte_usec_ssd", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3116     .set_default(0.011)
3117     .set_description("Cost per byte in microseconds to consider per OSD (for solid state media)")
3118     .set_long_description("This option specifies the cost per byte to consider in microseconds per OSD for solid state device type. This is considered by the mclock_scheduler to set an additional cost factor in QoS calculations. Only considered for osd_op_queue = mclock_scheduler")
3119     .set_flag(Option::FLAG_RUNTIME),
3120
3121     Option("osd_mclock_max_capacity_iops", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
3122     .set_default(0.0)
3123     .set_description("Max IOPs capacity (at 4KiB block size) to consider per OSD (overrides _ssd and _hdd if non-zero)")
3124     .set_long_description("This option specifies the max osd capacity in iops per OSD. Helps in QoS calculations when enabling a dmclock profile. Only considered for osd_op_queue = mclock_scheduler")
3125     .set_flag(Option::FLAG_RUNTIME),
3126
3127     Option("osd_mclock_max_capacity_iops_hdd", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
3128     .set_default(315.0)
3129     .set_description("Max IOPs capacity (at 4KiB block size) to consider per OSD (for rotational media)")
3130     .set_long_description("This option specifies the max OSD capacity in iops per OSD. Helps in QoS calculations when enabling a dmclock profile. Only considered for osd_op_queue = mclock_scheduler")
3131     .set_flag(Option::FLAG_RUNTIME),
3132
3133     Option("osd_mclock_max_capacity_iops_ssd", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
3134     .set_default(21500.0)
3135     .set_description("Max IOPs capacity (at 4KiB block size) to consider per OSD (for solid state media)")
3136     .set_long_description("This option specifies the max OSD capacity in iops per OSD. Helps in QoS calculations when enabling a dmclock profile. Only considered for osd_op_queue = mclock_scheduler")
3137     .set_flag(Option::FLAG_RUNTIME),
3138
3139     Option("osd_mclock_profile", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3140     .set_default("high_client_ops")
3141     .set_enum_allowed( { "balanced", "high_recovery_ops", "high_client_ops", "custom" } )
3142     .set_description("Which mclock profile to use")
3143     .set_long_description("This option specifies the mclock profile to enable - one among the set of built-in profiles or a custom profile. Only considered for osd_op_queue = mclock_scheduler")
3144     .set_flag(Option::FLAG_RUNTIME)
3145     .add_see_also("osd_op_queue"),
3146
3147     Option("osd_ignore_stale_divergent_priors", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3148     .set_default(false)
3149     .set_description(""),
3150
3151     Option("osd_read_ec_check_for_errors", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3152     .set_default(false)
3153     .set_description(""),
3154
3155     // Only use clone_overlap for recovery if there are fewer than
3156     // osd_recover_clone_overlap_limit entries in the overlap set
3157     Option("osd_recover_clone_overlap_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3158     .set_default(10)
3159     .set_description("")
3160     .set_flag(Option::FLAG_RUNTIME),
3161
3162     Option("osd_debug_feed_pullee", Option::TYPE_INT, Option::LEVEL_DEV)
3163     .set_default(-1)
3164     .set_description("Feed a pullee, and force primary to pull "
3165                      "a currently missing object from it"),
3166
3167     Option("osd_backfill_scan_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3168     .set_default(64)
3169     .set_description(""),
3170
3171     Option("osd_backfill_scan_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3172     .set_default(512)
3173     .set_description(""),
3174
3175     Option("osd_op_thread_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3176     .set_default(15)
3177     .set_description(""),
3178
3179     Option("osd_op_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3180     .set_default(150)
3181     .set_description(""),
3182
3183     Option("osd_recovery_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3184     .set_default(0)
3185     .set_flag(Option::FLAG_RUNTIME)
3186     .set_description("Time in seconds to sleep before next recovery or backfill op"),
3187
3188     Option("osd_recovery_sleep_hdd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3189     .set_default(0.1)
3190     .set_flag(Option::FLAG_RUNTIME)
3191     .set_description("Time in seconds to sleep before next recovery or backfill op for HDDs"),
3192
3193     Option("osd_recovery_sleep_ssd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3194     .set_default(0)
3195     .set_flag(Option::FLAG_RUNTIME)
3196     .set_description("Time in seconds to sleep before next recovery or backfill op for SSDs")
3197     .add_see_also("osd_recovery_sleep"),
3198
3199     Option("osd_recovery_sleep_hybrid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3200     .set_default(0.025)
3201     .set_flag(Option::FLAG_RUNTIME)
3202     .set_description("Time in seconds to sleep before next recovery or backfill op when data is on HDD and journal is on SSD")
3203     .add_see_also("osd_recovery_sleep"),
3204
3205     Option("osd_snap_trim_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3206     .set_default(0)
3207     .set_description("Time in seconds to sleep before next snap trim (overrides values below)"),
3208
3209     Option("osd_snap_trim_sleep_hdd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3210     .set_default(5)
3211     .set_description("Time in seconds to sleep before next snap trim for HDDs"),
3212
3213     Option("osd_snap_trim_sleep_ssd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3214     .set_default(0)
3215     .set_description("Time in seconds to sleep before next snap trim for SSDs"),
3216
3217     Option("osd_snap_trim_sleep_hybrid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3218     .set_default(2)
3219     .set_description("Time in seconds to sleep before next snap trim when data is on HDD and journal is on SSD"),
3220
3221     Option("osd_scrub_invalid_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3222     .set_default(true)
3223     .set_description(""),
3224
3225     Option("osd_command_thread_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3226     .set_default(10_min)
3227     .set_description(""),
3228
3229     Option("osd_command_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3230     .set_default(15_min)
3231     .set_description(""),
3232
3233     Option("osd_heartbeat_interval", Option::TYPE_INT, Option::LEVEL_DEV)
3234     .set_default(6)
3235     .set_min_max(1, 60)
3236     .set_description("Interval (in seconds) between peer pings"),
3237
3238     Option("osd_heartbeat_grace", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3239     .set_default(20)
3240     .set_description(""),
3241
3242     Option("osd_heartbeat_stale", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3243     .set_default(600)
3244     .set_description("Interval (in seconds) we mark an unresponsive heartbeat peer as stale.")
3245     .set_long_description("Automatically mark unresponsive heartbeat sessions as stale and tear them down. "
3246                           "The primary benefit is that OSD doesn't need to keep a flood of "
3247                           "blocked heartbeat messages around in memory."),
3248
3249     Option("osd_heartbeat_min_peers", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3250     .set_default(10)
3251     .set_description(""),
3252
3253     Option("osd_heartbeat_use_min_delay_socket", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3254     .set_default(false)
3255     .set_description(""),
3256
3257     Option("osd_heartbeat_min_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3258     .set_default(2000)
3259     .set_description("Minimum heartbeat packet size in bytes. Will add dummy payload if heartbeat packet is smaller than this."),
3260
3261     Option("osd_pg_max_concurrent_snap_trims", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3262     .set_default(2)
3263     .set_description(""),
3264
3265     Option("osd_max_trimming_pgs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3266     .set_default(2)
3267     .set_description(""),
3268
3269     Option("osd_heartbeat_min_healthy_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3270     .set_default(.33)
3271     .set_description(""),
3272
3273     Option("osd_mon_heartbeat_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3274     .set_default(30)
3275     .set_description(""),
3276
3277     Option("osd_mon_heartbeat_stat_stale", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3278     .set_default(1_hr)
3279     .set_description("Stop reporting on heartbeat ping times not updated for this many seconds.")
3280     .set_long_description("Stop reporting on old heartbeat information unless this is set to zero"),
3281
3282     Option("osd_mon_report_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3283     .set_default(5)
3284     .set_description("Frequency of OSD reports to mon for peer failures, fullness status changes"),
3285
3286     Option("osd_mon_report_max_in_flight", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3287     .set_default(2)
3288     .set_description(""),
3289
3290     Option("osd_beacon_report_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3291     .set_default(300)
3292     .set_description(""),
3293
3294     Option("osd_pg_stat_report_interval_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3295     .set_default(500)
3296     .set_description(""),
3297
3298     Option("osd_max_snap_prune_intervals_per_epoch", Option::TYPE_UINT, Option::LEVEL_DEV)
3299     .set_default(512)
3300     .set_description("Max number of snap intervals to report to mgr in pg_stat_t"),
3301
3302     Option("osd_default_data_pool_replay_window", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3303     .set_default(45)
3304     .set_description(""),
3305
3306     Option("osd_auto_mark_unfound_lost", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3307     .set_default(false)
3308     .set_description(""),
3309
3310     Option("osd_recovery_delay_start", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3311     .set_default(0)
3312     .set_description(""),
3313
3314     Option("osd_recovery_max_active", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3315     .set_default(0)
3316     .set_flag(Option::FLAG_RUNTIME)
3317     .set_description("Number of simultaneous active recovery operations per OSD (overrides _ssd and _hdd if non-zero)")
3318     .add_see_also("osd_recovery_max_active_hdd")
3319     .add_see_also("osd_recovery_max_active_ssd"),
3320
3321     Option("osd_recovery_max_active_hdd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3322     .set_default(3)
3323     .set_flag(Option::FLAG_RUNTIME)
3324     .set_description("Number of simultaneous active recovery operations per OSD (for rotational devices)")
3325     .add_see_also("osd_recovery_max_active")
3326     .add_see_also("osd_recovery_max_active_ssd"),
3327
3328     Option("osd_recovery_max_active_ssd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3329     .set_default(10)
3330     .set_flag(Option::FLAG_RUNTIME)
3331     .set_description("Number of simultaneous active recovery operations per OSD (for non-rotational solid state devices)")
3332     .add_see_also("osd_recovery_max_active")
3333     .add_see_also("osd_recovery_max_active_hdd"),
3334
3335     Option("osd_recovery_max_single_start", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3336     .set_default(1)
3337     .set_description(""),
3338
3339     Option("osd_recovery_max_chunk", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3340     .set_default(8_M)
3341     .set_description(""),
3342
3343     Option("osd_recovery_max_omap_entries_per_chunk", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3344     .set_default(8096)
3345     .set_description(""),
3346
3347     Option("osd_copyfrom_max_chunk", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3348     .set_default(8_M)
3349     .set_description(""),
3350
3351     Option("osd_push_per_object_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3352     .set_default(1000)
3353     .set_description(""),
3354
3355     Option("osd_max_push_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3356     .set_default(8<<20)
3357     .set_description(""),
3358
3359     Option("osd_max_push_objects", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3360     .set_default(10)
3361     .set_description(""),
3362
3363     Option("osd_max_scrubs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3364     .set_default(1)
3365     .set_description("Maximum concurrent scrubs on a single OSD"),
3366
3367     Option("osd_scrub_during_recovery", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3368     .set_default(false)
3369     .set_description("Allow scrubbing when PGs on the OSD are undergoing recovery"),
3370
3371     Option("osd_repair_during_recovery", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3372     .set_default(false)
3373     .set_description("Allow requested repairing when PGs on the OSD are undergoing recovery"),
3374
3375     Option("osd_scrub_begin_hour", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3376     .set_default(0)
3377     .set_min_max(0, 23)
3378     .set_description("Restrict scrubbing to this hour of the day or later")
3379     .set_long_description("Use osd_scrub_begin_hour=0 and osd_scrub_end_hour=0 for the entire day.")
3380     .add_see_also("osd_scrub_end_hour"),
3381
3382     Option("osd_scrub_end_hour", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3383     .set_default(0)
3384     .set_min_max(0, 23)
3385     .set_description("Restrict scrubbing to hours of the day earlier than this")
3386     .set_long_description("Use osd_scrub_begin_hour=0 and osd_scrub_end_hour=0 for the entire day.")
3387     .add_see_also("osd_scrub_begin_hour"),
3388
3389     Option("osd_scrub_begin_week_day", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3390     .set_default(0)
3391     .set_min_max(0, 6)
3392     .set_description("Restrict scrubbing to this day of the week or later")
3393     .set_long_description("0 = Sunday, 1 = Monday, etc. Use osd_scrub_begin_week_day=0 osd_scrub_end_week_day=0 for the entire week.")
3394     .add_see_also("osd_scrub_end_week_day"),
3395
3396     Option("osd_scrub_end_week_day", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3397     .set_default(0)
3398     .set_min_max(0, 6)
3399     .set_description("Restrict scrubbing to days of the week earlier than this")
3400     .set_long_description("0 = Sunday, 1 = Monday, etc. Use osd_scrub_begin_week_day=0 osd_scrub_end_week_day=0 for the entire week.")
3401     .add_see_also("osd_scrub_begin_week_day"),
3402
3403     Option("osd_scrub_load_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3404     .set_default(0.5)
3405     .set_description("Allow scrubbing when system load divided by number of CPUs is below this value"),
3406
3407     Option("osd_scrub_min_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3408     .set_default(1_day)
3409     .set_description("Scrub each PG no more often than this interval")
3410     .add_see_also("osd_scrub_max_interval"),
3411
3412     Option("osd_scrub_max_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3413     .set_default(7_day)
3414     .set_description("Scrub each PG no less often than this interval")
3415     .add_see_also("osd_scrub_min_interval"),
3416
3417     Option("osd_scrub_interval_randomize_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3418     .set_default(0.5)
3419     .set_description("Ratio of scrub interval to randomly vary")
3420     .set_long_description("This prevents a scrub 'stampede' by randomly varying the scrub intervals so that they are soon uniformly distributed over the week")
3421     .add_see_also("osd_scrub_min_interval"),
3422
3423     Option("osd_scrub_backoff_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3424     .set_default(.66)
3425     .set_long_description("This is the precentage of ticks that do NOT schedule scrubs, 66% means that 1 out of 3 ticks will schedule scrubs")
3426     .set_description("Backoff ratio for scheduling scrubs"),
3427
3428     Option("osd_scrub_chunk_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3429     .set_default(5)
3430     .set_description("Minimum number of objects to scrub in a single chunk")
3431     .add_see_also("osd_scrub_chunk_max"),
3432
3433     Option("osd_scrub_chunk_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3434     .set_default(25)
3435     .set_description("Maximum number of objects to scrub in a single chunk")
3436     .add_see_also("osd_scrub_chunk_min"),
3437
3438     Option("osd_scrub_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3439     .set_default(0)
3440     .set_description("Duration to inject a delay during scrubbing"),
3441
3442     Option("osd_scrub_extended_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3443     .set_default(0)
3444     .set_description("Duration to inject a delay during scrubbing out of scrubbing hours")
3445     .add_see_also("osd_scrub_begin_hour")
3446     .add_see_also("osd_scrub_end_hour")
3447     .add_see_also("osd_scrub_begin_week_day")
3448     .add_see_also("osd_scrub_end_week_day"),
3449
3450     Option("osd_scrub_auto_repair", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3451     .set_default(false)
3452     .set_description("Automatically repair damaged objects detected during scrub"),
3453
3454     Option("osd_scrub_auto_repair_num_errors", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3455     .set_default(5)
3456     .set_description("Maximum number of detected errors to automatically repair")
3457     .add_see_also("osd_scrub_auto_repair"),
3458
3459     Option("osd_scrub_max_preemptions", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3460     .set_default(5)
3461     .set_min_max(0, 30)
3462     .set_description("Set the maximum number of times we will preempt a deep scrub due to a client operation before blocking client IO to complete the scrub"),
3463
3464     Option("osd_deep_scrub_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3465     .set_default(7_day)
3466     .set_description("Deep scrub each PG (i.e., verify data checksums) at least this often"),
3467
3468     Option("osd_deep_scrub_randomize_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3469     .set_default(0.15)
3470     .set_description("Scrubs will randomly become deep scrubs at this rate (0.15 -> 15% of scrubs are deep)")
3471     .set_long_description("This prevents a deep scrub 'stampede' by spreading deep scrubs so they are uniformly distributed over the week"),
3472
3473     Option("osd_deep_scrub_stride", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3474     .set_default(512_K)
3475     .set_description("Number of bytes to read from an object at a time during deep scrub"),
3476
3477     Option("osd_deep_scrub_keys", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3478     .set_default(1024)
3479     .set_description("Number of keys to read from an object at a time during deep scrub"),
3480
3481     Option("osd_deep_scrub_update_digest_min_age", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3482     .set_default(2_hr)
3483     .set_description("Update overall object digest only if object was last modified longer ago than this"),
3484
3485     Option("osd_deep_scrub_large_omap_object_key_threshold", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3486     .set_default(200000)
3487     .set_description("Warn when we encounter an object with more omap keys than this")
3488     .add_service("osd")
3489     .add_see_also("osd_deep_scrub_large_omap_object_value_sum_threshold"),
3490
3491     Option("osd_deep_scrub_large_omap_object_value_sum_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3492     .set_default(1_G)
3493     .set_description("Warn when we encounter an object with more omap key bytes than this")
3494     .add_service("osd")
3495     .add_see_also("osd_deep_scrub_large_omap_object_key_threshold"),
3496
3497     Option("osd_class_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3498     .set_default(CEPH_LIBDIR "/rados-classes")
3499     .set_description(""),
3500
3501     Option("osd_open_classes_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3502     .set_default(true)
3503     .set_description(""),
3504
3505     Option("osd_class_load_list", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3506     .set_default("cephfs hello journal lock log numops " "otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo")
3507     .set_description(""),
3508
3509     Option("osd_class_default_list", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3510     .set_default("cephfs hello journal lock log numops " "otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo")
3511     .set_description(""),
3512
3513     Option("osd_check_for_log_corruption", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3514     .set_default(false)
3515     .set_description(""),
3516
3517     Option("osd_use_stale_snap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3518     .set_default(false)
3519     .set_description(""),
3520
3521     Option("osd_rollback_to_cluster_snap", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3522     .set_default("")
3523     .set_description(""),
3524
3525     Option("osd_default_notify_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3526     .set_default(30)
3527     .set_description(""),
3528
3529     Option("osd_kill_backfill_at", Option::TYPE_INT, Option::LEVEL_DEV)
3530     .set_default(0)
3531     .set_description(""),
3532
3533     Option("osd_pg_epoch_persisted_max_stale", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3534     .set_default(40)
3535     .set_description(""),
3536
3537     Option("osd_target_pg_log_entries_per_osd", Option::TYPE_UINT, Option::LEVEL_DEV)
3538     .set_default(3000 * 100)
3539     .set_description("target number of PG entries total on an OSD - limited per pg by the min and max options below")
3540     .add_see_also("osd_max_pg_log_entries")
3541     .add_see_also("osd_min_pg_log_entries"),
3542
3543     Option("osd_min_pg_log_entries", Option::TYPE_UINT, Option::LEVEL_DEV)
3544     .set_default(250)
3545     .set_description("minimum number of entries to maintain in the PG log")
3546     .add_service("osd")
3547     .add_see_also("osd_max_pg_log_entries")
3548     .add_see_also("osd_pg_log_dups_tracked")
3549     .add_see_also("osd_target_pg_log_entries_per_osd"),
3550
3551     Option("osd_max_pg_log_entries", Option::TYPE_UINT, Option::LEVEL_DEV)
3552     .set_default(10000)
3553     .set_description("maximum number of entries to maintain in the PG log")
3554     .add_service("osd")
3555     .add_see_also("osd_min_pg_log_entries")
3556     .add_see_also("osd_pg_log_dups_tracked")
3557     .add_see_also("osd_target_pg_log_entries_per_osd"),
3558
3559     Option("osd_pg_log_dups_tracked", Option::TYPE_UINT, Option::LEVEL_DEV)
3560     .set_default(3000)
3561     .set_description("how many versions back to track in order to detect duplicate ops; this is combined with both the regular pg log entries and additional minimal dup detection entries")
3562     .add_service("osd")
3563     .add_see_also("osd_min_pg_log_entries")
3564     .add_see_also("osd_max_pg_log_entries"),
3565
3566     Option("osd_object_clean_region_max_num_intervals", Option::TYPE_INT, Option::LEVEL_DEV)
3567     .set_default(10)
3568     .set_description("number of intervals in clean_offsets")
3569     .set_long_description("partial recovery uses multiple intervals to record the clean part of the object"
3570         "when the number of intervals is greater than osd_object_clean_region_max_num_intervals, minimum interval will be trimmed"
3571         "(0 will recovery the entire object data interval)")
3572     .add_service("osd"),
3573
3574     Option("osd_force_recovery_pg_log_entries_factor", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3575     .set_default(1.3)
3576     .set_description(""),
3577
3578     Option("osd_pg_log_trim_min", Option::TYPE_UINT, Option::LEVEL_DEV)
3579     .set_default(100)
3580     .set_description("Minimum number of log entries to trim at once. This lets us trim in larger batches rather than with each write.")
3581     .add_see_also("osd_max_pg_log_entries")
3582     .add_see_also("osd_min_pg_log_entries"),
3583
3584     Option("osd_force_auth_primary_missing_objects", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3585     .set_default(100)
3586     .set_description("Approximate missing objects above which to force auth_log_shard to be primary temporarily"),
3587
3588     Option("osd_async_recovery_min_cost", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3589     .set_default(100)
3590     .set_flag(Option::FLAG_RUNTIME)
3591     .set_description("A mixture measure of number of current log entries difference "
3592                      "and historical missing objects,  above which we switch to use "
3593                      "asynchronous recovery when appropriate"),
3594
3595     Option("osd_max_pg_per_osd_hard_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3596     .set_default(3)
3597     .set_min(1)
3598     .set_description("Maximum number of PG per OSD, a factor of 'mon_max_pg_per_osd'")
3599     .set_long_description("OSD will refuse to instantiate PG if the number of PG it serves exceeds this number.")
3600     .add_see_also("mon_max_pg_per_osd"),
3601
3602     Option("osd_pg_log_trim_max", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3603     .set_default(10000)
3604     .set_description("maximum number of entries to remove at once from the PG log")
3605     .add_service("osd")
3606     .add_see_also("osd_min_pg_log_entries")
3607     .add_see_also("osd_max_pg_log_entries"),
3608
3609     Option("osd_op_complaint_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3610     .set_default(30)
3611     .set_description(""),
3612
3613     Option("osd_command_max_records", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3614     .set_default(256)
3615     .set_description(""),
3616
3617     Option("osd_max_pg_blocked_by", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3618     .set_default(16)
3619     .set_description(""),
3620
3621     Option("osd_op_log_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3622     .set_default(5)
3623     .set_description(""),
3624
3625     Option("osd_backoff_on_unfound", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3626     .set_default(true)
3627     .set_description(""),
3628
3629     Option("osd_backoff_on_degraded", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3630     .set_default(false)
3631     .set_description(""),
3632
3633     Option("osd_backoff_on_peering", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3634     .set_default(false)
3635     .set_description(""),
3636
3637     Option("osd_debug_shutdown", Option::TYPE_BOOL, Option::LEVEL_DEV)
3638     .set_default(false)
3639     .set_description("Turn up debug levels during shutdown"),
3640
3641     Option("osd_debug_crash_on_ignored_backoff", Option::TYPE_BOOL, Option::LEVEL_DEV)
3642     .set_default(false)
3643     .set_description(""),
3644
3645     Option("osd_debug_inject_dispatch_delay_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3646     .set_default(0)
3647     .set_description(""),
3648
3649     Option("osd_debug_inject_dispatch_delay_duration", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3650     .set_default(.1)
3651     .set_description(""),
3652
3653     Option("osd_debug_drop_ping_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3654     .set_default(0)
3655     .set_description(""),
3656
3657     Option("osd_debug_drop_ping_duration", Option::TYPE_INT, Option::LEVEL_DEV)
3658     .set_default(0)
3659     .set_description(""),
3660
3661     Option("osd_debug_op_order", Option::TYPE_BOOL, Option::LEVEL_DEV)
3662     .set_default(false)
3663     .set_description(""),
3664
3665     Option("osd_debug_verify_missing_on_start", Option::TYPE_BOOL, Option::LEVEL_DEV)
3666     .set_default(false)
3667     .set_description(""),
3668
3669     Option("osd_debug_verify_snaps", Option::TYPE_BOOL, Option::LEVEL_DEV)
3670     .set_default(false)
3671     .set_description(""),
3672
3673     Option("osd_debug_verify_stray_on_activate", Option::TYPE_BOOL, Option::LEVEL_DEV)
3674     .set_default(false)
3675     .set_description(""),
3676
3677     Option("osd_debug_skip_full_check_in_backfill_reservation", Option::TYPE_BOOL, Option::LEVEL_DEV)
3678     .set_default(false)
3679     .set_description(""),
3680
3681     Option("osd_debug_reject_backfill_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3682     .set_default(0)
3683     .set_description(""),
3684
3685     Option("osd_debug_inject_copyfrom_error", Option::TYPE_BOOL, Option::LEVEL_DEV)
3686     .set_default(false)
3687     .set_description(""),
3688
3689     Option("osd_debug_misdirected_ops", Option::TYPE_BOOL, Option::LEVEL_DEV)
3690     .set_default(false)
3691     .set_description(""),
3692
3693     Option("osd_debug_skip_full_check_in_recovery", Option::TYPE_BOOL, Option::LEVEL_DEV)
3694     .set_default(false)
3695     .set_description(""),
3696
3697     Option("osd_debug_random_push_read_error", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3698     .set_default(0)
3699     .set_description(""),
3700
3701     Option("osd_debug_verify_cached_snaps", Option::TYPE_BOOL, Option::LEVEL_DEV)
3702     .set_default(false)
3703     .set_description(""),
3704
3705     Option("osd_debug_deep_scrub_sleep", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3706     .set_default(0)
3707     .set_description("Inject an expensive sleep during deep scrub IO to make it easier to induce preemption"),
3708
3709     Option("osd_debug_no_acting_change", Option::TYPE_BOOL, Option::LEVEL_DEV)
3710     .set_default(false),
3711     Option("osd_debug_no_purge_strays", Option::TYPE_BOOL, Option::LEVEL_DEV)
3712     .set_default(false),
3713
3714     Option("osd_debug_pretend_recovery_active", Option::TYPE_BOOL, Option::LEVEL_DEV)
3715     .set_default(false)
3716     .set_description(""),
3717
3718     Option("osd_enable_op_tracker", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3719     .set_default(true)
3720     .set_description(""),
3721
3722     Option("osd_num_op_tracker_shard", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3723     .set_default(32)
3724     .set_description(""),
3725
3726     Option("osd_op_history_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3727     .set_default(20)
3728     .set_description(""),
3729
3730     Option("osd_op_history_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3731     .set_default(600)
3732     .set_description(""),
3733
3734     Option("osd_op_history_slow_op_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3735     .set_default(20)
3736     .set_description(""),
3737
3738     Option("osd_op_history_slow_op_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3739     .set_default(10.0)
3740     .set_description(""),
3741
3742     Option("osd_target_transaction_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3743     .set_default(30)
3744     .set_description(""),
3745
3746     Option("osd_delete_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3747     .set_default(0)
3748     .set_description("Time in seconds to sleep before next removal transaction (overrides values below)"),
3749
3750     Option("osd_delete_sleep_hdd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3751     .set_default(5)
3752     .set_description("Time in seconds to sleep before next removal transaction for HDDs"),
3753
3754     Option("osd_delete_sleep_ssd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3755     .set_default(1)
3756     .set_description("Time in seconds to sleep before next removal transaction for SSDs"),
3757
3758     Option("osd_delete_sleep_hybrid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3759     .set_default(1)
3760     .set_description("Time in seconds to sleep before next removal transaction when data is on HDD and journal is on SSD"),
3761
3762     Option("osd_failsafe_full_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3763     .set_default(.97)
3764     .set_description(""),
3765
3766     Option("osd_fast_shutdown", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3767     .set_default(true)
3768     .set_description("Fast, immediate shutdown")
3769     .set_long_description("Setting this to false makes the OSD do a slower teardown of all state when it receives a SIGINT or SIGTERM or when shutting down for any other reason.  That slow shutdown is primarilyy useful for doing memory leak checking with valgrind."),
3770
3771     Option("osd_fast_shutdown_notify_mon", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3772     .set_default(false)
3773     .set_description("Tell mon about OSD shutdown on immediate shutdown")
3774     .set_long_description("Tell the monitor the OSD is shutting down on immediate shutdown. This helps with cluster log messages from other OSDs reporting it immediately failed.")
3775     .add_see_also({"osd_fast_shutdown", "osd_mon_shutdown_timeout"}),
3776
3777     Option("osd_fast_fail_on_connection_refused", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3778     .set_default(true)
3779     .set_description(""),
3780
3781     Option("osd_pg_object_context_cache_count", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3782     .set_default(64)
3783     .set_description(""),
3784
3785     Option("osd_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3786     .set_default(false)
3787     .set_description(""),
3788
3789     Option("osd_function_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3790     .set_default(false)
3791     .set_description(""),
3792
3793     Option("osd_fast_info", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3794     .set_default(true)
3795     .set_description(""),
3796
3797     Option("osd_debug_pg_log_writeout", Option::TYPE_BOOL, Option::LEVEL_DEV)
3798     .set_default(false)
3799     .set_description(""),
3800
3801     Option("osd_loop_before_reset_tphandle", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3802     .set_default(64)
3803     .set_description(""),
3804
3805     Option("threadpool_default_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3806     .set_default(60)
3807     .set_description(""),
3808
3809     Option("threadpool_empty_queue_max_wait", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3810     .set_default(2)
3811     .set_description(""),
3812
3813     Option("leveldb_log_to_ceph_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3814     .set_default(true)
3815     .set_description(""),
3816
3817     Option("leveldb_write_buffer_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3818     .set_default(8_M)
3819     .set_description(""),
3820
3821     Option("leveldb_cache_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3822     .set_default(128_M)
3823     .set_description(""),
3824
3825     Option("leveldb_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3826     .set_default(0)
3827     .set_description(""),
3828
3829     Option("leveldb_bloom_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3830     .set_default(0)
3831     .set_description(""),
3832
3833     Option("leveldb_max_open_files", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3834     .set_default(0)
3835     .set_description(""),
3836
3837     Option("leveldb_compression", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3838     .set_default(true)
3839     .set_description(""),
3840
3841     Option("leveldb_paranoid", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3842     .set_default(false)
3843     .set_description(""),
3844
3845     Option("leveldb_log", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3846     .set_default("/dev/null")
3847     .set_description(""),
3848
3849     Option("leveldb_compact_on_mount", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3850     .set_default(false)
3851     .set_description(""),
3852
3853     Option("rocksdb_log_to_ceph_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3854     .set_default(true)
3855     .set_description(""),
3856
3857     Option("rocksdb_cache_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3858     .set_default(512_M)
3859     .set_flag(Option::FLAG_RUNTIME)
3860     .set_description(""),
3861
3862     Option("rocksdb_cache_row_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3863     .set_default(0)
3864     .set_description(""),
3865
3866     Option("rocksdb_cache_shard_bits", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3867     .set_default(4)
3868     .set_description(""),
3869
3870     Option("rocksdb_cache_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3871     .set_default("binned_lru")
3872     .set_description(""),
3873
3874     Option("rocksdb_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3875     .set_default(4_K)
3876     .set_description(""),
3877
3878     Option("rocksdb_perf", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3879     .set_default(false)
3880     .set_description(""),
3881
3882     Option("rocksdb_collect_compaction_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3883     .set_default(false)
3884     .set_description(""),
3885
3886     Option("rocksdb_collect_extended_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3887     .set_default(false)
3888     .set_description(""),
3889
3890     Option("rocksdb_collect_memory_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3891     .set_default(false)
3892     .set_description(""),
3893
3894     Option("rocksdb_delete_range_threshold", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3895     .set_default(1048576)
3896     .set_description("The number of keys required to invoke DeleteRange when deleting muliple keys."),
3897
3898     Option("rocksdb_bloom_bits_per_key", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3899     .set_default(20)
3900     .set_description("Number of bits per key to use for RocksDB's bloom filters.")
3901     .set_long_description("RocksDB bloom filters can be used to quickly answer the question of whether or not a key may exist or definitely does not exist in a given RocksDB SST file without having to read all keys into memory.  Using a higher bit value decreases the likelihood of false positives at the expense of additional disk space and memory consumption when the filter is loaded into RAM.  The current default value of 20 was found to provide significant performance gains when getattr calls are made (such as during new object creation in bluestore) without significant memory overhead or cache pollution when combined with rocksdb partitioned index filters.  See: https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters for more information."),
3902
3903     Option("rocksdb_cache_index_and_filter_blocks", Option::TYPE_BOOL, Option::LEVEL_DEV)
3904     .set_default(true)
3905     .set_description("Whether to cache indices and filters in block cache")
3906     .set_long_description("By default RocksDB will load an SST file's index and bloom filters into memory when it is opened and remove them from memory when an SST file is closed.  Thus, memory consumption by indices and bloom filters is directly tied to the number of concurrent SST files allowed to be kept open.  This option instead stores cached indicies and filters in the block cache where they directly compete with other cached data.  By default we set this option to true to better account for and bound rocksdb memory usage and keep filters in memory even when an SST file is closed."),
3907
3908     Option("rocksdb_cache_index_and_filter_blocks_with_high_priority", Option::TYPE_BOOL, Option::LEVEL_DEV)
3909     .set_default(false)
3910     .set_description("Whether to cache indices and filters in the block cache with high priority")
3911     .set_long_description("A downside of setting rocksdb_cache_index_and_filter_blocks to true is that regular data can push indices and filters out of memory.  Setting this option to true means they are cached with higher priority than other data and should typically stay in the block cache."),
3912
3913     Option("rocksdb_pin_l0_filter_and_index_blocks_in_cache", Option::TYPE_BOOL, Option::LEVEL_DEV)
3914     .set_default(false)
3915     .set_description("Whether to pin Level 0 indices and bloom filters in the block cache")
3916     .set_long_description("A downside of setting rocksdb_cache_index_and_filter_blocks to true is that regular data can push indices and filters out of memory.  Setting this option to true means that level 0 SST files will always have their indices and filters pinned in the block cache."),
3917
3918     Option("rocksdb_index_type", Option::TYPE_STR, Option::LEVEL_DEV)
3919     .set_default("binary_search")
3920     .set_description("Type of index for SST files: binary_search, hash_search, two_level")
3921     .set_long_description("This option controls the table index type.  binary_search is a space efficient index block that is optimized for block-search-based index. hash_search may improve prefix lookup performance at the expense of higher disk and memory usage and potentially slower compactions.  two_level is an experimental index type that uses two binary search indexes and works in conjunction with partition filters.  See: http://rocksdb.org/blog/2017/05/12/partitioned-index-filter.html"),
3922
3923     Option("rocksdb_partition_filters", Option::TYPE_BOOL, Option::LEVEL_DEV)
3924     .set_default(false)
3925     .set_description("(experimental) partition SST index/filters into smaller blocks")
3926     .set_long_description("This is an experimental option for rocksdb that works in conjunction with two_level indices to avoid having to keep the entire filter/index in cache when cache_index_and_filter_blocks is true.  The idea is to keep a much smaller top-level index in heap/cache and then opportunistically cache the lower level indices.  See: https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters"),
3927
3928     Option("rocksdb_metadata_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
3929     .set_default(4_K)
3930     .set_description("The block size for index partitions. (0 = rocksdb default)"),
3931
3932     Option("mon_rocksdb_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3933     .set_default("write_buffer_size=33554432,"
3934                  "compression=kNoCompression,"
3935                  "level_compaction_dynamic_level_bytes=true")
3936     .set_description(""),
3937
3938     Option("osd_client_op_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3939     .set_default(63)
3940     .set_description(""),
3941
3942     Option("osd_recovery_op_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3943     .set_default(3)
3944     .set_description("Priority to use for recovery operations if not specified for the pool"),
3945
3946     Option("osd_peering_op_priority", Option::TYPE_UINT, Option::LEVEL_DEV)
3947     .set_default(255)
3948     .set_description(""),
3949
3950     Option("osd_snap_trim_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3951     .set_default(5)
3952     .set_description(""),
3953
3954     Option("osd_snap_trim_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3955     .set_default(1<<20)
3956     .set_description(""),
3957
3958     Option("osd_pg_delete_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3959     .set_default(5)
3960     .set_description(""),
3961
3962     Option("osd_pg_delete_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3963     .set_default(1<<20)
3964     .set_description(""),
3965
3966     Option("osd_scrub_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3967     .set_default(5)
3968     .set_description("Priority for scrub operations in work queue"),
3969
3970     Option("osd_scrub_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3971     .set_default(50<<20)
3972     .set_description("Cost for scrub operations in work queue"),
3973
3974     Option("osd_requested_scrub_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3975     .set_default(120)
3976     .set_description(""),
3977
3978     Option("osd_recovery_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3979     .set_default(5)
3980     .set_description("Priority of recovery in the work queue")
3981     .set_long_description("Not related to a pool's recovery_priority"),
3982
3983     Option("osd_recovery_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3984     .set_default(20<<20)
3985     .set_description(""),
3986
3987     Option("osd_recovery_op_warn_multiple", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3988     .set_default(16)
3989     .set_description(""),
3990
3991     Option("osd_mon_shutdown_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3992     .set_default(5)
3993     .set_description(""),
3994
3995     Option("osd_shutdown_pgref_assert", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3996     .set_default(false)
3997     .set_description(""),
3998
3999     Option("osd_max_object_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4000     .set_default(128_M)
4001     .set_description(""),
4002
4003     Option("osd_max_object_name_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4004     .set_default(2048)
4005     .set_description(""),
4006
4007     Option("osd_max_object_namespace_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4008     .set_default(256)
4009     .set_description(""),
4010
4011     Option("osd_max_attr_name_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4012     .set_default(100)
4013     .set_description(""),
4014
4015     Option("osd_max_attr_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4016     .set_default(0)
4017     .set_description(""),
4018
4019     Option("osd_max_omap_entries_per_request", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4020     .set_default(1024)
4021     .set_description(""),
4022
4023     Option("osd_max_omap_bytes_per_request", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4024     .set_default(1_G)
4025     .set_description(""),
4026
4027     Option("osd_max_write_op_reply_len", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4028     .set_default(32)
4029     .set_description("Max size of the per-op payload for requests with the RETURNVEC flag set")
4030     .set_long_description("This value caps the amount of data (per op; a request may have many ops) that will be sent back to the client and recorded in the PG log."),
4031
4032     Option("osd_objectstore", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4033     .set_default("bluestore")
4034     .set_enum_allowed({"bluestore", "filestore", "memstore", "kstore"})
4035     .set_flag(Option::FLAG_CREATE)
4036     .set_description("backend type for an OSD (like filestore or bluestore)"),
4037
4038     Option("osd_objectstore_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4039     .set_default(false)
4040     .set_description(""),
4041
4042     Option("osd_objectstore_fuse", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4043     .set_default(false)
4044     .set_description(""),
4045
4046     Option("osd_bench_small_size_max_iops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4047     .set_default(100)
4048     .set_description(""),
4049
4050     Option("osd_bench_large_size_max_throughput", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4051     .set_default(100_M)
4052     .set_description(""),
4053
4054     Option("osd_bench_max_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4055     .set_default(64_M)
4056     .set_description(""),
4057
4058     Option("osd_bench_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4059     .set_default(30)
4060     .set_description(""),
4061
4062     Option("osd_blkin_trace_all", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4063     .set_default(false)
4064     .set_description(""),
4065
4066     Option("osdc_blkin_trace_all", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4067     .set_default(false)
4068     .set_description(""),
4069
4070     Option("osd_discard_disconnected_ops", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4071     .set_default(true)
4072     .set_description(""),
4073
4074     Option("osd_memory_target", Option::TYPE_SIZE, Option::LEVEL_BASIC)
4075     .set_default(4_G)
4076     .set_min(896_M)
4077     .set_flag(Option::FLAG_RUNTIME)
4078     .add_see_also("bluestore_cache_autotune")
4079     .add_see_also("osd_memory_cache_min")
4080     .add_see_also("osd_memory_base")
4081     .set_description("When tcmalloc and cache autotuning is enabled, try to keep this many bytes mapped in memory.")
4082     .set_long_description("The minimum value must be at least equal to osd_memory_base + osd_memory_cache_min."),
4083
4084     Option("osd_memory_target_cgroup_limit_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4085     .set_default(0.8)
4086     .set_min_max(0.0, 1.0)
4087     .add_see_also("osd_memory_target")
4088     .set_description("Set the default value for osd_memory_target to the cgroup memory limit (if set) times this value")
4089     .set_long_description("A value of 0 disables this feature."),
4090
4091     Option("osd_memory_base", Option::TYPE_SIZE, Option::LEVEL_DEV)
4092     .set_default(768_M)
4093     .set_flag(Option::FLAG_RUNTIME)
4094     .add_see_also("bluestore_cache_autotune")
4095     .set_description("When tcmalloc and cache autotuning is enabled, estimate the minimum amount of memory in bytes the OSD will need."),
4096
4097     Option("osd_memory_expected_fragmentation", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4098     .set_default(0.15)
4099     .set_min_max(0.0, 1.0)
4100     .set_flag(Option::FLAG_RUNTIME)
4101     .add_see_also("bluestore_cache_autotune")
4102     .set_description("When tcmalloc and cache autotuning is enabled, estimate the percent of memory fragmentation."),
4103
4104     Option("osd_memory_cache_min", Option::TYPE_SIZE, Option::LEVEL_DEV)
4105     .set_default(128_M)
4106     .set_min(128_M)
4107     .set_flag(Option::FLAG_RUNTIME)
4108     .add_see_also("bluestore_cache_autotune")
4109     .set_description("When tcmalloc and cache autotuning is enabled, set the minimum amount of memory used for caches."),
4110
4111     Option("osd_memory_cache_resize_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4112     .set_default(1)
4113     .add_see_also("bluestore_cache_autotune")
4114     .set_description("When tcmalloc and cache autotuning is enabled, wait this many seconds between resizing caches."),
4115
4116     Option("memstore_device_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4117     .set_default(1_G)
4118     .set_description(""),
4119
4120     Option("memstore_page_set", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4121     .set_default(false)
4122     .set_description(""),
4123
4124     Option("memstore_page_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4125     .set_default(64_K)
4126     .set_description(""),
4127
4128     Option("memstore_debug_omit_block_device_write", Option::TYPE_BOOL, Option::LEVEL_DEV)
4129     .set_default(false)
4130     .add_see_also("bluestore_debug_omit_block_device_write")
4131     .set_description("write metadata only"),
4132
4133     Option("objectstore_blackhole", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4134     .set_default(false)
4135     .set_description(""),
4136
4137     // --------------------------
4138     // bluestore
4139
4140     Option("bdev_debug_inflight_ios", Option::TYPE_BOOL, Option::LEVEL_DEV)
4141     .set_default(false)
4142     .set_description(""),
4143
4144     Option("bdev_inject_crash", Option::TYPE_INT, Option::LEVEL_DEV)
4145     .set_default(0)
4146     .set_description(""),
4147
4148     Option("bdev_inject_crash_flush_delay", Option::TYPE_INT, Option::LEVEL_DEV)
4149     .set_default(2)
4150     .set_description(""),
4151
4152     Option("bdev_aio", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4153     .set_default(true)
4154     .set_description(""),
4155
4156     Option("bdev_aio_poll_ms", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4157     .set_default(250)
4158     .set_description(""),
4159
4160     Option("bdev_aio_max_queue_depth", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4161     .set_default(1024)
4162     .set_description(""),
4163
4164     Option("bdev_aio_reap_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4165     .set_default(16)
4166     .set_description(""),
4167
4168     Option("bdev_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4169     .set_default(4_K)
4170     .set_description(""),
4171
4172     Option("bdev_debug_aio", Option::TYPE_BOOL, Option::LEVEL_DEV)
4173     .set_default(false)
4174     .set_description(""),
4175
4176     Option("bdev_debug_aio_suicide_timeout", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4177     .set_default(60.0)
4178     .set_description(""),
4179
4180     Option("bdev_debug_aio_log_age", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4181     .set_default(5.0)
4182     .set_description(""),
4183
4184     Option("bdev_nvme_unbind_from_kernel", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4185     .set_default(false)
4186     .set_description(""),
4187
4188     Option("bdev_enable_discard", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4189     .set_default(false)
4190     .set_description(""),
4191
4192     Option("bdev_async_discard", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4193     .set_default(false)
4194     .set_description(""),
4195
4196     Option("bdev_flock_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4197     .set_default(0.1)
4198     .set_description("interval to retry the flock"),
4199
4200     Option("bdev_flock_retry", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4201     .set_default(3)
4202     .set_description("times to retry the flock")
4203     .set_long_description(
4204         "The number of times to retry on getting the block device lock. "
4205         "Programs such as systemd-udevd may compete with Ceph for this lock. "
4206         "0 means 'unlimited'."),
4207
4208     Option("bluefs_alloc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4209     .set_default(1_M)
4210     .set_description("Allocation unit size for DB and WAL devices"),
4211
4212     Option("bluefs_shared_alloc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4213     .set_default(64_K)
4214     .set_description("Allocation unit size for primary/shared device"),
4215
4216     Option("bluefs_max_prefetch", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4217     .set_default(1_M)
4218     .set_description(""),
4219
4220     Option("bluefs_min_log_runway", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4221     .set_default(1_M)
4222     .set_description(""),
4223
4224     Option("bluefs_max_log_runway", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4225     .set_default(4194304)
4226     .set_description(""),
4227
4228     Option("bluefs_log_compact_min_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4229     .set_default(5.0)
4230     .set_description(""),
4231
4232     Option("bluefs_log_compact_min_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4233     .set_default(16_M)
4234     .set_description(""),
4235
4236     Option("bluefs_min_flush_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4237     .set_default(512_K)
4238     .set_description(""),
4239
4240     Option("bluefs_compact_log_sync", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4241     .set_default(false)
4242     .set_description(""),
4243
4244     Option("bluefs_buffered_io", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4245     .set_default(true)
4246     .set_description("Enabled buffered IO for bluefs reads.")
4247     .set_long_description("When this option is enabled, bluefs will in some cases perform buffered reads.  This allows the kernel page cache to act as a secondary cache for things like RocksDB compaction.  For example, if the rocksdb block cache isn't large enough to hold blocks from the compressed SST files itself, they can be read from page cache instead of from the disk."),
4248
4249     Option("bluefs_sync_write", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4250     .set_default(false)
4251     .set_description(""),
4252
4253     Option("bluefs_allocator", Option::TYPE_STR, Option::LEVEL_DEV)
4254     .set_default("hybrid")
4255     .set_enum_allowed({"bitmap", "stupid", "avl", "hybrid"})
4256     .set_description(""),
4257
4258     Option("bluefs_log_replay_check_allocations", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4259       .set_default(true)
4260       .set_description("Enables checks for allocations consistency during log replay"),
4261
4262     Option("bluefs_replay_recovery", Option::TYPE_BOOL, Option::LEVEL_DEV)
4263     .set_default(false)
4264     .set_description("Attempt to read bluefs log so large that it became unreadable.")
4265     .set_long_description("If BlueFS log grows to extreme sizes (200GB+) it is likely that it becames unreadable. "
4266                           "This options enables heuristics that scans devices for missing data. "
4267                           "DO NOT ENABLE BY DEFAULT"),
4268
4269     Option("bluefs_replay_recovery_disable_compact", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4270     .set_default(false)
4271     .set_description(""),
4272
4273     Option("bluefs_check_for_zeros", Option::TYPE_BOOL, Option::LEVEL_DEV)
4274     .set_default(false)
4275     .set_flag(Option::FLAG_RUNTIME)
4276     .set_description("Check data read for suspicious pages")
4277     .set_long_description("Looks into data read to check if there is a 4K block entirely filled with zeros. "
4278                           "If this happens, we re-read data. If there is difference, we print error to log.")
4279     .add_see_also("bluestore_retry_disk_reads"),
4280
4281     Option("bluestore_bluefs", Option::TYPE_BOOL, Option::LEVEL_DEV)
4282     .set_default(true)
4283     .set_flag(Option::FLAG_CREATE)
4284     .set_description("Use BlueFS to back rocksdb")
4285     .set_long_description("BlueFS allows rocksdb to share the same physical device(s) as the rest of BlueStore.  It should be used in all cases unless testing/developing an alternative metadata database for BlueStore."),
4286
4287     Option("bluestore_bluefs_env_mirror", Option::TYPE_BOOL, Option::LEVEL_DEV)
4288     .set_default(false)
4289     .set_flag(Option::FLAG_CREATE)
4290     .set_description("Mirror bluefs data to file system for testing/validation"),
4291
4292     Option("bluestore_bluefs_alloc_failure_dump_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4293     .set_default(0)
4294     .set_description("How frequently (in seconds) to dump allocator on"
4295       "BlueFS space allocation failure"),
4296
4297     Option("bluestore_spdk_mem", Option::TYPE_SIZE, Option::LEVEL_DEV)
4298     .set_default(512)
4299     .set_description("Amount of dpdk memory size in MB")
4300     .set_long_description("If running multiple SPDK instances per node, you must specify the amount of dpdk memory size in MB each instance will use, to make sure each instance uses its own dpdk memory"),
4301
4302     Option("bluestore_spdk_coremask", Option::TYPE_STR, Option::LEVEL_DEV)
4303     .set_default("0x1")
4304     .set_description("A hexadecimal bit mask of the cores to run on. Note the core numbering can change between platforms and should be determined beforehand"),
4305
4306     Option("bluestore_spdk_max_io_completion", Option::TYPE_UINT, Option::LEVEL_DEV)
4307     .set_default(0)
4308     .set_description("Maximal I/Os to be batched completed while checking queue pair completions, 0 means let spdk library determine it"),
4309
4310     Option("bluestore_spdk_io_sleep", Option::TYPE_UINT, Option::LEVEL_DEV)
4311     .set_default(5)
4312     .set_description("Time period to wait if there is no completed I/O from polling"),
4313
4314     Option("bluestore_block_path", Option::TYPE_STR, Option::LEVEL_DEV)
4315     .set_default("")
4316     .set_flag(Option::FLAG_CREATE)
4317     .set_description("Path to block device/file"),
4318
4319     Option("bluestore_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4320     .set_default(100_G)
4321     .set_flag(Option::FLAG_CREATE)
4322     .set_description("Size of file to create for backing bluestore"),
4323
4324     Option("bluestore_block_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
4325     .set_default(true)
4326     .set_flag(Option::FLAG_CREATE)
4327     .set_description("Create bluestore_block_path if it doesn't exist")
4328     .add_see_also("bluestore_block_path").add_see_also("bluestore_block_size"),
4329
4330     Option("bluestore_block_db_path", Option::TYPE_STR, Option::LEVEL_DEV)
4331     .set_default("")
4332     .set_flag(Option::FLAG_CREATE)
4333     .set_description("Path for db block device"),
4334
4335     Option("bluestore_block_db_size", Option::TYPE_UINT, Option::LEVEL_DEV)
4336     .set_default(0)
4337     .set_flag(Option::FLAG_CREATE)
4338     .set_description("Size of file to create for bluestore_block_db_path"),
4339
4340     Option("bluestore_block_db_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
4341     .set_default(false)
4342     .set_flag(Option::FLAG_CREATE)
4343     .set_description("Create bluestore_block_db_path if it doesn't exist")
4344     .add_see_also("bluestore_block_db_path")
4345     .add_see_also("bluestore_block_db_size"),
4346
4347     Option("bluestore_block_wal_path", Option::TYPE_STR, Option::LEVEL_DEV)
4348     .set_default("")
4349     .set_flag(Option::FLAG_CREATE)
4350     .set_description("Path to block device/file backing bluefs wal"),
4351
4352     Option("bluestore_block_wal_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4353     .set_default(96_M)
4354     .set_flag(Option::FLAG_CREATE)
4355     .set_description("Size of file to create for bluestore_block_wal_path"),
4356
4357     Option("bluestore_block_wal_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
4358     .set_default(false)
4359     .set_flag(Option::FLAG_CREATE)
4360     .set_description("Create bluestore_block_wal_path if it doesn't exist")
4361     .add_see_also("bluestore_block_wal_path")
4362     .add_see_also("bluestore_block_wal_size"),
4363
4364     Option("bluestore_block_preallocate_file", Option::TYPE_BOOL, Option::LEVEL_DEV)
4365     .set_default(false)
4366     .set_flag(Option::FLAG_CREATE)
4367     .set_description("Preallocate file created via bluestore_block*_create"),
4368
4369     Option("bluestore_ignore_data_csum", Option::TYPE_BOOL, Option::LEVEL_DEV)
4370     .set_default(false)
4371     .set_flag(Option::FLAG_RUNTIME)
4372     .set_description("Ignore checksum errors on read and do not generate an EIO error"),
4373
4374     Option("bluestore_csum_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4375     .set_default("crc32c")
4376     .set_enum_allowed({"none", "crc32c", "crc32c_16", "crc32c_8", "xxhash32", "xxhash64"})
4377     .set_flag(Option::FLAG_RUNTIME)
4378     .set_description("Default checksum algorithm to use")
4379     .set_long_description("crc32c, xxhash32, and xxhash64 are available.  The _16 and _8 variants use only a subset of the bits for more compact (but less reliable) checksumming."),
4380
4381     Option("bluestore_retry_disk_reads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4382     .set_default(3)
4383     .set_min_max(0, 255)
4384     .set_flag(Option::FLAG_RUNTIME)
4385     .set_description("Number of read retries on checksum validation error")
4386     .set_long_description("Retries to read data from the disk this many times when checksum validation fails to handle spurious read errors gracefully."),
4387
4388     Option("bluestore_min_alloc_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4389     .set_default(0)
4390     .set_flag(Option::FLAG_CREATE)
4391     .set_description("Minimum allocation size to allocate for an object")
4392     .set_long_description("A smaller allocation size generally means less data is read and then rewritten when a copy-on-write operation is triggered (e.g., when writing to something that was recently snapshotted).  Similarly, less data is journaled before performing an overwrite (writes smaller than min_alloc_size must first pass through the BlueStore journal).  Larger values of min_alloc_size reduce the amount of metadata required to describe the on-disk layout and reduce overall fragmentation."),
4393
4394     Option("bluestore_min_alloc_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4395     .set_default(4_K)
4396     .set_flag(Option::FLAG_CREATE)
4397     .set_description("Default min_alloc_size value for rotational media")
4398     .add_see_also("bluestore_min_alloc_size"),
4399
4400     Option("bluestore_min_alloc_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4401     .set_default(4_K)
4402     .set_flag(Option::FLAG_CREATE)
4403     .set_description("Default min_alloc_size value for non-rotational (solid state)  media")
4404     .add_see_also("bluestore_min_alloc_size"),
4405
4406     Option("bluestore_max_alloc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4407     .set_default(0)
4408     .set_flag(Option::FLAG_CREATE)
4409     .set_description("Maximum size of a single allocation (0 for no max)"),
4410
4411     Option("bluestore_prefer_deferred_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4412     .set_default(0)
4413     .set_flag(Option::FLAG_RUNTIME)
4414     .set_description("Writes smaller than this size will be written to the journal and then asynchronously written to the device.  This can be beneficial when using rotational media where seeks are expensive, and is helpful both with and without solid state journal/wal devices."),
4415
4416     Option("bluestore_prefer_deferred_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4417     .set_default(64_K)
4418     .set_flag(Option::FLAG_RUNTIME)
4419     .set_description("Default bluestore_prefer_deferred_size for rotational media")
4420     .add_see_also("bluestore_prefer_deferred_size"),
4421
4422     Option("bluestore_prefer_deferred_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4423     .set_default(0)
4424     .set_flag(Option::FLAG_RUNTIME)
4425     .set_description("Default bluestore_prefer_deferred_size for non-rotational (solid state) media")
4426     .add_see_also("bluestore_prefer_deferred_size"),
4427
4428     Option("bluestore_compression_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4429     .set_default("none")
4430     .set_enum_allowed({"none", "passive", "aggressive", "force"})
4431     .set_flag(Option::FLAG_RUNTIME)
4432     .set_description("Default policy for using compression when pool does not specify")
4433     .set_long_description("'none' means never use compression.  'passive' means use compression when clients hint that data is compressible.  'aggressive' means use compression unless clients hint that data is not compressible.  This option is used when the per-pool property for the compression mode is not present."),
4434
4435     Option("bluestore_compression_algorithm", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4436     .set_default("snappy")
4437     .set_enum_allowed({"", "snappy", "zlib", "zstd", "lz4"})
4438     .set_flag(Option::FLAG_RUNTIME)
4439     .set_description("Default compression algorithm to use when writing object data")
4440     .set_long_description("This controls the default compressor to use (if any) if the per-pool property is not set.  Note that zstd is *not* recommended for bluestore due to high CPU overhead when compressing small amounts of data."),
4441
4442     Option("bluestore_compression_min_blob_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4443     .set_default(0)
4444     .set_flag(Option::FLAG_RUNTIME)
4445     .set_description("Maximum chunk size to apply compression to when random access is expected for an object.")
4446     .set_long_description("Chunks larger than this are broken into smaller chunks before being compressed"),
4447
4448     Option("bluestore_compression_min_blob_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4449     .set_default(8_K)
4450     .set_flag(Option::FLAG_RUNTIME)
4451     .set_description("Default value of bluestore_compression_min_blob_size for rotational media")
4452     .add_see_also("bluestore_compression_min_blob_size"),
4453
4454     Option("bluestore_compression_min_blob_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4455     .set_default(8_K)
4456     .set_flag(Option::FLAG_RUNTIME)
4457     .set_description("Default value of bluestore_compression_min_blob_size for non-rotational (solid state) media")
4458     .add_see_also("bluestore_compression_min_blob_size"),
4459
4460     Option("bluestore_compression_max_blob_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4461     .set_default(0)
4462     .set_flag(Option::FLAG_RUNTIME)
4463     .set_description("Maximum chunk size to apply compression to when non-random access is expected for an object.")
4464     .set_long_description("Chunks larger than this are broken into smaller chunks before being compressed"),
4465
4466     Option("bluestore_compression_max_blob_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4467     .set_default(64_K)
4468     .set_flag(Option::FLAG_RUNTIME)
4469     .set_description("Default value of bluestore_compression_max_blob_size for rotational media")
4470     .add_see_also("bluestore_compression_max_blob_size"),
4471
4472     Option("bluestore_compression_max_blob_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4473     .set_default(64_K)
4474     .set_flag(Option::FLAG_RUNTIME)
4475     .set_description("Default value of bluestore_compression_max_blob_size for non-rotational (solid state) media")
4476     .add_see_also("bluestore_compression_max_blob_size"),
4477
4478     Option("bluestore_gc_enable_blob_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
4479     .set_default(0)
4480     .set_flag(Option::FLAG_RUNTIME)
4481     .set_description(""),
4482
4483     Option("bluestore_gc_enable_total_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
4484     .set_default(0)
4485     .set_flag(Option::FLAG_RUNTIME)
4486     .set_description(""),
4487
4488     Option("bluestore_max_blob_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4489     .set_default(0)
4490     .set_flag(Option::FLAG_RUNTIME)
4491     .set_description("")
4492     .set_long_description("Bluestore blobs are collections of extents (ie on-disk data) originating from one or more objects.  Blobs can be compressed, typically have checksum data, may be overwritten, may be shared (with an extent ref map), or split.  This setting controls the maximum size a blob is allowed to be."),
4493
4494     Option("bluestore_max_blob_size_hdd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4495     .set_default(64_K)
4496     .set_flag(Option::FLAG_RUNTIME)
4497     .set_description("")
4498     .add_see_also("bluestore_max_blob_size"),
4499
4500     Option("bluestore_max_blob_size_ssd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4501     .set_default(64_K)
4502     .set_flag(Option::FLAG_RUNTIME)
4503     .set_description("")
4504     .add_see_also("bluestore_max_blob_size"),
4505
4506     Option("bluestore_compression_required_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4507     .set_default(.875)
4508     .set_flag(Option::FLAG_RUNTIME)
4509     .set_description("Compression ratio required to store compressed data")
4510     .set_long_description("If we compress data and get less than this we discard the result and store the original uncompressed data."),
4511
4512     Option("bluestore_extent_map_shard_max_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4513     .set_default(1200)
4514     .set_description("Max size (bytes) for a single extent map shard before splitting"),
4515
4516     Option("bluestore_extent_map_shard_target_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4517     .set_default(500)
4518     .set_description("Target size (bytes) for a single extent map shard"),
4519
4520     Option("bluestore_extent_map_shard_min_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4521     .set_default(150)
4522     .set_description("Min size (bytes) for a single extent map shard before merging"),
4523
4524     Option("bluestore_extent_map_shard_target_size_slop", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4525     .set_default(.2)
4526     .set_description("Ratio above/below target for a shard when trying to align to an existing extent or blob boundary"),
4527
4528     Option("bluestore_extent_map_inline_shard_prealloc_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4529     .set_default(256)
4530     .set_description("Preallocated buffer for inline shards"),
4531
4532     Option("bluestore_cache_trim_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4533     .set_default(.05)
4534     .set_description("How frequently we trim the bluestore cache"),
4535
4536     Option("bluestore_cache_trim_max_skip_pinned", Option::TYPE_UINT, Option::LEVEL_DEV)
4537     .set_default(64)
4538     .set_description("Max pinned cache entries we consider before giving up"),
4539
4540     Option("bluestore_cache_type", Option::TYPE_STR, Option::LEVEL_DEV)
4541     .set_default("2q")
4542     .set_enum_allowed({"2q", "lru"})
4543     .set_description("Cache replacement algorithm"),
4544
4545     Option("bluestore_2q_cache_kin_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4546     .set_default(.5)
4547     .set_description("2Q paper suggests .5"),
4548
4549     Option("bluestore_2q_cache_kout_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4550     .set_default(.5)
4551     .set_description("2Q paper suggests .5"),
4552
4553     Option("bluestore_cache_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4554     .set_default(0)
4555     .set_description("Cache size (in bytes) for BlueStore")
4556     .set_long_description("This includes data and metadata cached by BlueStore as well as memory devoted to rocksdb's cache(s)."),
4557
4558     Option("bluestore_cache_size_hdd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4559     .set_default(1_G)
4560     .set_description("Default bluestore_cache_size for rotational media")
4561     .add_see_also("bluestore_cache_size"),
4562
4563     Option("bluestore_cache_size_ssd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4564     .set_default(3_G)
4565     .set_description("Default bluestore_cache_size for non-rotational (solid state) media")
4566     .add_see_also("bluestore_cache_size"),
4567
4568     Option("bluestore_cache_meta_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4569     .set_default(.45)
4570     .add_see_also("bluestore_cache_size")
4571     .set_description("Ratio of bluestore cache to devote to metadata"),
4572
4573     Option("bluestore_cache_kv_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4574     .set_default(.45)
4575     .add_see_also("bluestore_cache_size")
4576     .set_description("Ratio of bluestore cache to devote to kv database (rocksdb)"),
4577
4578     Option("bluestore_cache_kv_onode_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4579     .set_default(.04)
4580     .add_see_also("bluestore_cache_size")
4581     .set_description("Ratio of bluestore cache to devote to kv onode column family (rocksdb)"),
4582
4583     Option("bluestore_cache_autotune", Option::TYPE_BOOL, Option::LEVEL_DEV)
4584     .set_default(true)
4585     .add_see_also("bluestore_cache_size")
4586     .add_see_also("bluestore_cache_meta_ratio")
4587     .set_description("Automatically tune the ratio of caches while respecting min values."),
4588
4589     Option("bluestore_cache_autotune_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4590     .set_default(5)
4591     .add_see_also("bluestore_cache_autotune")
4592     .set_description("The number of seconds to wait between rebalances when cache autotune is enabled."),
4593
4594     Option("bluestore_alloc_stats_dump_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4595       .set_default(3600 * 24)
4596       .set_description("The period (in second) for logging allocation statistics."),
4597
4598     Option("bluestore_kvbackend", Option::TYPE_STR, Option::LEVEL_DEV)
4599     .set_default("rocksdb")
4600     .set_flag(Option::FLAG_CREATE)
4601     .set_description("Key value database to use for bluestore"),
4602
4603     Option("bluestore_allocator", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4604     .set_default("hybrid")
4605     .set_enum_allowed({"bitmap", "stupid", "avl", "hybrid", "zoned"})
4606     .set_description("Allocator policy")
4607     .set_long_description("Allocator to use for bluestore.  Stupid should only be used for testing."),
4608
4609     Option("bluestore_freelist_blocks_per_key", Option::TYPE_SIZE, Option::LEVEL_DEV)
4610     .set_default(128)
4611     .set_description("Block (and bits) per database key"),
4612
4613     Option("bluestore_bitmapallocator_blocks_per_zone", Option::TYPE_SIZE, Option::LEVEL_DEV)
4614     .set_default(1024)
4615     .set_description(""),
4616
4617     Option("bluestore_bitmapallocator_span_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4618     .set_default(1024)
4619     .set_description(""),
4620
4621     Option("bluestore_max_deferred_txc", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4622     .set_default(32)
4623     .set_description("Max transactions with deferred writes that can accumulate before we force flush deferred writes"),
4624
4625     Option("bluestore_max_defer_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4626     .set_default(3)
4627     .set_description("max duration to force deferred submit"),
4628
4629     Option("bluestore_rocksdb_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4630     .set_default("compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2,max_total_wal_size=1073741824")
4631     .set_description("Full set of rocksdb settings to override"),
4632
4633     Option("bluestore_rocksdb_options_annex", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4634     .set_default("")
4635     .set_description("An addition to bluestore_rocksdb_options. Allows setting rocksdb options without repeating the existing defaults."),
4636
4637     Option("bluestore_rocksdb_cf", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4638     .set_default(true)
4639 #ifdef WITH_SEASTAR
4640     // This is necessary as the Seastar's allocator imposes restrictions
4641     // on the number of threads that entered malloc/free/*. Unfortunately,
4642     // RocksDB sharding in BlueStore dramatically lifted the number of
4643     // threads spawn during RocksDB's init.
4644     .set_validator([](std::string *value, std::string *error_message){
4645       if (const bool parsed_value = strict_strtob(value->c_str(), error_message);
4646           error_message->empty() && parsed_value) {
4647         *error_message = "invalid BlueStore sharding configuration."
4648                          " Be aware any change takes effect only on mkfs!";
4649         return -EINVAL;
4650       } else {
4651         return 0;
4652       }
4653     })
4654 #endif
4655     .set_description("Enable use of rocksdb column families for bluestore metadata"),
4656
4657     Option("bluestore_rocksdb_cfs", Option::TYPE_STR, Option::LEVEL_DEV)
4658     .set_default("m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L P")
4659     .set_description("Definition of column families and their sharding")
4660     .set_long_description("Space separated list of elements: column_def [ '=' rocksdb_options ]. "
4661                           "column_def := column_name [ '(' shard_count [ ',' hash_begin '-' [ hash_end ] ] ')' ]. "
4662                           "Example: 'I=write_buffer_size=1048576 O(6) m(7,10-)'. "
4663                           "Interval [hash_begin..hash_end) defines characters to use for hash calculation. "
4664                           "Recommended hash ranges: O(0-13) P(0-8) m(0-16). "
4665                           "Sharding of S,T,C,M,B prefixes is inadvised"),
4666
4667     Option("bluestore_fsck_on_mount", Option::TYPE_BOOL, Option::LEVEL_DEV)
4668     .set_default(false)
4669     .set_description("Run fsck at mount"),
4670
4671     Option("bluestore_fsck_on_mount_deep", Option::TYPE_BOOL, Option::LEVEL_DEV)
4672     .set_default(false)
4673     .set_description("Run deep fsck at mount when bluestore_fsck_on_mount is set to true"),
4674
4675     Option("bluestore_fsck_quick_fix_on_mount", Option::TYPE_BOOL, Option::LEVEL_DEV)
4676       .set_default(false)
4677       .set_description("Do quick-fix for the store at mount"),
4678
4679     Option("bluestore_fsck_on_umount", Option::TYPE_BOOL, Option::LEVEL_DEV)
4680     .set_default(false)
4681     .set_description("Run fsck at umount"),
4682
4683     Option("bluestore_fsck_on_umount_deep", Option::TYPE_BOOL, Option::LEVEL_DEV)
4684     .set_default(false)
4685     .set_description("Run deep fsck at umount when bluestore_fsck_on_umount is set to true"),
4686
4687     Option("bluestore_fsck_on_mkfs", Option::TYPE_BOOL, Option::LEVEL_DEV)
4688     .set_default(true)
4689     .set_description("Run fsck after mkfs"),
4690
4691     Option("bluestore_fsck_on_mkfs_deep", Option::TYPE_BOOL, Option::LEVEL_DEV)
4692     .set_default(false)
4693     .set_description("Run deep fsck after mkfs"),
4694
4695     Option("bluestore_sync_submit_transaction", Option::TYPE_BOOL, Option::LEVEL_DEV)
4696     .set_default(false)
4697     .set_description("Try to submit metadata transaction to rocksdb in queuing thread context"),
4698
4699     Option("bluestore_fsck_read_bytes_cap", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4700     .set_default(64_M)
4701     .set_flag(Option::FLAG_RUNTIME)
4702     .set_description("Maximum bytes read at once by deep fsck"),
4703
4704     Option("bluestore_fsck_quick_fix_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4705       .set_default(2)
4706       .set_description("Number of additional threads to perform quick-fix (shallow fsck) command"),
4707
4708     Option("bluestore_throttle_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4709     .set_default(64_M)
4710     .set_flag(Option::FLAG_RUNTIME)
4711     .set_description("Maximum bytes in flight before we throttle IO submission"),
4712
4713     Option("bluestore_throttle_deferred_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4714     .set_default(128_M)
4715     .set_flag(Option::FLAG_RUNTIME)
4716     .set_description("Maximum bytes for deferred writes before we throttle IO submission"),
4717
4718     Option("bluestore_throttle_cost_per_io", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4719     .set_default(0)
4720     .set_flag(Option::FLAG_RUNTIME)
4721     .set_description("Overhead added to transaction cost (in bytes) for each IO"),
4722
4723   Option("bluestore_throttle_cost_per_io_hdd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4724     .set_default(670000)
4725     .set_flag(Option::FLAG_RUNTIME)
4726     .set_description("Default bluestore_throttle_cost_per_io for rotational media")
4727     .add_see_also("bluestore_throttle_cost_per_io"),
4728
4729     Option("bluestore_throttle_cost_per_io_ssd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4730     .set_default(4000)
4731     .set_flag(Option::FLAG_RUNTIME)
4732     .set_description("Default bluestore_throttle_cost_per_io for non-rotation (solid state) media")
4733     .add_see_also("bluestore_throttle_cost_per_io"),
4734
4735     Option("bluestore_deferred_batch_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4736     .set_default(0)
4737     .set_min_max(0, 65535)
4738     .set_flag(Option::FLAG_RUNTIME)
4739     .set_description("Max number of deferred writes before we flush the deferred write queue"),
4740
4741     Option("bluestore_deferred_batch_ops_hdd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4742     .set_default(64)
4743     .set_min_max(0, 65535)
4744     .set_flag(Option::FLAG_RUNTIME)
4745     .set_description("Default bluestore_deferred_batch_ops for rotational media")
4746     .add_see_also("bluestore_deferred_batch_ops"),
4747
4748     Option("bluestore_deferred_batch_ops_ssd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4749     .set_default(16)
4750     .set_min_max(0, 65535)
4751     .set_flag(Option::FLAG_RUNTIME)
4752     .set_description("Default bluestore_deferred_batch_ops for non-rotational (solid state) media")
4753     .add_see_also("bluestore_deferred_batch_ops"),
4754
4755     Option("bluestore_nid_prealloc", Option::TYPE_INT, Option::LEVEL_DEV)
4756     .set_default(1024)
4757     .set_description("Number of unique object ids to preallocate at a time"),
4758
4759     Option("bluestore_blobid_prealloc", Option::TYPE_UINT, Option::LEVEL_DEV)
4760     .set_default(10240)
4761     .set_description("Number of unique blob ids to preallocate at a time"),
4762
4763     Option("bluestore_clone_cow", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4764     .set_default(true)
4765     .set_flag(Option::FLAG_RUNTIME)
4766     .set_description("Use copy-on-write when cloning objects (versus reading and rewriting them at clone time)"),
4767
4768     Option("bluestore_default_buffered_read", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4769     .set_default(true)
4770     .set_flag(Option::FLAG_RUNTIME)
4771     .set_description("Cache read results by default (unless hinted NOCACHE or WONTNEED)"),
4772
4773     Option("bluestore_default_buffered_write", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4774     .set_default(false)
4775     .set_flag(Option::FLAG_RUNTIME)
4776     .set_description("Cache writes by default (unless hinted NOCACHE or WONTNEED)"),
4777
4778     Option("bluestore_debug_no_reuse_blocks", Option::TYPE_BOOL, Option::LEVEL_DEV)
4779     .set_default(false)
4780     .set_description(""),
4781
4782     Option("bluestore_debug_small_allocations", Option::TYPE_INT, Option::LEVEL_DEV)
4783     .set_default(0)
4784     .set_description(""),
4785
4786     Option("bluestore_debug_too_many_blobs_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
4787     .set_default(24*1024)
4788     .set_description(""),
4789
4790     Option("bluestore_debug_freelist", Option::TYPE_BOOL, Option::LEVEL_DEV)
4791     .set_default(false)
4792     .set_description(""),
4793
4794     Option("bluestore_debug_prefill", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4795     .set_default(0)
4796     .set_description("simulate fragmentation"),
4797
4798     Option("bluestore_debug_prefragment_max", Option::TYPE_SIZE, Option::LEVEL_DEV)
4799     .set_default(1_M)
4800     .set_description(""),
4801
4802     Option("bluestore_debug_inject_read_err", Option::TYPE_BOOL, Option::LEVEL_DEV)
4803     .set_default(false)
4804     .set_description(""),
4805
4806     Option("bluestore_debug_randomize_serial_transaction", Option::TYPE_INT, Option::LEVEL_DEV)
4807     .set_default(0)
4808     .set_description(""),
4809
4810     Option("bluestore_debug_omit_block_device_write", Option::TYPE_BOOL, Option::LEVEL_DEV)
4811     .set_default(false)
4812     .set_description(""),
4813
4814     Option("bluestore_debug_fsck_abort", Option::TYPE_BOOL, Option::LEVEL_DEV)
4815     .set_default(false)
4816     .set_description(""),
4817
4818     Option("bluestore_debug_omit_kv_commit", Option::TYPE_BOOL, Option::LEVEL_DEV)
4819     .set_default(false)
4820     .set_description(""),
4821
4822     Option("bluestore_debug_permit_any_bdev_label", Option::TYPE_BOOL, Option::LEVEL_DEV)
4823     .set_default(false)
4824     .set_description(""),
4825
4826     Option("bluestore_debug_random_read_err", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4827     .set_default(0)
4828     .set_description(""),
4829
4830     Option("bluestore_debug_inject_bug21040", Option::TYPE_BOOL, Option::LEVEL_DEV)
4831     .set_default(false)
4832     .set_description(""),
4833
4834     Option("bluestore_debug_inject_csum_err_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4835     .set_default(0.0)
4836     .set_description("inject crc verification errors into bluestore device reads"),
4837
4838     Option("bluestore_fsck_error_on_no_per_pool_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4839     .set_default(false)
4840     .set_description("Make fsck error (instead of warn) when bluestore lacks per-pool stats, e.g., after an upgrade"),
4841
4842     Option("bluestore_warn_on_bluefs_spillover", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4843     .set_default(true)
4844     .set_description("Enable health indication on bluefs slow device usage"),
4845
4846     Option("bluestore_warn_on_legacy_statfs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4847     .set_default(true)
4848     .set_description("Enable health indication on lack of per-pool statfs reporting from bluestore"),
4849
4850     Option("bluestore_warn_on_spurious_read_errors", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4851     .set_default(true)
4852     .set_description("Enable health indication when spurious read errors are observed by OSD"),
4853
4854     Option("bluestore_fsck_error_on_no_per_pool_omap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4855     .set_default(false)
4856     .set_description("Make fsck error (instead of warn) when objects without per-pool omap are found"),
4857
4858     Option("bluestore_fsck_error_on_no_per_pg_omap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4859     .set_default(false)
4860     .set_description("Make fsck error (instead of warn) when objects without per-pg omap are found"),
4861
4862     Option("bluestore_warn_on_no_per_pool_omap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4863     .set_default(true)
4864     .set_description("Enable health indication on lack of per-pool omap"),
4865
4866     Option("bluestore_warn_on_no_per_pg_omap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4867     .set_default(false)
4868     .set_description("Enable health indication on lack of per-pg omap"),
4869
4870     Option("bluestore_log_op_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4871     .set_default(5)
4872     .set_description("log operation if it's slower than this age (seconds)"),
4873
4874     Option("bluestore_log_omap_iterator_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4875     .set_default(5)
4876     .set_description("log omap iteration operation if it's slower than this age (seconds)"),
4877
4878     Option("bluestore_log_collection_list_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4879     .set_default(60)
4880     .set_description("log collection list operation if it's slower than this age (seconds)"),
4881
4882     Option("bluestore_debug_enforce_settings", Option::TYPE_STR, Option::LEVEL_DEV)
4883     .set_default("default")
4884     .set_enum_allowed({"default", "hdd", "ssd"})
4885     .set_description("Enforces specific hw profile settings")
4886     .set_long_description("'hdd' enforces settings intended for BlueStore above a rotational drive. 'ssd' enforces settings intended for BlueStore above a solid drive. 'default' - using settings for the actual hardware."),
4887
4888     Option("bluestore_avl_alloc_bf_threshold", Option::TYPE_UINT, Option::LEVEL_DEV)
4889     .set_default(131072)
4890     .set_description("Sets threshold at which shrinking max free chunk size triggers enabling best-fit mode.")
4891     .set_long_description("AVL allocator works in two modes: near-fit and best-fit. By default, it uses very fast near-fit mode, "
4892                           "in which it tries to fit a new block near the last allocated block of similar size. The second mode "
4893                           "is much slower best-fit mode, in which it tries to find an exact match for the requested allocation. "
4894                           "This mode is used when either the device gets fragmented or when it is low on free space. "
4895                           "When the largest free block is smaller than 'bluestore_avl_alloc_bf_threshold', best-fit mode is used.")
4896     .add_see_also("bluestore_avl_alloc_bf_free_pct"),
4897
4898     Option("bluestore_avl_alloc_bf_free_pct", Option::TYPE_UINT, Option::LEVEL_DEV)
4899     .set_default(4)
4900     .set_description("Sets threshold at which shrinking free space (in %, integer) triggers enabling best-fit mode.")
4901     .set_long_description("AVL allocator works in two modes: near-fit and best-fit. By default, it uses very fast near-fit mode, "
4902                           "in which it tries to fit a new block near the last allocated block of similar size. The second mode "
4903                           "is much slower best-fit mode, in which it tries to find an exact match for the requested allocation. "
4904                           "This mode is used when either the device gets fragmented or when it is low on free space. "
4905                           "When free space is smaller than 'bluestore_avl_alloc_bf_free_pct', best-fit mode is used.")
4906     .add_see_also("bluestore_avl_alloc_bf_threshold"),
4907
4908     Option("bluestore_hybrid_alloc_mem_cap", Option::TYPE_UINT, Option::LEVEL_DEV)
4909     .set_default(64_M)
4910     .set_description("Maximum RAM hybrid allocator should use before enabling bitmap supplement"),
4911
4912     Option("bluestore_volume_selection_policy", Option::TYPE_STR, Option::LEVEL_DEV)
4913     .set_default("use_some_extra")
4914     .set_enum_allowed({ "rocksdb_original", "use_some_extra", "fit_to_fast" })
4915     .set_description("Determines bluefs volume selection policy")
4916     .set_long_description("Determines bluefs volume selection policy. 'use_some_extra' policy allows to override RocksDB level "
4917                           "granularity and put high level's data to faster device even when the level doesn't completely fit there. "
4918                           "'fit_to_fast' policy enables using 100% of faster disk capacity and allows the user to turn on "
4919                           "'level_compaction_dynamic_level_bytes' option in RocksDB options."),
4920
4921     Option("bluestore_volume_selection_reserved_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4922       .set_flag(Option::FLAG_STARTUP)
4923       .set_default(2.0)
4924       .set_description("DB level size multiplier. Determines amount of space at DB device to bar from the usage when 'use some extra' policy is in action. Reserved size is determined as sum(L_max_size[0], L_max_size[L-1]) + L_max_size[L] * this_factor"),
4925
4926     Option("bluestore_volume_selection_reserved", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4927       .set_flag(Option::FLAG_STARTUP)
4928       .set_default(0)
4929       .set_description("Space reserved at DB device and not allowed for 'use some extra' policy usage. Overrides 'bluestore_volume_selection_reserved_factor' setting and introduces straightforward limit."),
4930
4931     Option("bdev_ioring", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4932     .set_default(false)
4933     .set_description("Enables Linux io_uring API instead of libaio"),
4934
4935     Option("bdev_ioring_hipri", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4936     .set_default(false)
4937     .set_description("Enables Linux io_uring API Use polled IO completions"),
4938
4939     Option("bdev_ioring_sqthread_poll", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4940     .set_default(false)
4941     .set_description("Enables Linux io_uring API Offload submission/completion to kernel thread"),
4942
4943     Option("bluestore_kv_sync_util_logging_s", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4944     .set_default(10.0)
4945     .set_flag(Option::FLAG_RUNTIME)
4946     .set_description("KV sync thread utilization logging period")
4947     .set_long_description("How often (in seconds) to print KV sync thread utilization, "
4948       "not logged when set to 0 or when utilization is 0%"),
4949
4950
4951     // -----------------------------------------
4952     // kstore
4953
4954     Option("kstore_max_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4955     .set_default(512)
4956     .set_description(""),
4957
4958     Option("kstore_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4959     .set_default(64_M)
4960     .set_description(""),
4961
4962     Option("kstore_backend", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4963     .set_default("rocksdb")
4964     .set_description(""),
4965
4966     Option("kstore_rocksdb_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4967     .set_default("compression=kNoCompression")
4968     .set_description("Options to pass through when RocksDB is used as the KeyValueDB for kstore."),
4969
4970     Option("kstore_fsck_on_mount", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4971     .set_default(false)
4972     .set_description("Whether or not to run fsck on mount for kstore."),
4973
4974     Option("kstore_fsck_on_mount_deep", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4975     .set_default(true)
4976     .set_description("Whether or not to run deep fsck on mount for kstore"),
4977
4978     Option("kstore_nid_prealloc", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4979     .set_default(1024)
4980     .set_description(""),
4981
4982     Option("kstore_sync_transaction", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4983     .set_default(false)
4984     .set_description(""),
4985
4986     Option("kstore_sync_submit_transaction", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4987     .set_default(false)
4988     .set_description(""),
4989
4990     Option("kstore_onode_map_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4991     .set_default(1024)
4992     .set_description(""),
4993
4994     Option("kstore_default_stripe_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4995     .set_default(65536)
4996     .set_description(""),
4997
4998     // ---------------------
4999     // filestore
5000
5001     Option("filestore_rocksdb_options", Option::TYPE_STR, Option::LEVEL_DEV)
5002     .set_default("max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression")
5003     .set_description("Options to pass through when RocksDB is used as the KeyValueDB for filestore."),
5004
5005     Option("filestore_omap_backend", Option::TYPE_STR, Option::LEVEL_DEV)
5006     .set_default("rocksdb")
5007     .set_enum_allowed({"leveldb", "rocksdb"})
5008     .set_description("The KeyValueDB to use for filestore metadata (ie omap)."),
5009
5010     Option("filestore_omap_backend_path", Option::TYPE_STR, Option::LEVEL_DEV)
5011     .set_default("")
5012     .set_description("The path where the filestore KeyValueDB should store it's database(s)."),
5013
5014     Option("filestore_wbthrottle_enable", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5015     .set_default(true)
5016     .set_description("Enabling throttling of operations to backing file system"),
5017
5018     Option("filestore_wbthrottle_btrfs_bytes_start_flusher", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5019     .set_default(41943040)
5020     .set_description("Start flushing (fsyncing) when this many bytes are written(btrfs)"),
5021
5022     Option("filestore_wbthrottle_btrfs_bytes_hard_limit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5023     .set_default(419430400)
5024     .set_description("Block writes when this many bytes haven't been flushed (fsynced) (btrfs)"),
5025
5026     Option("filestore_wbthrottle_btrfs_ios_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5027     .set_default(500)
5028     .set_description("Start flushing (fsyncing) when this many IOs are written (brtrfs)"),
5029
5030     Option("filestore_wbthrottle_btrfs_ios_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5031     .set_default(5000)
5032     .set_description("Block writes when this many IOs haven't been flushed (fsynced) (btrfs)"),
5033
5034     Option("filestore_wbthrottle_btrfs_inodes_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5035     .set_default(500)
5036     .set_description("Start flushing (fsyncing) when this many distinct inodes have been modified (btrfs)"),
5037
5038     Option("filestore_wbthrottle_xfs_bytes_start_flusher", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5039     .set_default(41943040)
5040     .set_description("Start flushing (fsyncing) when this many bytes are written(xfs)"),
5041
5042     Option("filestore_wbthrottle_xfs_bytes_hard_limit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5043     .set_default(419430400)
5044     .set_description("Block writes when this many bytes haven't been flushed (fsynced) (xfs)"),
5045
5046     Option("filestore_wbthrottle_xfs_ios_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5047     .set_default(500)
5048     .set_description("Start flushing (fsyncing) when this many IOs are written (xfs)"),
5049
5050     Option("filestore_wbthrottle_xfs_ios_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5051     .set_default(5000)
5052     .set_description("Block writes when this many IOs haven't been flushed (fsynced) (xfs)"),
5053
5054     Option("filestore_wbthrottle_xfs_inodes_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5055     .set_default(500)
5056     .set_description("Start flushing (fsyncing) when this many distinct inodes have been modified (xfs)"),
5057
5058     Option("filestore_wbthrottle_btrfs_inodes_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5059     .set_default(5000)
5060     .set_description("Block writing when this many inodes have outstanding writes (btrfs)"),
5061
5062     Option("filestore_wbthrottle_xfs_inodes_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5063     .set_default(5000)
5064     .set_description("Block writing when this many inodes have outstanding writes (xfs)"),
5065
5066     Option("filestore_odsync_write", Option::TYPE_BOOL, Option::LEVEL_DEV)
5067     .set_default(false)
5068     .set_description("Write with O_DSYNC"),
5069
5070     Option("filestore_index_retry_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5071     .set_default(0)
5072     .set_description(""),
5073
5074     Option("filestore_debug_inject_read_err", Option::TYPE_BOOL, Option::LEVEL_DEV)
5075     .set_default(false)
5076     .set_description(""),
5077
5078     Option("filestore_debug_random_read_err", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5079     .set_default(0)
5080     .set_description(""),
5081
5082     Option("filestore_debug_omap_check", Option::TYPE_BOOL, Option::LEVEL_DEV)
5083     .set_default(false)
5084     .set_description(""),
5085
5086     Option("filestore_omap_header_cache_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5087     .set_default(1024)
5088     .set_description(""),
5089
5090     Option("filestore_max_inline_xattr_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5091     .set_default(0)
5092     .set_description(""),
5093
5094     Option("filestore_max_inline_xattr_size_xfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
5095     .set_default(65536)
5096     .set_description(""),
5097
5098     Option("filestore_max_inline_xattr_size_btrfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
5099     .set_default(2048)
5100     .set_description(""),
5101
5102     Option("filestore_max_inline_xattr_size_other", Option::TYPE_SIZE, Option::LEVEL_DEV)
5103     .set_default(512)
5104     .set_description(""),
5105
5106     Option("filestore_max_inline_xattrs", Option::TYPE_UINT, Option::LEVEL_DEV)
5107     .set_default(0)
5108     .set_description(""),
5109
5110     Option("filestore_max_inline_xattrs_xfs", Option::TYPE_UINT, Option::LEVEL_DEV)
5111     .set_default(10)
5112     .set_description(""),
5113
5114     Option("filestore_max_inline_xattrs_btrfs", Option::TYPE_UINT, Option::LEVEL_DEV)
5115     .set_default(10)
5116     .set_description(""),
5117
5118     Option("filestore_max_inline_xattrs_other", Option::TYPE_UINT, Option::LEVEL_DEV)
5119     .set_default(2)
5120     .set_description(""),
5121
5122     Option("filestore_max_xattr_value_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5123     .set_default(0)
5124     .set_description(""),
5125
5126     Option("filestore_max_xattr_value_size_xfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
5127     .set_default(64_K)
5128     .set_description(""),
5129
5130     Option("filestore_max_xattr_value_size_btrfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
5131     .set_default(64_K)
5132     .set_description(""),
5133
5134     Option("filestore_max_xattr_value_size_other", Option::TYPE_SIZE, Option::LEVEL_DEV)
5135     .set_default(1_K)
5136     .set_description(""),
5137
5138     Option("filestore_sloppy_crc", Option::TYPE_BOOL, Option::LEVEL_DEV)
5139     .set_default(false)
5140     .set_description(""),
5141
5142     Option("filestore_sloppy_crc_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5143     .set_default(65536)
5144     .set_description(""),
5145
5146     Option("filestore_max_alloc_hint_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5147     .set_default(1ULL << 20)
5148     .set_description(""),
5149
5150     Option("filestore_max_sync_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5151     .set_default(5)
5152     .set_description("Period between calls to syncfs(2) and journal trims (seconds)"),
5153
5154     Option("filestore_min_sync_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5155     .set_default(.01)
5156     .set_description("Minimum period between calls to syncfs(2)"),
5157
5158     Option("filestore_btrfs_snap", Option::TYPE_BOOL, Option::LEVEL_DEV)
5159     .set_default(true)
5160     .set_description(""),
5161
5162     Option("filestore_btrfs_clone_range", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5163     .set_default(true)
5164     .set_description("Use btrfs clone_range ioctl to efficiently duplicate objects"),
5165
5166     Option("filestore_zfs_snap", Option::TYPE_BOOL, Option::LEVEL_DEV)
5167     .set_default(false)
5168     .set_description(""),
5169
5170     Option("filestore_fsync_flushes_journal_data", Option::TYPE_BOOL, Option::LEVEL_DEV)
5171     .set_default(false)
5172     .set_description(""),
5173
5174     Option("filestore_fiemap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5175     .set_default(false)
5176     .set_description("Use fiemap ioctl(2) to determine which parts of objects are sparse"),
5177
5178     Option("filestore_punch_hole", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5179     .set_default(false)
5180     .set_description("Use fallocate(2) FALLOC_FL_PUNCH_HOLE to efficiently zero ranges of objects"),
5181
5182     Option("filestore_seek_data_hole", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5183     .set_default(false)
5184     .set_description("Use lseek(2) SEEK_HOLE and SEEK_DATA to determine which parts of objects are sparse"),
5185
5186     Option("filestore_splice", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5187     .set_default(false)
5188     .set_description("Use splice(2) to more efficiently copy data between files"),
5189
5190     Option("filestore_fadvise", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5191     .set_default(true)
5192     .set_description("Use posix_fadvise(2) to pass hints to file system"),
5193
5194     Option("filestore_collect_device_partition_information", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5195     .set_default(true)
5196     .set_description("Collect metadata about the backing file system on OSD startup"),
5197
5198     Option("filestore_xfs_extsize", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5199     .set_default(false)
5200     .set_description("Use XFS extsize ioctl(2) to hint allocator about expected write sizes"),
5201
5202     Option("filestore_journal_parallel", Option::TYPE_BOOL, Option::LEVEL_DEV)
5203     .set_default(false)
5204     .set_description(""),
5205
5206     Option("filestore_journal_writeahead", Option::TYPE_BOOL, Option::LEVEL_DEV)
5207     .set_default(false)
5208     .set_description(""),
5209
5210     Option("filestore_journal_trailing", Option::TYPE_BOOL, Option::LEVEL_DEV)
5211     .set_default(false)
5212     .set_description(""),
5213
5214     Option("filestore_queue_max_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5215     .set_default(50)
5216     .set_description("Max IO operations in flight"),
5217
5218     Option("filestore_queue_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5219     .set_default(100_M)
5220     .set_description("Max (written) bytes in flight"),
5221
5222     Option("filestore_caller_concurrency", Option::TYPE_INT, Option::LEVEL_DEV)
5223     .set_default(10)
5224     .set_description(""),
5225
5226     Option("filestore_expected_throughput_bytes", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5227     .set_default(200_M)
5228     .set_description("Expected throughput of backend device (aids throttling calculations)"),
5229
5230     Option("filestore_expected_throughput_ops", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5231     .set_default(200)
5232     .set_description("Expected through of backend device in IOPS (aids throttling calculations)"),
5233
5234     Option("filestore_queue_max_delay_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5235     .set_default(0)
5236     .set_description(""),
5237
5238     Option("filestore_queue_high_delay_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5239     .set_default(0)
5240     .set_description(""),
5241
5242     Option("filestore_queue_max_delay_multiple_bytes", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5243     .set_default(0)
5244     .set_description(""),
5245
5246     Option("filestore_queue_high_delay_multiple_bytes", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5247     .set_default(0)
5248     .set_description(""),
5249
5250     Option("filestore_queue_max_delay_multiple_ops", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5251     .set_default(0)
5252     .set_description(""),
5253
5254     Option("filestore_queue_high_delay_multiple_ops", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5255     .set_default(0)
5256     .set_description(""),
5257
5258     Option("filestore_queue_low_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5259     .set_default(0.3)
5260     .set_description(""),
5261
5262     Option("filestore_queue_high_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5263     .set_default(0.9)
5264     .set_description(""),
5265
5266     Option("filestore_op_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5267     .set_default(2)
5268     .set_description("Threads used to apply changes to backing file system"),
5269
5270     Option("filestore_op_thread_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5271     .set_default(60)
5272     .set_description("Seconds before a worker thread is considered stalled"),
5273
5274     Option("filestore_op_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5275     .set_default(180)
5276     .set_description("Seconds before a worker thread is considered dead"),
5277
5278     Option("filestore_commit_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5279     .set_default(600)
5280     .set_description("Seconds before backing file system is considered hung"),
5281
5282     Option("filestore_fiemap_threshold", Option::TYPE_SIZE, Option::LEVEL_DEV)
5283     .set_default(4_K)
5284     .set_description(""),
5285
5286     Option("filestore_merge_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
5287     .set_default(-10)
5288     .set_description(""),
5289
5290     Option("filestore_split_multiple", Option::TYPE_INT, Option::LEVEL_DEV)
5291     .set_default(2)
5292     .set_description(""),
5293
5294     Option("filestore_split_rand_factor", Option::TYPE_UINT, Option::LEVEL_DEV)
5295     .set_default(20)
5296     .set_description(""),
5297
5298     Option("filestore_update_to", Option::TYPE_INT, Option::LEVEL_DEV)
5299     .set_default(1000)
5300     .set_description(""),
5301
5302     Option("filestore_blackhole", Option::TYPE_BOOL, Option::LEVEL_DEV)
5303     .set_default(false)
5304     .set_description(""),
5305
5306     Option("filestore_fd_cache_size", Option::TYPE_INT, Option::LEVEL_DEV)
5307     .set_default(128)
5308     .set_description(""),
5309
5310     Option("filestore_fd_cache_shards", Option::TYPE_INT, Option::LEVEL_DEV)
5311     .set_default(16)
5312     .set_description(""),
5313
5314     Option("filestore_ondisk_finisher_threads", Option::TYPE_INT, Option::LEVEL_DEV)
5315     .set_default(1)
5316     .set_description(""),
5317
5318     Option("filestore_apply_finisher_threads", Option::TYPE_INT, Option::LEVEL_DEV)
5319     .set_default(1)
5320     .set_description(""),
5321
5322     Option("filestore_dump_file", Option::TYPE_STR, Option::LEVEL_DEV)
5323     .set_default("")
5324     .set_description(""),
5325
5326     Option("filestore_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
5327     .set_default(0)
5328     .set_description(""),
5329
5330     Option("filestore_inject_stall", Option::TYPE_INT, Option::LEVEL_DEV)
5331     .set_default(0)
5332     .set_description(""),
5333
5334     Option("filestore_fail_eio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5335     .set_default(true)
5336     .set_description(""),
5337
5338     Option("filestore_debug_verify_split", Option::TYPE_BOOL, Option::LEVEL_DEV)
5339     .set_default(false)
5340     .set_description(""),
5341
5342     Option("journal_dio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5343     .set_default(true)
5344     .set_description(""),
5345
5346     Option("journal_aio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5347     .set_default(true)
5348     .set_description(""),
5349
5350     Option("journal_force_aio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5351     .set_default(false)
5352     .set_description(""),
5353
5354     Option("journal_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5355     .set_default(4_K)
5356     .set_description(""),
5357
5358     Option("journal_block_align", Option::TYPE_BOOL, Option::LEVEL_DEV)
5359     .set_default(true)
5360     .set_description(""),
5361
5362     Option("journal_write_header_frequency", Option::TYPE_UINT, Option::LEVEL_DEV)
5363     .set_default(0)
5364     .set_description(""),
5365
5366     Option("journal_max_write_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5367     .set_default(10_M)
5368     .set_description("Max bytes in flight to journal"),
5369
5370     Option("journal_max_write_entries", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5371     .set_default(100)
5372     .set_description("Max IOs in flight to journal"),
5373
5374     Option("journal_throttle_low_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5375     .set_default(0.6)
5376     .set_description(""),
5377
5378     Option("journal_throttle_high_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5379     .set_default(0.9)
5380     .set_description(""),
5381
5382     Option("journal_throttle_high_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5383     .set_default(0)
5384     .set_description(""),
5385
5386     Option("journal_throttle_max_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5387     .set_default(0)
5388     .set_description(""),
5389
5390     Option("journal_align_min_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5391     .set_default(64_K)
5392     .set_description(""),
5393
5394     Option("journal_replay_from", Option::TYPE_INT, Option::LEVEL_DEV)
5395     .set_default(0)
5396     .set_description(""),
5397
5398     Option("mgr_stats_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5399     .set_default((int64_t)PerfCountersBuilder::PRIO_USEFUL)
5400     .set_description("Lowest perfcounter priority collected by mgr")
5401     .set_long_description("Daemons only set perf counter data to the manager "
5402                           "daemon if the counter has a priority higher than this.")
5403     .set_min_max((int64_t)PerfCountersBuilder::PRIO_DEBUGONLY,
5404                  (int64_t)PerfCountersBuilder::PRIO_CRITICAL + 1),
5405
5406     Option("journal_zero_on_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
5407     .set_default(false)
5408     .set_description(""),
5409
5410     Option("journal_ignore_corruption", Option::TYPE_BOOL, Option::LEVEL_DEV)
5411     .set_default(false)
5412     .set_description(""),
5413
5414     Option("journal_discard", Option::TYPE_BOOL, Option::LEVEL_DEV)
5415     .set_default(false)
5416     .set_description(""),
5417
5418     Option("fio_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5419     .set_default("/tmp/fio")
5420     .set_description(""),
5421
5422     Option("rados_mon_op_timeout", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
5423     .set_default(0)
5424     .set_description("timeout for operations handled by monitors such as statfs (0 is unlimited)")
5425     .set_flag(Option::FLAG_RUNTIME)
5426     .set_min(0),
5427
5428     Option("rados_osd_op_timeout", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
5429     .set_default(0)
5430     .set_description("timeout for operations handled by osds such as write (0 is unlimited)")
5431     .set_flag(Option::FLAG_RUNTIME)
5432     .set_min(0),
5433
5434     Option("rados_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5435     .set_default(false)
5436     .set_description(""),
5437
5438     Option("cephadm_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5439     .set_default("/usr/sbin/cephadm")
5440     .add_service("mgr")
5441     .set_description("Path to cephadm utility"),
5442
5443     Option("mgr_module_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5444     .set_default(CEPH_DATADIR "/mgr")
5445     .add_service("mgr")
5446     .set_description("Filesystem path to manager modules."),
5447
5448     Option("mgr_disabled_modules", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5449 #ifdef MGR_DISABLED_MODULES
5450     .set_default(MGR_DISABLED_MODULES)
5451 #endif
5452     .set_flag(Option::FLAG_STARTUP)
5453     .add_service("mgr")
5454     .set_description("List of manager modules never get loaded")
5455     .set_long_description("A comma delimited list of module names. This list "
5456         "is read by manager when it starts. By default, manager loads all "
5457         "modules found in specified 'mgr_module_path', and it starts the "
5458         "enabled ones as instructed. The modules in this list will not be "
5459         "loaded at all.")
5460     .add_see_also("mgr_module_path"),
5461
5462     Option("mgr_initial_modules", Option::TYPE_STR, Option::LEVEL_BASIC)
5463     .set_default("restful iostat")
5464     .set_flag(Option::FLAG_NO_MON_UPDATE)
5465     .set_flag(Option::FLAG_CLUSTER_CREATE)
5466     .add_service("mon")
5467     .set_description("List of manager modules to enable when the cluster is "
5468                      "first started")
5469     .set_long_description("This list of module names is read by the monitor "
5470         "when the cluster is first started after installation, to populate "
5471         "the list of enabled manager modules.  Subsequent updates are done using "
5472         "the 'mgr module [enable|disable]' commands.  List may be comma "
5473         "or space separated."),
5474
5475     Option("mgr_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5476     .set_default("/var/lib/ceph/mgr/$cluster-$id")
5477     .set_flag(Option::FLAG_NO_MON_UPDATE)
5478     .add_service("mgr")
5479     .set_description("Filesystem path to the ceph-mgr data directory, used to "
5480                      "contain keyring."),
5481
5482     Option("mgr_tick_period", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
5483     .set_default(2)
5484     .add_service("mgr")
5485     .set_description("Period in seconds of beacon messages to monitor"),
5486
5487     Option("mgr_stats_period", Option::TYPE_INT, Option::LEVEL_BASIC)
5488     .set_default(5)
5489     .add_service("mgr")
5490     .set_description("Period in seconds of OSD/MDS stats reports to manager")
5491     .set_long_description("Use this setting to control the granularity of "
5492                           "time series data collection from daemons.  Adjust "
5493                           "upwards if the manager CPU load is too high, or "
5494                           "if you simply do not require the most up to date "
5495                           "performance counter data."),
5496
5497     Option("mgr_client_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5498     .set_default(128_M)
5499     .add_service("mgr"),
5500
5501     Option("mgr_client_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5502     .set_default(512)
5503     .add_service("mgr"),
5504
5505     Option("mgr_osd_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5506     .set_default(512_M)
5507     .add_service("mgr"),
5508
5509     Option("mgr_osd_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5510     .set_default(8192)
5511     .add_service("mgr"),
5512
5513     Option("mgr_mds_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5514     .set_default(128_M)
5515     .add_service("mgr"),
5516
5517     Option("mgr_mds_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5518     .set_default(128)
5519     .add_service("mgr"),
5520
5521     Option("mgr_mon_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5522     .set_default(128_M)
5523     .add_service("mgr"),
5524
5525     Option("mgr_mon_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5526     .set_default(128)
5527     .add_service("mgr"),
5528
5529     Option("mgr_connect_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5530     .set_default(1.0)
5531     .add_service("common"),
5532
5533     Option("mgr_service_beacon_grace", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5534     .set_default(60.0)
5535     .add_service("mgr")
5536     .set_description("Period in seconds from last beacon to manager dropping "
5537                      "state about a monitored service (RGW, rbd-mirror etc)"),
5538
5539     Option("mgr_client_service_daemon_unregister_timeout", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5540     .set_default(1.0)
5541     .set_description("Time to wait during shutdown to deregister service with mgr"),
5542
5543     Option("mgr_debug_aggressive_pg_num_changes", Option::TYPE_BOOL, Option::LEVEL_DEV)
5544     .set_default(false)
5545     .set_description("Bypass most throttling and safety checks in pg[p]_num controller")
5546     .add_service("mgr"),
5547
5548     Option("mon_mgr_digest_period", Option::TYPE_INT, Option::LEVEL_DEV)
5549     .set_default(5)
5550     .add_service("mon")
5551     .set_description("Period in seconds between monitor-to-manager "
5552                      "health/status updates"),
5553
5554     Option("mon_mgr_beacon_grace", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
5555     .set_default(30)
5556     .add_service("mon")
5557     .set_description("Period in seconds from last beacon to monitor marking "
5558                      "a manager daemon as failed"),
5559
5560     Option("mon_mgr_inactive_grace", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5561     .set_default(60)
5562     .add_service("mon")
5563     .set_description("Period in seconds after cluster creation during which "
5564                      "cluster may have no active manager")
5565     .set_long_description("This grace period enables the cluster to come "
5566                           "up cleanly without raising spurious health check "
5567                           "failures about managers that aren't online yet"),
5568
5569     Option("mon_mgr_mkfs_grace", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5570     .set_default(120)
5571     .add_service("mon")
5572     .set_description("Period in seconds that the cluster may have no active "
5573                      "manager before this is reported as an ERR rather than "
5574                      "a WARN"),
5575
5576     Option("throttler_perf_counter", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5577     .set_default(true)
5578     .set_description(""),
5579
5580     Option("event_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5581     .set_default(false)
5582     .set_description(""),
5583
5584     Option("bluestore_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5585     .set_default(false)
5586     .set_description("Enable bluestore event tracing."),
5587
5588     Option("bluestore_throttle_trace_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5589     .set_default(0)
5590     .set_description("Rate at which to sample bluestore transactions (per second)"),
5591
5592     Option("debug_deliberately_leak_memory", Option::TYPE_BOOL, Option::LEVEL_DEV)
5593     .set_default(false)
5594     .set_description(""),
5595
5596     Option("debug_asserts_on_shutdown", Option::TYPE_BOOL,Option::LEVEL_DEV)
5597     .set_default(false)
5598     .set_description("Enable certain asserts to check for refcounting bugs on shutdown; see http://tracker.ceph.com/issues/21738"),
5599
5600     Option("debug_asok_assert_abort", Option::TYPE_BOOL, Option::LEVEL_DEV)
5601     .set_default(false)
5602     .set_description("allow commands 'assert' and 'abort' via asok for testing crash dumps etc"),
5603
5604     Option("target_max_misplaced_ratio", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
5605     .set_default(.05)
5606     .set_description("Max ratio of misplaced objects to target when throttling data rebalancing activity"),
5607
5608     Option("device_failure_prediction_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
5609     .set_default("none")
5610     .set_flag(Option::FLAG_RUNTIME)
5611     .set_enum_allowed({"none", "local", "cloud"})
5612     .set_description("Method used to predict device failures")
5613     .set_long_description("To disable prediction, use 'none',  'local' uses a prediction model that runs inside the mgr daemon.  'cloud' will share metrics with a cloud service and query the service for devicelife expectancy."),
5614
5615     /*  KRB Authentication. */
5616     Option("gss_ktab_client_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5617     .set_default("/var/lib/ceph/$name/gss_client_$name.ktab")
5618     .set_description("GSS/KRB5 Keytab file for client authentication")
5619     .add_service({"mon", "osd"})
5620     .set_long_description("This sets the full path for the GSS/Kerberos client keytab file location."),
5621
5622     Option("gss_target_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5623     .set_default("ceph")
5624     .set_description("")
5625     .add_service({"mon", "osd"})
5626     .set_long_description("This sets the gss target service name."),
5627
5628     Option("debug_disable_randomized_ping", Option::TYPE_BOOL, Option::LEVEL_DEV)
5629     .set_default(false)
5630     .set_description("Disable heartbeat ping randomization for testing purposes"),
5631
5632     Option("debug_heartbeat_testing_span", Option::TYPE_INT, Option::LEVEL_DEV)
5633     .set_default(0)
5634     .set_description("Override 60 second periods for testing only"),
5635
5636     Option("librados_thread_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5637     .set_default(2)
5638     .set_min(1)
5639     .set_description("Size of thread pool for Objecter")
5640     .add_tag("client"),
5641
5642     Option("osd_asio_thread_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5643     .set_default(2)
5644     .set_min(1)
5645     .set_description("Size of thread pool for ASIO completions")
5646     .add_tag("osd"),
5647
5648     Option("cephsqlite_lock_renewal_interval", Option::TYPE_MILLISECS, Option::LEVEL_ADVANCED)
5649     .add_see_also("cephsqlite_lock_renewal_timeout")
5650     .add_tag("client")
5651     .set_default(2000)
5652     .set_description("number of milliseconds before lock is renewed")
5653     .set_min(100)
5654     ,
5655
5656     Option("cephsqlite_lock_renewal_timeout", Option::TYPE_MILLISECS, Option::LEVEL_ADVANCED)
5657     .add_see_also("cephsqlite_lock_renewal_interval")
5658     .add_tag("client")
5659     .set_default(30000)
5660     .set_description("number of milliseconds before transaction lock times out")
5661     .set_long_description("The amount of time before a running libcephsqlite VFS connection has to renew a lock on the database before the lock is automatically lost. If the lock is lost, the VFS will abort the process to prevent database corruption.")
5662     .set_min(100),
5663
5664     Option("cephsqlite_blocklist_dead_locker", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5665     .add_tag("client")
5666     .set_default(true)
5667     .set_description("blocklist the last dead owner of the database lock")
5668     .set_long_description("Require that the Ceph SQLite VFS blocklist the last dead owner of the database when cleanup was incomplete. DO NOT CHANGE THIS UNLESS YOU UNDERSTAND THE RAMIFICATIONS. CORRUPTION MAY RESULT."),
5669
5670     // ----------------------------
5671     // Crimson specific options
5672
5673     Option("crimson_osd_obc_lru_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5674     .set_default(10)
5675     .set_description("Number of obcs to cache"),
5676
5677     Option("crimson_osd_scheduler_concurrency", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5678     .set_default(0)
5679     .set_description("The maximum number concurrent IO operations, 0 for unlimited"),
5680
5681     // ----------------------------
5682     // blk specific options
5683     Option("bdev_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5684     .set_description("Explicitly set the device type to select the driver if it's needed")
5685     .set_enum_allowed({"aio", "spdk", "pmem", "hm_smr"})
5686
5687   });
5688 }
5689
5690 std::vector<Option> get_rgw_options() {
5691   return std::vector<Option>({
5692     Option("rgw_acl_grants_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5693     .set_default(100)
5694     .set_description("Max number of ACL grants in a single request"),
5695
5696     Option("rgw_cors_rules_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5697     .set_default(100)
5698     .set_description("Max number of cors rules in a single request"),
5699
5700     Option("rgw_delete_multi_obj_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5701     .set_default(1000)
5702     .set_description("Max number of objects in a single multi-object delete request"),
5703
5704     Option("rgw_website_routing_rules_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5705     .set_default(50)
5706     .set_description("Max number of website routing rules in a single request"),
5707
5708     Option("rgw_rados_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5709     .set_default(false)
5710     .set_description("true if LTTng-UST tracepoints should be enabled"),
5711
5712     Option("rgw_op_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5713     .set_default(false)
5714     .set_description("true if LTTng-UST tracepoints should be enabled"),
5715
5716     Option("rgw_max_chunk_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5717     .set_default(4_M)
5718     .set_description("Set RGW max chunk size")
5719     .set_long_description(
5720         "The chunk size is the size of RADOS I/O requests that RGW sends when accessing "
5721         "data objects. RGW read and write operation will never request more than this amount "
5722         "in a single request. This also defines the rgw object head size, as head operations "
5723         "need to be atomic, and anything larger than this would require more than a single "
5724         "operation."),
5725
5726     Option("rgw_put_obj_min_window_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5727     .set_default(16_M)
5728     .set_description("The minimum RADOS write window size (in bytes).")
5729     .set_long_description(
5730         "The window size determines the total concurrent RADOS writes of a single rgw object. "
5731         "When writing an object RGW will send multiple chunks to RADOS. The total size of the "
5732         "writes does not exceed the window size. The window size can be automatically "
5733         "in order to better utilize the pipe.")
5734     .add_see_also({"rgw_put_obj_max_window_size", "rgw_max_chunk_size"}),
5735
5736     Option("rgw_put_obj_max_window_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5737     .set_default(64_M)
5738     .set_description("The maximum RADOS write window size (in bytes).")
5739     .set_long_description("The window size may be dynamically adjusted, but will not surpass this value.")
5740     .add_see_also({"rgw_put_obj_min_window_size", "rgw_max_chunk_size"}),
5741
5742     Option("rgw_max_put_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5743     .set_default(5_G)
5744     .set_description("Max size (in bytes) of regular (non multi-part) object upload.")
5745     .set_long_description(
5746         "Plain object upload is capped at this amount of data. In order to upload larger "
5747         "objects, a special upload mechanism is required. The S3 API provides the "
5748         "multi-part upload, and Swift provides DLO and SLO."),
5749
5750     Option("rgw_max_put_param_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5751     .set_default(1_M)
5752     .set_description("The maximum size (in bytes) of data input of certain RESTful requests."),
5753
5754     Option("rgw_max_attr_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5755     .set_default(0)
5756     .set_description("The maximum length of metadata value. 0 skips the check"),
5757
5758     Option("rgw_max_attr_name_len", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5759     .set_default(0)
5760     .set_description("The maximum length of metadata name. 0 skips the check"),
5761
5762     Option("rgw_max_attrs_num_in_req", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5763     .set_default(0)
5764     .set_description("The maximum number of metadata items that can be put via single request"),
5765
5766     Option("rgw_override_bucket_index_max_shards", Option::TYPE_UINT, Option::LEVEL_DEV)
5767     .set_default(0)
5768     .set_description("The default number of bucket index shards for newly-created "
5769         "buckets. This value overrides bucket_index_max_shards stored in the zone. "
5770         "Setting this value in the zone is preferred, because it applies globally "
5771         "to all radosgw daemons running in the zone."),
5772
5773     Option("rgw_bucket_index_max_aio", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5774     .set_default(128)
5775     .set_description("Max number of concurrent RADOS requests when handling bucket shards."),
5776
5777     Option("rgw_enable_quota_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5778     .set_default(true)
5779     .set_description("Enables the quota maintenance thread.")
5780     .set_long_description(
5781         "The quota maintenance thread is responsible for quota related maintenance work. "
5782         "The thread itself can be disabled, but in order for quota to work correctly, at "
5783         "least one RGW in each zone needs to have this thread running. Having the thread "
5784         "enabled on multiple RGW processes within the same zone can spread "
5785         "some of the maintenance work between them.")
5786     .add_see_also({"rgw_enable_gc_threads", "rgw_enable_lc_threads"}),
5787
5788     Option("rgw_enable_gc_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5789     .set_default(true)
5790     .set_description("Enables the garbage collection maintenance thread.")
5791     .set_long_description(
5792         "The garbage collection maintenance thread is responsible for garbage collector "
5793         "maintenance work. The thread itself can be disabled, but in order for garbage "
5794         "collection to work correctly, at least one RGW in each zone needs to have this "
5795         "thread running.  Having the thread enabled on multiple RGW processes within the "
5796         "same zone can spread some of the maintenance work between them.")
5797     .add_see_also({"rgw_enable_quota_threads", "rgw_enable_lc_threads"}),
5798
5799     Option("rgw_enable_lc_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5800     .set_default(true)
5801     .set_description("Enables the lifecycle maintenance thread. This is required on at least one rgw for each zone.")
5802     .set_long_description(
5803         "The lifecycle maintenance thread is responsible for lifecycle related maintenance "
5804         "work. The thread itself can be disabled, but in order for lifecycle to work "
5805         "correctly, at least one RGW in each zone needs to have this thread running. Having"
5806         "the thread enabled on multiple RGW processes within the same zone can spread "
5807         "some of the maintenance work between them.")
5808     .add_see_also({"rgw_enable_gc_threads", "rgw_enable_quota_threads"}),
5809
5810     Option("rgw_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5811     .set_default("/var/lib/ceph/radosgw/$cluster-$id")
5812     .set_flag(Option::FLAG_NO_MON_UPDATE)
5813     .set_description("Alternative location for RGW configuration.")
5814     .set_long_description(
5815         "If this is set, the different Ceph system configurables (such as the keyring file "
5816         "will be located in the path that is specified here. "),
5817
5818     Option("rgw_enable_apis", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5819     .set_default("s3, s3website, swift, swift_auth, admin, sts, iam, notifications")
5820     .set_description("A list of set of RESTful APIs that rgw handles."),
5821
5822     Option("rgw_cache_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5823     .set_default(true)
5824     .set_description("Enable RGW metadata cache.")
5825     .set_long_description(
5826         "The metadata cache holds metadata entries that RGW requires for processing "
5827         "requests. Metadata entries can be user info, bucket info, and bucket instance "
5828         "info. If not found in the cache, entries will be fetched from the backing "
5829         "RADOS store.")
5830     .add_see_also("rgw_cache_lru_size"),
5831
5832     Option("rgw_cache_lru_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5833     .set_default(10000)
5834     .set_description("Max number of items in RGW metadata cache.")
5835     .set_long_description(
5836         "When full, the RGW metadata cache evicts least recently used entries.")
5837     .add_see_also("rgw_cache_enabled"),
5838
5839     Option("rgw_socket_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5840     .set_default("")
5841     .set_description("RGW FastCGI socket path (for FastCGI over Unix domain sockets).")
5842     .add_see_also("rgw_fcgi_socket_backlog"),
5843
5844     Option("rgw_host", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5845     .set_default("")
5846     .set_description("RGW FastCGI host name (for FastCGI over TCP)")
5847     .add_see_also({"rgw_port", "rgw_fcgi_socket_backlog"}),
5848
5849     Option("rgw_port", Option::TYPE_STR, Option::LEVEL_BASIC)
5850     .set_default("")
5851     .set_description("RGW FastCGI port number (for FastCGI over TCP)")
5852     .add_see_also({"rgw_host", "rgw_fcgi_socket_backlog"}),
5853
5854     Option("rgw_dns_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5855     .set_default("")
5856     .set_description("The host name that RGW uses.")
5857     .set_long_description(
5858         "This is Needed for virtual hosting of buckets to work properly, unless configured "
5859         "via zonegroup configuration."),
5860
5861     Option("rgw_dns_s3website_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5862     .set_default("")
5863     .set_description("The host name that RGW uses for static websites (S3)")
5864     .set_long_description(
5865         "This is needed for virtual hosting of buckets, unless configured via zonegroup "
5866         "configuration."),
5867
5868     Option("rgw_numa_node", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5869     .set_default(-1)
5870     .set_flag(Option::FLAG_STARTUP)
5871     .set_description("set rgw's cpu affinity to a numa node (-1 for none)"),
5872
5873     Option("rgw_service_provider_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5874     .set_default("")
5875     .set_description("Service provider name which is contained in http response headers")
5876     .set_long_description(
5877         "As S3 or other cloud storage providers do, http response headers should contain the name of the provider. "
5878         "This name will be placed in http header 'Server'."),
5879
5880     Option("rgw_content_length_compat", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5881     .set_default(false)
5882     .set_description("Multiple content length headers compatibility")
5883     .set_long_description(
5884         "Try to handle requests with abiguous multiple content length headers "
5885         "(Content-Length, Http-Content-Length)."),
5886
5887     Option("rgw_relaxed_region_enforcement", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5888     .set_default(false)
5889     .set_description("Disable region constraint enforcement")
5890     .set_long_description(
5891         "Enable requests such as bucket creation to succeed irrespective of region restrictions (Jewel compat)."),
5892
5893     Option("rgw_lifecycle_work_time", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5894     .set_default("00:00-06:00")
5895     .set_description("Lifecycle allowed work time")
5896     .set_long_description("Local time window in which the lifecycle maintenance thread can work."),
5897
5898     Option("rgw_lc_lock_max_time", Option::TYPE_INT, Option::LEVEL_DEV)
5899     .set_default(90)
5900     .set_description(""),
5901
5902     Option("rgw_lc_thread_delay", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5903     .set_default(0)
5904     .set_description("Delay after processing of bucket listing chunks (i.e., per 1000 entries) in milliseconds"),
5905
5906     Option("rgw_lc_max_worker", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5907     .set_default(3)
5908     .set_description("Number of LCWorker tasks that will be run in parallel")
5909     .set_long_description(
5910       "Number of LCWorker tasks that will run in parallel--used to permit >1 "
5911       "bucket/index shards to be processed simultaneously"),
5912
5913     Option("rgw_lc_max_wp_worker", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5914     .set_default(3)
5915     .set_description("Number of workpool threads per LCWorker")
5916     .set_long_description(
5917       "Number of threads in per-LCWorker workpools--used to accelerate "
5918       "per-bucket processing"),
5919
5920     Option("rgw_lc_max_objs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5921     .set_default(32)
5922     .set_description("Number of lifecycle data shards")
5923     .set_long_description(
5924           "Number of RADOS objects to use for storing lifecycle index. This "
5925           "affects concurrency of lifecycle maintenance, as shards can be "
5926           "processed in parallel."),
5927
5928     Option("rgw_lc_max_rules", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5929     .set_default(1000)
5930     .set_description("Max number of lifecycle rules set on one bucket")
5931     .set_long_description("Number of lifecycle rules set on one bucket should be limited."),
5932
5933     Option("rgw_lc_debug_interval", Option::TYPE_INT, Option::LEVEL_DEV)
5934     .set_default(-1)
5935     .set_description(""),
5936
5937     Option("rgw_mp_lock_max_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5938     .set_default(600)
5939     .set_description("Multipart upload max completion time")
5940     .set_long_description(
5941         "Time length to allow completion of a multipart upload operation. This is done "
5942         "to prevent concurrent completions on the same object with the same upload id."),
5943
5944     Option("rgw_script_uri", Option::TYPE_STR, Option::LEVEL_DEV)
5945     .set_default("")
5946     .set_description(""),
5947
5948     Option("rgw_request_uri", Option::TYPE_STR, Option::LEVEL_DEV)
5949     .set_default("")
5950     .set_description(""),
5951
5952     Option("rgw_ignore_get_invalid_range", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5953     .set_default(false)
5954     .set_description("Treat invalid (e.g., negative) range request as full")
5955     .set_long_description("Treat invalid (e.g., negative) range request "
5956                           "as request for the full object (AWS compatibility)"),
5957
5958     Option("rgw_swift_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5959     .set_default("")
5960     .set_description("Swift-auth storage URL")
5961     .set_long_description(
5962         "Used in conjunction with rgw internal swift authentication. This affects the "
5963         "X-Storage-Url response header value.")
5964     .add_see_also("rgw_swift_auth_entry"),
5965
5966     Option("rgw_swift_url_prefix", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5967     .set_default("swift")
5968     .set_description("Swift URL prefix")
5969     .set_long_description("The URL path prefix for swift requests."),
5970
5971     Option("rgw_swift_auth_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5972     .set_default("")
5973     .set_description("Swift auth URL")
5974     .set_long_description(
5975         "Default url to which RGW connects and verifies tokens for v1 auth (if not using "
5976         "internal swift auth)."),
5977
5978     Option("rgw_swift_auth_entry", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5979     .set_default("auth")
5980     .set_description("Swift auth URL prefix")
5981     .set_long_description("URL path prefix for internal swift auth requests.")
5982     .add_see_also("rgw_swift_url"),
5983
5984     Option("rgw_swift_tenant_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5985     .set_default("")
5986     .set_description("Swift tenant name")
5987     .set_long_description("Tenant name that is used when constructing the swift path.")
5988     .add_see_also("rgw_swift_account_in_url"),
5989
5990     Option("rgw_swift_account_in_url", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5991     .set_default(false)
5992     .set_description("Swift account encoded in URL")
5993     .set_long_description("Whether the swift account is encoded in the uri path (AUTH_<account>).")
5994     .add_see_also("rgw_swift_tenant_name"),
5995
5996     Option("rgw_swift_enforce_content_length", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5997     .set_default(false)
5998     .set_description("Send content length when listing containers (Swift)")
5999     .set_long_description(
6000         "Whether content length header is needed when listing containers. When this is "
6001         "set to false, RGW will send extra info for each entry in the response."),
6002
6003     Option("rgw_keystone_url", Option::TYPE_STR, Option::LEVEL_BASIC)
6004     .set_default("")
6005     .set_description("The URL to the Keystone server."),
6006
6007     Option("rgw_keystone_admin_token", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6008     .set_default("")
6009     .set_description("DEPRECATED: The admin token (shared secret) that is used for the Keystone requests."),
6010
6011     Option("rgw_keystone_admin_token_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6012     .set_default("")
6013     .set_description("Path to a file containing the admin token (shared secret) that is used for the Keystone requests."),
6014
6015     Option("rgw_keystone_admin_user", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6016     .set_default("")
6017     .set_description("Keystone admin user."),
6018
6019     Option("rgw_keystone_admin_password", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6020     .set_default("")
6021     .set_description("DEPRECATED: Keystone admin password."),
6022
6023     Option("rgw_keystone_admin_password_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6024     .set_default("")
6025     .set_description("Path to a file containing the Keystone admin password."),
6026
6027     Option("rgw_keystone_admin_tenant", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6028     .set_default("")
6029     .set_description("Keystone admin user tenant."),
6030
6031     Option("rgw_keystone_admin_project", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6032     .set_default("")
6033     .set_description("Keystone admin user project (for Keystone v3)."),
6034
6035     Option("rgw_keystone_admin_domain", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6036     .set_default("")
6037     .set_description("Keystone admin user domain (for Keystone v3)."),
6038
6039     Option("rgw_keystone_barbican_user", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6040     .set_default("")
6041     .set_description("Keystone user to access barbican secrets."),
6042
6043     Option("rgw_keystone_barbican_password", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6044     .set_default("")
6045     .set_description("Keystone password for barbican user."),
6046
6047     Option("rgw_keystone_barbican_tenant", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6048     .set_default("")
6049     .set_description("Keystone barbican user tenant (Keystone v2.0)."),
6050
6051     Option("rgw_keystone_barbican_project", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6052     .set_default("")
6053     .set_description("Keystone barbican user project (Keystone v3)."),
6054
6055     Option("rgw_keystone_barbican_domain", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6056     .set_default("")
6057     .set_description("Keystone barbican user domain."),
6058
6059     Option("rgw_keystone_api_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6060     .set_default(2)
6061     .set_description("Version of Keystone API to use (2 or 3)."),
6062
6063     Option("rgw_keystone_accepted_roles", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6064     .set_default("Member, admin")
6065     .set_description("Only users with one of these roles will be served when doing Keystone authentication."),
6066
6067     Option("rgw_keystone_accepted_admin_roles", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6068     .set_default("")
6069     .set_description("List of roles allowing user to gain admin privileges (Keystone)."),
6070
6071     Option("rgw_keystone_token_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6072     .set_default(10000)
6073     .set_description("Keystone token cache size")
6074     .set_long_description(
6075         "Max number of Keystone tokens that will be cached. Token that is not cached "
6076         "requires RGW to access the Keystone server when authenticating."),
6077
6078     Option("rgw_keystone_verify_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6079     .set_default(true)
6080     .set_description("Should RGW verify the Keystone server SSL certificate."),
6081
6082     Option("rgw_keystone_implicit_tenants", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6083     .set_default("false")
6084     .set_enum_allowed( { "false", "true", "swift", "s3", "both", "0", "1", "none" } )
6085     .set_description("RGW Keystone implicit tenants creation")
6086     .set_long_description(
6087         "Implicitly create new users in their own tenant with the same name when "
6088         "authenticating via Keystone.  Can be limited to s3 or swift only."),
6089
6090     Option("rgw_cross_domain_policy", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6091     .set_default("<allow-access-from domain=\"*\" secure=\"false\" />")
6092     .set_description("RGW handle cross domain policy")
6093     .set_long_description("Returned cross domain policy when accessing the crossdomain.xml "
6094                           "resource (Swift compatiility)."),
6095
6096     Option("rgw_healthcheck_disabling_path", Option::TYPE_STR, Option::LEVEL_DEV)
6097     .set_default("")
6098     .set_description("Swift health check api can be disabled if a file can be accessed in this path."),
6099
6100     Option("rgw_s3_auth_use_rados", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6101     .set_default(true)
6102     .set_description("Should S3 authentication use credentials stored in RADOS backend."),
6103
6104     Option("rgw_s3_auth_use_keystone", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6105     .set_default(false)
6106     .set_description("Should S3 authentication use Keystone."),
6107
6108     Option("rgw_s3_auth_order", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6109      .set_default("sts, external, local")
6110      .set_description("Authentication strategy order to use for s3 authentication")
6111      .set_long_description(
6112           "Order of authentication strategies to try for s3 authentication, the allowed "
6113            "options are a comma separated list of engines external, local. The "
6114            "default order is to try all the externally configured engines before "
6115            "attempting local rados based authentication"),
6116
6117     Option("rgw_barbican_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6118     .set_default("")
6119     .set_description("URL to barbican server."),
6120
6121     Option("rgw_ldap_uri", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6122     .set_default("ldaps://<ldap.your.domain>")
6123     .set_description("Space-separated list of LDAP servers in URI format."),
6124
6125     Option("rgw_ldap_binddn", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6126     .set_default("uid=admin,cn=users,dc=example,dc=com")
6127     .set_description("LDAP entry RGW will bind with (user match)."),
6128
6129     Option("rgw_ldap_searchdn", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6130     .set_default("cn=users,cn=accounts,dc=example,dc=com")
6131     .set_description("LDAP search base (basedn)."),
6132
6133     Option("rgw_ldap_dnattr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6134     .set_default("uid")
6135     .set_description("LDAP attribute containing RGW user names (to form binddns)."),
6136
6137     Option("rgw_ldap_secret", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6138     .set_default("/etc/openldap/secret")
6139     .set_description("Path to file containing credentials for rgw_ldap_binddn."),
6140
6141     Option("rgw_s3_auth_use_ldap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6142     .set_default(false)
6143     .set_description("Should S3 authentication use LDAP."),
6144
6145     Option("rgw_ldap_searchfilter", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6146     .set_default("")
6147     .set_description("LDAP search filter."),
6148
6149     Option("rgw_opa_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6150     .set_default("")
6151     .set_description("URL to OPA server."),
6152
6153     Option("rgw_opa_token", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6154     .set_default("")
6155     .set_description("The Bearer token OPA uses to authenticate client requests."),
6156
6157     Option("rgw_opa_verify_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6158     .set_default(true)
6159     .set_description("Should RGW verify the OPA server SSL certificate."),
6160
6161     Option("rgw_use_opa_authz", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6162     .set_default(false)
6163     .set_description("Should OPA be used to authorize client requests."),
6164
6165     Option("rgw_admin_entry", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6166     .set_default("admin")
6167     .set_description("Path prefix to be used for accessing RGW RESTful admin API."),
6168
6169     Option("rgw_enforce_swift_acls", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6170     .set_default(true)
6171     .set_description("RGW enforce swift acls")
6172     .set_long_description(
6173         "Should RGW enforce special Swift-only ACLs. Swift has a special ACL that gives "
6174         "permission to access all objects in a container."),
6175
6176     Option("rgw_swift_token_expiration", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6177     .set_default(1_day)
6178     .set_description("Expiration time (in seconds) for token generated through RGW Swift auth."),
6179
6180     Option("rgw_print_continue", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6181     .set_default(true)
6182     .set_description("RGW support of 100-continue")
6183     .set_long_description(
6184         "Should RGW explicitly send 100 (continue) responses. This is mainly relevant when "
6185         "using FastCGI, as some FastCGI modules do not fully support this feature."),
6186
6187     Option("rgw_print_prohibited_content_length", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6188     .set_default(false)
6189     .set_description("RGW RFC-7230 compatibility")
6190     .set_long_description(
6191         "Specifies whether RGW violates RFC 7230 and sends Content-Length with 204 or 304 "
6192         "statuses."),
6193
6194     Option("rgw_remote_addr_param", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6195     .set_default("REMOTE_ADDR")
6196     .set_description("HTTP header that holds the remote address in incoming requests.")
6197     .set_long_description(
6198         "RGW will use this header to extract requests origin. When RGW runs behind "
6199         "a reverse proxy, the remote address header will point at the proxy's address "
6200         "and not at the originator's address. Therefore it is sometimes possible to "
6201         "have the proxy add the originator's address in a separate HTTP header, which "
6202         "will allow RGW to log it correctly."
6203         )
6204     .add_see_also("rgw_enable_ops_log"),
6205
6206     Option("rgw_op_thread_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
6207     .set_default(10*60)
6208     .set_description("Timeout for async rados coroutine operations."),
6209
6210     Option("rgw_op_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
6211     .set_default(0)
6212     .set_description(""),
6213
6214     Option("rgw_thread_pool_size", Option::TYPE_INT, Option::LEVEL_BASIC)
6215     .set_default(512)
6216     .set_description("RGW requests handling thread pool size.")
6217     .set_long_description(
6218         "This parameter determines the number of concurrent requests RGW can process "
6219         "when using either the civetweb, or the fastcgi frontends. The higher this "
6220         "number is, RGW will be able to deal with more concurrent requests at the "
6221         "cost of more resource utilization."),
6222
6223     Option("rgw_num_control_oids", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6224     .set_default(8)
6225     .set_description("Number of control objects used for cross-RGW communication.")
6226     .set_long_description(
6227         "RGW uses certain control objects to send messages between different RGW "
6228         "processes running on the same zone. These messages include metadata cache "
6229         "invalidation info that is being sent when metadata is modified (such as "
6230         "user or bucket information). A higher number of control objects allows "
6231         "better concurrency of these messages, at the cost of more resource "
6232         "utilization."),
6233
6234     Option("rgw_num_rados_handles", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6235     .set_default(1)
6236     .set_description("Number of librados handles that RGW uses.")
6237     .set_long_description(
6238         "This param affects the number of separate librados handles it uses to "
6239         "connect to the RADOS backend, which directly affects the number of connections "
6240         "RGW will have to each OSD. A higher number affects resource utilization."),
6241
6242     Option("rgw_verify_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6243     .set_default(true)
6244     .set_description("Should RGW verify SSL when connecing to a remote HTTP server")
6245     .set_long_description(
6246         "RGW can send requests to other RGW servers (e.g., in multi-site sync work). "
6247         "This configurable selects whether RGW should verify the certificate for "
6248         "the remote peer and host.")
6249     .add_see_also("rgw_keystone_verify_ssl"),
6250
6251     Option("rgw_nfs_lru_lanes", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6252     .set_default(5)
6253     .set_description(""),
6254
6255     Option("rgw_nfs_lru_lane_hiwat", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6256     .set_default(911)
6257     .set_description(""),
6258
6259     Option("rgw_nfs_fhcache_partitions", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6260     .set_default(3)
6261     .set_description(""),
6262
6263     Option("rgw_nfs_fhcache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6264     .set_default(2017)
6265     .set_description(""),
6266
6267     Option("rgw_nfs_namespace_expire_secs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6268     .set_default(300)
6269     .set_min(1)
6270     .set_description(""),
6271
6272     Option("rgw_nfs_max_gc", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6273     .set_default(300)
6274     .set_min(1)
6275     .set_description(""),
6276
6277     Option("rgw_nfs_write_completion_interval_s", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6278     .set_default(10)
6279     .set_description(""),
6280
6281     Option("rgw_nfs_s3_fast_attrs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6282     .set_default(false)
6283     .set_description("use fast S3 attrs from bucket index (immutable only)")
6284     .set_long_description("use fast S3 attrs from bucket index (assumes NFS "
6285                           "mounts are immutable)"),
6286
6287     Option("rgw_nfs_run_gc_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6288     .set_default(false)
6289     .set_description("run GC threads in librgw (default off)"),
6290
6291     Option("rgw_nfs_run_lc_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6292     .set_default(false)
6293     .set_description("run lifecycle threads in librgw (default off)"),
6294
6295     Option("rgw_nfs_run_quota_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6296     .set_default(false)
6297     .set_description("run quota threads in librgw (default off)"),
6298
6299     Option("rgw_nfs_run_sync_thread", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6300     .set_default(false)
6301     .set_description("run sync thread in librgw (default off)"),
6302
6303     Option("rgw_rados_pool_autoscale_bias", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
6304     .set_default(4.0)
6305     .set_min_max(0.01, 100000.0)
6306     .set_description("pg_autoscale_bias value for RGW metadata (omap-heavy) pools"),
6307
6308     Option("rgw_rados_pool_pg_num_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6309     .set_default(8)
6310     .set_min_max(1, 1024)
6311     .set_description("pg_num_min value for RGW metadata (omap-heavy) pools"),
6312
6313     Option("rgw_rados_pool_recovery_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6314     .set_default(5)
6315     .set_min_max(-10, 10)
6316     .set_description("recovery_priority value for RGW metadata (omap-heavy) pools"),
6317
6318     Option("rgw_zone", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6319     .set_default("")
6320     .set_description("Zone name")
6321     .add_see_also({"rgw_zonegroup", "rgw_realm"}),
6322
6323     Option("rgw_zone_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6324     .set_default(".rgw.root")
6325     .set_description("Zone root pool name")
6326     .set_long_description(
6327         "The zone root pool, is the pool where the RGW zone configuration located."
6328     )
6329     .add_see_also({"rgw_zonegroup_root_pool", "rgw_realm_root_pool", "rgw_period_root_pool"}),
6330
6331     Option("rgw_default_zone_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6332     .set_default("default.zone")
6333     .set_description("Default zone info object id")
6334     .set_long_description(
6335         "Name of the RADOS object that holds the default zone information."
6336     ),
6337
6338     Option("rgw_region", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6339     .set_default("")
6340     .set_description("Region name")
6341     .set_long_description(
6342         "Obsolete config option. The rgw_zonegroup option should be used instead.")
6343     .add_see_also("rgw_zonegroup"),
6344
6345     Option("rgw_region_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6346     .set_default(".rgw.root")
6347     .set_description("Region root pool")
6348     .set_long_description(
6349         "Obsolete config option. The rgw_zonegroup_root_pool should be used instead.")
6350     .add_see_also("rgw_zonegroup_root_pool"),
6351
6352     Option("rgw_default_region_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6353     .set_default("default.region")
6354     .set_description("Default region info object id")
6355     .set_long_description(
6356         "Obsolete config option. The rgw_default_zonegroup_info_oid should be used instead.")
6357     .add_see_also("rgw_default_zonegroup_info_oid"),
6358
6359     Option("rgw_zonegroup", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6360     .set_default("")
6361     .set_description("Zonegroup name")
6362     .add_see_also({"rgw_zone", "rgw_realm"}),
6363
6364     Option("rgw_zonegroup_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6365     .set_default(".rgw.root")
6366     .set_description("Zonegroup root pool")
6367     .set_long_description(
6368         "The zonegroup root pool, is the pool where the RGW zonegroup configuration located."
6369     )
6370     .add_see_also({"rgw_zone_root_pool", "rgw_realm_root_pool", "rgw_period_root_pool"}),
6371
6372     Option("rgw_default_zonegroup_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6373     .set_default("default.zonegroup")
6374     .set_description(""),
6375
6376     Option("rgw_realm", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6377     .set_default("")
6378     .set_description(""),
6379
6380     Option("rgw_realm_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6381     .set_default(".rgw.root")
6382     .set_description("Realm root pool")
6383     .set_long_description(
6384         "The realm root pool, is the pool where the RGW realm configuration located."
6385     )
6386     .add_see_also({"rgw_zonegroup_root_pool", "rgw_zone_root_pool", "rgw_period_root_pool"}),
6387
6388     Option("rgw_default_realm_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6389     .set_default("default.realm")
6390     .set_description(""),
6391
6392     Option("rgw_period_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6393     .set_default(".rgw.root")
6394     .set_description("Period root pool")
6395     .set_long_description(
6396         "The period root pool, is the pool where the RGW period configuration located."
6397     )
6398     .add_see_also({"rgw_zonegroup_root_pool", "rgw_zone_root_pool", "rgw_realm_root_pool"}),
6399
6400     Option("rgw_period_latest_epoch_info_oid", Option::TYPE_STR, Option::LEVEL_DEV)
6401     .set_default(".latest_epoch")
6402     .set_description(""),
6403
6404     Option("rgw_log_nonexistent_bucket", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6405     .set_default(false)
6406     .set_description("Should RGW log operations on bucket that does not exist")
6407     .set_long_description(
6408         "This config option applies to the ops log. When this option is set, the ops log "
6409         "will log operations that are sent to non existing buckets. These operations "
6410         "inherently fail, and do not correspond to a specific user.")
6411     .add_see_also("rgw_enable_ops_log"),
6412
6413     Option("rgw_log_object_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6414     .set_default("%Y-%m-%d-%H-%i-%n")
6415     .set_description("Ops log object name format")
6416     .set_long_description(
6417         "Defines the format of the RADOS objects names that ops log uses to store ops "
6418         "log data")
6419     .add_see_also("rgw_enable_ops_log"),
6420
6421     Option("rgw_log_object_name_utc", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6422     .set_default(false)
6423     .set_description("Should ops log object name based on UTC")
6424     .set_long_description(
6425         "If set, the names of the RADOS objects that hold the ops log data will be based "
6426         "on UTC time zone. If not set, it will use the local time zone.")
6427     .add_see_also({"rgw_enable_ops_log", "rgw_log_object_name"}),
6428
6429     Option("rgw_usage_max_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6430     .set_default(32)
6431     .set_description("Number of shards for usage log.")
6432     .set_long_description(
6433         "The number of RADOS objects that RGW will use in order to store the usage log "
6434         "data.")
6435     .add_see_also("rgw_enable_usage_log"),
6436
6437     Option("rgw_usage_max_user_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6438     .set_default(1)
6439     .set_min(1)
6440     .set_description("Number of shards for single user in usage log")
6441     .set_long_description(
6442         "The number of shards that a single user will span over in the usage log.")
6443     .add_see_also("rgw_enable_usage_log"),
6444
6445     Option("rgw_enable_ops_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6446     .set_default(false)
6447     .set_description("Enable ops log")
6448     .add_see_also({"rgw_log_nonexistent_bucket", "rgw_log_object_name", "rgw_ops_log_rados",
6449                "rgw_ops_log_socket_path"}),
6450
6451     Option("rgw_enable_usage_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6452     .set_default(false)
6453     .set_description("Enable usage log")
6454     .add_see_also("rgw_usage_max_shards"),
6455
6456     Option("rgw_ops_log_rados", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6457     .set_default(true)
6458     .set_description("Use RADOS for ops log")
6459     .set_long_description(
6460        "If set, RGW will store ops log information in RADOS.")
6461     .add_see_also({"rgw_enable_ops_log"}),
6462
6463     Option("rgw_ops_log_socket_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6464     .set_default("")
6465     .set_description("Unix domain socket path for ops log.")
6466     .set_long_description(
6467         "Path to unix domain socket that RGW will listen for connection on. When connected, "
6468         "RGW will send ops log data through it.")
6469     .add_see_also({"rgw_enable_ops_log", "rgw_ops_log_data_backlog"}),
6470
6471     Option("rgw_ops_log_data_backlog", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6472     .set_default(5 << 20)
6473     .set_description("Ops log socket backlog")
6474     .set_long_description(
6475         "Maximum amount of data backlog that RGW can keep when ops log is configured to "
6476         "send info through unix domain socket. When data backlog is higher than this, "
6477         "ops log entries will be lost. In order to avoid ops log information loss, the "
6478         "listener needs to clear data (by reading it) quickly enough.")
6479     .add_see_also({"rgw_enable_ops_log", "rgw_ops_log_socket_path"}),
6480
6481     Option("rgw_fcgi_socket_backlog", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6482     .set_default(1024)
6483     .set_description("FastCGI socket connection backlog")
6484     .set_long_description(
6485         "Size of FastCGI connection backlog. This reflects the maximum number of new "
6486         "connection requests that RGW can handle concurrently without dropping any. ")
6487     .add_see_also({"rgw_host", "rgw_socket_path"}),
6488
6489     Option("rgw_usage_log_flush_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6490     .set_default(1024)
6491     .set_description("Number of entries in usage log before flushing")
6492     .set_long_description(
6493         "This is the max number of entries that will be held in the usage log, before it "
6494         "will be flushed to the backend. Note that the usage log is periodically flushed, "
6495         "even if number of entries does not reach this threshold. A usage log entry "
6496         "corresponds to one or more operations on a single bucket.i")
6497     .add_see_also({"rgw_enable_usage_log", "rgw_usage_log_tick_interval"}),
6498
6499     Option("rgw_usage_log_tick_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6500     .set_default(30)
6501     .set_description("Number of seconds between usage log flush cycles")
6502     .set_long_description(
6503         "The number of seconds between consecutive usage log flushes. The usage log will "
6504         "also flush itself to the backend if the number of pending entries reaches a "
6505         "certain threshold.")
6506     .add_see_also({"rgw_enable_usage_log", "rgw_usage_log_flush_threshold"}),
6507
6508     Option("rgw_init_timeout", Option::TYPE_INT, Option::LEVEL_BASIC)
6509     .set_default(300)
6510     .set_description("Initialization timeout")
6511     .set_long_description(
6512         "The time length (in seconds) that RGW will allow for its initialization. RGW "
6513         "process will give up and quit if initialization is not complete after this amount "
6514         "of time."),
6515
6516     Option("rgw_mime_types_file", Option::TYPE_STR, Option::LEVEL_BASIC)
6517     .set_default("/etc/mime.types")
6518     .set_description("Path to local mime types file")
6519     .set_long_description(
6520         "The mime types file is needed in Swift when uploading an object. If object's "
6521         "content type is not specified, RGW will use data from this file to assign "
6522         "a content type to the object."),
6523
6524     Option("rgw_gc_max_objs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6525     .set_default(32)
6526     .set_description("Number of shards for garbage collector data")
6527     .set_long_description(
6528         "The number of garbage collector data shards, is the number of RADOS objects that "
6529         "RGW will use to store the garbage collection information on.")
6530     .add_see_also({"rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_processor_period", "rgw_gc_max_concurrent_io"}),
6531
6532     Option("rgw_gc_obj_min_wait", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6533     .set_default(2_hr)
6534     .set_description("Garbage collection object expiration time")
6535     .set_long_description(
6536        "The length of time (in seconds) that the RGW collector will wait before purging "
6537        "a deleted object's data. RGW will not remove object immediately, as object could "
6538        "still have readers. A mechanism exists to increase the object's expiration time "
6539        "when it's being read. The recommended value of its lower limit is 30 minutes")
6540     .add_see_also({"rgw_gc_max_objs", "rgw_gc_processor_max_time", "rgw_gc_processor_period", "rgw_gc_max_concurrent_io"}),
6541
6542     Option("rgw_gc_processor_max_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6543     .set_default(1_hr)
6544     .set_description("Length of time GC processor can lease shard")
6545     .set_long_description(
6546         "Garbage collection thread in RGW process holds a lease on its data shards. These "
6547         "objects contain the information about the objects that need to be removed. RGW "
6548         "takes a lease in order to prevent multiple RGW processes from handling the same "
6549         "objects concurrently. This time signifies that maximum amount of time (in seconds) that RGW "
6550         "is allowed to hold that lease. In the case where RGW goes down uncleanly, this "
6551         "is the amount of time where processing of that data shard will be blocked.")
6552     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_period", "rgw_gc_max_concurrent_io"}),
6553
6554     Option("rgw_gc_processor_period", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6555     .set_default(1_hr)
6556     .set_description("Garbage collector cycle run time")
6557     .set_long_description(
6558         "The amount of time between the start of consecutive runs of the garbage collector "
6559         "threads. If garbage collector runs takes more than this period, it will not wait "
6560         "before running again.")
6561     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_max_concurrent_io", "rgw_gc_max_trim_chunk"}),
6562
6563     Option("rgw_gc_max_concurrent_io", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6564     .set_default(10)
6565     .set_description("Max concurrent RADOS IO operations for garbage collection")
6566     .set_long_description(
6567         "The maximum number of concurrent IO operations that the RGW garbage collection "
6568         "thread will use when purging old data.")
6569     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_max_trim_chunk"}),
6570
6571     Option("rgw_gc_max_trim_chunk", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6572     .set_default(16)
6573     .set_description("Max number of keys to remove from garbage collector log in a single operation")
6574     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_max_concurrent_io"}),
6575
6576     Option("rgw_gc_max_deferred_entries_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6577     .set_default(3072)
6578     .set_description("maximum allowed size of deferred entries in queue head for gc"),
6579
6580     Option("rgw_gc_max_queue_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6581     .set_default(134213632)
6582     .set_description("Maximum allowed queue size for gc")
6583     .set_long_description(
6584         "The maximum allowed size of each gc queue, and its value should not "
6585         "be greater than (osd_max_object_size - rgw_gc_max_deferred_entries_size - 1K).")
6586     .add_see_also({"osd_max_object_size", "rgw_gc_max_deferred_entries_size"}),
6587
6588     Option("rgw_gc_max_deferred", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6589     .set_default(50)
6590     .set_description("Number of maximum deferred data entries to be stored in queue for gc"),
6591
6592     Option("rgw_s3_success_create_obj_status", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6593     .set_default(0)
6594     .set_description("HTTP return code override for object creation")
6595     .set_long_description(
6596         "If not zero, this is the HTTP return code that will be returned on a successful S3 "
6597         "object creation."),
6598
6599     Option("rgw_resolve_cname", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6600     .set_default(false)
6601     .set_description("Support vanity domain names via CNAME")
6602     .set_long_description(
6603         "If true, RGW will query DNS when detecting that it's serving a request that was "
6604         "sent to a host in another domain. If a CNAME record is configured for that domain "
6605         "it will use it instead. This gives user to have the ability of creating a unique "
6606         "domain of their own to point at data in their bucket."),
6607
6608     Option("rgw_obj_stripe_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6609     .set_default(4_M)
6610     .set_description("RGW object stripe size")
6611     .set_long_description(
6612         "The size of an object stripe for RGW objects. This is the maximum size a backing "
6613         "RADOS object will have. RGW objects that are larger than this will span over "
6614         "multiple objects."),
6615
6616     Option("rgw_extended_http_attrs", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6617     .set_default("")
6618     .set_description("RGW support extended HTTP attrs")
6619     .set_long_description(
6620         "Add new set of attributes that could be set on an object. These extra attributes "
6621         "can be set through HTTP header fields when putting the objects. If set, these "
6622         "attributes will return as HTTP fields when doing GET/HEAD on the object."),
6623
6624     Option("rgw_exit_timeout_secs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6625     .set_default(120)
6626     .set_description("RGW shutdown timeout")
6627     .set_long_description("Number of seconds to wait for a process before exiting unconditionally."),
6628
6629     Option("rgw_get_obj_window_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6630     .set_default(16_M)
6631     .set_description("RGW object read window size")
6632     .set_long_description("The window size in bytes for a single object read request"),
6633
6634     Option("rgw_get_obj_max_req_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6635     .set_default(4_M)
6636     .set_description("RGW object read chunk size")
6637     .set_long_description(
6638         "The maximum request size of a single object read operation sent to RADOS"),
6639
6640     Option("rgw_relaxed_s3_bucket_names", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6641     .set_default(false)
6642     .set_description("RGW enable relaxed S3 bucket names")
6643     .set_long_description("RGW enable relaxed S3 bucket name rules for US region buckets."),
6644
6645     Option("rgw_defer_to_bucket_acls", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6646     .set_default("")
6647     .set_description("Bucket ACLs override object ACLs")
6648     .set_long_description(
6649         "If not empty, a string that selects that mode of operation. 'recurse' will use "
6650         "bucket's ACL for the authorizaton. 'full-control' will allow users that users "
6651         "that have full control permission on the bucket have access to the object."),
6652
6653     Option("rgw_list_buckets_max_chunk", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6654     .set_default(1000)
6655     .set_description("Max number of buckets to retrieve in a single listing operation")
6656     .set_long_description(
6657         "When RGW fetches lists of user's buckets from the backend, this is the max number "
6658         "of entries it will try to retrieve in a single operation. Note that the backend "
6659         "may choose to return a smaller number of entries."),
6660
6661     Option("rgw_md_log_max_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6662     .set_default(64)
6663     .set_description("RGW number of metadata log shards")
6664     .set_long_description(
6665         "The number of shards the RGW metadata log entries will reside in. This affects "
6666         "the metadata sync parallelism as a shard can only be processed by a single "
6667         "RGW at a time"),
6668
6669     Option("rgw_curl_buffersize", Option::TYPE_INT, Option::LEVEL_DEV)
6670     .set_default(512_K)
6671     .set_min_max(1_K, 512_K)
6672     .set_long_description(
6673         "Pass a long specifying your preferred size (in bytes) for the receive"
6674         "buffer in libcurl. "
6675         "See: https://curl.se/libcurl/c/CURLOPT_BUFFERSIZE.html"),
6676
6677     Option("rgw_curl_wait_timeout_ms", Option::TYPE_INT, Option::LEVEL_DEV)
6678     .set_default(1000)
6679     .set_description(""),
6680
6681     Option("rgw_curl_low_speed_limit", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6682     .set_default(1024)
6683     .set_long_description(
6684         "It contains the average transfer speed in bytes per second that the "
6685         "transfer should be below during rgw_curl_low_speed_time seconds for libcurl "
6686         "to consider it to be too slow and abort. Set it zero to disable this."),
6687
6688     Option("rgw_curl_low_speed_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6689     .set_default(300)
6690     .set_long_description(
6691         "It contains the time in number seconds that the transfer speed should be below "
6692         "the rgw_curl_low_speed_limit for the library to consider it too slow and abort. "
6693         "Set it zero to disable this."),
6694
6695     Option("rgw_copy_obj_progress", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6696     .set_default(true)
6697     .set_description("Send progress report through copy operation")
6698     .set_long_description(
6699         "If true, RGW will send progress information when copy operation is executed. "),
6700
6701     Option("rgw_copy_obj_progress_every_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6702     .set_default(1_M)
6703     .set_description("Send copy-object progress info after these many bytes"),
6704
6705     Option("rgw_sync_obj_etag_verify", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6706     .set_default(false)
6707     .set_description("Verify if the object copied from remote is identical to its source")
6708     .set_long_description(
6709         "If true, this option computes the MD5 checksum of the data which is written at the "
6710         "destination and checks if it is identical to the ETAG stored in the source. "
6711         "It ensures integrity of the objects fetched from a remote server over HTTP including "
6712         "multisite sync."),
6713
6714     Option("rgw_obj_tombstone_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6715     .set_default(1000)
6716     .set_description("Max number of entries to keep in tombstone cache")
6717     .set_long_description(
6718         "The tombstone cache is used when doing a multi-zone data sync. RGW keeps "
6719         "there information about removed objects which is needed in order to prevent "
6720         "re-syncing of objects that were already removed."),
6721
6722     Option("rgw_data_log_window", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6723     .set_default(30)
6724     .set_description("Data log time window")
6725     .set_long_description(
6726         "The data log keeps information about buckets that have objectst that were "
6727         "modified within a specific timeframe. The sync process then knows which buckets "
6728         "are needed to be scanned for data sync."),
6729
6730     Option("rgw_data_log_changes_size", Option::TYPE_INT, Option::LEVEL_DEV)
6731     .set_default(1000)
6732     .set_description("Max size of pending changes in data log")
6733     .set_long_description(
6734         "RGW will trigger update to the data log if the number of pending entries reached "
6735         "this number."),
6736
6737     Option("rgw_data_log_num_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6738     .set_default(128)
6739     .set_description("Number of data log shards")
6740     .set_long_description(
6741         "The number of shards the RGW data log entries will reside in. This affects the "
6742         "data sync parallelism as a shard can only be processed by a single RGW at a time."),
6743
6744     Option("rgw_data_log_obj_prefix", Option::TYPE_STR, Option::LEVEL_DEV)
6745     .set_default("data_log")
6746     .set_description(""),
6747
6748     Option("rgw_bucket_quota_ttl", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6749     .set_default(600)
6750     .set_description("Bucket quota stats cache TTL")
6751     .set_long_description(
6752         "Length of time for bucket stats to be cached within RGW instance."),
6753
6754     Option("rgw_bucket_quota_soft_threshold", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
6755     .set_default(0.95)
6756     .set_description("RGW quota soft threshold")
6757     .set_long_description(
6758         "Threshold from which RGW doesn't rely on cached info for quota "
6759         "decisions. This is done for higher accuracy of the quota mechanism at "
6760         "cost of performance, when getting close to the quota limit. The value "
6761         "configured here is the ratio between the data usage to the max usage "
6762         "as specified by the quota."),
6763
6764     Option("rgw_bucket_quota_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6765     .set_default(10000)
6766     .set_description("RGW quota stats cache size")
6767     .set_long_description(
6768         "Maximum number of entries in the quota stats cache."),
6769
6770     Option("rgw_bucket_default_quota_max_objects", Option::TYPE_INT, Option::LEVEL_BASIC)
6771     .set_default(-1)
6772     .set_description("Default quota for max objects in a bucket")
6773     .set_long_description(
6774         "The default quota configuration for max number of objects in a bucket. A "
6775         "negative number means 'unlimited'."),
6776
6777     Option("rgw_bucket_default_quota_max_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6778     .set_default(-1)
6779     .set_description("Default quota for total size in a bucket")
6780     .set_long_description(
6781         "The default quota configuration for total size of objects in a bucket. A "
6782         "negative number means 'unlimited'."),
6783
6784     Option("rgw_expose_bucket", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6785     .set_default(false)
6786     .set_description("Send Bucket HTTP header with the response")
6787     .set_long_description(
6788         "If true, RGW will send a Bucket HTTP header with the responses. The header will "
6789         "contain the name of the bucket the operation happened on."),
6790
6791     Option("rgw_frontends", Option::TYPE_STR, Option::LEVEL_BASIC)
6792     .set_default("beast port=7480")
6793     .set_description("RGW frontends configuration")
6794     .set_long_description(
6795         "A comma delimited list of frontends configuration. Each configuration contains "
6796         "the type of the frontend followed by an optional space delimited set of "
6797         "key=value config parameters."),
6798
6799     Option("rgw_frontend_defaults", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6800     .set_default("beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key")
6801     .set_description("RGW frontends default configuration")
6802     .set_long_description(
6803         "A comma delimited list of default frontends configuration."),
6804
6805     Option("rgw_user_quota_bucket_sync_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6806     .set_default(180)
6807     .set_description("User quota bucket sync interval")
6808     .set_long_description(
6809         "Time period for accumulating modified buckets before syncing these stats."),
6810
6811     Option("rgw_user_quota_sync_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6812     .set_default(1_day)
6813     .set_description("User quota sync interval")
6814     .set_long_description(
6815         "Time period for accumulating modified buckets before syncing entire user stats."),
6816
6817     Option("rgw_user_quota_sync_idle_users", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6818     .set_default(false)
6819     .set_description("Should sync idle users quota")
6820     .set_long_description(
6821         "Whether stats for idle users be fully synced."),
6822
6823     Option("rgw_user_quota_sync_wait_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6824     .set_default(1_day)
6825     .set_description("User quota full-sync wait time")
6826     .set_long_description(
6827         "Minimum time between two full stats sync for non-idle users."),
6828
6829     Option("rgw_user_default_quota_max_objects", Option::TYPE_INT, Option::LEVEL_BASIC)
6830     .set_default(-1)
6831     .set_description("User quota max objects")
6832     .set_long_description(
6833         "The default quota configuration for total number of objects for a single user. A "
6834         "negative number means 'unlimited'."),
6835
6836     Option("rgw_user_default_quota_max_size", Option::TYPE_INT, Option::LEVEL_BASIC)
6837     .set_default(-1)
6838     .set_description("User quota max size")
6839     .set_long_description(
6840         "The default quota configuration for total size of objects for a single user. A "
6841         "negative number means 'unlimited'."),
6842
6843     Option("rgw_multipart_min_part_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6844     .set_default(5_M)
6845     .set_description("Minimum S3 multipart-upload part size")
6846     .set_long_description(
6847         "When doing a multipart upload, each part (other than the last part) should be "
6848         "at least this size."),
6849
6850     Option("rgw_multipart_part_upload_limit", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6851     .set_default(10000)
6852     .set_description("Max number of parts in multipart upload"),
6853
6854     Option("rgw_max_slo_entries", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6855     .set_default(1000)
6856     .set_description("Max number of entries in Swift Static Large Object manifest"),
6857
6858     Option("rgw_olh_pending_timeout_sec", Option::TYPE_INT, Option::LEVEL_DEV)
6859     .set_default(1_hr)
6860     .set_description("Max time for pending OLH change to complete")
6861     .set_long_description(
6862         "OLH is a versioned object's logical head. Operations on it are journaled and "
6863         "as pending before completion. If an operation doesn't complete with this amount "
6864         "of seconds, we remove the operation from the journal."),
6865
6866     Option("rgw_user_max_buckets", Option::TYPE_INT, Option::LEVEL_BASIC)
6867     .set_default(1000)
6868     .set_description("Max number of buckets per user")
6869     .set_long_description(
6870       "A user can create at most this number of buckets. Zero means "
6871       "no limit; a negative value means users cannot create any new "
6872       "buckets, although users will retain buckets already created."),
6873
6874     Option("rgw_objexp_gc_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6875     .set_default(10_min)
6876     .set_description("Swift objects expirer garbage collector interval"),
6877
6878     Option("rgw_objexp_hints_num_shards", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6879     .set_default(127)
6880     .set_description("Number of object expirer data shards")
6881     .set_long_description(
6882         "The number of shards the (Swift) object expirer will store its data on."),
6883
6884     Option("rgw_objexp_chunk_size", Option::TYPE_UINT, Option::LEVEL_DEV)
6885     .set_default(100)
6886     .set_description(""),
6887
6888     Option("rgw_enable_static_website", Option::TYPE_BOOL, Option::LEVEL_BASIC)
6889     .set_default(false)
6890     .set_description("Enable static website APIs")
6891     .set_long_description(
6892         "This configurable controls whether RGW handles the website control APIs. RGW can "
6893         "server static websites if s3website hostnames are configured, and unrelated to "
6894         "this configurable."),
6895
6896      Option("rgw_user_unique_email", Option::TYPE_BOOL, Option::LEVEL_BASIC)
6897     .set_default(true)
6898     .set_description("Require local RGW users to have unique email addresses")
6899     .set_long_description(
6900         "Enforce builtin user accounts to have unique email addresses.  This "
6901         "setting is historical.  In future, non-enforcement of email address "
6902         "uniqueness is likely to become the default."),
6903
6904     Option("rgw_log_http_headers", Option::TYPE_STR, Option::LEVEL_BASIC)
6905     .set_default("")
6906     .set_description("List of HTTP headers to log")
6907     .set_long_description(
6908         "A comma delimited list of HTTP headers to log when seen, ignores case (e.g., "
6909         "http_x_forwarded_for)."),
6910
6911     Option("rgw_num_async_rados_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6912     .set_default(32)
6913     .set_description("Number of concurrent RADOS operations in multisite sync")
6914     .set_long_description(
6915         "The number of concurrent RADOS IO operations that will be triggered for handling "
6916         "multisite sync operations. This includes control related work, and not the actual "
6917         "sync operations."),
6918
6919     Option("rgw_md_notify_interval_msec", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6920     .set_default(200)
6921     .set_description("Length of time to aggregate metadata changes")
6922     .set_long_description(
6923         "Length of time (in milliseconds) in which the master zone aggregates all the "
6924         "metadata changes that occurred, before sending notifications to all the other "
6925         "zones."),
6926
6927     Option("rgw_run_sync_thread", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6928     .set_default(true)
6929     .set_description("Should run sync thread"),
6930
6931     Option("rgw_sync_lease_period", Option::TYPE_INT, Option::LEVEL_DEV)
6932     .set_default(120)
6933     .set_description(""),
6934
6935     Option("rgw_sync_log_trim_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6936     .set_default(1200)
6937     .set_description("Sync log trim interval")
6938     .set_long_description(
6939         "Time in seconds between attempts to trim sync logs."),
6940
6941     Option("rgw_sync_log_trim_max_buckets", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6942     .set_default(16)
6943     .set_description("Maximum number of buckets to trim per interval")
6944     .set_long_description("The maximum number of buckets to consider for bucket index log trimming each trim interval, regardless of the number of bucket index shards. Priority is given to buckets with the most sync activity over the last trim interval.")
6945     .add_see_also("rgw_sync_log_trim_interval")
6946     .add_see_also("rgw_sync_log_trim_min_cold_buckets")
6947     .add_see_also("rgw_sync_log_trim_concurrent_buckets"),
6948
6949     Option("rgw_sync_log_trim_min_cold_buckets", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6950     .set_default(4)
6951     .set_description("Minimum number of cold buckets to trim per interval")
6952     .set_long_description("Of the `rgw_sync_log_trim_max_buckets` selected for bucket index log trimming each trim interval, at least this many of them must be 'cold' buckets. These buckets are selected in order from the list of all bucket instances, to guarantee that all buckets will be visited eventually.")
6953     .add_see_also("rgw_sync_log_trim_interval")
6954     .add_see_also("rgw_sync_log_trim_max_buckets")
6955     .add_see_also("rgw_sync_log_trim_concurrent_buckets"),
6956
6957     Option("rgw_sync_log_trim_concurrent_buckets", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6958     .set_default(4)
6959     .set_description("Maximum number of buckets to trim in parallel")
6960     .add_see_also("rgw_sync_log_trim_interval")
6961     .add_see_also("rgw_sync_log_trim_max_buckets")
6962     .add_see_also("rgw_sync_log_trim_min_cold_buckets"),
6963
6964     Option("rgw_sync_data_inject_err_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
6965     .set_default(0)
6966     .set_description(""),
6967
6968     Option("rgw_sync_meta_inject_err_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
6969     .set_default(0)
6970     .set_description(""),
6971
6972     Option("rgw_sync_trace_history_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6973     .set_default(4096)
6974     .set_description("Sync trace history size")
6975     .set_long_description(
6976       "Maximum number of complete sync trace entries to keep."),
6977
6978     Option("rgw_sync_trace_per_node_log_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6979     .set_default(32)
6980     .set_description("Sync trace per-node log size")
6981     .set_long_description(
6982         "The number of log entries to keep per sync-trace node."),
6983
6984     Option("rgw_sync_trace_servicemap_update_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6985     .set_default(10)
6986     .set_description("Sync-trace service-map update interval")
6987     .set_long_description(
6988         "Number of seconds between service-map updates of sync-trace events."),
6989
6990     Option("rgw_period_push_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
6991     .set_default(2)
6992     .set_description("Period push interval")
6993     .set_long_description(
6994         "Number of seconds to wait before retrying 'period push' operation."),
6995
6996     Option("rgw_period_push_interval_max", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
6997     .set_default(30)
6998     .set_description("Period push maximum interval")
6999     .set_long_description(
7000         "The max number of seconds to wait before retrying 'period push' after exponential "
7001         "backoff."),
7002
7003     Option("rgw_safe_max_objects_per_shard", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7004     .set_default(100*1024)
7005     .set_description("Safe number of objects per shard")
7006     .set_long_description(
7007         "This is the max number of objects per bucket index shard that RGW considers "
7008         "safe. RGW will warn if it identifies a bucket where its per-shard count is "
7009         "higher than a percentage of this number.")
7010     .add_see_also("rgw_shard_warning_threshold"),
7011
7012     Option("rgw_shard_warning_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7013     .set_default(90)
7014     .set_description("Warn about max objects per shard")
7015     .set_long_description(
7016         "Warn if number of objects per shard in a specific bucket passed this percentage "
7017         "of the safe number.")
7018     .add_see_also("rgw_safe_max_objects_per_shard"),
7019
7020     Option("rgw_swift_versioning_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7021     .set_default(false)
7022     .set_description("Enable Swift versioning"),
7023
7024     Option("rgw_swift_custom_header", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7025     .set_default("")
7026     .set_description("Enable swift custom header")
7027     .set_long_description(
7028         "If not empty, specifies a name of HTTP header that can include custom data. When "
7029         "uploading an object, if this header is passed RGW will store this header info "
7030         "and it will be available when listing the bucket."),
7031
7032     Option("rgw_swift_need_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7033     .set_default(true)
7034     .set_description("Enable stats on bucket listing in Swift"),
7035
7036     Option("rgw_reshard_num_logs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7037     .set_default(16)
7038     .set_min(1)
7039     .set_description("")
7040     .add_service("rgw"),
7041
7042     Option("rgw_reshard_bucket_lock_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7043     .set_default(360)
7044     .set_min(30)
7045     .set_description("Number of seconds the timeout on the reshard locks (bucket reshard lock and reshard log lock) are set to. As a reshard proceeds these locks can be renewed/extended. If too short, reshards cannot complete and will fail, causing a future reshard attempt. If too long a hung or crashed reshard attempt will keep the bucket locked for an extended period, not allowing RGW to detect the failed reshard attempt and recover.")
7046     .add_tag("performance")
7047     .add_service("rgw"),
7048
7049     Option("rgw_reshard_batch_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7050     .set_default(64)
7051     .set_min(8)
7052     .set_description("Number of reshard entries to batch together before sending the operations to the CLS back-end")
7053     .add_tag("performance")
7054     .add_service("rgw"),
7055
7056     Option("rgw_reshard_max_aio", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7057     .set_default(128)
7058     .set_min(16)
7059     .set_description("Maximum number of outstanding asynchronous I/O operations to allow at a time during resharding")
7060     .add_tag("performance")
7061     .add_service("rgw"),
7062
7063     Option("rgw_trust_forwarded_https", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7064     .set_default(false)
7065     .set_description("Trust Forwarded and X-Forwarded-Proto headers")
7066     .set_long_description(
7067         "When a proxy in front of radosgw is used for ssl termination, radosgw "
7068         "does not know whether incoming http connections are secure. Enable "
7069         "this option to trust the Forwarded and X-Forwarded-Proto headers sent "
7070         "by the proxy when determining whether the connection is secure. This "
7071         "is required for some features, such as server side encryption. "
7072         "(Never enable this setting if you do not have a trusted proxy in "
7073         "front of radosgw, or else malicious users will be able to set these "
7074         "headers in any request.)")
7075     .add_see_also("rgw_crypt_require_ssl"),
7076
7077     Option("rgw_crypt_require_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7078     .set_default(true)
7079     .set_description("Requests including encryption key headers must be sent over ssl"),
7080
7081     Option("rgw_crypt_default_encryption_key", Option::TYPE_STR, Option::LEVEL_DEV)
7082     .set_default("")
7083     .set_description(""),
7084
7085     Option("rgw_crypt_s3_kms_backend", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7086     .set_default("barbican")
7087     .set_enum_allowed({"barbican", "vault", "testing", "kmip"})
7088     .set_description(
7089         "Where the SSE-KMS encryption keys are stored. Supported KMS "
7090         "systems are OpenStack Barbican ('barbican', the default) and HashiCorp "
7091         "Vault ('vault')."),
7092
7093     Option("rgw_crypt_s3_kms_encryption_keys", Option::TYPE_STR, Option::LEVEL_DEV)
7094     .set_default("")
7095     .set_description(""),
7096
7097     Option("rgw_crypt_vault_auth", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7098     .set_default("token")
7099     .set_enum_allowed({"token", "agent"})
7100     .set_description(
7101         "Type of authentication method to be used with Vault. ")
7102     .add_see_also({
7103         "rgw_crypt_s3_kms_backend",
7104         "rgw_crypt_vault_addr",
7105         "rgw_crypt_vault_token_file"}),
7106
7107     Option("rgw_crypt_vault_token_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7108     .set_default("")
7109     .set_description(
7110         "If authentication method is 'token', provide a path to the token file, "
7111         "which for security reasons should readable only by Rados Gateway.")
7112     .add_see_also({
7113       "rgw_crypt_s3_kms_backend",
7114       "rgw_crypt_vault_auth",
7115       "rgw_crypt_vault_addr"}),
7116
7117     Option("rgw_crypt_vault_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7118     .set_default("")
7119     .set_description("Vault server base address.")
7120     .add_see_also({
7121       "rgw_crypt_s3_kms_backend",
7122       "rgw_crypt_vault_auth",
7123       "rgw_crypt_vault_prefix"}),
7124
7125     Option("rgw_crypt_vault_prefix", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7126     .set_default("")
7127     .set_description("Vault secret URL prefix, which can be used to restrict "
7128                      "access to a particular subset of the Vault secret space.")
7129     .add_see_also({
7130       "rgw_crypt_s3_kms_backend",
7131       "rgw_crypt_vault_addr",
7132       "rgw_crypt_vault_auth"}),
7133
7134
7135     Option("rgw_crypt_vault_secret_engine", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7136     .set_default("transit")
7137     .set_description(
7138         "Vault Secret Engine to be used to retrieve encryption keys.")
7139     .add_see_also({
7140       "rgw_crypt_s3_kms_backend",
7141       "rgw_crypt_vault_auth",
7142       "rgw_crypt_vault_addr"}),
7143
7144     Option("rgw_crypt_vault_namespace", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7145     .set_default("")
7146     .set_description("Vault Namespace to be used to select your tenant")
7147     .add_see_also({
7148       "rgw_crypt_s3_kms_backend",
7149       "rgw_crypt_vault_auth",
7150       "rgw_crypt_vault_addr"}),
7151
7152     Option("rgw_crypt_kmip_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7153     .set_default("")
7154     .set_description("kmip server address"),
7155
7156     Option("rgw_crypt_kmip_ca_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7157     .set_default("")
7158     .set_description("ca for kmip servers"),
7159
7160     Option("rgw_crypt_kmip_username", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7161     .set_default("")
7162     .set_description("when authenticating via username"),
7163
7164     Option("rgw_crypt_kmip_password", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7165     .set_default("")
7166     .set_description("optional w/ username"),
7167
7168     Option("rgw_crypt_kmip_client_cert", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7169     .set_default("")
7170     .set_description("connect using client certificate"),
7171
7172     Option("rgw_crypt_kmip_client_key", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7173     .set_default("")
7174     .set_description("connect using client certificate"),
7175
7176     Option("rgw_crypt_kmip_kms_key_template", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7177     .set_default("")
7178     .set_description("sse-kms; kmip key names"),
7179
7180     Option("rgw_crypt_kmip_s3_key_template", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7181     .set_default("$keyid")
7182     .set_description("sse-s3; kmip key template"),
7183
7184     Option("rgw_crypt_suppress_logs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7185     .set_default(true)
7186     .set_description("Suppress logs that might print client key"),
7187
7188     Option("rgw_list_bucket_min_readahead", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7189     .set_default(1000)
7190     .set_description("Minimum number of entries to request from rados for bucket listing"),
7191
7192     Option("rgw_rest_getusage_op_compat", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7193     .set_default(false)
7194     .set_description("REST GetUsage request backward compatibility"),
7195
7196     Option("rgw_torrent_flag", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7197     .set_default(false)
7198     .set_description("When true, uploaded objects will calculate and store "
7199                      "a SHA256 hash of object data so the object can be "
7200                      "retrieved as a torrent file"),
7201
7202     Option("rgw_torrent_tracker", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7203     .set_default("")
7204     .set_description("Torrent field announce and announce list"),
7205
7206     Option("rgw_torrent_createby", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7207     .set_default("")
7208     .set_description("torrent field created by"),
7209
7210     Option("rgw_torrent_comment", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7211     .set_default("")
7212     .set_description("Torrent field comment"),
7213
7214     Option("rgw_torrent_encoding", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7215     .set_default("")
7216     .set_description("torrent field encoding"),
7217
7218     Option("rgw_data_notify_interval_msec", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7219     .set_default(200)
7220     .set_description("data changes notification interval to followers"),
7221
7222     Option("rgw_torrent_origin", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7223     .set_default("")
7224     .set_description("Torrent origin"),
7225
7226     Option("rgw_torrent_sha_unit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7227     .set_default(512*1024)
7228     .set_description(""),
7229
7230     Option("rgw_dynamic_resharding", Option::TYPE_BOOL, Option::LEVEL_BASIC)
7231     .set_default(true)
7232     .set_description("Enable dynamic resharding")
7233     .set_long_description(
7234         "If true, RGW will dynamicall increase the number of shards in buckets that have "
7235         "a high number of objects per shard.")
7236     .add_see_also("rgw_max_objs_per_shard")
7237     .add_see_also("rgw_max_dynamic_shards"),
7238
7239     Option("rgw_max_objs_per_shard", Option::TYPE_UINT, Option::LEVEL_BASIC)
7240     .set_default(100000)
7241     .set_description("Max objects per shard for dynamic resharding")
7242     .set_long_description(
7243         "This is the max number of objects per bucket index shard that RGW will "
7244         "allow with dynamic resharding. RGW will trigger an automatic reshard operation "
7245         "on the bucket if it exceeds this number.")
7246     .add_see_also("rgw_dynamic_resharding")
7247     .add_see_also("rgw_max_dynamic_shards"),
7248
7249     Option("rgw_max_dynamic_shards", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7250     .set_default(1999)
7251     .set_min(1)
7252     .set_description("Max shards that dynamic resharding can create")
7253     .set_long_description(
7254         "This is the maximum number of bucket index shards that dynamic "
7255         "sharding is able to create on its own. This does not limit user "
7256         "requested resharding. Ideally this value is a prime number.")
7257     .add_see_also("rgw_dynamic_resharding")
7258     .add_see_also("rgw_max_objs_per_shard"),
7259
7260     Option("rgw_reshard_thread_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7261     .set_default(10_min)
7262     .set_min(10)
7263     .set_description("Number of seconds between processing of reshard log entries"),
7264
7265     Option("rgw_cache_expiry_interval", Option::TYPE_UINT,
7266            Option::LEVEL_ADVANCED)
7267     .set_default(15_min)
7268     .set_description("Number of seconds before entries in the cache are "
7269                      "assumed stale and re-fetched. Zero is never.")
7270     .add_tag("performance")
7271     .add_service("rgw")
7272     .set_long_description("The Rados Gateway stores metadata and objects in "
7273                           "an internal cache. This should be kept consistent "
7274                           "by the OSD's relaying notify events between "
7275                           "multiple watching RGW processes. In the event "
7276                           "that this notification protocol fails, bounding "
7277                           "the length of time that any data in the cache will "
7278                           "be assumed valid will ensure that any RGW instance "
7279                           "that falls out of sync will eventually recover. "
7280                           "This seems to be an issue mostly for large numbers "
7281                           "of RGW instances under heavy use. If you would like "
7282                           "to turn off cache expiry, set this value to zero."),
7283
7284     Option("rgw_inject_notify_timeout_probability", Option::TYPE_FLOAT,
7285            Option::LEVEL_DEV)
7286     .set_default(0)
7287     .add_tag("fault injection")
7288     .add_tag("testing")
7289     .add_service("rgw")
7290     .set_min_max(0.0, 1.0)
7291     .set_description("Likelihood of ignoring a notify")
7292     .set_long_description("This is the probability that the RGW cache will "
7293                           "ignore a cache notify message. It exists to help "
7294                           "with the development and testing of cache "
7295                           "consistency and recovery improvements. Please "
7296                           "do not set it in a production cluster, as it "
7297                           "actively causes failures. Set this to a floating "
7298                           "point value between 0 and 1."),
7299     Option("rgw_max_notify_retries", Option::TYPE_UINT,
7300            Option::LEVEL_ADVANCED)
7301     .set_default(3)
7302     .add_tag("error recovery")
7303     .add_service("rgw")
7304     .set_description("Number of attempts to notify peers before giving up.")
7305     .set_long_description("The number of times we will attempt to update "
7306                           "a peer's cache in the event of error before giving "
7307                           "up. This is unlikely to be an issue unless your "
7308                           "cluster is very heavily loaded. Beware that "
7309                           "increasing this value may cause some operations to "
7310                           "take longer in exceptional cases and thus may, "
7311                           "rarely, cause clients to time out."),
7312     Option("rgw_sts_entry", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7313     .set_default("sts")
7314     .set_description("STS URL prefix")
7315     .set_long_description("URL path prefix for internal STS requests."),
7316
7317     Option("rgw_sts_key", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7318     .set_default("sts")
7319     .set_description("STS Key")
7320     .set_long_description("Key used for encrypting/ decrypting session token."),
7321
7322     Option("rgw_s3_auth_use_sts", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7323     .set_default(false)
7324     .set_description("Should S3 authentication use STS."),
7325
7326     Option("rgw_sts_max_session_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7327     .set_default(43200)
7328     .set_description("Session token max duration")
7329     .set_long_description("Max duration in seconds for which the session token is valid."),
7330
7331     Option("rgw_sts_min_session_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7332     .set_default(900)
7333     .set_description("Minimum allowed duration of a session"),
7334
7335     Option("rgw_max_listing_results", Option::TYPE_UINT,
7336            Option::LEVEL_ADVANCED)
7337     .set_default(1000)
7338     .set_min_max(1, 100000)
7339     .add_service("rgw")
7340     .set_description("Upper bound on results in listing operations, ListBucket max-keys")
7341     .set_long_description("This caps the maximum permitted value for listing-like operations in RGW S3. "
7342                           "Affects ListBucket(max-keys), "
7343                           "ListBucketVersions(max-keys), "
7344                           "ListBucketMultipartUploads(max-uploads), "
7345                           "ListMultipartUploadParts(max-parts)"),
7346
7347     Option("rgw_sts_token_introspection_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7348     .set_default("")
7349     .set_description("STS Web Token introspection URL")
7350     .set_long_description("URL for introspecting an STS Web Token."),
7351
7352     Option("rgw_sts_client_id", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7353     .set_default("")
7354     .set_description("Client Id")
7355     .set_long_description("Client Id needed for introspecting a Web Token."),
7356
7357     Option("rgw_sts_client_secret", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7358     .set_default("")
7359     .set_description("Client Secret")
7360     .set_long_description("Client Secret needed for introspecting a Web Token."),
7361
7362     Option("rgw_max_concurrent_requests", Option::TYPE_INT, Option::LEVEL_BASIC)
7363     .set_default(1024)
7364     .set_description("Maximum number of concurrent HTTP requests.")
7365     .set_long_description(
7366         "Maximum number of concurrent HTTP requests that the beast frontend "
7367         "will process. Tuning this can help to limit memory usage under heavy "
7368         "load.")
7369     .add_tag("performance")
7370     .add_see_also("rgw_frontends"),
7371
7372     Option("rgw_scheduler_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7373     .set_default("throttler")
7374     .set_description("Set the type of dmclock scheduler, defaults to throttler "
7375                      "Other valid values are dmclock which is experimental"),
7376
7377     Option("rgw_dmclock_admin_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7378     .set_default(100.0)
7379     .set_description("mclock reservation for admin requests")
7380     .add_see_also("rgw_dmclock_admin_wgt")
7381     .add_see_also("rgw_dmclock_admin_lim"),
7382
7383     Option("rgw_dmclock_admin_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7384     .set_default(100.0)
7385     .set_description("mclock weight for admin requests")
7386     .add_see_also("rgw_dmclock_admin_res")
7387     .add_see_also("rgw_dmclock_admin_lim"),
7388
7389     Option("rgw_dmclock_admin_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7390     .set_default(0.0)
7391     .set_description("mclock limit for admin requests")
7392     .add_see_also("rgw_dmclock_admin_res")
7393     .add_see_also("rgw_dmclock_admin_wgt"),
7394
7395     Option("rgw_dmclock_auth_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7396     .set_default(200.0)
7397     .set_description("mclock reservation for object data requests")
7398     .add_see_also("rgw_dmclock_auth_wgt")
7399     .add_see_also("rgw_dmclock_auth_lim"),
7400
7401     Option("rgw_dmclock_auth_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7402     .set_default(100.0)
7403     .set_description("mclock weight for object data requests")
7404     .add_see_also("rgw_dmclock_auth_res")
7405     .add_see_also("rgw_dmclock_auth_lim"),
7406
7407     Option("rgw_dmclock_auth_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7408     .set_default(0.0)
7409     .set_description("mclock limit for object data requests")
7410     .add_see_also("rgw_dmclock_auth_res")
7411     .add_see_also("rgw_dmclock_auth_wgt"),
7412
7413     Option("rgw_dmclock_data_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7414     .set_default(500.0)
7415     .set_description("mclock reservation for object data requests")
7416     .add_see_also("rgw_dmclock_data_wgt")
7417     .add_see_also("rgw_dmclock_data_lim"),
7418
7419     Option("rgw_dmclock_data_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7420     .set_default(500.0)
7421     .set_description("mclock weight for object data requests")
7422     .add_see_also("rgw_dmclock_data_res")
7423     .add_see_also("rgw_dmclock_data_lim"),
7424
7425     Option("rgw_dmclock_data_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7426     .set_default(0.0)
7427     .set_description("mclock limit for object data requests")
7428     .add_see_also("rgw_dmclock_data_res")
7429     .add_see_also("rgw_dmclock_data_wgt"),
7430
7431     Option("rgw_dmclock_metadata_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7432     .set_default(500.0)
7433     .set_description("mclock reservation for metadata requests")
7434     .add_see_also("rgw_dmclock_metadata_wgt")
7435     .add_see_also("rgw_dmclock_metadata_lim"),
7436
7437     Option("rgw_dmclock_metadata_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7438     .set_default(500.0)
7439     .set_description("mclock weight for metadata requests")
7440     .add_see_also("rgw_dmclock_metadata_res")
7441     .add_see_also("rgw_dmclock_metadata_lim"),
7442
7443     Option("rgw_dmclock_metadata_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7444     .set_default(0.0)
7445     .set_description("mclock limit for metadata requests")
7446     .add_see_also("rgw_dmclock_metadata_res")
7447     .add_see_also("rgw_dmclock_metadata_wgt"),
7448
7449    Option("rgw_default_data_log_backing", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7450     .set_default("fifo")
7451     .set_enum_allowed( { "fifo", "omap" } )
7452     .set_description("Default backing store for the RGW data sync log")
7453     .set_long_description(
7454         "Whether to use the older OMAP backing store or the high performance "
7455         "FIFO based backing store by default. This only covers the creation of "
7456         "the log on startup if none exists."),
7457
7458    Option("rgw_luarocks_location", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7459      .set_flag(Option::FLAG_STARTUP)
7460 #ifdef WITH_RADOSGW_LUA_PACKAGES
7461     .set_default("/tmp/luarocks")
7462 #else
7463     // if set to empty string, system default luarocks package location (if exist) will be used
7464     .set_default("")
7465 #endif
7466     .set_description("Directory where luarocks install packages from allowlist"),
7467   });
7468 }
7469
7470 static std::vector<Option> get_rbd_options() {
7471   return std::vector<Option>({
7472     Option("rbd_default_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7473     .set_default("rbd")
7474     .set_description("default pool for storing new images")
7475     .set_validator([](std::string *value, std::string *error_message){
7476       std::regex pattern("^[^@/]+$");
7477       if (!std::regex_match (*value, pattern)) {
7478         *value = "rbd";
7479         *error_message = "invalid RBD default pool, resetting to 'rbd'";
7480       }
7481       return 0;
7482     }),
7483
7484     Option("rbd_default_data_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7485     .set_default("")
7486     .set_description("default pool for storing data blocks for new images")
7487     .set_validator([](std::string *value, std::string *error_message){
7488       std::regex pattern("^[^@/]*$");
7489       if (!std::regex_match (*value, pattern)) {
7490         *value = "";
7491         *error_message = "ignoring invalid RBD data pool";
7492       }
7493       return 0;
7494     }),
7495
7496     Option("rbd_default_features", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7497     .set_default("layering,exclusive-lock,object-map,fast-diff,deep-flatten")
7498     .set_description("default v2 image features for new images")
7499     .set_long_description(
7500         "RBD features are only applicable for v2 images. This setting accepts "
7501         "either an integer bitmask value or comma-delimited string of RBD "
7502         "feature names. This setting is always internally stored as an integer "
7503         "bitmask value. The mapping between feature bitmask value and feature "
7504         "name is as follows: +1 -> layering, +2 -> striping, "
7505         "+4 -> exclusive-lock, +8 -> object-map, +16 -> fast-diff, "
7506         "+32 -> deep-flatten, +64 -> journaling, +128 -> data-pool")
7507     .set_flag(Option::FLAG_RUNTIME)
7508     .set_validator([](std::string *value, std::string *error_message) {
7509         ostringstream ss;
7510         uint64_t features = librbd::rbd_features_from_string(*value, &ss);
7511         // Leave this in integer form to avoid breaking Cinder.  Someday
7512         // we would like to present this in string form instead...
7513         *value = stringify(features);
7514         if (ss.str().size()) {
7515           return -EINVAL;
7516         }
7517         return 0;
7518       }),
7519
7520     Option("rbd_op_threads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7521     .set_default(1)
7522     .set_description("number of threads to utilize for internal processing"),
7523
7524     Option("rbd_op_thread_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7525     .set_default(60)
7526     .set_description("time in seconds for detecting a hung thread"),
7527
7528     Option("rbd_disable_zero_copy_writes", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7529     .set_default(true)
7530     .set_description("Disable the use of zero-copy writes to ensure unstable "
7531                      "writes from clients cannot cause a CRC mismatch"),
7532
7533     Option("rbd_non_blocking_aio", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7534     .set_default(true)
7535     .set_description("process AIO ops from a dispatch thread to prevent blocking"),
7536
7537     Option("rbd_cache", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7538     .set_default(true)
7539     .set_description("whether to enable caching (writeback unless rbd_cache_max_dirty is 0)"),
7540
7541     Option("rbd_cache_policy", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7542     .set_enum_allowed({"writethrough", "writeback", "writearound"})
7543     .set_default("writearound")
7544     .set_description("cache policy for handling writes."),
7545
7546     Option("rbd_cache_writethrough_until_flush", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7547     .set_default(true)
7548     .set_description("whether to make writeback caching writethrough until "
7549                      "flush is called, to be sure the user of librbd will send "
7550                      "flushes so that writeback is safe"),
7551
7552     Option("rbd_cache_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7553     .set_default(32_M)
7554     .set_description("cache size in bytes"),
7555
7556     Option("rbd_cache_max_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7557     .set_default(24_M)
7558     .set_description("dirty limit in bytes - set to 0 for write-through caching"),
7559
7560     Option("rbd_cache_target_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7561     .set_default(16_M)
7562     .set_description("target dirty limit in bytes"),
7563
7564     Option("rbd_cache_max_dirty_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7565     .set_default(1.0)
7566     .set_description("seconds in cache before writeback starts"),
7567
7568     Option("rbd_cache_max_dirty_object", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7569     .set_default(0)
7570     .set_description("dirty limit for objects - set to 0 for auto calculate from rbd_cache_size"),
7571
7572     Option("rbd_cache_block_writes_upfront", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7573     .set_default(false)
7574     .set_description("whether to block writes to the cache before the aio_write call completes"),
7575
7576     Option("rbd_parent_cache_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7577     .set_default(false)
7578     .set_description("whether to enable rbd shared ro cache"),
7579
7580     Option("rbd_concurrent_management_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7581     .set_default(10)
7582     .set_min(1)
7583     .set_description("how many operations can be in flight for a management operation like deleting or resizing an image"),
7584
7585     Option("rbd_balance_snap_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7586     .set_default(false)
7587     .set_description("distribute snap read requests to random OSD"),
7588
7589     Option("rbd_localize_snap_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7590     .set_default(false)
7591     .set_description("localize snap read requests to closest OSD"),
7592
7593     Option("rbd_balance_parent_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7594     .set_default(false)
7595     .set_description("distribute parent read requests to random OSD"),
7596
7597     Option("rbd_localize_parent_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7598     .set_default(false)
7599     .set_description("localize parent requests to closest OSD"),
7600
7601     Option("rbd_sparse_read_threshold_bytes", Option::TYPE_SIZE,
7602            Option::LEVEL_ADVANCED)
7603     .set_default(64_K)
7604     .set_description("threshold for issuing a sparse-read")
7605     .set_long_description("minimum number of sequential bytes to read against "
7606                           "an object before issuing a sparse-read request to "
7607                           "the cluster. 0 implies it must be a full object read "
7608                           "to issue a sparse-read, 1 implies always use "
7609                           "sparse-read, and any value larger than the maximum "
7610                           "object size will disable sparse-read for all "
7611                           "requests"),
7612
7613     Option("rbd_readahead_trigger_requests", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7614     .set_default(10)
7615     .set_description("number of sequential requests necessary to trigger readahead"),
7616
7617     Option("rbd_readahead_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7618     .set_default(512_K)
7619     .set_description("set to 0 to disable readahead"),
7620
7621     Option("rbd_readahead_disable_after_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7622     .set_default(50_M)
7623     .set_description("how many bytes are read in total before readahead is disabled"),
7624
7625     Option("rbd_clone_copy_on_read", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7626     .set_default(false)
7627     .set_description("copy-up parent image blocks to clone upon read request"),
7628
7629     Option("rbd_blocklist_on_break_lock", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7630     .set_default(true)
7631     .set_description("whether to blocklist clients whose lock was broken"),
7632
7633     Option("rbd_blocklist_expire_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7634     .set_default(0)
7635     .set_description("number of seconds to blocklist - set to 0 for OSD default"),
7636
7637     Option("rbd_request_timed_out_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7638     .set_default(30)
7639     .set_description("number of seconds before maintenance request times out"),
7640
7641     Option("rbd_skip_partial_discard", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7642     .set_default(true)
7643     .set_description("skip discard (zero) of unaligned extents within an object"),
7644
7645     Option("rbd_discard_granularity_bytes", Option::TYPE_UINT,
7646            Option::LEVEL_ADVANCED)
7647     .set_default(64_K)
7648     .set_min_max(4_K, 32_M)
7649     .set_validator([](std::string *value, std::string *error_message){
7650         uint64_t f = strict_si_cast<uint64_t>(value->c_str(), error_message);
7651         if (!error_message->empty()) {
7652           return -EINVAL;
7653         } else if (!isp2(f)) {
7654           *error_message = "value must be a power of two";
7655           return -EINVAL;
7656         }
7657         return 0;
7658       })
7659     .set_description("minimum aligned size of discard operations"),
7660
7661     Option("rbd_enable_alloc_hint", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7662     .set_default(true)
7663     .set_description("when writing a object, it will issue a hint to osd backend to indicate the expected size object need"),
7664
7665     Option("rbd_compression_hint", Option::TYPE_STR, Option::LEVEL_BASIC)
7666     .set_enum_allowed({"none", "compressible", "incompressible"})
7667     .set_default("none")
7668     .set_description("Compression hint to send to the OSDs during writes")
7669     .set_flag(Option::FLAG_RUNTIME),
7670
7671     Option("rbd_read_from_replica_policy", Option::TYPE_STR, Option::LEVEL_BASIC)
7672     .set_enum_allowed({"default", "balance", "localize"})
7673     .set_default("default")
7674     .set_description("Read replica policy send to the OSDS during reads")
7675     .set_flag(Option::FLAG_RUNTIME),
7676
7677     Option("rbd_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7678     .set_default(false)
7679     .set_description("true if LTTng-UST tracepoints should be enabled"),
7680
7681     Option("rbd_blkin_trace_all", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7682     .set_default(false)
7683     .set_description("create a blkin trace for all RBD requests"),
7684
7685     Option("rbd_validate_pool", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7686     .set_default(true)
7687     .set_description("validate empty pools for RBD compatibility"),
7688
7689     Option("rbd_validate_names", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7690     .set_default(true)
7691     .set_description("validate new image names for RBD compatibility"),
7692
7693     Option("rbd_invalidate_object_map_on_timeout", Option::TYPE_BOOL, Option::LEVEL_DEV)
7694     .set_default(true)
7695     .set_description("true if object map should be invalidated when load or update timeout"),
7696
7697     Option("rbd_auto_exclusive_lock_until_manual_request", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7698     .set_default(true)
7699     .set_description("automatically acquire/release exclusive lock until it is explicitly requested"),
7700
7701     Option("rbd_move_to_trash_on_remove", Option::TYPE_BOOL, Option::LEVEL_BASIC)
7702     .set_default(false)
7703     .set_description("automatically move images to the trash when deleted"),
7704
7705     Option("rbd_move_to_trash_on_remove_expire_seconds", Option::TYPE_UINT, Option::LEVEL_BASIC)
7706     .set_default(0)
7707     .set_description("default number of seconds to protect deleted images in the trash"),
7708
7709     Option("rbd_move_parent_to_trash_on_remove", Option::TYPE_BOOL, Option::LEVEL_BASIC)
7710     .set_default(false)
7711     .set_description("move parent with clone format v2 children to the trash when deleted"),
7712
7713     Option("rbd_mirroring_resync_after_disconnect", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7714     .set_default(false)
7715     .set_description("automatically start image resync after mirroring is disconnected due to being laggy"),
7716
7717     Option("rbd_mirroring_delete_delay", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7718     .set_default(0)
7719     .set_description("time-delay in seconds for rbd-mirror delete propagation"),
7720
7721     Option("rbd_mirroring_replay_delay", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7722     .set_default(0)
7723     .set_description("time-delay in seconds for rbd-mirror asynchronous replication"),
7724
7725     Option("rbd_mirroring_max_mirroring_snapshots", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7726     .set_default(3)
7727     .set_min(3)
7728     .set_description("mirroring snapshots limit"),
7729
7730     Option("rbd_default_format", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7731     .set_default(2)
7732     .set_description("default image format for new images"),
7733
7734     Option("rbd_default_order", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7735     .set_default(22)
7736     .set_description("default order (data block object size) for new images"),
7737
7738     Option("rbd_default_stripe_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7739     .set_default(0)
7740     .set_description("default stripe count for new images"),
7741
7742     Option("rbd_default_stripe_unit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7743     .set_default(0)
7744     .set_description("default stripe width for new images"),
7745
7746     Option("rbd_default_map_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7747     .set_default("")
7748     .set_description("default krbd map options"),
7749
7750     Option("rbd_default_clone_format", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7751     .set_enum_allowed({"1", "2", "auto"})
7752     .set_default("auto")
7753     .set_description("default internal format for handling clones")
7754     .set_long_description("This sets the internal format for tracking cloned "
7755                           "images. The setting of '1' requires attaching to "
7756                           "protected snapshots that cannot be removed until "
7757                           "the clone is removed/flattened. The setting of '2' "
7758                           "will allow clones to be attached to any snapshot "
7759                           "and permits removing in-use parent snapshots but "
7760                           "requires Mimic or later clients. The default "
7761                           "setting of 'auto' will use the v2 format if the "
7762                           "cluster is configured to require mimic or later "
7763                           "clients.")
7764     .set_flag(Option::FLAG_RUNTIME),
7765
7766     Option("rbd_journal_order", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7767     .set_min_max(12, 26)
7768     .set_default(24)
7769     .set_description("default order (object size) for journal data objects"),
7770
7771     Option("rbd_journal_splay_width", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7772     .set_default(4)
7773     .set_description("number of active journal objects"),
7774
7775     Option("rbd_journal_commit_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7776     .set_default(5)
7777     .set_description("commit time interval, seconds"),
7778
7779     Option("rbd_journal_object_writethrough_until_flush", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7780     .set_default(true)
7781     .set_description("when enabled, the rbd_journal_object_flush* configuration "
7782                      "options are ignored until the first flush so that batched "
7783                      "journal IO is known to be safe for consistency"),
7784
7785     Option("rbd_journal_object_flush_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7786     .set_default(0)
7787     .set_description("maximum number of pending commits per journal object"),
7788
7789     Option("rbd_journal_object_flush_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7790     .set_default(1_M)
7791     .set_description("maximum number of pending bytes per journal object"),
7792
7793     Option("rbd_journal_object_flush_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7794     .set_default(0)
7795     .set_description("maximum age (in seconds) for pending commits"),
7796
7797     Option("rbd_journal_object_max_in_flight_appends", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7798     .set_default(0)
7799     .set_description("maximum number of in-flight appends per journal object"),
7800
7801     Option("rbd_journal_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7802     .set_default("")
7803     .set_description("pool for journal objects"),
7804
7805     Option("rbd_journal_max_payload_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7806     .set_default(16384)
7807     .set_description("maximum journal payload size before splitting"),
7808
7809     Option("rbd_journal_max_concurrent_object_sets", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7810     .set_default(0)
7811     .set_description("maximum number of object sets a journal client can be behind before it is automatically unregistered"),
7812
7813     Option("rbd_qos_iops_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7814     .set_default(0)
7815     .set_description("the desired limit of IO operations per second"),
7816
7817     Option("rbd_qos_bps_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7818     .set_default(0)
7819     .set_description("the desired limit of IO bytes per second"),
7820
7821     Option("rbd_qos_read_iops_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7822     .set_default(0)
7823     .set_description("the desired limit of read operations per second"),
7824
7825     Option("rbd_qos_write_iops_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7826     .set_default(0)
7827     .set_description("the desired limit of write operations per second"),
7828
7829     Option("rbd_qos_read_bps_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7830     .set_default(0)
7831     .set_description("the desired limit of read bytes per second"),
7832
7833     Option("rbd_qos_write_bps_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7834     .set_default(0)
7835     .set_description("the desired limit of write bytes per second"),
7836
7837     Option("rbd_qos_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7838     .set_default(0)
7839     .set_description("the desired burst limit of IO operations"),
7840
7841     Option("rbd_qos_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7842     .set_default(0)
7843     .set_description("the desired burst limit of IO bytes"),
7844
7845     Option("rbd_qos_read_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7846     .set_default(0)
7847     .set_description("the desired burst limit of read operations"),
7848
7849     Option("rbd_qos_write_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7850     .set_default(0)
7851     .set_description("the desired burst limit of write operations"),
7852
7853     Option("rbd_qos_read_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7854     .set_default(0)
7855     .set_description("the desired burst limit of read bytes"),
7856
7857     Option("rbd_qos_write_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7858     .set_default(0)
7859     .set_description("the desired burst limit of write bytes"),
7860
7861     Option("rbd_qos_iops_burst_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7862     .set_default(1)
7863     .set_min(1)
7864     .set_description("the desired burst duration in seconds of IO operations"),
7865
7866     Option("rbd_qos_bps_burst_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7867     .set_default(1)
7868     .set_min(1)
7869     .set_description("the desired burst duration in seconds of IO bytes"),
7870
7871     Option("rbd_qos_read_iops_burst_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7872     .set_default(1)
7873     .set_min(1)
7874     .set_description("the desired burst duration in seconds of read operations"),
7875
7876     Option("rbd_qos_write_iops_burst_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7877     .set_default(1)
7878     .set_min(1)
7879     .set_description("the desired burst duration in seconds of write operations"),
7880
7881     Option("rbd_qos_read_bps_burst_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7882     .set_default(1)
7883     .set_min(1)
7884     .set_description("the desired burst duration in seconds of read bytes"),
7885
7886     Option("rbd_qos_write_bps_burst_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7887     .set_default(1)
7888     .set_min(1)
7889     .set_description("the desired burst duration in seconds of write bytes"),
7890
7891     Option("rbd_qos_schedule_tick_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7892     .set_default(50)
7893     .set_min(1)
7894     .set_description("minimum schedule tick (in milliseconds) for QoS"),
7895
7896     Option("rbd_discard_on_zeroed_write_same", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7897     .set_default(true)
7898     .set_description("discard data on zeroed write same instead of writing zero"),
7899
7900     Option("rbd_mtime_update_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7901     .set_default(60)
7902     .set_min(0)
7903     .set_description("RBD Image modify timestamp refresh interval. Set to 0 to disable modify timestamp update."),
7904
7905     Option("rbd_atime_update_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7906     .set_default(60)
7907     .set_min(0)
7908     .set_description("RBD Image access timestamp refresh interval. Set to 0 to disable access timestamp update."),
7909
7910     Option("rbd_io_scheduler", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7911     .set_default("simple")
7912     .set_enum_allowed({"none", "simple"})
7913     .set_description("RBD IO scheduler"),
7914
7915     Option("rbd_io_scheduler_simple_max_delay", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7916     .set_default(0)
7917     .set_min(0)
7918     .set_description("maximum io delay (in milliseconds) for simple io scheduler (if set to 0 dalay is calculated based on latency stats)"),
7919
7920     Option("rbd_persistent_cache_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7921     .set_default("disabled")
7922     .set_enum_allowed({"disabled", "rwl", "ssd"})
7923     .set_description("enable persistent write back cache for this volume"),
7924
7925     Option("rbd_persistent_cache_log_periodic_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7926     .set_default(false)
7927     .set_description("emit periodic perf stats to debug log"),
7928
7929     Option("rbd_persistent_cache_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7930     .set_default(1073741824)
7931     .set_min(1073741824)
7932     .set_description("size of the persistent write back cache for this volume"),
7933
7934     Option("rbd_persistent_cache_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7935     .set_default("/tmp")
7936     .set_description("location of the persistent write back cache in a DAX-enabled filesystem on persistent memory"),
7937
7938     Option("rbd_quiesce_notification_attempts", Option::TYPE_UINT, Option::LEVEL_DEV)
7939     .set_default(10)
7940     .set_min(1)
7941     .set_description("the number of quiesce notification attempts"),
7942
7943     Option("rbd_default_snapshot_quiesce_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7944     .set_default("required")
7945     .set_enum_allowed({"required", "ignore-error", "skip"})
7946     .set_description("default snapshot quiesce mode"),
7947
7948     Option("rbd_plugins", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7949     .set_default("")
7950     .set_description("comma-delimited list of librbd plugins to enable"),
7951
7952     Option("rbd_config_pool_override_update_timestamp", Option::TYPE_UINT,
7953            Option::LEVEL_DEV)
7954     .set_default(0)
7955     .set_description("timestamp of last update to pool-level config overrides"),
7956
7957   });
7958 }
7959
7960 static std::vector<Option> get_rbd_mirror_options() {
7961   return std::vector<Option>({
7962     Option("rbd_mirror_journal_commit_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7963     .set_default(5)
7964     .set_description("commit time interval, seconds"),
7965
7966     Option("rbd_mirror_journal_poll_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7967     .set_default(5)
7968     .set_description("maximum age (in seconds) between successive journal polls"),
7969
7970     Option("rbd_mirror_sync_point_update_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7971     .set_default(30)
7972     .set_description("number of seconds between each update of the image sync point object number"),
7973
7974     Option("rbd_mirror_concurrent_image_syncs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7975     .set_default(5)
7976     .set_description("maximum number of image syncs in parallel"),
7977
7978     Option("rbd_mirror_pool_replayers_refresh_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7979     .set_default(30)
7980     .set_description("interval to refresh peers in rbd-mirror daemon"),
7981
7982     Option("rbd_mirror_concurrent_image_deletions", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7983     .set_default(1)
7984     .set_min(1)
7985     .set_description("maximum number of image deletions in parallel"),
7986
7987     Option("rbd_mirror_delete_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7988     .set_default(30)
7989     .set_description("interval to check and retry the failed deletion requests"),
7990
7991     Option("rbd_mirror_image_state_check_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7992     .set_default(30)
7993     .set_min(1)
7994     .set_description("interval to get images from pool watcher and set sources in replayer"),
7995
7996     Option("rbd_mirror_leader_heartbeat_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7997     .set_default(5)
7998     .set_min(1)
7999     .set_description("interval (in seconds) between mirror leader heartbeats"),
8000
8001     Option("rbd_mirror_leader_max_missed_heartbeats", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8002     .set_default(2)
8003     .set_description("number of missed heartbeats for non-lock owner to attempt to acquire lock"),
8004
8005     Option("rbd_mirror_leader_max_acquire_attempts_before_break", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8006     .set_default(3)
8007     .set_description("number of failed attempts to acquire lock after missing heartbeats before breaking lock"),
8008
8009     Option("rbd_mirror_image_policy_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8010     .set_default("simple")
8011     .set_enum_allowed({"none", "simple"})
8012     .set_description("active/active policy type for mapping images to instances"),
8013
8014     Option("rbd_mirror_image_policy_migration_throttle", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8015     .set_default(300)
8016     .set_description("number of seconds after which an image can be reshuffled (migrated) again"),
8017
8018     Option("rbd_mirror_image_policy_update_throttle_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8019     .set_default(1)
8020     .set_min(1)
8021     .set_description("interval (in seconds) to throttle images for mirror daemon peer updates"),
8022
8023     Option("rbd_mirror_image_policy_rebalance_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8024     .set_default(0)
8025     .set_description("number of seconds policy should be idle before trigerring reshuffle (rebalance) of images"),
8026
8027     Option("rbd_mirror_perf_stats_prio", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8028     .set_default((int64_t)PerfCountersBuilder::PRIO_USEFUL)
8029     .set_description("Priority level for mirror daemon replication perf counters")
8030     .set_long_description("The daemon will send perf counter data to the "
8031                           "manager daemon if the priority is not lower than "
8032                           "mgr_stats_threshold.")
8033     .set_min_max((int64_t)PerfCountersBuilder::PRIO_DEBUGONLY,
8034                  (int64_t)PerfCountersBuilder::PRIO_CRITICAL + 1),
8035
8036     Option("rbd_mirror_image_perf_stats_prio", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8037     .set_default((int64_t)PerfCountersBuilder::PRIO_USEFUL)
8038     .set_description("Priority level for mirror daemon per-image replication perf counters")
8039     .set_long_description("The daemon will send per-image perf counter data to the "
8040                           "manager daemon if the priority is not lower than "
8041                           "mgr_stats_threshold.")
8042     .set_min_max((int64_t)PerfCountersBuilder::PRIO_DEBUGONLY,
8043                  (int64_t)PerfCountersBuilder::PRIO_CRITICAL + 1),
8044
8045     Option("rbd_mirror_memory_autotune", Option::TYPE_BOOL, Option::LEVEL_DEV)
8046     .set_default(true)
8047     .add_see_also("rbd_mirror_memory_target")
8048     .set_description("Automatically tune the ratio of caches while respecting min values."),
8049
8050     Option("rbd_mirror_memory_target", Option::TYPE_SIZE, Option::LEVEL_BASIC)
8051     .set_default(4_G)
8052     .add_see_also("rbd_mirror_memory_autotune")
8053     .set_description("When tcmalloc and cache autotuning is enabled, try to keep this many bytes mapped in memory."),
8054
8055     Option("rbd_mirror_memory_base", Option::TYPE_SIZE, Option::LEVEL_DEV)
8056     .set_default(768_M)
8057     .add_see_also("rbd_mirror_memory_autotune")
8058     .set_description("When tcmalloc and cache autotuning is enabled, estimate the minimum amount of memory in bytes the rbd-mirror daemon will need."),
8059
8060     Option("rbd_mirror_memory_expected_fragmentation", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8061     .set_default(0.15)
8062     .set_min_max(0.0, 1.0)
8063     .add_see_also("rbd_mirror_memory_autotune")
8064     .set_description("When tcmalloc and cache autotuning is enabled, estimate the percent of memory fragmentation."),
8065
8066     Option("rbd_mirror_memory_cache_min", Option::TYPE_SIZE, Option::LEVEL_DEV)
8067     .set_default(128_M)
8068     .add_see_also("rbd_mirror_memory_autotune")
8069     .set_description("When tcmalloc and cache autotuning is enabled, set the minimum amount of memory used for cache."),
8070
8071     Option("rbd_mirror_memory_cache_resize_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8072     .set_default(5)
8073     .add_see_also("rbd_mirror_memory_autotune")
8074     .set_description("When tcmalloc and cache autotuning is enabled, wait this many seconds between resizing caches."),
8075
8076     Option("rbd_mirror_memory_cache_autotune_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8077     .set_default(30)
8078     .add_see_also("rbd_mirror_memory_autotune")
8079     .set_description("The number of seconds to wait between rebalances when cache autotune is enabled."),
8080   });
8081 }
8082
8083 static std::vector<Option> get_immutable_object_cache_options() {
8084   return std::vector<Option>({
8085     Option("immutable_object_cache_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8086     .set_default("/tmp/ceph_immutable_object_cache")
8087     .set_description("immutable object cache data dir"),
8088
8089     Option("immutable_object_cache_sock", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8090     .set_default("/var/run/ceph/immutable_object_cache_sock")
8091     .set_description("immutable object cache domain socket"),
8092
8093     Option("immutable_object_cache_max_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8094     .set_default(1_G)
8095     .set_description("max immutable object cache data size"),
8096
8097     Option("immutable_object_cache_max_inflight_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8098     .set_default(128)
8099     .set_description("max inflight promoting requests for immutable object cache daemon"),
8100
8101     Option("immutable_object_cache_client_dedicated_thread_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8102     .set_default(2)
8103     .set_description("immutable object cache client dedicated thread number"),
8104
8105     Option("immutable_object_cache_watermark", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8106     .set_default(0.9)
8107     .set_description("immutable object cache water mark"),
8108
8109     Option("immutable_object_cache_qos_schedule_tick_min", Option::TYPE_MILLISECS, Option::LEVEL_ADVANCED)
8110     .set_default(50)
8111     .set_min(1)
8112     .set_description("minimum schedule tick for immutable object cache"),
8113
8114     Option("immutable_object_cache_qos_iops_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8115     .set_default(0)
8116     .set_description("the desired immutable object cache IO operations limit per second"),
8117
8118     Option("immutable_object_cache_qos_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8119     .set_default(0)
8120     .set_description("the desired burst limit of immutable object cache IO operations"),
8121
8122     Option("immutable_object_cache_qos_iops_burst_seconds", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
8123     .set_default(1)
8124     .set_min(1)
8125     .set_description("the desired burst duration in seconds of immutable object cache IO operations"),
8126
8127     Option("immutable_object_cache_qos_bps_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8128     .set_default(0)
8129     .set_description("the desired immutable object cache IO bytes limit per second"),
8130
8131     Option("immutable_object_cache_qos_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8132     .set_default(0)
8133     .set_description("the desired burst limit of immutable object cache IO bytes"),
8134
8135     Option("immutable_object_cache_qos_bps_burst_seconds", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
8136     .set_default(1)
8137     .set_min(1)
8138     .set_description("the desired burst duration in seconds of immutable object cache IO bytes"),
8139   });
8140 }
8141
8142 std::vector<Option> get_mds_options() {
8143   return std::vector<Option>({
8144     Option("mds_alternate_name_max", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8145     .set_default(8192)
8146     .set_flag(Option::FLAG_RUNTIME)
8147     .set_description("set the maximum length of alternate names for dentries"),
8148
8149     Option("mds_valgrind_exit", Option::TYPE_BOOL, Option::LEVEL_DEV)
8150     .set_default(false)
8151     .set_flag(Option::FLAG_RUNTIME),
8152
8153     Option("mds_numa_node", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8154     .set_default(-1)
8155     .set_flag(Option::FLAG_STARTUP)
8156     .set_description("set mds's cpu affinity to a numa node (-1 for none)"),
8157
8158     Option("mds_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8159     .set_default("/var/lib/ceph/mds/$cluster-$id")
8160     .set_flag(Option::FLAG_NO_MON_UPDATE)
8161     .set_description("path to MDS data and keyring"),
8162
8163     Option("mds_join_fs", Option::TYPE_STR, Option::LEVEL_BASIC)
8164     .set_default("")
8165     .set_description("file system MDS prefers to join")
8166     .set_long_description("This setting indicates which file system name the MDS should prefer to join (affinity). The monitors will try to have the MDS cluster safely reach a state where all MDS have strong affinity, even via failovers to a standby.")
8167     .set_flag(Option::FLAG_RUNTIME),
8168
8169     Option("mds_max_xattr_pairs_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8170     .set_default(64_K)
8171     .set_description("maximum aggregate size of extended attributes on a file"),
8172
8173     Option("mds_cache_trim_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
8174     .set_default(1)
8175     .set_description("interval in seconds between cache trimming")
8176     .set_flag(Option::FLAG_RUNTIME),
8177
8178     Option("mds_cache_release_free_interval", Option::TYPE_SECS, Option::LEVEL_DEV)
8179     .set_default(10)
8180     .set_description("interval in seconds between heap releases")
8181     .set_flag(Option::FLAG_RUNTIME),
8182
8183     Option("mds_cache_memory_limit", Option::TYPE_SIZE, Option::LEVEL_BASIC)
8184     .set_default(4_G)
8185     .set_description("target maximum memory usage of MDS cache")
8186     .set_flag(Option::FLAG_RUNTIME)
8187     .set_long_description("This sets a target maximum memory usage of the MDS cache and is the primary tunable to limit the MDS memory usage. The MDS will try to stay under a reservation of this limit (by default 95%; 1 - mds_cache_reservation) by trimming unused metadata in its cache and recalling cached items in the client caches. It is possible for the MDS to exceed this limit due to slow recall from clients. The mds_health_cache_threshold (150%) sets a cache full threshold for when the MDS signals a cluster health warning."),
8188
8189     Option("mds_cache_reservation", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8190     .set_default(.05)
8191     .set_description("amount of memory to reserve for future cached objects")
8192     .set_flag(Option::FLAG_RUNTIME),
8193
8194     Option("mds_health_cache_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8195     .set_default(1.5)
8196     .set_description("threshold for cache size to generate health warning"),
8197
8198     Option("mds_cache_mid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8199     .set_default(.7)
8200     .set_description("midpoint for MDS cache LRU"),
8201
8202     Option("mds_cache_trim_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8203     .set_default(1)
8204     .set_description("decay rate for trimming MDS cache throttle")
8205     .set_flag(Option::FLAG_RUNTIME),
8206
8207     Option("mds_cache_trim_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8208     .set_default(256_K)
8209     .set_description("threshold for number of dentries that can be trimmed")
8210     .set_flag(Option::FLAG_RUNTIME),
8211
8212     Option("mds_max_file_recover", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8213     .set_default(32)
8214     .set_description("maximum number of files to recover file sizes in parallel"),
8215
8216     Option("mds_dir_max_commit_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8217     .set_default(10)
8218     .set_description("maximum size in megabytes for a RADOS write to a directory"),
8219
8220     Option("mds_dir_keys_per_op", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8221     .set_default(16384)
8222     .set_description("number of directory entries to read in one RADOS operation"),
8223
8224     Option("mds_decay_halflife", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8225     .set_default(5)
8226     .set_description("rate of decay for temperature counters on each directory for balancing"),
8227
8228     Option("mds_beacon_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8229     .set_default(4)
8230     .set_description("interval in seconds between MDS beacons to monitors"),
8231
8232     Option("mds_beacon_grace", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8233     .set_default(15)
8234     .set_description("tolerance in seconds for missed MDS beacons to monitors"),
8235
8236     Option("mds_heartbeat_grace", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8237     .set_default(15)
8238     .set_description("tolerance in seconds for MDS internal heartbeat"),
8239
8240     Option("mds_enforce_unique_name", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8241     .set_default(true)
8242     .set_description("require MDS name is unique in the cluster"),
8243
8244     Option("mds_session_blocklist_on_timeout", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8245     .set_default(true)
8246     .set_description("blocklist clients whose sessions have become stale"),
8247
8248     Option("mds_session_blocklist_on_evict", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8249     .set_default(true)
8250     .set_description("blocklist clients that have been evicted"),
8251
8252     Option("mds_sessionmap_keys_per_op", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8253     .set_default(1024)
8254     .set_description("number of omap keys to read from the SessionMap in one operation"),
8255
8256     Option("mds_recall_max_caps", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8257     .set_default(30000)
8258     .set_description("maximum number of caps to recall from client session in single recall")
8259     .set_flag(Option::FLAG_RUNTIME),
8260
8261     Option("mds_recall_max_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8262     .set_default(1.5)
8263     .set_description("decay rate for throttle on recalled caps on a session")
8264     .set_flag(Option::FLAG_RUNTIME),
8265
8266     Option("mds_recall_max_decay_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8267     .set_default(128_K)
8268     .set_description("decay threshold for throttle on recalled caps on a session")
8269     .set_flag(Option::FLAG_RUNTIME),
8270
8271     Option("mds_recall_global_max_decay_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8272     .set_default(128_K)
8273     .set_description("decay threshold for throttle on recalled caps globally")
8274     .set_flag(Option::FLAG_RUNTIME),
8275
8276     Option("mds_recall_warning_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8277     .set_default(256_K)
8278     .set_description("decay threshold for warning on slow session cap recall")
8279     .set_flag(Option::FLAG_RUNTIME),
8280
8281     Option("mds_recall_warning_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8282     .set_default(60.0)
8283     .set_description("decay rate for warning on slow session cap recall")
8284     .set_flag(Option::FLAG_RUNTIME),
8285
8286     Option("mds_session_cache_liveness_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8287     .add_see_also("mds_session_cache_liveness_magnitude")
8288     .set_default(5_min)
8289     .set_description("decay rate for session liveness leading to preemptive cap recall")
8290     .set_flag(Option::FLAG_RUNTIME)
8291     .set_long_description("This determines how long a session needs to be quiescent before the MDS begins preemptively recalling capabilities. The default of 5 minutes will cause 10 halvings of the decay counter after 1 hour, or 1/1024. The default magnitude of 10 (1^10 or 1024) is chosen so that the MDS considers a previously chatty session (approximately) to be quiescent after 1 hour."),
8292
8293     Option("mds_session_cache_liveness_magnitude", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8294     .add_see_also("mds_session_cache_liveness_decay_rate")
8295     .set_default(10)
8296     .set_description("decay magnitude for preemptively recalling caps on quiet client")
8297     .set_flag(Option::FLAG_RUNTIME)
8298     .set_long_description("This is the order of magnitude difference (in base 2) of the internal liveness decay counter and the number of capabilities the session holds. When this difference occurs, the MDS treats the session as quiescent and begins recalling capabilities."),
8299
8300     Option("mds_session_cap_acquisition_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8301     .set_default(10)
8302     .set_description("decay rate for session readdir caps leading to readdir throttle")
8303     .set_flag(Option::FLAG_RUNTIME)
8304     .set_long_description("The half-life for the session cap acquisition counter of caps acquired by readdir. This is used for throttling readdir requests from clients slow to release caps."),
8305
8306     Option("mds_session_cap_acquisition_throttle", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8307     .set_default(500000)
8308     .set_description("throttle point for cap acquisition decay counter"),
8309
8310     Option("mds_session_max_caps_throttle_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8311     .set_default(1.1)
8312     .set_description("ratio of mds_max_maps_per_client that client must exceed before readdir may be throttled by cap acquisition throttle"),
8313
8314     Option("mds_cap_acquisition_throttle_retry_request_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8315     .set_default(0.5)
8316     .set_description("timeout in seconds after which a client request is retried due to cap acquisition throttling"),
8317
8318     Option("mds_freeze_tree_timeout", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8319     .set_default(30)
8320     .set_description(""),
8321
8322     Option("mds_health_summarize_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8323     .set_default(10)
8324     .set_description("threshold of number of clients to summarize late client recall"),
8325
8326     Option("mds_reconnect_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8327     .set_default(45)
8328     .set_description("timeout in seconds to wait for clients to reconnect during MDS reconnect recovery state"),
8329
8330     Option("mds_deny_all_reconnect", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8331     .set_default(false)
8332     .set_flag(Option::FLAG_RUNTIME)
8333     .set_description("flag to deny all client reconnects during failover"),
8334
8335     Option("mds_tick_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8336     .set_default(5)
8337     .set_description("time in seconds between upkeep tasks"),
8338
8339     Option("mds_dirstat_min_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8340     .set_default(1)
8341     .set_description(""),
8342
8343     Option("mds_scatter_nudge_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8344     .set_default(5)
8345     .set_description("minimum interval between scatter lock updates"),
8346
8347     Option("mds_client_prealloc_inos", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8348     .set_default(1000)
8349     .set_description("number of unused inodes to pre-allocate to clients for file creation"),
8350
8351     Option("mds_client_delegate_inos_pct", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8352     .set_default(50)
8353     .set_flag(Option::FLAG_RUNTIME)
8354     .set_description("percentage of preallocated inos to delegate to client"),
8355
8356     Option("mds_early_reply", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8357     .set_default(true)
8358     .set_description("additional reply to clients that metadata requests are complete but not yet durable"),
8359
8360     Option("mds_replay_unsafe_with_closed_session", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8361     .set_default(false)
8362     .set_flag(Option::FLAG_STARTUP)
8363     .set_description("complete all the replay request when mds is restarted, no matter the session is closed or not"),
8364
8365     Option("mds_default_dir_hash", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8366     .set_default(CEPH_STR_HASH_RJENKINS)
8367     .set_description("hash function to select directory fragment for dentry name"),
8368
8369     Option("mds_log_pause", Option::TYPE_BOOL, Option::LEVEL_DEV)
8370     .set_default(false)
8371     .set_description(""),
8372
8373     Option("mds_log_skip_corrupt_events", Option::TYPE_BOOL, Option::LEVEL_DEV)
8374     .set_default(false)
8375     .set_description(""),
8376
8377     Option("mds_log_max_events", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8378     .set_default(-1)
8379     .set_description("maximum number of events in the MDS journal (-1 is unlimited)"),
8380
8381     Option("mds_log_events_per_segment", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8382     .set_default(1024)
8383     .set_description("maximum number of events in an MDS journal segment"),
8384
8385     Option("mds_log_segment_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8386     .set_default(0)
8387     .set_description("size in bytes of each MDS log segment"),
8388
8389     Option("mds_log_max_segments", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8390     .set_default(128)
8391     .set_description("maximum number of segments which may be untrimmed"),
8392
8393     Option("mds_log_warn_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8394     .set_default(2.0)
8395     .set_min(1.0)
8396     .set_flag(Option::FLAG_RUNTIME)
8397     .set_description("trigger MDS_HEALTH_TRIM warning when the mds log is longer than mds_log_max_segments * mds_log_warn_factor"),
8398
8399     Option("mds_bal_export_pin", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8400     .set_default(true)
8401     .set_description("allow setting directory export pins to particular ranks"),
8402
8403     Option("mds_export_ephemeral_random", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8404     .set_default(true)
8405     .set_flag(Option::FLAG_RUNTIME)
8406     .set_description("allow ephemeral random pinning of the loaded subtrees")
8407     .set_long_description("probabilistically pin the loaded directory inode and the subtree beneath it to an MDS based on the consistent hash of the inode number. The higher this value the more likely the loaded subtrees get pinned"),
8408
8409     Option("mds_export_ephemeral_random_max", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8410     .set_default(0.01)
8411     .set_flag(Option::FLAG_RUNTIME)
8412     .set_description("the maximum percent permitted for random ephemeral pin policy")
8413     .set_min_max(0.0, 1.0)
8414     .add_see_also("mds_export_ephemeral_random"),
8415
8416     Option("mds_export_ephemeral_distributed", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8417     .set_default(true)
8418     .set_flag(Option::FLAG_RUNTIME)
8419     .set_description("allow ephemeral distributed pinning of the loaded subtrees")
8420     .set_long_description("pin the immediate child directories of the loaded directory inode based on the consistent hash of the child's inode number. "),
8421
8422     Option("mds_export_ephemeral_distributed_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8423     .set_default(2.0)
8424     .set_min_max(1.0, 100.0)
8425     .set_flag(Option::FLAG_RUNTIME)
8426     .set_description("multiple of max_mds for splitting and distributing directory"),
8427
8428     Option("mds_bal_sample_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8429     .set_default(3.0)
8430     .set_description("interval in seconds between balancer ticks"),
8431
8432     Option("mds_bal_replicate_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8433     .set_default(8000)
8434     .set_description("hot popularity threshold to replicate a subtree"),
8435
8436     Option("mds_bal_unreplicate_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8437     .set_default(0)
8438     .set_description("cold popularity threshold to merge subtrees"),
8439
8440     Option("mds_bal_split_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8441     .set_default(10000)
8442     .set_description("minimum size of directory fragment before splitting"),
8443
8444     Option("mds_bal_split_rd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8445     .set_default(25000)
8446     .set_description("hot read popularity threshold for splitting a directory fragment"),
8447
8448     Option("mds_bal_split_wr", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8449     .set_default(10000)
8450     .set_description("hot write popularity threshold for splitting a directory fragment"),
8451
8452     Option("mds_bal_split_bits", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8453     .set_default(3)
8454     .set_min_max(1, 24)
8455     .set_description("power of two child fragments for a fragment on split"),
8456
8457     Option("mds_bal_merge_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8458     .set_default(50)
8459     .set_description("size of fragments where merging should occur"),
8460
8461     Option("mds_bal_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8462     .set_default(10)
8463     .set_description("interval between MDS balancer cycles"),
8464
8465     Option("mds_bal_fragment_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8466     .set_default(5)
8467     .set_description("delay in seconds before interrupting client IO to perform splits"),
8468
8469     Option("mds_bal_fragment_size_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8470     .set_default(10000*10)
8471     .set_description("maximum size of a directory fragment before new creat/links fail"),
8472
8473     Option("mds_bal_fragment_fast_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8474     .set_default(1.5)
8475     .set_description("ratio of mds_bal_split_size at which fast fragment splitting occurs"),
8476
8477     Option("mds_bal_fragment_dirs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8478     .set_default(true)
8479     .set_description("enable directory fragmentation")
8480     .set_long_description("Directory fragmentation is a standard feature of CephFS that allows sharding directories across multiple objects for performance and stability. Additionally, this allows fragments to be distributed across multiple active MDSs to increase throughput. Disabling (new) fragmentation should only be done in exceptional circumstances and may lead to performance issues."),
8481
8482     Option("mds_bal_idle_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8483     .set_default(0)
8484     .set_description("idle metadata popularity threshold before rebalancing"),
8485
8486     Option("mds_bal_max", Option::TYPE_INT, Option::LEVEL_DEV)
8487     .set_default(-1)
8488     .set_description(""),
8489
8490     Option("mds_bal_max_until", Option::TYPE_INT, Option::LEVEL_DEV)
8491     .set_default(-1)
8492     .set_description(""),
8493
8494     Option("mds_bal_mode", Option::TYPE_INT, Option::LEVEL_DEV)
8495     .set_default(0)
8496     .set_description(""),
8497
8498     Option("mds_bal_min_rebalance", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8499     .set_default(.1)
8500     .set_description("amount overloaded over internal target before balancer begins offloading"),
8501
8502     Option("mds_bal_min_start", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8503     .set_default(.2)
8504     .set_description(""),
8505
8506     Option("mds_bal_need_min", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8507     .set_default(.8)
8508     .set_description(""),
8509
8510     Option("mds_bal_need_max", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8511     .set_default(1.2)
8512     .set_description(""),
8513
8514     Option("mds_bal_midchunk", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8515     .set_default(.3)
8516     .set_description(""),
8517
8518     Option("mds_bal_minchunk", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8519     .set_default(.001)
8520     .set_description(""),
8521
8522     Option("mds_bal_target_decay", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8523     .set_default(10.0)
8524     .set_description("rate of decay for export targets communicated to clients"),
8525
8526     Option("mds_oft_prefetch_dirfrags", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8527     .set_default(true)
8528     .set_description("prefetch dirfrags recorded in open file table on startup")
8529     .set_flag(Option::FLAG_STARTUP),
8530
8531     Option("mds_replay_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8532     .set_default(1.0)
8533     .set_description("time in seconds between replay of updates to journal by standby replay MDS"),
8534
8535     Option("mds_shutdown_check", Option::TYPE_INT, Option::LEVEL_DEV)
8536     .set_default(0)
8537     .set_description(""),
8538
8539     Option("mds_thrash_exports", Option::TYPE_INT, Option::LEVEL_DEV)
8540     .set_default(0)
8541     .set_description(""),
8542
8543     Option("mds_thrash_fragments", Option::TYPE_INT, Option::LEVEL_DEV)
8544     .set_default(0)
8545     .set_description(""),
8546
8547     Option("mds_dump_cache_on_map", Option::TYPE_BOOL, Option::LEVEL_DEV)
8548     .set_default(false)
8549     .set_description(""),
8550
8551     Option("mds_dump_cache_after_rejoin", Option::TYPE_BOOL, Option::LEVEL_DEV)
8552     .set_default(false)
8553     .set_description(""),
8554
8555     Option("mds_verify_scatter", Option::TYPE_BOOL, Option::LEVEL_DEV)
8556     .set_default(false)
8557     .set_description(""),
8558
8559     Option("mds_debug_scatterstat", Option::TYPE_BOOL, Option::LEVEL_DEV)
8560     .set_default(false)
8561     .set_description(""),
8562
8563     Option("mds_debug_frag", Option::TYPE_BOOL, Option::LEVEL_DEV)
8564     .set_default(false)
8565     .set_description(""),
8566
8567     Option("mds_debug_auth_pins", Option::TYPE_BOOL, Option::LEVEL_DEV)
8568     .set_default(false)
8569     .set_description(""),
8570
8571     Option("mds_debug_subtrees", Option::TYPE_BOOL, Option::LEVEL_DEV)
8572     .set_default(false)
8573     .set_description(""),
8574
8575     Option("mds_kill_mdstable_at", Option::TYPE_INT, Option::LEVEL_DEV)
8576     .set_default(0)
8577     .set_description(""),
8578
8579     Option("mds_max_export_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
8580     .set_default(20_M)
8581     .set_description(""),
8582
8583     Option("mds_kill_export_at", Option::TYPE_INT, Option::LEVEL_DEV)
8584     .set_default(0)
8585     .set_description(""),
8586
8587     Option("mds_kill_import_at", Option::TYPE_INT, Option::LEVEL_DEV)
8588     .set_default(0)
8589     .set_description(""),
8590
8591     Option("mds_kill_link_at", Option::TYPE_INT, Option::LEVEL_DEV)
8592     .set_default(0)
8593     .set_description(""),
8594
8595     Option("mds_kill_rename_at", Option::TYPE_INT, Option::LEVEL_DEV)
8596     .set_default(0)
8597     .set_description(""),
8598
8599     Option("mds_kill_openc_at", Option::TYPE_INT, Option::LEVEL_DEV)
8600     .set_default(0)
8601     .set_description(""),
8602
8603     Option("mds_kill_journal_at", Option::TYPE_INT, Option::LEVEL_DEV)
8604     .set_default(0)
8605     .set_description(""),
8606
8607     Option("mds_kill_journal_expire_at", Option::TYPE_INT, Option::LEVEL_DEV)
8608     .set_default(0)
8609     .set_description(""),
8610
8611     Option("mds_kill_journal_replay_at", Option::TYPE_INT, Option::LEVEL_DEV)
8612     .set_default(0)
8613     .set_description(""),
8614
8615     Option("mds_journal_format", Option::TYPE_UINT, Option::LEVEL_DEV)
8616     .set_default(1)
8617     .set_description(""),
8618
8619     Option("mds_kill_create_at", Option::TYPE_INT, Option::LEVEL_DEV)
8620     .set_default(0)
8621     .set_description(""),
8622
8623     Option("mds_inject_traceless_reply_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8624     .set_default(0)
8625     .set_description(""),
8626
8627     Option("mds_wipe_sessions", Option::TYPE_BOOL, Option::LEVEL_DEV)
8628     .set_default(0)
8629     .set_description(""),
8630
8631     Option("mds_wipe_ino_prealloc", Option::TYPE_BOOL, Option::LEVEL_DEV)
8632     .set_default(0)
8633     .set_description(""),
8634
8635     Option("mds_skip_ino", Option::TYPE_INT, Option::LEVEL_DEV)
8636     .set_default(0)
8637     .set_description(""),
8638
8639     Option("mds_enable_op_tracker", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8640     .set_default(true)
8641     .set_description("track remote operation progression and statistics"),
8642
8643     Option("mds_op_history_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8644     .set_default(20)
8645     .set_description("maximum size for list of historical operations"),
8646
8647     Option("mds_op_history_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8648     .set_default(600)
8649     .set_description("expiration time in seconds of historical operations"),
8650
8651     Option("mds_op_complaint_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8652     .set_default(30)
8653     .set_description("time in seconds to consider an operation blocked after no updates"),
8654
8655     Option("mds_op_log_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
8656     .set_default(5)
8657     .set_description(""),
8658
8659     Option("mds_snap_min_uid", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8660     .set_default(0)
8661     .set_description("minimum uid of client to perform snapshots"),
8662
8663     Option("mds_snap_max_uid", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8664     .set_default(4294967294)
8665     .set_description("maximum uid of client to perform snapshots"),
8666
8667     Option("mds_snap_rstat", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8668     .set_default(false)
8669     .set_description("enabled nested rstat for snapshots"),
8670
8671     Option("mds_verify_backtrace", Option::TYPE_UINT, Option::LEVEL_DEV)
8672     .set_default(1)
8673     .set_description(""),
8674
8675     Option("mds_max_completed_flushes", Option::TYPE_UINT, Option::LEVEL_DEV)
8676     .set_default(100000)
8677     .set_description(""),
8678
8679     Option("mds_max_completed_requests", Option::TYPE_UINT, Option::LEVEL_DEV)
8680     .set_default(100000)
8681     .set_description(""),
8682
8683     Option("mds_action_on_write_error", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8684     .set_default(1)
8685     .set_description("action to take when MDS cannot write to RADOS (0:ignore, 1:read-only, 2:suicide)"),
8686
8687     Option("mds_mon_shutdown_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8688     .set_default(5)
8689     .set_description("time to wait for mon to receive damaged MDS rank notification"),
8690
8691     Option("mds_max_purge_files", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8692     .set_default(64)
8693     .set_description("maximum number of deleted files to purge in parallel"),
8694
8695     Option("mds_max_purge_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8696     .set_default(8192)
8697     .set_description("maximum number of purge operations performed in parallel"),
8698
8699     Option("mds_max_purge_ops_per_pg", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8700     .set_default(0.5)
8701     .set_description("number of parallel purge operations performed per PG"),
8702
8703     Option("mds_purge_queue_busy_flush_period", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8704     .set_default(1.0)
8705     .set_description(""),
8706
8707     Option("mds_root_ino_uid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8708     .set_default(0)
8709     .set_description("default uid for new root directory"),
8710
8711     Option("mds_root_ino_gid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8712     .set_default(0)
8713     .set_description("default gid for new root directory"),
8714
8715     Option("mds_max_scrub_ops_in_progress", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8716     .set_default(5)
8717     .set_description("maximum number of scrub operations performed in parallel"),
8718
8719     Option("mds_forward_all_requests_to_auth", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8720     .set_default(false)
8721     .set_flag(Option::FLAG_RUNTIME)
8722     .set_description("always process op on auth mds"),
8723
8724     Option("mds_damage_table_max_entries", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8725     .set_default(10000)
8726     .set_description("maximum number of damage table entries"),
8727
8728     Option("mds_client_writeable_range_max_inc_objs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8729     .set_default(1024)
8730     .set_description("maximum number of objects in writeable range of a file for a client"),
8731
8732     Option("mds_min_caps_per_client", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8733     .set_default(100)
8734     .set_description("minimum number of capabilities a client may hold"),
8735
8736     Option("mds_min_caps_working_set", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8737     .set_default(10000)
8738     .set_flag(Option::FLAG_RUNTIME)
8739     .set_description("number of capabilities a client may hold without cache pressure warnings generated"),
8740
8741     Option("mds_max_caps_per_client", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8742     .set_default(1_M)
8743     .set_description("maximum number of capabilities a client may hold"),
8744
8745     Option("mds_hack_allow_loading_invalid_metadata", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8746      .set_default(0)
8747      .set_description("INTENTIONALLY CAUSE DATA LOSS by bypasing checks for invalid metadata on disk. Allows testing repair tools."),
8748
8749     Option("mds_defer_session_stale", Option::TYPE_BOOL, Option::LEVEL_DEV)
8750      .set_default(true),
8751
8752     Option("mds_inject_migrator_session_race", Option::TYPE_BOOL, Option::LEVEL_DEV)
8753      .set_default(false),
8754
8755     Option("mds_request_load_average_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8756     .set_default(60)
8757     .set_description("rate of decay in seconds for calculating request load average"),
8758
8759     Option("mds_cap_revoke_eviction_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8760      .set_default(0)
8761      .set_description("number of seconds after which clients which have not responded to cap revoke messages by the MDS are evicted."),
8762
8763     Option("mds_max_retries_on_remount_failure", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8764      .set_default(5)
8765      .set_description("number of consecutive failed remount attempts for invalidating kernel dcache after which client would abort."),
8766
8767     Option("mds_dump_cache_threshold_formatter", Option::TYPE_SIZE, Option::LEVEL_DEV)
8768      .set_default(1_G)
8769      .set_description("threshold for cache usage to disallow \"dump cache\" operation to formatter")
8770      .set_long_description("Disallow MDS from dumping caches to formatter via \"dump cache\" command if cache usage exceeds this threshold."),
8771
8772     Option("mds_dump_cache_threshold_file", Option::TYPE_SIZE, Option::LEVEL_DEV)
8773      .set_default(0)
8774      .set_description("threshold for cache usage to disallow \"dump cache\" operation to file")
8775      .set_long_description("Disallow MDS from dumping caches to file via \"dump cache\" command if cache usage exceeds this threshold."),
8776
8777     Option("mds_task_status_update_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8778      .set_default(2.0)
8779      .set_description("task status update interval to manager")
8780      .set_long_description("interval (in seconds) for sending mds task status to ceph manager"),
8781
8782     Option("mds_max_snaps_per_dir", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8783      .set_default(100)
8784      .set_min_max(0, 4096)
8785      .set_flag(Option::FLAG_RUNTIME)
8786      .set_description("max snapshots per directory")
8787      .set_long_description("maximum number of snapshots that can be created per directory"),
8788
8789     Option("mds_asio_thread_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8790     .set_default(2)
8791     .set_min(1)
8792     .set_description("Size of thread pool for ASIO completions")
8793     .add_tag("mds"),
8794
8795     Option("mds_ping_grace", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
8796      .set_default(15)
8797      .set_flag(Option::FLAG_RUNTIME)
8798      .set_description("timeout after which an MDS is considered laggy by rank 0 MDS.")
8799      .set_long_description("timeout for replying to a ping message sent by rank 0 after which an active MDS considered laggy (delayed metrics) by rank 0."),
8800
8801     Option("mds_ping_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
8802      .set_default(5)
8803      .set_flag(Option::FLAG_RUNTIME)
8804      .set_description("interval in seconds for sending ping messages to active MDSs.")
8805      .set_long_description("interval in seconds for rank 0 to send ping messages to all active MDSs."),
8806
8807     Option("mds_metrics_update_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
8808      .set_default(2)
8809      .set_flag(Option::FLAG_RUNTIME)
8810      .set_description("interval in seconds for metrics data update.")
8811      .set_long_description("interval in seconds after which active MDSs send client metrics data to rank 0.")
8812   });
8813 }
8814
8815 std::vector<Option> get_mds_client_options() {
8816   return std::vector<Option>({
8817     Option("client_cache_size", Option::TYPE_SIZE, Option::LEVEL_BASIC)
8818     .set_default(16384)
8819     .set_description("soft maximum number of directory entries in client cache"),
8820
8821     Option("client_cache_mid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8822     .set_default(.75)
8823     .set_description("mid-point of client cache LRU"),
8824
8825     Option("client_use_random_mds", Option::TYPE_BOOL, Option::LEVEL_DEV)
8826     .set_default(false)
8827     .set_description("issue new requests to a random active MDS"),
8828
8829     Option("client_mount_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8830     .set_default(300.0)
8831     .set_description("timeout for mounting CephFS (seconds)"),
8832
8833     Option("client_tick_interval", Option::TYPE_SECS, Option::LEVEL_DEV)
8834     .set_default(1)
8835     .set_description("seconds between client upkeep ticks"),
8836
8837     Option("client_trace", Option::TYPE_STR, Option::LEVEL_DEV)
8838     .set_default("")
8839     .set_description("file containing trace of client operations"),
8840
8841     Option("client_readahead_min", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8842     .set_default(128*1024)
8843     .set_description("minimum bytes to readahead in a file"),
8844
8845     Option("client_readahead_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8846     .set_default(0)
8847     .set_description("maximum bytes to readahead in a file (zero is unlimited)"),
8848
8849     Option("client_readahead_max_periods", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8850     .set_default(4)
8851     .set_description("maximum stripe periods to readahead in a file"),
8852
8853     Option("client_reconnect_stale", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8854     .set_default(false)
8855     .set_description("reconnect when the session becomes stale"),
8856
8857     Option("client_snapdir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8858     .set_default(".snap")
8859     .set_description("pseudo directory for snapshot access to a directory"),
8860
8861     Option("client_mountpoint", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8862     .set_default("/")
8863     .set_description("default mount-point"),
8864
8865     Option("client_mount_uid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8866     .set_default(-1)
8867     .set_description("uid to mount as"),
8868
8869     Option("client_mount_gid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8870     .set_default(-1)
8871     .set_description("gid to mount as"),
8872
8873     /* RADOS client option */
8874     Option("client_notify_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
8875     .set_default(10)
8876     .set_description(""),
8877
8878     /* RADOS client option */
8879     Option("osd_client_watch_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
8880     .set_default(30)
8881     .set_description(""),
8882
8883     Option("client_caps_release_delay", Option::TYPE_INT, Option::LEVEL_DEV)
8884     .set_default(5)
8885     .set_description(""),
8886
8887     Option("client_quota_df", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8888     .set_default(true)
8889     .set_description("show quota usage for statfs (df)"),
8890
8891     Option("client_oc", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8892     .set_default(true)
8893     .set_description("enable object caching"),
8894
8895     Option("client_oc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8896     .set_flag(Option::FLAG_RUNTIME)
8897     .set_default(200_M)
8898     .set_description("maximum size of object cache"),
8899
8900     Option("client_oc_max_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8901     .set_flag(Option::FLAG_RUNTIME)
8902     .set_default(100_M)
8903     .set_description("maximum size of dirty pages in object cache"),
8904
8905     Option("client_oc_target_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8906     .set_flag(Option::FLAG_RUNTIME)
8907     .set_default(8_M)
8908     .set_description("target size of dirty pages object cache"),
8909
8910     Option("client_oc_max_dirty_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8911     .set_flag(Option::FLAG_RUNTIME)
8912     .set_default(5.0)
8913     .set_description("maximum age of dirty pages in object cache (seconds)"),
8914
8915     Option("client_oc_max_objects", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8916     .set_flag(Option::FLAG_RUNTIME)
8917     .set_default(1000)
8918     .set_description("maximum number of objects in cache"),
8919
8920     Option("client_debug_getattr_caps", Option::TYPE_BOOL, Option::LEVEL_DEV)
8921     .set_default(false)
8922     .set_description(""),
8923
8924     Option("client_debug_force_sync_read", Option::TYPE_BOOL, Option::LEVEL_DEV)
8925     .set_default(false)
8926     .set_description(""),
8927
8928     Option("client_debug_inject_tick_delay", Option::TYPE_SECS, Option::LEVEL_DEV)
8929     .set_default(0)
8930     .set_description(""),
8931
8932     Option("client_max_inline_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
8933     .set_default(4_K)
8934     .set_description(""),
8935
8936     Option("client_inject_release_failure", Option::TYPE_BOOL, Option::LEVEL_DEV)
8937     .set_default(false)
8938     .set_description(""),
8939
8940     Option("client_inject_fixed_oldest_tid", Option::TYPE_BOOL, Option::LEVEL_DEV)
8941     .set_default(false)
8942     .set_description(""),
8943
8944     Option("client_metadata", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8945     .set_default("")
8946     .set_description("metadata key=value comma-delimited pairs appended to session metadata"),
8947
8948     Option("client_acl_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8949     .set_default("")
8950     .set_description("ACL type to enforce (none or \"posix_acl\")"),
8951
8952     Option("client_permissions", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8953     .set_default(true)
8954     .set_description("client-enforced permission checking"),
8955
8956     Option("client_dirsize_rbytes", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8957     .set_default(true)
8958     .set_description("set the directory size as the number of file bytes recursively used")
8959     .set_long_description("This option enables a CephFS feature that stores the recursive directory size (the bytes used by files in the directory and its descendents) in the st_size field of the stat structure."),
8960
8961     Option("client_force_lazyio", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8962     .set_default(false)
8963     .set_description(""),
8964
8965     // note: the max amount of "in flight" dirty data is roughly (max - target)
8966     Option("fuse_use_invalidate_cb", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8967     .set_default(true)
8968     .set_description("use fuse 2.8+ invalidate callback to keep page cache consistent"),
8969
8970     Option("fuse_disable_pagecache", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8971     .set_default(false)
8972     .set_description("disable page caching in the kernel for this FUSE mount"),
8973
8974     Option("fuse_allow_other", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8975     .set_default(true)
8976     .set_description("pass allow_other to FUSE on mount"),
8977
8978     Option("fuse_default_permissions", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8979     .set_default(false)
8980     .set_description("pass default_permisions to FUSE on mount")
8981     .set_flag(Option::FLAG_STARTUP),
8982
8983     Option("fuse_splice_read", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8984     .set_default(true)
8985     .set_description("enable splice read to reduce the memory copies"),
8986
8987     Option("fuse_splice_write", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8988     .set_default(true)
8989     .set_description("enable splice write to reduce the memory copies"),
8990
8991     Option("fuse_splice_move", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8992     .set_default(true)
8993     .set_description("enable splice move to reduce the memory copies"),
8994
8995     Option("fuse_big_writes", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8996     .set_default(true)
8997     .set_description("big_writes is deprecated in libfuse 3.0.0"),
8998
8999     Option("fuse_max_write", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
9000     .set_default(0)
9001     .set_description("set the maximum number of bytes in a single write operation")
9002     .set_long_description("Set the maximum number of bytes in a single write operation that may pass atomically through FUSE. The FUSE default is 128kB and may be indicated by setting this option to 0."),
9003
9004     Option("fuse_atomic_o_trunc", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
9005     .set_default(true)
9006     .set_description("pass atomic_o_trunc flag to FUSE on mount"),
9007
9008     Option("fuse_debug", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
9009     .set_default(false)
9010     .set_flag(Option::FLAG_STARTUP)
9011     .set_flag(Option::FLAG_NO_MON_UPDATE)
9012     .set_description("enable debugging for the libfuse"),
9013
9014     Option("fuse_multithreaded", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
9015     .set_default(true)
9016     .set_description("allow parallel processing through FUSE library"),
9017
9018     Option("fuse_require_active_mds", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
9019     .set_default(true)
9020     .set_description("require active MDSs in the file system when mounting"),
9021
9022     Option("fuse_syncfs_on_mksnap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
9023     .set_default(true)
9024     .set_description("synchronize all local metadata/file changes after snapshot"),
9025
9026     Option("fuse_set_user_groups", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
9027     .set_default(true)
9028     .set_description("check for ceph-fuse to consider supplementary groups for permissions"),
9029
9030     Option("client_try_dentry_invalidate", Option::TYPE_BOOL, Option::LEVEL_DEV)
9031     .set_default(false)
9032     .set_description(""),
9033
9034     Option("client_die_on_failed_remount", Option::TYPE_BOOL, Option::LEVEL_DEV)
9035     .set_default(false)
9036     .set_description(""),
9037
9038     Option("client_die_on_failed_dentry_invalidate", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
9039     .set_default(true)
9040     .set_description("kill the client when no dentry invalidation options are available")
9041     .set_long_description("The CephFS client requires a mechanism to invalidate dentries in the caller (e.g. the kernel for ceph-fuse) when capabilities must be recalled. If the client cannot do this then the MDS cache cannot shrink which can cause the MDS to fail."),
9042
9043     Option("client_check_pool_perm", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
9044     .set_default(true)
9045     .set_description("confirm access to inode's data pool/namespace described in file layout"),
9046
9047     Option("client_use_faked_inos", Option::TYPE_BOOL, Option::LEVEL_DEV)
9048     .set_default(false)
9049     .set_description(""),
9050
9051     Option("client_fs", Option::TYPE_STR, Option::LEVEL_ADVANCED)
9052     .set_flag(Option::FLAG_STARTUP)
9053     .set_default("")
9054     .set_description("CephFS file system name to mount")
9055     .set_long_description("Use this with ceph-fuse, or with any process "
9056         "that uses libcephfs.  Programs using libcephfs may also pass "
9057         "the filesystem name into mount(), which will override this setting. "
9058         "If no filesystem name is given in mount() or this setting, the default "
9059         "filesystem will be mounted (usually the first created)."),
9060
9061     /* Alias for client_fs. Deprecated */
9062     Option("client_mds_namespace", Option::TYPE_STR, Option::LEVEL_DEV)
9063     .set_flag(Option::FLAG_STARTUP)
9064     .set_default(""),
9065
9066     Option("fake_statfs_for_testing", Option::TYPE_INT, Option::LEVEL_DEV)
9067     .set_default(0)
9068     .set_description("Set a value for kb and compute kb_used from total of num_bytes"),
9069
9070     Option("debug_allow_any_pool_priority", Option::TYPE_BOOL, Option::LEVEL_DEV)
9071     .set_default(false)
9072     .set_description("Allow any pool priority to be set to test conversion to new range"),
9073
9074     Option("client_asio_thread_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
9075     .set_default(2)
9076     .set_min(1)
9077     .set_description("Size of thread pool for ASIO completions")
9078     .add_tag("client"),
9079
9080     Option("client_shutdown_timeout", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
9081     .set_flag(Option::FLAG_RUNTIME)
9082     .set_default(30)
9083     .set_min(0)
9084     .set_description("timeout for shutting down CephFS")
9085     .set_long_description("Timeout for shutting down CephFS via unmount or shutdown.")
9086     .add_tag("client")
9087     });
9088 }
9089
9090 std::vector<Option> get_cephfs_mirror_options() {
9091   return std::vector<Option>({
9092     Option("cephfs_mirror_max_concurrent_directory_syncs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
9093     .set_default(3)
9094     .set_min(1)
9095     .set_description("maximum number of concurrent snapshot synchronization threads")
9096     .set_long_description("maximum number of directory snapshots that can be synchronized concurrently by cephfs-mirror daemon. Controls the number of synchronization threads."),
9097
9098     Option("cephfs_mirror_action_update_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
9099     .set_default(2)
9100     .set_min(1)
9101     .set_description("interval for driving asynchornous mirror actions")
9102     .set_long_description("Interval in seconds to process pending mirror update actions."),
9103
9104     Option("cephfs_mirror_restart_mirror_on_blocklist_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
9105     .set_default(30)
9106     .set_min(0)
9107     .set_description("interval to restart blocklisted instances")
9108     .set_long_description("Interval in seconds to restart blocklisted mirror instances. Setting to zero (0) disables restarting blocklisted instances."),
9109
9110     Option("cephfs_mirror_max_snapshot_sync_per_cycle", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
9111     .set_default(3)
9112     .set_min(1)
9113     .set_description("number of snapshots to mirror in one cycle")
9114     .set_long_description("maximum number of snapshots to mirror when a directory is picked up for mirroring by worker threads."),
9115
9116     Option("cephfs_mirror_directory_scan_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
9117     .set_default(10)
9118     .set_min(1)
9119     .set_description("interval to scan directories to mirror snapshots")
9120     .set_long_description("interval in seconds to scan configured directories for snapshot mirroring."),
9121
9122     Option("cephfs_mirror_max_consecutive_failures_per_directory", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
9123     .set_default(10)
9124     .set_min(0)
9125     .set_description("consecutive failed directory synchronization attempts before marking a directory as \"failed\"")
9126     .set_long_description("number of consecutive snapshot synchronization failues to mark a directory as \"failed\". failed directories are retried for synchronization less frequently."),
9127
9128     Option("cephfs_mirror_retry_failed_directories_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
9129     .set_default(60)
9130     .set_min(1)
9131     .set_description("failed directory retry interval for synchronization")
9132     .set_long_description("interval in seconds to retry synchronization for failed directories."),
9133
9134     Option("cephfs_mirror_restart_mirror_on_failure_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
9135     .set_default(20)
9136     .set_min(0)
9137     .set_description("interval to restart failed mirror instances")
9138     .set_long_description("Interval in seconds to restart failed mirror instances. Setting to zero (0) disables restarting failed mirror instances."),
9139     });
9140 }
9141
9142 static std::vector<Option> build_options()
9143 {
9144   std::vector<Option> result = get_global_options();
9145
9146   auto ingest = [&result](std::vector<Option>&& options, const char* svc) {
9147     for (auto &o : options) {
9148       o.add_service(svc);
9149       result.push_back(std::move(o));
9150     }
9151   };
9152
9153   ingest(get_rgw_options(), "rgw");
9154   ingest(get_rbd_options(), "rbd");
9155   ingest(get_rbd_mirror_options(), "rbd-mirror");
9156   ingest(get_immutable_object_cache_options(), "immutable-objet-cache");
9157   ingest(get_mds_options(), "mds");
9158   ingest(get_mds_client_options(), "mds_client");
9159   ingest(get_cephfs_mirror_options(), "cephfs-mirror");
9160
9161   return result;
9162 }
9163
9164 const std::vector<Option> ceph_options = build_options();