ceph/src/common/options.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "acconfig.h"
   5 #include "options.h"
   6 #include "common/Formatter.h"
   7
   8 // Helpers for validators
   9 #include "include/stringify.h"
  10 #include <boost/algorithm/string.hpp>
  11 #include <boost/lexical_cast.hpp>
  12 #include <regex>
  13
  14 // Definitions for enums
  15 #include "common/perf_counters.h"
  16
  17 // rbd feature validation
  18 #include "librbd/Features.h"
  19
  20 namespace {
  21 class printer : public boost::static_visitor<> {
  22   ostream& out;
  23 public:
  24   explicit printer(ostream& os)
  25     : out(os) {}
  26   template<typename T>
  27   void operator()(const T& v) const {
  28     out << v;
  29   }
  30   void operator()(boost::blank blank) const {
  31     return;
  32   }
  33   void operator()(bool v) const {
  34     out << (v ? "true" : "false");
  35   }
  36   void operator()(double v) const {
  37     out << std::fixed << v << std::defaultfloat;
  38   }
  39   void operator()(const Option::size_t& v) const {
  40     out << v.value;
  41   }
  42   void operator()(const std::chrono::seconds v) const {
  43     out << v.count();
  44   }
  45 };
  46 }
  47
  48 ostream& operator<<(ostream& os, const Option::value_t& v) {
  49   printer p{os};
  50   v.apply_visitor(p);
  51   return os;
  52 }
  53
  54 void Option::dump_value(const char *field_name,
  55     const Option::value_t &v, Formatter *f) const
  56 {
  57   if (boost::get<boost::blank>(&v)) {
  58     // This should be nil but Formatter doesn't allow it.
  59     f->dump_string(field_name, "");
  60     return;
  61   }
  62   switch (type) {
  63   case TYPE_INT:
  64     f->dump_int(field_name, boost::get<int64_t>(v)); break;
  65   case TYPE_UINT:
  66     f->dump_unsigned(field_name, boost::get<uint64_t>(v)); break;
  67   case TYPE_STR:
  68     f->dump_string(field_name, boost::get<std::string>(v)); break;
  69   case TYPE_FLOAT:
  70     f->dump_float(field_name, boost::get<double>(v)); break;
  71   case TYPE_BOOL:
  72     f->dump_bool(field_name, boost::get<bool>(v)); break;
  73   default:
  74     f->dump_stream(field_name) << v; break;
  75   }
  76 }
  77
  78 int Option::pre_validate(std::string *new_value, std::string *err) const
  79 {
  80   if (validator) {
  81     return validator(new_value, err);
  82   } else {
  83     return 0;
  84   }
  85 }
  86
  87 int Option::validate(const Option::value_t &new_value, std::string *err) const
  88 {
  89   // Generic validation: min
  90   if (!boost::get<boost::blank>(&(min))) {
  91     if (new_value < min) {
  92       std::ostringstream oss;
  93       oss << "Value '" << new_value << "' is below minimum " << min;
  94       *err = oss.str();
  95       return -EINVAL;
  96     }
  97   }
  98
  99   // Generic validation: max
 100   if (!boost::get<boost::blank>(&(max))) {
 101     if (new_value > max) {
 102       std::ostringstream oss;
 103       oss << "Value '" << new_value << "' exceeds maximum " << max;
 104       *err = oss.str();
 105       return -EINVAL;
 106     }
 107   }
 108
 109   // Generic validation: enum
 110   if (!enum_allowed.empty() && type == Option::TYPE_STR) {
 111     auto found = std::find(enum_allowed.begin(), enum_allowed.end(),
 112                            boost::get<std::string>(new_value));
 113     if (found == enum_allowed.end()) {
 114       std::ostringstream oss;
 115       oss << "'" << new_value << "' is not one of the permitted "
 116                  "values: " << joinify(enum_allowed.begin(),
 117                                        enum_allowed.end(),
 118                                        std::string(", "));
 119       *err = oss.str();
 120       return -EINVAL;
 121     }
 122   }
 123
 124   return 0;
 125 }
 126
 127 int Option::parse_value(
 128   const std::string& raw_val,
 129   value_t *out,
 130   std::string *error_message,
 131   std::string *normalized_value) const
 132 {
 133   std::string val = raw_val;
 134
 135   int r = pre_validate(&val, error_message);
 136   if (r != 0) {
 137     return r;
 138   }
 139
 140   if (type == Option::TYPE_INT) {
 141     int64_t f = strict_si_cast<int64_t>(val.c_str(), error_message);
 142     if (!error_message->empty()) {
 143       return -EINVAL;
 144     }
 145     *out = f;
 146   } else if (type == Option::TYPE_UINT) {
 147     uint64_t f = strict_si_cast<uint64_t>(val.c_str(), error_message);
 148     if (!error_message->empty()) {
 149       return -EINVAL;
 150     }
 151     *out = f;
 152   } else if (type == Option::TYPE_STR) {
 153     *out = val;
 154   } else if (type == Option::TYPE_FLOAT) {
 155     double f = strict_strtod(val.c_str(), error_message);
 156     if (!error_message->empty()) {
 157       return -EINVAL;
 158     } else {
 159       *out = f;
 160     }
 161   } else if (type == Option::TYPE_BOOL) {
 162     if (strcasecmp(val.c_str(), "false") == 0) {
 163       *out = false;
 164     } else if (strcasecmp(val.c_str(), "true") == 0) {
 165       *out = true;
 166     } else {
 167       int b = strict_strtol(val.c_str(), 10, error_message);
 168       if (!error_message->empty()) {
 169         return -EINVAL;
 170       }
 171       *out = (bool)!!b;
 172     }
 173   } else if (type == Option::TYPE_ADDR) {
 174     entity_addr_t addr;
 175     if (!addr.parse(val.c_str())){
 176       return -EINVAL;
 177     }
 178     *out = addr;
 179   } else if (type == Option::TYPE_ADDR) {
 180     entity_addrvec_t addr;
 181     if (!addr.parse(val.c_str())){
 182       return -EINVAL;
 183     }
 184     *out = addr;
 185   } else if (type == Option::TYPE_UUID) {
 186     uuid_d uuid;
 187     if (!uuid.parse(val.c_str())) {
 188       return -EINVAL;
 189     }
 190     *out = uuid;
 191   } else if (type == Option::TYPE_SIZE) {
 192     Option::size_t sz{strict_iecstrtoll(val.c_str(), error_message)};
 193     if (!error_message->empty()) {
 194       return -EINVAL;
 195     }
 196     *out = sz;
 197   } else if (type == Option::TYPE_SECS) {
 198     try {
 199       *out = parse_timespan(val);
 200     } catch (const invalid_argument& e) {
 201       *error_message = e.what();
 202       return -EINVAL;
 203     }
 204   } else {
 205     ceph_abort();
 206   }
 207
 208   r = validate(*out, error_message);
 209   if (r != 0) {
 210     return r;
 211   }
 212
 213   if (normalized_value) {
 214     *normalized_value = to_str(*out);
 215   }
 216   return 0;
 217 }
 218
 219 void Option::dump(Formatter *f) const
 220 {
 221   f->dump_string("name", name);
 222
 223   f->dump_string("type", type_to_str(type));
 224
 225   f->dump_string("level", level_to_str(level));
 226
 227   f->dump_string("desc", desc);
 228   f->dump_string("long_desc", long_desc);
 229
 230   dump_value("default", value, f);
 231   dump_value("daemon_default", daemon_value, f);
 232
 233   f->open_array_section("tags");
 234   for (const auto t : tags) {
 235     f->dump_string("tag", t);
 236   }
 237   f->close_section();
 238
 239   f->open_array_section("services");
 240   for (const auto s : services) {
 241     f->dump_string("service", s);
 242   }
 243   f->close_section();
 244
 245   f->open_array_section("see_also");
 246   for (const auto sa : see_also) {
 247     f->dump_string("see_also", sa);
 248   }
 249   f->close_section();
 250
 251   if (type == TYPE_STR) {
 252     f->open_array_section("enum_values");
 253     for (const auto &ea : enum_allowed) {
 254       f->dump_string("enum_value", ea);
 255     }
 256     f->close_section();
 257   }
 258
 259   dump_value("min", min, f);
 260   dump_value("max", max, f);
 261
 262   f->dump_bool("can_update_at_runtime", can_update_at_runtime());
 263
 264   f->open_array_section("flags");
 265   if (has_flag(FLAG_RUNTIME)) {
 266     f->dump_string("option", "runtime");
 267   }
 268   if (has_flag(FLAG_NO_MON_UPDATE)) {
 269     f->dump_string("option", "no_mon_update");
 270   }
 271   if (has_flag(FLAG_STARTUP)) {
 272     f->dump_string("option", "startup");
 273   }
 274   if (has_flag(FLAG_CLUSTER_CREATE)) {
 275     f->dump_string("option", "cluster_create");
 276   }
 277   if (has_flag(FLAG_CREATE)) {
 278     f->dump_string("option", "create");
 279   }
 280   f->close_section();
 281 }
 282
 283 std::string Option::to_str(const Option::value_t& v)
 284 {
 285   return stringify(v);
 286 }
 287
 288 void Option::print(ostream *out) const
 289 {
 290   *out << name << " - " << desc << "\n";
 291   *out << "  (" << type_to_str(type) << ", " << level_to_str(level) << ")\n";
 292   if (!boost::get<boost::blank>(&daemon_value)) {
 293     *out << "  Default (non-daemon): " << stringify(value) << "\n";
 294     *out << "  Default (daemon): " << stringify(daemon_value) << "\n";
 295   } else {
 296     *out << "  Default: " << stringify(value) << "\n";
 297   }
 298   if (!enum_allowed.empty()) {
 299     *out << "  Possible values: ";
 300     for (auto& i : enum_allowed) {
 301       *out << " " << stringify(i);
 302     }
 303     *out << "\n";
 304   }
 305   if (!boost::get<boost::blank>(&min)) {
 306     *out << "  Minimum: " << stringify(min) << "\n"
 307          << "  Maximum: " << stringify(max) << "\n";
 308   }
 309   *out << "  Can update at runtime: "
 310        << (can_update_at_runtime() ? "true" : "false") << "\n";
 311   if (!services.empty()) {
 312     *out << "  Services: " << services << "\n";
 313   }
 314   if (!tags.empty()) {
 315     *out << "  Tags: " << tags << "\n";
 316   }
 317   if (!see_also.empty()) {
 318     *out << "  See also: " << see_also << "\n";
 319   }
 320
 321   if (long_desc.size()) {
 322     *out << "\n" << long_desc << "\n";
 323   }
 324 }
 325
 326 constexpr unsigned long long operator"" _min (unsigned long long min) {
 327   return min * 60;
 328 }
 329 constexpr unsigned long long operator"" _hr (unsigned long long hr) {
 330   return hr * 60 * 60;
 331 }
 332 constexpr unsigned long long operator"" _day (unsigned long long day) {
 333   return day * 60 * 60 * 24;
 334 }
 335 constexpr unsigned long long operator"" _K (unsigned long long n) {
 336   return n << 10;
 337 }
 338 constexpr unsigned long long operator"" _M (unsigned long long n) {
 339   return n << 20;
 340 }
 341 constexpr unsigned long long operator"" _G (unsigned long long n) {
 342   return n << 30;
 343 }
 344
 345 std::vector<Option> get_global_options() {
 346   return std::vector<Option>({
 347     Option("host", Option::TYPE_STR, Option::LEVEL_BASIC)
 348     .set_description("local hostname")
 349     .set_long_description("if blank, ceph assumes the short hostname (hostname -s)")
 350     .set_flag(Option::FLAG_NO_MON_UPDATE)
 351     .add_service("common")
 352     .add_tag("network"),
 353
 354     Option("fsid", Option::TYPE_UUID, Option::LEVEL_BASIC)
 355     .set_description("cluster fsid (uuid)")
 356     .set_flag(Option::FLAG_NO_MON_UPDATE)
 357     .set_flag(Option::FLAG_STARTUP)
 358     .add_service("common")
 359     .add_tag("service"),
 360
 361     Option("public_addr", Option::TYPE_ADDR, Option::LEVEL_BASIC)
 362     .set_description("public-facing address to bind to")
 363     .set_flag(Option::FLAG_STARTUP)
 364     .add_service({"mon", "mds", "osd", "mgr"}),
 365
 366     Option("public_bind_addr", Option::TYPE_ADDR, Option::LEVEL_ADVANCED)
 367     .set_default(entity_addr_t())
 368     .set_flag(Option::FLAG_STARTUP)
 369     .add_service("mon")
 370     .set_description(""),
 371
 372     Option("cluster_addr", Option::TYPE_ADDR, Option::LEVEL_BASIC)
 373     .set_description("cluster-facing address to bind to")
 374     .add_service("osd")
 375     .set_flag(Option::FLAG_STARTUP)
 376     .add_tag("network"),
 377
 378     Option("public_network", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 379     .add_service({"mon", "mds", "osd", "mgr"})
 380     .set_flag(Option::FLAG_STARTUP)
 381     .add_tag("network")
 382     .set_description("Network(s) from which to choose a public address to bind to"),
 383
 384     Option("public_network_interface", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 385     .add_service({"mon", "mds", "osd", "mgr"})
 386     .add_tag("network")
 387     .set_flag(Option::FLAG_STARTUP)
 388     .set_description("Interface name(s) from which to choose an address from a public_network to bind to; public_network must also be specified.")
 389     .add_see_also("public_network"),
 390
 391     Option("cluster_network", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 392     .add_service("osd")
 393     .set_flag(Option::FLAG_STARTUP)
 394     .add_tag("network")
 395     .set_description("Network(s) from which to choose a cluster address to bind to"),
 396
 397     Option("cluster_network_interface", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 398     .add_service({"mon", "mds", "osd", "mgr"})
 399     .set_flag(Option::FLAG_STARTUP)
 400     .add_tag("network")
 401     .set_description("Interface name(s) from which to choose an address from a cluster_network to bind to; cluster_network must also be specified.")
 402     .add_see_also("cluster_network"),
 403
 404     Option("monmap", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 405     .set_description("path to MonMap file")
 406     .set_long_description("This option is normally used during mkfs, but can also "
 407                         "be used to identify which monitors to connect to.")
 408     .set_flag(Option::FLAG_NO_MON_UPDATE)
 409     .add_service("mon")
 410     .set_flag(Option::FLAG_CREATE),
 411
 412     Option("mon_host", Option::TYPE_STR, Option::LEVEL_BASIC)
 413     .set_description("list of hosts or addresses to search for a monitor")
 414     .set_long_description("This is a comma, whitespace, or semicolon separated "
 415                         "list of IP addresses or hostnames. Hostnames are "
 416                         "resolved via DNS and all A or AAAA records are "
 417                         "included in the search list.")
 418     .set_flag(Option::FLAG_NO_MON_UPDATE)
 419     .set_flag(Option::FLAG_STARTUP)
 420     .add_service("common"),
 421
 422     Option("mon_dns_srv_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 423     .set_default("ceph-mon")
 424     .set_description("name of DNS SRV record to check for monitor addresses")
 425     .set_flag(Option::FLAG_NO_MON_UPDATE)
 426     .set_flag(Option::FLAG_STARTUP)
 427     .add_service("common")
 428     .add_tag("network")
 429     .add_see_also("mon_host"),
 430
 431     // lockdep
 432     Option("lockdep", Option::TYPE_BOOL, Option::LEVEL_DEV)
 433     .set_description("enable lockdep lock dependency analyzer")
 434     .set_flag(Option::FLAG_NO_MON_UPDATE)
 435     .set_flag(Option::FLAG_STARTUP)
 436     .add_service("common"),
 437
 438     Option("lockdep_force_backtrace", Option::TYPE_BOOL, Option::LEVEL_DEV)
 439     .set_description("always gather current backtrace at every lock")
 440     .set_flag(Option::FLAG_STARTUP)
 441     .add_service("common")
 442     .add_see_also("lockdep"),
 443
 444     Option("run_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 445     .set_default("/var/run/ceph")
 446     .set_flag(Option::FLAG_STARTUP)
 447     .set_description("path for the 'run' directory for storing pid and socket files")
 448     .add_service("common")
 449     .add_see_also("admin_socket"),
 450
 451     Option("admin_socket", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 452     .set_default("")
 453     .set_daemon_default("$run_dir/$cluster-$name.asok")
 454     .set_flag(Option::FLAG_STARTUP)
 455     .set_description("path for the runtime control socket file, used by the 'ceph daemon' command")
 456     .add_service("common"),
 457
 458     Option("admin_socket_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 459     .set_description("file mode to set for the admin socket file, e.g, '0755'")
 460     .set_flag(Option::FLAG_STARTUP)
 461     .add_service("common")
 462     .add_see_also("admin_socket"),
 463
 464     // daemon
 465     Option("daemonize", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 466     .set_default(false)
 467     .set_daemon_default(true)
 468     .set_description("whether to daemonize (background) after startup")
 469     .set_flag(Option::FLAG_STARTUP)
 470     .set_flag(Option::FLAG_NO_MON_UPDATE)
 471     .add_service({"mon", "mgr", "osd", "mds"})
 472     .add_tag("service")
 473     .add_see_also({"pid_file", "chdir"}),
 474
 475     Option("setuser", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 476     .set_flag(Option::FLAG_STARTUP)
 477     .set_description("uid or user name to switch to on startup")
 478     .set_long_description("This is normally specified by the systemd unit file.")
 479     .add_service({"mon", "mgr", "osd", "mds"})
 480     .add_tag("service")
 481     .add_see_also("setgroup"),
 482
 483     Option("setgroup", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 484     .set_flag(Option::FLAG_STARTUP)
 485     .set_description("gid or group name to switch to on startup")
 486     .set_long_description("This is normally specified by the systemd unit file.")
 487     .add_service({"mon", "mgr", "osd", "mds"})
 488     .add_tag("service")
 489     .add_see_also("setuser"),
 490
 491     Option("setuser_match_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 492     .set_flag(Option::FLAG_STARTUP)
 493     .set_description("if set, setuser/setgroup is condition on this path matching ownership")
 494     .set_long_description("If setuser or setgroup are specified, and this option is non-empty, then the uid/gid of the daemon will only be changed if the file or directory specified by this option has a matching uid and/or gid.  This exists primarily to allow switching to user ceph for OSDs to be conditional on whether the osd data contents have also been chowned after an upgrade.  This is normally specified by the systemd unit file.")
 495     .add_service({"mon", "mgr", "osd", "mds"})
 496     .add_tag("service")
 497     .add_see_also({"setuser", "setgroup"}),
 498
 499     Option("pid_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 500     .set_flag(Option::FLAG_STARTUP)
 501     .set_description("path to write a pid file (if any)")
 502     .add_service({"mon", "mgr", "osd", "mds"})
 503     .add_tag("service"),
 504
 505     Option("chdir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 506     .set_description("path to chdir(2) to after daemonizing")
 507     .set_flag(Option::FLAG_STARTUP)
 508     .set_flag(Option::FLAG_NO_MON_UPDATE)
 509     .add_service({"mon", "mgr", "osd", "mds"})
 510     .add_tag("service")
 511     .add_see_also("daemonize"),
 512
 513     Option("fatal_signal_handlers", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 514     .set_default(true)
 515     .set_flag(Option::FLAG_STARTUP)
 516     .set_description("whether to register signal handlers for SIGABRT etc that dump a stack trace")
 517     .set_long_description("This is normally true for daemons and values for libraries.")
 518     .add_service({"mon", "mgr", "osd", "mds"})
 519     .add_tag("service"),
 520
 521     Option("crash_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 522     .set_flag(Option::FLAG_STARTUP)
 523     .set_default("/var/lib/ceph/crash")
 524     .set_description("Directory where crash reports are archived"),
 525
 526     // restapi
 527     Option("restapi_log_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 528     .set_description("default set by python code"),
 529
 530     Option("restapi_base_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 531     .set_description("default set by python code"),
 532
 533     Option("erasure_code_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 534     .set_default(CEPH_PKGLIBDIR"/erasure-code")
 535     .set_flag(Option::FLAG_STARTUP)
 536     .set_description("directory where erasure-code plugins can be found")
 537     .add_service({"mon", "osd"}),
 538
 539     // logging
 540     Option("log_file", Option::TYPE_STR, Option::LEVEL_BASIC)
 541     .set_default("")
 542     .set_daemon_default("/var/log/ceph/$cluster-$name.log")
 543     .set_description("path to log file")
 544     .add_see_also({"log_to_file",
 545                    "log_to_stderr",
 546                    "err_to_stderr",
 547                    "log_to_syslog",
 548                    "err_to_syslog"}),
 549
 550     Option("log_max_new", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 551     .set_default(1000)
 552     .set_description("max unwritten log entries to allow before waiting to flush to the log")
 553     .add_see_also("log_max_recent"),
 554
 555     Option("log_max_recent", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 556     .set_default(500)
 557     .set_daemon_default(10000)
 558     .set_description("recent log entries to keep in memory to dump in the event of a crash")
 559     .set_long_description("The purpose of this option is to log at a higher debug level only to the in-memory buffer, and write out the detailed log messages only if there is a crash.  Only log entries below the lower log level will be written unconditionally to the log.  For example, debug_osd=1/5 will write everything <= 1 to the log unconditionally but keep entries at levels 2-5 in memory.  If there is a seg fault or assertion failure, all entries will be dumped to the log."),
 560
 561     Option("log_to_file", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 562     .set_default(true)
 563     .set_description("send log lines to a file")
 564     .add_see_also("log_file"),
 565
 566     Option("log_to_stderr", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 567     .set_default(true)
 568     .set_daemon_default(false)
 569     .set_description("send log lines to stderr"),
 570
 571     Option("err_to_stderr", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 572     .set_default(false)
 573     .set_daemon_default(true)
 574     .set_description("send critical error log lines to stderr"),
 575
 576     Option("log_stderr_prefix", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 577     .set_description("String to prefix log messages with when sent to stderr")
 578     .set_long_description("This is useful in container environments when combined with mon_cluster_log_to_stderr.  The mon log prefixes each line with the channel name (e.g., 'default', 'audit'), while log_stderr_prefix can be set to 'debug '.")
 579     .add_see_also("mon_cluster_log_to_stderr"),
 580
 581     Option("log_to_syslog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 582     .set_default(false)
 583     .set_description("send log lines to syslog facility"),
 584
 585     Option("err_to_syslog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 586     .set_default(false)
 587     .set_description("send critical error log lines to syslog facility"),
 588
 589     Option("log_flush_on_exit", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 590     .set_default(false)
 591     .set_description("set a process exit handler to ensure the log is flushed on exit"),
 592
 593     Option("log_stop_at_utilization", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
 594     .set_default(.97)
 595     .set_min_max(0.0, 1.0)
 596     .set_description("stop writing to the log file when device utilization reaches this ratio")
 597     .add_see_also("log_file"),
 598
 599     Option("log_to_graylog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 600     .set_default(false)
 601     .set_description("send log lines to remote graylog server")
 602     .add_see_also({"err_to_graylog",
 603                    "log_graylog_host",
 604                    "log_graylog_port"}),
 605
 606     Option("err_to_graylog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 607     .set_default(false)
 608     .set_description("send critical error log lines to remote graylog server")
 609     .add_see_also({"log_to_graylog",
 610                    "log_graylog_host",
 611                    "log_graylog_port"}),
 612
 613     Option("log_graylog_host", Option::TYPE_STR, Option::LEVEL_BASIC)
 614     .set_default("127.0.0.1")
 615     .set_description("address or hostname of graylog server to log to")
 616     .add_see_also({"log_to_graylog",
 617                    "err_to_graylog",
 618                    "log_graylog_port"}),
 619
 620     Option("log_graylog_port", Option::TYPE_INT, Option::LEVEL_BASIC)
 621     .set_default(12201)
 622     .set_description("port number for the remote graylog server")
 623     .add_see_also("log_graylog_host"),
 624
 625     Option("log_coarse_timestamps", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 626     .set_default(true)
 627     .set_description("timestamp log entries from coarse system clock "
 628                      "to improve performance")
 629     .add_service("common")
 630     .add_tag("performance")
 631     .add_tag("service"),
 632
 633
 634     // unmodified
 635     Option("clog_to_monitors", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 636     .set_default("default=true")
 637     .set_flag(Option::FLAG_RUNTIME)
 638     .set_description("Make daemons send cluster log messages to monitors"),
 639
 640     Option("clog_to_syslog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 641     .set_default("false")
 642     .set_flag(Option::FLAG_RUNTIME)
 643     .set_description("Make daemons send cluster log messages to syslog"),
 644
 645     Option("clog_to_syslog_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 646     .set_default("info")
 647     .set_flag(Option::FLAG_RUNTIME)
 648     .set_description("Syslog level for cluster log messages")
 649     .add_see_also("clog_to_syslog"),
 650
 651     Option("clog_to_syslog_facility", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 652     .set_default("default=daemon audit=local0")
 653     .set_flag(Option::FLAG_RUNTIME)
 654     .set_description("Syslog facility for cluster log messages")
 655     .add_see_also("clog_to_syslog"),
 656
 657     Option("clog_to_graylog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 658     .set_default("false")
 659     .set_flag(Option::FLAG_RUNTIME)
 660     .set_description("Make daemons send cluster log to graylog"),
 661
 662     Option("clog_to_graylog_host", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 663     .set_default("127.0.0.1")
 664     .set_flag(Option::FLAG_RUNTIME)
 665     .set_description("Graylog host to cluster log messages")
 666     .add_see_also("clog_to_graylog"),
 667
 668     Option("clog_to_graylog_port", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 669     .set_default("12201")
 670     .set_flag(Option::FLAG_RUNTIME)
 671     .set_description("Graylog port number for cluster log messages")
 672     .add_see_also("clog_to_graylog"),
 673
 674     Option("mon_cluster_log_to_stderr", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 675     .set_default(false)
 676     .add_service("mon")
 677     .set_flag(Option::FLAG_RUNTIME)
 678     .set_description("Make monitor send cluster log messages to stderr (prefixed by channel)")
 679     .add_see_also("log_stderr_prefix"),
 680
 681     Option("mon_cluster_log_to_syslog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 682     .set_default("default=false")
 683     .set_flag(Option::FLAG_RUNTIME)
 684     .add_service("mon")
 685     .set_description("Make monitor send cluster log messages to syslog"),
 686
 687     Option("mon_cluster_log_to_syslog_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 688     .set_default("info")
 689     .add_service("mon")
 690     .set_flag(Option::FLAG_RUNTIME)
 691     .set_description("Syslog level for cluster log messages")
 692     .add_see_also("mon_cluster_log_to_syslog"),
 693
 694     Option("mon_cluster_log_to_syslog_facility", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 695     .set_default("daemon")
 696     .add_service("mon")
 697     .set_flag(Option::FLAG_RUNTIME)
 698     .set_description("Syslog facility for cluster log messages")
 699     .add_see_also("mon_cluster_log_to_syslog"),
 700
 701     Option("mon_cluster_log_to_file", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 702     .set_default(true)
 703     .set_flag(Option::FLAG_RUNTIME)
 704     .add_service("mon")
 705     .set_description("Make monitor send cluster log messages to file")
 706     .add_see_also("mon_cluster_log_file"),
 707
 708     Option("mon_cluster_log_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 709     .set_default("default=/var/log/ceph/$cluster.$channel.log cluster=/var/log/ceph/$cluster.log")
 710     .set_flag(Option::FLAG_RUNTIME)
 711     .add_service("mon")
 712     .set_description("File(s) to write cluster log to")
 713     .set_long_description("This can either be a simple file name to receive all messages, or a list of key/value pairs where the key is the log channel and the value is the filename, which may include $cluster and $channel metavariables")
 714     .add_see_also("mon_cluster_log_to_file"),
 715
 716     Option("mon_cluster_log_file_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 717     .set_default("debug")
 718     .set_flag(Option::FLAG_RUNTIME)
 719     .add_service("mon")
 720     .set_description("Lowest level to include is cluster log file")
 721     .add_see_also("mon_cluster_log_file"),
 722
 723     Option("mon_cluster_log_to_graylog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 724     .set_default("false")
 725     .set_flag(Option::FLAG_RUNTIME)
 726     .add_service("mon")
 727     .set_description("Make monitor send cluster log to graylog"),
 728
 729     Option("mon_cluster_log_to_graylog_host", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 730     .set_default("127.0.0.1")
 731     .set_flag(Option::FLAG_RUNTIME)
 732     .add_service("mon")
 733     .set_description("Graylog host for cluster log messages")
 734     .add_see_also("mon_cluster_log_to_graylog"),
 735
 736     Option("mon_cluster_log_to_graylog_port", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 737     .set_default("12201")
 738     .set_flag(Option::FLAG_RUNTIME)
 739     .add_service("mon")
 740     .set_description("Graylog port for cluster log messages")
 741     .add_see_also("mon_cluster_log_to_graylog"),
 742
 743     Option("enable_experimental_unrecoverable_data_corrupting_features", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 744     .set_flag(Option::FLAG_RUNTIME)
 745     .set_default("")
 746     .set_description("Enable named (or all with '*') experimental features that may be untested, dangerous, and/or cause permanent data loss"),
 747
 748     Option("plugin_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 749     .set_default(CEPH_PKGLIBDIR)
 750     .set_flag(Option::FLAG_STARTUP)
 751     .add_service({"mon", "osd"})
 752     .set_description("Base directory for dynamically loaded plugins"),
 753
 754     // XIO
 755     Option("xio_trace_mempool", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 756     .set_default(false)
 757     .set_description(""),
 758
 759     Option("xio_trace_msgcnt", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 760     .set_default(false)
 761     .set_description(""),
 762
 763     Option("xio_trace_xcon", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 764     .set_default(false)
 765     .set_description(""),
 766
 767     Option("xio_queue_depth", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 768     .set_default(128)
 769     .set_description(""),
 770
 771     Option("xio_mp_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 772     .set_default(128)
 773     .set_description(""),
 774
 775     Option("xio_mp_max_64", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 776     .set_default(65536)
 777     .set_description(""),
 778
 779     Option("xio_mp_max_256", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 780     .set_default(8192)
 781     .set_description(""),
 782
 783     Option("xio_mp_max_1k", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 784     .set_default(8192)
 785     .set_description(""),
 786
 787     Option("xio_mp_max_page", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 788     .set_default(4096)
 789     .set_description(""),
 790
 791     Option("xio_mp_max_hint", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 792     .set_default(4096)
 793     .set_description(""),
 794
 795     Option("xio_portal_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 796     .set_default(2)
 797     .set_description(""),
 798
 799     Option("xio_max_conns_per_portal", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 800     .set_default(32)
 801     .set_description(""),
 802
 803     Option("xio_transport_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 804     .set_default("rdma")
 805     .set_description(""),
 806
 807     Option("xio_max_send_inline", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 808     .set_default(512)
 809     .set_description(""),
 810
 811     // Compressor
 812     Option("compressor_zlib_isal", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 813     .set_default(false)
 814     .set_description("Use Intel ISA-L accelerated zlib implementation if available"),
 815
 816     Option("compressor_zlib_level", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 817     .set_default(5)
 818     .set_description("Zlib compression level to use"),
 819
 820     Option("qat_compressor_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 821     .set_default(false)
 822     .set_description("Enable Intel QAT acceleration support for compression if available"),
 823
 824     Option("plugin_crypto_accelerator", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 825     .set_default("crypto_isal")
 826     .set_description("Crypto accelerator library to use"),
 827
 828     Option("mempool_debug", Option::TYPE_BOOL, Option::LEVEL_DEV)
 829     .set_default(false)
 830     .set_flag(Option::FLAG_NO_MON_UPDATE)
 831     .set_description(""),
 832
 833     Option("key", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 834     .set_default("")
 835     .set_description("Authentication key")
 836     .set_long_description("A CephX authentication key, base64 encoded.  It normally looks something like 'AQAtut9ZdMbNJBAAHz6yBAWyJyz2yYRyeMWDag=='.")
 837     .set_flag(Option::FLAG_STARTUP)
 838     .set_flag(Option::FLAG_NO_MON_UPDATE)
 839     .add_see_also("keyfile")
 840     .add_see_also("keyring"),
 841
 842     Option("keyfile", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 843     .set_default("")
 844     .set_description("Path to a file containing a key")
 845     .set_long_description("The file should contain a CephX authentication key and optionally a trailing newline, but nothing else.")
 846     .set_flag(Option::FLAG_STARTUP)
 847     .set_flag(Option::FLAG_NO_MON_UPDATE)
 848     .add_see_also("key"),
 849
 850     Option("keyring", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 851     .set_default(
 852       "/etc/ceph/$cluster.$name.keyring,/etc/ceph/$cluster.keyring,"
 853       "/etc/ceph/keyring,/etc/ceph/keyring.bin,"
 854   #if defined(__FreeBSD)
 855       "/usr/local/etc/ceph/$cluster.$name.keyring,"
 856       "/usr/local/etc/ceph/$cluster.keyring,"
 857       "/usr/local/etc/ceph/keyring,/usr/local/etc/ceph/keyring.bin,"
 858   #endif
 859     )
 860     .set_description("Path to a keyring file.")
 861     .set_long_description("A keyring file is an INI-style formatted file where the section names are client or daemon names (e.g., 'osd.0') and each section contains a 'key' property with CephX authentication key as the value.")
 862     .set_flag(Option::FLAG_STARTUP)
 863     .set_flag(Option::FLAG_NO_MON_UPDATE)
 864     .add_see_also("key")
 865     .add_see_also("keyfile"),
 866
 867     Option("heartbeat_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 868     .set_default(5)
 869     .set_flag(Option::FLAG_STARTUP)
 870     .set_description("Frequency of internal heartbeat checks (seconds)"),
 871
 872     Option("heartbeat_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 873     .set_default("")
 874     .set_flag(Option::FLAG_STARTUP)
 875     .set_description("File to touch on successful internal heartbeat")
 876     .set_long_description("If set, this file will be touched every time an internal heartbeat check succeeds.")
 877     .add_see_also("heartbeat_interval"),
 878
 879     Option("heartbeat_inject_failure", Option::TYPE_INT, Option::LEVEL_DEV)
 880     .set_default(0)
 881     .set_description(""),
 882
 883     Option("perf", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 884     .set_default(true)
 885     .set_description("Enable internal performance metrics")
 886     .set_long_description("If enabled, collect and expose internal health metrics"),
 887
 888     Option("ms_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 889     .set_flag(Option::FLAG_STARTUP)
 890     .set_default("async+posix")
 891     .set_description("Messenger implementation to use for network communication"),
 892
 893     Option("ms_public_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 894     .set_default("")
 895     .set_flag(Option::FLAG_STARTUP)
 896     .set_description("Messenger implementation to use for the public network")
 897     .set_long_description("If not specified, use ms_type")
 898     .add_see_also("ms_type"),
 899
 900     Option("ms_cluster_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 901     .set_default("")
 902     .set_flag(Option::FLAG_STARTUP)
 903     .set_description("Messenger implementation to use for the internal cluster network")
 904     .set_long_description("If not specified, use ms_type")
 905     .add_see_also("ms_type"),
 906
 907     Option("ms_mon_cluster_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 908     .set_default("secure crc")
 909     .set_flag(Option::FLAG_STARTUP)
 910     .set_description("Connection modes (crc, secure) for intra-mon connections in order of preference")
 911     .add_see_also("ms_mon_service_mode")
 912     .add_see_also("ms_mon_client_mode")
 913     .add_see_also("ms_service_mode")
 914     .add_see_also("ms_cluster_mode")
 915     .add_see_also("ms_client_mode"),
 916
 917     Option("ms_mon_service_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 918     .set_default("secure crc")
 919     .set_flag(Option::FLAG_STARTUP)
 920     .set_description("Allowed connection modes (crc, secure) for connections to mons")
 921     .add_see_also("ms_service_mode")
 922     .add_see_also("ms_mon_cluster_mode")
 923     .add_see_also("ms_mon_client_mode")
 924     .add_see_also("ms_cluster_mode")
 925     .add_see_also("ms_client_mode"),
 926
 927     Option("ms_mon_client_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 928     .set_default("secure crc")
 929     .set_flag(Option::FLAG_STARTUP)
 930     .set_description("Connection modes (crc, secure) for connections from clients to monitors in order of preference")
 931     .add_see_also("ms_mon_service_mode")
 932     .add_see_also("ms_mon_cluster_mode")
 933     .add_see_also("ms_service_mode")
 934     .add_see_also("ms_cluster_mode")
 935     .add_see_also("ms_client_mode"),
 936
 937     Option("ms_cluster_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 938     .set_default("crc secure")
 939     .set_flag(Option::FLAG_STARTUP)
 940     .set_description("Connection modes (crc, secure) for intra-cluster connections in order of preference")
 941     .add_see_also("ms_service_mode")
 942     .add_see_also("ms_client_mode"),
 943
 944     Option("ms_service_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 945     .set_default("crc secure")
 946     .set_flag(Option::FLAG_STARTUP)
 947     .set_description("Allowed connection modes (crc, secure) for connections to daemons")
 948     .add_see_also("ms_cluster_mode")
 949     .add_see_also("ms_client_mode"),
 950
 951     Option("ms_client_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 952     .set_default("crc secure")
 953     .set_flag(Option::FLAG_STARTUP)
 954     .set_description("Connection modes (crc, secure) for connections from clients in order of preference")
 955     .add_see_also("ms_cluster_mode")
 956     .add_see_also("ms_service_mode"),
 957
 958     Option("ms_learn_addr_from_peer", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 959     .set_default(true)
 960     .set_description("Learn address from what IP our first peer thinks we connect from")
 961     .set_long_description("Use the IP address our first peer (usually a monitor) sees that we are connecting from.  This is useful if a client is behind some sort of NAT and we want to see it identified by its local (not NATed) address."),
 962
 963     Option("ms_tcp_nodelay", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 964     .set_default(true)
 965     .set_description("Disable Nagle's algorithm and send queued network traffic immediately"),
 966
 967     Option("ms_tcp_rcvbuf", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 968     .set_default(0)
 969     .set_description("Size of TCP socket receive buffer"),
 970
 971     Option("ms_tcp_prefetch_max_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 972     .set_default(4_K)
 973     .set_description("Maximum amount of data to prefetch out of the socket receive buffer"),
 974
 975     Option("ms_initial_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
 976     .set_default(.2)
 977     .set_description("Initial backoff after a network error is detected (seconds)"),
 978
 979     Option("ms_max_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
 980     .set_default(15.0)
 981     .set_description("Maximum backoff after a network error before retrying (seconds)")
 982     .add_see_also("ms_initial_backoff"),
 983
 984     Option("ms_crc_data", Option::TYPE_BOOL, Option::LEVEL_DEV)
 985     .set_default(true)
 986     .set_description("Set and/or verify crc32c checksum on data payload sent over network"),
 987
 988     Option("ms_crc_header", Option::TYPE_BOOL, Option::LEVEL_DEV)
 989     .set_default(true)
 990     .set_description("Set and/or verify crc32c checksum on header payload sent over network"),
 991
 992     Option("ms_die_on_bad_msg", Option::TYPE_BOOL, Option::LEVEL_DEV)
 993     .set_default(false)
 994     .set_description("Induce a daemon crash/exit when a bad network message is received"),
 995
 996     Option("ms_die_on_unhandled_msg", Option::TYPE_BOOL, Option::LEVEL_DEV)
 997     .set_default(false)
 998     .set_description("Induce a daemon crash/exit when an unrecognized message is received"),
 999
1000     Option("ms_die_on_old_message", Option::TYPE_BOOL, Option::LEVEL_DEV)
1001     .set_default(false)
1002     .set_description("Induce a daemon crash/exit when a old, undecodable message is received"),
1003
1004     Option("ms_die_on_skipped_message", Option::TYPE_BOOL, Option::LEVEL_DEV)
1005     .set_default(false)
1006     .set_description("Induce a daemon crash/exit if sender skips a message sequence number"),
1007
1008     Option("ms_die_on_bug", Option::TYPE_BOOL, Option::LEVEL_DEV)
1009     .set_default(false)
1010     .set_description("Induce a crash/exit on various bugs (for testing purposes)"),
1011
1012     Option("ms_dispatch_throttle_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1013     .set_default(100_M)
1014     .set_description("Limit messages that are read off the network but still being processed"),
1015
1016     Option("ms_msgr2_sign_messages", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1017     .set_default(false)
1018     .set_description("Sign msgr2 frames' payload")
1019     .add_see_also("ms_msgr2_encrypt_messages"),
1020
1021     Option("ms_msgr2_encrypt_messages", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1022     .set_default(false)
1023     .set_description("Encrypt msgr2 frames' payload")
1024     .add_see_also("ms_msgr2_sign_messages"),
1025
1026     Option("ms_bind_ipv4", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1027     .set_default(true)
1028     .set_description("Bind servers to IPv4 address(es)")
1029     .add_see_also("ms_bind_ipv6"),
1030
1031     Option("ms_bind_ipv6", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1032     .set_default(false)
1033     .set_description("Bind servers to IPv6 address(es)")
1034     .add_see_also("ms_bind_ipv4"),
1035
1036     Option("ms_bind_prefer_ipv4", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1037     .set_default(false)
1038     .set_description("Prefer IPV4 over IPV6 address(es)"),
1039
1040     Option("ms_bind_msgr1", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1041     .set_default(true)
1042     .set_description("Bind servers to msgr1 (legacy) protocol address(es)")
1043     .add_see_also("ms_bind_msgr2"),
1044
1045     Option("ms_bind_msgr2", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1046     .set_default(true)
1047     .set_description("Bind servers to msgr2 (nautilus+) protocol address(es)")
1048     .add_see_also("ms_bind_msgr1"),
1049
1050     Option("ms_bind_port_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1051     .set_default(6800)
1052     .set_description("Lowest port number to bind daemon(s) to"),
1053
1054     Option("ms_bind_port_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1055     .set_default(7300)
1056     .set_description("Highest port number to bind daemon(s) to"),
1057
1058     Option("ms_bind_retry_count", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1059   #if !defined(__FreeBSD__)
1060     .set_default(3)
1061   #else
1062     // FreeBSD does not use SO_REAUSEADDR so allow for a bit more time per default
1063     .set_default(6)
1064   #endif
1065     .set_description("Number of attempts to make while bind(2)ing to a port"),
1066
1067     Option("ms_bind_retry_delay", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1068   #if !defined(__FreeBSD__)
1069     .set_default(5)
1070   #else
1071     // FreeBSD does not use SO_REAUSEADDR so allow for a bit more time per default
1072     .set_default(6)
1073   #endif
1074     .set_description("Delay between bind(2) attempts (seconds)"),
1075
1076     Option("ms_bind_before_connect", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1077     .set_default(false)
1078     .set_description("Call bind(2) on client sockets"),
1079
1080     Option("ms_tcp_listen_backlog", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1081     .set_default(512)
1082     .set_description("Size of queue of incoming connections for accept(2)"),
1083
1084     Option("ms_rwthread_stack_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1085     .set_default(1_M)
1086     .set_description("Size of stack for SimpleMessenger read/write threads"),
1087
1088     Option("ms_connection_ready_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1089     .set_default(10)
1090     .set_description("Time before we declare a not yet ready connection as dead (seconds)"),
1091
1092     Option("ms_connection_idle_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1093     .set_default(900)
1094     .set_description("Time before an idle connection is closed (seconds)"),
1095
1096     Option("ms_pq_max_tokens_per_priority", Option::TYPE_UINT, Option::LEVEL_DEV)
1097     .set_default(16777216)
1098     .set_description(""),
1099
1100     Option("ms_pq_min_cost", Option::TYPE_SIZE, Option::LEVEL_DEV)
1101     .set_default(65536)
1102     .set_description(""),
1103
1104     Option("ms_inject_socket_failures", Option::TYPE_UINT, Option::LEVEL_DEV)
1105     .set_default(0)
1106     .set_description("Inject a socket failure every Nth socket operation"),
1107
1108     Option("ms_inject_delay_type", Option::TYPE_STR, Option::LEVEL_DEV)
1109     .set_default("")
1110     .set_description("Entity type to inject delays for")
1111     .set_flag(Option::FLAG_RUNTIME),
1112
1113     Option("ms_inject_delay_msg_type", Option::TYPE_STR, Option::LEVEL_DEV)
1114     .set_default("")
1115     .set_description("Message type to inject delays for"),
1116
1117     Option("ms_inject_delay_max", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1118     .set_default(1)
1119     .set_description("Max delay to inject"),
1120
1121     Option("ms_inject_delay_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1122     .set_default(0)
1123     .set_description(""),
1124
1125     Option("ms_inject_internal_delays", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1126     .set_default(0)
1127     .set_description("Inject various internal delays to induce races (seconds)"),
1128
1129     Option("ms_dump_on_send", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1130     .set_default(false)
1131     .set_description("Hexdump message to debug log on message send"),
1132
1133     Option("ms_dump_corrupt_message_level", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1134     .set_default(1)
1135     .set_description("Log level at which to hexdump corrupt messages we receive"),
1136
1137     Option("ms_async_op_threads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1138     .set_default(3)
1139     .set_min_max(1, 24)
1140     .set_description("Threadpool size for AsyncMessenger (ms_type=async)"),
1141
1142     Option("ms_async_max_op_threads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1143     .set_default(5)
1144     .set_description("Maximum threadpool size of AsyncMessenger")
1145     .add_see_also("ms_async_op_threads"),
1146
1147     Option("ms_async_rdma_device_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1148     .set_default("")
1149     .set_description(""),
1150
1151     Option("ms_async_rdma_enable_hugepage", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1152     .set_default(false)
1153     .set_description(""),
1154
1155     Option("ms_async_rdma_buffer_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1156     .set_default(128_K)
1157     .set_description(""),
1158
1159     Option("ms_async_rdma_send_buffers", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1160     .set_default(1_K)
1161     .set_description(""),
1162
1163     Option("ms_async_rdma_receive_buffers", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1164     .set_default(32768)
1165     .set_description(""),
1166
1167     Option("ms_async_rdma_receive_queue_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1168     .set_default(4096)
1169     .set_description(""),
1170
1171     Option("ms_async_rdma_support_srq", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1172     .set_default(true)
1173     .set_description(""),
1174
1175     Option("ms_async_rdma_port_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1176     .set_default(1)
1177     .set_description(""),
1178
1179     Option("ms_async_rdma_polling_us", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1180     .set_default(1000)
1181     .set_description(""),
1182
1183     Option("ms_async_rdma_local_gid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1184     .set_default("")
1185     .set_description(""),
1186
1187     Option("ms_async_rdma_roce_ver", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1188     .set_default(1)
1189     .set_description(""),
1190
1191     Option("ms_async_rdma_sl", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1192     .set_default(3)
1193     .set_description(""),
1194
1195     Option("ms_async_rdma_dscp", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1196     .set_default(96)
1197     .set_description(""),
1198
1199     Option("ms_max_accept_failures", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1200     .set_default(4)
1201     .set_description("The maximum number of consecutive failed accept() calls before "
1202                      "considering the daemon is misconfigured and abort it."),
1203
1204     Option("ms_async_rdma_cm", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1205     .set_default(false)
1206     .set_description(""),
1207
1208     Option("ms_async_rdma_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1209     .set_default("ib")
1210     .set_description(""),
1211
1212     Option("ms_dpdk_port_id", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1213     .set_default(0)
1214     .set_description(""),
1215
1216     Option("ms_dpdk_coremask", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1217     .set_default("0xF")         //begin with 0x for the string
1218     .set_description("")
1219     .add_see_also("ms_async_op_threads"),
1220
1221     Option("ms_dpdk_memory_channel", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1222     .set_default("4")
1223     .set_description(""),
1224
1225     Option("ms_dpdk_hugepages", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1226     .set_default("")
1227     .set_description(""),
1228
1229     Option("ms_dpdk_pmd", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1230     .set_default("")
1231     .set_description(""),
1232
1233     Option("ms_dpdk_host_ipv4_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1234     .set_default("")
1235     .set_description(""),
1236
1237     Option("ms_dpdk_gateway_ipv4_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1238     .set_default("")
1239     .set_description(""),
1240
1241     Option("ms_dpdk_netmask_ipv4_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1242     .set_default("")
1243     .set_description(""),
1244
1245     Option("ms_dpdk_lro", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1246     .set_default(true)
1247     .set_description(""),
1248
1249     Option("ms_dpdk_hw_flow_control", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1250     .set_default(true)
1251     .set_description(""),
1252
1253     Option("ms_dpdk_hw_queue_weight", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1254     .set_default(1)
1255     .set_description(""),
1256
1257     Option("ms_dpdk_debug_allow_loopback", Option::TYPE_BOOL, Option::LEVEL_DEV)
1258     .set_default(false)
1259     .set_description(""),
1260
1261     Option("ms_dpdk_rx_buffer_count_per_core", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1262     .set_default(8192)
1263     .set_description(""),
1264
1265     Option("inject_early_sigterm", Option::TYPE_BOOL, Option::LEVEL_DEV)
1266     .set_default(false)
1267     .set_description("send ourselves a SIGTERM early during startup"),
1268
1269     // MON
1270     Option("mon_enable_op_tracker", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1271     .set_default(true)
1272     .add_service("mon")
1273     .set_description("enable/disable MON op tracking"),
1274
1275     Option("mon_op_complaint_time", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
1276     .set_default(30)
1277     .add_service("mon")
1278     .set_description("time after which to consider a monitor operation blocked "
1279                      "after no updates"),
1280
1281     Option("mon_op_log_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1282     .set_default(5)
1283     .add_service("mon")
1284     .set_description("max number of slow ops to display"),
1285
1286     Option("mon_op_history_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1287     .set_default(20)
1288     .add_service("mon")
1289     .set_description("max number of completed ops to track"),
1290
1291     Option("mon_op_history_duration", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
1292     .set_default(600)
1293     .add_service("mon")
1294     .set_description("expiration time in seconds of historical MON OPS"),
1295
1296     Option("mon_op_history_slow_op_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1297     .set_default(20)
1298     .add_service("mon")
1299     .set_description("max number of slow historical MON OPS to keep"),
1300
1301     Option("mon_op_history_slow_op_threshold", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
1302     .set_default(10)
1303     .add_service("mon")
1304     .set_description("duration of an op to be considered as a historical slow op"),
1305
1306     Option("mon_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1307     .set_flag(Option::FLAG_NO_MON_UPDATE)
1308     .set_default("/var/lib/ceph/mon/$cluster-$id")
1309     .add_service("mon")
1310     .set_description("path to mon database"),
1311
1312     Option("mon_initial_members", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1313     .set_default("")
1314     .add_service("mon")
1315     .set_flag(Option::FLAG_NO_MON_UPDATE)
1316     .set_flag(Option::FLAG_CLUSTER_CREATE)
1317     .set_description(""),
1318
1319     Option("mon_compact_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1320     .set_default(false)
1321     .add_service("mon")
1322     .set_description(""),
1323
1324     Option("mon_compact_on_bootstrap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1325     .set_default(false)
1326     .add_service("mon")
1327     .set_description(""),
1328
1329     Option("mon_compact_on_trim", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1330     .set_default(true)
1331     .add_service("mon")
1332     .set_description(""),
1333
1334     /* -- mon: osdmap prune (begin) -- */
1335     Option("mon_osdmap_full_prune_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1336     .set_default(true)
1337     .add_service("mon")
1338     .set_description("enables pruning full osdmap versions when we go over a given number of maps")
1339     .add_see_also("mon_osdmap_full_prune_min")
1340     .add_see_also("mon_osdmap_full_prune_interval")
1341     .add_see_also("mon_osdmap_full_prune_txsize"),
1342
1343     Option("mon_osdmap_full_prune_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1344     .set_default(10000)
1345     .add_service("mon")
1346     .set_description("minimum number of versions in the store to trigger full map pruning")
1347     .add_see_also("mon_osdmap_full_prune_enabled")
1348     .add_see_also("mon_osdmap_full_prune_interval")
1349     .add_see_also("mon_osdmap_full_prune_txsize"),
1350
1351     Option("mon_osdmap_full_prune_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1352     .set_default(10)
1353     .add_service("mon")
1354     .set_description("interval between maps that will not be pruned; maps in the middle will be pruned.")
1355     .add_see_also("mon_osdmap_full_prune_enabled")
1356     .add_see_also("mon_osdmap_full_prune_interval")
1357     .add_see_also("mon_osdmap_full_prune_txsize"),
1358
1359     Option("mon_osdmap_full_prune_txsize", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1360     .set_default(100)
1361     .add_service("mon")
1362     .set_description("number of maps we will prune per iteration")
1363     .add_see_also("mon_osdmap_full_prune_enabled")
1364     .add_see_also("mon_osdmap_full_prune_interval")
1365     .add_see_also("mon_osdmap_full_prune_txsize"),
1366     /* -- mon: osdmap prune (end) -- */
1367
1368     Option("mon_osd_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1369     .set_default(500)
1370     .add_service("mon")
1371     .set_description("maximum number of OSDMaps to cache in memory"),
1372
1373     Option("mon_osd_cache_size_min", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1374     .set_default(128_M)
1375     .add_service("mon")
1376     .set_description("The minimum amount of bytes to be kept mapped in memory for osd monitor caches."),
1377
1378     Option("mon_memory_target", Option::TYPE_SIZE, Option::LEVEL_BASIC)
1379     .set_default(2_G)
1380     .set_flag(Option::FLAG_RUNTIME)
1381     .add_service("mon")
1382     .set_description("The amount of bytes pertaining to osd monitor caches and kv cache to be kept mapped in memory with cache auto-tuning enabled"),
1383
1384     Option("mon_memory_autotune", Option::TYPE_BOOL, Option::LEVEL_BASIC)
1385     .set_default(true)
1386     .set_flag(Option::FLAG_RUNTIME)
1387     .add_service("mon")
1388     .set_description("Autotune the cache memory being used for osd monitors and kv database"),
1389
1390     Option("mon_cpu_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1391     .set_default(4)
1392     .add_service("mon")
1393     .set_description("worker threads for CPU intensive background work"),
1394
1395     Option("mon_osd_mapping_pgs_per_chunk", Option::TYPE_INT, Option::LEVEL_DEV)
1396     .set_default(4096)
1397     .add_service("mon")
1398     .set_description("granularity of PG placement calculation background work"),
1399
1400     Option("mon_clean_pg_upmaps_per_chunk", Option::TYPE_INT, Option::LEVEL_DEV)
1401     .set_default(256)
1402     .add_service("mon")
1403     .set_description("granularity of PG upmap validation background work"),
1404
1405     Option("mon_osd_max_creating_pgs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1406     .set_default(1024)
1407     .add_service("mon")
1408     .set_description("maximum number of PGs the mon will create at once"),
1409
1410     Option("mon_osd_max_initial_pgs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1411     .set_default(1024)
1412     .add_service("mon")
1413     .set_description("maximum number of PGs a pool will created with")
1414     .set_long_description("If the user specifies more PGs than this, the cluster will subsequently split PGs after the pool is created in order to reach the target."),
1415
1416     Option("mon_tick_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1417     .set_default(5)
1418     .add_service("mon")
1419     .set_description("interval for internal mon background checks"),
1420
1421     Option("mon_session_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1422     .set_default(300)
1423     .add_service("mon")
1424     .set_description("close inactive mon client connections after this many seconds"),
1425
1426     Option("mon_subscribe_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1427     .set_default(1_day)
1428     .add_service("mon")
1429     .set_description("subscribe interval for pre-jewel clients"),
1430
1431     Option("mon_delta_reset_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1432     .set_default(10)
1433     .add_service("mon")
1434     .add_service("mon")
1435     .set_description("window duration for rate calculations in 'ceph status'"),
1436
1437     Option("mon_osd_laggy_halflife", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1438     .set_default(1_hr)
1439     .add_service("mon")
1440     .set_description("halflife of OSD 'lagginess' factor"),
1441
1442     Option("mon_osd_laggy_weight", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1443     .set_default(.3)
1444     .set_min_max(0.0, 1.0)
1445     .add_service("mon")
1446     .set_description("how heavily to weight OSD marking itself back up in overall laggy_probability")
1447     .set_long_description("1.0 means that an OSD marking itself back up (because it was marked down but not actually dead) means a 100% laggy_probability; 0.0 effectively disables tracking of laggy_probability."),
1448
1449     Option("mon_osd_laggy_max_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1450     .set_default(300)
1451     .add_service("mon")
1452     .set_description("cap value for period for OSD to be marked for laggy_interval calculation"),
1453
1454     Option("mon_osd_adjust_heartbeat_grace", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1455     .set_default(true)
1456     .add_service("mon")
1457     .set_description("increase OSD heartbeat grace if peers appear to be laggy")
1458     .set_long_description("If an OSD is marked down but then marks itself back up, it implies it wasn't actually down but was unable to respond to heartbeats.  If this option is true, we can use the laggy_probability and laggy_interval values calculated to model this situation to increase the heartbeat grace period for this OSD so that it isn't marked down again.  laggy_probability is an estimated probability that the given OSD is down because it is laggy (not actually down), and laggy_interval is an estiate on how long it stays down when it is laggy.")
1459     .add_see_also("mon_osd_laggy_halflife")
1460     .add_see_also("mon_osd_laggy_weight")
1461     .add_see_also("mon_osd_laggy_max_interval"),
1462
1463     Option("mon_osd_adjust_down_out_interval", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1464     .set_default(true)
1465     .add_service("mon")
1466     .set_description("increase the mon_osd_down_out_interval if an OSD appears to be laggy")
1467     .add_see_also("mon_osd_adjust_heartbeat_grace"),
1468
1469     Option("mon_osd_auto_mark_in", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1470     .set_default(false)
1471     .add_service("mon")
1472     .set_description("mark any OSD that comes up 'in'"),
1473
1474     Option("mon_osd_auto_mark_auto_out_in", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1475     .set_default(true)
1476     .add_service("mon")
1477     .set_description("mark any OSD that comes up that was automatically marked 'out' back 'in'")
1478     .add_see_also("mon_osd_down_out_interval"),
1479
1480     Option("mon_osd_auto_mark_new_in", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1481     .set_default(true)
1482     .add_service("mon")
1483     .set_description("mark any new OSD that comes up 'in'"),
1484
1485     Option("mon_osd_destroyed_out_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1486     .set_default(600)
1487     .add_service("mon")
1488     .set_description("mark any OSD 'out' that has been 'destroy'ed for this long (seconds)"),
1489
1490     Option("mon_osd_down_out_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1491     .set_default(600)
1492     .add_service("mon")
1493     .set_description("mark any OSD 'out' that has been 'down' for this long (seconds)"),
1494
1495     Option("mon_osd_down_out_subtree_limit", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1496     .set_default("rack")
1497     .set_flag(Option::FLAG_RUNTIME)
1498     .add_service("mon")
1499     .set_description("do not automatically mark OSDs 'out' if an entire subtree of this size is down")
1500     .add_see_also("mon_osd_down_out_interval"),
1501
1502     Option("mon_osd_min_up_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1503     .set_default(.3)
1504     .add_service("mon")
1505     .set_description("do not automatically mark OSDs 'out' if fewer than this many OSDs are 'up'")
1506     .add_see_also("mon_osd_down_out_interval"),
1507
1508     Option("mon_osd_min_in_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1509     .set_default(.75)
1510     .add_service("mon")
1511     .set_description("do not automatically mark OSDs 'out' if fewer than this many OSDs are 'in'")
1512     .add_see_also("mon_osd_down_out_interval"),
1513
1514     Option("mon_osd_warn_op_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1515     .set_default(32)
1516     .add_service("mgr")
1517     .set_description("issue REQUEST_SLOW health warning if OSD ops are slower than this age (seconds)"),
1518
1519     Option("mon_osd_err_op_age_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1520     .set_default(128)
1521     .add_service("mgr")
1522     .set_description("issue REQUEST_STUCK health error if OSD ops are slower than is age (seconds)"),
1523
1524     Option("mon_osd_prime_pg_temp", Option::TYPE_BOOL, Option::LEVEL_DEV)
1525     .set_default(true)
1526     .add_service("mon")
1527     .set_description("minimize peering work by priming pg_temp values after a map change"),
1528
1529     Option("mon_osd_prime_pg_temp_max_time", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1530     .set_default(.5)
1531     .add_service("mon")
1532     .set_description("maximum time to spend precalculating PG mappings on map change (seconds)"),
1533
1534     Option("mon_osd_prime_pg_temp_max_estimate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1535     .set_default(.25)
1536     .add_service("mon")
1537     .set_description("calculate all PG mappings if estimated fraction of PGs that change is above this amount"),
1538
1539     Option("mon_stat_smooth_intervals", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1540     .set_default(6)
1541     .set_min(1)
1542     .add_service("mgr")
1543     .set_description("number of PGMaps stats over which we calc the average read/write throughput of the whole cluster"),
1544
1545     Option("mon_election_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1546     .set_default(5)
1547     .add_service("mon")
1548     .set_description("maximum time for a mon election (seconds)"),
1549
1550     Option("mon_lease", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1551     .set_default(5)
1552     .add_service("mon")
1553     .set_description("lease interval between quorum monitors (seconds)")
1554     .set_long_description("This setting controls how sensitive your mon quorum is to intermittent network issues or other failures."),
1555
1556     Option("mon_lease_renew_interval_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1557     .set_default(.6)
1558     .set_min_max((double)0.0, (double).9999999)
1559     .add_service("mon")
1560     .set_description("multiple of mon_lease for the lease renewal interval")
1561     .set_long_description("Leases must be renewed before they time out.  A smaller value means frequent renewals, while a value close to 1 makes a lease expiration more likely.")
1562     .add_see_also("mon_lease"),
1563
1564     Option("mon_lease_ack_timeout_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1565     .set_default(2.0)
1566     .set_min_max(1.0001, 100.0)
1567     .add_service("mon")
1568     .set_description("multiple of mon_lease for the lease ack interval before calling new election")
1569     .add_see_also("mon_lease"),
1570
1571     Option("mon_accept_timeout_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1572     .set_default(2.0)
1573     .add_service("mon")
1574     .set_description("multiple of mon_lease for follower mons to accept proposed state changes before calling a new election")
1575     .add_see_also("mon_lease"),
1576
1577     Option("mon_clock_drift_allowed", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1578     .set_default(.050)
1579     .add_service("mon")
1580     .set_description("allowed clock drift (in seconds) between mons before issuing a health warning"),
1581
1582     Option("mon_clock_drift_warn_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1583     .set_default(5)
1584     .add_service("mon")
1585     .set_description("exponential backoff factor for logging clock drift warnings in the cluster log"),
1586
1587     Option("mon_timecheck_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1588     .set_default(300.0)
1589     .add_service("mon")
1590     .set_description("frequency of clock synchronization checks between monitors (seconds)"),
1591
1592     Option("mon_timecheck_skew_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1593     .set_default(30.0)
1594     .add_service("mon")
1595     .set_description("frequency of clock synchronization (re)checks between monitors while clocks are believed to be skewed (seconds)")
1596     .add_see_also("mon_timecheck_interval"),
1597
1598     Option("mon_pg_stuck_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1599     .set_default(60)
1600     .set_description("number of seconds after which pgs can be considered stuck inactive, unclean, etc")
1601     .set_long_description("see doc/control.rst under dump_stuck for more info")
1602     .add_service("mgr"),
1603
1604     Option("mon_pg_warn_min_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1605     .set_default(30)
1606     .add_service("mgr")
1607     .set_description("minimal number PGs per (in) osd before we warn the admin"),
1608
1609     Option("mon_max_pg_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1610     .set_min(1)
1611     .set_default(250)
1612     .add_service("mgr")
1613     .set_description("Max number of PGs per OSD the cluster will allow")
1614     .set_long_description("If the number of PGs per OSD exceeds this, a "
1615         "health warning will be visible in `ceph status`.  This is also used "
1616         "in automated PG management, as the threshold at which some pools' "
1617         "pg_num may be shrunk in order to enable increasing the pg_num of "
1618         "others."),
1619
1620     Option("mon_target_pg_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1621     .set_min(1)
1622     .set_default(100)
1623     .set_description("Automated PG management creates this many PGs per OSD")
1624     .set_long_description("When creating pools, the automated PG management "
1625         "logic will attempt to reach this target.  In some circumstances, it "
1626         "may exceed this target, up to the ``mon_max_pg_per_osd`` limit. "
1627         "Conversely, a lower number of PGs per OSD may be created if the "
1628         "cluster is not yet fully utilised"),
1629
1630     Option("mon_pg_warn_max_object_skew", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1631     .set_default(10.0)
1632     .set_description("max skew few average in objects per pg")
1633     .add_service("mgr"),
1634
1635     Option("mon_pg_warn_min_objects", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1636     .set_default(10000)
1637     .set_description("do not warn below this object #")
1638     .add_service("mgr"),
1639
1640     Option("mon_pg_warn_min_pool_objects", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1641     .set_default(1000)
1642     .set_description("do not warn on pools below this object #")
1643     .add_service("mgr"),
1644
1645     Option("mon_pg_check_down_all_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1646     .set_default(.5)
1647     .set_description("threshold of down osds after which we check all pgs")
1648     .add_service("mgr"),
1649
1650     Option("mon_cache_target_full_warn_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1651     .set_default(.66)
1652     .add_service("mgr")
1653     .set_flag(Option::FLAG_NO_MON_UPDATE)
1654     .set_flag(Option::FLAG_CLUSTER_CREATE)
1655     .set_description("issue CACHE_POOL_NEAR_FULL health warning when cache pool utilization exceeds this ratio of usable space"),
1656
1657     Option("mon_osd_full_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1658     .set_default(.95)
1659     .set_flag(Option::FLAG_NO_MON_UPDATE)
1660     .set_flag(Option::FLAG_CLUSTER_CREATE)
1661     .set_description("full ratio of OSDs to be set during initial creation of the cluster"),
1662
1663     Option("mon_osd_backfillfull_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1664     .set_default(.90)
1665     .set_flag(Option::FLAG_NO_MON_UPDATE)
1666     .set_flag(Option::FLAG_CLUSTER_CREATE)
1667     .set_description(""),
1668
1669     Option("mon_osd_nearfull_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1670     .set_default(.85)
1671     .set_flag(Option::FLAG_NO_MON_UPDATE)
1672     .set_flag(Option::FLAG_CLUSTER_CREATE)
1673     .set_description("nearfull ratio for OSDs to be set during initial creation of cluster"),
1674
1675     Option("mon_osd_initial_require_min_compat_client", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1676     .set_default("jewel")
1677     .set_flag(Option::FLAG_NO_MON_UPDATE)
1678     .set_flag(Option::FLAG_CLUSTER_CREATE)
1679     .set_description(""),
1680
1681     Option("mon_allow_pool_delete", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1682     .set_default(false)
1683     .add_service("mon")
1684     .set_description("allow pool deletions"),
1685
1686     Option("mon_fake_pool_delete", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1687     .set_default(false)
1688     .add_service("mon")
1689     .set_description("fake pool deletions by renaming the rados pool"),
1690
1691     Option("mon_globalid_prealloc", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1692     .set_default(10000)
1693     .add_service("mon")
1694     .set_description("number of globalid values to preallocate")
1695     .set_long_description("This setting caps how many new clients can authenticate with the cluster before the monitors have to perform a write to preallocate more.  Large values burn through the 64-bit ID space more quickly."),
1696
1697     Option("mon_osd_report_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1698     .set_default(900)
1699     .add_service("mon")
1700     .set_description("time before OSDs who do not report to the mons are marked down (seconds)"),
1701
1702     Option("mon_warn_on_msgr2_not_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1703     .set_default(true)
1704     .add_service("mon")
1705     .set_description("issue MON_MSGR2_NOT_ENABLED health warning if monitors are all running Nautilus but not all binding to a msgr2 port")
1706     .add_see_also("ms_bind_msgr2"),
1707
1708     Option("mon_warn_on_legacy_crush_tunables", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1709     .set_default(true)
1710     .add_service("mgr")
1711     .set_description("issue OLD_CRUSH_TUNABLES health warning if CRUSH tunables are older than mon_crush_min_required_version")
1712     .add_see_also("mon_crush_min_required_version"),
1713
1714     Option("mon_crush_min_required_version", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1715     .set_default("hammer")
1716     .add_service("mgr")
1717     .set_description("minimum ceph release to use for mon_warn_on_legacy_crush_tunables")
1718     .add_see_also("mon_warn_on_legacy_crush_tunables"),
1719
1720     Option("mon_warn_on_crush_straw_calc_version_zero", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1721     .set_default(true)
1722     .add_service("mgr")
1723     .set_description("issue OLD_CRUSH_STRAW_CALC_VERSION health warning if the CRUSH map's straw_calc_version is zero"),
1724
1725     Option("mon_warn_on_osd_down_out_interval_zero", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1726     .set_default(true)
1727     .add_service("mgr")
1728     .set_description("issue OSD_NO_DOWN_OUT_INTERVAL health warning if mon_osd_down_out_interval is zero")
1729     .set_long_description("Having mon_osd_down_out_interval set to 0 means that down OSDs are not marked out automatically and the cluster does not heal itself without administrator intervention.")
1730     .add_see_also("mon_osd_down_out_interval"),
1731
1732     Option("mon_warn_on_cache_pools_without_hit_sets", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1733     .set_default(true)
1734     .add_service("mgr")
1735     .set_description("issue CACHE_POOL_NO_HIT_SET health warning for cache pools that do not have hit sets configured"),
1736
1737     Option("mon_warn_on_pool_no_app", Option::TYPE_BOOL, Option::LEVEL_DEV)
1738     .set_default(true)
1739     .add_service("mgr")
1740     .set_description("issue POOL_APP_NOT_ENABLED health warning if pool has not application enabled"),
1741
1742     Option("mon_warn_on_misplaced", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1743     .set_default(false)
1744     .add_service("mgr")
1745     .set_description("Issue a health warning if there are misplaced objects"),
1746
1747     Option("mon_warn_on_too_few_osds", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1748     .set_default(true)
1749     .add_service("mgr")
1750     .set_description("Issue a health warning if there are fewer OSDs than osd_pool_default_size"),
1751
1752     Option("mon_warn_on_slow_ping_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1753     .set_default(0)
1754     .add_service("mgr")
1755     .set_description("Override mon_warn_on_slow_ping_ratio with specified threshold in milliseconds")
1756     .add_see_also("mon_warn_on_slow_ping_ratio"),
1757
1758     Option("mon_warn_on_slow_ping_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1759     .set_default(.05)
1760     .add_service("mgr")
1761     .set_description("Issue a health warning if heartbeat ping longer than percentage of osd_heartbeat_grace")
1762     .add_see_also("osd_heartbeat_grace")
1763     .add_see_also("mon_warn_on_slow_ping_time"),
1764
1765     Option("mon_max_snap_prune_per_epoch", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1766     .set_default(100)
1767     .add_service("mon")
1768     .set_description("max number of pruned snaps we will process in a single OSDMap epoch"),
1769
1770     Option("mon_min_osdmap_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1771     .set_default(500)
1772     .add_service("mon")
1773     .set_description("min number of OSDMaps to store"),
1774
1775     Option("mon_max_log_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1776     .set_default(500)
1777     .add_service("mon")
1778     .set_description("max number of past cluster log epochs to store"),
1779
1780     Option("mon_max_mdsmap_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1781     .set_default(500)
1782     .add_service("mon")
1783     .set_description("max number of FSMaps/MDSMaps to store"),
1784
1785     Option("mon_max_mgrmap_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1786     .set_default(500)
1787     .add_service("mon")
1788     .set_description("max number of MgrMaps to store"),
1789
1790     Option("mon_max_osd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1791     .set_default(10000)
1792     .add_service("mon")
1793     .set_description("max number of OSDs in a cluster"),
1794
1795     Option("mon_probe_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1796     .set_default(2.0)
1797     .add_service("mon")
1798     .set_description("timeout for querying other mons during bootstrap pre-election phase (seconds)"),
1799
1800     Option("mon_client_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1801     .set_default(100ul << 20)
1802     .add_service("mon")
1803     .set_description("max bytes of outstanding client messages mon will read off the network"),
1804
1805     Option("mon_daemon_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1806     .set_default(400ul << 20)
1807     .add_service("mon")
1808     .set_description("max bytes of outstanding mon messages mon will read off the network"),
1809
1810     Option("mon_mgr_proxy_client_bytes_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1811     .set_default(.3)
1812     .add_service("mon")
1813     .set_description("ratio of mon_client_bytes that can be consumed by "
1814                      "proxied mgr commands before we error out to client"),
1815
1816     Option("mon_log_max_summary", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1817     .set_default(50)
1818     .add_service("mon")
1819     .set_description("number of recent cluster log messages to retain"),
1820
1821     Option("mon_max_log_entries_per_event", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1822     .set_default(4096)
1823     .add_service("mon")
1824     .set_description("max cluster log entries per paxos event"),
1825
1826     Option("mon_reweight_min_pgs_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1827     .set_default(10)
1828     .add_service("mgr")
1829     .set_description(""),
1830
1831     Option("mon_reweight_min_bytes_per_osd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1832     .set_default(100_M)
1833     .add_service("mgr")
1834     .set_description(""),
1835
1836     Option("mon_reweight_max_osds", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1837     .set_default(4)
1838     .add_service("mgr")
1839     .set_description(""),
1840
1841     Option("mon_reweight_max_change", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1842     .set_default(0.05)
1843     .add_service("mgr")
1844     .set_description(""),
1845
1846     Option("mon_health_to_clog", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1847     .set_default(true)
1848     .add_service("mon")
1849     .set_description("log monitor health to cluster log"),
1850
1851     Option("mon_health_to_clog_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1852     .set_default(1_hr)
1853     .add_service("mon")
1854     .set_description("frequency to log monitor health to cluster log")
1855     .add_see_also("mon_health_to_clog"),
1856
1857     Option("mon_health_to_clog_tick_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1858     .set_default(60.0)
1859     .add_service("mon")
1860     .set_description(""),
1861
1862     Option("mon_health_max_detail", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1863     .set_default(50)
1864     .add_service("mon")
1865     .set_description("max detailed pgs to report in health detail"),
1866
1867     Option("mon_health_log_update_period", Option::TYPE_INT, Option::LEVEL_DEV)
1868     .set_default(5)
1869     .add_service("mon")
1870     .set_description("minimum time in seconds between log messages about "
1871                      "each health check")
1872     .set_min(0),
1873
1874     Option("mon_data_avail_crit", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1875     .set_default(5)
1876     .add_service("mon")
1877     .set_description("issue MON_DISK_CRIT health error when mon available space below this percentage"),
1878
1879     Option("mon_data_avail_warn", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1880     .set_default(30)
1881     .add_service("mon")
1882     .set_description("issue MON_DISK_LOW health warning when mon available space below this percentage"),
1883
1884     Option("mon_data_size_warn", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1885     .set_default(15_G)
1886     .add_service("mon")
1887     .set_description("issue MON_DISK_BIG health warning when mon database is above this size"),
1888
1889     Option("mon_warn_pg_not_scrubbed_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1890     .set_default(0.5)
1891     .set_min(0)
1892     .set_description("Percentage of the scrub max interval past the scrub max interval to warn")
1893     .set_long_description("")
1894     .add_see_also("osd_scrub_max_interval"),
1895
1896     Option("mon_warn_pg_not_deep_scrubbed_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1897     .set_default(0.75)
1898     .set_min(0)
1899     .set_description("Percentage of the deep scrub interval past the deep scrub interval to warn")
1900     .set_long_description("")
1901     .add_see_also("osd_deep_scrub_interval"),
1902
1903     Option("mon_scrub_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1904     .set_default(1_day)
1905     .add_service("mon")
1906     .set_description("frequency for scrubbing mon database"),
1907
1908     Option("mon_scrub_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1909     .set_default(5_min)
1910     .add_service("mon")
1911     .set_description("timeout to restart scrub of mon quorum participant does not respond for the latest chunk"),
1912
1913     Option("mon_scrub_max_keys", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1914     .set_default(100)
1915     .add_service("mon")
1916     .set_description("max keys per on scrub chunk/step"),
1917
1918     Option("mon_scrub_inject_crc_mismatch", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1919     .set_default(0.0)
1920     .add_service("mon")
1921     .set_description("probability for injecting crc mismatches into mon scrub"),
1922
1923     Option("mon_scrub_inject_missing_keys", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1924     .set_default(0.0)
1925     .add_service("mon")
1926     .set_description("probability for injecting missing keys into mon scrub"),
1927
1928     Option("mon_config_key_max_entry_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1929     .set_default(64_K)
1930     .add_service("mon")
1931     .set_description("Defines the number of bytes allowed to be held in a "
1932                      "single config-key entry"),
1933
1934     Option("mon_sync_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1935     .set_default(60.0)
1936     .add_service("mon")
1937     .set_description("timeout before canceling sync if syncing mon does not respond"),
1938
1939     Option("mon_sync_max_payload_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1940     .set_default(1_M)
1941     .add_service("mon")
1942     .set_description("target max message payload for mon sync"),
1943
1944     Option("mon_sync_debug", Option::TYPE_BOOL, Option::LEVEL_DEV)
1945     .set_default(false)
1946     .add_service("mon")
1947     .set_description("enable extra debugging during mon sync"),
1948
1949     Option("mon_inject_sync_get_chunk_delay", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1950     .set_default(0)
1951     .add_service("mon")
1952     .set_description("inject delay during sync (seconds)"),
1953
1954     Option("mon_osd_min_down_reporters", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1955     .set_default(2)
1956     .add_service("mon")
1957     .set_description("number of OSDs from different subtrees who need to report a down OSD for it to count")
1958     .add_see_also("mon_osd_reporter_subtree_level"),
1959
1960     Option("mon_osd_reporter_subtree_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1961     .set_default("host")
1962     .add_service("mon")
1963     .set_flag(Option::FLAG_RUNTIME)
1964     .set_description("in which level of parent bucket the reporters are counted"),
1965
1966     Option("mon_osd_snap_trim_queue_warn_on", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1967     .set_default(32768)
1968     .add_service("mon")
1969     .set_description("Warn when snap trim queue is that large (or larger).")
1970     .set_long_description("Warn when snap trim queue length for at least one PG crosses this value, as this is indicator of snap trimmer not keeping up, wasting disk space"),
1971
1972     Option("mon_osd_force_trim_to", Option::TYPE_INT, Option::LEVEL_DEV)
1973     .set_default(0)
1974     .add_service("mon")
1975     .set_description("force mons to trim osdmaps through this epoch"),
1976
1977     Option("mon_mds_force_trim_to", Option::TYPE_INT, Option::LEVEL_DEV)
1978     .set_default(0)
1979     .add_service("mon")
1980     .set_description("force mons to trim mdsmaps/fsmaps through this epoch"),
1981
1982     Option("mon_mds_skip_sanity", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1983     .set_default(false)
1984     .add_service("mon")
1985     .set_description("skip sanity checks on fsmap/mdsmap"),
1986
1987     Option("mon_debug_extra_checks", Option::TYPE_BOOL, Option::LEVEL_DEV)
1988     .set_default(false)
1989     .add_service("mon")
1990     .set_description("Enable some additional monitor checks")
1991     .set_long_description(
1992         "Enable some additional monitor checks that would be too expensive "
1993         "to run on production systems, or would only be relevant while "
1994         "testing or debugging."),
1995
1996     Option("mon_debug_block_osdmap_trim", Option::TYPE_BOOL, Option::LEVEL_DEV)
1997     .set_default(false)
1998     .add_service("mon")
1999     .set_description("Block OSDMap trimming while the option is enabled.")
2000     .set_long_description(
2001         "Blocking OSDMap trimming may be quite helpful to easily reproduce "
2002         "states in which the monitor keeps (hundreds of) thousands of "
2003         "osdmaps."),
2004
2005     Option("mon_debug_deprecated_as_obsolete", Option::TYPE_BOOL, Option::LEVEL_DEV)
2006     .set_default(false)
2007     .add_service("mon")
2008     .set_description("treat deprecated mon commands as obsolete"),
2009
2010     Option("mon_debug_dump_transactions", Option::TYPE_BOOL, Option::LEVEL_DEV)
2011     .set_default(false)
2012     .add_service("mon")
2013     .set_description("dump paxos transactions to log")
2014     .add_see_also("mon_debug_dump_location"),
2015
2016     Option("mon_debug_dump_json", Option::TYPE_BOOL, Option::LEVEL_DEV)
2017     .set_default(false)
2018     .add_service("mon")
2019     .set_description("dump paxos transasctions to log as json")
2020     .add_see_also("mon_debug_dump_transactions"),
2021
2022     Option("mon_debug_dump_location", Option::TYPE_STR, Option::LEVEL_DEV)
2023     .set_default("/var/log/ceph/$cluster-$name.tdump")
2024     .add_service("mon")
2025     .set_description("file to dump paxos transactions to")
2026     .add_see_also("mon_debug_dump_transactions"),
2027
2028     Option("mon_debug_no_require_mimic", Option::TYPE_BOOL, Option::LEVEL_DEV)
2029     .set_default(false)
2030     .add_service("mon")
2031     .set_flag(Option::FLAG_CLUSTER_CREATE)
2032     .set_description("do not set mimic feature for new mon clusters"),
2033
2034     Option("mon_debug_no_require_nautilus", Option::TYPE_BOOL, Option::LEVEL_DEV)
2035     .set_default(false)
2036     .add_service("mon")
2037     .set_flag(Option::FLAG_CLUSTER_CREATE)
2038     .set_description("do not set nautilus feature for new mon clusters"),
2039
2040     Option("mon_debug_no_require_bluestore_for_ec_overwrites", Option::TYPE_BOOL, Option::LEVEL_DEV)
2041     .set_default(false)
2042     .add_service("mon")
2043     .set_description("do not require bluestore OSDs to enable EC overwrites on a rados pool"),
2044
2045     Option("mon_debug_no_initial_persistent_features", Option::TYPE_BOOL, Option::LEVEL_DEV)
2046     .set_default(false)
2047     .add_service("mon")
2048     .set_flag(Option::FLAG_CLUSTER_CREATE)
2049     .set_description("do not set any monmap features for new mon clusters"),
2050
2051     Option("mon_inject_transaction_delay_max", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2052     .set_default(10.0)
2053     .add_service("mon")
2054     .set_description("max duration of injected delay in paxos"),
2055
2056     Option("mon_inject_transaction_delay_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2057     .set_default(0)
2058     .add_service("mon")
2059     .set_description("probability of injecting a delay in paxos"),
2060
2061     Option("mon_inject_pg_merge_bounce_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2062     .set_default(0)
2063     .add_service("mon")
2064     .set_description("probability of failing and reverting a pg_num decrement"),
2065
2066     Option("mon_sync_provider_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
2067     .set_default(0)
2068     .add_service("mon")
2069     .set_description("kill mon sync requester at specific point"),
2070
2071     Option("mon_sync_requester_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
2072     .set_default(0)
2073     .add_service("mon")
2074     .set_description("kill mon sync requestor at specific point"),
2075
2076     Option("mon_force_quorum_join", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2077     .set_default(false)
2078     .add_service("mon")
2079     .set_description("force mon to rejoin quorum even though it was just removed"),
2080
2081     Option("mon_keyvaluedb", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2082     .set_default("rocksdb")
2083     .set_enum_allowed({"leveldb", "rocksdb"})
2084     .set_flag(Option::FLAG_CREATE)
2085     .add_service("mon")
2086     .set_description("database backend to use for the mon database"),
2087
2088     Option("mon_debug_unsafe_allow_tier_with_nonempty_snaps", Option::TYPE_BOOL, Option::LEVEL_DEV)
2089     .set_default(false)
2090     .add_service("mon")
2091     .set_description(""),
2092
2093     Option("mon_osd_blacklist_default_expire", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2094     .set_default(1_hr)
2095     .add_service("mon")
2096     .set_description("Duration in seconds that blacklist entries for clients "
2097                      "remain in the OSD map"),
2098
2099     Option("mon_mds_blacklist_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2100     .set_default(1_day)
2101     .set_min(1_hr)
2102     .add_service("mon")
2103     .set_description("Duration in seconds that blacklist entries for MDS "
2104                      "daemons remain in the OSD map"),
2105
2106     Option("mon_osd_crush_smoke_test", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2107     .set_default(true)
2108     .add_service("mon")
2109     .set_description("perform a smoke test on any new CRUSH map before accepting changes"),
2110
2111     Option("mon_smart_report_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2112     .set_default(5)
2113     .add_service("mon")
2114     .set_description("Timeout (in seconds) for smarctl to run, default is set to 5"),
2115
2116
2117     // PAXOS
2118
2119     Option("paxos_stash_full_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2120     .set_default(25)
2121     .add_service("mon")
2122     .set_description(""),
2123
2124     Option("paxos_max_join_drift", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2125     .set_default(10)
2126     .add_service("mon")
2127     .set_description(""),
2128
2129     Option("paxos_propose_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2130     .set_default(1.0)
2131     .add_service("mon")
2132     .set_description(""),
2133
2134     Option("paxos_min_wait", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2135     .set_default(0.05)
2136     .add_service("mon")
2137     .set_description(""),
2138
2139     Option("paxos_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2140     .set_default(500)
2141     .add_service("mon")
2142     .set_description(""),
2143
2144     Option("paxos_trim_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2145     .set_default(250)
2146     .add_service("mon")
2147     .set_description(""),
2148
2149     Option("paxos_trim_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2150     .set_default(500)
2151     .add_service("mon")
2152     .set_description(""),
2153
2154     Option("paxos_service_trim_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2155     .set_default(250)
2156     .add_service("mon")
2157     .set_description(""),
2158
2159     Option("paxos_service_trim_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2160     .set_default(500)
2161     .add_service("mon")
2162     .set_description(""),
2163
2164     Option("paxos_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
2165     .set_default(0)
2166     .add_service("mon")
2167     .set_description(""),
2168
2169
2170     // AUTH
2171
2172     Option("auth_cluster_required", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2173     .set_default("cephx")
2174     .set_description("authentication methods required by the cluster"),
2175
2176     Option("auth_service_required", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2177     .set_default("cephx")
2178     .set_description("authentication methods required by service daemons"),
2179
2180     Option("auth_client_required", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2181     .set_default("cephx, none")
2182     .set_flag(Option::FLAG_MINIMAL_CONF)
2183     .set_description("authentication methods allowed by clients"),
2184
2185     Option("auth_supported", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2186     .set_default("")
2187     .set_description("authentication methods required (deprecated)"),
2188
2189     Option("max_rotating_auth_attempts", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2190     .set_default(10)
2191     .set_description("number of attempts to initialize rotating keys before giving up"),
2192
2193     Option("rotating_keys_bootstrap_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2194     .set_default(30)
2195     .set_description("timeout for obtaining rotating keys during bootstrap phase (seconds)"),
2196
2197     Option("rotating_keys_renewal_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2198     .set_default(10)
2199     .set_description("timeout for updating rotating keys (seconds)"),
2200
2201     Option("cephx_require_signatures", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2202     .set_default(false)
2203     .set_description(""),
2204
2205     Option("cephx_require_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2206     .set_default(1)
2207     .set_description("Cephx version required (1 = pre-mimic, 2 = mimic+)"),
2208
2209     Option("cephx_cluster_require_signatures", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2210     .set_default(false)
2211     .set_description(""),
2212
2213     Option("cephx_cluster_require_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2214     .set_default(1)
2215     .set_description("Cephx version required by the cluster from clients (1 = pre-mimic, 2 = mimic+)"),
2216
2217     Option("cephx_service_require_signatures", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2218     .set_default(false)
2219     .set_description(""),
2220
2221     Option("cephx_service_require_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2222     .set_default(1)
2223     .set_description("Cephx version required from ceph services (1 = pre-mimic, 2 = mimic+)"),
2224
2225     Option("cephx_sign_messages", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2226     .set_default(true)
2227     .set_description(""),
2228
2229     Option("auth_mon_ticket_ttl", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2230     .set_default(12_hr)
2231     .set_description(""),
2232
2233     Option("auth_service_ticket_ttl", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2234     .set_default(1_hr)
2235     .set_description(""),
2236
2237     Option("auth_debug", Option::TYPE_BOOL, Option::LEVEL_DEV)
2238     .set_default(false)
2239     .set_description(""),
2240
2241     Option("mon_client_hunt_parallel", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2242     .set_default(3)
2243     .set_description(""),
2244
2245     Option("mon_client_hunt_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2246     .set_default(3.0)
2247     .set_description(""),
2248
2249     Option("mon_client_ping_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2250     .set_default(10.0)
2251     .set_description(""),
2252
2253     Option("mon_client_ping_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2254     .set_default(30.0)
2255     .set_description(""),
2256
2257     Option("mon_client_hunt_interval_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2258     .set_default(1.5)
2259     .set_description(""),
2260
2261     Option("mon_client_hunt_interval_min_multiple", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2262     .set_default(1.0)
2263     .set_description(""),
2264
2265     Option("mon_client_hunt_interval_max_multiple", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2266     .set_default(10.0)
2267     .set_description(""),
2268
2269     Option("mon_client_max_log_entries_per_message", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2270     .set_default(1000)
2271     .set_description(""),
2272
2273     Option("mon_client_directed_command_retry", Option::TYPE_INT, Option::LEVEL_DEV)
2274     .set_default(2)
2275     .set_description("Number of times to try sending a comamnd directed at a specific monitor"),
2276
2277     Option("mon_max_pool_pg_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2278     .set_default(65536)
2279     .set_description(""),
2280
2281     Option("mon_pool_quota_warn_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2282     .set_default(0)
2283     .set_description("percent of quota at which to issue warnings")
2284     .add_service("mgr"),
2285
2286     Option("mon_pool_quota_crit_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2287     .set_default(0)
2288     .set_description("percent of quota at which to issue errors")
2289     .add_service("mgr"),
2290
2291     Option("crush_location", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2292     .set_default("")
2293     .set_description(""),
2294
2295     Option("crush_location_hook", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2296     .set_default("")
2297     .set_description(""),
2298
2299     Option("crush_location_hook_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2300     .set_default(10)
2301     .set_description(""),
2302
2303     Option("objecter_tick_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2304     .set_default(5.0)
2305     .set_description(""),
2306
2307     Option("objecter_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2308     .set_default(10.0)
2309     .set_description("Seconds before in-flight op is considered 'laggy' and we query mon for the latest OSDMap"),
2310
2311     Option("objecter_inflight_op_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2312     .set_default(100_M)
2313     .set_description("Max in-flight data in bytes (both directions)"),
2314
2315     Option("objecter_inflight_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2316     .set_default(1024)
2317     .set_description("Max in-flight operations"),
2318
2319     Option("objecter_completion_locks_per_session", Option::TYPE_UINT, Option::LEVEL_DEV)
2320     .set_default(32)
2321     .set_description(""),
2322
2323     Option("objecter_inject_no_watch_ping", Option::TYPE_BOOL, Option::LEVEL_DEV)
2324     .set_default(false)
2325     .set_description(""),
2326
2327     Option("objecter_retry_writes_after_first_reply", Option::TYPE_BOOL, Option::LEVEL_DEV)
2328     .set_default(false)
2329     .set_description(""),
2330
2331     Option("objecter_debug_inject_relock_delay", Option::TYPE_BOOL, Option::LEVEL_DEV)
2332     .set_default(false)
2333     .set_description(""),
2334
2335     Option("filer_max_purge_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2336     .set_default(10)
2337     .set_description("Max in-flight operations for purging a striped range (e.g., MDS journal)"),
2338
2339     Option("filer_max_truncate_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2340     .set_default(128)
2341     .set_description("Max in-flight operations for truncating/deleting a striped sequence (e.g., MDS journal)"),
2342
2343     Option("journaler_write_head_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2344     .set_default(15)
2345     .set_description("Interval in seconds between journal header updates (to help bound replay time)"),
2346
2347     // * journal object size
2348     Option("journaler_prefetch_periods", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2349     .set_default(10)
2350     .set_min(2)                 // we need at least 2 periods to make progress.
2351     .set_description("Number of striping periods to prefetch while reading MDS journal"),
2352
2353     // * journal object size
2354     Option("journaler_prezero_periods", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2355     .set_default(5)
2356     // we need to zero at least two periods, minimum, to ensure that we
2357     // have a full empty object/period in front of us.
2358     .set_min(2)
2359     .set_description("Number of striping periods to zero head of MDS journal write position"),
2360
2361     // -- OSD --
2362     Option("osd_calc_pg_upmaps_aggressively", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2363     .set_default(true)
2364     .set_flag(Option::FLAG_RUNTIME)
2365     .set_description("try to calculate PG upmaps more aggressively, e.g., "
2366                      "by doing a fairly exhaustive search of existing PGs "
2367                      "that can be unmapped or upmapped"),
2368
2369     Option("osd_calc_pg_upmaps_max_stddev", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2370     .set_default(1.0)
2371     .set_flag(Option::FLAG_RUNTIME)
2372     .set_description("standard deviation below which there is no attempt made "
2373                      "while trying to calculate PG upmaps"),
2374
2375     Option("osd_calc_pg_upmaps_local_fallback_retries", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2376     .set_default(100)
2377     .set_flag(Option::FLAG_RUNTIME)
2378     .set_description("Maximum number of PGs we can attempt to unmap or upmap "
2379                      "for a specific overfull or underfull osd per iteration "),
2380
2381     Option("osd_numa_prefer_iface", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2382     .set_default(true)
2383     .set_flag(Option::FLAG_STARTUP)
2384     .set_description("prefer IP on network interface on same numa node as storage")
2385     .add_see_also("osd_numa_auto_affinity"),
2386
2387     Option("osd_numa_auto_affinity", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2388     .set_default(true)
2389     .set_flag(Option::FLAG_STARTUP)
2390     .set_description("automatically set affinity to numa node when storage and network match"),
2391
2392     Option("osd_numa_node", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2393     .set_default(-1)
2394     .set_flag(Option::FLAG_STARTUP)
2395     .set_description("set affinity to a numa node (-1 for none)")
2396     .add_see_also("osd_numa_auto_affinity"),
2397
2398     Option("osd_smart_report_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2399     .set_default(5)
2400     .set_description("Timeout (in seconds) for smarctl to run, default is set to 5"),
2401
2402     Option("osd_check_max_object_name_len_on_startup", Option::TYPE_BOOL, Option::LEVEL_DEV)
2403     .set_default(true)
2404     .set_description(""),
2405
2406     Option("osd_max_backfills", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2407     .set_default(1)
2408     .set_description("Maximum number of concurrent local and remote backfills or recoveries per OSD ")
2409     .set_long_description("There can be osd_max_backfills local reservations AND the same remote reservations per OSD. So a value of 1 lets this OSD participate as 1 PG primary in recovery and 1 shard of another recovering PG."),
2410
2411     Option("osd_min_recovery_priority", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2412     .set_default(0)
2413     .set_description("Minimum priority below which recovery is not performed")
2414     .set_long_description("The purpose here is to prevent the cluster from doing *any* lower priority work (e.g., rebalancing) below this threshold and focus solely on higher priority work (e.g., replicating degraded objects)."),
2415
2416     Option("osd_backfill_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2417     .set_default(30.0)
2418     .set_description("how frequently to retry backfill reservations after being denied (e.g., due to a full OSD)"),
2419
2420     Option("osd_recovery_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2421     .set_default(30.0)
2422     .set_description("how frequently to retry recovery reservations after being denied (e.g., due to a full OSD)"),
2423
2424     Option("osd_agent_max_ops", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2425     .set_default(4)
2426     .set_description("maximum concurrent tiering operations for tiering agent"),
2427
2428     Option("osd_agent_max_low_ops", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2429     .set_default(2)
2430     .set_description("maximum concurrent low-priority tiering operations for tiering agent"),
2431
2432     Option("osd_agent_min_evict_effort", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2433     .set_default(.1)
2434     .set_min_max(0.0, .99)
2435     .set_description("minimum effort to expend evicting clean objects"),
2436
2437     Option("osd_agent_quantize_effort", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2438     .set_default(.1)
2439     .set_description("size of quantize unit for eviction effort"),
2440
2441     Option("osd_agent_delay_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2442     .set_default(5.0)
2443     .set_description("how long agent should sleep if it has no work to do"),
2444
2445     Option("osd_find_best_info_ignore_history_les", Option::TYPE_BOOL, Option::LEVEL_DEV)
2446     .set_default(false)
2447     .set_description("ignore last_epoch_started value when peering AND PROBABLY LOSE DATA")
2448     .set_long_description("THIS IS AN EXTREMELY DANGEROUS OPTION THAT SHOULD ONLY BE USED AT THE DIRECTION OF A DEVELOPER.  It makes peering ignore the last_epoch_started value when peering, which can allow the OSD to believe an OSD has an authoritative view of a PG's contents even when it is in fact old and stale, typically leading to data loss (by believing a stale PG is up to date)."),
2449
2450     Option("osd_agent_hist_halflife", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2451     .set_default(1000)
2452     .set_description("halflife of agent atime and temp histograms"),
2453
2454     Option("osd_agent_slop", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2455     .set_default(.02)
2456     .set_description("slop factor to avoid switching tiering flush and eviction mode"),
2457
2458     Option("osd_uuid", Option::TYPE_UUID, Option::LEVEL_ADVANCED)
2459     .set_default(uuid_d())
2460     .set_flag(Option::FLAG_CREATE)
2461     .set_description("uuid label for a new OSD"),
2462
2463     Option("osd_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2464     .set_default("/var/lib/ceph/osd/$cluster-$id")
2465     .set_flag(Option::FLAG_NO_MON_UPDATE)
2466     .set_description("path to OSD data"),
2467
2468     Option("osd_journal", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2469     .set_default("/var/lib/ceph/osd/$cluster-$id/journal")
2470     .set_flag(Option::FLAG_NO_MON_UPDATE)
2471     .set_description("path to OSD journal (when FileStore backend is in use)"),
2472
2473     Option("osd_journal_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2474     .set_default(5120)
2475     .set_flag(Option::FLAG_CREATE)
2476     .set_description("size of FileStore journal (in MiB)"),
2477
2478     Option("osd_journal_flush_on_shutdown", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2479     .set_default(true)
2480     .set_description("flush FileStore journal contents during clean OSD shutdown"),
2481
2482     Option("osd_os_flags", Option::TYPE_UINT, Option::LEVEL_DEV)
2483     .set_default(0)
2484     .set_description("flags to skip filestore omap or journal initialization"),
2485
2486     Option("osd_max_write_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2487     .set_min(4)
2488     .set_default(90)
2489     .set_description("Maximum size of a RADOS write operation in megabytes")
2490     .set_long_description("This setting prevents clients from doing "
2491         "very large writes to RADOS.  If you set this to a value "
2492         "below what clients expect, they will receive an error "
2493         "when attempting to write to the cluster."),
2494
2495     Option("osd_max_pgls", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2496     .set_default(1024)
2497     .set_description("maximum number of results when listing objects in a pool"),
2498
2499     Option("osd_client_message_size_cap", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2500     .set_default(500_M)
2501     .set_description("maximum memory to devote to in-flight client requests")
2502     .set_long_description("If this value is exceeded, the OSD will not read any new client data off of the network until memory is freed."),
2503
2504     Option("osd_client_message_cap", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2505     .set_default(100)
2506     .set_description("maximum number of in-flight client requests"),
2507
2508     Option("osd_crush_update_weight_set", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2509     .set_default(true)
2510     .set_description("update CRUSH weight-set weights when updating weights")
2511     .set_long_description("If this setting is true, we will update the weight-set weights when adjusting an item's weight, effectively making changes take effect immediately, and discarding any previous optimization in the weight-set value.  Setting this value to false will leave it to the balancer to (slowly, presumably) adjust weights to approach the new target value."),
2512
2513     Option("osd_crush_chooseleaf_type", Option::TYPE_INT, Option::LEVEL_DEV)
2514     .set_default(1)
2515     .set_flag(Option::FLAG_CREATE)
2516     .set_description("default chooseleaf type for osdmaptool --create"),
2517
2518     Option("osd_pool_use_gmt_hitset", Option::TYPE_BOOL, Option::LEVEL_DEV)
2519     .set_default(true)
2520     .set_description("use UTC for hitset timestamps")
2521     .set_long_description("This setting only exists for compatibility with hammer (and older) clusters."),
2522
2523     Option("osd_crush_update_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2524     .set_default(true)
2525     .set_description("update OSD CRUSH location on startup"),
2526
2527     Option("osd_class_update_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2528     .set_default(true)
2529     .set_description("set OSD device class on startup"),
2530
2531     Option("osd_crush_initial_weight", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2532     .set_default(-1)
2533     .set_description("if >= 0, initial CRUSH weight for newly created OSDs")
2534     .set_long_description("If this value is negative, the size of the OSD in TiB is used."),
2535
2536     Option("osd_pool_default_ec_fast_read", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2537     .set_default(false)
2538     .set_description("set ec_fast_read for new erasure-coded pools")
2539     .add_service("mon"),
2540
2541     Option("osd_pool_default_crush_rule", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2542     .set_default(-1)
2543     .set_description("CRUSH rule for newly created pools")
2544     .add_service("mon"),
2545
2546     Option("osd_pool_erasure_code_stripe_unit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2547     .set_default(4_K)
2548     .set_description("the amount of data (in bytes) in a data chunk, per stripe")
2549     .add_service("mon"),
2550
2551     Option("osd_pool_default_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2552     .set_default(3)
2553     .set_min_max(0, 10)
2554     .set_flag(Option::FLAG_RUNTIME)
2555     .set_description("the number of copies of an object for new replicated pools")
2556     .add_service("mon"),
2557
2558     Option("osd_pool_default_min_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2559     .set_default(0)
2560     .set_min_max(0, 255)
2561     .set_flag(Option::FLAG_RUNTIME)
2562     .set_description("the minimal number of copies allowed to write to a degraded pool for new replicated pools")
2563     .set_long_description("0 means no specific default; ceph will use size-size/2")
2564     .add_see_also("osd_pool_default_size")
2565     .add_service("mon"),
2566
2567     Option("osd_pool_default_pg_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2568     .set_default(8)
2569     .set_description("number of PGs for new pools")
2570     .set_flag(Option::FLAG_RUNTIME)
2571     .add_service("mon"),
2572
2573     Option("osd_pool_default_pgp_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2574     .set_default(0)
2575     .set_description("number of PGs for placement purposes (0 to match pg_num)")
2576     .add_see_also("osd_pool_default_pg_num")
2577     .set_flag(Option::FLAG_RUNTIME)
2578     .add_service("mon"),
2579
2580     Option("osd_pool_default_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2581     .set_default("replicated")
2582     .set_enum_allowed({"replicated", "erasure"})
2583     .set_flag(Option::FLAG_RUNTIME)
2584     .set_description("default type of pool to create")
2585     .add_service("mon"),
2586
2587     Option("osd_pool_default_erasure_code_profile", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2588     .set_default("plugin=jerasure technique=reed_sol_van k=2 m=1")
2589     .set_flag(Option::FLAG_RUNTIME)
2590     .set_description("default erasure code profile for new erasure-coded pools")
2591     .add_service("mon"),
2592
2593     Option("osd_erasure_code_plugins", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2594     .set_default("jerasure lrc"
2595   #ifdef HAVE_BETTER_YASM_ELF64
2596          " isa"
2597   #endif
2598         )
2599     .set_flag(Option::FLAG_STARTUP)
2600     .set_description("erasure code plugins to load")
2601     .add_service("mon")
2602     .add_service("osd"),
2603
2604     Option("osd_allow_recovery_below_min_size", Option::TYPE_BOOL, Option::LEVEL_DEV)
2605     .set_default(true)
2606     .set_description("allow replicated pools to recover with < min_size active members")
2607     .add_service("osd"),
2608
2609     Option("osd_pool_default_flags", Option::TYPE_INT, Option::LEVEL_DEV)
2610     .set_default(0)
2611     .set_description("(integer) flags to set on new pools")
2612     .add_service("mon"),
2613
2614     Option("osd_pool_default_flag_hashpspool", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2615     .set_default(true)
2616     .set_description("set hashpspool (better hashing scheme) flag on new pools")
2617     .add_service("mon"),
2618
2619     Option("osd_pool_default_flag_nodelete", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2620     .set_default(false)
2621     .set_description("set nodelete flag on new pools")
2622     .add_service("mon"),
2623
2624     Option("osd_pool_default_flag_nopgchange", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2625     .set_default(false)
2626     .set_description("set nopgchange flag on new pools")
2627     .add_service("mon"),
2628
2629     Option("osd_pool_default_flag_nosizechange", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2630     .set_default(false)
2631     .set_description("set nosizechange flag on new pools")
2632     .add_service("mon"),
2633
2634     Option("osd_pool_default_hit_set_bloom_fpp", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2635     .set_default(.05)
2636     .set_description("")
2637     .add_see_also("osd_tier_default_cache_hit_set_type")
2638     .add_service("mon"),
2639
2640     Option("osd_pool_default_cache_target_dirty_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2641     .set_default(.4)
2642     .set_description(""),
2643
2644     Option("osd_pool_default_cache_target_dirty_high_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2645     .set_default(.6)
2646     .set_description(""),
2647
2648     Option("osd_pool_default_cache_target_full_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2649     .set_default(.8)
2650     .set_description(""),
2651
2652     Option("osd_pool_default_cache_min_flush_age", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2653     .set_default(0)
2654     .set_description(""),
2655
2656     Option("osd_pool_default_cache_min_evict_age", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2657     .set_default(0)
2658     .set_description(""),
2659
2660     Option("osd_pool_default_cache_max_evict_check_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2661     .set_default(10)
2662     .set_description(""),
2663
2664     Option("osd_pool_default_pg_autoscale_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2665     .set_default("warn")
2666     .set_flag(Option::FLAG_RUNTIME)
2667     .set_enum_allowed({"off", "warn", "on"})
2668     .set_description("Default PG autoscaling behavior for new pools"),
2669
2670     Option("osd_hit_set_min_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2671     .set_default(1000)
2672     .set_description(""),
2673
2674     Option("osd_hit_set_max_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2675     .set_default(100000)
2676     .set_description(""),
2677
2678     Option("osd_hit_set_namespace", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2679     .set_default(".ceph-internal")
2680     .set_description(""),
2681
2682     Option("osd_tier_promote_max_objects_sec", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2683     .set_default(25)
2684     .set_description(""),
2685
2686     Option("osd_tier_promote_max_bytes_sec", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2687     .set_default(5_M)
2688     .set_description(""),
2689
2690     Option("osd_tier_default_cache_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2691     .set_default("writeback")
2692     .set_enum_allowed({"none", "writeback", "forward",
2693                        "readonly", "readforward", "readproxy", "proxy"})
2694     .set_flag(Option::FLAG_RUNTIME)
2695     .set_description(""),
2696
2697     Option("osd_tier_default_cache_hit_set_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2698     .set_default(4)
2699     .set_description(""),
2700
2701     Option("osd_tier_default_cache_hit_set_period", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2702     .set_default(1200)
2703     .set_description(""),
2704
2705     Option("osd_tier_default_cache_hit_set_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2706     .set_default("bloom")
2707     .set_enum_allowed({"bloom", "explicit_hash", "explicit_object"})
2708     .set_flag(Option::FLAG_RUNTIME)
2709     .set_description(""),
2710
2711     Option("osd_tier_default_cache_min_read_recency_for_promote", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2712     .set_default(1)
2713     .set_description("number of recent HitSets the object must appear in to be promoted (on read)"),
2714
2715     Option("osd_tier_default_cache_min_write_recency_for_promote", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2716     .set_default(1)
2717     .set_description("number of recent HitSets the object must appear in to be promoted (on write)"),
2718
2719     Option("osd_tier_default_cache_hit_set_grade_decay_rate", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2720     .set_default(20)
2721     .set_description(""),
2722
2723     Option("osd_tier_default_cache_hit_set_search_last_n", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2724     .set_default(1)
2725     .set_description(""),
2726
2727     Option("osd_objecter_finishers", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2728     .set_default(1)
2729     .set_flag(Option::FLAG_STARTUP)
2730     .set_description(""),
2731
2732     Option("osd_map_dedup", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2733     .set_default(true)
2734     .set_description(""),
2735
2736     Option("osd_map_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2737     .set_default(50)
2738     .set_description(""),
2739
2740     Option("osd_map_message_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2741     .set_default(40)
2742     .set_description("maximum number of OSDMaps to include in a single message"),
2743
2744     Option("osd_map_message_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2745     .set_default(10_M)
2746     .set_description("maximum number of bytes worth of OSDMaps to include in a single message"),
2747
2748     Option("osd_map_share_max_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2749     .set_default(40)
2750     .set_description(""),
2751
2752     Option("osd_pg_epoch_max_lag_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2753     .set_default(2.0)
2754     .set_description("Max multiple of the map cache that PGs can lag before we throttle map injest")
2755     .add_see_also("osd_map_cache_size"),
2756
2757     Option("osd_inject_bad_map_crc_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2758     .set_default(0)
2759     .set_description(""),
2760
2761     Option("osd_inject_failure_on_pg_removal", Option::TYPE_BOOL, Option::LEVEL_DEV)
2762     .set_default(false)
2763     .set_description(""),
2764
2765     Option("osd_max_markdown_period", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2766     .set_default(600)
2767     .set_description(""),
2768
2769     Option("osd_max_markdown_count", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2770     .set_default(5)
2771     .set_description(""),
2772
2773     Option("osd_op_pq_max_tokens_per_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2774     .set_default(4194304)
2775     .set_description(""),
2776
2777     Option("osd_op_pq_min_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2778     .set_default(65536)
2779     .set_description(""),
2780
2781     Option("osd_recover_clone_overlap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2782     .set_default(true)
2783     .set_description(""),
2784
2785     Option("osd_op_num_threads_per_shard", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2786     .set_default(0)
2787     .set_flag(Option::FLAG_STARTUP)
2788     .set_description(""),
2789
2790     Option("osd_op_num_threads_per_shard_hdd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2791     .set_default(1)
2792     .set_flag(Option::FLAG_STARTUP)
2793     .set_description("")
2794     .add_see_also("osd_op_num_threads_per_shard"),
2795
2796     Option("osd_op_num_threads_per_shard_ssd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2797     .set_default(2)
2798     .set_flag(Option::FLAG_STARTUP)
2799     .set_description("")
2800     .add_see_also("osd_op_num_threads_per_shard"),
2801
2802     Option("osd_op_num_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2803     .set_default(0)
2804     .set_flag(Option::FLAG_STARTUP)
2805     .set_description(""),
2806
2807     Option("osd_op_num_shards_hdd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2808     .set_default(5)
2809     .set_flag(Option::FLAG_STARTUP)
2810     .set_description("")
2811     .add_see_also("osd_op_num_shards"),
2812
2813     Option("osd_op_num_shards_ssd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2814     .set_default(8)
2815     .set_flag(Option::FLAG_STARTUP)
2816     .set_description("")
2817     .add_see_also("osd_op_num_shards"),
2818
2819     Option("osd_skip_data_digest", Option::TYPE_BOOL, Option::LEVEL_DEV)
2820     .set_default(false)
2821     .set_description("Do not store full-object checksums if the backend (bluestore) does its own checksums.  Only usable with all BlueStore OSDs."),
2822
2823     Option("osd_op_queue", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2824     .set_default("wpq")
2825     .set_enum_allowed( { "wpq", "prioritized", "mclock_opclass", "mclock_client", "debug_random" } )
2826     .set_description("which operation queue algorithm to use")
2827     .set_long_description("which operation queue algorithm to use; mclock_opclass and mclock_client are currently experimental")
2828     .set_flag(Option::FLAG_STARTUP)
2829     .add_see_also("osd_op_queue_cut_off"),
2830
2831     Option("osd_op_queue_cut_off", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2832     .set_default("low")
2833     .set_enum_allowed( { "low", "high", "debug_random" } )
2834     .set_description("the threshold between high priority ops and low priority ops")
2835     .set_long_description("the threshold between high priority ops that use strict priority ordering and low priority ops that use a fairness algorithm that may or may not incorporate priority")
2836     .add_see_also("osd_op_queue"),
2837
2838     Option("osd_op_queue_mclock_client_op_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2839     .set_default(1000.0)
2840     .set_description("mclock reservation of client operator requests")
2841     .set_long_description("mclock reservation of client operator requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the reservation")
2842     .add_see_also("osd_op_queue")
2843     .add_see_also("osd_op_queue_mclock_client_op_wgt")
2844     .add_see_also("osd_op_queue_mclock_client_op_lim")
2845     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
2846     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
2847     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
2848     .add_see_also("osd_op_queue_mclock_snap_res")
2849     .add_see_also("osd_op_queue_mclock_snap_wgt")
2850     .add_see_also("osd_op_queue_mclock_snap_lim")
2851     .add_see_also("osd_op_queue_mclock_recov_res")
2852     .add_see_also("osd_op_queue_mclock_recov_wgt")
2853     .add_see_also("osd_op_queue_mclock_recov_lim")
2854     .add_see_also("osd_op_queue_mclock_scrub_res")
2855     .add_see_also("osd_op_queue_mclock_scrub_wgt")
2856     .add_see_also("osd_op_queue_mclock_scrub_lim")
2857     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
2858
2859     Option("osd_op_queue_mclock_client_op_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2860     .set_default(500.0)
2861     .set_description("mclock weight of client operator requests")
2862     .set_long_description("mclock weight of client operator requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the weight")
2863     .add_see_also("osd_op_queue")
2864     .add_see_also("osd_op_queue_mclock_client_op_res")
2865     .add_see_also("osd_op_queue_mclock_client_op_lim")
2866     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
2867     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
2868     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
2869     .add_see_also("osd_op_queue_mclock_snap_res")
2870     .add_see_also("osd_op_queue_mclock_snap_wgt")
2871     .add_see_also("osd_op_queue_mclock_snap_lim")
2872     .add_see_also("osd_op_queue_mclock_recov_res")
2873     .add_see_also("osd_op_queue_mclock_recov_wgt")
2874     .add_see_also("osd_op_queue_mclock_recov_lim")
2875     .add_see_also("osd_op_queue_mclock_scrub_res")
2876     .add_see_also("osd_op_queue_mclock_scrub_wgt")
2877     .add_see_also("osd_op_queue_mclock_scrub_lim")
2878     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
2879
2880     Option("osd_op_queue_mclock_client_op_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2881     .set_default(0.0)
2882     .set_description("mclock limit of client operator requests")
2883     .set_long_description("mclock limit of client operator requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the limit")
2884     .add_see_also("osd_op_queue")
2885     .add_see_also("osd_op_queue_mclock_client_op_res")
2886     .add_see_also("osd_op_queue_mclock_client_op_wgt")
2887     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
2888     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
2889     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
2890     .add_see_also("osd_op_queue_mclock_snap_res")
2891     .add_see_also("osd_op_queue_mclock_snap_wgt")
2892     .add_see_also("osd_op_queue_mclock_snap_lim")
2893     .add_see_also("osd_op_queue_mclock_recov_res")
2894     .add_see_also("osd_op_queue_mclock_recov_wgt")
2895     .add_see_also("osd_op_queue_mclock_recov_lim")
2896     .add_see_also("osd_op_queue_mclock_scrub_res")
2897     .add_see_also("osd_op_queue_mclock_scrub_wgt")
2898     .add_see_also("osd_op_queue_mclock_scrub_lim")
2899     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
2900
2901     Option("osd_op_queue_mclock_osd_rep_op_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2902     .set_default(1000.0)
2903     .set_description("mclock reservation of osd replication operation requests and replies")
2904     .set_long_description("mclock reservation of replication operation requests and replies when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the reservation")
2905     .add_see_also("osd_op_queue")
2906     .add_see_also("osd_op_queue_mclock_client_op_res")
2907     .add_see_also("osd_op_queue_mclock_client_op_wgt")
2908     .add_see_also("osd_op_queue_mclock_client_op_lim")
2909     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
2910     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
2911     .add_see_also("osd_op_queue_mclock_snap_res")
2912     .add_see_also("osd_op_queue_mclock_snap_wgt")
2913     .add_see_also("osd_op_queue_mclock_snap_lim")
2914     .add_see_also("osd_op_queue_mclock_recov_res")
2915     .add_see_also("osd_op_queue_mclock_recov_wgt")
2916     .add_see_also("osd_op_queue_mclock_recov_lim")
2917     .add_see_also("osd_op_queue_mclock_scrub_res")
2918     .add_see_also("osd_op_queue_mclock_scrub_wgt")
2919     .add_see_also("osd_op_queue_mclock_scrub_lim")
2920     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
2921
2922     Option("osd_op_queue_mclock_osd_rep_op_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2923     .set_default(500.0)
2924     .set_description("mclock weight of osd replication operation requests and replies")
2925     .set_long_description("mclock weight of osd replication operation requests and replies when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the weight")
2926     .add_see_also("osd_op_queue")
2927     .add_see_also("osd_op_queue_mclock_client_op_res")
2928     .add_see_also("osd_op_queue_mclock_client_op_wgt")
2929     .add_see_also("osd_op_queue_mclock_client_op_lim")
2930     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
2931     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
2932     .add_see_also("osd_op_queue_mclock_snap_res")
2933     .add_see_also("osd_op_queue_mclock_snap_wgt")
2934     .add_see_also("osd_op_queue_mclock_snap_lim")
2935     .add_see_also("osd_op_queue_mclock_recov_res")
2936     .add_see_also("osd_op_queue_mclock_recov_wgt")
2937     .add_see_also("osd_op_queue_mclock_recov_lim")
2938     .add_see_also("osd_op_queue_mclock_scrub_res")
2939     .add_see_also("osd_op_queue_mclock_scrub_wgt")
2940     .add_see_also("osd_op_queue_mclock_scrub_lim")
2941     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
2942
2943     Option("osd_op_queue_mclock_osd_rep_op_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2944     .set_default(0.0)
2945     .set_description("mclock limit of osd replication operation requests and replies")
2946     .set_long_description("mclock limit of osd sub-operation requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the limit")
2947     .add_see_also("osd_op_queue")
2948     .add_see_also("osd_op_queue_mclock_client_op_res")
2949     .add_see_also("osd_op_queue_mclock_client_op_wgt")
2950     .add_see_also("osd_op_queue_mclock_client_op_lim")
2951     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
2952     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
2953     .add_see_also("osd_op_queue_mclock_snap_res")
2954     .add_see_also("osd_op_queue_mclock_snap_wgt")
2955     .add_see_also("osd_op_queue_mclock_snap_lim")
2956     .add_see_also("osd_op_queue_mclock_recov_res")
2957     .add_see_also("osd_op_queue_mclock_recov_wgt")
2958     .add_see_also("osd_op_queue_mclock_recov_lim")
2959     .add_see_also("osd_op_queue_mclock_scrub_res")
2960     .add_see_also("osd_op_queue_mclock_scrub_wgt")
2961     .add_see_also("osd_op_queue_mclock_scrub_lim")
2962     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
2963
2964     Option("osd_op_queue_mclock_snap_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2965     .set_default(0.0)
2966     .set_description("mclock reservation of snaptrim requests")
2967     .set_long_description("mclock reservation of snaptrim requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the reservation")
2968     .add_see_also("osd_op_queue")
2969     .add_see_also("osd_op_queue_mclock_client_op_res")
2970     .add_see_also("osd_op_queue_mclock_client_op_wgt")
2971     .add_see_also("osd_op_queue_mclock_client_op_lim")
2972     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
2973     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
2974     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
2975     .add_see_also("osd_op_queue_mclock_snap_wgt")
2976     .add_see_also("osd_op_queue_mclock_snap_lim")
2977     .add_see_also("osd_op_queue_mclock_recov_res")
2978     .add_see_also("osd_op_queue_mclock_recov_wgt")
2979     .add_see_also("osd_op_queue_mclock_recov_lim")
2980     .add_see_also("osd_op_queue_mclock_scrub_res")
2981     .add_see_also("osd_op_queue_mclock_scrub_wgt")
2982     .add_see_also("osd_op_queue_mclock_scrub_lim")
2983     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
2984
2985     Option("osd_op_queue_mclock_snap_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2986     .set_default(1.0)
2987     .set_description("mclock weight of snaptrim requests")
2988     .set_long_description("mclock weight of snaptrim requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the weight")
2989     .add_see_also("osd_op_queue")
2990     .add_see_also("osd_op_queue_mclock_client_op_res")
2991     .add_see_also("osd_op_queue_mclock_client_op_wgt")
2992     .add_see_also("osd_op_queue_mclock_client_op_lim")
2993     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
2994     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
2995     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
2996     .add_see_also("osd_op_queue_mclock_snap_res")
2997     .add_see_also("osd_op_queue_mclock_snap_lim")
2998     .add_see_also("osd_op_queue_mclock_recov_res")
2999     .add_see_also("osd_op_queue_mclock_recov_wgt")
3000     .add_see_also("osd_op_queue_mclock_recov_lim")
3001     .add_see_also("osd_op_queue_mclock_scrub_res")
3002     .add_see_also("osd_op_queue_mclock_scrub_wgt")
3003     .add_see_also("osd_op_queue_mclock_scrub_lim")
3004     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
3005
3006     Option("osd_op_queue_mclock_snap_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3007     .set_default(0.001)
3008     .set_description("")
3009     .set_description("mclock limit of snaptrim requests")
3010     .set_long_description("mclock limit of snaptrim requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the limit")
3011     .add_see_also("osd_op_queue_mclock_client_op_res")
3012     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3013     .add_see_also("osd_op_queue_mclock_client_op_lim")
3014     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3015     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3016     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3017     .add_see_also("osd_op_queue_mclock_snap_res")
3018     .add_see_also("osd_op_queue_mclock_snap_wgt")
3019     .add_see_also("osd_op_queue_mclock_recov_res")
3020     .add_see_also("osd_op_queue_mclock_recov_wgt")
3021     .add_see_also("osd_op_queue_mclock_recov_lim")
3022     .add_see_also("osd_op_queue_mclock_scrub_res")
3023     .add_see_also("osd_op_queue_mclock_scrub_wgt")
3024     .add_see_also("osd_op_queue_mclock_scrub_lim")
3025     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
3026
3027     Option("osd_op_queue_mclock_recov_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3028     .set_default(0.0)
3029     .set_description("mclock reservation of recovery requests")
3030     .set_long_description("mclock reservation of recovery requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the reservation")
3031     .add_see_also("osd_op_queue")
3032     .add_see_also("osd_op_queue_mclock_client_op_res")
3033     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3034     .add_see_also("osd_op_queue_mclock_client_op_lim")
3035     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3036     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3037     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3038     .add_see_also("osd_op_queue_mclock_snap_res")
3039     .add_see_also("osd_op_queue_mclock_snap_wgt")
3040     .add_see_also("osd_op_queue_mclock_snap_lim")
3041     .add_see_also("osd_op_queue_mclock_recov_wgt")
3042     .add_see_also("osd_op_queue_mclock_recov_lim")
3043     .add_see_also("osd_op_queue_mclock_scrub_res")
3044     .add_see_also("osd_op_queue_mclock_scrub_wgt")
3045     .add_see_also("osd_op_queue_mclock_scrub_lim")
3046     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
3047
3048     Option("osd_op_queue_mclock_recov_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3049     .set_default(1.0)
3050     .set_description("mclock weight of recovery requests")
3051     .set_long_description("mclock weight of recovery requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the weight")
3052     .add_see_also("osd_op_queue")
3053     .add_see_also("osd_op_queue_mclock_client_op_res")
3054     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3055     .add_see_also("osd_op_queue_mclock_client_op_lim")
3056     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3057     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3058     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3059     .add_see_also("osd_op_queue_mclock_snap_res")
3060     .add_see_also("osd_op_queue_mclock_snap_wgt")
3061     .add_see_also("osd_op_queue_mclock_snap_lim")
3062     .add_see_also("osd_op_queue_mclock_recov_res")
3063     .add_see_also("osd_op_queue_mclock_recov_lim")
3064     .add_see_also("osd_op_queue_mclock_scrub_res")
3065     .add_see_also("osd_op_queue_mclock_scrub_wgt")
3066     .add_see_also("osd_op_queue_mclock_scrub_lim")
3067     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
3068
3069     Option("osd_op_queue_mclock_recov_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3070     .set_default(0.001)
3071     .set_description("mclock limit of recovery requests")
3072     .set_long_description("mclock limit of recovery requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the limit")
3073     .add_see_also("osd_op_queue")
3074     .add_see_also("osd_op_queue_mclock_client_op_res")
3075     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3076     .add_see_also("osd_op_queue_mclock_client_op_lim")
3077     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3078     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3079     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3080     .add_see_also("osd_op_queue_mclock_snap_res")
3081     .add_see_also("osd_op_queue_mclock_snap_wgt")
3082     .add_see_also("osd_op_queue_mclock_snap_lim")
3083     .add_see_also("osd_op_queue_mclock_recov_res")
3084     .add_see_also("osd_op_queue_mclock_recov_wgt")
3085     .add_see_also("osd_op_queue_mclock_scrub_res")
3086     .add_see_also("osd_op_queue_mclock_scrub_wgt")
3087     .add_see_also("osd_op_queue_mclock_scrub_lim")
3088     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
3089
3090     Option("osd_op_queue_mclock_scrub_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3091     .set_default(0.0)
3092     .set_description("mclock reservation of scrub requests")
3093     .set_long_description("mclock reservation of scrub requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the reservation")
3094     .add_see_also("osd_op_queue")
3095     .add_see_also("osd_op_queue_mclock_client_op_res")
3096     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3097     .add_see_also("osd_op_queue_mclock_client_op_lim")
3098     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3099     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3100     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3101     .add_see_also("osd_op_queue_mclock_snap_res")
3102     .add_see_also("osd_op_queue_mclock_snap_wgt")
3103     .add_see_also("osd_op_queue_mclock_snap_lim")
3104     .add_see_also("osd_op_queue_mclock_recov_res")
3105     .add_see_also("osd_op_queue_mclock_recov_wgt")
3106     .add_see_also("osd_op_queue_mclock_recov_lim")
3107     .add_see_also("osd_op_queue_mclock_scrub_wgt")
3108     .add_see_also("osd_op_queue_mclock_scrub_lim")
3109     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
3110
3111     Option("osd_op_queue_mclock_scrub_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3112     .set_default(1.0)
3113     .set_description("mclock weight of scrub requests")
3114     .set_long_description("mclock weight of scrub requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the weight")
3115     .add_see_also("osd_op_queue")
3116     .add_see_also("osd_op_queue_mclock_client_op_res")
3117     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3118     .add_see_also("osd_op_queue_mclock_client_op_lim")
3119     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3120     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3121     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3122     .add_see_also("osd_op_queue_mclock_snap_res")
3123     .add_see_also("osd_op_queue_mclock_snap_wgt")
3124     .add_see_also("osd_op_queue_mclock_snap_lim")
3125     .add_see_also("osd_op_queue_mclock_recov_res")
3126     .add_see_also("osd_op_queue_mclock_recov_wgt")
3127     .add_see_also("osd_op_queue_mclock_recov_lim")
3128     .add_see_also("osd_op_queue_mclock_scrub_res")
3129     .add_see_also("osd_op_queue_mclock_scrub_lim")
3130     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
3131
3132     Option("osd_op_queue_mclock_scrub_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3133     .set_default(0.001)
3134     .set_description("mclock weight of limit requests")
3135     .set_long_description("mclock weight of limit requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the limit")
3136     .add_see_also("osd_op_queue")
3137     .add_see_also("osd_op_queue_mclock_client_op_res")
3138     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3139     .add_see_also("osd_op_queue_mclock_client_op_lim")
3140     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3141     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3142     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3143     .add_see_also("osd_op_queue_mclock_snap_res")
3144     .add_see_also("osd_op_queue_mclock_snap_wgt")
3145     .add_see_also("osd_op_queue_mclock_snap_lim")
3146     .add_see_also("osd_op_queue_mclock_recov_res")
3147     .add_see_also("osd_op_queue_mclock_recov_wgt")
3148     .add_see_also("osd_op_queue_mclock_recov_lim")
3149     .add_see_also("osd_op_queue_mclock_scrub_res")
3150     .add_see_also("osd_op_queue_mclock_scrub_wgt")
3151     .add_see_also("osd_op_queue_mclock_anticipation_timeout"),
3152
3153     Option("osd_op_queue_mclock_anticipation_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3154     .set_default(0.0)
3155     .set_description("mclock anticipation timeout in seconds")
3156     .set_long_description("the amount of time that mclock waits until the unused resource is forfeited")
3157     .add_see_also("osd_op_queue")
3158     .add_see_also("osd_op_queue_mclock_client_op_res")
3159     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3160     .add_see_also("osd_op_queue_mclock_client_op_lim")
3161     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3162     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3163     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3164     .add_see_also("osd_op_queue_mclock_snap_res")
3165     .add_see_also("osd_op_queue_mclock_snap_wgt")
3166     .add_see_also("osd_op_queue_mclock_snap_lim")
3167     .add_see_also("osd_op_queue_mclock_recov_res")
3168     .add_see_also("osd_op_queue_mclock_recov_wgt")
3169     .add_see_also("osd_op_queue_mclock_recov_lim")
3170     .add_see_also("osd_op_queue_mclock_scrub_res")
3171     .add_see_also("osd_op_queue_mclock_scrub_wgt")
3172     .add_see_also("osd_op_queue_mclock_scrub_lim"),
3173
3174     Option("osd_op_queue_mclock_pg_delete_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3175     .set_default(0.0)
3176     .set_description("mclock reservation of pg delete work")
3177     .set_long_description("mclock reservation of pg delete work when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the reservation")
3178     .add_see_also("osd_op_queue")
3179     .add_see_also("osd_op_queue_mclock_client_op_res")
3180     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3181     .add_see_also("osd_op_queue_mclock_client_op_lim")
3182     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3183     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3184     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3185     .add_see_also("osd_op_queue_mclock_snap_res")
3186     .add_see_also("osd_op_queue_mclock_snap_wgt")
3187     .add_see_also("osd_op_queue_mclock_snap_lim")
3188     .add_see_also("osd_op_queue_mclock_recov_res")
3189     .add_see_also("osd_op_queue_mclock_recov_wgt")
3190     .add_see_also("osd_op_queue_mclock_recov_lim")
3191     .add_see_also("osd_op_queue_mclock_scrub_wgt")
3192     .add_see_also("osd_op_queue_mclock_scrub_lim"),
3193
3194     Option("osd_op_queue_mclock_pg_delete_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3195     .set_default(1.0)
3196     .set_description("mclock weight of pg delete work")
3197     .set_long_description("mclock weight of pg delete work when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the weight")
3198     .add_see_also("osd_op_queue")
3199     .add_see_also("osd_op_queue_mclock_client_op_res")
3200     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3201     .add_see_also("osd_op_queue_mclock_client_op_lim")
3202     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3203     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3204     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3205     .add_see_also("osd_op_queue_mclock_snap_res")
3206     .add_see_also("osd_op_queue_mclock_snap_wgt")
3207     .add_see_also("osd_op_queue_mclock_snap_lim")
3208     .add_see_also("osd_op_queue_mclock_recov_res")
3209     .add_see_also("osd_op_queue_mclock_recov_wgt")
3210     .add_see_also("osd_op_queue_mclock_recov_lim")
3211     .add_see_also("osd_op_queue_mclock_scrub_res")
3212     .add_see_also("osd_op_queue_mclock_scrub_lim"),
3213
3214     Option("osd_op_queue_mclock_pg_delete_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3215     .set_default(0.001)
3216     .set_description("mclock weight of pg delete work limit requests")
3217     .set_long_description("mclock weight of limit pg delete work when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the limit")
3218     .add_see_also("osd_op_queue")
3219     .add_see_also("osd_op_queue_mclock_client_op_res")
3220     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3221     .add_see_also("osd_op_queue_mclock_client_op_lim")
3222     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3223     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3224     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3225     .add_see_also("osd_op_queue_mclock_snap_res")
3226     .add_see_also("osd_op_queue_mclock_snap_wgt")
3227     .add_see_also("osd_op_queue_mclock_snap_lim")
3228     .add_see_also("osd_op_queue_mclock_recov_res")
3229     .add_see_also("osd_op_queue_mclock_recov_wgt")
3230     .add_see_also("osd_op_queue_mclock_recov_lim")
3231     .add_see_also("osd_op_queue_mclock_scrub_res")
3232     .add_see_also("osd_op_queue_mclock_scrub_wgt"),
3233
3234     Option("osd_op_queue_mclock_peering_event_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3235     .set_default(0.0)
3236     .set_description("mclock reservation of peering events")
3237     .set_long_description("mclock reservation of scrub requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the reservation")
3238     .add_see_also("osd_op_queue")
3239     .add_see_also("osd_op_queue_mclock_client_op_res")
3240     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3241     .add_see_also("osd_op_queue_mclock_client_op_lim")
3242     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3243     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3244     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3245     .add_see_also("osd_op_queue_mclock_snap_res")
3246     .add_see_also("osd_op_queue_mclock_snap_wgt")
3247     .add_see_also("osd_op_queue_mclock_snap_lim")
3248     .add_see_also("osd_op_queue_mclock_recov_res")
3249     .add_see_also("osd_op_queue_mclock_recov_wgt")
3250     .add_see_also("osd_op_queue_mclock_recov_lim")
3251     .add_see_also("osd_op_queue_mclock_scrub_wgt")
3252     .add_see_also("osd_op_queue_mclock_scrub_lim"),
3253
3254     Option("osd_op_queue_mclock_peering_event_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3255     .set_default(1.0)
3256     .set_description("mclock weight of peering events")
3257     .set_long_description("mclock weight of scrub requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the weight")
3258     .add_see_also("osd_op_queue")
3259     .add_see_also("osd_op_queue_mclock_client_op_res")
3260     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3261     .add_see_also("osd_op_queue_mclock_client_op_lim")
3262     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3263     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3264     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3265     .add_see_also("osd_op_queue_mclock_snap_res")
3266     .add_see_also("osd_op_queue_mclock_snap_wgt")
3267     .add_see_also("osd_op_queue_mclock_snap_lim")
3268     .add_see_also("osd_op_queue_mclock_recov_res")
3269     .add_see_also("osd_op_queue_mclock_recov_wgt")
3270     .add_see_also("osd_op_queue_mclock_recov_lim")
3271     .add_see_also("osd_op_queue_mclock_scrub_res")
3272     .add_see_also("osd_op_queue_mclock_scrub_lim"),
3273
3274     Option("osd_op_queue_mclock_peering_event_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3275     .set_default(0.001)
3276     .set_description("mclock weight of limit peering events")
3277     .set_long_description("mclock weight of limit requests when osd_op_queue is either 'mclock_opclass' or 'mclock_client'; higher values increase the limit")
3278     .add_see_also("osd_op_queue")
3279     .add_see_also("osd_op_queue_mclock_client_op_res")
3280     .add_see_also("osd_op_queue_mclock_client_op_wgt")
3281     .add_see_also("osd_op_queue_mclock_client_op_lim")
3282     .add_see_also("osd_op_queue_mclock_osd_rep_op_res")
3283     .add_see_also("osd_op_queue_mclock_osd_rep_op_wgt")
3284     .add_see_also("osd_op_queue_mclock_osd_rep_op_lim")
3285     .add_see_also("osd_op_queue_mclock_snap_res")
3286     .add_see_also("osd_op_queue_mclock_snap_wgt")
3287     .add_see_also("osd_op_queue_mclock_snap_lim")
3288     .add_see_also("osd_op_queue_mclock_recov_res")
3289     .add_see_also("osd_op_queue_mclock_recov_wgt")
3290     .add_see_also("osd_op_queue_mclock_recov_lim")
3291     .add_see_also("osd_op_queue_mclock_scrub_res")
3292     .add_see_also("osd_op_queue_mclock_scrub_wgt"),
3293
3294     Option("osd_ignore_stale_divergent_priors", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3295     .set_default(false)
3296     .set_description(""),
3297
3298     Option("osd_read_ec_check_for_errors", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3299     .set_default(false)
3300     .set_description(""),
3301
3302     Option("osd_recover_clone_overlap_limit", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3303     .set_default(10)
3304     .set_description(""),
3305
3306     Option("osd_debug_feed_pullee", Option::TYPE_INT, Option::LEVEL_DEV)
3307     .set_default(-1)
3308     .set_description("Feed a pullee, and force primary to pull "
3309                      "a currently missing object from it"),
3310
3311     Option("osd_backfill_scan_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3312     .set_default(64)
3313     .set_description(""),
3314
3315     Option("osd_backfill_scan_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3316     .set_default(512)
3317     .set_description(""),
3318
3319     Option("osd_op_thread_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3320     .set_default(15)
3321     .set_description(""),
3322
3323     Option("osd_op_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3324     .set_default(150)
3325     .set_description(""),
3326
3327     Option("osd_recovery_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3328     .set_default(0)
3329     .set_description("Time in seconds to sleep before next recovery or backfill op"),
3330
3331     Option("osd_recovery_sleep_hdd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3332     .set_default(0.1)
3333     .set_description("Time in seconds to sleep before next recovery or backfill op for HDDs"),
3334
3335     Option("osd_recovery_sleep_ssd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3336     .set_default(0)
3337     .set_description("Time in seconds to sleep before next recovery or backfill op for SSDs")
3338     .add_see_also("osd_recovery_sleep"),
3339
3340     Option("osd_recovery_sleep_hybrid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3341     .set_default(0.025)
3342     .set_description("Time in seconds to sleep before next recovery or backfill op when data is on HDD and journal is on SSD")
3343     .add_see_also("osd_recovery_sleep"),
3344
3345     Option("osd_snap_trim_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3346     .set_default(0)
3347     .set_description("Time in seconds to sleep before next snap trim (overrides values below)"),
3348
3349     Option("osd_snap_trim_sleep_hdd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3350     .set_default(5)
3351     .set_description("Time in seconds to sleep before next snap trim for HDDs"),
3352
3353     Option("osd_snap_trim_sleep_ssd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3354     .set_default(0)
3355     .set_description("Time in seconds to sleep before next snap trim for SSDs"),
3356
3357     Option("osd_snap_trim_sleep_hybrid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3358     .set_default(2)
3359     .set_description("Time in seconds to sleep before next snap trim when data is on HDD and journal is on SSD"),
3360
3361     Option("osd_scrub_invalid_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3362     .set_default(true)
3363     .set_description(""),
3364
3365     Option("osd_command_thread_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3366     .set_default(10_min)
3367     .set_description(""),
3368
3369     Option("osd_command_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3370     .set_default(15_min)
3371     .set_description(""),
3372
3373     Option("osd_heartbeat_interval", Option::TYPE_INT, Option::LEVEL_DEV)
3374     .set_default(6)
3375     .set_min_max(1, 60)
3376     .set_description("Interval (in seconds) between peer pings"),
3377
3378     Option("osd_heartbeat_grace", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3379     .set_default(20)
3380     .set_description(""),
3381
3382     Option("osd_heartbeat_stale", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3383     .set_default(600)
3384     .set_description("Interval (in seconds) we mark an unresponsive heartbeat peer as stale.")
3385     .set_long_description("Automatically mark unresponsive heartbeat sessions as stale and tear them down. "
3386                           "The primary benefit is that OSD doesn't need to keep a flood of "
3387                           "blocked heartbeat messages around in memory."),
3388
3389     Option("osd_heartbeat_min_peers", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3390     .set_default(10)
3391     .set_description(""),
3392
3393     Option("osd_heartbeat_use_min_delay_socket", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3394     .set_default(false)
3395     .set_description(""),
3396
3397     Option("osd_heartbeat_min_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3398     .set_default(2000)
3399     .set_description("Minimum heartbeat packet size in bytes. Will add dummy payload if heartbeat packet is smaller than this."),
3400
3401     Option("osd_pg_max_concurrent_snap_trims", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3402     .set_default(2)
3403     .set_description(""),
3404
3405     Option("osd_max_trimming_pgs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3406     .set_default(2)
3407     .set_description(""),
3408
3409     Option("osd_heartbeat_min_healthy_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3410     .set_default(.33)
3411     .set_description(""),
3412
3413     Option("osd_mon_heartbeat_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3414     .set_default(30)
3415     .set_description(""),
3416
3417     Option("osd_mon_heartbeat_stat_stale", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3418     .set_default(1_hr)
3419     .set_description("Stop reporting on heartbeat ping times not updated for this many seconds.")
3420     .set_long_description("Stop reporting on old heartbeat information unless this is set to zero"),
3421
3422     Option("osd_mon_report_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3423     .set_default(5)
3424     .set_description("Frequency of OSD reports to mon for peer failures, fullness status changes"),
3425
3426     Option("osd_mon_report_max_in_flight", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3427     .set_default(2)
3428     .set_description(""),
3429
3430     Option("osd_beacon_report_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3431     .set_default(300)
3432     .set_description(""),
3433
3434     Option("osd_pg_stat_report_interval_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3435     .set_default(500)
3436     .set_description(""),
3437
3438     Option("osd_mon_ack_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3439     .set_default(30.0)
3440     .set_description(""),
3441
3442     Option("osd_stats_ack_timeout_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3443     .set_default(2.0)
3444     .set_description(""),
3445
3446     Option("osd_stats_ack_timeout_decay", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3447     .set_default(.9)
3448     .set_description(""),
3449
3450     Option("osd_max_snap_prune_intervals_per_epoch", Option::TYPE_UINT, Option::LEVEL_DEV)
3451     .set_default(512)
3452     .set_description("Max number of snap intervals to report to mgr in pg_stat_t"),
3453
3454     Option("osd_default_data_pool_replay_window", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3455     .set_default(45)
3456     .set_description(""),
3457
3458     Option("osd_auto_mark_unfound_lost", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3459     .set_default(false)
3460     .set_description(""),
3461
3462     Option("osd_recovery_delay_start", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3463     .set_default(0)
3464     .set_description(""),
3465
3466     Option("osd_recovery_max_active", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3467     .set_default(3)
3468     .set_description(""),
3469
3470     Option("osd_recovery_max_single_start", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3471     .set_default(1)
3472     .set_description(""),
3473
3474     Option("osd_recovery_max_chunk", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3475     .set_default(8_M)
3476     .set_description(""),
3477
3478     Option("osd_recovery_max_omap_entries_per_chunk", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3479     .set_default(8096)
3480     .set_description(""),
3481
3482     Option("osd_copyfrom_max_chunk", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3483     .set_default(8_M)
3484     .set_description(""),
3485
3486     Option("osd_push_per_object_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3487     .set_default(1000)
3488     .set_description(""),
3489
3490     Option("osd_max_push_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3491     .set_default(8<<20)
3492     .set_description(""),
3493
3494     Option("osd_max_push_objects", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3495     .set_default(10)
3496     .set_description(""),
3497
3498     Option("osd_max_scrubs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3499     .set_default(1)
3500     .set_description("Maximum concurrent scrubs on a single OSD"),
3501
3502     Option("osd_scrub_during_recovery", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3503     .set_default(false)
3504     .set_description("Allow scrubbing when PGs on the OSD are undergoing recovery"),
3505
3506     Option("osd_repair_during_recovery", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3507     .set_default(false)
3508     .set_description("Allow requested repairing when PGs on the OSD are undergoing recovery"),
3509
3510     Option("osd_scrub_begin_hour", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3511     .set_default(0)
3512     .set_description("Restrict scrubbing to this hour of the day or later")
3513     .add_see_also("osd_scrub_end_hour"),
3514
3515     Option("osd_scrub_end_hour", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3516     .set_default(24)
3517     .set_description("Restrict scrubbing to hours of the day earlier than this")
3518     .add_see_also("osd_scrub_begin_hour"),
3519
3520     Option("osd_scrub_begin_week_day", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3521     .set_default(0)
3522     .set_description("Restrict scrubbing to this day of the week or later")
3523     .set_long_description("0 or 7 = Sunday, 1 = Monday, etc.")
3524     .add_see_also("osd_scrub_end_week_day"),
3525
3526     Option("osd_scrub_end_week_day", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3527     .set_default(7)
3528     .set_description("Restrict scrubbing to days of the week earlier than this")
3529     .set_long_description("0 or 7 = Sunday, 1 = Monday, etc.")
3530     .add_see_also("osd_scrub_begin_week_day"),
3531
3532     Option("osd_scrub_load_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3533     .set_default(0.5)
3534     .set_description("Allow scrubbing when system load divided by number of CPUs is below this value"),
3535
3536     Option("osd_scrub_min_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3537     .set_default(1_day)
3538     .set_description("Scrub each PG no more often than this interval")
3539     .add_see_also("osd_scrub_max_interval"),
3540
3541     Option("osd_scrub_max_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3542     .set_default(7_day)
3543     .set_description("Scrub each PG no less often than this interval")
3544     .add_see_also("osd_scrub_min_interval"),
3545
3546     Option("osd_scrub_interval_randomize_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3547     .set_default(0.5)
3548     .set_description("Ratio of scrub interval to randomly vary")
3549     .set_long_description("This prevents a scrub 'stampede' by randomly varying the scrub intervals so that they are soon uniformly distributed over the week")
3550     .add_see_also("osd_scrub_min_interval"),
3551
3552     Option("osd_scrub_backoff_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3553     .set_default(.66)
3554     .set_long_description("This is the precentage of ticks that do NOT schedule scrubs, 66% means that 1 out of 3 ticks will schedule scrubs")
3555     .set_description("Backoff ratio for scheduling scrubs"),
3556
3557     Option("osd_scrub_chunk_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3558     .set_default(5)
3559     .set_description("Minimum number of objects to scrub in a single chunk")
3560     .add_see_also("osd_scrub_chunk_max"),
3561
3562     Option("osd_scrub_chunk_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3563     .set_default(25)
3564     .set_description("Maximum number of objects to scrub in a single chunk")
3565     .add_see_also("osd_scrub_chunk_min"),
3566
3567     Option("osd_scrub_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3568     .set_default(0)
3569     .set_description("Duration to inject a delay during scrubbing"),
3570
3571     Option("osd_scrub_auto_repair", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3572     .set_default(false)
3573     .set_description("Automatically repair damaged objects detected during scrub"),
3574
3575     Option("osd_scrub_auto_repair_num_errors", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3576     .set_default(5)
3577     .set_description("Maximum number of detected errors to automatically repair")
3578     .add_see_also("osd_scrub_auto_repair"),
3579
3580     Option("osd_scrub_max_preemptions", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3581     .set_default(5)
3582     .set_description("Set the maximum number of times we will preempt a deep scrub due to a client operation before blocking client IO to complete the scrub"),
3583
3584     Option("osd_deep_scrub_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3585     .set_default(7_day)
3586     .set_description("Deep scrub each PG (i.e., verify data checksums) at least this often"),
3587
3588     Option("osd_deep_scrub_randomize_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3589     .set_default(0.15)
3590     .set_description("Scrubs will randomly become deep scrubs at this rate (0.15 -> 15% of scrubs are deep)")
3591     .set_long_description("This prevents a deep scrub 'stampede' by spreading deep scrubs so they are uniformly distributed over the week"),
3592
3593     Option("osd_deep_scrub_stride", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3594     .set_default(512_K)
3595     .set_description("Number of bytes to read from an object at a time during deep scrub"),
3596
3597     Option("osd_deep_scrub_keys", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3598     .set_default(1024)
3599     .set_description("Number of keys to read from an object at a time during deep scrub"),
3600
3601     Option("osd_deep_scrub_update_digest_min_age", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3602     .set_default(2_hr)
3603     .set_description("Update overall object digest only if object was last modified longer ago than this"),
3604
3605     Option("osd_deep_scrub_large_omap_object_key_threshold", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3606     .set_default(200000)
3607     .set_description("Warn when we encounter an object with more omap keys than this")
3608     .add_service("osd")
3609     .add_see_also("osd_deep_scrub_large_omap_object_value_sum_threshold"),
3610
3611     Option("osd_deep_scrub_large_omap_object_value_sum_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3612     .set_default(1_G)
3613     .set_description("Warn when we encounter an object with more omap key bytes than this")
3614     .add_service("osd")
3615     .add_see_also("osd_deep_scrub_large_omap_object_key_threshold"),
3616
3617     Option("osd_class_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3618     .set_default(CEPH_LIBDIR "/rados-classes")
3619     .set_description(""),
3620
3621     Option("osd_open_classes_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3622     .set_default(true)
3623     .set_description(""),
3624
3625     Option("osd_class_load_list", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3626     .set_default("cephfs hello journal lock log numops " "otp rbd refcount rgw timeindex user version cas")
3627     .set_description(""),
3628
3629     Option("osd_class_default_list", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3630     .set_default("cephfs hello journal lock log numops " "otp rbd refcount rgw timeindex user version cas")
3631     .set_description(""),
3632
3633     Option("osd_check_for_log_corruption", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3634     .set_default(false)
3635     .set_description(""),
3636
3637     Option("osd_use_stale_snap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3638     .set_default(false)
3639     .set_description(""),
3640
3641     Option("osd_rollback_to_cluster_snap", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3642     .set_default("")
3643     .set_description(""),
3644
3645     Option("osd_default_notify_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3646     .set_default(30)
3647     .set_description(""),
3648
3649     Option("osd_kill_backfill_at", Option::TYPE_INT, Option::LEVEL_DEV)
3650     .set_default(0)
3651     .set_description(""),
3652
3653     Option("osd_pg_epoch_persisted_max_stale", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3654     .set_default(40)
3655     .set_description(""),
3656
3657     Option("osd_min_pg_log_entries", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3658     .set_default(3000)
3659     .set_description("minimum number of entries to maintain in the PG log")
3660     .add_service("osd")
3661     .add_see_also("osd_max_pg_log_entries")
3662     .add_see_also("osd_pg_log_dups_tracked"),
3663
3664     Option("osd_max_pg_log_entries", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3665     .set_default(3000)
3666     .set_description("maximum number of entries to maintain in the PG log when degraded before we trim")
3667     .add_service("osd")
3668     .add_see_also("osd_min_pg_log_entries")
3669     .add_see_also("osd_pg_log_dups_tracked"),
3670
3671     Option("osd_pg_log_dups_tracked", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3672     .set_default(3000)
3673     .set_description("how many versions back to track in order to detect duplicate ops; this is combined with both the regular pg log entries and additional minimal dup detection entries")
3674     .add_service("osd")
3675     .add_see_also("osd_min_pg_log_entries")
3676     .add_see_also("osd_max_pg_log_entries"),
3677
3678     Option("osd_force_recovery_pg_log_entries_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3679     .set_default(1.3)
3680     .set_description(""),
3681
3682     Option("osd_pg_log_trim_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3683     .set_default(100)
3684     .set_description(""),
3685
3686     Option("osd_force_auth_primary_missing_objects", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3687     .set_default(100)
3688     .set_description("Approximate missing objects above which to force auth_log_shard to be primary temporarily"),
3689
3690     Option("osd_async_recovery_min_cost", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3691     .set_default(100)
3692     .set_description("A mixture measure of number of current log entries difference "
3693                      "and historical missing objects,  above which we switch to use "
3694                      "asynchronous recovery when appropriate"),
3695
3696     Option("osd_max_pg_per_osd_hard_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3697     .set_default(3)
3698     .set_min(1)
3699     .set_description("Maximum number of PG per OSD, a factor of 'mon_max_pg_per_osd'")
3700     .set_long_description("OSD will refuse to instantiate PG if the number of PG it serves exceeds this number.")
3701     .add_see_also("mon_max_pg_per_osd"),
3702
3703     Option("osd_pg_log_trim_max", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3704     .set_default(10000)
3705     .set_description("maximum number of entries to remove at once from the PG log")
3706     .add_service("osd")
3707     .add_see_also("osd_min_pg_log_entries")
3708     .add_see_also("osd_max_pg_log_entries"),
3709
3710     Option("osd_op_complaint_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3711     .set_default(30)
3712     .set_description(""),
3713
3714     Option("osd_command_max_records", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3715     .set_default(256)
3716     .set_description(""),
3717
3718     Option("osd_max_pg_blocked_by", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3719     .set_default(16)
3720     .set_description(""),
3721
3722     Option("osd_op_log_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3723     .set_default(5)
3724     .set_description(""),
3725
3726     Option("osd_verify_sparse_read_holes", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3727     .set_default(false)
3728     .set_description(""),
3729
3730     Option("osd_backoff_on_unfound", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3731     .set_default(true)
3732     .set_description(""),
3733
3734     Option("osd_backoff_on_degraded", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3735     .set_default(false)
3736     .set_description(""),
3737
3738     Option("osd_backoff_on_peering", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3739     .set_default(false)
3740     .set_description(""),
3741
3742     Option("osd_debug_shutdown", Option::TYPE_BOOL, Option::LEVEL_DEV)
3743     .set_default(false)
3744     .set_description("Turn up debug levels during shutdown"),
3745
3746     Option("osd_debug_crash_on_ignored_backoff", Option::TYPE_BOOL, Option::LEVEL_DEV)
3747     .set_default(false)
3748     .set_description(""),
3749
3750     Option("osd_debug_inject_dispatch_delay_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3751     .set_default(0)
3752     .set_description(""),
3753
3754     Option("osd_debug_inject_dispatch_delay_duration", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3755     .set_default(.1)
3756     .set_description(""),
3757
3758     Option("osd_debug_drop_ping_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3759     .set_default(0)
3760     .set_description(""),
3761
3762     Option("osd_debug_drop_ping_duration", Option::TYPE_INT, Option::LEVEL_DEV)
3763     .set_default(0)
3764     .set_description(""),
3765
3766     Option("osd_debug_op_order", Option::TYPE_BOOL, Option::LEVEL_DEV)
3767     .set_default(false)
3768     .set_description(""),
3769
3770     Option("osd_debug_verify_missing_on_start", Option::TYPE_BOOL, Option::LEVEL_DEV)
3771     .set_default(false)
3772     .set_description(""),
3773
3774     Option("osd_debug_verify_snaps", Option::TYPE_BOOL, Option::LEVEL_DEV)
3775     .set_default(false)
3776     .set_description(""),
3777
3778     Option("osd_debug_verify_stray_on_activate", Option::TYPE_BOOL, Option::LEVEL_DEV)
3779     .set_default(false)
3780     .set_description(""),
3781
3782     Option("osd_debug_skip_full_check_in_backfill_reservation", Option::TYPE_BOOL, Option::LEVEL_DEV)
3783     .set_default(false)
3784     .set_description(""),
3785
3786     Option("osd_debug_reject_backfill_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3787     .set_default(0)
3788     .set_description(""),
3789
3790     Option("osd_debug_inject_copyfrom_error", Option::TYPE_BOOL, Option::LEVEL_DEV)
3791     .set_default(false)
3792     .set_description(""),
3793
3794     Option("osd_debug_misdirected_ops", Option::TYPE_BOOL, Option::LEVEL_DEV)
3795     .set_default(false)
3796     .set_description(""),
3797
3798     Option("osd_debug_skip_full_check_in_recovery", Option::TYPE_BOOL, Option::LEVEL_DEV)
3799     .set_default(false)
3800     .set_description(""),
3801
3802     Option("osd_debug_random_push_read_error", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3803     .set_default(0)
3804     .set_description(""),
3805
3806     Option("osd_debug_verify_cached_snaps", Option::TYPE_BOOL, Option::LEVEL_DEV)
3807     .set_default(false)
3808     .set_description(""),
3809
3810     Option("osd_debug_deep_scrub_sleep", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3811     .set_default(0)
3812     .set_description("Inject an expensive sleep during deep scrub IO to make it easier to induce preemption"),
3813
3814     Option("osd_debug_no_acting_change", Option::TYPE_BOOL, Option::LEVEL_DEV)
3815     .set_default(false),
3816     Option("osd_debug_no_purge_strays", Option::TYPE_BOOL, Option::LEVEL_DEV)
3817     .set_default(false),
3818
3819     Option("osd_debug_pretend_recovery_active", Option::TYPE_BOOL, Option::LEVEL_DEV)
3820     .set_default(false)
3821     .set_description(""),
3822
3823     Option("osd_enable_op_tracker", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3824     .set_default(true)
3825     .set_description(""),
3826
3827     Option("osd_num_op_tracker_shard", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3828     .set_default(32)
3829     .set_description(""),
3830
3831     Option("osd_op_history_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3832     .set_default(20)
3833     .set_description(""),
3834
3835     Option("osd_op_history_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3836     .set_default(600)
3837     .set_description(""),
3838
3839     Option("osd_op_history_slow_op_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3840     .set_default(20)
3841     .set_description(""),
3842
3843     Option("osd_op_history_slow_op_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3844     .set_default(10.0)
3845     .set_description(""),
3846
3847     Option("osd_target_transaction_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3848     .set_default(30)
3849     .set_description(""),
3850
3851     Option("osd_delete_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3852     .set_default(0)
3853     .set_description("Time in seconds to sleep before next removal transaction (overrides values below)"),
3854
3855     Option("osd_delete_sleep_hdd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3856     .set_default(5)
3857     .set_description("Time in seconds to sleep before next removal transaction for HDDs"),
3858
3859     Option("osd_delete_sleep_ssd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3860     .set_default(0)
3861     .set_description("Time in seconds to sleep before next removal transaction for SSDs"),
3862
3863     Option("osd_delete_sleep_hybrid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3864     .set_default(2)
3865     .set_description("Time in seconds to sleep before next removal transaction when data is on HDD and journal is on SSD"),
3866
3867     Option("osd_failsafe_full_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3868     .set_default(.97)
3869     .set_description(""),
3870
3871     Option("osd_fast_fail_on_connection_refused", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3872     .set_default(true)
3873     .set_description(""),
3874
3875     Option("osd_pg_object_context_cache_count", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3876     .set_default(64)
3877     .set_description(""),
3878
3879     Option("osd_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3880     .set_default(false)
3881     .set_description(""),
3882
3883     Option("osd_function_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3884     .set_default(false)
3885     .set_description(""),
3886
3887     Option("osd_fast_info", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3888     .set_default(true)
3889     .set_description(""),
3890
3891     Option("osd_debug_pg_log_writeout", Option::TYPE_BOOL, Option::LEVEL_DEV)
3892     .set_default(false)
3893     .set_description(""),
3894
3895     Option("osd_loop_before_reset_tphandle", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3896     .set_default(64)
3897     .set_description(""),
3898
3899     Option("threadpool_default_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3900     .set_default(60)
3901     .set_description(""),
3902
3903     Option("threadpool_empty_queue_max_wait", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3904     .set_default(2)
3905     .set_description(""),
3906
3907     Option("leveldb_log_to_ceph_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3908     .set_default(true)
3909     .set_description(""),
3910
3911     Option("leveldb_write_buffer_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3912     .set_default(8_M)
3913     .set_description(""),
3914
3915     Option("leveldb_cache_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3916     .set_default(128_M)
3917     .set_description(""),
3918
3919     Option("leveldb_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3920     .set_default(0)
3921     .set_description(""),
3922
3923     Option("leveldb_bloom_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3924     .set_default(0)
3925     .set_description(""),
3926
3927     Option("leveldb_max_open_files", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3928     .set_default(0)
3929     .set_description(""),
3930
3931     Option("leveldb_compression", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3932     .set_default(true)
3933     .set_description(""),
3934
3935     Option("leveldb_paranoid", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3936     .set_default(false)
3937     .set_description(""),
3938
3939     Option("leveldb_log", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3940     .set_default("/dev/null")
3941     .set_description(""),
3942
3943     Option("leveldb_compact_on_mount", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3944     .set_default(false)
3945     .set_description(""),
3946
3947     Option("kinetic_host", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3948     .set_default("")
3949     .set_description(""),
3950
3951     Option("kinetic_port", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3952     .set_default(8123)
3953     .set_description(""),
3954
3955     Option("kinetic_user_id", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3956     .set_default(1)
3957     .set_description(""),
3958
3959     Option("kinetic_hmac_key", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3960     .set_default("asdfasdf")
3961     .set_description(""),
3962
3963     Option("kinetic_use_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3964     .set_default(false)
3965     .set_description(""),
3966
3967     Option("rocksdb_log_to_ceph_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3968     .set_default(true)
3969     .set_description(""),
3970
3971     Option("rocksdb_cache_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3972     .set_default(512_M)
3973     .set_flag(Option::FLAG_RUNTIME)
3974     .set_description(""),
3975
3976     Option("rocksdb_cache_row_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3977     .set_default(0)
3978     .set_description(""),
3979
3980     Option("rocksdb_cache_shard_bits", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3981     .set_default(4)
3982     .set_description(""),
3983
3984     Option("rocksdb_cache_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3985     .set_default("binned_lru")
3986     .set_description(""),
3987
3988     Option("rocksdb_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3989     .set_default(4_K)
3990     .set_description(""),
3991
3992     Option("rocksdb_perf", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3993     .set_default(false)
3994     .set_description(""),
3995
3996     Option("rocksdb_collect_compaction_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3997     .set_default(false)
3998     .set_description(""),
3999
4000     Option("rocksdb_collect_extended_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4001     .set_default(false)
4002     .set_description(""),
4003
4004     Option("rocksdb_collect_memory_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4005     .set_default(false)
4006     .set_description(""),
4007
4008     Option("rocksdb_enable_rmrange", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4009     .set_default(false)
4010     .set_description("Refer to github.com/facebook/rocksdb/wiki/DeleteRange-Implementation"),
4011
4012     Option("rocksdb_max_items_rmrange", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4013     .set_default(1024)
4014     .set_description("Delete Range will be called if number of keys exceeded, must enable rocksdb_enable_rmrange first")
4015     .add_see_also("rocksdb_enable_rmrange"),
4016
4017     Option("rocksdb_bloom_bits_per_key", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4018     .set_default(20)
4019     .set_description("Number of bits per key to use for RocksDB's bloom filters.")
4020     .set_long_description("RocksDB bloom filters can be used to quickly answer the question of whether or not a key may exist or definitely does not exist in a given RocksDB SST file without having to read all keys into memory.  Using a higher bit value decreases the likelihood of false positives at the expense of additional disk space and memory consumption when the filter is loaded into RAM.  The current default value of 20 was found to provide significant performance gains when getattr calls are made (such as during new object creation in bluestore) without significant memory overhead or cache pollution when combined with rocksdb partitioned index filters.  See: https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters for more information."),
4021
4022     Option("rocksdb_cache_index_and_filter_blocks", Option::TYPE_BOOL, Option::LEVEL_DEV)
4023     .set_default(true)
4024     .set_description("Whether to cache indices and filters in block cache")
4025     .set_long_description("By default RocksDB will load an SST file's index and bloom filters into memory when it is opened and remove them from memory when an SST file is closed.  Thus, memory consumption by indices and bloom filters is directly tied to the number of concurrent SST files allowed to be kept open.  This option instead stores cached indicies and filters in the block cache where they directly compete with other cached data.  By default we set this option to true to better account for and bound rocksdb memory usage and keep filters in memory even when an SST file is closed."),
4026
4027     Option("rocksdb_cache_index_and_filter_blocks_with_high_priority", Option::TYPE_BOOL, Option::LEVEL_DEV)
4028     .set_default(true)
4029     .set_description("Whether to cache indices and filters in the block cache with high priority")
4030     .set_long_description("A downside of setting rocksdb_cache_index_and_filter_blocks to true is that regular data can push indices and filters out of memory.  Setting this option to true means they are cached with higher priority than other data and should typically stay in the block cache."),
4031
4032     Option("rocksdb_pin_l0_filter_and_index_blocks_in_cache", Option::TYPE_BOOL, Option::LEVEL_DEV)
4033     .set_default(false)
4034     .set_description("Whether to pin Level 0 indices and bloom filters in the block cache")
4035     .set_long_description("A downside of setting rocksdb_cache_index_and_filter_blocks to true is that regular data can push indices and filters out of memory.  Setting this option to true means that level 0 SST files will always have their indices and filters pinned in the block cache."),
4036
4037     Option("rocksdb_index_type", Option::TYPE_STR, Option::LEVEL_DEV)
4038     .set_default("binary_search")
4039     .set_description("Type of index for SST files: binary_search, hash_search, two_level")
4040     .set_long_description("This option controls the table index type.  binary_search is a space efficient index block that is optimized for block-search-based index. hash_search may improve prefix lookup performance at the expense of higher disk and memory usage and potentially slower compactions.  two_level is an experimental index type that uses two binary search indexes and works in conjunction with partition filters.  See: http://rocksdb.org/blog/2017/05/12/partitioned-index-filter.html"),
4041
4042     Option("rocksdb_partition_filters", Option::TYPE_BOOL, Option::LEVEL_DEV)
4043     .set_default(false)
4044     .set_description("(experimental) partition SST index/filters into smaller blocks")
4045     .set_long_description("This is an experimental option for rocksdb that works in conjunction with two_level indices to avoid having to keep the entire filter/index in cache when cache_index_and_filter_blocks is true.  The idea is to keep a much smaller top-level index in heap/cache and then opportunistically cache the lower level indices.  See: https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters"),
4046
4047     Option("rocksdb_metadata_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4048     .set_default(4_K)
4049     .set_description("The block size for index partitions. (0 = rocksdb default)"),
4050
4051     Option("mon_rocksdb_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4052     .set_default("write_buffer_size=33554432,"
4053                  "compression=kNoCompression,"
4054                  "level_compaction_dynamic_level_bytes=true")
4055     .set_description(""),
4056
4057     Option("osd_client_op_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4058     .set_default(63)
4059     .set_description(""),
4060
4061     Option("osd_recovery_op_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4062     .set_default(3)
4063     .set_description("Priority to use for recovery operations if not specified for the pool"),
4064
4065     Option("osd_peering_op_priority", Option::TYPE_UINT, Option::LEVEL_DEV)
4066     .set_default(255)
4067     .set_description(""),
4068
4069     Option("osd_snap_trim_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4070     .set_default(5)
4071     .set_description(""),
4072
4073     Option("osd_snap_trim_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4074     .set_default(1<<20)
4075     .set_description(""),
4076
4077     Option("osd_pg_delete_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4078     .set_default(5)
4079     .set_description(""),
4080
4081     Option("osd_pg_delete_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4082     .set_default(1<<20)
4083     .set_description(""),
4084
4085     Option("osd_scrub_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4086     .set_default(5)
4087     .set_description("Priority for scrub operations in work queue"),
4088
4089     Option("osd_scrub_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4090     .set_default(50<<20)
4091     .set_description("Cost for scrub operations in work queue"),
4092
4093     Option("osd_requested_scrub_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4094     .set_default(120)
4095     .set_description(""),
4096
4097     Option("osd_recovery_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4098     .set_default(5)
4099     .set_description("Priority of recovery in the work queue")
4100     .set_long_description("Not related to a pool's recovery_priority"),
4101
4102     Option("osd_recovery_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4103     .set_default(20<<20)
4104     .set_description(""),
4105
4106     Option("osd_recovery_op_warn_multiple", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4107     .set_default(16)
4108     .set_description(""),
4109
4110     Option("osd_mon_shutdown_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4111     .set_default(5)
4112     .set_description(""),
4113
4114     Option("osd_shutdown_pgref_assert", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4115     .set_default(false)
4116     .set_description(""),
4117
4118     Option("osd_max_object_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4119     .set_default(128_M)
4120     .set_description(""),
4121
4122     Option("osd_max_object_name_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4123     .set_default(2048)
4124     .set_description(""),
4125
4126     Option("osd_max_object_namespace_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4127     .set_default(256)
4128     .set_description(""),
4129
4130     Option("osd_max_attr_name_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4131     .set_default(100)
4132     .set_description(""),
4133
4134     Option("osd_max_attr_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4135     .set_default(0)
4136     .set_description(""),
4137
4138     Option("osd_max_omap_entries_per_request", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4139     .set_default(131072)
4140     .set_description(""),
4141
4142     Option("osd_max_omap_bytes_per_request", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4143     .set_default(1_G)
4144     .set_description(""),
4145
4146     Option("osd_objectstore", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4147     .set_default("bluestore")
4148     .set_enum_allowed({"bluestore", "filestore", "memstore", "kstore"})
4149     .set_flag(Option::FLAG_CREATE)
4150     .set_description("backend type for an OSD (like filestore or bluestore)"),
4151
4152     Option("osd_objectstore_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4153     .set_default(false)
4154     .set_description(""),
4155
4156     Option("osd_objectstore_fuse", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4157     .set_default(false)
4158     .set_description(""),
4159
4160     Option("osd_bench_small_size_max_iops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4161     .set_default(100)
4162     .set_description(""),
4163
4164     Option("osd_bench_large_size_max_throughput", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4165     .set_default(100_M)
4166     .set_description(""),
4167
4168     Option("osd_bench_max_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4169     .set_default(64_M)
4170     .set_description(""),
4171
4172     Option("osd_bench_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4173     .set_default(30)
4174     .set_description(""),
4175
4176     Option("osd_blkin_trace_all", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4177     .set_default(false)
4178     .set_description(""),
4179
4180     Option("osdc_blkin_trace_all", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4181     .set_default(false)
4182     .set_description(""),
4183
4184     Option("osd_discard_disconnected_ops", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4185     .set_default(true)
4186     .set_description(""),
4187
4188     Option("osd_memory_target", Option::TYPE_SIZE, Option::LEVEL_BASIC)
4189     .set_default(4_G)
4190     .add_see_also("bluestore_cache_autotune")
4191     .set_description("When tcmalloc and cache autotuning is enabled, try to keep this many bytes mapped in memory."),
4192
4193     Option("osd_memory_target_cgroup_limit_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4194     .set_default(0.8)
4195     .set_min_max(0.0, 1.0)
4196     .add_see_also("osd_memory_target")
4197     .set_description("Set the default value for osd_memory_target to the cgroup memory limit (if set) times this value")
4198     .set_long_description("A value of 0 disables this feature."),
4199
4200     Option("osd_memory_base", Option::TYPE_SIZE, Option::LEVEL_DEV)
4201     .set_default(768_M)
4202     .add_see_also("bluestore_cache_autotune")
4203     .set_description("When tcmalloc and cache autotuning is enabled, estimate the minimum amount of memory in bytes the OSD will need."),
4204
4205     Option("osd_memory_expected_fragmentation", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4206     .set_default(0.15)
4207     .set_min_max(0.0, 1.0)
4208     .add_see_also("bluestore_cache_autotune")
4209     .set_description("When tcmalloc and cache autotuning is enabled, estimate the percent of memory fragmentation."),
4210
4211     Option("osd_memory_cache_min", Option::TYPE_SIZE, Option::LEVEL_DEV)
4212     .set_default(128_M)
4213     .add_see_also("bluestore_cache_autotune")
4214     .set_description("When tcmalloc and cache autotuning is enabled, set the minimum amount of memory used for caches."),
4215
4216     Option("osd_memory_cache_resize_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4217     .set_default(1)
4218     .add_see_also("bluestore_cache_autotune")
4219     .set_description("When tcmalloc and cache autotuning is enabled, wait this many seconds between resizing caches."),
4220
4221     Option("memstore_device_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4222     .set_default(1_G)
4223     .set_description(""),
4224
4225     Option("memstore_page_set", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4226     .set_default(false)
4227     .set_description(""),
4228
4229     Option("memstore_page_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4230     .set_default(64_K)
4231     .set_description(""),
4232
4233     Option("objectstore_blackhole", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4234     .set_default(false)
4235     .set_description(""),
4236
4237     // --------------------------
4238     // bluestore
4239
4240     Option("bdev_debug_inflight_ios", Option::TYPE_BOOL, Option::LEVEL_DEV)
4241     .set_default(false)
4242     .set_description(""),
4243
4244     Option("bdev_inject_crash", Option::TYPE_INT, Option::LEVEL_DEV)
4245     .set_default(0)
4246     .set_description(""),
4247
4248     Option("bdev_inject_crash_flush_delay", Option::TYPE_INT, Option::LEVEL_DEV)
4249     .set_default(2)
4250     .set_description(""),
4251
4252     Option("bdev_aio", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4253     .set_default(true)
4254     .set_description(""),
4255
4256     Option("bdev_aio_poll_ms", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4257     .set_default(250)
4258     .set_description(""),
4259
4260     Option("bdev_aio_max_queue_depth", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4261     .set_default(1024)
4262     .set_description(""),
4263
4264     Option("bdev_aio_reap_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4265     .set_default(16)
4266     .set_description(""),
4267
4268     Option("bdev_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4269     .set_default(4_K)
4270     .set_description(""),
4271
4272     Option("bdev_debug_aio", Option::TYPE_BOOL, Option::LEVEL_DEV)
4273     .set_default(false)
4274     .set_description(""),
4275
4276     Option("bdev_debug_aio_suicide_timeout", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4277     .set_default(60.0)
4278     .set_description(""),
4279
4280     Option("bdev_debug_aio_log_age", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4281     .set_default(5.0)
4282     .set_description(""),
4283
4284     Option("bdev_nvme_unbind_from_kernel", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4285     .set_default(false)
4286     .set_description(""),
4287
4288     Option("bdev_nvme_retry_count", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4289     .set_default(-1)
4290     .set_description(""),
4291
4292     Option("bdev_enable_discard", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4293     .set_default(false)
4294     .set_description(""),
4295
4296     Option("bdev_async_discard", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4297     .set_default(false)
4298     .set_description(""),
4299
4300     Option("bluefs_alloc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4301     .set_default(1_M)
4302     .set_description("Allocation unit size for DB and WAL devices"),
4303
4304     Option("bluefs_shared_alloc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4305     .set_default(64_K)
4306     .set_description("Allocation unit size for primary/shared device"),
4307
4308     Option("bluefs_max_prefetch", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4309     .set_default(1_M)
4310     .set_description(""),
4311
4312     Option("bluefs_min_log_runway", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4313     .set_default(1_M)
4314     .set_description(""),
4315
4316     Option("bluefs_max_log_runway", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4317     .set_default(4194304)
4318     .set_description(""),
4319
4320     Option("bluefs_log_compact_min_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4321     .set_default(5.0)
4322     .set_description(""),
4323
4324     Option("bluefs_log_compact_min_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4325     .set_default(16_M)
4326     .set_description(""),
4327
4328     Option("bluefs_min_flush_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4329     .set_default(512_K)
4330     .set_description(""),
4331
4332     Option("bluefs_compact_log_sync", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4333     .set_default(false)
4334     .set_description(""),
4335
4336     Option("bluefs_buffered_io", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4337     .set_default(true)
4338     .set_description(""),
4339
4340     Option("bluefs_sync_write", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4341     .set_default(false)
4342     .set_description(""),
4343
4344     Option("bluefs_allocator", Option::TYPE_STR, Option::LEVEL_DEV)
4345     .set_default("bitmap")
4346     .set_description(""),
4347
4348     Option("bluefs_preextend_wal_files", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4349     .set_default(false)
4350     .set_description(""),
4351
4352     Option("bluestore_bluefs", Option::TYPE_BOOL, Option::LEVEL_DEV)
4353     .set_default(true)
4354     .set_flag(Option::FLAG_CREATE)
4355     .set_description("Use BlueFS to back rocksdb")
4356     .set_long_description("BlueFS allows rocksdb to share the same physical device(s) as the rest of BlueStore.  It should be used in all cases unless testing/developing an alternative metadata database for BlueStore."),
4357
4358     Option("bluestore_bluefs_env_mirror", Option::TYPE_BOOL, Option::LEVEL_DEV)
4359     .set_default(false)
4360     .set_flag(Option::FLAG_CREATE)
4361     .set_description("Mirror bluefs data to file system for testing/validation"),
4362
4363     Option("bluestore_bluefs_min", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4364     .set_default(1_G)
4365     .set_description("minimum disk space allocated to BlueFS (e.g., at mkfs)"),
4366
4367     Option("bluestore_bluefs_min_free", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4368     .set_default(1_G)
4369     .set_description("minimum free space allocated to BlueFS"),
4370
4371     Option("bluestore_bluefs_min_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4372     .set_default(.02)
4373     .set_description("Minimum fraction of free space devoted to BlueFS"),
4374
4375     Option("bluestore_bluefs_max_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4376     .set_default(.90)
4377     .set_description("Maximum fraction of free storage devoted to BlueFS"),
4378
4379     Option("bluestore_bluefs_gift_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4380     .set_default(.02)
4381     .set_description("Maximum fraction of free space to give to BlueFS at once"),
4382
4383     Option("bluestore_bluefs_reclaim_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4384     .set_default(.20)
4385     .set_description("Maximum fraction of free space to reclaim from BlueFS at once"),
4386
4387     Option("bluestore_bluefs_balance_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4388     .set_default(1)
4389     .set_description("How frequently (in seconds) to balance free space between BlueFS and BlueStore"),
4390
4391     Option("bluestore_bluefs_alloc_failure_dump_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4392     .set_default(0)
4393     .set_description("How frequently (in seconds) to dump allocator on"
4394       "BlueFS space allocation failure"),
4395
4396     Option("bluestore_bluefs_db_compatibility", Option::TYPE_BOOL, Option::LEVEL_DEV)
4397     .set_default(true)
4398     .set_description("Sync db with legacy bluefs extents info")
4399     .set_long_description("Enforces db sync with legacy bluefs extents information on close."
4400                           " Enables downgrades to pre-nautilus releases"),
4401
4402     Option("bluestore_spdk_mem", Option::TYPE_SIZE, Option::LEVEL_DEV)
4403     .set_default(512)
4404     .set_description("Amount of dpdk memory size in MB")
4405     .set_long_description("If running multiple SPDK instances per node, you must specify the amount of dpdk memory size in MB each instance will use, to make sure each instance uses its own dpdk memory"),
4406
4407     Option("bluestore_spdk_coremask", Option::TYPE_STR, Option::LEVEL_DEV)
4408     .set_default("0x1")
4409     .set_description("A hexadecimal bit mask of the cores to run on. Note the core numbering can change between platforms and should be determined beforehand"),
4410
4411     Option("bluestore_spdk_max_io_completion", Option::TYPE_UINT, Option::LEVEL_DEV)
4412     .set_default(0)
4413     .set_description("Maximal I/Os to be batched completed while checking queue pair completions, 0 means let spdk library determine it"),
4414
4415     Option("bluestore_spdk_io_sleep", Option::TYPE_UINT, Option::LEVEL_DEV)
4416     .set_default(5)
4417     .set_description("Time period to wait if there is no completed I/O from polling"),
4418
4419     Option("bluestore_block_path", Option::TYPE_STR, Option::LEVEL_DEV)
4420     .set_default("")
4421     .set_flag(Option::FLAG_CREATE)
4422     .set_description("Path to block device/file"),
4423
4424     Option("bluestore_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4425     .set_default(10_G)
4426     .set_flag(Option::FLAG_CREATE)
4427     .set_description("Size of file to create for backing bluestore"),
4428
4429     Option("bluestore_block_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
4430     .set_default(true)
4431     .set_flag(Option::FLAG_CREATE)
4432     .set_description("Create bluestore_block_path if it doesn't exist")
4433     .add_see_also("bluestore_block_path").add_see_also("bluestore_block_size"),
4434
4435     Option("bluestore_block_db_path", Option::TYPE_STR, Option::LEVEL_DEV)
4436     .set_default("")
4437     .set_flag(Option::FLAG_CREATE)
4438     .set_description("Path for db block device"),
4439
4440     Option("bluestore_block_db_size", Option::TYPE_UINT, Option::LEVEL_DEV)
4441     .set_default(0)
4442     .set_flag(Option::FLAG_CREATE)
4443     .set_description("Size of file to create for bluestore_block_db_path"),
4444
4445     Option("bluestore_block_db_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
4446     .set_default(false)
4447     .set_flag(Option::FLAG_CREATE)
4448     .set_description("Create bluestore_block_db_path if it doesn't exist")
4449     .add_see_also("bluestore_block_db_path")
4450     .add_see_also("bluestore_block_db_size"),
4451
4452     Option("bluestore_block_wal_path", Option::TYPE_STR, Option::LEVEL_DEV)
4453     .set_default("")
4454     .set_flag(Option::FLAG_CREATE)
4455     .set_description("Path to block device/file backing bluefs wal"),
4456
4457     Option("bluestore_block_wal_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4458     .set_default(96_M)
4459     .set_flag(Option::FLAG_CREATE)
4460     .set_description("Size of file to create for bluestore_block_wal_path"),
4461
4462     Option("bluestore_block_wal_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
4463     .set_default(false)
4464     .set_flag(Option::FLAG_CREATE)
4465     .set_description("Create bluestore_block_wal_path if it doesn't exist")
4466     .add_see_also("bluestore_block_wal_path")
4467     .add_see_also("bluestore_block_wal_size"),
4468
4469     Option("bluestore_block_preallocate_file", Option::TYPE_BOOL, Option::LEVEL_DEV)
4470     .set_default(false)
4471     .set_flag(Option::FLAG_CREATE)
4472     .set_description("Preallocate file created via bluestore_block*_create"),
4473
4474     Option("bluestore_ignore_data_csum", Option::TYPE_BOOL, Option::LEVEL_DEV)
4475     .set_default(false)
4476     .set_flag(Option::FLAG_RUNTIME)
4477     .set_description("Ignore checksum errors on read and do not generate an EIO error"),
4478
4479     Option("bluestore_csum_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4480     .set_default("crc32c")
4481     .set_enum_allowed({"none", "crc32c", "crc32c_16", "crc32c_8", "xxhash32", "xxhash64"})
4482     .set_flag(Option::FLAG_RUNTIME)
4483     .set_description("Default checksum algorithm to use")
4484     .set_long_description("crc32c, xxhash32, and xxhash64 are available.  The _16 and _8 variants use only a subset of the bits for more compact (but less reliable) checksumming."),
4485
4486     Option("bluestore_retry_disk_reads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4487     .set_default(3)
4488     .set_min_max(0, 255)
4489     .set_flag(Option::FLAG_RUNTIME)
4490     .set_description("Number of read retries on checksum validation error")
4491     .set_long_description("Retries to read data from the disk this many times when checksum validation fails to handle spurious read errors gracefully."),
4492
4493     Option("bluestore_min_alloc_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4494     .set_default(0)
4495     .set_flag(Option::FLAG_CREATE)
4496     .set_description("Minimum allocation size to allocate for an object")
4497     .set_long_description("A smaller allocation size generally means less data is read and then rewritten when a copy-on-write operation is triggered (e.g., when writing to something that was recently snapshotted).  Similarly, less data is journaled before performing an overwrite (writes smaller than min_alloc_size must first pass through the BlueStore journal).  Larger values of min_alloc_size reduce the amount of metadata required to describe the on-disk layout and reduce overall fragmentation."),
4498
4499     Option("bluestore_min_alloc_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4500     .set_default(64_K)
4501     .set_flag(Option::FLAG_CREATE)
4502     .set_description("Default min_alloc_size value for rotational media")
4503     .add_see_also("bluestore_min_alloc_size"),
4504
4505     Option("bluestore_min_alloc_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4506     .set_default(16_K)
4507     .set_flag(Option::FLAG_CREATE)
4508     .set_description("Default min_alloc_size value for non-rotational (solid state)  media")
4509     .add_see_also("bluestore_min_alloc_size"),
4510
4511     Option("bluestore_max_alloc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4512     .set_default(0)
4513     .set_flag(Option::FLAG_CREATE)
4514     .set_description("Maximum size of a single allocation (0 for no max)"),
4515
4516     Option("bluestore_prefer_deferred_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4517     .set_default(0)
4518     .set_flag(Option::FLAG_RUNTIME)
4519     .set_description("Writes smaller than this size will be written to the journal and then asynchronously written to the device.  This can be beneficial when using rotational media where seeks are expensive, and is helpful both with and without solid state journal/wal devices."),
4520
4521     Option("bluestore_prefer_deferred_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4522     .set_default(32768)
4523     .set_flag(Option::FLAG_RUNTIME)
4524     .set_description("Default bluestore_prefer_deferred_size for rotational media")
4525     .add_see_also("bluestore_prefer_deferred_size"),
4526
4527     Option("bluestore_prefer_deferred_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4528     .set_default(0)
4529     .set_flag(Option::FLAG_RUNTIME)
4530     .set_description("Default bluestore_prefer_deferred_size for non-rotational (solid state) media")
4531     .add_see_also("bluestore_prefer_deferred_size"),
4532
4533     Option("bluestore_compression_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4534     .set_default("none")
4535     .set_enum_allowed({"none", "passive", "aggressive", "force"})
4536     .set_flag(Option::FLAG_RUNTIME)
4537     .set_description("Default policy for using compression when pool does not specify")
4538     .set_long_description("'none' means never use compression.  'passive' means use compression when clients hint that data is compressible.  'aggressive' means use compression unless clients hint that data is not compressible.  This option is used when the per-pool property for the compression mode is not present."),
4539
4540     Option("bluestore_compression_algorithm", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4541     .set_default("snappy")
4542     .set_enum_allowed({"", "snappy", "zlib", "zstd", "lz4"})
4543     .set_flag(Option::FLAG_RUNTIME)
4544     .set_description("Default compression algorithm to use when writing object data")
4545     .set_long_description("This controls the default compressor to use (if any) if the per-pool property is not set.  Note that zstd is *not* recommended for bluestore due to high CPU overhead when compressing small amounts of data."),
4546
4547     Option("bluestore_compression_min_blob_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4548     .set_default(0)
4549     .set_flag(Option::FLAG_RUNTIME)
4550     .set_description("Maximum chunk size to apply compression to when random access is expected for an object.")
4551     .set_long_description("Chunks larger than this are broken into smaller chunks before being compressed"),
4552
4553     Option("bluestore_compression_min_blob_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4554     .set_default(128_K)
4555     .set_flag(Option::FLAG_RUNTIME)
4556     .set_description("Default value of bluestore_compression_min_blob_size for rotational media")
4557     .add_see_also("bluestore_compression_min_blob_size"),
4558
4559     Option("bluestore_compression_min_blob_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4560     .set_default(8_K)
4561     .set_flag(Option::FLAG_RUNTIME)
4562     .set_description("Default value of bluestore_compression_min_blob_size for non-rotational (solid state) media")
4563     .add_see_also("bluestore_compression_min_blob_size"),
4564
4565     Option("bluestore_compression_max_blob_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4566     .set_default(0)
4567     .set_flag(Option::FLAG_RUNTIME)
4568     .set_description("Maximum chunk size to apply compression to when non-random access is expected for an object.")
4569     .set_long_description("Chunks larger than this are broken into smaller chunks before being compressed"),
4570
4571     Option("bluestore_compression_max_blob_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4572     .set_default(512_K)
4573     .set_flag(Option::FLAG_RUNTIME)
4574     .set_description("Default value of bluestore_compression_max_blob_size for rotational media")
4575     .add_see_also("bluestore_compression_max_blob_size"),
4576
4577     Option("bluestore_compression_max_blob_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4578     .set_default(64_K)
4579     .set_flag(Option::FLAG_RUNTIME)
4580     .set_description("Default value of bluestore_compression_max_blob_size for non-rotational (solid state) media")
4581     .add_see_also("bluestore_compression_max_blob_size"),
4582
4583     Option("bluestore_gc_enable_blob_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
4584     .set_default(0)
4585     .set_flag(Option::FLAG_RUNTIME)
4586     .set_description(""),
4587
4588     Option("bluestore_gc_enable_total_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
4589     .set_default(0)
4590     .set_flag(Option::FLAG_RUNTIME)
4591     .set_description(""),
4592
4593     Option("bluestore_max_blob_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4594     .set_default(0)
4595     .set_flag(Option::FLAG_RUNTIME)
4596     .set_description("")
4597     .set_long_description("Bluestore blobs are collections of extents (ie on-disk data) originating from one or more objects.  Blobs can be compressed, typically have checksum data, may be overwritten, may be shared (with an extent ref map), or split.  This setting controls the maximum size a blob is allowed to be."),
4598
4599     Option("bluestore_max_blob_size_hdd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4600     .set_default(512_K)
4601     .set_flag(Option::FLAG_RUNTIME)
4602     .set_description("")
4603     .add_see_also("bluestore_max_blob_size"),
4604
4605     Option("bluestore_max_blob_size_ssd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4606     .set_default(64_K)
4607     .set_flag(Option::FLAG_RUNTIME)
4608     .set_description("")
4609     .add_see_also("bluestore_max_blob_size"),
4610
4611     Option("bluestore_compression_required_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4612     .set_default(.875)
4613     .set_flag(Option::FLAG_RUNTIME)
4614     .set_description("Compression ratio required to store compressed data")
4615     .set_long_description("If we compress data and get less than this we discard the result and store the original uncompressed data."),
4616
4617     Option("bluestore_extent_map_shard_max_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4618     .set_default(1200)
4619     .set_description("Max size (bytes) for a single extent map shard before splitting"),
4620
4621     Option("bluestore_extent_map_shard_target_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4622     .set_default(500)
4623     .set_description("Target size (bytes) for a single extent map shard"),
4624
4625     Option("bluestore_extent_map_shard_min_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4626     .set_default(150)
4627     .set_description("Min size (bytes) for a single extent map shard before merging"),
4628
4629     Option("bluestore_extent_map_shard_target_size_slop", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4630     .set_default(.2)
4631     .set_description("Ratio above/below target for a shard when trying to align to an existing extent or blob boundary"),
4632
4633     Option("bluestore_extent_map_inline_shard_prealloc_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4634     .set_default(256)
4635     .set_description("Preallocated buffer for inline shards"),
4636
4637     Option("bluestore_cache_trim_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4638     .set_default(.05)
4639     .set_description("How frequently we trim the bluestore cache"),
4640
4641     Option("bluestore_cache_trim_max_skip_pinned", Option::TYPE_UINT, Option::LEVEL_DEV)
4642     .set_default(64)
4643     .set_description("Max pinned cache entries we consider before giving up"),
4644
4645     Option("bluestore_cache_type", Option::TYPE_STR, Option::LEVEL_DEV)
4646     .set_default("2q")
4647     .set_enum_allowed({"2q", "lru"})
4648     .set_description("Cache replacement algorithm"),
4649
4650     Option("bluestore_2q_cache_kin_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4651     .set_default(.5)
4652     .set_description("2Q paper suggests .5"),
4653
4654     Option("bluestore_2q_cache_kout_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4655     .set_default(.5)
4656     .set_description("2Q paper suggests .5"),
4657
4658     Option("bluestore_cache_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4659     .set_default(0)
4660     .set_description("Cache size (in bytes) for BlueStore")
4661     .set_long_description("This includes data and metadata cached by BlueStore as well as memory devoted to rocksdb's cache(s)."),
4662
4663     Option("bluestore_cache_size_hdd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4664     .set_default(1_G)
4665     .set_description("Default bluestore_cache_size for rotational media")
4666     .add_see_also("bluestore_cache_size"),
4667
4668     Option("bluestore_cache_size_ssd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4669     .set_default(3_G)
4670     .set_description("Default bluestore_cache_size for non-rotational (solid state) media")
4671     .add_see_also("bluestore_cache_size"),
4672
4673     Option("bluestore_cache_meta_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4674     .set_default(.4)
4675     .add_see_also("bluestore_cache_size")
4676     .set_description("Ratio of bluestore cache to devote to metadata"),
4677
4678     Option("bluestore_cache_kv_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4679     .set_default(.4)
4680     .add_see_also("bluestore_cache_size")
4681     .set_description("Ratio of bluestore cache to devote to kv database (rocksdb)"),
4682
4683     Option("bluestore_cache_autotune", Option::TYPE_BOOL, Option::LEVEL_DEV)
4684     .set_default(true)
4685     .add_see_also("bluestore_cache_size")
4686     .add_see_also("bluestore_cache_meta_ratio")
4687     .set_description("Automatically tune the ratio of caches while respecting min values."),
4688
4689     Option("bluestore_cache_autotune_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4690     .set_default(5)
4691     .add_see_also("bluestore_cache_autotune")
4692     .set_description("The number of seconds to wait between rebalances when cache autotune is enabled."),
4693
4694     Option("bluestore_kvbackend", Option::TYPE_STR, Option::LEVEL_DEV)
4695     .set_default("rocksdb")
4696     .set_flag(Option::FLAG_CREATE)
4697     .set_description("Key value database to use for bluestore"),
4698
4699     Option("bluestore_allocator", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4700     .set_default("bitmap")
4701     .set_enum_allowed({"bitmap", "stupid"})
4702     .set_description("Allocator policy")
4703     .set_long_description("Allocator to use for bluestore.  Stupid should only be used for testing."),
4704
4705     Option("bluestore_freelist_blocks_per_key", Option::TYPE_SIZE, Option::LEVEL_DEV)
4706     .set_default(128)
4707     .set_description("Block (and bits) per database key"),
4708
4709     Option("bluestore_bitmapallocator_blocks_per_zone", Option::TYPE_SIZE, Option::LEVEL_DEV)
4710     .set_default(1024)
4711     .set_description(""),
4712
4713     Option("bluestore_bitmapallocator_span_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4714     .set_default(1024)
4715     .set_description(""),
4716
4717     Option("bluestore_max_deferred_txc", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4718     .set_default(32)
4719     .set_description("Max transactions with deferred writes that can accumulate before we force flush deferred writes"),
4720
4721     Option("bluestore_rocksdb_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4722     .set_default("compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2")
4723     .set_description("Rocksdb options"),
4724
4725     Option("bluestore_rocksdb_cf", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4726     .set_default(false)
4727     .set_description("Enable use of rocksdb column families for bluestore metadata"),
4728
4729     Option("bluestore_rocksdb_cfs", Option::TYPE_STR, Option::LEVEL_DEV)
4730     .set_default("M= P= L=")
4731     .set_description("List of whitespace-separate key/value pairs where key is CF name and value is CF options"),
4732
4733     Option("bluestore_fsck_on_mount", Option::TYPE_BOOL, Option::LEVEL_DEV)
4734     .set_default(false)
4735     .set_description("Run fsck at mount"),
4736
4737     Option("bluestore_fsck_on_mount_deep", Option::TYPE_BOOL, Option::LEVEL_DEV)
4738     .set_default(false)
4739     .set_description("Run deep fsck at mount when bluestore_fsck_on_mount is set to true"),
4740
4741     Option("bluestore_fsck_quick_fix_on_mount", Option::TYPE_BOOL, Option::LEVEL_DEV)
4742       .set_default(true)
4743       .set_description("Do quick-fix for the store at mount"),
4744
4745     Option("bluestore_fsck_on_umount", Option::TYPE_BOOL, Option::LEVEL_DEV)
4746     .set_default(false)
4747     .set_description("Run fsck at umount"),
4748
4749     Option("bluestore_fsck_on_umount_deep", Option::TYPE_BOOL, Option::LEVEL_DEV)
4750     .set_default(false)
4751     .set_description("Run deep fsck at umount when bluestore_fsck_on_umount is set to true"),
4752
4753     Option("bluestore_fsck_on_mkfs", Option::TYPE_BOOL, Option::LEVEL_DEV)
4754     .set_default(true)
4755     .set_description("Run fsck after mkfs"),
4756
4757     Option("bluestore_fsck_on_mkfs_deep", Option::TYPE_BOOL, Option::LEVEL_DEV)
4758     .set_default(false)
4759     .set_description("Run deep fsck after mkfs"),
4760
4761     Option("bluestore_sync_submit_transaction", Option::TYPE_BOOL, Option::LEVEL_DEV)
4762     .set_default(false)
4763     .set_description("Try to submit metadata transaction to rocksdb in queuing thread context"),
4764
4765     Option("bluestore_fsck_read_bytes_cap", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4766     .set_default(64_M)
4767     .set_flag(Option::FLAG_RUNTIME)
4768     .set_description("Maximum bytes read at once by deep fsck"),
4769
4770     Option("bluestore_fsck_quick_fix_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4771       .set_default(2)
4772       .set_description("Number of additional threads to perform quick-fix (shallow fsck) command"),
4773
4774     Option("bluestore_throttle_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4775     .set_default(64_M)
4776     .set_flag(Option::FLAG_RUNTIME)
4777     .set_description("Maximum bytes in flight before we throttle IO submission"),
4778
4779     Option("bluestore_throttle_deferred_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4780     .set_default(128_M)
4781     .set_flag(Option::FLAG_RUNTIME)
4782     .set_description("Maximum bytes for deferred writes before we throttle IO submission"),
4783
4784     Option("bluestore_throttle_cost_per_io", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4785     .set_default(0)
4786     .set_flag(Option::FLAG_RUNTIME)
4787     .set_description("Overhead added to transaction cost (in bytes) for each IO"),
4788
4789   Option("bluestore_throttle_cost_per_io_hdd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4790     .set_default(670000)
4791     .set_flag(Option::FLAG_RUNTIME)
4792     .set_description("Default bluestore_throttle_cost_per_io for rotational media")
4793     .add_see_also("bluestore_throttle_cost_per_io"),
4794
4795     Option("bluestore_throttle_cost_per_io_ssd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4796     .set_default(4000)
4797     .set_flag(Option::FLAG_RUNTIME)
4798     .set_description("Default bluestore_throttle_cost_per_io for non-rotation (solid state) media")
4799     .add_see_also("bluestore_throttle_cost_per_io"),
4800
4801     Option("bluestore_deferred_batch_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4802     .set_default(0)
4803     .set_flag(Option::FLAG_RUNTIME)
4804     .set_description("Max number of deferred writes before we flush the deferred write queue"),
4805
4806     Option("bluestore_deferred_batch_ops_hdd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4807     .set_default(64)
4808     .set_flag(Option::FLAG_RUNTIME)
4809     .set_description("Default bluestore_deferred_batch_ops for rotational media")
4810     .add_see_also("bluestore_deferred_batch_ops"),
4811
4812     Option("bluestore_deferred_batch_ops_ssd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4813     .set_default(16)
4814     .set_flag(Option::FLAG_RUNTIME)
4815     .set_description("Default bluestore_deferred_batch_ops for non-rotational (solid state) media")
4816     .add_see_also("bluestore_deferred_batch_ops"),
4817
4818     Option("bluestore_nid_prealloc", Option::TYPE_INT, Option::LEVEL_DEV)
4819     .set_default(1024)
4820     .set_description("Number of unique object ids to preallocate at a time"),
4821
4822     Option("bluestore_blobid_prealloc", Option::TYPE_UINT, Option::LEVEL_DEV)
4823     .set_default(10240)
4824     .set_description("Number of unique blob ids to preallocate at a time"),
4825
4826     Option("bluestore_clone_cow", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4827     .set_default(true)
4828     .set_flag(Option::FLAG_RUNTIME)
4829     .set_description("Use copy-on-write when cloning objects (versus reading and rewriting them at clone time)"),
4830
4831     Option("bluestore_default_buffered_read", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4832     .set_default(true)
4833     .set_flag(Option::FLAG_RUNTIME)
4834     .set_description("Cache read results by default (unless hinted NOCACHE or WONTNEED)"),
4835
4836     Option("bluestore_default_buffered_write", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4837     .set_default(false)
4838     .set_flag(Option::FLAG_RUNTIME)
4839     .set_description("Cache writes by default (unless hinted NOCACHE or WONTNEED)"),
4840
4841     Option("bluestore_debug_misc", Option::TYPE_BOOL, Option::LEVEL_DEV)
4842     .set_default(false)
4843     .set_description(""),
4844
4845     Option("bluestore_debug_no_reuse_blocks", Option::TYPE_BOOL, Option::LEVEL_DEV)
4846     .set_default(false)
4847     .set_description(""),
4848
4849     Option("bluestore_debug_small_allocations", Option::TYPE_INT, Option::LEVEL_DEV)
4850     .set_default(0)
4851     .set_description(""),
4852
4853     Option("bluestore_debug_freelist", Option::TYPE_BOOL, Option::LEVEL_DEV)
4854     .set_default(false)
4855     .set_description(""),
4856
4857     Option("bluestore_debug_prefill", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4858     .set_default(0)
4859     .set_description("simulate fragmentation"),
4860
4861     Option("bluestore_debug_prefragment_max", Option::TYPE_SIZE, Option::LEVEL_DEV)
4862     .set_default(1_M)
4863     .set_description(""),
4864
4865     Option("bluestore_debug_inject_read_err", Option::TYPE_BOOL, Option::LEVEL_DEV)
4866     .set_default(false)
4867     .set_description(""),
4868
4869     Option("bluestore_debug_randomize_serial_transaction", Option::TYPE_INT, Option::LEVEL_DEV)
4870     .set_default(0)
4871     .set_description(""),
4872
4873     Option("bluestore_debug_omit_block_device_write", Option::TYPE_BOOL, Option::LEVEL_DEV)
4874     .set_default(false)
4875     .set_description(""),
4876
4877     Option("bluestore_debug_fsck_abort", Option::TYPE_BOOL, Option::LEVEL_DEV)
4878     .set_default(false)
4879     .set_description(""),
4880
4881     Option("bluestore_debug_omit_kv_commit", Option::TYPE_BOOL, Option::LEVEL_DEV)
4882     .set_default(false)
4883     .set_description(""),
4884
4885     Option("bluestore_debug_permit_any_bdev_label", Option::TYPE_BOOL, Option::LEVEL_DEV)
4886     .set_default(false)
4887     .set_description(""),
4888
4889     Option("bluestore_debug_random_read_err", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4890     .set_default(0)
4891     .set_description(""),
4892
4893     Option("bluestore_debug_inject_bug21040", Option::TYPE_BOOL, Option::LEVEL_DEV)
4894     .set_default(false)
4895     .set_description(""),
4896
4897     Option("bluestore_debug_inject_csum_err_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4898     .set_default(0.0)
4899     .set_description("inject crc verification errors into bluestore device reads"),
4900
4901     Option("bluestore_fsck_error_on_no_per_pool_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4902     .set_default(false)
4903     .set_description("Make fsck error (instead of warn) when bluestore lacks per-pool stats, e.g., after an upgrade"),
4904
4905     Option("bluestore_warn_on_bluefs_spillover", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4906     .set_default(true)
4907     .set_description("Enable health indication on bluefs slow device usage"),
4908
4909     Option("bluestore_warn_on_legacy_statfs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4910     .set_default(true)
4911     .set_description("Enable health indication on lack of per-pool statfs reporting from bluestore"),
4912
4913     Option("bluestore_log_op_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4914     .set_default(5)
4915     .set_description("log operation if it's slower than this age (seconds)"),
4916
4917     Option("bluestore_log_omap_iterator_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4918     .set_default(5)
4919     .set_description("log omap iteration operation if it's slower than this age (seconds)"),
4920
4921     Option("bluestore_log_collection_list_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4922     .set_default(60)
4923     .set_description("log collection list operation if it's slower than this age (seconds)"),
4924
4925     // -----------------------------------------
4926     // kstore
4927
4928     Option("kstore_max_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4929     .set_default(512)
4930     .set_description(""),
4931
4932     Option("kstore_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4933     .set_default(64_M)
4934     .set_description(""),
4935
4936     Option("kstore_backend", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4937     .set_default("rocksdb")
4938     .set_description(""),
4939
4940     Option("kstore_rocksdb_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4941     .set_default("compression=kNoCompression")
4942     .set_description("Options to pass through when RocksDB is used as the KeyValueDB for kstore."),
4943
4944     Option("kstore_fsck_on_mount", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4945     .set_default(false)
4946     .set_description("Whether or not to run fsck on mount for kstore."),
4947
4948     Option("kstore_fsck_on_mount_deep", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4949     .set_default(true)
4950     .set_description("Whether or not to run deep fsck on mount for kstore"),
4951
4952     Option("kstore_nid_prealloc", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4953     .set_default(1024)
4954     .set_description(""),
4955
4956     Option("kstore_sync_transaction", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4957     .set_default(false)
4958     .set_description(""),
4959
4960     Option("kstore_sync_submit_transaction", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4961     .set_default(false)
4962     .set_description(""),
4963
4964     Option("kstore_onode_map_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4965     .set_default(1024)
4966     .set_description(""),
4967
4968     Option("kstore_default_stripe_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4969     .set_default(65536)
4970     .set_description(""),
4971
4972     // ---------------------
4973     // filestore
4974
4975     Option("filestore_rocksdb_options", Option::TYPE_STR, Option::LEVEL_DEV)
4976     .set_default("max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression")
4977     .set_description("Options to pass through when RocksDB is used as the KeyValueDB for filestore."),
4978
4979     Option("filestore_omap_backend", Option::TYPE_STR, Option::LEVEL_DEV)
4980     .set_default("rocksdb")
4981     .set_enum_allowed({"leveldb", "rocksdb"})
4982     .set_description("The KeyValueDB to use for filestore metadata (ie omap)."),
4983
4984     Option("filestore_omap_backend_path", Option::TYPE_STR, Option::LEVEL_DEV)
4985     .set_default("")
4986     .set_description("The path where the filestore KeyValueDB should store it's database(s)."),
4987
4988     Option("filestore_wbthrottle_enable", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4989     .set_default(true)
4990     .set_description("Enabling throttling of operations to backing file system"),
4991
4992     Option("filestore_wbthrottle_btrfs_bytes_start_flusher", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4993     .set_default(41943040)
4994     .set_description("Start flushing (fsyncing) when this many bytes are written(btrfs)"),
4995
4996     Option("filestore_wbthrottle_btrfs_bytes_hard_limit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4997     .set_default(419430400)
4998     .set_description("Block writes when this many bytes haven't been flushed (fsynced) (btrfs)"),
4999
5000     Option("filestore_wbthrottle_btrfs_ios_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5001     .set_default(500)
5002     .set_description("Start flushing (fsyncing) when this many IOs are written (brtrfs)"),
5003
5004     Option("filestore_wbthrottle_btrfs_ios_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5005     .set_default(5000)
5006     .set_description("Block writes when this many IOs haven't been flushed (fsynced) (btrfs)"),
5007
5008     Option("filestore_wbthrottle_btrfs_inodes_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5009     .set_default(500)
5010     .set_description("Start flushing (fsyncing) when this many distinct inodes have been modified (btrfs)"),
5011
5012     Option("filestore_wbthrottle_xfs_bytes_start_flusher", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5013     .set_default(41943040)
5014     .set_description("Start flushing (fsyncing) when this many bytes are written(xfs)"),
5015
5016     Option("filestore_wbthrottle_xfs_bytes_hard_limit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5017     .set_default(419430400)
5018     .set_description("Block writes when this many bytes haven't been flushed (fsynced) (xfs)"),
5019
5020     Option("filestore_wbthrottle_xfs_ios_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5021     .set_default(500)
5022     .set_description("Start flushing (fsyncing) when this many IOs are written (xfs)"),
5023
5024     Option("filestore_wbthrottle_xfs_ios_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5025     .set_default(5000)
5026     .set_description("Block writes when this many IOs haven't been flushed (fsynced) (xfs)"),
5027
5028     Option("filestore_wbthrottle_xfs_inodes_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5029     .set_default(500)
5030     .set_description("Start flushing (fsyncing) when this many distinct inodes have been modified (xfs)"),
5031
5032     Option("filestore_wbthrottle_btrfs_inodes_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5033     .set_default(5000)
5034     .set_description("Block writing when this many inodes have outstanding writes (btrfs)"),
5035
5036     Option("filestore_wbthrottle_xfs_inodes_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5037     .set_default(5000)
5038     .set_description("Block writing when this many inodes have outstanding writes (xfs)"),
5039
5040     Option("filestore_odsync_write", Option::TYPE_BOOL, Option::LEVEL_DEV)
5041     .set_default(false)
5042     .set_description("Write with O_DSYNC"),
5043
5044     Option("filestore_index_retry_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5045     .set_default(0)
5046     .set_description(""),
5047
5048     Option("filestore_debug_inject_read_err", Option::TYPE_BOOL, Option::LEVEL_DEV)
5049     .set_default(false)
5050     .set_description(""),
5051
5052     Option("filestore_debug_random_read_err", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5053     .set_default(0)
5054     .set_description(""),
5055
5056     Option("filestore_debug_omap_check", Option::TYPE_BOOL, Option::LEVEL_DEV)
5057     .set_default(false)
5058     .set_description(""),
5059
5060     Option("filestore_omap_header_cache_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5061     .set_default(1024)
5062     .set_description(""),
5063
5064     Option("filestore_max_inline_xattr_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5065     .set_default(0)
5066     .set_description(""),
5067
5068     Option("filestore_max_inline_xattr_size_xfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
5069     .set_default(65536)
5070     .set_description(""),
5071
5072     Option("filestore_max_inline_xattr_size_btrfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
5073     .set_default(2048)
5074     .set_description(""),
5075
5076     Option("filestore_max_inline_xattr_size_other", Option::TYPE_SIZE, Option::LEVEL_DEV)
5077     .set_default(512)
5078     .set_description(""),
5079
5080     Option("filestore_max_inline_xattrs", Option::TYPE_UINT, Option::LEVEL_DEV)
5081     .set_default(0)
5082     .set_description(""),
5083
5084     Option("filestore_max_inline_xattrs_xfs", Option::TYPE_UINT, Option::LEVEL_DEV)
5085     .set_default(10)
5086     .set_description(""),
5087
5088     Option("filestore_max_inline_xattrs_btrfs", Option::TYPE_UINT, Option::LEVEL_DEV)
5089     .set_default(10)
5090     .set_description(""),
5091
5092     Option("filestore_max_inline_xattrs_other", Option::TYPE_UINT, Option::LEVEL_DEV)
5093     .set_default(2)
5094     .set_description(""),
5095
5096     Option("filestore_max_xattr_value_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5097     .set_default(0)
5098     .set_description(""),
5099
5100     Option("filestore_max_xattr_value_size_xfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
5101     .set_default(64_K)
5102     .set_description(""),
5103
5104     Option("filestore_max_xattr_value_size_btrfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
5105     .set_default(64_K)
5106     .set_description(""),
5107
5108     Option("filestore_max_xattr_value_size_other", Option::TYPE_SIZE, Option::LEVEL_DEV)
5109     .set_default(1_K)
5110     .set_description(""),
5111
5112     Option("filestore_sloppy_crc", Option::TYPE_BOOL, Option::LEVEL_DEV)
5113     .set_default(false)
5114     .set_description(""),
5115
5116     Option("filestore_sloppy_crc_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5117     .set_default(65536)
5118     .set_description(""),
5119
5120     Option("filestore_max_alloc_hint_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5121     .set_default(1ULL << 20)
5122     .set_description(""),
5123
5124     Option("filestore_max_sync_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5125     .set_default(5)
5126     .set_description("Period between calls to syncfs(2) and journal trims (seconds)"),
5127
5128     Option("filestore_min_sync_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5129     .set_default(.01)
5130     .set_description("Minimum period between calls to syncfs(2)"),
5131
5132     Option("filestore_btrfs_snap", Option::TYPE_BOOL, Option::LEVEL_DEV)
5133     .set_default(true)
5134     .set_description(""),
5135
5136     Option("filestore_btrfs_clone_range", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5137     .set_default(true)
5138     .set_description("Use btrfs clone_range ioctl to efficiently duplicate objects"),
5139
5140     Option("filestore_zfs_snap", Option::TYPE_BOOL, Option::LEVEL_DEV)
5141     .set_default(false)
5142     .set_description(""),
5143
5144     Option("filestore_fsync_flushes_journal_data", Option::TYPE_BOOL, Option::LEVEL_DEV)
5145     .set_default(false)
5146     .set_description(""),
5147
5148     Option("filestore_fiemap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5149     .set_default(false)
5150     .set_description("Use fiemap ioctl(2) to determine which parts of objects are sparse"),
5151
5152     Option("filestore_punch_hole", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5153     .set_default(false)
5154     .set_description("Use fallocate(2) FALLOC_FL_PUNCH_HOLE to efficiently zero ranges of objects"),
5155
5156     Option("filestore_seek_data_hole", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5157     .set_default(false)
5158     .set_description("Use lseek(2) SEEK_HOLE and SEEK_DATA to determine which parts of objects are sparse"),
5159
5160     Option("filestore_splice", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5161     .set_default(false)
5162     .set_description("Use splice(2) to more efficiently copy data between files"),
5163
5164     Option("filestore_fadvise", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5165     .set_default(true)
5166     .set_description("Use posix_fadvise(2) to pass hints to file system"),
5167
5168     Option("filestore_collect_device_partition_information", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5169     .set_default(true)
5170     .set_description("Collect metadata about the backing file system on OSD startup"),
5171
5172     Option("filestore_xfs_extsize", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5173     .set_default(false)
5174     .set_description("Use XFS extsize ioctl(2) to hint allocator about expected write sizes"),
5175
5176     Option("filestore_journal_parallel", Option::TYPE_BOOL, Option::LEVEL_DEV)
5177     .set_default(false)
5178     .set_description(""),
5179
5180     Option("filestore_journal_writeahead", Option::TYPE_BOOL, Option::LEVEL_DEV)
5181     .set_default(false)
5182     .set_description(""),
5183
5184     Option("filestore_journal_trailing", Option::TYPE_BOOL, Option::LEVEL_DEV)
5185     .set_default(false)
5186     .set_description(""),
5187
5188     Option("filestore_queue_max_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5189     .set_default(50)
5190     .set_description("Max IO operations in flight"),
5191
5192     Option("filestore_queue_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5193     .set_default(100_M)
5194     .set_description("Max (written) bytes in flight"),
5195
5196     Option("filestore_caller_concurrency", Option::TYPE_INT, Option::LEVEL_DEV)
5197     .set_default(10)
5198     .set_description(""),
5199
5200     Option("filestore_expected_throughput_bytes", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5201     .set_default(200_M)
5202     .set_description("Expected throughput of backend device (aids throttling calculations)"),
5203
5204     Option("filestore_expected_throughput_ops", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5205     .set_default(200)
5206     .set_description("Expected through of backend device in IOPS (aids throttling calculations)"),
5207
5208     Option("filestore_queue_max_delay_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5209     .set_default(0)
5210     .set_description(""),
5211
5212     Option("filestore_queue_high_delay_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5213     .set_default(0)
5214     .set_description(""),
5215
5216     Option("filestore_queue_max_delay_multiple_bytes", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5217     .set_default(0)
5218     .set_description(""),
5219
5220     Option("filestore_queue_high_delay_multiple_bytes", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5221     .set_default(0)
5222     .set_description(""),
5223
5224     Option("filestore_queue_max_delay_multiple_ops", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5225     .set_default(0)
5226     .set_description(""),
5227
5228     Option("filestore_queue_high_delay_multiple_ops", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5229     .set_default(0)
5230     .set_description(""),
5231
5232     Option("filestore_queue_low_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5233     .set_default(0.3)
5234     .set_description(""),
5235
5236     Option("filestore_queue_high_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5237     .set_default(0.9)
5238     .set_description(""),
5239
5240     Option("filestore_op_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5241     .set_default(2)
5242     .set_description("Threads used to apply changes to backing file system"),
5243
5244     Option("filestore_op_thread_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5245     .set_default(60)
5246     .set_description("Seconds before a worker thread is considered stalled"),
5247
5248     Option("filestore_op_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5249     .set_default(180)
5250     .set_description("Seconds before a worker thread is considered dead"),
5251
5252     Option("filestore_commit_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5253     .set_default(600)
5254     .set_description("Seconds before backing file system is considered hung"),
5255
5256     Option("filestore_fiemap_threshold", Option::TYPE_SIZE, Option::LEVEL_DEV)
5257     .set_default(4_K)
5258     .set_description(""),
5259
5260     Option("filestore_merge_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
5261     .set_default(-10)
5262     .set_description(""),
5263
5264     Option("filestore_split_multiple", Option::TYPE_INT, Option::LEVEL_DEV)
5265     .set_default(2)
5266     .set_description(""),
5267
5268     Option("filestore_split_rand_factor", Option::TYPE_UINT, Option::LEVEL_DEV)
5269     .set_default(20)
5270     .set_description(""),
5271
5272     Option("filestore_update_to", Option::TYPE_INT, Option::LEVEL_DEV)
5273     .set_default(1000)
5274     .set_description(""),
5275
5276     Option("filestore_blackhole", Option::TYPE_BOOL, Option::LEVEL_DEV)
5277     .set_default(false)
5278     .set_description(""),
5279
5280     Option("filestore_fd_cache_size", Option::TYPE_INT, Option::LEVEL_DEV)
5281     .set_default(128)
5282     .set_description(""),
5283
5284     Option("filestore_fd_cache_shards", Option::TYPE_INT, Option::LEVEL_DEV)
5285     .set_default(16)
5286     .set_description(""),
5287
5288     Option("filestore_ondisk_finisher_threads", Option::TYPE_INT, Option::LEVEL_DEV)
5289     .set_default(1)
5290     .set_description(""),
5291
5292     Option("filestore_apply_finisher_threads", Option::TYPE_INT, Option::LEVEL_DEV)
5293     .set_default(1)
5294     .set_description(""),
5295
5296     Option("filestore_dump_file", Option::TYPE_STR, Option::LEVEL_DEV)
5297     .set_default("")
5298     .set_description(""),
5299
5300     Option("filestore_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
5301     .set_default(0)
5302     .set_description(""),
5303
5304     Option("filestore_inject_stall", Option::TYPE_INT, Option::LEVEL_DEV)
5305     .set_default(0)
5306     .set_description(""),
5307
5308     Option("filestore_fail_eio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5309     .set_default(true)
5310     .set_description(""),
5311
5312     Option("filestore_debug_verify_split", Option::TYPE_BOOL, Option::LEVEL_DEV)
5313     .set_default(false)
5314     .set_description(""),
5315
5316     Option("journal_dio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5317     .set_default(true)
5318     .set_description(""),
5319
5320     Option("journal_aio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5321     .set_default(true)
5322     .set_description(""),
5323
5324     Option("journal_force_aio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5325     .set_default(false)
5326     .set_description(""),
5327
5328     Option("journal_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5329     .set_default(4_K)
5330     .set_description(""),
5331
5332     Option("journal_block_align", Option::TYPE_BOOL, Option::LEVEL_DEV)
5333     .set_default(true)
5334     .set_description(""),
5335
5336     Option("journal_write_header_frequency", Option::TYPE_UINT, Option::LEVEL_DEV)
5337     .set_default(0)
5338     .set_description(""),
5339
5340     Option("journal_max_write_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5341     .set_default(10_M)
5342     .set_description("Max bytes in flight to journal"),
5343
5344     Option("journal_max_write_entries", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5345     .set_default(100)
5346     .set_description("Max IOs in flight to journal"),
5347
5348     Option("journal_throttle_low_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5349     .set_default(0.6)
5350     .set_description(""),
5351
5352     Option("journal_throttle_high_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5353     .set_default(0.9)
5354     .set_description(""),
5355
5356     Option("journal_throttle_high_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5357     .set_default(0)
5358     .set_description(""),
5359
5360     Option("journal_throttle_max_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5361     .set_default(0)
5362     .set_description(""),
5363
5364     Option("journal_align_min_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5365     .set_default(64_K)
5366     .set_description(""),
5367
5368     Option("journal_replay_from", Option::TYPE_INT, Option::LEVEL_DEV)
5369     .set_default(0)
5370     .set_description(""),
5371
5372   Option("mgr_stats_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5373   .set_default((int64_t)PerfCountersBuilder::PRIO_USEFUL)
5374   .set_description("Lowest perfcounter priority collected by mgr")
5375   .set_long_description("Daemons only set perf counter data to the manager "
5376     "daemon if the counter has a priority higher than this.")
5377   .set_min_max((int64_t)PerfCountersBuilder::PRIO_DEBUGONLY,
5378                (int64_t)PerfCountersBuilder::PRIO_CRITICAL + 1),
5379
5380     Option("journal_zero_on_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
5381     .set_default(false)
5382     .set_description(""),
5383
5384     Option("journal_ignore_corruption", Option::TYPE_BOOL, Option::LEVEL_DEV)
5385     .set_default(false)
5386     .set_description(""),
5387
5388     Option("journal_discard", Option::TYPE_BOOL, Option::LEVEL_DEV)
5389     .set_default(false)
5390     .set_description(""),
5391
5392     Option("fio_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5393     .set_default("/tmp/fio")
5394     .set_description(""),
5395
5396     Option("rados_mon_op_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5397     .set_default(0)
5398     .set_description(""),
5399
5400     Option("rados_osd_op_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5401     .set_default(0)
5402     .set_description(""),
5403
5404     Option("rados_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5405     .set_default(false)
5406     .set_description(""),
5407
5408     Option("nss_db_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5409     .set_default("")
5410     .set_description(""),
5411
5412     Option("mgr_module_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5413     .set_default(CEPH_DATADIR "/mgr")
5414     .add_service("mgr")
5415     .set_description("Filesystem path to manager modules."),
5416
5417     Option("mgr_initial_modules", Option::TYPE_STR, Option::LEVEL_BASIC)
5418     .set_default("restful iostat")
5419     .set_flag(Option::FLAG_NO_MON_UPDATE)
5420     .set_flag(Option::FLAG_CLUSTER_CREATE)
5421     .add_service("mon")
5422     .set_description("List of manager modules to enable when the cluster is "
5423                      "first started")
5424     .set_long_description("This list of module names is read by the monitor "
5425         "when the cluster is first started after installation, to populate "
5426         "the list of enabled manager modules.  Subsequent updates are done using "
5427         "the 'mgr module [enable|disable]' commands.  List may be comma "
5428         "or space separated."),
5429
5430     Option("mgr_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5431     .set_default("/var/lib/ceph/mgr/$cluster-$id")
5432     .set_flag(Option::FLAG_NO_MON_UPDATE)
5433     .add_service("mgr")
5434     .set_description("Filesystem path to the ceph-mgr data directory, used to "
5435                      "contain keyring."),
5436
5437     Option("mgr_tick_period", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
5438     .set_default(2)
5439     .add_service("mgr")
5440     .set_description("Period in seconds of beacon messages to monitor"),
5441
5442     Option("mgr_stats_period", Option::TYPE_INT, Option::LEVEL_BASIC)
5443     .set_default(5)
5444     .add_service("mgr")
5445     .set_description("Period in seconds of OSD/MDS stats reports to manager")
5446     .set_long_description("Use this setting to control the granularity of "
5447                           "time series data collection from daemons.  Adjust "
5448                           "upwards if the manager CPU load is too high, or "
5449                           "if you simply do not require the most up to date "
5450                           "performance counter data."),
5451
5452     Option("mgr_client_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5453     .set_default(128_M)
5454     .add_service("mgr"),
5455
5456     Option("mgr_client_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5457     .set_default(512)
5458     .add_service("mgr"),
5459
5460     Option("mgr_osd_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5461     .set_default(512_M)
5462     .add_service("mgr"),
5463
5464     Option("mgr_osd_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5465     .set_default(8192)
5466     .add_service("mgr"),
5467
5468     Option("mgr_mds_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5469     .set_default(128_M)
5470     .add_service("mgr"),
5471
5472     Option("mgr_mds_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5473     .set_default(128)
5474     .add_service("mgr"),
5475
5476     Option("mgr_mon_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5477     .set_default(128_M)
5478     .add_service("mgr"),
5479
5480     Option("mgr_mon_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5481     .set_default(128)
5482     .add_service("mgr"),
5483
5484     Option("mgr_connect_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5485     .set_default(1.0)
5486     .add_service("common"),
5487
5488     Option("mgr_service_beacon_grace", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5489     .set_default(60.0)
5490     .add_service("mgr")
5491     .set_description("Period in seconds from last beacon to manager dropping "
5492                      "state about a monitored service (RGW, rbd-mirror etc)"),
5493
5494     Option("mgr_client_service_daemon_unregister_timeout", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5495     .set_default(1.0)
5496     .set_description("Time to wait during shutdown to deregister service with mgr"),
5497
5498     Option("mgr_debug_aggressive_pg_num_changes", Option::TYPE_BOOL, Option::LEVEL_DEV)
5499     .set_default(false)
5500     .set_description("Bypass most throttling and safety checks in pg[p]_num controller")
5501     .add_service("mgr"),
5502
5503     Option("mon_mgr_digest_period", Option::TYPE_INT, Option::LEVEL_DEV)
5504     .set_default(5)
5505     .add_service("mon")
5506     .set_description("Period in seconds between monitor-to-manager "
5507                      "health/status updates"),
5508
5509     Option("mon_mgr_beacon_grace", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
5510     .set_default(30)
5511     .add_service("mon")
5512     .set_description("Period in seconds from last beacon to monitor marking "
5513                      "a manager daemon as failed"),
5514
5515     Option("mon_mgr_inactive_grace", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5516     .set_default(60)
5517     .add_service("mon")
5518     .set_description("Period in seconds after cluster creation during which "
5519                      "cluster may have no active manager")
5520     .set_long_description("This grace period enables the cluster to come "
5521                           "up cleanly without raising spurious health check "
5522                           "failures about managers that aren't online yet"),
5523
5524     Option("mon_mgr_mkfs_grace", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5525     .set_default(120)
5526     .add_service("mon")
5527     .set_description("Period in seconds that the cluster may have no active "
5528                      "manager before this is reported as an ERR rather than "
5529                      "a WARN"),
5530
5531     Option("throttler_perf_counter", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5532     .set_default(true)
5533     .set_description(""),
5534
5535     Option("event_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5536     .set_default(false)
5537     .set_description(""),
5538
5539     Option("debug_deliberately_leak_memory", Option::TYPE_BOOL, Option::LEVEL_DEV)
5540     .set_default(false)
5541     .set_description(""),
5542
5543     Option("debug_asserts_on_shutdown", Option::TYPE_BOOL,Option::LEVEL_DEV)
5544     .set_default(false)
5545     .set_description("Enable certain asserts to check for refcounting bugs on shutdown; see http://tracker.ceph.com/issues/21738"),
5546
5547     Option("debug_asok_assert_abort", Option::TYPE_BOOL, Option::LEVEL_DEV)
5548     .set_default(false)
5549     .set_description("allow commands 'assert' and 'abort' via asok for testing crash dumps etc"),
5550
5551     Option("target_max_misplaced_ratio", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
5552     .set_default(.05)
5553     .set_description("Max ratio of misplaced objects to target when throttling data rebalancing activity"),
5554
5555     Option("device_failure_prediction_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
5556     .set_default("none")
5557     .set_flag(Option::FLAG_RUNTIME)
5558     .set_enum_allowed({"none", "local", "cloud"})
5559     .set_description("Method used to predict device failures")
5560     .set_long_description("To disable prediction, use 'none',  'local' uses a prediction model that runs inside the mgr daemon.  'cloud' will share metrics with a cloud service and query the service for devicelife expectancy."),
5561
5562     /*  KRB Authentication. */
5563     Option("gss_ktab_client_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5564     .set_default("/var/lib/ceph/$name/gss_client_$name.ktab")
5565     .set_description("GSS/KRB5 Keytab file for client authentication")
5566     .add_service({"mon", "osd"})
5567     .set_long_description("This sets the full path for the GSS/Kerberos client keytab file location."),
5568
5569     Option("gss_target_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5570     .set_default("ceph")
5571     .set_description("")
5572     .add_service({"mon", "osd"})
5573     .set_long_description("This sets the gss target service name."),
5574
5575     Option("debug_disable_randomized_ping", Option::TYPE_BOOL, Option::LEVEL_DEV)
5576     .set_default(false)
5577     .set_description("Disable heartbeat ping randomization for testing purposes"),
5578
5579     Option("debug_heartbeat_testing_span", Option::TYPE_INT, Option::LEVEL_DEV)
5580     .set_default(0)
5581     .set_description("Override 60 second periods for testing only"),
5582   });
5583 }
5584
5585 std::vector<Option> get_rgw_options() {
5586   return std::vector<Option>({
5587     Option("rgw_acl_grants_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5588     .set_default(100)
5589     .set_description("Max number of ACL grants in a single request"),
5590
5591     Option("rgw_cors_rules_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5592     .set_default(100)
5593     .set_description("Max number of cors rules in a single request"),
5594
5595     Option("rgw_delete_multi_obj_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5596     .set_default(1000)
5597     .set_description("Max number of objects in a single multi-object delete request"),
5598
5599     Option("rgw_website_routing_rules_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5600     .set_default(50)
5601     .set_description("Max number of website routing rules in a single request"),
5602
5603     Option("rgw_rados_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5604     .set_default(false)
5605     .set_description("true if LTTng-UST tracepoints should be enabled"),
5606
5607     Option("rgw_op_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5608     .set_default(false)
5609     .set_description("true if LTTng-UST tracepoints should be enabled"),
5610
5611     Option("rgw_max_chunk_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5612     .set_default(4_M)
5613     .set_description("Set RGW max chunk size")
5614     .set_long_description(
5615         "The chunk size is the size of RADOS I/O requests that RGW sends when accessing "
5616         "data objects. RGW read and write operation will never request more than this amount "
5617         "in a single request. This also defines the rgw object head size, as head operations "
5618         "need to be atomic, and anything larger than this would require more than a single "
5619         "operation."),
5620
5621     Option("rgw_put_obj_min_window_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5622     .set_default(16_M)
5623     .set_description("The minimum RADOS write window size (in bytes).")
5624     .set_long_description(
5625         "The window size determines the total concurrent RADOS writes of a single rgw object. "
5626         "When writing an object RGW will send multiple chunks to RADOS. The total size of the "
5627         "writes does not exceed the window size. The window size can be automatically "
5628         "in order to better utilize the pipe.")
5629     .add_see_also({"rgw_put_obj_max_window_size", "rgw_max_chunk_size"}),
5630
5631     Option("rgw_put_obj_max_window_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5632     .set_default(64_M)
5633     .set_description("The maximum RADOS write window size (in bytes).")
5634     .set_long_description("The window size may be dynamically adjusted, but will not surpass this value.")
5635     .add_see_also({"rgw_put_obj_min_window_size", "rgw_max_chunk_size"}),
5636
5637     Option("rgw_max_put_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5638     .set_default(5_G)
5639     .set_description("Max size (in bytes) of regular (non multi-part) object upload.")
5640     .set_long_description(
5641         "Plain object upload is capped at this amount of data. In order to upload larger "
5642         "objects, a special upload mechanism is required. The S3 API provides the "
5643         "multi-part upload, and Swift provides DLO and SLO."),
5644
5645     Option("rgw_max_put_param_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5646     .set_default(1_M)
5647     .set_description("The maximum size (in bytes) of data input of certain RESTful requests."),
5648
5649     Option("rgw_max_attr_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5650     .set_default(0)
5651     .set_description("The maximum length of metadata value. 0 skips the check"),
5652
5653     Option("rgw_max_attr_name_len", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5654     .set_default(0)
5655     .set_description("The maximum length of metadata name. 0 skips the check"),
5656
5657     Option("rgw_max_attrs_num_in_req", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5658     .set_default(0)
5659     .set_description("The maximum number of metadata items that can be put via single request"),
5660
5661     Option("rgw_override_bucket_index_max_shards", Option::TYPE_UINT, Option::LEVEL_DEV)
5662     .set_default(0)
5663     .set_description(""),
5664
5665     Option("rgw_bucket_index_max_aio", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5666     .set_default(8)
5667     .set_description("Max number of concurrent RADOS requests when handling bucket shards."),
5668
5669     Option("rgw_enable_quota_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5670     .set_default(true)
5671     .set_description("Enables the quota maintenance thread.")
5672     .set_long_description(
5673         "The quota maintenance thread is responsible for quota related maintenance work. "
5674         "The thread itself can be disabled, but in order for quota to work correctly, at "
5675         "least one RGW in each zone needs to have this thread running. Having the thread "
5676         "enabled on multiple RGW processes within the same zone can spread "
5677         "some of the maintenance work between them.")
5678     .add_see_also({"rgw_enable_gc_threads", "rgw_enable_lc_threads"}),
5679
5680     Option("rgw_enable_gc_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5681     .set_default(true)
5682     .set_description("Enables the garbage collection maintenance thread.")
5683     .set_long_description(
5684         "The garbage collection maintenance thread is responsible for garbage collector "
5685         "maintenance work. The thread itself can be disabled, but in order for garbage "
5686         "collection to work correctly, at least one RGW in each zone needs to have this "
5687         "thread running.  Having the thread enabled on multiple RGW processes within the "
5688         "same zone can spread some of the maintenance work between them.")
5689     .add_see_also({"rgw_enable_quota_threads", "rgw_enable_lc_threads"}),
5690
5691     Option("rgw_enable_lc_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5692     .set_default(true)
5693     .set_description("Enables the lifecycle maintenance thread. This is required on at least one rgw for each zone.")
5694     .set_long_description(
5695         "The lifecycle maintenance thread is responsible for lifecycle related maintenance "
5696         "work. The thread itself can be disabled, but in order for lifecycle to work "
5697         "correctly, at least one RGW in each zone needs to have this thread running. Having"
5698         "the thread enabled on multiple RGW processes within the same zone can spread "
5699         "some of the maintenance work between them.")
5700     .add_see_also({"rgw_enable_gc_threads", "rgw_enable_quota_threads"}),
5701
5702     Option("rgw_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5703     .set_default("/var/lib/ceph/radosgw/$cluster-$id")
5704     .set_flag(Option::FLAG_NO_MON_UPDATE)
5705     .set_description("Alternative location for RGW configuration.")
5706     .set_long_description(
5707         "If this is set, the different Ceph system configurables (such as the keyring file "
5708         "will be located in the path that is specified here. "),
5709
5710     Option("rgw_enable_apis", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5711     .set_default("s3, s3website, swift, swift_auth, admin, sts, pubsub")
5712     .set_description("A list of set of RESTful APIs that rgw handles."),
5713
5714     Option("rgw_cache_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5715     .set_default(true)
5716     .set_description("Enable RGW metadata cache.")
5717     .set_long_description(
5718         "The metadata cache holds metadata entries that RGW requires for processing "
5719         "requests. Metadata entries can be user info, bucket info, and bucket instance "
5720         "info. If not found in the cache, entries will be fetched from the backing "
5721         "RADOS store.")
5722     .add_see_also("rgw_cache_lru_size"),
5723
5724     Option("rgw_cache_lru_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5725     .set_default(10000)
5726     .set_description("Max number of items in RGW metadata cache.")
5727     .set_long_description(
5728         "When full, the RGW metadata cache evicts least recently used entries.")
5729     .add_see_also("rgw_cache_enabled"),
5730
5731     Option("rgw_socket_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5732     .set_default("")
5733     .set_description("RGW FastCGI socket path (for FastCGI over Unix domain sockets).")
5734     .add_see_also("rgw_fcgi_socket_backlog"),
5735
5736     Option("rgw_host", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5737     .set_default("")
5738     .set_description("RGW FastCGI host name (for FastCGI over TCP)")
5739     .add_see_also({"rgw_port", "rgw_fcgi_socket_backlog"}),
5740
5741     Option("rgw_port", Option::TYPE_STR, Option::LEVEL_BASIC)
5742     .set_default("")
5743     .set_description("RGW FastCGI port number (for FastCGI over TCP)")
5744     .add_see_also({"rgw_host", "rgw_fcgi_socket_backlog"}),
5745
5746     Option("rgw_dns_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5747     .set_default("")
5748     .set_description("The host name that RGW uses.")
5749     .set_long_description(
5750         "This is Needed for virtual hosting of buckets to work properly, unless configured "
5751         "via zonegroup configuration."),
5752
5753     Option("rgw_dns_s3website_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5754     .set_default("")
5755     .set_description("The host name that RGW uses for static websites (S3)")
5756     .set_long_description(
5757         "This is needed for virtual hosting of buckets, unless configured via zonegroup "
5758         "configuration."),
5759
5760     Option("rgw_service_provider_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5761     .set_default("")
5762     .set_description("Service provider name which is contained in http response headers")
5763     .set_long_description(
5764         "As S3 or other cloud storage providers do, http response headers should contain the name of the provider. "
5765         "This name will be placed in http header 'Server'."),
5766
5767     Option("rgw_content_length_compat", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5768     .set_default(false)
5769     .set_description("Multiple content length headers compatibility")
5770     .set_long_description(
5771         "Try to handle requests with abiguous multiple content length headers "
5772         "(Content-Length, Http-Content-Length)."),
5773
5774     Option("rgw_relaxed_region_enforcement", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5775     .set_default(false)
5776     .set_description("Disable region constraint enforcement")
5777     .set_long_description(
5778         "Enable requests such as bucket creation to succeed irrespective of region restrictions (Jewel compat)."),
5779
5780     Option("rgw_lifecycle_work_time", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5781     .set_default("00:00-06:00")
5782     .set_description("Lifecycle allowed work time")
5783     .set_long_description("Local time window in which the lifecycle maintenance thread can work."),
5784
5785     Option("rgw_lc_lock_max_time", Option::TYPE_INT, Option::LEVEL_DEV)
5786     .set_default(60)
5787     .set_description(""),
5788
5789     Option("rgw_lc_thread_delay", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5790     .set_default(0)
5791     .set_description("Delay after processing of bucket listing chunks (i.e., per 1000 entries) in milliseconds"),
5792
5793     Option("rgw_lc_max_objs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5794     .set_default(32)
5795     .set_description("Number of lifecycle data shards")
5796     .set_long_description(
5797           "Number of RADOS objects to use for storing lifecycle index. This can affect "
5798           "concurrency of lifecycle maintenance, but requires multiple RGW processes "
5799           "running on the zone to be utilized."),
5800
5801     Option("rgw_lc_max_rules", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5802     .set_default(1000)
5803     .set_description("Max number of lifecycle rules set on one bucket")
5804     .set_long_description("Number of lifecycle rules set on one bucket should be limited."),
5805
5806     Option("rgw_lc_debug_interval", Option::TYPE_INT, Option::LEVEL_DEV)
5807     .set_default(-1)
5808     .set_description(""),
5809
5810     Option("rgw_mp_lock_max_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5811     .set_default(600)
5812     .set_description("Multipart upload max completion time")
5813     .set_long_description(
5814         "Time length to allow completion of a multipart upload operation. This is done "
5815         "to prevent concurrent completions on the same object with the same upload id."),
5816
5817     Option("rgw_script_uri", Option::TYPE_STR, Option::LEVEL_DEV)
5818     .set_default("")
5819     .set_description(""),
5820
5821     Option("rgw_request_uri", Option::TYPE_STR, Option::LEVEL_DEV)
5822     .set_default("")
5823     .set_description(""),
5824
5825     Option("rgw_ignore_get_invalid_range", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5826     .set_default(false)
5827     .set_description("Treat invalid (e.g., negative) range request as full")
5828     .set_long_description("Treat invalid (e.g., negative) range request "
5829                           "as request for the full object (AWS compatibility)"),
5830
5831     Option("rgw_swift_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5832     .set_default("")
5833     .set_description("Swift-auth storage URL")
5834     .set_long_description(
5835         "Used in conjunction with rgw internal swift authentication. This affects the "
5836         "X-Storage-Url response header value.")
5837     .add_see_also("rgw_swift_auth_entry"),
5838
5839     Option("rgw_swift_url_prefix", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5840     .set_default("swift")
5841     .set_description("Swift URL prefix")
5842     .set_long_description("The URL path prefix for swift requests."),
5843
5844     Option("rgw_swift_auth_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5845     .set_default("")
5846     .set_description("Swift auth URL")
5847     .set_long_description(
5848         "Default url to which RGW connects and verifies tokens for v1 auth (if not using "
5849         "internal swift auth)."),
5850
5851     Option("rgw_swift_auth_entry", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5852     .set_default("auth")
5853     .set_description("Swift auth URL prefix")
5854     .set_long_description("URL path prefix for internal swift auth requests.")
5855     .add_see_also("rgw_swift_url"),
5856
5857     Option("rgw_swift_tenant_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5858     .set_default("")
5859     .set_description("Swift tenant name")
5860     .set_long_description("Tenant name that is used when constructing the swift path.")
5861     .add_see_also("rgw_swift_account_in_url"),
5862
5863     Option("rgw_swift_account_in_url", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5864     .set_default(false)
5865     .set_description("Swift account encoded in URL")
5866     .set_long_description("Whether the swift account is encoded in the uri path (AUTH_<account>).")
5867     .add_see_also("rgw_swift_tenant_name"),
5868
5869     Option("rgw_swift_enforce_content_length", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5870     .set_default(false)
5871     .set_description("Send content length when listing containers (Swift)")
5872     .set_long_description(
5873         "Whether content length header is needed when listing containers. When this is "
5874         "set to false, RGW will send extra info for each entry in the response."),
5875
5876     Option("rgw_keystone_url", Option::TYPE_STR, Option::LEVEL_BASIC)
5877     .set_default("")
5878     .set_description("The URL to the Keystone server."),
5879
5880     Option("rgw_keystone_admin_token", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5881     .set_default("")
5882     .set_description("DEPRECATED: The admin token (shared secret) that is used for the Keystone requests."),
5883
5884     Option("rgw_keystone_admin_token_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5885     .set_default("")
5886     .set_description("Path to a file containing the admin token (shared secret) that is used for the Keystone requests."),
5887
5888     Option("rgw_keystone_admin_user", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5889     .set_default("")
5890     .set_description("Keystone admin user."),
5891
5892     Option("rgw_keystone_admin_password", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5893     .set_default("")
5894     .set_description("DEPRECATED: Keystone admin password."),
5895
5896     Option("rgw_keystone_admin_password_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5897     .set_default("")
5898     .set_description("Path to a file containing the Keystone admin password."),
5899
5900     Option("rgw_keystone_admin_tenant", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5901     .set_default("")
5902     .set_description("Keystone admin user tenant."),
5903
5904     Option("rgw_keystone_admin_project", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5905     .set_default("")
5906     .set_description("Keystone admin user project (for Keystone v3)."),
5907
5908     Option("rgw_keystone_admin_domain", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5909     .set_default("")
5910     .set_description("Keystone admin user domain (for Keystone v3)."),
5911
5912     Option("rgw_keystone_barbican_user", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5913     .set_default("")
5914     .set_description("Keystone user to access barbican secrets."),
5915
5916     Option("rgw_keystone_barbican_password", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5917     .set_default("")
5918     .set_description("Keystone password for barbican user."),
5919
5920     Option("rgw_keystone_barbican_tenant", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5921     .set_default("")
5922     .set_description("Keystone barbican user tenant (Keystone v2.0)."),
5923
5924     Option("rgw_keystone_barbican_project", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5925     .set_default("")
5926     .set_description("Keystone barbican user project (Keystone v3)."),
5927
5928     Option("rgw_keystone_barbican_domain", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5929     .set_default("")
5930     .set_description("Keystone barbican user domain."),
5931
5932     Option("rgw_keystone_api_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5933     .set_default(2)
5934     .set_description("Version of Keystone API to use (2 or 3)."),
5935
5936     Option("rgw_keystone_accepted_roles", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5937     .set_default("Member, admin")
5938     .set_description("Only users with one of these roles will be served when doing Keystone authentication."),
5939
5940     Option("rgw_keystone_accepted_admin_roles", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5941     .set_default("")
5942     .set_description("List of roles allowing user to gain admin privileges (Keystone)."),
5943
5944     Option("rgw_keystone_token_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5945     .set_default(10000)
5946     .set_description("Keystone token cache size")
5947     .set_long_description(
5948         "Max number of Keystone tokens that will be cached. Token that is not cached "
5949         "requires RGW to access the Keystone server when authenticating."),
5950
5951     Option("rgw_keystone_revocation_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5952     .set_default(15_min)
5953     .set_description("Keystone cache revocation interval")
5954     .set_long_description(
5955         "Time (in seconds) that RGW waits between requests to Keystone for getting a list "
5956         "of revoked tokens. A revoked token might still be considered valid by RGW for "
5957         "this amount of time."),
5958
5959     Option("rgw_keystone_verify_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5960     .set_default(true)
5961     .set_description("Should RGW verify the Keystone server SSL certificate."),
5962
5963     Option("rgw_keystone_implicit_tenants", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5964     .set_default(false)
5965     .set_description("RGW Keystone implicit tenants creation")
5966     .set_long_description(
5967         "Implicitly create new users in their own tenant with the same name when "
5968         "authenticating via Keystone."),
5969
5970     Option("rgw_cross_domain_policy", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5971     .set_default("<allow-access-from domain=\"*\" secure=\"false\" />")
5972     .set_description("RGW handle cross domain policy")
5973     .set_long_description("Returned cross domain policy when accessing the crossdomain.xml "
5974                           "resource (Swift compatiility)."),
5975
5976     Option("rgw_healthcheck_disabling_path", Option::TYPE_STR, Option::LEVEL_DEV)
5977     .set_default("")
5978     .set_description("Swift health check api can be disabled if a file can be accessed in this path."),
5979
5980     Option("rgw_s3_auth_use_rados", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5981     .set_default(true)
5982     .set_description("Should S3 authentication use credentials stored in RADOS backend."),
5983
5984     Option("rgw_s3_auth_use_keystone", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5985     .set_default(false)
5986     .set_description("Should S3 authentication use Keystone."),
5987
5988     Option("rgw_s3_auth_order", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5989      .set_default("sts, external, local")
5990      .set_description("Authentication strategy order to use for s3 authentication")
5991      .set_long_description(
5992           "Order of authentication strategies to try for s3 authentication, the allowed "
5993            "options are a comma separated list of engines external, local. The "
5994            "default order is to try all the externally configured engines before "
5995            "attempting local rados based authentication"),
5996
5997     Option("rgw_barbican_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5998     .set_default("")
5999     .set_description("URL to barbican server."),
6000
6001     Option("rgw_ldap_uri", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6002     .set_default("ldaps://<ldap.your.domain>")
6003     .set_description("Space-separated list of LDAP servers in URI format."),
6004
6005     Option("rgw_ldap_binddn", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6006     .set_default("uid=admin,cn=users,dc=example,dc=com")
6007     .set_description("LDAP entry RGW will bind with (user match)."),
6008
6009     Option("rgw_ldap_searchdn", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6010     .set_default("cn=users,cn=accounts,dc=example,dc=com")
6011     .set_description("LDAP search base (basedn)."),
6012
6013     Option("rgw_ldap_dnattr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6014     .set_default("uid")
6015     .set_description("LDAP attribute containing RGW user names (to form binddns)."),
6016
6017     Option("rgw_ldap_secret", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6018     .set_default("/etc/openldap/secret")
6019     .set_description("Path to file containing credentials for rgw_ldap_binddn."),
6020
6021     Option("rgw_s3_auth_use_ldap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6022     .set_default(false)
6023     .set_description("Should S3 authentication use LDAP."),
6024
6025     Option("rgw_ldap_searchfilter", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6026     .set_default("")
6027     .set_description("LDAP search filter."),
6028
6029     Option("rgw_opa_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6030     .set_default("")
6031     .set_description("URL to OPA server."),
6032
6033     Option("rgw_opa_token", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6034     .set_default("")
6035     .set_description("The Bearer token OPA uses to authenticate client requests."),
6036
6037     Option("rgw_opa_verify_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6038     .set_default(true)
6039     .set_description("Should RGW verify the OPA server SSL certificate."),
6040
6041     Option("rgw_use_opa_authz", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6042     .set_default(false)
6043     .set_description("Should OPA be used to authorize client requests."),
6044
6045     Option("rgw_admin_entry", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6046     .set_default("admin")
6047     .set_description("Path prefix to be used for accessing RGW RESTful admin API."),
6048
6049     Option("rgw_enforce_swift_acls", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6050     .set_default(true)
6051     .set_description("RGW enforce swift acls")
6052     .set_long_description(
6053         "Should RGW enforce special Swift-only ACLs. Swift has a special ACL that gives "
6054         "permission to access all objects in a container."),
6055
6056     Option("rgw_swift_token_expiration", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6057     .set_default(1_day)
6058     .set_description("Expiration time (in seconds) for token generated through RGW Swift auth."),
6059
6060     Option("rgw_print_continue", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6061     .set_default(true)
6062     .set_description("RGW support of 100-continue")
6063     .set_long_description(
6064         "Should RGW explicitly send 100 (continue) responses. This is mainly relevant when "
6065         "using FastCGI, as some FastCGI modules do not fully support this feature."),
6066
6067     Option("rgw_print_prohibited_content_length", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6068     .set_default(false)
6069     .set_description("RGW RFC-7230 compatibility")
6070     .set_long_description(
6071         "Specifies whether RGW violates RFC 7230 and sends Content-Length with 204 or 304 "
6072         "statuses."),
6073
6074     Option("rgw_remote_addr_param", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6075     .set_default("REMOTE_ADDR")
6076     .set_description("HTTP header that holds the remote address in incoming requests.")
6077     .set_long_description(
6078         "RGW will use this header to extract requests origin. When RGW runs behind "
6079         "a reverse proxy, the remote address header will point at the proxy's address "
6080         "and not at the originator's address. Therefore it is sometimes possible to "
6081         "have the proxy add the originator's address in a separate HTTP header, which "
6082         "will allow RGW to log it correctly."
6083         )
6084     .add_see_also("rgw_enable_ops_log"),
6085
6086     Option("rgw_op_thread_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
6087     .set_default(10*60)
6088     .set_description("Timeout for async rados coroutine operations."),
6089
6090     Option("rgw_op_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
6091     .set_default(0)
6092     .set_description(""),
6093
6094     Option("rgw_thread_pool_size", Option::TYPE_INT, Option::LEVEL_BASIC)
6095     .set_default(512)
6096     .set_description("RGW requests handling thread pool size.")
6097     .set_long_description(
6098         "This parameter determines the number of concurrent requests RGW can process "
6099         "when using either the civetweb, or the fastcgi frontends. The higher this "
6100         "number is, RGW will be able to deal with more concurrent requests at the "
6101         "cost of more resource utilization."),
6102
6103     Option("rgw_num_control_oids", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6104     .set_default(8)
6105     .set_description("Number of control objects used for cross-RGW communication.")
6106     .set_long_description(
6107         "RGW uses certain control objects to send messages between different RGW "
6108         "processes running on the same zone. These messages include metadata cache "
6109         "invalidation info that is being sent when metadata is modified (such as "
6110         "user or bucket information). A higher number of control objects allows "
6111         "better concurrency of these messages, at the cost of more resource "
6112         "utilization."),
6113
6114     Option("rgw_num_rados_handles", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6115     .set_default(1)
6116     .set_description("Number of librados handles that RGW uses.")
6117     .set_long_description(
6118         "This param affects the number of separate librados handles it uses to "
6119         "connect to the RADOS backend, which directly affects the number of connections "
6120         "RGW will have to each OSD. A higher number affects resource utilization."),
6121
6122     Option("rgw_verify_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6123     .set_default(true)
6124     .set_description("Should RGW verify SSL when connecing to a remote HTTP server")
6125     .set_long_description(
6126         "RGW can send requests to other RGW servers (e.g., in multi-site sync work). "
6127         "This configurable selects whether RGW should verify the certificate for "
6128         "the remote peer and host.")
6129     .add_see_also("rgw_keystone_verify_ssl"),
6130
6131     Option("rgw_nfs_lru_lanes", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6132     .set_default(5)
6133     .set_description(""),
6134
6135     Option("rgw_nfs_lru_lane_hiwat", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6136     .set_default(911)
6137     .set_description(""),
6138
6139     Option("rgw_nfs_fhcache_partitions", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6140     .set_default(3)
6141     .set_description(""),
6142
6143     Option("rgw_nfs_fhcache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6144     .set_default(2017)
6145     .set_description(""),
6146
6147     Option("rgw_nfs_namespace_expire_secs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6148     .set_default(300)
6149     .set_min(1)
6150     .set_description(""),
6151
6152     Option("rgw_nfs_max_gc", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6153     .set_default(300)
6154     .set_min(1)
6155     .set_description(""),
6156
6157     Option("rgw_nfs_write_completion_interval_s", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6158     .set_default(10)
6159     .set_description(""),
6160
6161     Option("rgw_nfs_s3_fast_attrs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6162     .set_default(false)
6163     .set_description("use fast S3 attrs from bucket index (immutable only)")
6164     .set_long_description("use fast S3 attrs from bucket index (assumes NFS "
6165                           "mounts are immutable)"),
6166
6167     Option("rgw_rados_pool_autoscale_bias", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
6168     .set_default(4.0)
6169     .set_min_max(0.01, 100000.0)
6170     .set_description("pg_autoscale_bias value for RGW metadata (omap-heavy) pools"),
6171
6172     Option("rgw_rados_pool_pg_num_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6173     .set_default(8)
6174     .set_min_max(1, 1024)
6175     .set_description("pg_num_min value for RGW metadata (omap-heavy) pools"),
6176
6177     Option("rgw_zone", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6178     .set_default("")
6179     .set_description("Zone name")
6180     .add_see_also({"rgw_zonegroup", "rgw_realm"}),
6181
6182     Option("rgw_zone_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6183     .set_default(".rgw.root")
6184     .set_description("Zone root pool name")
6185     .set_long_description(
6186         "The zone root pool, is the pool where the RGW zone configuration located."
6187     )
6188     .add_see_also({"rgw_zonegroup_root_pool", "rgw_realm_root_pool", "rgw_period_root_pool"}),
6189
6190     Option("rgw_default_zone_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6191     .set_default("default.zone")
6192     .set_description("Default zone info object id")
6193     .set_long_description(
6194         "Name of the RADOS object that holds the default zone information."
6195     ),
6196
6197     Option("rgw_region", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6198     .set_default("")
6199     .set_description("Region name")
6200     .set_long_description(
6201         "Obsolete config option. The rgw_zonegroup option should be used instead.")
6202     .add_see_also("rgw_zonegroup"),
6203
6204     Option("rgw_region_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6205     .set_default(".rgw.root")
6206     .set_description("Region root pool")
6207     .set_long_description(
6208         "Obsolete config option. The rgw_zonegroup_root_pool should be used instead.")
6209     .add_see_also("rgw_zonegroup_root_pool"),
6210
6211     Option("rgw_default_region_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6212     .set_default("default.region")
6213     .set_description("Default region info object id")
6214     .set_long_description(
6215         "Obsolete config option. The rgw_default_zonegroup_info_oid should be used instead.")
6216     .add_see_also("rgw_default_zonegroup_info_oid"),
6217
6218     Option("rgw_zonegroup", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6219     .set_default("")
6220     .set_description("Zonegroup name")
6221     .add_see_also({"rgw_zone", "rgw_realm"}),
6222
6223     Option("rgw_zonegroup_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6224     .set_default(".rgw.root")
6225     .set_description("Zonegroup root pool")
6226     .set_long_description(
6227         "The zonegroup root pool, is the pool where the RGW zonegroup configuration located."
6228     )
6229     .add_see_also({"rgw_zone_root_pool", "rgw_realm_root_pool", "rgw_period_root_pool"}),
6230
6231     Option("rgw_default_zonegroup_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6232     .set_default("default.zonegroup")
6233     .set_description(""),
6234
6235     Option("rgw_realm", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6236     .set_default("")
6237     .set_description(""),
6238
6239     Option("rgw_realm_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6240     .set_default(".rgw.root")
6241     .set_description("Realm root pool")
6242     .set_long_description(
6243         "The realm root pool, is the pool where the RGW realm configuration located."
6244     )
6245     .add_see_also({"rgw_zonegroup_root_pool", "rgw_zone_root_pool", "rgw_period_root_pool"}),
6246
6247     Option("rgw_default_realm_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6248     .set_default("default.realm")
6249     .set_description(""),
6250
6251     Option("rgw_period_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6252     .set_default(".rgw.root")
6253     .set_description("Period root pool")
6254     .set_long_description(
6255         "The period root pool, is the pool where the RGW period configuration located."
6256     )
6257     .add_see_also({"rgw_zonegroup_root_pool", "rgw_zone_root_pool", "rgw_realm_root_pool"}),
6258
6259     Option("rgw_period_latest_epoch_info_oid", Option::TYPE_STR, Option::LEVEL_DEV)
6260     .set_default(".latest_epoch")
6261     .set_description(""),
6262
6263     Option("rgw_log_nonexistent_bucket", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6264     .set_default(false)
6265     .set_description("Should RGW log operations on bucket that does not exist")
6266     .set_long_description(
6267         "This config option applies to the ops log. When this option is set, the ops log "
6268         "will log operations that are sent to non existing buckets. These operations "
6269         "inherently fail, and do not correspond to a specific user.")
6270     .add_see_also("rgw_enable_ops_log"),
6271
6272     Option("rgw_log_object_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6273     .set_default("%Y-%m-%d-%H-%i-%n")
6274     .set_description("Ops log object name format")
6275     .set_long_description(
6276         "Defines the format of the RADOS objects names that ops log uses to store ops "
6277         "log data")
6278     .add_see_also("rgw_enable_ops_log"),
6279
6280     Option("rgw_log_object_name_utc", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6281     .set_default(false)
6282     .set_description("Should ops log object name based on UTC")
6283     .set_long_description(
6284         "If set, the names of the RADOS objects that hold the ops log data will be based "
6285         "on UTC time zone. If not set, it will use the local time zone.")
6286     .add_see_also({"rgw_enable_ops_log", "rgw_log_object_name"}),
6287
6288     Option("rgw_usage_max_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6289     .set_default(32)
6290     .set_description("Number of shards for usage log.")
6291     .set_long_description(
6292         "The number of RADOS objects that RGW will use in order to store the usage log "
6293         "data.")
6294     .add_see_also("rgw_enable_usage_log"),
6295
6296     Option("rgw_usage_max_user_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6297     .set_default(1)
6298     .set_min(1)
6299     .set_description("Number of shards for single user in usage log")
6300     .set_long_description(
6301         "The number of shards that a single user will span over in the usage log.")
6302     .add_see_also("rgw_enable_usage_log"),
6303
6304     Option("rgw_enable_ops_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6305     .set_default(false)
6306     .set_description("Enable ops log")
6307     .add_see_also({"rgw_log_nonexistent_bucket", "rgw_log_object_name", "rgw_ops_log_rados",
6308                "rgw_ops_log_socket_path"}),
6309
6310     Option("rgw_enable_usage_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6311     .set_default(false)
6312     .set_description("Enable usage log")
6313     .add_see_also("rgw_usage_max_shards"),
6314
6315     Option("rgw_ops_log_rados", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6316     .set_default(true)
6317     .set_description("Use RADOS for ops log")
6318     .set_long_description(
6319        "If set, RGW will store ops log information in RADOS.")
6320     .add_see_also({"rgw_enable_ops_log"}),
6321
6322     Option("rgw_ops_log_socket_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6323     .set_default("")
6324     .set_description("Unix domain socket path for ops log.")
6325     .set_long_description(
6326         "Path to unix domain socket that RGW will listen for connection on. When connected, "
6327         "RGW will send ops log data through it.")
6328     .add_see_also({"rgw_enable_ops_log", "rgw_ops_log_data_backlog"}),
6329
6330     Option("rgw_ops_log_data_backlog", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6331     .set_default(5 << 20)
6332     .set_description("Ops log socket backlog")
6333     .set_long_description(
6334         "Maximum amount of data backlog that RGW can keep when ops log is configured to "
6335         "send info through unix domain socket. When data backlog is higher than this, "
6336         "ops log entries will be lost. In order to avoid ops log information loss, the "
6337         "listener needs to clear data (by reading it) quickly enough.")
6338     .add_see_also({"rgw_enable_ops_log", "rgw_ops_log_socket_path"}),
6339
6340     Option("rgw_fcgi_socket_backlog", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6341     .set_default(1024)
6342     .set_description("FastCGI socket connection backlog")
6343     .set_long_description(
6344         "Size of FastCGI connection backlog. This reflects the maximum number of new "
6345         "connection requests that RGW can handle concurrently without dropping any. ")
6346     .add_see_also({"rgw_host", "rgw_socket_path"}),
6347
6348     Option("rgw_usage_log_flush_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6349     .set_default(1024)
6350     .set_description("Number of entries in usage log before flushing")
6351     .set_long_description(
6352         "This is the max number of entries that will be held in the usage log, before it "
6353         "will be flushed to the backend. Note that the usage log is periodically flushed, "
6354         "even if number of entries does not reach this threshold. A usage log entry "
6355         "corresponds to one or more operations on a single bucket.i")
6356     .add_see_also({"rgw_enable_usage_log", "rgw_usage_log_tick_interval"}),
6357
6358     Option("rgw_usage_log_tick_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6359     .set_default(30)
6360     .set_description("Number of seconds between usage log flush cycles")
6361     .set_long_description(
6362         "The number of seconds between consecutive usage log flushes. The usage log will "
6363         "also flush itself to the backend if the number of pending entries reaches a "
6364         "certain threshold.")
6365     .add_see_also({"rgw_enable_usage_log", "rgw_usage_log_flush_threshold"}),
6366
6367     Option("rgw_init_timeout", Option::TYPE_INT, Option::LEVEL_BASIC)
6368     .set_default(300)
6369     .set_description("Initialization timeout")
6370     .set_long_description(
6371         "The time length (in seconds) that RGW will allow for its initialization. RGW "
6372         "process will give up and quit if initialization is not complete after this amount "
6373         "of time."),
6374
6375     Option("rgw_mime_types_file", Option::TYPE_STR, Option::LEVEL_BASIC)
6376     .set_default("/etc/mime.types")
6377     .set_description("Path to local mime types file")
6378     .set_long_description(
6379         "The mime types file is needed in Swift when uploading an object. If object's "
6380         "content type is not specified, RGW will use data from this file to assign "
6381         "a content type to the object."),
6382
6383     Option("rgw_gc_max_objs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6384     .set_default(32)
6385     .set_description("Number of shards for garbage collector data")
6386     .set_long_description(
6387         "The number of garbage collector data shards, is the number of RADOS objects that "
6388         "RGW will use to store the garbage collection information on.")
6389     .add_see_also({"rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_processor_period", "rgw_gc_max_concurrent_io"}),
6390
6391     Option("rgw_gc_obj_min_wait", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6392     .set_default(2_hr)
6393     .set_description("Garbage collection object expiration time")
6394     .set_long_description(
6395        "The length of time (in seconds) that the RGW collector will wait before purging "
6396        "a deleted object's data. RGW will not remove object immediately, as object could "
6397        "still have readers. A mechanism exists to increase the object's expiration time "
6398        "when it's being read.")
6399     .add_see_also({"rgw_gc_max_objs", "rgw_gc_processor_max_time", "rgw_gc_processor_period", "rgw_gc_max_concurrent_io"}),
6400
6401     Option("rgw_gc_processor_max_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6402     .set_default(1_hr)
6403     .set_description("Length of time GC processor can lease shard")
6404     .set_long_description(
6405         "Garbage collection thread in RGW process holds a lease on its data shards. These "
6406         "objects contain the information about the objects that need to be removed. RGW "
6407         "takes a lease in order to prevent multiple RGW processes from handling the same "
6408         "objects concurrently. This time signifies that maximum amount of time (in seconds) that RGW "
6409         "is allowed to hold that lease. In the case where RGW goes down uncleanly, this "
6410         "is the amount of time where processing of that data shard will be blocked.")
6411     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_period", "rgw_gc_max_concurrent_io"}),
6412
6413     Option("rgw_gc_processor_period", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6414     .set_default(1_hr)
6415     .set_description("Garbage collector cycle run time")
6416     .set_long_description(
6417         "The amount of time between the start of consecutive runs of the garbage collector "
6418         "threads. If garbage collector runs takes more than this period, it will not wait "
6419         "before running again.")
6420     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_max_concurrent_io", "rgw_gc_max_trim_chunk"}),
6421
6422     Option("rgw_gc_max_concurrent_io", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6423     .set_default(10)
6424     .set_description("Max concurrent RADOS IO operations for garbage collection")
6425     .set_long_description(
6426         "The maximum number of concurrent IO operations that the RGW garbage collection "
6427         "thread will use when purging old data.")
6428     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_max_trim_chunk"}),
6429
6430     Option("rgw_gc_max_trim_chunk", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6431     .set_default(16)
6432     .set_description("Max number of keys to remove from garbage collector log in a single operation")
6433     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_max_concurrent_io"}),
6434
6435     Option("rgw_s3_success_create_obj_status", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6436     .set_default(0)
6437     .set_description("HTTP return code override for object creation")
6438     .set_long_description(
6439         "If not zero, this is the HTTP return code that will be returned on a successful S3 "
6440         "object creation."),
6441
6442     Option("rgw_resolve_cname", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6443     .set_default(false)
6444     .set_description("Support vanity domain names via CNAME")
6445     .set_long_description(
6446         "If true, RGW will query DNS when detecting that it's serving a request that was "
6447         "sent to a host in another domain. If a CNAME record is configured for that domain "
6448         "it will use it instead. This gives user to have the ability of creating a unique "
6449         "domain of their own to point at data in their bucket."),
6450
6451     Option("rgw_obj_stripe_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6452     .set_default(4_M)
6453     .set_description("RGW object stripe size")
6454     .set_long_description(
6455         "The size of an object stripe for RGW objects. This is the maximum size a backing "
6456         "RADOS object will have. RGW objects that are larger than this will span over "
6457         "multiple objects."),
6458
6459     Option("rgw_extended_http_attrs", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6460     .set_default("")
6461     .set_description("RGW support extended HTTP attrs")
6462     .set_long_description(
6463         "Add new set of attributes that could be set on an object. These extra attributes "
6464         "can be set through HTTP header fields when putting the objects. If set, these "
6465         "attributes will return as HTTP fields when doing GET/HEAD on the object."),
6466
6467     Option("rgw_exit_timeout_secs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6468     .set_default(120)
6469     .set_description("RGW shutdown timeout")
6470     .set_long_description("Number of seconds to wait for a process before exiting unconditionally."),
6471
6472     Option("rgw_get_obj_window_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6473     .set_default(16_M)
6474     .set_description("RGW object read window size")
6475     .set_long_description("The window size in bytes for a single object read request"),
6476
6477     Option("rgw_get_obj_max_req_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6478     .set_default(4_M)
6479     .set_description("RGW object read chunk size")
6480     .set_long_description(
6481         "The maximum request size of a single object read operation sent to RADOS"),
6482
6483     Option("rgw_relaxed_s3_bucket_names", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6484     .set_default(false)
6485     .set_description("RGW enable relaxed S3 bucket names")
6486     .set_long_description("RGW enable relaxed S3 bucket name rules for US region buckets."),
6487
6488     Option("rgw_defer_to_bucket_acls", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6489     .set_default("")
6490     .set_description("Bucket ACLs override object ACLs")
6491     .set_long_description(
6492         "If not empty, a string that selects that mode of operation. 'recurse' will use "
6493         "bucket's ACL for the authorizaton. 'full-control' will allow users that users "
6494         "that have full control permission on the bucket have access to the object."),
6495
6496     Option("rgw_list_buckets_max_chunk", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6497     .set_default(1000)
6498     .set_description("Max number of buckets to retrieve in a single listing operation")
6499     .set_long_description(
6500         "When RGW fetches lists of user's buckets from the backend, this is the max number "
6501         "of entries it will try to retrieve in a single operation. Note that the backend "
6502         "may choose to return a smaller number of entries."),
6503
6504     Option("rgw_md_log_max_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6505     .set_default(64)
6506     .set_description("RGW number of metadata log shards")
6507     .set_long_description(
6508         "The number of shards the RGW metadata log entries will reside in. This affects "
6509         "the metadata sync parallelism as a shard can only be processed by a single "
6510         "RGW at a time"),
6511
6512     Option("rgw_curl_wait_timeout_ms", Option::TYPE_INT, Option::LEVEL_DEV)
6513     .set_default(1000)
6514     .set_description(""),
6515
6516     Option("rgw_curl_low_speed_limit", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6517     .set_default(1024)
6518     .set_long_description(
6519         "It contains the average transfer speed in bytes per second that the "
6520         "transfer should be below during rgw_curl_low_speed_time seconds for libcurl "
6521         "to consider it to be too slow and abort. Set it zero to disable this."),
6522
6523     Option("rgw_curl_low_speed_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6524     .set_default(300)
6525     .set_long_description(
6526         "It contains the time in number seconds that the transfer speed should be below "
6527         "the rgw_curl_low_speed_limit for the library to consider it too slow and abort. "
6528         "Set it zero to disable this."),
6529
6530     Option("rgw_copy_obj_progress", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6531     .set_default(true)
6532     .set_description("Send progress report through copy operation")
6533     .set_long_description(
6534         "If true, RGW will send progress information when copy operation is executed. "),
6535
6536     Option("rgw_copy_obj_progress_every_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6537     .set_default(1_M)
6538     .set_description("Send copy-object progress info after these many bytes"),
6539
6540     Option("rgw_obj_tombstone_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6541     .set_default(1000)
6542     .set_description("Max number of entries to keep in tombstone cache")
6543     .set_long_description(
6544         "The tombstone cache is used when doing a multi-zone data sync. RGW keeps "
6545         "there information about removed objects which is needed in order to prevent "
6546         "re-syncing of objects that were already removed."),
6547
6548     Option("rgw_data_log_window", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6549     .set_default(30)
6550     .set_description("Data log time window")
6551     .set_long_description(
6552         "The data log keeps information about buckets that have objectst that were "
6553         "modified within a specific timeframe. The sync process then knows which buckets "
6554         "are needed to be scanned for data sync."),
6555
6556     Option("rgw_data_log_changes_size", Option::TYPE_INT, Option::LEVEL_DEV)
6557     .set_default(1000)
6558     .set_description("Max size of pending changes in data log")
6559     .set_long_description(
6560         "RGW will trigger update to the data log if the number of pending entries reached "
6561         "this number."),
6562
6563     Option("rgw_data_log_num_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6564     .set_default(128)
6565     .set_description("Number of data log shards")
6566     .set_long_description(
6567         "The number of shards the RGW data log entries will reside in. This affects the "
6568         "data sync parallelism as a shard can only be processed by a single RGW at a time."),
6569
6570     Option("rgw_data_log_obj_prefix", Option::TYPE_STR, Option::LEVEL_DEV)
6571     .set_default("data_log")
6572     .set_description(""),
6573
6574     Option("rgw_bucket_quota_ttl", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6575     .set_default(600)
6576     .set_description("Bucket quota stats cache TTL")
6577     .set_long_description(
6578         "Length of time for bucket stats to be cached within RGW instance."),
6579
6580     Option("rgw_bucket_quota_soft_threshold", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
6581     .set_default(0.95)
6582     .set_description("RGW quota soft threshold")
6583     .set_long_description(
6584         "Threshold from which RGW doesn't rely on cached info for quota "
6585         "decisions. This is done for higher accuracy of the quota mechanism at "
6586         "cost of performance, when getting close to the quota limit. The value "
6587         "configured here is the ratio between the data usage to the max usage "
6588         "as specified by the quota."),
6589
6590     Option("rgw_bucket_quota_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6591     .set_default(10000)
6592     .set_description("RGW quota stats cache size")
6593     .set_long_description(
6594         "Maximum number of entries in the quota stats cache."),
6595
6596     Option("rgw_bucket_default_quota_max_objects", Option::TYPE_INT, Option::LEVEL_BASIC)
6597     .set_default(-1)
6598     .set_description("Default quota for max objects in a bucket")
6599     .set_long_description(
6600         "The default quota configuration for max number of objects in a bucket. A "
6601         "negative number means 'unlimited'."),
6602
6603     Option("rgw_bucket_default_quota_max_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6604     .set_default(-1)
6605     .set_description("Default quota for total size in a bucket")
6606     .set_long_description(
6607         "The default quota configuration for total size of objects in a bucket. A "
6608         "negative number means 'unlimited'."),
6609
6610     Option("rgw_expose_bucket", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6611     .set_default(false)
6612     .set_description("Send Bucket HTTP header with the response")
6613     .set_long_description(
6614         "If true, RGW will send a Bucket HTTP header with the responses. The header will "
6615         "contain the name of the bucket the operation happened on."),
6616
6617     Option("rgw_frontends", Option::TYPE_STR, Option::LEVEL_BASIC)
6618     .set_default("beast port=7480")
6619     .set_description("RGW frontends configuration")
6620     .set_long_description(
6621         "A comma delimited list of frontends configuration. Each configuration contains "
6622         "the type of the frontend followed by an optional space delimited set of "
6623         "key=value config parameters."),
6624
6625     Option("rgw_user_quota_bucket_sync_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6626     .set_default(180)
6627     .set_description("User quota bucket sync interval")
6628     .set_long_description(
6629         "Time period for accumulating modified buckets before syncing these stats."),
6630
6631     Option("rgw_user_quota_sync_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6632     .set_default(1_day)
6633     .set_description("User quota sync interval")
6634     .set_long_description(
6635         "Time period for accumulating modified buckets before syncing entire user stats."),
6636
6637     Option("rgw_user_quota_sync_idle_users", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6638     .set_default(false)
6639     .set_description("Should sync idle users quota")
6640     .set_long_description(
6641         "Whether stats for idle users be fully synced."),
6642
6643     Option("rgw_user_quota_sync_wait_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6644     .set_default(1_day)
6645     .set_description("User quota full-sync wait time")
6646     .set_long_description(
6647         "Minimum time between two full stats sync for non-idle users."),
6648
6649     Option("rgw_user_default_quota_max_objects", Option::TYPE_INT, Option::LEVEL_BASIC)
6650     .set_default(-1)
6651     .set_description("User quota max objects")
6652     .set_long_description(
6653         "The default quota configuration for total number of objects for a single user. A "
6654         "negative number means 'unlimited'."),
6655
6656     Option("rgw_user_default_quota_max_size", Option::TYPE_INT, Option::LEVEL_BASIC)
6657     .set_default(-1)
6658     .set_description("User quota max size")
6659     .set_long_description(
6660         "The default quota configuration for total size of objects for a single user. A "
6661         "negative number means 'unlimited'."),
6662
6663     Option("rgw_multipart_min_part_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6664     .set_default(5_M)
6665     .set_description("Minimum S3 multipart-upload part size")
6666     .set_long_description(
6667         "When doing a multipart upload, each part (other than the last part) should be "
6668         "at least this size."),
6669
6670     Option("rgw_multipart_part_upload_limit", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6671     .set_default(10000)
6672     .set_description("Max number of parts in multipart upload"),
6673
6674     Option("rgw_max_slo_entries", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6675     .set_default(1000)
6676     .set_description("Max number of entries in Swift Static Large Object manifest"),
6677
6678     Option("rgw_olh_pending_timeout_sec", Option::TYPE_INT, Option::LEVEL_DEV)
6679     .set_default(1_hr)
6680     .set_description("Max time for pending OLH change to complete")
6681     .set_long_description(
6682         "OLH is a versioned object's logical head. Operations on it are journaled and "
6683         "as pending before completion. If an operation doesn't complete with this amount "
6684         "of seconds, we remove the operation from the journal."),
6685
6686     Option("rgw_user_max_buckets", Option::TYPE_INT, Option::LEVEL_BASIC)
6687     .set_default(1000)
6688     .set_description("Max number of buckets per user")
6689     .set_long_description(
6690         "A user can create this many buckets. Zero means unlimited, negative number means "
6691         "user cannot create any buckets (although user will retain buckets already created."),
6692
6693     Option("rgw_objexp_gc_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6694     .set_default(10_min)
6695     .set_description("Swift objects expirer garbage collector interval"),
6696
6697     Option("rgw_objexp_hints_num_shards", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6698     .set_default(127)
6699     .set_description("Number of object expirer data shards")
6700     .set_long_description(
6701         "The number of shards the (Swift) object expirer will store its data on."),
6702
6703     Option("rgw_objexp_chunk_size", Option::TYPE_UINT, Option::LEVEL_DEV)
6704     .set_default(100)
6705     .set_description(""),
6706
6707     Option("rgw_enable_static_website", Option::TYPE_BOOL, Option::LEVEL_BASIC)
6708     .set_default(false)
6709     .set_description("Enable static website APIs")
6710     .set_long_description(
6711         "This configurable controls whether RGW handles the website control APIs. RGW can "
6712         "server static websites if s3website hostnames are configured, and unrelated to "
6713         "this configurable."),
6714
6715      Option("rgw_user_unique_email", Option::TYPE_BOOL, Option::LEVEL_BASIC)
6716     .set_default(true)
6717     .set_description("Require local RGW users to have unique email addresses")
6718     .set_long_description(
6719         "Enforce builtin user accounts to have unique email addresses.  This "
6720         "setting is historical.  In future, non-enforcement of email address "
6721         "uniqueness is likely to become the default."),
6722
6723     Option("rgw_log_http_headers", Option::TYPE_STR, Option::LEVEL_BASIC)
6724     .set_default("")
6725     .set_description("List of HTTP headers to log")
6726     .set_long_description(
6727         "A comma delimited list of HTTP headers to log when seen, ignores case (e.g., "
6728         "http_x_forwarded_for)."),
6729
6730     Option("rgw_num_async_rados_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6731     .set_default(32)
6732     .set_description("Number of concurrent RADOS operations in multisite sync")
6733     .set_long_description(
6734         "The number of concurrent RADOS IO operations that will be triggered for handling "
6735         "multisite sync operations. This includes control related work, and not the actual "
6736         "sync operations."),
6737
6738     Option("rgw_md_notify_interval_msec", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6739     .set_default(200)
6740     .set_description("Length of time to aggregate metadata changes")
6741     .set_long_description(
6742         "Length of time (in milliseconds) in which the master zone aggregates all the "
6743         "metadata changes that occurred, before sending notifications to all the other "
6744         "zones."),
6745
6746     Option("rgw_run_sync_thread", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6747     .set_default(true)
6748     .set_description("Should run sync thread"),
6749
6750     Option("rgw_sync_lease_period", Option::TYPE_INT, Option::LEVEL_DEV)
6751     .set_default(120)
6752     .set_description(""),
6753
6754     Option("rgw_sync_log_trim_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6755     .set_default(1200)
6756     .set_description("Sync log trim interval")
6757     .set_long_description(
6758         "Time in seconds between attempts to trim sync logs."),
6759
6760     Option("rgw_sync_log_trim_max_buckets", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6761     .set_default(16)
6762     .set_description("Maximum number of buckets to trim per interval")
6763     .set_long_description("The maximum number of buckets to consider for bucket index log trimming each trim interval, regardless of the number of bucket index shards. Priority is given to buckets with the most sync activity over the last trim interval.")
6764     .add_see_also("rgw_sync_log_trim_interval")
6765     .add_see_also("rgw_sync_log_trim_min_cold_buckets")
6766     .add_see_also("rgw_sync_log_trim_concurrent_buckets"),
6767
6768     Option("rgw_sync_log_trim_min_cold_buckets", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6769     .set_default(4)
6770     .set_description("Minimum number of cold buckets to trim per interval")
6771     .set_long_description("Of the `rgw_sync_log_trim_max_buckets` selected for bucket index log trimming each trim interval, at least this many of them must be 'cold' buckets. These buckets are selected in order from the list of all bucket instances, to guarantee that all buckets will be visited eventually.")
6772     .add_see_also("rgw_sync_log_trim_interval")
6773     .add_see_also("rgw_sync_log_trim_max_buckets")
6774     .add_see_also("rgw_sync_log_trim_concurrent_buckets"),
6775
6776     Option("rgw_sync_log_trim_concurrent_buckets", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6777     .set_default(4)
6778     .set_description("Maximum number of buckets to trim in parallel")
6779     .add_see_also("rgw_sync_log_trim_interval")
6780     .add_see_also("rgw_sync_log_trim_max_buckets")
6781     .add_see_also("rgw_sync_log_trim_min_cold_buckets"),
6782
6783     Option("rgw_sync_data_inject_err_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
6784     .set_default(0)
6785     .set_description(""),
6786
6787     Option("rgw_sync_meta_inject_err_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
6788     .set_default(0)
6789     .set_description(""),
6790
6791     Option("rgw_sync_trace_history_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6792     .set_default(4096)
6793     .set_description("Sync trace history size")
6794     .set_long_description(
6795       "Maximum number of complete sync trace entries to keep."),
6796
6797     Option("rgw_sync_trace_per_node_log_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6798     .set_default(32)
6799     .set_description("Sync trace per-node log size")
6800     .set_long_description(
6801         "The number of log entries to keep per sync-trace node."),
6802
6803     Option("rgw_sync_trace_servicemap_update_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6804     .set_default(10)
6805     .set_description("Sync-trace service-map update interval")
6806     .set_long_description(
6807         "Number of seconds between service-map updates of sync-trace events."),
6808
6809     Option("rgw_period_push_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
6810     .set_default(2)
6811     .set_description("Period push interval")
6812     .set_long_description(
6813         "Number of seconds to wait before retrying 'period push' operation."),
6814
6815     Option("rgw_period_push_interval_max", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
6816     .set_default(30)
6817     .set_description("Period push maximum interval")
6818     .set_long_description(
6819         "The max number of seconds to wait before retrying 'period push' after exponential "
6820         "backoff."),
6821
6822     Option("rgw_safe_max_objects_per_shard", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6823     .set_default(100*1024)
6824     .set_description("Safe number of objects per shard")
6825     .set_long_description(
6826         "This is the max number of objects per bucket index shard that RGW considers "
6827         "safe. RGW will warn if it identifies a bucket where its per-shard count is "
6828         "higher than a percentage of this number.")
6829     .add_see_also("rgw_shard_warning_threshold"),
6830
6831     Option("rgw_shard_warning_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
6832     .set_default(90)
6833     .set_description("Warn about max objects per shard")
6834     .set_long_description(
6835         "Warn if number of objects per shard in a specific bucket passed this percentage "
6836         "of the safe number.")
6837     .add_see_also("rgw_safe_max_objects_per_shard"),
6838
6839     Option("rgw_swift_versioning_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6840     .set_default(false)
6841     .set_description("Enable Swift versioning"),
6842
6843     Option("rgw_swift_custom_header", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6844     .set_default("")
6845     .set_description("Enable swift custom header")
6846     .set_long_description(
6847         "If not empty, specifies a name of HTTP header that can include custom data. When "
6848         "uploading an object, if this header is passed RGW will store this header info "
6849         "and it will be available when listing the bucket."),
6850
6851     Option("rgw_swift_need_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6852     .set_default(true)
6853     .set_description("Enable stats on bucket listing in Swift"),
6854
6855     Option("rgw_reshard_num_logs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6856     .set_default(16)
6857     .set_min(1)
6858     .set_description("")
6859     .add_service("rgw"),
6860
6861     Option("rgw_reshard_bucket_lock_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6862     .set_default(360)
6863     .set_min(30)
6864     .set_description("Number of seconds the timeout on the reshard locks (bucket reshard lock and reshard log lock) are set to. As a reshard proceeds these locks can be renewed/extended. If too short, reshards cannot complete and will fail, causing a future reshard attempt. If too long a hung or crashed reshard attempt will keep the bucket locked for an extended period, not allowing RGW to detect the failed reshard attempt and recover.")
6865     .add_tag("performance")
6866     .add_service("rgw"),
6867
6868     Option("rgw_reshard_batch_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6869     .set_default(64)
6870     .set_min(8)
6871     .set_description("Number of reshard entries to batch together before sending the operations to the CLS back-end")
6872     .add_tag("performance")
6873     .add_service("rgw"),
6874
6875     Option("rgw_reshard_max_aio", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6876     .set_default(128)
6877     .set_min(16)
6878     .set_description("Maximum number of outstanding asynchronous I/O operations to allow at a time during resharding")
6879     .add_tag("performance")
6880     .add_service("rgw"),
6881
6882     Option("rgw_trust_forwarded_https", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6883     .set_default(false)
6884     .set_description("Trust Forwarded and X-Forwarded-Proto headers")
6885     .set_long_description(
6886         "When a proxy in front of radosgw is used for ssl termination, radosgw "
6887         "does not know whether incoming http connections are secure. Enable "
6888         "this option to trust the Forwarded and X-Forwarded-Proto headers sent "
6889         "by the proxy when determining whether the connection is secure. This "
6890         "is required for some features, such as server side encryption.")
6891     .add_see_also("rgw_crypt_require_ssl"),
6892
6893     Option("rgw_crypt_require_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6894     .set_default(true)
6895     .set_description("Requests including encryption key headers must be sent over ssl"),
6896
6897     Option("rgw_crypt_default_encryption_key", Option::TYPE_STR, Option::LEVEL_DEV)
6898     .set_default("")
6899     .set_description(""),
6900
6901     Option("rgw_crypt_s3_kms_encryption_keys", Option::TYPE_STR, Option::LEVEL_DEV)
6902     .set_default("")
6903     .set_description(""),
6904
6905     Option("rgw_crypt_suppress_logs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6906     .set_default(true)
6907     .set_description("Suppress logs that might print client key"),
6908
6909     Option("rgw_list_bucket_min_readahead", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6910     .set_default(1000)
6911     .set_description("Minimum number of entries to request from rados for bucket listing"),
6912
6913     Option("rgw_rest_getusage_op_compat", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6914     .set_default(false)
6915     .set_description("REST GetUsage request backward compatibility"),
6916
6917     Option("rgw_torrent_flag", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6918     .set_default(false)
6919     .set_description("When true, uploaded objects will calculate and store "
6920                      "a SHA256 hash of object data so the object can be "
6921                      "retrieved as a torrent file"),
6922
6923     Option("rgw_torrent_tracker", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6924     .set_default("")
6925     .set_description("Torrent field announce and announce list"),
6926
6927     Option("rgw_torrent_createby", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6928     .set_default("")
6929     .set_description("torrent field created by"),
6930
6931     Option("rgw_torrent_comment", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6932     .set_default("")
6933     .set_description("Torrent field comment"),
6934
6935     Option("rgw_torrent_encoding", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6936     .set_default("")
6937     .set_description("torrent field encoding"),
6938
6939     Option("rgw_data_notify_interval_msec", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6940     .set_default(200)
6941     .set_description("data changes notification interval to followers"),
6942
6943     Option("rgw_torrent_origin", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6944     .set_default("")
6945     .set_description("Torrent origin"),
6946
6947     Option("rgw_torrent_sha_unit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6948     .set_default(512*1024)
6949     .set_description(""),
6950
6951     Option("rgw_dynamic_resharding", Option::TYPE_BOOL, Option::LEVEL_BASIC)
6952     .set_default(true)
6953     .set_description("Enable dynamic resharding")
6954     .set_long_description(
6955         "If true, RGW will dynamicall increase the number of shards in buckets that have "
6956         "a high number of objects per shard.")
6957     .add_see_also("rgw_max_objs_per_shard"),
6958
6959     Option("rgw_max_objs_per_shard", Option::TYPE_UINT, Option::LEVEL_BASIC)
6960     .set_default(100000)
6961     .set_description("Max objects per shard for dynamic resharding")
6962     .set_long_description(
6963         "This is the max number of objects per bucket index shard that RGW will "
6964         "allow with dynamic resharding. RGW will trigger an automatic reshard operation "
6965         "on the bucket if it exceeds this number.")
6966     .add_see_also("rgw_dynamic_resharding"),
6967
6968     Option("rgw_reshard_thread_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6969     .set_default(10_min)
6970     .set_min(10_min)
6971     .set_description("Number of seconds between processing of reshard log entries"),
6972
6973     Option("rgw_cache_expiry_interval", Option::TYPE_UINT,
6974            Option::LEVEL_ADVANCED)
6975     .set_default(15_min)
6976     .set_description("Number of seconds before entries in the cache are "
6977                      "assumed stale and re-fetched. Zero is never.")
6978     .add_tag("performance")
6979     .add_service("rgw")
6980     .set_long_description("The Rados Gateway stores metadata and objects in "
6981                           "an internal cache. This should be kept consistent "
6982                           "by the OSD's relaying notify events between "
6983                           "multiple watching RGW processes. In the event "
6984                           "that this notification protocol fails, bounding "
6985                           "the length of time that any data in the cache will "
6986                           "be assumed valid will ensure that any RGW instance "
6987                           "that falls out of sync will eventually recover. "
6988                           "This seems to be an issue mostly for large numbers "
6989                           "of RGW instances under heavy use. If you would like "
6990                           "to turn off cache expiry, set this value to zero."),
6991
6992     Option("rgw_inject_notify_timeout_probability", Option::TYPE_FLOAT,
6993            Option::LEVEL_DEV)
6994     .set_default(0)
6995     .add_tag("fault injection")
6996     .add_tag("testing")
6997     .add_service("rgw")
6998     .set_min_max(0.0, 1.0)
6999     .set_description("Likelihood of ignoring a notify")
7000     .set_long_description("This is the probability that the RGW cache will "
7001                           "ignore a cache notify message. It exists to help "
7002                           "with the development and testing of cache "
7003                           "consistency and recovery improvements. Please "
7004                           "do not set it in a production cluster, as it "
7005                           "actively causes failures. Set this to a floating "
7006                           "point value between 0 and 1."),
7007     Option("rgw_max_notify_retries", Option::TYPE_UINT,
7008            Option::LEVEL_ADVANCED)
7009     .set_default(3)
7010     .add_tag("error recovery")
7011     .add_service("rgw")
7012     .set_description("Number of attempts to notify peers before giving up.")
7013     .set_long_description("The number of times we will attempt to update "
7014                           "a peer's cache in the event of error before giving "
7015                           "up. This is unlikely to be an issue unless your "
7016                           "cluster is very heavily loaded. Beware that "
7017                           "increasing this value may cause some operations to "
7018                           "take longer in exceptional cases and thus may, "
7019                           "rarely, cause clients to time out."),
7020     Option("rgw_sts_entry", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7021     .set_default("sts")
7022     .set_description("STS URL prefix")
7023     .set_long_description("URL path prefix for internal STS requests."),
7024
7025     Option("rgw_sts_key", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7026     .set_default("sts")
7027     .set_description("STS Key")
7028     .set_long_description("Key used for encrypting/ decrypting session token."),
7029
7030     Option("rgw_s3_auth_use_sts", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7031     .set_default(false)
7032     .set_description("Should S3 authentication use STS."),
7033
7034     Option("rgw_sts_max_session_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7035     .set_default(43200)
7036     .set_description("Session token max duration")
7037     .set_long_description("Max duration in seconds for which the session token is valid."),
7038
7039     Option("rgw_max_listing_results", Option::TYPE_UINT,
7040            Option::LEVEL_ADVANCED)
7041     .set_default(1000)
7042     .set_min_max(1, 100000)
7043     .add_service("rgw")
7044     .set_description("Upper bound on results in listing operations, ListBucket max-keys")
7045     .set_long_description("This caps the maximum permitted value for listing-like operations in RGW S3. "
7046                           "Affects ListBucket(max-keys), "
7047                           "ListBucketVersions(max-keys), "
7048                           "ListBucketMultiPartUploads(max-uploads), "
7049                           "ListMultipartUploadParts(max-parts)"),
7050
7051     Option("rgw_sts_token_introspection_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7052     .set_default("")
7053     .set_description("STS Web Token introspection URL")
7054     .set_long_description("URL for introspecting an STS Web Token."),
7055
7056     Option("rgw_sts_client_id", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7057     .set_default("")
7058     .set_description("Client Id")
7059     .set_long_description("Client Id needed for introspecting a Web Token."),
7060
7061     Option("rgw_sts_client_secret", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7062     .set_default("")
7063     .set_description("Client Secret")
7064     .set_long_description("Client Secret needed for introspecting a Web Token."),
7065
7066     Option("rgw_max_concurrent_requests", Option::TYPE_INT, Option::LEVEL_BASIC)
7067     .set_default(1024)
7068     .set_description("Maximum number of concurrent HTTP requests.")
7069     .set_long_description(
7070         "Maximum number of concurrent HTTP requests that the beast frontend "
7071         "will process. Tuning this can help to limit memory usage under heavy "
7072         "load.")
7073     .add_tag("performance")
7074     .add_see_also("rgw_frontends"),
7075
7076     Option("rgw_scheduler_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7077     .set_default("throttler")
7078     .set_description("Set the type of dmclock scheduler, defaults to throttler "
7079                      "Other valid values are dmclock which is experimental"),
7080
7081     Option("rgw_dmclock_admin_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7082     .set_default(100.0)
7083     .set_description("mclock reservation for admin requests")
7084     .add_see_also("rgw_dmclock_admin_wgt")
7085     .add_see_also("rgw_dmclock_admin_lim"),
7086
7087     Option("rgw_dmclock_admin_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7088     .set_default(100.0)
7089     .set_description("mclock weight for admin requests")
7090     .add_see_also("rgw_dmclock_admin_res")
7091     .add_see_also("rgw_dmclock_admin_lim"),
7092
7093     Option("rgw_dmclock_admin_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7094     .set_default(0.0)
7095     .set_description("mclock limit for admin requests")
7096     .add_see_also("rgw_dmclock_admin_res")
7097     .add_see_also("rgw_dmclock_admin_wgt"),
7098
7099     Option("rgw_dmclock_auth_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7100     .set_default(200.0)
7101     .set_description("mclock reservation for object data requests")
7102     .add_see_also("rgw_dmclock_auth_wgt")
7103     .add_see_also("rgw_dmclock_auth_lim"),
7104
7105     Option("rgw_dmclock_auth_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7106     .set_default(100.0)
7107     .set_description("mclock weight for object data requests")
7108     .add_see_also("rgw_dmclock_auth_res")
7109     .add_see_also("rgw_dmclock_auth_lim"),
7110
7111     Option("rgw_dmclock_auth_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7112     .set_default(0.0)
7113     .set_description("mclock limit for object data requests")
7114     .add_see_also("rgw_dmclock_auth_res")
7115     .add_see_also("rgw_dmclock_auth_wgt"),
7116
7117     Option("rgw_dmclock_data_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7118     .set_default(500.0)
7119     .set_description("mclock reservation for object data requests")
7120     .add_see_also("rgw_dmclock_data_wgt")
7121     .add_see_also("rgw_dmclock_data_lim"),
7122
7123     Option("rgw_dmclock_data_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7124     .set_default(500.0)
7125     .set_description("mclock weight for object data requests")
7126     .add_see_also("rgw_dmclock_data_res")
7127     .add_see_also("rgw_dmclock_data_lim"),
7128
7129     Option("rgw_dmclock_data_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7130     .set_default(0.0)
7131     .set_description("mclock limit for object data requests")
7132     .add_see_also("rgw_dmclock_data_res")
7133     .add_see_also("rgw_dmclock_data_wgt"),
7134
7135     Option("rgw_dmclock_metadata_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7136     .set_default(500.0)
7137     .set_description("mclock reservation for metadata requests")
7138     .add_see_also("rgw_dmclock_metadata_wgt")
7139     .add_see_also("rgw_dmclock_metadata_lim"),
7140
7141     Option("rgw_dmclock_metadata_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7142     .set_default(500.0)
7143     .set_description("mclock weight for metadata requests")
7144     .add_see_also("rgw_dmclock_metadata_res")
7145     .add_see_also("rgw_dmclock_metadata_lim"),
7146
7147     Option("rgw_dmclock_metadata_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7148     .set_default(0.0)
7149     .set_description("mclock limit for metadata requests")
7150     .add_see_also("rgw_dmclock_metadata_res")
7151     .add_see_also("rgw_dmclock_metadata_wgt"),
7152
7153   });
7154 }
7155
7156 static std::vector<Option> get_rbd_options() {
7157   return std::vector<Option>({
7158     Option("rbd_default_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7159     .set_default("rbd")
7160     .set_description("default pool for storing new images")
7161     .set_validator([](std::string *value, std::string *error_message){
7162       std::regex pattern("^[^@/]+$");
7163       if (!std::regex_match (*value, pattern)) {
7164         *value = "rbd";
7165         *error_message = "invalid RBD default pool, resetting to 'rbd'";
7166       }
7167       return 0;
7168     }),
7169
7170     Option("rbd_default_data_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7171     .set_default("")
7172     .set_description("default pool for storing data blocks for new images")
7173     .set_validator([](std::string *value, std::string *error_message){
7174       std::regex pattern("^[^@/]*$");
7175       if (!std::regex_match (*value, pattern)) {
7176         *value = "";
7177         *error_message = "ignoring invalid RBD data pool";
7178       }
7179       return 0;
7180     }),
7181
7182     Option("rbd_default_features", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7183     .set_default("layering,exclusive-lock,object-map,fast-diff,deep-flatten")
7184     .set_description("default v2 image features for new images")
7185     .set_long_description(
7186         "RBD features are only applicable for v2 images. This setting accepts "
7187         "either an integer bitmask value or comma-delimited string of RBD "
7188         "feature names. This setting is always internally stored as an integer "
7189         "bitmask value. The mapping between feature bitmask value and feature "
7190         "name is as follows: +1 -> layering, +2 -> striping, "
7191         "+4 -> exclusive-lock, +8 -> object-map, +16 -> fast-diff, "
7192         "+32 -> deep-flatten, +64 -> journaling, +128 -> data-pool")
7193     .set_flag(Option::FLAG_RUNTIME)
7194     .set_validator([](std::string *value, std::string *error_message) {
7195         ostringstream ss;
7196         uint64_t features = librbd::rbd_features_from_string(*value, &ss);
7197         // Leave this in integer form to avoid breaking Cinder.  Someday
7198         // we would like to present this in string form instead...
7199         *value = stringify(features);
7200         if (ss.str().size()) {
7201           return -EINVAL;
7202         }
7203         return 0;
7204       }),
7205
7206     Option("rbd_op_threads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7207     .set_default(1)
7208     .set_description("number of threads to utilize for internal processing"),
7209
7210     Option("rbd_op_thread_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7211     .set_default(60)
7212     .set_description("time in seconds for detecting a hung thread"),
7213
7214     Option("rbd_non_blocking_aio", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7215     .set_default(true)
7216     .set_description("process AIO ops from a dispatch thread to prevent blocking"),
7217
7218     Option("rbd_cache", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7219     .set_default(true)
7220     .set_description("whether to enable caching (writeback unless rbd_cache_max_dirty is 0)"),
7221
7222     Option("rbd_cache_writethrough_until_flush", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7223     .set_default(true)
7224     .set_description("whether to make writeback caching writethrough until "
7225                      "flush is called, to be sure the user of librbd will send "
7226                      "flushes so that writeback is safe"),
7227
7228     Option("rbd_cache_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7229     .set_default(32_M)
7230     .set_description("cache size in bytes"),
7231
7232     Option("rbd_cache_max_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7233     .set_default(24_M)
7234     .set_description("dirty limit in bytes - set to 0 for write-through caching"),
7235
7236     Option("rbd_cache_target_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7237     .set_default(16_M)
7238     .set_description("target dirty limit in bytes"),
7239
7240     Option("rbd_cache_max_dirty_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7241     .set_default(1.0)
7242     .set_description("seconds in cache before writeback starts"),
7243
7244     Option("rbd_cache_max_dirty_object", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7245     .set_default(0)
7246     .set_description("dirty limit for objects - set to 0 for auto calculate from rbd_cache_size"),
7247
7248     Option("rbd_cache_block_writes_upfront", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7249     .set_default(false)
7250     .set_description("whether to block writes to the cache before the aio_write call completes"),
7251
7252     Option("rbd_concurrent_management_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7253     .set_default(10)
7254     .set_min(1)
7255     .set_description("how many operations can be in flight for a management operation like deleting or resizing an image"),
7256
7257     Option("rbd_balance_snap_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7258     .set_default(false)
7259     .set_description("distribute snap read requests to random OSD"),
7260
7261     Option("rbd_localize_snap_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7262     .set_default(false)
7263     .set_description("localize snap read requests to closest OSD"),
7264
7265     Option("rbd_balance_parent_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7266     .set_default(false)
7267     .set_description("distribute parent read requests to random OSD"),
7268
7269     Option("rbd_localize_parent_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7270     .set_default(false)
7271     .set_description("localize parent requests to closest OSD"),
7272
7273     Option("rbd_sparse_read_threshold_bytes", Option::TYPE_SIZE,
7274            Option::LEVEL_ADVANCED)
7275     .set_default(64_K)
7276     .set_description("threshold for issuing a sparse-read")
7277     .set_long_description("minimum number of sequential bytes to read against "
7278                           "an object before issuing a sparse-read request to "
7279                           "the cluster. 0 implies it must be a full object read "
7280                           "to issue a sparse-read, 1 implies always use "
7281                           "sparse-read, and any value larger than the maximum "
7282                           "object size will disable sparse-read for all "
7283                           "requests"),
7284
7285     Option("rbd_readahead_trigger_requests", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7286     .set_default(10)
7287     .set_description("number of sequential requests necessary to trigger readahead"),
7288
7289     Option("rbd_readahead_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7290     .set_default(512_K)
7291     .set_description("set to 0 to disable readahead"),
7292
7293     Option("rbd_readahead_disable_after_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7294     .set_default(50_M)
7295     .set_description("how many bytes are read in total before readahead is disabled"),
7296
7297     Option("rbd_clone_copy_on_read", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7298     .set_default(false)
7299     .set_description("copy-up parent image blocks to clone upon read request"),
7300
7301     Option("rbd_blacklist_on_break_lock", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7302     .set_default(true)
7303     .set_description("whether to blacklist clients whose lock was broken"),
7304
7305     Option("rbd_blacklist_expire_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7306     .set_default(0)
7307     .set_description("number of seconds to blacklist - set to 0 for OSD default"),
7308
7309     Option("rbd_request_timed_out_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7310     .set_default(30)
7311     .set_description("number of seconds before maintenance request times out"),
7312
7313     Option("rbd_skip_partial_discard", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7314     .set_default(true)
7315     .set_description("skip discard (zero) of unaligned extents within an object"),
7316
7317     Option("rbd_discard_granularity_bytes", Option::TYPE_UINT,
7318            Option::LEVEL_ADVANCED)
7319     .set_default(64_K)
7320     .set_min_max(4_K, 32_M)
7321     .set_validator([](std::string *value, std::string *error_message){
7322         uint64_t f = strict_si_cast<uint64_t>(value->c_str(), error_message);
7323         if (!error_message->empty()) {
7324           return -EINVAL;
7325         } else if (!isp2(f)) {
7326           *error_message = "value must be a power of two";
7327           return -EINVAL;
7328         }
7329         return 0;
7330       })
7331     .set_description("minimum aligned size of discard operations"),
7332
7333     Option("rbd_enable_alloc_hint", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7334     .set_default(true)
7335     .set_description("when writing a object, it will issue a hint to osd backend to indicate the expected size object need"),
7336
7337     Option("rbd_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7338     .set_default(false)
7339     .set_description("true if LTTng-UST tracepoints should be enabled"),
7340
7341     Option("rbd_blkin_trace_all", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7342     .set_default(false)
7343     .set_description("create a blkin trace for all RBD requests"),
7344
7345     Option("rbd_validate_pool", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7346     .set_default(true)
7347     .set_description("validate empty pools for RBD compatibility"),
7348
7349     Option("rbd_validate_names", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7350     .set_default(true)
7351     .set_description("validate new image names for RBD compatibility"),
7352
7353     Option("rbd_auto_exclusive_lock_until_manual_request", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7354     .set_default(true)
7355     .set_description("automatically acquire/release exclusive lock until it is explicitly requested"),
7356
7357     Option("rbd_move_to_trash_on_remove", Option::TYPE_BOOL, Option::LEVEL_BASIC)
7358     .set_default(false)
7359     .set_description("automatically move images to the trash when deleted"),
7360
7361     Option("rbd_move_to_trash_on_remove_expire_seconds", Option::TYPE_UINT, Option::LEVEL_BASIC)
7362     .set_default(0)
7363     .set_description("default number of seconds to protect deleted images in the trash"),
7364
7365     Option("rbd_mirroring_resync_after_disconnect", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7366     .set_default(false)
7367     .set_description("automatically start image resync after mirroring is disconnected due to being laggy"),
7368
7369     Option("rbd_mirroring_delete_delay", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7370     .set_default(0)
7371     .set_description("time-delay in seconds for rbd-mirror delete propagation"),
7372
7373     Option("rbd_mirroring_replay_delay", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7374     .set_default(0)
7375     .set_description("time-delay in seconds for rbd-mirror asynchronous replication"),
7376
7377     Option("rbd_default_format", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7378     .set_default(2)
7379     .set_description("default image format for new images"),
7380
7381     Option("rbd_default_order", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7382     .set_default(22)
7383     .set_description("default order (data block object size) for new images"),
7384
7385     Option("rbd_default_stripe_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7386     .set_default(0)
7387     .set_description("default stripe count for new images"),
7388
7389     Option("rbd_default_stripe_unit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7390     .set_default(0)
7391     .set_description("default stripe width for new images"),
7392
7393     Option("rbd_default_map_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7394     .set_default("")
7395     .set_description("default krbd map options"),
7396
7397     Option("rbd_default_clone_format", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7398     .set_enum_allowed({"1", "2", "auto"})
7399     .set_default("auto")
7400     .set_description("default internal format for handling clones")
7401     .set_long_description("This sets the internal format for tracking cloned "
7402                           "images. The setting of '1' requires attaching to "
7403                           "protected snapshots that cannot be removed until "
7404                           "the clone is removed/flattened. The setting of '2' "
7405                           "will allow clones to be attached to any snapshot "
7406                           "and permits removing in-use parent snapshots but "
7407                           "requires Mimic or later clients. The default "
7408                           "setting of 'auto' will use the v2 format if the "
7409                           "cluster is configured to require mimic or later "
7410                           "clients.")
7411     .set_flag(Option::FLAG_RUNTIME),
7412
7413     Option("rbd_journal_order", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7414     .set_min_max(12, 26)
7415     .set_default(24)
7416     .set_description("default order (object size) for journal data objects"),
7417
7418     Option("rbd_journal_splay_width", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7419     .set_default(4)
7420     .set_description("number of active journal objects"),
7421
7422     Option("rbd_journal_commit_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7423     .set_default(5)
7424     .set_description("commit time interval, seconds"),
7425
7426     Option("rbd_journal_object_writethrough_until_flush", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7427     .set_default(true)
7428     .set_description("when enabled, the rbd_journal_object_flush* configuration "
7429                      "options are ignored until the first flush so that batched "
7430                      "journal IO is known to be safe for consistency"),
7431
7432     Option("rbd_journal_object_flush_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7433     .set_default(0)
7434     .set_description("maximum number of pending commits per journal object"),
7435
7436     Option("rbd_journal_object_flush_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7437     .set_default(1_M)
7438     .set_description("maximum number of pending bytes per journal object"),
7439
7440     Option("rbd_journal_object_flush_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7441     .set_default(0)
7442     .set_description("maximum age (in seconds) for pending commits"),
7443
7444     Option("rbd_journal_object_max_in_flight_appends", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7445     .set_default(0)
7446     .set_description("maximum number of in-flight appends per journal object"),
7447
7448     Option("rbd_journal_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7449     .set_default("")
7450     .set_description("pool for journal objects"),
7451
7452     Option("rbd_journal_max_payload_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7453     .set_default(16384)
7454     .set_description("maximum journal payload size before splitting"),
7455
7456     Option("rbd_journal_max_concurrent_object_sets", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7457     .set_default(0)
7458     .set_description("maximum number of object sets a journal client can be behind before it is automatically unregistered"),
7459
7460     Option("rbd_qos_iops_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7461     .set_default(0)
7462     .set_description("the desired limit of IO operations per second"),
7463
7464     Option("rbd_qos_bps_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7465     .set_default(0)
7466     .set_description("the desired limit of IO bytes per second"),
7467
7468     Option("rbd_qos_read_iops_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7469     .set_default(0)
7470     .set_description("the desired limit of read operations per second"),
7471
7472     Option("rbd_qos_write_iops_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7473     .set_default(0)
7474     .set_description("the desired limit of write operations per second"),
7475
7476     Option("rbd_qos_read_bps_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7477     .set_default(0)
7478     .set_description("the desired limit of read bytes per second"),
7479
7480     Option("rbd_qos_write_bps_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7481     .set_default(0)
7482     .set_description("the desired limit of write bytes per second"),
7483
7484     Option("rbd_qos_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7485     .set_default(0)
7486     .set_description("the desired burst limit of IO operations"),
7487
7488     Option("rbd_qos_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7489     .set_default(0)
7490     .set_description("the desired burst limit of IO bytes"),
7491
7492     Option("rbd_qos_read_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7493     .set_default(0)
7494     .set_description("the desired burst limit of read operations"),
7495
7496     Option("rbd_qos_write_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7497     .set_default(0)
7498     .set_description("the desired burst limit of write operations"),
7499
7500     Option("rbd_qos_read_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7501     .set_default(0)
7502     .set_description("the desired burst limit of read bytes"),
7503
7504     Option("rbd_qos_write_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7505     .set_default(0)
7506     .set_description("the desired burst limit of write bytes"),
7507
7508     Option("rbd_qos_schedule_tick_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7509     .set_default(50)
7510     .set_min(1)
7511     .set_description("minimum schedule tick (in milliseconds) for QoS"),
7512
7513     Option("rbd_discard_on_zeroed_write_same", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7514     .set_default(true)
7515     .set_description("discard data on zeroed write same instead of writing zero"),
7516
7517     Option("rbd_mtime_update_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7518     .set_default(60)
7519     .set_min(0)
7520     .set_description("RBD Image modify timestamp refresh interval. Set to 0 to disable modify timestamp update."),
7521
7522     Option("rbd_atime_update_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7523     .set_default(60)
7524     .set_min(0)
7525     .set_description("RBD Image access timestamp refresh interval. Set to 0 to disable access timestamp update."),
7526   });
7527 }
7528
7529 static std::vector<Option> get_rbd_mirror_options() {
7530   return std::vector<Option>({
7531     Option("rbd_mirror_journal_commit_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7532     .set_default(5)
7533     .set_description("commit time interval, seconds"),
7534
7535     Option("rbd_mirror_journal_poll_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7536     .set_default(5)
7537     .set_description("maximum age (in seconds) between successive journal polls"),
7538
7539     Option("rbd_mirror_journal_max_fetch_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7540     .set_default(32768)
7541     .set_description("maximum bytes to read from each journal data object per fetch"),
7542
7543     Option("rbd_mirror_sync_point_update_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7544     .set_default(30)
7545     .set_description("number of seconds between each update of the image sync point object number"),
7546
7547     Option("rbd_mirror_concurrent_image_syncs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7548     .set_default(5)
7549     .set_description("maximum number of image syncs in parallel"),
7550
7551     Option("rbd_mirror_pool_replayers_refresh_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7552     .set_default(30)
7553     .set_description("interval to refresh peers in rbd-mirror daemon"),
7554
7555     Option("rbd_mirror_concurrent_image_deletions", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7556     .set_default(1)
7557     .set_min(1)
7558     .set_description("maximum number of image deletions in parallel"),
7559
7560     Option("rbd_mirror_delete_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7561     .set_default(30)
7562     .set_description("interval to check and retry the failed deletion requests"),
7563
7564     Option("rbd_mirror_image_state_check_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7565     .set_default(30)
7566     .set_min(1)
7567     .set_description("interval to get images from pool watcher and set sources in replayer"),
7568
7569     Option("rbd_mirror_leader_heartbeat_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7570     .set_default(5)
7571     .set_min(1)
7572     .set_description("interval (in seconds) between mirror leader heartbeats"),
7573
7574     Option("rbd_mirror_leader_max_missed_heartbeats", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7575     .set_default(2)
7576     .set_description("number of missed heartbeats for non-lock owner to attempt to acquire lock"),
7577
7578     Option("rbd_mirror_leader_max_acquire_attempts_before_break", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7579     .set_default(3)
7580     .set_description("number of failed attempts to acquire lock after missing heartbeats before breaking lock"),
7581
7582     Option("rbd_mirror_image_policy_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7583     .set_default("simple")
7584     .set_enum_allowed({"none", "simple"})
7585     .set_description("active/active policy type for mapping images to instances"),
7586
7587     Option("rbd_mirror_image_policy_migration_throttle", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7588     .set_default(300)
7589     .set_description("number of seconds after which an image can be reshuffled (migrated) again"),
7590
7591     Option("rbd_mirror_image_policy_update_throttle_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7592     .set_default(1)
7593     .set_min(1)
7594     .set_description("interval (in seconds) to throttle images for mirror daemon peer updates"),
7595
7596     Option("rbd_mirror_image_policy_rebalance_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7597     .set_default(0)
7598     .set_description("number of seconds policy should be idle before trigerring reshuffle (rebalance) of images"),
7599
7600     Option("rbd_mirror_perf_stats_prio", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7601     .set_default((int64_t)PerfCountersBuilder::PRIO_USEFUL)
7602     .set_description("Priority level for mirror daemon replication perf counters")
7603     .set_long_description("The daemon will send perf counter data to the "
7604                           "manager daemon if the priority is not lower than "
7605                           "mgr_stats_threshold.")
7606     .set_min_max((int64_t)PerfCountersBuilder::PRIO_DEBUGONLY,
7607                  (int64_t)PerfCountersBuilder::PRIO_CRITICAL + 1),
7608   });
7609 }
7610
7611 std::vector<Option> get_mds_options() {
7612   return std::vector<Option>({
7613     Option("mds_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7614     .set_default("/var/lib/ceph/mds/$cluster-$id")
7615     .set_flag(Option::FLAG_NO_MON_UPDATE)
7616     .set_description("path to MDS data and keyring"),
7617
7618     Option("mds_max_xattr_pairs_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7619     .set_default(64_K)
7620     .set_description("maximum aggregate size of extended attributes on a file"),
7621
7622     Option("mds_cache_trim_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
7623     .set_default(1)
7624     .set_description("interval in seconds between cache trimming"),
7625
7626     Option("mds_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7627     .set_default(0)
7628     .set_description("maximum number of inodes in MDS cache (<=0 is unlimited)")
7629     .set_long_description("This tunable is no longer recommended. Use mds_cache_memory_limit."),
7630
7631     Option("mds_cache_memory_limit", Option::TYPE_SIZE, Option::LEVEL_BASIC)
7632     .set_default(1*(1LL<<30))
7633     .set_description("target maximum memory usage of MDS cache")
7634     .set_long_description("This sets a target maximum memory usage of the MDS cache and is the primary tunable to limit the MDS memory usage. The MDS will try to stay under a reservation of this limit (by default 95%; 1 - mds_cache_reservation) by trimming unused metadata in its cache and recalling cached items in the client caches. It is possible for the MDS to exceed this limit due to slow recall from clients. The mds_health_cache_threshold (150%) sets a cache full threshold for when the MDS signals a cluster health warning."),
7635
7636     Option("mds_cache_reservation", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7637     .set_default(.05)
7638     .set_description("amount of memory to reserve for future cached objects"),
7639
7640     Option("mds_health_cache_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7641     .set_default(1.5)
7642     .set_description("threshold for cache size to generate health warning"),
7643
7644     Option("mds_cache_mid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7645     .set_default(.7)
7646     .set_description("midpoint for MDS cache LRU"),
7647
7648     Option("mds_cache_trim_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7649     .set_default(1)
7650     .set_description("decay rate for trimming MDS cache throttle"),
7651
7652     Option("mds_cache_trim_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7653     .set_default(64_K)
7654     .set_description("threshold for number of dentries that can be trimmed"),
7655
7656     Option("mds_max_file_recover", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7657     .set_default(32)
7658     .set_description("maximum number of files to recover file sizes in parallel"),
7659
7660     Option("mds_dir_max_commit_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7661     .set_default(10)
7662     .set_description("maximum size in megabytes for a RADOS write to a directory"),
7663
7664     Option("mds_dir_keys_per_op", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7665     .set_default(16384)
7666     .set_description("number of directory entries to read in one RADOS operation"),
7667
7668     Option("mds_decay_halflife", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7669     .set_default(5)
7670     .set_description("rate of decay for temperature counters on each directory for balancing"),
7671
7672     Option("mds_beacon_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7673     .set_default(4)
7674     .set_description("interval in seconds between MDS beacons to monitors"),
7675
7676     Option("mds_beacon_grace", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7677     .set_default(15)
7678     .set_description("tolerance in seconds for missed MDS beacons to monitors"),
7679
7680     Option("mds_heartbeat_grace", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7681     .set_default(15)
7682     .set_description("tolerance in seconds for MDS internal heartbeat"),
7683
7684     Option("mds_enforce_unique_name", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7685     .set_default(true)
7686     .set_description("require MDS name is unique in the cluster"),
7687
7688     Option("mds_session_blacklist_on_timeout", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7689     .set_default(true)
7690     .set_description("blacklist clients whose sessions have become stale"),
7691
7692     Option("mds_session_blacklist_on_evict", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7693     .set_default(true)
7694     .set_description("blacklist clients that have been evicted"),
7695
7696     Option("mds_sessionmap_keys_per_op", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7697     .set_default(1024)
7698     .set_description("number of omap keys to read from the SessionMap in one operation"),
7699
7700     Option("mds_recall_max_caps", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7701     .set_default(5000)
7702     .set_description("maximum number of caps to recall from client session in single recall"),
7703
7704     Option("mds_recall_max_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7705     .set_default(2.5)
7706     .set_description("decay rate for throttle on recalled caps on a session"),
7707
7708     Option("mds_recall_max_decay_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7709     .set_default(16_K)
7710     .set_description("decay threshold for throttle on recalled caps on a session"),
7711
7712     Option("mds_recall_global_max_decay_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7713     .set_default(64_K)
7714     .set_description("decay threshold for throttle on recalled caps globally"),
7715
7716     Option("mds_recall_warning_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7717     .set_default(32_K)
7718     .set_description("decay threshold for warning on slow session cap recall"),
7719
7720     Option("mds_recall_warning_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7721     .set_default(60.0)
7722     .set_description("decay rate for warning on slow session cap recall"),
7723
7724     Option("mds_freeze_tree_timeout", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7725     .set_default(30)
7726     .set_description(""),
7727
7728     Option("mds_health_summarize_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7729     .set_default(10)
7730     .set_description("threshold of number of clients to summarize late client recall"),
7731
7732     Option("mds_reconnect_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7733     .set_default(45)
7734     .set_description("timeout in seconds to wait for clients to reconnect during MDS reconnect recovery state"),
7735
7736     Option("mds_tick_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7737     .set_default(5)
7738     .set_description("time in seconds between upkeep tasks"),
7739
7740     Option("mds_dirstat_min_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7741     .set_default(1)
7742     .set_description(""),
7743
7744     Option("mds_scatter_nudge_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7745     .set_default(5)
7746     .set_description("minimum interval between scatter lock updates"),
7747
7748     Option("mds_client_prealloc_inos", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7749     .set_default(1000)
7750     .set_description("number of unused inodes to pre-allocate to clients for file creation"),
7751
7752     Option("mds_early_reply", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7753     .set_default(true)
7754     .set_description("additional reply to clients that metadata requests are complete but not yet durable"),
7755
7756     Option("mds_default_dir_hash", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7757     .set_default(CEPH_STR_HASH_RJENKINS)
7758     .set_description("hash function to select directory fragment for dentry name"),
7759
7760     Option("mds_log_pause", Option::TYPE_BOOL, Option::LEVEL_DEV)
7761     .set_default(false)
7762     .set_description(""),
7763
7764     Option("mds_log_skip_corrupt_events", Option::TYPE_BOOL, Option::LEVEL_DEV)
7765     .set_default(false)
7766     .set_description(""),
7767
7768     Option("mds_log_max_events", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7769     .set_default(-1)
7770     .set_description("maximum number of events in the MDS journal (-1 is unlimited)"),
7771
7772     Option("mds_log_events_per_segment", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7773     .set_default(1024)
7774     .set_description("maximum number of events in an MDS journal segment"),
7775
7776     Option("mds_log_segment_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7777     .set_default(0)
7778     .set_description("size in bytes of each MDS log segment"),
7779
7780     Option("mds_log_max_segments", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7781     .set_default(128)
7782     .set_description("maximum number of segments which may be untrimmed"),
7783
7784     Option("mds_bal_export_pin", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7785     .set_default(true)
7786     .set_description("allow setting directory export pins to particular ranks"),
7787
7788     Option("mds_bal_sample_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7789     .set_default(3.0)
7790     .set_description("interval in seconds between balancer ticks"),
7791
7792     Option("mds_bal_replicate_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7793     .set_default(8000)
7794     .set_description("hot popularity threshold to replicate a subtree"),
7795
7796     Option("mds_bal_unreplicate_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7797     .set_default(0)
7798     .set_description("cold popularity threshold to merge subtrees"),
7799
7800     Option("mds_bal_split_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7801     .set_default(10000)
7802     .set_description("minimum size of directory fragment before splitting"),
7803
7804     Option("mds_bal_split_rd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7805     .set_default(25000)
7806     .set_description("hot read popularity threshold for splitting a directory fragment"),
7807
7808     Option("mds_bal_split_wr", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7809     .set_default(10000)
7810     .set_description("hot write popularity threshold for splitting a directory fragment"),
7811
7812     Option("mds_bal_split_bits", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7813     .set_default(3)
7814     .set_min_max(1, 24)
7815     .set_description("power of two child fragments for a fragment on split"),
7816
7817     Option("mds_bal_merge_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7818     .set_default(50)
7819     .set_description("size of fragments where merging should occur"),
7820
7821     Option("mds_bal_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7822     .set_default(10)
7823     .set_description("interval between MDS balancer cycles"),
7824
7825     Option("mds_bal_fragment_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7826     .set_default(5)
7827     .set_description("delay in seconds before interrupting client IO to perform splits"),
7828
7829     Option("mds_bal_fragment_size_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7830     .set_default(10000*10)
7831     .set_description("maximum size of a directory fragment before new creat/links fail"),
7832
7833     Option("mds_bal_fragment_fast_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7834     .set_default(1.5)
7835     .set_description("ratio of mds_bal_split_size at which fast fragment splitting occurs"),
7836
7837     Option("mds_bal_fragment_dirs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7838     .set_default(true)
7839     .set_description("enable directory fragmentation")
7840     .set_long_description("Directory fragmentation is a standard feature of CephFS that allows sharding directories across multiple objects for performance and stability. Additionally, this allows fragments to be distributed across multiple active MDSs to increase throughput. Disabling (new) fragmentation should only be done in exceptional circumstances and may lead to performance issues."),
7841
7842     Option("mds_bal_idle_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7843     .set_default(0)
7844     .set_description("idle metadata popularity threshold before rebalancing"),
7845
7846     Option("mds_bal_max", Option::TYPE_INT, Option::LEVEL_DEV)
7847     .set_default(-1)
7848     .set_description(""),
7849
7850     Option("mds_bal_max_until", Option::TYPE_INT, Option::LEVEL_DEV)
7851     .set_default(-1)
7852     .set_description(""),
7853
7854     Option("mds_bal_mode", Option::TYPE_INT, Option::LEVEL_DEV)
7855     .set_default(0)
7856     .set_description(""),
7857
7858     Option("mds_bal_min_rebalance", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7859     .set_default(.1)
7860     .set_description("amount overloaded over internal target before balancer begins offloading"),
7861
7862     Option("mds_bal_min_start", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7863     .set_default(.2)
7864     .set_description(""),
7865
7866     Option("mds_bal_need_min", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7867     .set_default(.8)
7868     .set_description(""),
7869
7870     Option("mds_bal_need_max", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7871     .set_default(1.2)
7872     .set_description(""),
7873
7874     Option("mds_bal_midchunk", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7875     .set_default(.3)
7876     .set_description(""),
7877
7878     Option("mds_bal_minchunk", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7879     .set_default(.001)
7880     .set_description(""),
7881
7882     Option("mds_bal_target_decay", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7883     .set_default(10.0)
7884     .set_description("rate of decay for export targets communicated to clients"),
7885
7886     Option("mds_replay_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7887     .set_default(1.0)
7888     .set_description("time in seconds between replay of updates to journal by standby replay MDS"),
7889
7890     Option("mds_shutdown_check", Option::TYPE_INT, Option::LEVEL_DEV)
7891     .set_default(0)
7892     .set_description(""),
7893
7894     Option("mds_thrash_exports", Option::TYPE_INT, Option::LEVEL_DEV)
7895     .set_default(0)
7896     .set_description(""),
7897
7898     Option("mds_thrash_fragments", Option::TYPE_INT, Option::LEVEL_DEV)
7899     .set_default(0)
7900     .set_description(""),
7901
7902     Option("mds_dump_cache_on_map", Option::TYPE_BOOL, Option::LEVEL_DEV)
7903     .set_default(false)
7904     .set_description(""),
7905
7906     Option("mds_dump_cache_after_rejoin", Option::TYPE_BOOL, Option::LEVEL_DEV)
7907     .set_default(false)
7908     .set_description(""),
7909
7910     Option("mds_verify_scatter", Option::TYPE_BOOL, Option::LEVEL_DEV)
7911     .set_default(false)
7912     .set_description(""),
7913
7914     Option("mds_debug_scatterstat", Option::TYPE_BOOL, Option::LEVEL_DEV)
7915     .set_default(false)
7916     .set_description(""),
7917
7918     Option("mds_debug_frag", Option::TYPE_BOOL, Option::LEVEL_DEV)
7919     .set_default(false)
7920     .set_description(""),
7921
7922     Option("mds_debug_auth_pins", Option::TYPE_BOOL, Option::LEVEL_DEV)
7923     .set_default(false)
7924     .set_description(""),
7925
7926     Option("mds_debug_subtrees", Option::TYPE_BOOL, Option::LEVEL_DEV)
7927     .set_default(false)
7928     .set_description(""),
7929
7930     Option("mds_kill_mdstable_at", Option::TYPE_INT, Option::LEVEL_DEV)
7931     .set_default(0)
7932     .set_description(""),
7933
7934     Option("mds_max_export_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
7935     .set_default(20_M)
7936     .set_description(""),
7937
7938     Option("mds_kill_export_at", Option::TYPE_INT, Option::LEVEL_DEV)
7939     .set_default(0)
7940     .set_description(""),
7941
7942     Option("mds_kill_import_at", Option::TYPE_INT, Option::LEVEL_DEV)
7943     .set_default(0)
7944     .set_description(""),
7945
7946     Option("mds_kill_link_at", Option::TYPE_INT, Option::LEVEL_DEV)
7947     .set_default(0)
7948     .set_description(""),
7949
7950     Option("mds_kill_rename_at", Option::TYPE_INT, Option::LEVEL_DEV)
7951     .set_default(0)
7952     .set_description(""),
7953
7954     Option("mds_kill_openc_at", Option::TYPE_INT, Option::LEVEL_DEV)
7955     .set_default(0)
7956     .set_description(""),
7957
7958     Option("mds_kill_journal_at", Option::TYPE_INT, Option::LEVEL_DEV)
7959     .set_default(0)
7960     .set_description(""),
7961
7962     Option("mds_kill_journal_expire_at", Option::TYPE_INT, Option::LEVEL_DEV)
7963     .set_default(0)
7964     .set_description(""),
7965
7966     Option("mds_kill_journal_replay_at", Option::TYPE_INT, Option::LEVEL_DEV)
7967     .set_default(0)
7968     .set_description(""),
7969
7970     Option("mds_journal_format", Option::TYPE_UINT, Option::LEVEL_DEV)
7971     .set_default(1)
7972     .set_description(""),
7973
7974     Option("mds_kill_create_at", Option::TYPE_INT, Option::LEVEL_DEV)
7975     .set_default(0)
7976     .set_description(""),
7977
7978     Option("mds_inject_traceless_reply_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7979     .set_default(0)
7980     .set_description(""),
7981
7982     Option("mds_wipe_sessions", Option::TYPE_BOOL, Option::LEVEL_DEV)
7983     .set_default(0)
7984     .set_description(""),
7985
7986     Option("mds_wipe_ino_prealloc", Option::TYPE_BOOL, Option::LEVEL_DEV)
7987     .set_default(0)
7988     .set_description(""),
7989
7990     Option("mds_skip_ino", Option::TYPE_INT, Option::LEVEL_DEV)
7991     .set_default(0)
7992     .set_description(""),
7993
7994     Option("mds_enable_op_tracker", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7995     .set_default(true)
7996     .set_description("track remote operation progression and statistics"),
7997
7998     Option("mds_op_history_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7999     .set_default(20)
8000     .set_description("maximum size for list of historical operations"),
8001
8002     Option("mds_op_history_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8003     .set_default(600)
8004     .set_description("expiration time in seconds of historical operations"),
8005
8006     Option("mds_op_complaint_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8007     .set_default(30)
8008     .set_description("time in seconds to consider an operation blocked after no updates"),
8009
8010     Option("mds_op_log_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
8011     .set_default(5)
8012     .set_description(""),
8013
8014     Option("mds_snap_min_uid", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8015     .set_default(0)
8016     .set_description("minimum uid of client to perform snapshots"),
8017
8018     Option("mds_snap_max_uid", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8019     .set_default(4294967294)
8020     .set_description("maximum uid of client to perform snapshots"),
8021
8022     Option("mds_snap_rstat", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8023     .set_default(false)
8024     .set_description("enabled nested rstat for snapshots"),
8025
8026     Option("mds_verify_backtrace", Option::TYPE_UINT, Option::LEVEL_DEV)
8027     .set_default(1)
8028     .set_description(""),
8029
8030     Option("mds_max_completed_flushes", Option::TYPE_UINT, Option::LEVEL_DEV)
8031     .set_default(100000)
8032     .set_description(""),
8033
8034     Option("mds_max_completed_requests", Option::TYPE_UINT, Option::LEVEL_DEV)
8035     .set_default(100000)
8036     .set_description(""),
8037
8038     Option("mds_action_on_write_error", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8039     .set_default(1)
8040     .set_description("action to take when MDS cannot write to RADOS (0:ignore, 1:read-only, 2:suicide)"),
8041
8042     Option("mds_mon_shutdown_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8043     .set_default(5)
8044     .set_description("time to wait for mon to receive damaged MDS rank notification"),
8045
8046     Option("mds_max_purge_files", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8047     .set_default(64)
8048     .set_description("maximum number of deleted files to purge in parallel"),
8049
8050     Option("mds_max_purge_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8051     .set_default(8192)
8052     .set_description("maximum number of purge operations performed in parallel"),
8053
8054     Option("mds_max_purge_ops_per_pg", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8055     .set_default(0.5)
8056     .set_description("number of parallel purge operations performed per PG"),
8057
8058     Option("mds_purge_queue_busy_flush_period", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8059     .set_default(1.0)
8060     .set_description(""),
8061
8062     Option("mds_root_ino_uid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8063     .set_default(0)
8064     .set_description("default uid for new root directory"),
8065
8066     Option("mds_root_ino_gid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8067     .set_default(0)
8068     .set_description("default gid for new root directory"),
8069
8070     Option("mds_max_scrub_ops_in_progress", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8071     .set_default(5)
8072     .set_description("maximum number of scrub operations performed in parallel"),
8073
8074     Option("mds_damage_table_max_entries", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8075     .set_default(10000)
8076     .set_description("maximum number of damage table entries"),
8077
8078     Option("mds_client_writeable_range_max_inc_objs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8079     .set_default(1024)
8080     .set_description("maximum number of objects in writeable range of a file for a client"),
8081
8082     Option("mds_min_caps_per_client", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8083     .set_default(100)
8084     .set_description("minimum number of capabilities a client may hold"),
8085
8086     Option("mds_max_caps_per_client", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8087     .set_default(1_M)
8088     .set_description("maximum number of capabilities a client may hold"),
8089
8090     Option("mds_hack_allow_loading_invalid_metadata", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8091      .set_default(0)
8092      .set_description("INTENTIONALLY CAUSE DATA LOSS by bypasing checks for invalid metadata on disk. Allows testing repair tools."),
8093
8094     Option("mds_defer_session_stale", Option::TYPE_BOOL, Option::LEVEL_DEV)
8095      .set_default(true),
8096
8097     Option("mds_inject_migrator_session_race", Option::TYPE_BOOL, Option::LEVEL_DEV)
8098      .set_default(false),
8099
8100     Option("mds_request_load_average_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8101     .set_default(60)
8102     .set_description("rate of decay in seconds for calculating request load average"),
8103
8104     Option("mds_cap_revoke_eviction_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8105      .set_default(0)
8106      .set_description("number of seconds after which clients which have not responded to cap revoke messages by the MDS are evicted."),
8107
8108     Option("mds_max_retries_on_remount_failure", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8109      .set_default(5)
8110      .set_description("number of consecutive failed remount attempts for invalidating kernel dcache after which client would abort."),
8111
8112     Option("mds_dump_cache_threshold_formatter", Option::TYPE_SIZE, Option::LEVEL_DEV)
8113      .set_default(1_G)
8114      .set_description("threshold for cache usage to disallow \"dump cache\" operation to formatter")
8115      .set_long_description("Disallow MDS from dumping caches to formatter via \"dump cache\" command if cache usage exceeds this threshold."),
8116
8117     Option("mds_dump_cache_threshold_file", Option::TYPE_SIZE, Option::LEVEL_DEV)
8118      .set_default(0)
8119      .set_description("threshold for cache usage to disallow \"dump cache\" operation to file")
8120      .set_long_description("Disallow MDS from dumping caches to file via \"dump cache\" command if cache usage exceeds this threshold."),
8121   });
8122 }
8123
8124 std::vector<Option> get_mds_client_options() {
8125   return std::vector<Option>({
8126     Option("client_cache_size", Option::TYPE_SIZE, Option::LEVEL_BASIC)
8127     .set_default(16384)
8128     .set_description("soft maximum number of directory entries in client cache"),
8129
8130     Option("client_cache_mid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8131     .set_default(.75)
8132     .set_description("mid-point of client cache LRU"),
8133
8134     Option("client_use_random_mds", Option::TYPE_BOOL, Option::LEVEL_DEV)
8135     .set_default(false)
8136     .set_description("issue new requests to a random active MDS"),
8137
8138     Option("client_mount_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8139     .set_default(300.0)
8140     .set_description("timeout for mounting CephFS (seconds)"),
8141
8142     Option("client_tick_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8143     .set_default(1.0)
8144     .set_description("seconds between client upkeep ticks"),
8145
8146     Option("client_trace", Option::TYPE_STR, Option::LEVEL_DEV)
8147     .set_default("")
8148     .set_description("file containing trace of client operations"),
8149
8150     Option("client_readahead_min", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8151     .set_default(128*1024)
8152     .set_description("minimum bytes to readahead in a file"),
8153
8154     Option("client_readahead_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8155     .set_default(0)
8156     .set_description("maximum bytes to readahead in a file (zero is unlimited)"),
8157
8158     Option("client_readahead_max_periods", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8159     .set_default(4)
8160     .set_description("maximum stripe periods to readahead in a file"),
8161
8162     Option("client_reconnect_stale", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8163     .set_default(false)
8164     .set_description("reconnect when the session becomes stale"),
8165
8166     Option("client_snapdir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8167     .set_default(".snap")
8168     .set_description("pseudo directory for snapshot access to a directory"),
8169
8170     Option("client_mountpoint", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8171     .set_default("/")
8172     .set_description("default mount-point"),
8173
8174     Option("client_mount_uid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8175     .set_default(-1)
8176     .set_description("uid to mount as"),
8177
8178     Option("client_mount_gid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8179     .set_default(-1)
8180     .set_description("gid to mount as"),
8181
8182     /* RADOS client option */
8183     Option("client_notify_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
8184     .set_default(10)
8185     .set_description(""),
8186
8187     /* RADOS client option */
8188     Option("osd_client_watch_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
8189     .set_default(30)
8190     .set_description(""),
8191
8192     Option("client_caps_release_delay", Option::TYPE_INT, Option::LEVEL_DEV)
8193     .set_default(5)
8194     .set_description(""),
8195
8196     Option("client_quota_df", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8197     .set_default(true)
8198     .set_description("show quota usage for statfs (df)"),
8199
8200     Option("client_oc", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8201     .set_default(true)
8202     .set_description("enable object caching"),
8203
8204     Option("client_oc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8205     .set_default(200_M)
8206     .set_description("maximum size of object cache"),
8207
8208     Option("client_oc_max_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8209     .set_default(100_M)
8210     .set_description("maximum size of dirty pages in object cache"),
8211
8212     Option("client_oc_target_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8213     .set_default(8_M)
8214     .set_description("target size of dirty pages object cache"),
8215
8216     Option("client_oc_max_dirty_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8217     .set_default(5.0)
8218     .set_description("maximum age of dirty pages in object cache (seconds)"),
8219
8220     Option("client_oc_max_objects", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8221     .set_default(1000)
8222     .set_description("maximum number of objects in cache"),
8223
8224     Option("client_debug_getattr_caps", Option::TYPE_BOOL, Option::LEVEL_DEV)
8225     .set_default(false)
8226     .set_description(""),
8227
8228     Option("client_debug_force_sync_read", Option::TYPE_BOOL, Option::LEVEL_DEV)
8229     .set_default(false)
8230     .set_description(""),
8231
8232     Option("client_debug_inject_tick_delay", Option::TYPE_INT, Option::LEVEL_DEV)
8233     .set_default(0)
8234     .set_description(""),
8235
8236     Option("client_max_inline_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
8237     .set_default(4_K)
8238     .set_description(""),
8239
8240     Option("client_inject_release_failure", Option::TYPE_BOOL, Option::LEVEL_DEV)
8241     .set_default(false)
8242     .set_description(""),
8243
8244     Option("client_inject_fixed_oldest_tid", Option::TYPE_BOOL, Option::LEVEL_DEV)
8245     .set_default(false)
8246     .set_description(""),
8247
8248     Option("client_metadata", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8249     .set_default("")
8250     .set_description("metadata key=value comma-delimited pairs appended to session metadata"),
8251
8252     Option("client_acl_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8253     .set_default("")
8254     .set_description("ACL type to enforce (none or \"posix_acl\")"),
8255
8256     Option("client_permissions", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8257     .set_default(true)
8258     .set_description("client-enforced permission checking"),
8259
8260     Option("client_dirsize_rbytes", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8261     .set_default(true)
8262     .set_description("set the directory size as the number of file bytes recursively used")
8263     .set_long_description("This option enables a CephFS feature that stores the recursive directory size (the bytes used by files in the directory and its descendents) in the st_size field of the stat structure."),
8264
8265     Option("client_force_lazyio", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8266     .set_default(false)
8267     .set_description(""),
8268
8269     // note: the max amount of "in flight" dirty data is roughly (max - target)
8270     Option("fuse_use_invalidate_cb", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8271     .set_default(true)
8272     .set_description("use fuse 2.8+ invalidate callback to keep page cache consistent"),
8273
8274     Option("fuse_disable_pagecache", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8275     .set_default(false)
8276     .set_description("disable page caching in the kernel for this FUSE mount"),
8277
8278     Option("fuse_allow_other", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8279     .set_default(true)
8280     .set_description("pass allow_other to FUSE on mount"),
8281
8282     Option("fuse_default_permissions", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8283     .set_default(false)
8284     .set_description("pass default_permisions to FUSE on mount"),
8285
8286     Option("fuse_big_writes", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8287     .set_default(false)
8288     .set_description("big_writes is deprecated in libfuse 3.0.0"),
8289
8290     Option("fuse_max_write", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8291     .set_default(0)
8292     .set_description("set the maximum number of bytes in a single write operation")
8293     .set_long_description("Set the maximum number of bytes in a single write operation that may pass atomically through FUSE. The FUSE default is 128kB and may be indicated by setting this option to 0."),
8294
8295     Option("fuse_atomic_o_trunc", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8296     .set_default(true)
8297     .set_description("pass atomic_o_trunc flag to FUSE on mount"),
8298
8299     Option("fuse_debug", Option::TYPE_BOOL, Option::LEVEL_DEV)
8300     .set_default(false)
8301     .set_description(""),
8302
8303     Option("fuse_multithreaded", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8304     .set_default(true)
8305     .set_description("allow parallel processing through FUSE library"),
8306
8307     Option("fuse_require_active_mds", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8308     .set_default(true)
8309     .set_description("require active MDSs in the file system when mounting"),
8310
8311     Option("fuse_syncfs_on_mksnap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8312     .set_default(true)
8313     .set_description("synchronize all local metadata/file changes after snapshot"),
8314
8315     Option("fuse_set_user_groups", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8316     .set_default(true)
8317     .set_description("check for ceph-fuse to consider supplementary groups for permissions"),
8318
8319     Option("client_try_dentry_invalidate", Option::TYPE_BOOL, Option::LEVEL_DEV)
8320     .set_default(false)
8321     .set_description(""),
8322
8323     Option("client_die_on_failed_remount", Option::TYPE_BOOL, Option::LEVEL_DEV)
8324     .set_default(false)
8325     .set_description(""),
8326
8327     Option("client_die_on_failed_dentry_invalidate", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8328     .set_default(true)
8329     .set_description("kill the client when no dentry invalidation options are available")
8330     .set_long_description("The CephFS client requires a mechanism to invalidate dentries in the caller (e.g. the kernel for ceph-fuse) when capabilities must be recalled. If the client cannot do this then the MDS cache cannot shrink which can cause the MDS to fail."),
8331
8332     Option("client_check_pool_perm", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8333     .set_default(true)
8334     .set_description("confirm access to inode's data pool/namespace described in file layout"),
8335
8336     Option("client_use_faked_inos", Option::TYPE_BOOL, Option::LEVEL_DEV)
8337     .set_default(false)
8338     .set_description(""),
8339
8340     Option("client_mds_namespace", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8341     .set_default("")
8342
8343     .set_description("CephFS file system name to mount")
8344     .set_long_description("Use this with ceph-fuse, or with any process "
8345         "that uses libcephfs.  Programs using libcephfs may also pass "
8346         "the filesystem name into mount(), which will override this setting. "
8347         "If no filesystem name is given in mount() or this setting, the default "
8348         "filesystem will be mounted (usually the first created)."),
8349
8350     Option("fake_statfs_for_testing", Option::TYPE_INT, Option::LEVEL_DEV)
8351     .set_default(0)
8352     .set_description("Set a value for kb and compute kb_used from total of num_bytes"),
8353
8354     Option("debug_allow_any_pool_priority", Option::TYPE_BOOL, Option::LEVEL_DEV)
8355     .set_default(false)
8356     .set_description("Allow any pool priority to be set to test conversion to new range"),
8357   });
8358 }
8359
8360
8361 static std::vector<Option> build_options()
8362 {
8363   std::vector<Option> result = get_global_options();
8364
8365   auto ingest = [&result](std::vector<Option>&& options, const char* svc) {
8366     for (auto &o : options) {
8367       o.add_service(svc);
8368       result.push_back(std::move(o));
8369     }
8370   };
8371
8372   ingest(get_rgw_options(), "rgw");
8373   ingest(get_rbd_options(), "rbd");
8374   ingest(get_rbd_mirror_options(), "rbd-mirror");
8375   ingest(get_mds_options(), "mds");
8376   ingest(get_mds_client_options(), "mds_client");
8377
8378   return result;
8379 }
8380
8381 const std::vector<Option> ceph_options = build_options();