ceph/src/common/options.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "acconfig.h"
   5 #include "options.h"
   6 #include "common/Formatter.h"
   7
   8 // Helpers for validators
   9 #include "include/stringify.h"
  10 #include "include/common_fwd.h"
  11 #include <boost/algorithm/string.hpp>
  12 #include <boost/lexical_cast.hpp>
  13 #include <regex>
  14
  15 // Definitions for enums
  16 #include "common/perf_counters.h"
  17
  18 // rbd feature validation
  19 #include "librbd/Features.h"
  20
  21 namespace {
  22 class printer : public boost::static_visitor<> {
  23   ostream& out;
  24 public:
  25   explicit printer(ostream& os)
  26     : out(os) {}
  27   template<typename T>
  28   void operator()(const T& v) const {
  29     out << v;
  30   }
  31   void operator()(boost::blank blank) const {
  32     return;
  33   }
  34   void operator()(bool v) const {
  35     out << (v ? "true" : "false");
  36   }
  37   void operator()(double v) const {
  38     out << std::fixed << v << std::defaultfloat;
  39   }
  40   void operator()(const Option::size_t& v) const {
  41     out << v.value;
  42   }
  43   void operator()(const std::chrono::seconds v) const {
  44     out << v.count();
  45   }
  46 };
  47 }
  48
  49 ostream& operator<<(ostream& os, const Option::value_t& v) {
  50   printer p{os};
  51   v.apply_visitor(p);
  52   return os;
  53 }
  54
  55 void Option::dump_value(const char *field_name,
  56     const Option::value_t &v, Formatter *f) const
  57 {
  58   if (boost::get<boost::blank>(&v)) {
  59     // This should be nil but Formatter doesn't allow it.
  60     f->dump_string(field_name, "");
  61     return;
  62   }
  63   switch (type) {
  64   case TYPE_INT:
  65     f->dump_int(field_name, boost::get<int64_t>(v)); break;
  66   case TYPE_UINT:
  67     f->dump_unsigned(field_name, boost::get<uint64_t>(v)); break;
  68   case TYPE_STR:
  69     f->dump_string(field_name, boost::get<std::string>(v)); break;
  70   case TYPE_FLOAT:
  71     f->dump_float(field_name, boost::get<double>(v)); break;
  72   case TYPE_BOOL:
  73     f->dump_bool(field_name, boost::get<bool>(v)); break;
  74   default:
  75     f->dump_stream(field_name) << v; break;
  76   }
  77 }
  78
  79 int Option::pre_validate(std::string *new_value, std::string *err) const
  80 {
  81   if (validator) {
  82     return validator(new_value, err);
  83   } else {
  84     return 0;
  85   }
  86 }
  87
  88 int Option::validate(const Option::value_t &new_value, std::string *err) const
  89 {
  90   // Generic validation: min
  91   if (!boost::get<boost::blank>(&(min))) {
  92     if (new_value < min) {
  93       std::ostringstream oss;
  94       oss << "Value '" << new_value << "' is below minimum " << min;
  95       *err = oss.str();
  96       return -EINVAL;
  97     }
  98   }
  99
 100   // Generic validation: max
 101   if (!boost::get<boost::blank>(&(max))) {
 102     if (new_value > max) {
 103       std::ostringstream oss;
 104       oss << "Value '" << new_value << "' exceeds maximum " << max;
 105       *err = oss.str();
 106       return -EINVAL;
 107     }
 108   }
 109
 110   // Generic validation: enum
 111   if (!enum_allowed.empty() && type == Option::TYPE_STR) {
 112     auto found = std::find(enum_allowed.begin(), enum_allowed.end(),
 113                            boost::get<std::string>(new_value));
 114     if (found == enum_allowed.end()) {
 115       std::ostringstream oss;
 116       oss << "'" << new_value << "' is not one of the permitted "
 117                  "values: " << joinify(enum_allowed.begin(),
 118                                        enum_allowed.end(),
 119                                        std::string(", "));
 120       *err = oss.str();
 121       return -EINVAL;
 122     }
 123   }
 124
 125   return 0;
 126 }
 127
 128 int Option::parse_value(
 129   const std::string& raw_val,
 130   value_t *out,
 131   std::string *error_message,
 132   std::string *normalized_value) const
 133 {
 134   std::string val = raw_val;
 135
 136   int r = pre_validate(&val, error_message);
 137   if (r != 0) {
 138     return r;
 139   }
 140
 141   if (type == Option::TYPE_INT) {
 142     int64_t f = strict_si_cast<int64_t>(val.c_str(), error_message);
 143     if (!error_message->empty()) {
 144       return -EINVAL;
 145     }
 146     *out = f;
 147   } else if (type == Option::TYPE_UINT) {
 148     uint64_t f = strict_si_cast<uint64_t>(val.c_str(), error_message);
 149     if (!error_message->empty()) {
 150       return -EINVAL;
 151     }
 152     *out = f;
 153   } else if (type == Option::TYPE_STR) {
 154     *out = val;
 155   } else if (type == Option::TYPE_FLOAT) {
 156     double f = strict_strtod(val.c_str(), error_message);
 157     if (!error_message->empty()) {
 158       return -EINVAL;
 159     } else {
 160       *out = f;
 161     }
 162   } else if (type == Option::TYPE_BOOL) {
 163     if (strcasecmp(val.c_str(), "false") == 0) {
 164       *out = false;
 165     } else if (strcasecmp(val.c_str(), "true") == 0) {
 166       *out = true;
 167     } else {
 168       int b = strict_strtol(val.c_str(), 10, error_message);
 169       if (!error_message->empty()) {
 170         return -EINVAL;
 171       }
 172       *out = (bool)!!b;
 173     }
 174   } else if (type == Option::TYPE_ADDR) {
 175     entity_addr_t addr;
 176     if (!addr.parse(val.c_str())){
 177       return -EINVAL;
 178     }
 179     *out = addr;
 180   } else if (type == Option::TYPE_ADDRVEC) {
 181     entity_addrvec_t addr;
 182     if (!addr.parse(val.c_str())){
 183       return -EINVAL;
 184     }
 185     *out = addr;
 186   } else if (type == Option::TYPE_UUID) {
 187     uuid_d uuid;
 188     if (!uuid.parse(val.c_str())) {
 189       return -EINVAL;
 190     }
 191     *out = uuid;
 192   } else if (type == Option::TYPE_SIZE) {
 193     Option::size_t sz{strict_iecstrtoll(val.c_str(), error_message)};
 194     if (!error_message->empty()) {
 195       return -EINVAL;
 196     }
 197     *out = sz;
 198   } else if (type == Option::TYPE_SECS) {
 199     try {
 200       *out = parse_timespan(val);
 201     } catch (const invalid_argument& e) {
 202       *error_message = e.what();
 203       return -EINVAL;
 204     }
 205   } else {
 206     ceph_abort();
 207   }
 208
 209   r = validate(*out, error_message);
 210   if (r != 0) {
 211     return r;
 212   }
 213
 214   if (normalized_value) {
 215     *normalized_value = to_str(*out);
 216   }
 217   return 0;
 218 }
 219
 220 void Option::dump(Formatter *f) const
 221 {
 222   f->dump_string("name", name);
 223
 224   f->dump_string("type", type_to_str(type));
 225
 226   f->dump_string("level", level_to_str(level));
 227
 228   f->dump_string("desc", desc);
 229   f->dump_string("long_desc", long_desc);
 230
 231   dump_value("default", value, f);
 232   dump_value("daemon_default", daemon_value, f);
 233
 234   f->open_array_section("tags");
 235   for (const auto t : tags) {
 236     f->dump_string("tag", t);
 237   }
 238   f->close_section();
 239
 240   f->open_array_section("services");
 241   for (const auto s : services) {
 242     f->dump_string("service", s);
 243   }
 244   f->close_section();
 245
 246   f->open_array_section("see_also");
 247   for (const auto sa : see_also) {
 248     f->dump_string("see_also", sa);
 249   }
 250   f->close_section();
 251
 252   if (type == TYPE_STR) {
 253     f->open_array_section("enum_values");
 254     for (const auto &ea : enum_allowed) {
 255       f->dump_string("enum_value", ea);
 256     }
 257     f->close_section();
 258   }
 259
 260   dump_value("min", min, f);
 261   dump_value("max", max, f);
 262
 263   f->dump_bool("can_update_at_runtime", can_update_at_runtime());
 264
 265   f->open_array_section("flags");
 266   if (has_flag(FLAG_RUNTIME)) {
 267     f->dump_string("option", "runtime");
 268   }
 269   if (has_flag(FLAG_NO_MON_UPDATE)) {
 270     f->dump_string("option", "no_mon_update");
 271   }
 272   if (has_flag(FLAG_STARTUP)) {
 273     f->dump_string("option", "startup");
 274   }
 275   if (has_flag(FLAG_CLUSTER_CREATE)) {
 276     f->dump_string("option", "cluster_create");
 277   }
 278   if (has_flag(FLAG_CREATE)) {
 279     f->dump_string("option", "create");
 280   }
 281   f->close_section();
 282 }
 283
 284 std::string Option::to_str(const Option::value_t& v)
 285 {
 286   return stringify(v);
 287 }
 288
 289 void Option::print(ostream *out) const
 290 {
 291   *out << name << " - " << desc << "\n";
 292   *out << "  (" << type_to_str(type) << ", " << level_to_str(level) << ")\n";
 293   if (!boost::get<boost::blank>(&daemon_value)) {
 294     *out << "  Default (non-daemon): " << stringify(value) << "\n";
 295     *out << "  Default (daemon): " << stringify(daemon_value) << "\n";
 296   } else {
 297     *out << "  Default: " << stringify(value) << "\n";
 298   }
 299   if (!enum_allowed.empty()) {
 300     *out << "  Possible values: ";
 301     for (auto& i : enum_allowed) {
 302       *out << " " << stringify(i);
 303     }
 304     *out << "\n";
 305   }
 306   if (!boost::get<boost::blank>(&min)) {
 307     *out << "  Minimum: " << stringify(min) << "\n"
 308          << "  Maximum: " << stringify(max) << "\n";
 309   }
 310   *out << "  Can update at runtime: "
 311        << (can_update_at_runtime() ? "true" : "false") << "\n";
 312   if (!services.empty()) {
 313     *out << "  Services: " << services << "\n";
 314   }
 315   if (!tags.empty()) {
 316     *out << "  Tags: " << tags << "\n";
 317   }
 318   if (!see_also.empty()) {
 319     *out << "  See also: " << see_also << "\n";
 320   }
 321
 322   if (long_desc.size()) {
 323     *out << "\n" << long_desc << "\n";
 324   }
 325 }
 326
 327 constexpr unsigned long long operator"" _min (unsigned long long min) {
 328   return min * 60;
 329 }
 330 constexpr unsigned long long operator"" _hr (unsigned long long hr) {
 331   return hr * 60 * 60;
 332 }
 333 constexpr unsigned long long operator"" _day (unsigned long long day) {
 334   return day * 60 * 60 * 24;
 335 }
 336 constexpr unsigned long long operator"" _K (unsigned long long n) {
 337   return n << 10;
 338 }
 339 constexpr unsigned long long operator"" _M (unsigned long long n) {
 340   return n << 20;
 341 }
 342 constexpr unsigned long long operator"" _G (unsigned long long n) {
 343   return n << 30;
 344 }
 345 constexpr unsigned long long operator"" _T (unsigned long long n) {
 346   return n << 40;
 347 }
 348
 349 std::vector<Option> get_global_options() {
 350   return std::vector<Option>({
 351     Option("host", Option::TYPE_STR, Option::LEVEL_BASIC)
 352     .set_description("local hostname")
 353     .set_long_description("if blank, ceph assumes the short hostname (hostname -s)")
 354     .set_flag(Option::FLAG_NO_MON_UPDATE)
 355     .add_service("common")
 356     .add_tag("network"),
 357
 358     Option("fsid", Option::TYPE_UUID, Option::LEVEL_BASIC)
 359     .set_description("cluster fsid (uuid)")
 360     .set_flag(Option::FLAG_NO_MON_UPDATE)
 361     .set_flag(Option::FLAG_STARTUP)
 362     .add_service("common")
 363     .add_tag("service"),
 364
 365     Option("public_addr", Option::TYPE_ADDR, Option::LEVEL_BASIC)
 366     .set_description("public-facing address to bind to")
 367     .set_flag(Option::FLAG_STARTUP)
 368     .add_service({"mon", "mds", "osd", "mgr"}),
 369
 370     Option("public_addrv", Option::TYPE_ADDRVEC, Option::LEVEL_BASIC)
 371     .set_description("public-facing address to bind to")
 372     .set_flag(Option::FLAG_STARTUP)
 373     .add_service({"mon", "mds", "osd", "mgr"}),
 374
 375     Option("public_bind_addr", Option::TYPE_ADDR, Option::LEVEL_ADVANCED)
 376     .set_default(entity_addr_t())
 377     .set_flag(Option::FLAG_STARTUP)
 378     .add_service("mon")
 379     .set_description(""),
 380
 381     Option("cluster_addr", Option::TYPE_ADDR, Option::LEVEL_BASIC)
 382     .set_description("cluster-facing address to bind to")
 383     .add_service("osd")
 384     .set_flag(Option::FLAG_STARTUP)
 385     .add_tag("network"),
 386
 387     Option("public_network", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 388     .add_service({"mon", "mds", "osd", "mgr"})
 389     .set_flag(Option::FLAG_STARTUP)
 390     .add_tag("network")
 391     .set_description("Network(s) from which to choose a public address to bind to"),
 392
 393     Option("public_network_interface", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 394     .add_service({"mon", "mds", "osd", "mgr"})
 395     .add_tag("network")
 396     .set_flag(Option::FLAG_STARTUP)
 397     .set_description("Interface name(s) from which to choose an address from a public_network to bind to; public_network must also be specified.")
 398     .add_see_also("public_network"),
 399
 400     Option("cluster_network", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 401     .add_service("osd")
 402     .set_flag(Option::FLAG_STARTUP)
 403     .add_tag("network")
 404     .set_description("Network(s) from which to choose a cluster address to bind to"),
 405
 406     Option("cluster_network_interface", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 407     .add_service({"mon", "mds", "osd", "mgr"})
 408     .set_flag(Option::FLAG_STARTUP)
 409     .add_tag("network")
 410     .set_description("Interface name(s) from which to choose an address from a cluster_network to bind to; cluster_network must also be specified.")
 411     .add_see_also("cluster_network"),
 412
 413     Option("monmap", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 414     .set_description("path to MonMap file")
 415     .set_long_description("This option is normally used during mkfs, but can also "
 416                         "be used to identify which monitors to connect to.")
 417     .set_flag(Option::FLAG_NO_MON_UPDATE)
 418     .add_service("mon")
 419     .set_flag(Option::FLAG_CREATE),
 420
 421     Option("mon_host", Option::TYPE_STR, Option::LEVEL_BASIC)
 422     .set_description("list of hosts or addresses to search for a monitor")
 423     .set_long_description("This is a comma, whitespace, or semicolon separated "
 424                         "list of IP addresses or hostnames. Hostnames are "
 425                         "resolved via DNS and all A or AAAA records are "
 426                         "included in the search list.")
 427     .set_flag(Option::FLAG_NO_MON_UPDATE)
 428     .set_flag(Option::FLAG_STARTUP)
 429     .add_service("common"),
 430
 431     Option("mon_host_override", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 432     .set_description("monitor(s) to use overriding the MonMap")
 433     .set_flag(Option::FLAG_NO_MON_UPDATE)
 434     .set_flag(Option::FLAG_STARTUP)
 435     .add_service("common"),
 436
 437     Option("mon_dns_srv_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 438     .set_default("ceph-mon")
 439     .set_description("name of DNS SRV record to check for monitor addresses")
 440     .set_flag(Option::FLAG_STARTUP)
 441     .add_service("common")
 442     .add_tag("network")
 443     .add_see_also("mon_host"),
 444
 445     Option("container_image", Option::TYPE_STR, Option::LEVEL_BASIC)
 446     .set_description("container image (used by cephadm orchestrator)")
 447     .set_flag(Option::FLAG_STARTUP)
 448     .set_default("docker.io/ceph/ceph:v15"),
 449
 450     Option("no_config_file", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 451     .set_default(false)
 452     .set_flag(Option::FLAG_NO_MON_UPDATE)
 453     .set_flag(Option::FLAG_STARTUP)
 454     .add_service("common")
 455     .add_tag("config")
 456     .set_description("signal that we don't require a config file to be present")
 457     .set_long_description("When specified, we won't be looking for a "
 458                           "configuration file, and will instead expect that "
 459                           "whatever options or values are required for us to "
 460                           "work will be passed as arguments."),
 461
 462     // lockdep
 463     Option("lockdep", Option::TYPE_BOOL, Option::LEVEL_DEV)
 464     .set_description("enable lockdep lock dependency analyzer")
 465     .set_flag(Option::FLAG_NO_MON_UPDATE)
 466     .set_flag(Option::FLAG_STARTUP)
 467     .add_service("common"),
 468
 469     Option("lockdep_force_backtrace", Option::TYPE_BOOL, Option::LEVEL_DEV)
 470     .set_description("always gather current backtrace at every lock")
 471     .set_flag(Option::FLAG_STARTUP)
 472     .add_service("common")
 473     .add_see_also("lockdep"),
 474
 475     Option("run_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 476     .set_default("/var/run/ceph")
 477     .set_flag(Option::FLAG_STARTUP)
 478     .set_description("path for the 'run' directory for storing pid and socket files")
 479     .add_service("common")
 480     .add_see_also("admin_socket"),
 481
 482     Option("admin_socket", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 483     .set_default("")
 484     .set_daemon_default("$run_dir/$cluster-$name.asok")
 485     .set_flag(Option::FLAG_STARTUP)
 486     .set_description("path for the runtime control socket file, used by the 'ceph daemon' command")
 487     .add_service("common"),
 488
 489     Option("admin_socket_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 490     .set_description("file mode to set for the admin socket file, e.g, '0755'")
 491     .set_flag(Option::FLAG_STARTUP)
 492     .add_service("common")
 493     .add_see_also("admin_socket"),
 494
 495     // daemon
 496     Option("daemonize", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 497     .set_default(false)
 498     .set_daemon_default(true)
 499     .set_description("whether to daemonize (background) after startup")
 500     .set_flag(Option::FLAG_STARTUP)
 501     .set_flag(Option::FLAG_NO_MON_UPDATE)
 502     .add_service({"mon", "mgr", "osd", "mds"})
 503     .add_tag("service")
 504     .add_see_also({"pid_file", "chdir"}),
 505
 506     Option("setuser", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 507     .set_flag(Option::FLAG_STARTUP)
 508     .set_description("uid or user name to switch to on startup")
 509     .set_long_description("This is normally specified by the systemd unit file.")
 510     .add_service({"mon", "mgr", "osd", "mds"})
 511     .add_tag("service")
 512     .add_see_also("setgroup"),
 513
 514     Option("setgroup", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 515     .set_flag(Option::FLAG_STARTUP)
 516     .set_description("gid or group name to switch to on startup")
 517     .set_long_description("This is normally specified by the systemd unit file.")
 518     .add_service({"mon", "mgr", "osd", "mds"})
 519     .add_tag("service")
 520     .add_see_also("setuser"),
 521
 522     Option("setuser_match_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 523     .set_flag(Option::FLAG_STARTUP)
 524     .set_description("if set, setuser/setgroup is condition on this path matching ownership")
 525     .set_long_description("If setuser or setgroup are specified, and this option is non-empty, then the uid/gid of the daemon will only be changed if the file or directory specified by this option has a matching uid and/or gid.  This exists primarily to allow switching to user ceph for OSDs to be conditional on whether the osd data contents have also been chowned after an upgrade.  This is normally specified by the systemd unit file.")
 526     .add_service({"mon", "mgr", "osd", "mds"})
 527     .add_tag("service")
 528     .add_see_also({"setuser", "setgroup"}),
 529
 530     Option("pid_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 531     .set_flag(Option::FLAG_STARTUP)
 532     .set_description("path to write a pid file (if any)")
 533     .add_service({"mon", "mgr", "osd", "mds"})
 534     .add_tag("service"),
 535
 536     Option("chdir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 537     .set_description("path to chdir(2) to after daemonizing")
 538     .set_flag(Option::FLAG_STARTUP)
 539     .set_flag(Option::FLAG_NO_MON_UPDATE)
 540     .add_service({"mon", "mgr", "osd", "mds"})
 541     .add_tag("service")
 542     .add_see_also("daemonize"),
 543
 544     Option("fatal_signal_handlers", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 545     .set_default(true)
 546     .set_flag(Option::FLAG_STARTUP)
 547     .set_description("whether to register signal handlers for SIGABRT etc that dump a stack trace")
 548     .set_long_description("This is normally true for daemons and values for libraries.")
 549     .add_service({"mon", "mgr", "osd", "mds"})
 550     .add_tag("service"),
 551
 552     Option("crash_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 553     .set_flag(Option::FLAG_STARTUP)
 554     .set_default("/var/lib/ceph/crash")
 555     .set_description("Directory where crash reports are archived"),
 556
 557     // restapi
 558     Option("restapi_log_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 559     .set_description("default set by python code"),
 560
 561     Option("restapi_base_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 562     .set_description("default set by python code"),
 563
 564     Option("erasure_code_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 565     .set_default(CEPH_PKGLIBDIR"/erasure-code")
 566     .set_flag(Option::FLAG_STARTUP)
 567     .set_description("directory where erasure-code plugins can be found")
 568     .add_service({"mon", "osd"}),
 569
 570     // logging
 571     Option("log_file", Option::TYPE_STR, Option::LEVEL_BASIC)
 572     .set_default("")
 573     .set_daemon_default("/var/log/ceph/$cluster-$name.log")
 574     .set_description("path to log file")
 575     .add_see_also({"log_to_file",
 576                    "log_to_stderr",
 577                    "err_to_stderr",
 578                    "log_to_syslog",
 579                    "err_to_syslog"}),
 580
 581     Option("log_max_new", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 582     .set_default(1000)
 583     .set_description("max unwritten log entries to allow before waiting to flush to the log")
 584     .add_see_also("log_max_recent"),
 585
 586     Option("log_max_recent", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 587     .set_default(500)
 588     .set_daemon_default(10000)
 589     .set_description("recent log entries to keep in memory to dump in the event of a crash")
 590     .set_long_description("The purpose of this option is to log at a higher debug level only to the in-memory buffer, and write out the detailed log messages only if there is a crash.  Only log entries below the lower log level will be written unconditionally to the log.  For example, debug_osd=1/5 will write everything <= 1 to the log unconditionally but keep entries at levels 2-5 in memory.  If there is a seg fault or assertion failure, all entries will be dumped to the log."),
 591
 592     Option("log_to_file", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 593     .set_default(true)
 594     .set_description("send log lines to a file")
 595     .add_see_also("log_file"),
 596
 597     Option("log_to_stderr", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 598     .set_default(true)
 599     .set_daemon_default(false)
 600     .set_description("send log lines to stderr"),
 601
 602     Option("err_to_stderr", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 603     .set_default(false)
 604     .set_daemon_default(true)
 605     .set_description("send critical error log lines to stderr"),
 606
 607     Option("log_stderr_prefix", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 608     .set_description("String to prefix log messages with when sent to stderr")
 609     .set_long_description("This is useful in container environments when combined with mon_cluster_log_to_stderr.  The mon log prefixes each line with the channel name (e.g., 'default', 'audit'), while log_stderr_prefix can be set to 'debug '.")
 610     .add_see_also("mon_cluster_log_to_stderr"),
 611
 612     Option("log_to_syslog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 613     .set_default(false)
 614     .set_description("send log lines to syslog facility"),
 615
 616     Option("err_to_syslog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 617     .set_default(false)
 618     .set_description("send critical error log lines to syslog facility"),
 619
 620     Option("log_flush_on_exit", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 621     .set_default(false)
 622     .set_description("set a process exit handler to ensure the log is flushed on exit"),
 623
 624     Option("log_stop_at_utilization", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
 625     .set_default(.97)
 626     .set_min_max(0.0, 1.0)
 627     .set_description("stop writing to the log file when device utilization reaches this ratio")
 628     .add_see_also("log_file"),
 629
 630     Option("log_to_graylog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 631     .set_default(false)
 632     .set_description("send log lines to remote graylog server")
 633     .add_see_also({"err_to_graylog",
 634                    "log_graylog_host",
 635                    "log_graylog_port"}),
 636
 637     Option("err_to_graylog", Option::TYPE_BOOL, Option::LEVEL_BASIC)
 638     .set_default(false)
 639     .set_description("send critical error log lines to remote graylog server")
 640     .add_see_also({"log_to_graylog",
 641                    "log_graylog_host",
 642                    "log_graylog_port"}),
 643
 644     Option("log_graylog_host", Option::TYPE_STR, Option::LEVEL_BASIC)
 645     .set_default("127.0.0.1")
 646     .set_description("address or hostname of graylog server to log to")
 647     .add_see_also({"log_to_graylog",
 648                    "err_to_graylog",
 649                    "log_graylog_port"}),
 650
 651     Option("log_graylog_port", Option::TYPE_INT, Option::LEVEL_BASIC)
 652     .set_default(12201)
 653     .set_description("port number for the remote graylog server")
 654     .add_see_also("log_graylog_host"),
 655
 656     Option("log_coarse_timestamps", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 657     .set_default(true)
 658     .set_description("timestamp log entries from coarse system clock "
 659                      "to improve performance")
 660     .add_service("common")
 661     .add_tag("performance")
 662     .add_tag("service"),
 663
 664
 665     // unmodified
 666     Option("clog_to_monitors", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 667     .set_default("default=true")
 668     .set_flag(Option::FLAG_RUNTIME)
 669     .set_description("Make daemons send cluster log messages to monitors"),
 670
 671     Option("clog_to_syslog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 672     .set_default("false")
 673     .set_flag(Option::FLAG_RUNTIME)
 674     .set_description("Make daemons send cluster log messages to syslog"),
 675
 676     Option("clog_to_syslog_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 677     .set_default("info")
 678     .set_flag(Option::FLAG_RUNTIME)
 679     .set_description("Syslog level for cluster log messages")
 680     .add_see_also("clog_to_syslog"),
 681
 682     Option("clog_to_syslog_facility", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 683     .set_default("default=daemon audit=local0")
 684     .set_flag(Option::FLAG_RUNTIME)
 685     .set_description("Syslog facility for cluster log messages")
 686     .add_see_also("clog_to_syslog"),
 687
 688     Option("clog_to_graylog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 689     .set_default("false")
 690     .set_flag(Option::FLAG_RUNTIME)
 691     .set_description("Make daemons send cluster log to graylog"),
 692
 693     Option("clog_to_graylog_host", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 694     .set_default("127.0.0.1")
 695     .set_flag(Option::FLAG_RUNTIME)
 696     .set_description("Graylog host to cluster log messages")
 697     .add_see_also("clog_to_graylog"),
 698
 699     Option("clog_to_graylog_port", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 700     .set_default("12201")
 701     .set_flag(Option::FLAG_RUNTIME)
 702     .set_description("Graylog port number for cluster log messages")
 703     .add_see_also("clog_to_graylog"),
 704
 705     Option("mon_cluster_log_to_stderr", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 706     .set_default(false)
 707     .add_service("mon")
 708     .set_flag(Option::FLAG_RUNTIME)
 709     .set_description("Make monitor send cluster log messages to stderr (prefixed by channel)")
 710     .add_see_also("log_stderr_prefix"),
 711
 712     Option("mon_cluster_log_to_syslog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 713     .set_default("default=false")
 714     .set_flag(Option::FLAG_RUNTIME)
 715     .add_service("mon")
 716     .set_description("Make monitor send cluster log messages to syslog"),
 717
 718     Option("mon_cluster_log_to_syslog_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 719     .set_default("info")
 720     .add_service("mon")
 721     .set_flag(Option::FLAG_RUNTIME)
 722     .set_description("Syslog level for cluster log messages")
 723     .add_see_also("mon_cluster_log_to_syslog"),
 724
 725     Option("mon_cluster_log_to_syslog_facility", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 726     .set_default("daemon")
 727     .add_service("mon")
 728     .set_flag(Option::FLAG_RUNTIME)
 729     .set_description("Syslog facility for cluster log messages")
 730     .add_see_also("mon_cluster_log_to_syslog"),
 731
 732     Option("mon_cluster_log_to_file", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 733     .set_default(true)
 734     .set_flag(Option::FLAG_RUNTIME)
 735     .add_service("mon")
 736     .set_description("Make monitor send cluster log messages to file")
 737     .add_see_also("mon_cluster_log_file"),
 738
 739     Option("mon_cluster_log_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 740     .set_default("default=/var/log/ceph/$cluster.$channel.log cluster=/var/log/ceph/$cluster.log")
 741     .set_flag(Option::FLAG_RUNTIME)
 742     .add_service("mon")
 743     .set_description("File(s) to write cluster log to")
 744     .set_long_description("This can either be a simple file name to receive all messages, or a list of key/value pairs where the key is the log channel and the value is the filename, which may include $cluster and $channel metavariables")
 745     .add_see_also("mon_cluster_log_to_file"),
 746
 747     Option("mon_cluster_log_file_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 748     .set_default("debug")
 749     .set_flag(Option::FLAG_RUNTIME)
 750     .add_service("mon")
 751     .set_description("Lowest level to include is cluster log file")
 752     .add_see_also("mon_cluster_log_file"),
 753
 754     Option("mon_cluster_log_to_graylog", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 755     .set_default("false")
 756     .set_flag(Option::FLAG_RUNTIME)
 757     .add_service("mon")
 758     .set_description("Make monitor send cluster log to graylog"),
 759
 760     Option("mon_cluster_log_to_graylog_host", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 761     .set_default("127.0.0.1")
 762     .set_flag(Option::FLAG_RUNTIME)
 763     .add_service("mon")
 764     .set_description("Graylog host for cluster log messages")
 765     .add_see_also("mon_cluster_log_to_graylog"),
 766
 767     Option("mon_cluster_log_to_graylog_port", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 768     .set_default("12201")
 769     .set_flag(Option::FLAG_RUNTIME)
 770     .add_service("mon")
 771     .set_description("Graylog port for cluster log messages")
 772     .add_see_also("mon_cluster_log_to_graylog"),
 773
 774     Option("enable_experimental_unrecoverable_data_corrupting_features", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 775     .set_flag(Option::FLAG_RUNTIME)
 776     .set_default("")
 777     .set_description("Enable named (or all with '*') experimental features that may be untested, dangerous, and/or cause permanent data loss"),
 778
 779     Option("plugin_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 780     .set_default(CEPH_PKGLIBDIR)
 781     .set_flag(Option::FLAG_STARTUP)
 782     .add_service({"mon", "osd"})
 783     .set_description("Base directory for dynamically loaded plugins"),
 784
 785     // Compressor
 786     Option("compressor_zlib_isal", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 787     .set_default(false)
 788     .set_description("Use Intel ISA-L accelerated zlib implementation if available"),
 789
 790     Option("compressor_zlib_level", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 791     .set_default(5)
 792     .set_description("Zlib compression level to use"),
 793
 794     Option("compressor_zstd_level", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 795     .set_default(1)
 796     .set_description("Zstd compression level to use"),
 797
 798     Option("qat_compressor_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 799     .set_default(false)
 800     .set_description("Enable Intel QAT acceleration support for compression if available"),
 801
 802     Option("plugin_crypto_accelerator", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 803     .set_default("crypto_isal")
 804     .set_description("Crypto accelerator library to use"),
 805
 806     Option("mempool_debug", Option::TYPE_BOOL, Option::LEVEL_DEV)
 807     .set_default(false)
 808     .set_flag(Option::FLAG_NO_MON_UPDATE)
 809     .set_description(""),
 810
 811     Option("thp", Option::TYPE_BOOL, Option::LEVEL_DEV)
 812     .set_default(false)
 813     .set_flag(Option::FLAG_STARTUP)
 814     .set_description("enable transparent huge page (THP) support")
 815     .set_long_description("Ceph is known to suffer from memory fragmentation due to THP use. This is indicated by RSS usage above configured memory targets. Enabling THP is currently discouraged until selective use of THP by Ceph is implemented."),
 816
 817     Option("key", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 818     .set_default("")
 819     .set_description("Authentication key")
 820     .set_long_description("A CephX authentication key, base64 encoded.  It normally looks something like 'AQAtut9ZdMbNJBAAHz6yBAWyJyz2yYRyeMWDag=='.")
 821     .set_flag(Option::FLAG_STARTUP)
 822     .set_flag(Option::FLAG_NO_MON_UPDATE)
 823     .add_see_also("keyfile")
 824     .add_see_also("keyring"),
 825
 826     Option("keyfile", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 827     .set_default("")
 828     .set_description("Path to a file containing a key")
 829     .set_long_description("The file should contain a CephX authentication key and optionally a trailing newline, but nothing else.")
 830     .set_flag(Option::FLAG_STARTUP)
 831     .set_flag(Option::FLAG_NO_MON_UPDATE)
 832     .add_see_also("key"),
 833
 834     Option("keyring", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 835     .set_default(
 836       "/etc/ceph/$cluster.$name.keyring,/etc/ceph/$cluster.keyring,"
 837       "/etc/ceph/keyring,/etc/ceph/keyring.bin,"
 838   #if defined(__FreeBSD)
 839       "/usr/local/etc/ceph/$cluster.$name.keyring,"
 840       "/usr/local/etc/ceph/$cluster.keyring,"
 841       "/usr/local/etc/ceph/keyring,/usr/local/etc/ceph/keyring.bin,"
 842   #endif
 843     )
 844     .set_description("Path to a keyring file.")
 845     .set_long_description("A keyring file is an INI-style formatted file where the section names are client or daemon names (e.g., 'osd.0') and each section contains a 'key' property with CephX authentication key as the value.")
 846     .set_flag(Option::FLAG_STARTUP)
 847     .set_flag(Option::FLAG_NO_MON_UPDATE)
 848     .add_see_also("key")
 849     .add_see_also("keyfile"),
 850
 851     Option("heartbeat_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
 852     .set_default(5)
 853     .set_flag(Option::FLAG_STARTUP)
 854     .set_description("Frequency of internal heartbeat checks (seconds)"),
 855
 856     Option("heartbeat_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 857     .set_default("")
 858     .set_flag(Option::FLAG_STARTUP)
 859     .set_description("File to touch on successful internal heartbeat")
 860     .set_long_description("If set, this file will be touched every time an internal heartbeat check succeeds.")
 861     .add_see_also("heartbeat_interval"),
 862
 863     Option("heartbeat_inject_failure", Option::TYPE_INT, Option::LEVEL_DEV)
 864     .set_default(0)
 865     .set_description(""),
 866
 867     Option("perf", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 868     .set_default(true)
 869     .set_description("Enable internal performance metrics")
 870     .set_long_description("If enabled, collect and expose internal health metrics"),
 871
 872     Option("ms_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 873     .set_flag(Option::FLAG_STARTUP)
 874     .set_default("async+posix")
 875     .set_description("Messenger implementation to use for network communication"),
 876
 877     Option("ms_public_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 878     .set_default("")
 879     .set_flag(Option::FLAG_STARTUP)
 880     .set_description("Messenger implementation to use for the public network")
 881     .set_long_description("If not specified, use ms_type")
 882     .add_see_also("ms_type"),
 883
 884     Option("ms_cluster_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
 885     .set_default("")
 886     .set_flag(Option::FLAG_STARTUP)
 887     .set_description("Messenger implementation to use for the internal cluster network")
 888     .set_long_description("If not specified, use ms_type")
 889     .add_see_also("ms_type"),
 890
 891     Option("ms_mon_cluster_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 892     .set_default("secure crc")
 893     .set_flag(Option::FLAG_STARTUP)
 894     .set_description("Connection modes (crc, secure) for intra-mon connections in order of preference")
 895     .add_see_also("ms_mon_service_mode")
 896     .add_see_also("ms_mon_client_mode")
 897     .add_see_also("ms_service_mode")
 898     .add_see_also("ms_cluster_mode")
 899     .add_see_also("ms_client_mode"),
 900
 901     Option("ms_mon_service_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 902     .set_default("secure crc")
 903     .set_flag(Option::FLAG_STARTUP)
 904     .set_description("Allowed connection modes (crc, secure) for connections to mons")
 905     .add_see_also("ms_service_mode")
 906     .add_see_also("ms_mon_cluster_mode")
 907     .add_see_also("ms_mon_client_mode")
 908     .add_see_also("ms_cluster_mode")
 909     .add_see_also("ms_client_mode"),
 910
 911     Option("ms_mon_client_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 912     .set_default("secure crc")
 913     .set_flag(Option::FLAG_STARTUP)
 914     .set_description("Connection modes (crc, secure) for connections from clients to monitors in order of preference")
 915     .add_see_also("ms_mon_service_mode")
 916     .add_see_also("ms_mon_cluster_mode")
 917     .add_see_also("ms_service_mode")
 918     .add_see_also("ms_cluster_mode")
 919     .add_see_also("ms_client_mode"),
 920
 921     Option("ms_cluster_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 922     .set_default("crc secure")
 923     .set_flag(Option::FLAG_STARTUP)
 924     .set_description("Connection modes (crc, secure) for intra-cluster connections in order of preference")
 925     .add_see_also("ms_service_mode")
 926     .add_see_also("ms_client_mode"),
 927
 928     Option("ms_service_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 929     .set_default("crc secure")
 930     .set_flag(Option::FLAG_STARTUP)
 931     .set_description("Allowed connection modes (crc, secure) for connections to daemons")
 932     .add_see_also("ms_cluster_mode")
 933     .add_see_also("ms_client_mode"),
 934
 935     Option("ms_client_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
 936     .set_default("crc secure")
 937     .set_flag(Option::FLAG_STARTUP)
 938     .set_description("Connection modes (crc, secure) for connections from clients in order of preference")
 939     .add_see_also("ms_cluster_mode")
 940     .add_see_also("ms_service_mode"),
 941
 942     Option("ms_learn_addr_from_peer", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 943     .set_default(true)
 944     .set_description("Learn address from what IP our first peer thinks we connect from")
 945     .set_long_description("Use the IP address our first peer (usually a monitor) sees that we are connecting from.  This is useful if a client is behind some sort of NAT and we want to see it identified by its local (not NATed) address."),
 946
 947     Option("ms_tcp_nodelay", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
 948     .set_default(true)
 949     .set_description("Disable Nagle's algorithm and send queued network traffic immediately"),
 950
 951     Option("ms_tcp_rcvbuf", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 952     .set_default(0)
 953     .set_description("Size of TCP socket receive buffer"),
 954
 955     Option("ms_tcp_prefetch_max_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 956     .set_default(4_K)
 957     .set_description("Maximum amount of data to prefetch out of the socket receive buffer"),
 958
 959     Option("ms_initial_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
 960     .set_default(.2)
 961     .set_description("Initial backoff after a network error is detected (seconds)"),
 962
 963     Option("ms_max_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
 964     .set_default(15.0)
 965     .set_description("Maximum backoff after a network error before retrying (seconds)")
 966     .add_see_also("ms_initial_backoff"),
 967
 968     Option("ms_crc_data", Option::TYPE_BOOL, Option::LEVEL_DEV)
 969     .set_default(true)
 970     .set_description("Set and/or verify crc32c checksum on data payload sent over network"),
 971
 972     Option("ms_crc_header", Option::TYPE_BOOL, Option::LEVEL_DEV)
 973     .set_default(true)
 974     .set_description("Set and/or verify crc32c checksum on header payload sent over network"),
 975
 976     Option("ms_die_on_bad_msg", Option::TYPE_BOOL, Option::LEVEL_DEV)
 977     .set_default(false)
 978     .set_description("Induce a daemon crash/exit when a bad network message is received"),
 979
 980     Option("ms_die_on_unhandled_msg", Option::TYPE_BOOL, Option::LEVEL_DEV)
 981     .set_default(false)
 982     .set_description("Induce a daemon crash/exit when an unrecognized message is received"),
 983
 984     Option("ms_die_on_old_message", Option::TYPE_BOOL, Option::LEVEL_DEV)
 985     .set_default(false)
 986     .set_description("Induce a daemon crash/exit when a old, undecodable message is received"),
 987
 988     Option("ms_die_on_skipped_message", Option::TYPE_BOOL, Option::LEVEL_DEV)
 989     .set_default(false)
 990     .set_description("Induce a daemon crash/exit if sender skips a message sequence number"),
 991
 992     Option("ms_die_on_bug", Option::TYPE_BOOL, Option::LEVEL_DEV)
 993     .set_default(false)
 994     .set_description("Induce a crash/exit on various bugs (for testing purposes)"),
 995
 996     Option("ms_dispatch_throttle_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
 997     .set_default(100_M)
 998     .set_description("Limit messages that are read off the network but still being processed"),
 999
1000     Option("ms_bind_ipv4", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1001     .set_default(true)
1002     .set_description("Bind servers to IPv4 address(es)")
1003     .add_see_also("ms_bind_ipv6"),
1004
1005     Option("ms_bind_ipv6", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1006     .set_default(false)
1007     .set_description("Bind servers to IPv6 address(es)")
1008     .add_see_also("ms_bind_ipv4"),
1009
1010     Option("ms_bind_prefer_ipv4", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1011     .set_default(false)
1012     .set_description("Prefer IPV4 over IPV6 address(es)"),
1013
1014     Option("ms_bind_msgr1", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1015     .set_default(true)
1016     .set_description("Bind servers to msgr1 (legacy) protocol address(es)")
1017     .add_see_also("ms_bind_msgr2"),
1018
1019     Option("ms_bind_msgr2", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1020     .set_default(true)
1021     .set_description("Bind servers to msgr2 (nautilus+) protocol address(es)")
1022     .add_see_also("ms_bind_msgr1"),
1023
1024     Option("ms_bind_port_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1025     .set_default(6800)
1026     .set_description("Lowest port number to bind daemon(s) to"),
1027
1028     Option("ms_bind_port_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1029     .set_default(7300)
1030     .set_description("Highest port number to bind daemon(s) to"),
1031
1032     Option("ms_bind_retry_count", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1033   #if !defined(__FreeBSD__)
1034     .set_default(3)
1035   #else
1036     // FreeBSD does not use SO_REAUSEADDR so allow for a bit more time per default
1037     .set_default(6)
1038   #endif
1039     .set_description("Number of attempts to make while bind(2)ing to a port"),
1040
1041     Option("ms_bind_retry_delay", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1042   #if !defined(__FreeBSD__)
1043     .set_default(5)
1044   #else
1045     // FreeBSD does not use SO_REAUSEADDR so allow for a bit more time per default
1046     .set_default(6)
1047   #endif
1048     .set_description("Delay between bind(2) attempts (seconds)"),
1049
1050     Option("ms_bind_before_connect", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1051     .set_default(false)
1052     .set_description("Call bind(2) on client sockets"),
1053
1054     Option("ms_tcp_listen_backlog", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1055     .set_default(512)
1056     .set_description("Size of queue of incoming connections for accept(2)"),
1057
1058
1059     Option("ms_connection_ready_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1060     .set_default(10)
1061     .set_description("Time before we declare a not yet ready connection as dead (seconds)"),
1062
1063     Option("ms_connection_idle_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1064     .set_default(900)
1065     .set_description("Time before an idle connection is closed (seconds)"),
1066
1067     Option("ms_pq_max_tokens_per_priority", Option::TYPE_UINT, Option::LEVEL_DEV)
1068     .set_default(16777216)
1069     .set_description(""),
1070
1071     Option("ms_pq_min_cost", Option::TYPE_SIZE, Option::LEVEL_DEV)
1072     .set_default(65536)
1073     .set_description(""),
1074
1075     Option("ms_inject_socket_failures", Option::TYPE_UINT, Option::LEVEL_DEV)
1076     .set_default(0)
1077     .set_description("Inject a socket failure every Nth socket operation"),
1078
1079     Option("ms_inject_delay_type", Option::TYPE_STR, Option::LEVEL_DEV)
1080     .set_default("")
1081     .set_description("Entity type to inject delays for")
1082     .set_flag(Option::FLAG_RUNTIME),
1083
1084     Option("ms_inject_delay_msg_type", Option::TYPE_STR, Option::LEVEL_DEV)
1085     .set_default("")
1086     .set_description("Message type to inject delays for"),
1087
1088     Option("ms_inject_delay_max", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1089     .set_default(1)
1090     .set_description("Max delay to inject"),
1091
1092     Option("ms_inject_delay_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1093     .set_default(0)
1094     .set_description(""),
1095
1096     Option("ms_inject_internal_delays", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1097     .set_default(0)
1098     .set_description("Inject various internal delays to induce races (seconds)"),
1099
1100     Option("ms_blackhole_osd", Option::TYPE_BOOL, Option::LEVEL_DEV)
1101     .set_default(false)
1102     .set_description(""),
1103
1104     Option("ms_blackhole_mon", Option::TYPE_BOOL, Option::LEVEL_DEV)
1105     .set_default(false)
1106     .set_description(""),
1107
1108     Option("ms_blackhole_mds", Option::TYPE_BOOL, Option::LEVEL_DEV)
1109     .set_default(false)
1110     .set_description(""),
1111
1112     Option("ms_blackhole_mgr", Option::TYPE_BOOL, Option::LEVEL_DEV)
1113     .set_default(false)
1114     .set_description(""),
1115
1116     Option("ms_blackhole_client", Option::TYPE_BOOL, Option::LEVEL_DEV)
1117     .set_default(false)
1118     .set_description(""),
1119
1120     Option("ms_dump_on_send", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1121     .set_default(false)
1122     .set_description("Hexdump message to debug log on message send"),
1123
1124     Option("ms_dump_corrupt_message_level", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1125     .set_default(1)
1126     .set_description("Log level at which to hexdump corrupt messages we receive"),
1127
1128     Option("ms_async_op_threads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1129     .set_default(3)
1130     .set_min_max(1, 24)
1131     .set_description("Threadpool size for AsyncMessenger (ms_type=async)"),
1132
1133     Option("ms_async_max_op_threads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1134     .set_default(5)
1135     .set_description("Maximum threadpool size of AsyncMessenger")
1136     .add_see_also("ms_async_op_threads"),
1137
1138     Option("ms_async_rdma_device_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1139     .set_default("")
1140     .set_description(""),
1141
1142     Option("ms_async_rdma_enable_hugepage", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1143     .set_default(false)
1144     .set_description(""),
1145
1146     Option("ms_async_rdma_buffer_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1147     .set_default(128_K)
1148     .set_description(""),
1149
1150     Option("ms_async_rdma_send_buffers", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1151     .set_default(1_K)
1152     .set_description(""),
1153
1154     Option("ms_async_rdma_receive_buffers", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1155     .set_default(32768)
1156     .set_description(""),
1157
1158     Option("ms_async_rdma_receive_queue_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1159     .set_default(4096)
1160     .set_description(""),
1161
1162     Option("ms_async_rdma_support_srq", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1163     .set_default(true)
1164     .set_description(""),
1165
1166     Option("ms_async_rdma_port_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1167     .set_default(1)
1168     .set_description(""),
1169
1170     Option("ms_async_rdma_polling_us", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1171     .set_default(1000)
1172     .set_description(""),
1173
1174     Option("ms_async_rdma_gid_idx", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1175     .set_default(0)
1176     .set_description("use gid_idx to select GID for choosing RoCEv1 or RoCEv2"),
1177
1178     Option("ms_async_rdma_local_gid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1179     .set_default("")
1180     .set_description(""),
1181
1182     Option("ms_async_rdma_roce_ver", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1183     .set_default(1)
1184     .set_description(""),
1185
1186     Option("ms_async_rdma_sl", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1187     .set_default(3)
1188     .set_description(""),
1189
1190     Option("ms_async_rdma_dscp", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1191     .set_default(96)
1192     .set_description(""),
1193
1194     Option("ms_max_accept_failures", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1195     .set_default(4)
1196     .set_description("The maximum number of consecutive failed accept() calls before "
1197                      "considering the daemon is misconfigured and abort it."),
1198
1199     Option("ms_async_rdma_cm", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1200     .set_default(false)
1201     .set_description(""),
1202
1203     Option("ms_async_rdma_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1204     .set_default("ib")
1205     .set_description(""),
1206
1207     Option("ms_dpdk_port_id", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1208     .set_default(0)
1209     .set_description(""),
1210
1211     Option("ms_dpdk_coremask", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1212     .set_default("0xF")         //begin with 0x for the string
1213     .set_description("")
1214     .add_see_also("ms_async_op_threads"),
1215
1216     Option("ms_dpdk_memory_channel", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1217     .set_default("4")
1218     .set_description(""),
1219
1220     Option("ms_dpdk_hugepages", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1221     .set_default("")
1222     .set_description(""),
1223
1224     Option("ms_dpdk_pmd", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1225     .set_default("")
1226     .set_description(""),
1227
1228     Option("ms_dpdk_host_ipv4_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1229     .set_default("")
1230     .set_description(""),
1231
1232     Option("ms_dpdk_gateway_ipv4_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1233     .set_default("")
1234     .set_description(""),
1235
1236     Option("ms_dpdk_netmask_ipv4_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1237     .set_default("")
1238     .set_description(""),
1239
1240     Option("ms_dpdk_lro", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1241     .set_default(true)
1242     .set_description(""),
1243
1244     Option("ms_dpdk_hw_flow_control", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1245     .set_default(true)
1246     .set_description(""),
1247
1248     Option("ms_dpdk_hw_queue_weight", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1249     .set_default(1)
1250     .set_description(""),
1251
1252     Option("ms_dpdk_debug_allow_loopback", Option::TYPE_BOOL, Option::LEVEL_DEV)
1253     .set_default(false)
1254     .set_description(""),
1255
1256     Option("ms_dpdk_rx_buffer_count_per_core", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1257     .set_default(8192)
1258     .set_description(""),
1259
1260     Option("inject_early_sigterm", Option::TYPE_BOOL, Option::LEVEL_DEV)
1261     .set_default(false)
1262     .set_description("send ourselves a SIGTERM early during startup"),
1263
1264     // MON
1265     Option("mon_enable_op_tracker", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1266     .set_default(true)
1267     .add_service("mon")
1268     .set_description("enable/disable MON op tracking"),
1269
1270     Option("mon_op_complaint_time", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
1271     .set_default(30)
1272     .add_service("mon")
1273     .set_description("time after which to consider a monitor operation blocked "
1274                      "after no updates"),
1275
1276     Option("mon_op_log_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1277     .set_default(5)
1278     .add_service("mon")
1279     .set_description("max number of slow ops to display"),
1280
1281     Option("mon_op_history_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1282     .set_default(20)
1283     .add_service("mon")
1284     .set_description("max number of completed ops to track"),
1285
1286     Option("mon_op_history_duration", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
1287     .set_default(600)
1288     .add_service("mon")
1289     .set_description("expiration time in seconds of historical MON OPS"),
1290
1291     Option("mon_op_history_slow_op_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1292     .set_default(20)
1293     .add_service("mon")
1294     .set_description("max number of slow historical MON OPS to keep"),
1295
1296     Option("mon_op_history_slow_op_threshold", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
1297     .set_default(10)
1298     .add_service("mon")
1299     .set_description("duration of an op to be considered as a historical slow op"),
1300
1301     Option("mon_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1302     .set_flag(Option::FLAG_NO_MON_UPDATE)
1303     .set_default("/var/lib/ceph/mon/$cluster-$id")
1304     .add_service("mon")
1305     .set_description("path to mon database"),
1306
1307     Option("mon_initial_members", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1308     .set_default("")
1309     .add_service("mon")
1310     .set_flag(Option::FLAG_NO_MON_UPDATE)
1311     .set_flag(Option::FLAG_CLUSTER_CREATE)
1312     .set_description(""),
1313
1314     Option("mon_compact_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1315     .set_default(false)
1316     .add_service("mon")
1317     .set_description(""),
1318
1319     Option("mon_compact_on_bootstrap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1320     .set_default(false)
1321     .add_service("mon")
1322     .set_description(""),
1323
1324     Option("mon_compact_on_trim", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1325     .set_default(true)
1326     .add_service("mon")
1327     .set_description(""),
1328
1329     /* -- mon: osdmap prune (begin) -- */
1330     Option("mon_osdmap_full_prune_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1331     .set_default(true)
1332     .add_service("mon")
1333     .set_description("enables pruning full osdmap versions when we go over a given number of maps")
1334     .add_see_also("mon_osdmap_full_prune_min")
1335     .add_see_also("mon_osdmap_full_prune_interval")
1336     .add_see_also("mon_osdmap_full_prune_txsize"),
1337
1338     Option("mon_osdmap_full_prune_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1339     .set_default(10000)
1340     .add_service("mon")
1341     .set_description("minimum number of versions in the store to trigger full map pruning")
1342     .add_see_also("mon_osdmap_full_prune_enabled")
1343     .add_see_also("mon_osdmap_full_prune_interval")
1344     .add_see_also("mon_osdmap_full_prune_txsize"),
1345
1346     Option("mon_osdmap_full_prune_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1347     .set_default(10)
1348     .add_service("mon")
1349     .set_description("interval between maps that will not be pruned; maps in the middle will be pruned.")
1350     .add_see_also("mon_osdmap_full_prune_enabled")
1351     .add_see_also("mon_osdmap_full_prune_interval")
1352     .add_see_also("mon_osdmap_full_prune_txsize"),
1353
1354     Option("mon_osdmap_full_prune_txsize", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1355     .set_default(100)
1356     .add_service("mon")
1357     .set_description("number of maps we will prune per iteration")
1358     .add_see_also("mon_osdmap_full_prune_enabled")
1359     .add_see_also("mon_osdmap_full_prune_interval")
1360     .add_see_also("mon_osdmap_full_prune_txsize"),
1361     /* -- mon: osdmap prune (end) -- */
1362
1363     Option("mon_osd_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1364     .set_default(500)
1365     .add_service("mon")
1366     .set_description("maximum number of OSDMaps to cache in memory"),
1367
1368     Option("mon_osd_cache_size_min", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1369     .set_default(128_M)
1370     .add_service("mon")
1371     .set_description("The minimum amount of bytes to be kept mapped in memory for osd monitor caches."),
1372
1373     Option("mon_memory_target", Option::TYPE_SIZE, Option::LEVEL_BASIC)
1374     .set_default(2_G)
1375     .set_flag(Option::FLAG_RUNTIME)
1376     .add_service("mon")
1377     .set_description("The amount of bytes pertaining to osd monitor caches and kv cache to be kept mapped in memory with cache auto-tuning enabled"),
1378
1379     Option("mon_memory_autotune", Option::TYPE_BOOL, Option::LEVEL_BASIC)
1380     .set_default(true)
1381     .set_flag(Option::FLAG_RUNTIME)
1382     .add_service("mon")
1383     .set_description("Autotune the cache memory being used for osd monitors and kv database"),
1384
1385     Option("mon_cpu_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1386     .set_default(4)
1387     .add_service("mon")
1388     .set_description("worker threads for CPU intensive background work"),
1389
1390     Option("mon_osd_mapping_pgs_per_chunk", Option::TYPE_INT, Option::LEVEL_DEV)
1391     .set_default(4096)
1392     .add_service("mon")
1393     .set_description("granularity of PG placement calculation background work"),
1394
1395     Option("mon_clean_pg_upmaps_per_chunk", Option::TYPE_UINT, Option::LEVEL_DEV)
1396     .set_default(256)
1397     .add_service("mon")
1398     .set_description("granularity of PG upmap validation background work"),
1399
1400     Option("mon_osd_max_creating_pgs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1401     .set_default(1024)
1402     .add_service("mon")
1403     .set_description("maximum number of PGs the mon will create at once"),
1404
1405     Option("mon_osd_max_initial_pgs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1406     .set_default(1024)
1407     .add_service("mon")
1408     .set_description("maximum number of PGs a pool will created with")
1409     .set_long_description("If the user specifies more PGs than this, the cluster will subsequently split PGs after the pool is created in order to reach the target."),
1410
1411     Option("mon_tick_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1412     .set_default(5)
1413     .add_service("mon")
1414     .set_description("interval for internal mon background checks"),
1415
1416     Option("mon_session_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1417     .set_default(300)
1418     .add_service("mon")
1419     .set_description("close inactive mon client connections after this many seconds"),
1420
1421     Option("mon_subscribe_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1422     .set_default(1_day)
1423     .add_service("mon")
1424     .set_description("subscribe interval for pre-jewel clients"),
1425
1426     Option("mon_delta_reset_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1427     .set_default(10)
1428     .add_service("mon")
1429     .add_service("mon")
1430     .set_description("window duration for rate calculations in 'ceph status'"),
1431
1432     Option("mon_osd_laggy_halflife", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1433     .set_default(1_hr)
1434     .add_service("mon")
1435     .set_description("halflife of OSD 'lagginess' factor"),
1436
1437     Option("mon_osd_laggy_weight", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1438     .set_default(.3)
1439     .set_min_max(0.0, 1.0)
1440     .add_service("mon")
1441     .set_description("how heavily to weight OSD marking itself back up in overall laggy_probability")
1442     .set_long_description("1.0 means that an OSD marking itself back up (because it was marked down but not actually dead) means a 100% laggy_probability; 0.0 effectively disables tracking of laggy_probability."),
1443
1444     Option("mon_osd_laggy_max_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1445     .set_default(300)
1446     .add_service("mon")
1447     .set_description("cap value for period for OSD to be marked for laggy_interval calculation"),
1448
1449     Option("mon_osd_adjust_heartbeat_grace", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1450     .set_default(true)
1451     .add_service("mon")
1452     .set_description("increase OSD heartbeat grace if peers appear to be laggy")
1453     .set_long_description("If an OSD is marked down but then marks itself back up, it implies it wasn't actually down but was unable to respond to heartbeats.  If this option is true, we can use the laggy_probability and laggy_interval values calculated to model this situation to increase the heartbeat grace period for this OSD so that it isn't marked down again.  laggy_probability is an estimated probability that the given OSD is down because it is laggy (not actually down), and laggy_interval is an estiate on how long it stays down when it is laggy.")
1454     .add_see_also("mon_osd_laggy_halflife")
1455     .add_see_also("mon_osd_laggy_weight")
1456     .add_see_also("mon_osd_laggy_max_interval"),
1457
1458     Option("mon_osd_adjust_down_out_interval", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1459     .set_default(true)
1460     .add_service("mon")
1461     .set_description("increase the mon_osd_down_out_interval if an OSD appears to be laggy")
1462     .add_see_also("mon_osd_adjust_heartbeat_grace"),
1463
1464     Option("mon_osd_auto_mark_in", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1465     .set_default(false)
1466     .add_service("mon")
1467     .set_description("mark any OSD that comes up 'in'"),
1468
1469     Option("mon_osd_auto_mark_auto_out_in", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1470     .set_default(true)
1471     .add_service("mon")
1472     .set_description("mark any OSD that comes up that was automatically marked 'out' back 'in'")
1473     .add_see_also("mon_osd_down_out_interval"),
1474
1475     Option("mon_osd_auto_mark_new_in", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1476     .set_default(true)
1477     .add_service("mon")
1478     .set_description("mark any new OSD that comes up 'in'"),
1479
1480     Option("mon_osd_destroyed_out_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1481     .set_default(600)
1482     .add_service("mon")
1483     .set_description("mark any OSD 'out' that has been 'destroy'ed for this long (seconds)"),
1484
1485     Option("mon_osd_down_out_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1486     .set_default(600)
1487     .add_service("mon")
1488     .set_description("mark any OSD 'out' that has been 'down' for this long (seconds)"),
1489
1490     Option("mon_osd_down_out_subtree_limit", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1491     .set_default("rack")
1492     .set_flag(Option::FLAG_RUNTIME)
1493     .add_service("mon")
1494     .set_description("do not automatically mark OSDs 'out' if an entire subtree of this size is down")
1495     .add_see_also("mon_osd_down_out_interval"),
1496
1497     Option("mon_osd_min_up_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1498     .set_default(.3)
1499     .add_service("mon")
1500     .set_description("do not automatically mark OSDs 'out' if fewer than this many OSDs are 'up'")
1501     .add_see_also("mon_osd_down_out_interval"),
1502
1503     Option("mon_osd_min_in_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1504     .set_default(.75)
1505     .add_service("mon")
1506     .set_description("do not automatically mark OSDs 'out' if fewer than this many OSDs are 'in'")
1507     .add_see_also("mon_osd_down_out_interval"),
1508
1509     Option("mon_osd_warn_op_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1510     .set_default(32)
1511     .add_service("mgr")
1512     .set_description("issue REQUEST_SLOW health warning if OSD ops are slower than this age (seconds)"),
1513
1514     Option("mon_osd_warn_num_repaired", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1515     .set_default(10)
1516     .add_service("mon")
1517     .set_description("issue OSD_TOO_MANY_REPAIRS health warning if an OSD has more than this many read repairs"),
1518
1519     Option("mon_osd_err_op_age_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1520     .set_default(128)
1521     .add_service("mgr")
1522     .set_description("issue REQUEST_STUCK health error if OSD ops are slower than is age (seconds)"),
1523
1524     Option("mon_osd_prime_pg_temp", Option::TYPE_BOOL, Option::LEVEL_DEV)
1525     .set_default(true)
1526     .add_service("mon")
1527     .set_description("minimize peering work by priming pg_temp values after a map change"),
1528
1529     Option("mon_osd_prime_pg_temp_max_time", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1530     .set_default(.5)
1531     .add_service("mon")
1532     .set_description("maximum time to spend precalculating PG mappings on map change (seconds)"),
1533
1534     Option("mon_osd_prime_pg_temp_max_estimate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1535     .set_default(.25)
1536     .add_service("mon")
1537     .set_description("calculate all PG mappings if estimated fraction of PGs that change is above this amount"),
1538
1539     Option("mon_stat_smooth_intervals", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1540     .set_default(6)
1541     .set_min(1)
1542     .add_service("mgr")
1543     .set_description("number of PGMaps stats over which we calc the average read/write throughput of the whole cluster"),
1544
1545     Option("mon_election_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1546     .set_default(5)
1547     .add_service("mon")
1548     .set_description("maximum time for a mon election (seconds)"),
1549
1550     Option("mon_lease", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1551     .set_default(5)
1552     .add_service("mon")
1553     .set_description("lease interval between quorum monitors (seconds)")
1554     .set_long_description("This setting controls how sensitive your mon quorum is to intermittent network issues or other failures."),
1555
1556     Option("mon_lease_renew_interval_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1557     .set_default(.6)
1558     .set_min_max((double)0.0, (double).9999999)
1559     .add_service("mon")
1560     .set_description("multiple of mon_lease for the lease renewal interval")
1561     .set_long_description("Leases must be renewed before they time out.  A smaller value means frequent renewals, while a value close to 1 makes a lease expiration more likely.")
1562     .add_see_also("mon_lease"),
1563
1564     Option("mon_lease_ack_timeout_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1565     .set_default(2.0)
1566     .set_min_max(1.0001, 100.0)
1567     .add_service("mon")
1568     .set_description("multiple of mon_lease for the lease ack interval before calling new election")
1569     .add_see_also("mon_lease"),
1570
1571     Option("mon_accept_timeout_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1572     .set_default(2.0)
1573     .add_service("mon")
1574     .set_description("multiple of mon_lease for follower mons to accept proposed state changes before calling a new election")
1575     .add_see_also("mon_lease"),
1576
1577     Option("mon_clock_drift_allowed", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1578     .set_default(.050)
1579     .add_service("mon")
1580     .set_description("allowed clock drift (in seconds) between mons before issuing a health warning"),
1581
1582     Option("mon_clock_drift_warn_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1583     .set_default(5)
1584     .add_service("mon")
1585     .set_description("exponential backoff factor for logging clock drift warnings in the cluster log"),
1586
1587     Option("mon_timecheck_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1588     .set_default(300.0)
1589     .add_service("mon")
1590     .set_description("frequency of clock synchronization checks between monitors (seconds)"),
1591
1592     Option("mon_timecheck_skew_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1593     .set_default(30.0)
1594     .add_service("mon")
1595     .set_description("frequency of clock synchronization (re)checks between monitors while clocks are believed to be skewed (seconds)")
1596     .add_see_also("mon_timecheck_interval"),
1597
1598     Option("mon_pg_stuck_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1599     .set_default(60)
1600     .set_description("number of seconds after which pgs can be considered stuck inactive, unclean, etc")
1601     .set_long_description("see doc/control.rst under dump_stuck for more info")
1602     .add_service("mgr"),
1603
1604     Option("mon_pg_warn_min_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1605     .set_default(0)
1606     .add_service("mgr")
1607     .set_description("minimal number PGs per (in) osd before we warn the admin"),
1608
1609     Option("mon_max_pg_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1610     .set_min(1)
1611     .set_default(250)
1612     .add_service("mgr")
1613     .set_description("Max number of PGs per OSD the cluster will allow")
1614     .set_long_description("If the number of PGs per OSD exceeds this, a "
1615         "health warning will be visible in `ceph status`.  This is also used "
1616         "in automated PG management, as the threshold at which some pools' "
1617         "pg_num may be shrunk in order to enable increasing the pg_num of "
1618         "others."),
1619
1620     Option("mon_target_pg_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1621     .set_min(1)
1622     .set_default(100)
1623     .set_description("Automated PG management creates this many PGs per OSD")
1624     .set_long_description("When creating pools, the automated PG management "
1625         "logic will attempt to reach this target.  In some circumstances, it "
1626         "may exceed this target, up to the ``mon_max_pg_per_osd`` limit. "
1627         "Conversely, a lower number of PGs per OSD may be created if the "
1628         "cluster is not yet fully utilised"),
1629
1630     Option("mon_pg_warn_max_object_skew", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1631     .set_default(10.0)
1632     .set_description("max skew few average in objects per pg")
1633     .add_service("mgr"),
1634
1635     Option("mon_pg_warn_min_objects", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1636     .set_default(10000)
1637     .set_description("do not warn below this object #")
1638     .add_service("mgr"),
1639
1640     Option("mon_pg_warn_min_pool_objects", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1641     .set_default(1000)
1642     .set_description("do not warn on pools below this object #")
1643     .add_service("mgr"),
1644
1645     Option("mon_pg_check_down_all_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1646     .set_default(.5)
1647     .set_description("threshold of down osds after which we check all pgs")
1648     .add_service("mgr"),
1649
1650     Option("mon_cache_target_full_warn_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1651     .set_default(.66)
1652     .add_service("mgr")
1653     .set_flag(Option::FLAG_NO_MON_UPDATE)
1654     .set_flag(Option::FLAG_CLUSTER_CREATE)
1655     .set_description("issue CACHE_POOL_NEAR_FULL health warning when cache pool utilization exceeds this ratio of usable space"),
1656
1657     Option("mon_osd_full_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1658     .set_default(.95)
1659     .set_flag(Option::FLAG_NO_MON_UPDATE)
1660     .set_flag(Option::FLAG_CLUSTER_CREATE)
1661     .set_description("full ratio of OSDs to be set during initial creation of the cluster"),
1662
1663     Option("mon_osd_backfillfull_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1664     .set_default(.90)
1665     .set_flag(Option::FLAG_NO_MON_UPDATE)
1666     .set_flag(Option::FLAG_CLUSTER_CREATE)
1667     .set_description(""),
1668
1669     Option("mon_osd_nearfull_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1670     .set_default(.85)
1671     .set_flag(Option::FLAG_NO_MON_UPDATE)
1672     .set_flag(Option::FLAG_CLUSTER_CREATE)
1673     .set_description("nearfull ratio for OSDs to be set during initial creation of cluster"),
1674
1675     Option("mon_osd_initial_require_min_compat_client", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1676     .set_default("jewel")
1677     .set_flag(Option::FLAG_NO_MON_UPDATE)
1678     .set_flag(Option::FLAG_CLUSTER_CREATE)
1679     .set_description(""),
1680
1681     Option("mon_allow_pool_delete", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1682     .set_default(false)
1683     .add_service("mon")
1684     .set_description("allow pool deletions"),
1685
1686     Option("mon_fake_pool_delete", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1687     .set_default(false)
1688     .add_service("mon")
1689     .set_description("fake pool deletions by renaming the rados pool"),
1690
1691     Option("mon_globalid_prealloc", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1692     .set_default(10000)
1693     .add_service("mon")
1694     .set_description("number of globalid values to preallocate")
1695     .set_long_description("This setting caps how many new clients can authenticate with the cluster before the monitors have to perform a write to preallocate more.  Large values burn through the 64-bit ID space more quickly."),
1696
1697     Option("mon_osd_report_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1698     .set_default(900)
1699     .add_service("mon")
1700     .set_description("time before OSDs who do not report to the mons are marked down (seconds)"),
1701
1702     Option("mon_warn_on_msgr2_not_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1703     .set_default(true)
1704     .add_service("mon")
1705     .set_description("issue MON_MSGR2_NOT_ENABLED health warning if monitors are all running Nautilus but not all binding to a msgr2 port")
1706     .add_see_also("ms_bind_msgr2"),
1707
1708     Option("mon_warn_on_legacy_crush_tunables", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1709     .set_default(true)
1710     .add_service("mgr")
1711     .set_description("issue OLD_CRUSH_TUNABLES health warning if CRUSH tunables are older than mon_crush_min_required_version")
1712     .add_see_also("mon_crush_min_required_version"),
1713
1714     Option("mon_crush_min_required_version", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1715     .set_default("hammer")
1716     .add_service("mgr")
1717     .set_description("minimum ceph release to use for mon_warn_on_legacy_crush_tunables")
1718     .add_see_also("mon_warn_on_legacy_crush_tunables"),
1719
1720     Option("mon_warn_on_crush_straw_calc_version_zero", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1721     .set_default(true)
1722     .add_service("mgr")
1723     .set_description("issue OLD_CRUSH_STRAW_CALC_VERSION health warning if the CRUSH map's straw_calc_version is zero"),
1724
1725     Option("mon_warn_on_osd_down_out_interval_zero", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1726     .set_default(true)
1727     .add_service("mgr")
1728     .set_description("issue OSD_NO_DOWN_OUT_INTERVAL health warning if mon_osd_down_out_interval is zero")
1729     .set_long_description("Having mon_osd_down_out_interval set to 0 means that down OSDs are not marked out automatically and the cluster does not heal itself without administrator intervention.")
1730     .add_see_also("mon_osd_down_out_interval"),
1731
1732     Option("mon_warn_on_cache_pools_without_hit_sets", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1733     .set_default(true)
1734     .add_service("mgr")
1735     .set_description("issue CACHE_POOL_NO_HIT_SET health warning for cache pools that do not have hit sets configured"),
1736
1737     Option("mon_warn_on_pool_no_app", Option::TYPE_BOOL, Option::LEVEL_DEV)
1738     .set_default(true)
1739     .add_service("mgr")
1740     .set_description("issue POOL_APP_NOT_ENABLED health warning if pool has not application enabled"),
1741
1742     Option("mon_warn_on_pool_pg_num_not_power_of_two", Option::TYPE_BOOL, Option::LEVEL_DEV)
1743     .set_default(true)
1744     .add_service("mon")
1745     .set_description("issue POOL_PG_NUM_NOT_POWER_OF_TWO warning if pool has a non-power-of-two pg_num value"),
1746
1747     Option("mon_warn_on_pool_no_redundancy", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1748     .set_default(true)
1749     .add_service("mon")
1750     .set_description("Issue a health warning if any pool is configured with no replicas")
1751     .add_see_also("osd_pool_default_size")
1752     .add_see_also("osd_pool_default_min_size"),
1753
1754     Option("mon_warn_on_misplaced", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1755     .set_default(false)
1756     .add_service("mgr")
1757     .set_description("Issue a health warning if there are misplaced objects"),
1758
1759     Option("mon_warn_on_too_few_osds", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1760     .set_default(true)
1761     .add_service("mgr")
1762     .set_description("Issue a health warning if there are fewer OSDs than osd_pool_default_size"),
1763
1764     Option("mon_warn_on_slow_ping_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1765     .set_default(0)
1766     .add_service("mgr")
1767     .set_description("Override mon_warn_on_slow_ping_ratio with specified threshold in milliseconds")
1768     .add_see_also("mon_warn_on_slow_ping_ratio"),
1769
1770     Option("mon_warn_on_slow_ping_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1771     .set_default(.05)
1772     .add_service("mgr")
1773     .set_description("Issue a health warning if heartbeat ping longer than percentage of osd_heartbeat_grace")
1774     .add_see_also("osd_heartbeat_grace")
1775     .add_see_also("mon_warn_on_slow_ping_time"),
1776
1777     Option("mon_max_snap_prune_per_epoch", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1778     .set_default(100)
1779     .add_service("mon")
1780     .set_description("max number of pruned snaps we will process in a single OSDMap epoch"),
1781
1782     Option("mon_min_osdmap_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1783     .set_default(500)
1784     .add_service("mon")
1785     .set_description("min number of OSDMaps to store"),
1786
1787     Option("mon_max_log_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1788     .set_default(500)
1789     .add_service("mon")
1790     .set_description("max number of past cluster log epochs to store"),
1791
1792     Option("mon_max_mdsmap_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1793     .set_default(500)
1794     .add_service("mon")
1795     .set_description("max number of FSMaps/MDSMaps to store"),
1796
1797     Option("mon_max_mgrmap_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1798     .set_default(500)
1799     .add_service("mon")
1800     .set_description("max number of MgrMaps to store"),
1801
1802     Option("mon_max_osd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1803     .set_default(10000)
1804     .add_service("mon")
1805     .set_description("max number of OSDs in a cluster"),
1806
1807     Option("mon_probe_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1808     .set_default(2.0)
1809     .add_service("mon")
1810     .set_description("timeout for querying other mons during bootstrap pre-election phase (seconds)"),
1811
1812     Option("mon_client_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1813     .set_default(100ul << 20)
1814     .add_service("mon")
1815     .set_description("max bytes of outstanding client messages mon will read off the network"),
1816
1817     Option("mon_daemon_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1818     .set_default(400ul << 20)
1819     .add_service("mon")
1820     .set_description("max bytes of outstanding mon messages mon will read off the network"),
1821
1822     Option("mon_mgr_proxy_client_bytes_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1823     .set_default(.3)
1824     .add_service("mon")
1825     .set_description("ratio of mon_client_bytes that can be consumed by "
1826                      "proxied mgr commands before we error out to client"),
1827
1828     Option("mon_log_max_summary", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1829     .set_default(50)
1830     .add_service("mon")
1831     .set_description("number of recent cluster log messages to retain"),
1832
1833     Option("mon_max_log_entries_per_event", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1834     .set_default(4096)
1835     .add_service("mon")
1836     .set_description("max cluster log entries per paxos event"),
1837
1838     Option("mon_reweight_min_pgs_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1839     .set_default(10)
1840     .add_service("mgr")
1841     .set_description(""),
1842
1843     Option("mon_reweight_min_bytes_per_osd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1844     .set_default(100_M)
1845     .add_service("mgr")
1846     .set_description(""),
1847
1848     Option("mon_reweight_max_osds", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1849     .set_default(4)
1850     .add_service("mgr")
1851     .set_description(""),
1852
1853     Option("mon_reweight_max_change", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1854     .set_default(0.05)
1855     .add_service("mgr")
1856     .set_description(""),
1857
1858     Option("mon_health_to_clog", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
1859     .set_default(true)
1860     .add_service("mon")
1861     .set_description("log monitor health to cluster log"),
1862
1863     Option("mon_health_to_clog_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1864     .set_default(10_min)
1865     .add_service("mon")
1866     .set_description("frequency to log monitor health to cluster log")
1867     .add_see_also("mon_health_to_clog"),
1868
1869     Option("mon_health_to_clog_tick_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1870     .set_default(60.0)
1871     .add_service("mon")
1872     .set_description(""),
1873
1874     Option("mon_health_detail_to_clog", Option::TYPE_BOOL, Option::LEVEL_DEV)
1875     .set_default(true)
1876     .set_description("log health detail to cluster log"),
1877
1878     Option("mon_health_max_detail", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1879     .set_default(50)
1880     .add_service("mon")
1881     .set_description("max detailed pgs to report in health detail"),
1882
1883     Option("mon_health_log_update_period", Option::TYPE_INT, Option::LEVEL_DEV)
1884     .set_default(5)
1885     .add_service("mon")
1886     .set_description("minimum time in seconds between log messages about "
1887                      "each health check")
1888     .set_min(0),
1889
1890     Option("mon_data_avail_crit", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1891     .set_default(5)
1892     .add_service("mon")
1893     .set_description("issue MON_DISK_CRIT health error when mon available space below this percentage"),
1894
1895     Option("mon_data_avail_warn", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1896     .set_default(30)
1897     .add_service("mon")
1898     .set_description("issue MON_DISK_LOW health warning when mon available space below this percentage"),
1899
1900     Option("mon_data_size_warn", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1901     .set_default(15_G)
1902     .add_service("mon")
1903     .set_description("issue MON_DISK_BIG health warning when mon database is above this size"),
1904
1905     Option("mon_warn_pg_not_scrubbed_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1906     .set_default(0.5)
1907     .set_min(0)
1908     .set_description("Percentage of the scrub max interval past the scrub max interval to warn")
1909     .set_long_description("")
1910     .add_see_also("osd_scrub_max_interval"),
1911
1912     Option("mon_warn_pg_not_deep_scrubbed_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1913     .set_default(0.75)
1914     .set_min(0)
1915     .set_description("Percentage of the deep scrub interval past the deep scrub interval to warn")
1916     .set_long_description("")
1917     .add_see_also("osd_deep_scrub_interval"),
1918
1919     Option("mon_scrub_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1920     .set_default(1_day)
1921     .add_service("mon")
1922     .set_description("frequency for scrubbing mon database"),
1923
1924     Option("mon_scrub_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1925     .set_default(5_min)
1926     .add_service("mon")
1927     .set_description("timeout to restart scrub of mon quorum participant does not respond for the latest chunk"),
1928
1929     Option("mon_scrub_max_keys", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1930     .set_default(100)
1931     .add_service("mon")
1932     .set_description("max keys per on scrub chunk/step"),
1933
1934     Option("mon_scrub_inject_crc_mismatch", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1935     .set_default(0.0)
1936     .add_service("mon")
1937     .set_description("probability for injecting crc mismatches into mon scrub"),
1938
1939     Option("mon_scrub_inject_missing_keys", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1940     .set_default(0.0)
1941     .add_service("mon")
1942     .set_description("probability for injecting missing keys into mon scrub"),
1943
1944     Option("mon_config_key_max_entry_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1945     .set_default(64_K)
1946     .add_service("mon")
1947     .set_description("Defines the number of bytes allowed to be held in a "
1948                      "single config-key entry"),
1949
1950     Option("mon_sync_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
1951     .set_default(60.0)
1952     .add_service("mon")
1953     .set_description("timeout before canceling sync if syncing mon does not respond"),
1954
1955     Option("mon_sync_max_payload_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
1956     .set_default(1_M)
1957     .add_service("mon")
1958     .set_description("target max message payload for mon sync"),
1959
1960     Option("mon_sync_max_payload_keys", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1961     .set_default(2000)
1962     .add_service("mon")
1963     .set_description("target max keys in message payload for mon sync"),
1964
1965     Option("mon_sync_debug", Option::TYPE_BOOL, Option::LEVEL_DEV)
1966     .set_default(false)
1967     .add_service("mon")
1968     .set_description("enable extra debugging during mon sync"),
1969
1970     Option("mon_inject_sync_get_chunk_delay", Option::TYPE_FLOAT, Option::LEVEL_DEV)
1971     .set_default(0)
1972     .add_service("mon")
1973     .set_description("inject delay during sync (seconds)"),
1974
1975     Option("mon_osd_min_down_reporters", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
1976     .set_default(2)
1977     .add_service("mon")
1978     .set_description("number of OSDs from different subtrees who need to report a down OSD for it to count")
1979     .add_see_also("mon_osd_reporter_subtree_level"),
1980
1981     Option("mon_osd_reporter_subtree_level", Option::TYPE_STR, Option::LEVEL_ADVANCED)
1982     .set_default("host")
1983     .add_service("mon")
1984     .set_flag(Option::FLAG_RUNTIME)
1985     .set_description("in which level of parent bucket the reporters are counted"),
1986
1987     Option("mon_osd_snap_trim_queue_warn_on", Option::TYPE_INT, Option::LEVEL_ADVANCED)
1988     .set_default(32768)
1989     .add_service("mon")
1990     .set_description("Warn when snap trim queue is that large (or larger).")
1991     .set_long_description("Warn when snap trim queue length for at least one PG crosses this value, as this is indicator of snap trimmer not keeping up, wasting disk space"),
1992
1993     Option("mon_osd_force_trim_to", Option::TYPE_INT, Option::LEVEL_DEV)
1994     .set_default(0)
1995     .add_service("mon")
1996     .set_description("force mons to trim osdmaps through this epoch"),
1997
1998     Option("mon_mds_force_trim_to", Option::TYPE_INT, Option::LEVEL_DEV)
1999     .set_default(0)
2000     .add_service("mon")
2001     .set_description("force mons to trim mdsmaps/fsmaps through this epoch"),
2002
2003     Option("mon_mds_skip_sanity", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2004     .set_default(false)
2005     .add_service("mon")
2006     .set_description("skip sanity checks on fsmap/mdsmap"),
2007
2008     Option("mon_debug_extra_checks", Option::TYPE_BOOL, Option::LEVEL_DEV)
2009     .set_default(false)
2010     .add_service("mon")
2011     .set_description("Enable some additional monitor checks")
2012     .set_long_description(
2013         "Enable some additional monitor checks that would be too expensive "
2014         "to run on production systems, or would only be relevant while "
2015         "testing or debugging."),
2016
2017     Option("mon_debug_block_osdmap_trim", Option::TYPE_BOOL, Option::LEVEL_DEV)
2018     .set_default(false)
2019     .add_service("mon")
2020     .set_description("Block OSDMap trimming while the option is enabled.")
2021     .set_long_description(
2022         "Blocking OSDMap trimming may be quite helpful to easily reproduce "
2023         "states in which the monitor keeps (hundreds of) thousands of "
2024         "osdmaps."),
2025
2026     Option("mon_debug_deprecated_as_obsolete", Option::TYPE_BOOL, Option::LEVEL_DEV)
2027     .set_default(false)
2028     .add_service("mon")
2029     .set_description("treat deprecated mon commands as obsolete"),
2030
2031     Option("mon_debug_dump_transactions", Option::TYPE_BOOL, Option::LEVEL_DEV)
2032     .set_default(false)
2033     .add_service("mon")
2034     .set_description("dump paxos transactions to log")
2035     .add_see_also("mon_debug_dump_location"),
2036
2037     Option("mon_debug_dump_json", Option::TYPE_BOOL, Option::LEVEL_DEV)
2038     .set_default(false)
2039     .add_service("mon")
2040     .set_description("dump paxos transasctions to log as json")
2041     .add_see_also("mon_debug_dump_transactions"),
2042
2043     Option("mon_debug_dump_location", Option::TYPE_STR, Option::LEVEL_DEV)
2044     .set_default("/var/log/ceph/$cluster-$name.tdump")
2045     .add_service("mon")
2046     .set_description("file to dump paxos transactions to")
2047     .add_see_also("mon_debug_dump_transactions"),
2048
2049     Option("mon_debug_no_require_nautilus", Option::TYPE_BOOL, Option::LEVEL_DEV)
2050     .set_default(false)
2051     .add_service("mon")
2052     .set_flag(Option::FLAG_CLUSTER_CREATE)
2053     .set_description("do not set nautilus feature for new mon clusters"),
2054
2055     Option("mon_debug_no_require_octopus", Option::TYPE_BOOL, Option::LEVEL_DEV)
2056     .set_default(false)
2057     .add_service("mon")
2058     .set_flag(Option::FLAG_CLUSTER_CREATE)
2059     .set_description("do not set octopus feature for new mon clusters"),
2060
2061     Option("mon_debug_no_require_bluestore_for_ec_overwrites", Option::TYPE_BOOL, Option::LEVEL_DEV)
2062     .set_default(false)
2063     .add_service("mon")
2064     .set_description("do not require bluestore OSDs to enable EC overwrites on a rados pool"),
2065
2066     Option("mon_debug_no_initial_persistent_features", Option::TYPE_BOOL, Option::LEVEL_DEV)
2067     .set_default(false)
2068     .add_service("mon")
2069     .set_flag(Option::FLAG_CLUSTER_CREATE)
2070     .set_description("do not set any monmap features for new mon clusters"),
2071
2072     Option("mon_inject_transaction_delay_max", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2073     .set_default(10.0)
2074     .add_service("mon")
2075     .set_description("max duration of injected delay in paxos"),
2076
2077     Option("mon_inject_transaction_delay_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2078     .set_default(0)
2079     .add_service("mon")
2080     .set_description("probability of injecting a delay in paxos"),
2081
2082     Option("mon_inject_pg_merge_bounce_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2083     .set_default(0)
2084     .add_service("mon")
2085     .set_description("probability of failing and reverting a pg_num decrement"),
2086
2087     Option("mon_sync_provider_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
2088     .set_default(0)
2089     .add_service("mon")
2090     .set_description("kill mon sync requester at specific point"),
2091
2092     Option("mon_sync_requester_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
2093     .set_default(0)
2094     .add_service("mon")
2095     .set_description("kill mon sync requestor at specific point"),
2096
2097     Option("mon_force_quorum_join", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2098     .set_default(false)
2099     .add_service("mon")
2100     .set_description("force mon to rejoin quorum even though it was just removed"),
2101
2102     Option("mon_keyvaluedb", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2103     .set_default("rocksdb")
2104     .set_enum_allowed({"leveldb", "rocksdb"})
2105     .set_flag(Option::FLAG_CREATE)
2106     .add_service("mon")
2107     .set_description("database backend to use for the mon database"),
2108
2109     Option("mon_debug_unsafe_allow_tier_with_nonempty_snaps", Option::TYPE_BOOL, Option::LEVEL_DEV)
2110     .set_default(false)
2111     .add_service("mon")
2112     .set_description(""),
2113
2114     Option("mon_osd_blacklist_default_expire", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2115     .set_default(1_hr)
2116     .add_service("mon")
2117     .set_description("Duration in seconds that blacklist entries for clients "
2118                      "remain in the OSD map"),
2119
2120     Option("mon_mds_blacklist_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2121     .set_default(1_day)
2122     .set_min(1_hr)
2123     .add_service("mon")
2124     .set_description("Duration in seconds that blacklist entries for MDS "
2125                      "daemons remain in the OSD map")
2126     .set_flag(Option::FLAG_RUNTIME),
2127
2128     Option("mon_mgr_blacklist_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2129     .set_default(1_day)
2130     .set_min(1_hr)
2131     .add_service("mon")
2132     .set_description("Duration in seconds that blacklist entries for mgr "
2133                      "daemons remain in the OSD map")
2134     .set_flag(Option::FLAG_RUNTIME),
2135
2136     Option("mon_osd_crush_smoke_test", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2137     .set_default(true)
2138     .add_service("mon")
2139     .set_description("perform a smoke test on any new CRUSH map before accepting changes"),
2140
2141     Option("mon_smart_report_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2142     .set_default(5)
2143     .add_service("mon")
2144     .set_description("Timeout (in seconds) for smarctl to run, default is set to 5"),
2145
2146
2147     // PAXOS
2148
2149     Option("paxos_stash_full_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2150     .set_default(25)
2151     .add_service("mon")
2152     .set_description(""),
2153
2154     Option("paxos_max_join_drift", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2155     .set_default(10)
2156     .add_service("mon")
2157     .set_description(""),
2158
2159     Option("paxos_propose_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2160     .set_default(1.0)
2161     .add_service("mon")
2162     .set_description(""),
2163
2164     Option("paxos_min_wait", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2165     .set_default(0.05)
2166     .add_service("mon")
2167     .set_description(""),
2168
2169     Option("paxos_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2170     .set_default(500)
2171     .add_service("mon")
2172     .set_description(""),
2173
2174     Option("paxos_trim_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2175     .set_default(250)
2176     .add_service("mon")
2177     .set_description(""),
2178
2179     Option("paxos_trim_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2180     .set_default(500)
2181     .add_service("mon")
2182     .set_description(""),
2183
2184     Option("paxos_service_trim_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2185     .set_default(250)
2186     .add_service("mon")
2187     .set_description(""),
2188
2189     Option("paxos_service_trim_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2190     .set_default(500)
2191     .add_service("mon")
2192     .set_description(""),
2193
2194     Option("paxos_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
2195     .set_default(0)
2196     .add_service("mon")
2197     .set_description(""),
2198
2199
2200     // AUTH
2201
2202     Option("auth_cluster_required", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2203     .set_default("cephx")
2204     .set_description("authentication methods required by the cluster"),
2205
2206     Option("auth_service_required", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2207     .set_default("cephx")
2208     .set_description("authentication methods required by service daemons"),
2209
2210     Option("auth_client_required", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2211     .set_default("cephx, none")
2212     .set_flag(Option::FLAG_MINIMAL_CONF)
2213     .set_description("authentication methods allowed by clients"),
2214
2215     Option("auth_supported", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2216     .set_default("")
2217     .set_description("authentication methods required (deprecated)"),
2218
2219     Option("max_rotating_auth_attempts", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2220     .set_default(10)
2221     .set_description("number of attempts to initialize rotating keys before giving up"),
2222
2223     Option("rotating_keys_bootstrap_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2224     .set_default(30)
2225     .set_description("timeout for obtaining rotating keys during bootstrap phase (seconds)"),
2226
2227     Option("rotating_keys_renewal_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2228     .set_default(10)
2229     .set_description("timeout for updating rotating keys (seconds)"),
2230
2231     Option("cephx_require_signatures", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2232     .set_default(false)
2233     .set_description(""),
2234
2235     Option("cephx_require_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2236     .set_default(1)
2237     .set_description("Cephx version required (1 = pre-mimic, 2 = mimic+)"),
2238
2239     Option("cephx_cluster_require_signatures", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2240     .set_default(false)
2241     .set_description(""),
2242
2243     Option("cephx_cluster_require_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2244     .set_default(1)
2245     .set_description("Cephx version required by the cluster from clients (1 = pre-mimic, 2 = mimic+)"),
2246
2247     Option("cephx_service_require_signatures", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2248     .set_default(false)
2249     .set_description(""),
2250
2251     Option("cephx_service_require_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2252     .set_default(1)
2253     .set_description("Cephx version required from ceph services (1 = pre-mimic, 2 = mimic+)"),
2254
2255     Option("cephx_sign_messages", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2256     .set_default(true)
2257     .set_description(""),
2258
2259     Option("auth_mon_ticket_ttl", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2260     .set_default(12_hr)
2261     .set_description(""),
2262
2263     Option("auth_service_ticket_ttl", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2264     .set_default(1_hr)
2265     .set_description(""),
2266
2267     Option("auth_debug", Option::TYPE_BOOL, Option::LEVEL_DEV)
2268     .set_default(false)
2269     .set_description(""),
2270
2271     Option("mon_client_hunt_parallel", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2272     .set_default(3)
2273     .set_description(""),
2274
2275     Option("mon_client_hunt_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2276     .set_default(3.0)
2277     .set_description(""),
2278
2279     Option("mon_client_log_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2280     .set_default(1.0)
2281     .set_description("How frequently we send queued cluster log messages to mon"),
2282
2283     Option("mon_client_ping_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2284     .set_default(10.0)
2285     .set_description(""),
2286
2287     Option("mon_client_ping_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2288     .set_default(30.0)
2289     .set_description(""),
2290
2291     Option("mon_client_hunt_interval_backoff", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2292     .set_default(1.5)
2293     .set_description(""),
2294
2295     Option("mon_client_hunt_interval_min_multiple", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2296     .set_default(1.0)
2297     .set_description(""),
2298
2299     Option("mon_client_hunt_interval_max_multiple", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2300     .set_default(10.0)
2301     .set_description(""),
2302
2303     Option("mon_client_max_log_entries_per_message", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2304     .set_default(1000)
2305     .set_description(""),
2306
2307     Option("mon_client_directed_command_retry", Option::TYPE_INT, Option::LEVEL_DEV)
2308     .set_default(2)
2309     .set_description("Number of times to try sending a comamnd directed at a specific monitor"),
2310
2311     Option("mon_max_pool_pg_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2312     .set_default(65536)
2313     .set_description(""),
2314
2315     Option("mon_pool_quota_warn_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2316     .set_default(0)
2317     .set_description("percent of quota at which to issue warnings")
2318     .add_service("mgr"),
2319
2320     Option("mon_pool_quota_crit_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2321     .set_default(0)
2322     .set_description("percent of quota at which to issue errors")
2323     .add_service("mgr"),
2324
2325     Option("crush_location", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2326     .set_default("")
2327     .set_description(""),
2328
2329     Option("crush_location_hook", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2330     .set_default("")
2331     .set_description(""),
2332
2333     Option("crush_location_hook_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2334     .set_default(10)
2335     .set_description(""),
2336
2337     Option("objecter_tick_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2338     .set_default(5.0)
2339     .set_description(""),
2340
2341     Option("objecter_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2342     .set_default(10.0)
2343     .set_description("Seconds before in-flight op is considered 'laggy' and we query mon for the latest OSDMap"),
2344
2345     Option("objecter_inflight_op_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2346     .set_default(100_M)
2347     .set_description("Max in-flight data in bytes (both directions)"),
2348
2349     Option("objecter_inflight_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2350     .set_default(1024)
2351     .set_description("Max in-flight operations"),
2352
2353     Option("objecter_completion_locks_per_session", Option::TYPE_UINT, Option::LEVEL_DEV)
2354     .set_default(32)
2355     .set_description(""),
2356
2357     Option("objecter_inject_no_watch_ping", Option::TYPE_BOOL, Option::LEVEL_DEV)
2358     .set_default(false)
2359     .set_description(""),
2360
2361     Option("objecter_retry_writes_after_first_reply", Option::TYPE_BOOL, Option::LEVEL_DEV)
2362     .set_default(false)
2363     .set_description(""),
2364
2365     Option("objecter_debug_inject_relock_delay", Option::TYPE_BOOL, Option::LEVEL_DEV)
2366     .set_default(false)
2367     .set_description(""),
2368
2369     Option("filer_max_purge_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2370     .set_default(10)
2371     .set_description("Max in-flight operations for purging a striped range (e.g., MDS journal)"),
2372
2373     Option("filer_max_truncate_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2374     .set_default(128)
2375     .set_description("Max in-flight operations for truncating/deleting a striped sequence (e.g., MDS journal)"),
2376
2377     Option("journaler_write_head_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2378     .set_default(15)
2379     .set_description("Interval in seconds between journal header updates (to help bound replay time)"),
2380
2381     // * journal object size
2382     Option("journaler_prefetch_periods", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2383     .set_default(10)
2384     .set_min(2)                 // we need at least 2 periods to make progress.
2385     .set_description("Number of striping periods to prefetch while reading MDS journal"),
2386
2387     // * journal object size
2388     Option("journaler_prezero_periods", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2389     .set_default(5)
2390     // we need to zero at least two periods, minimum, to ensure that we
2391     // have a full empty object/period in front of us.
2392     .set_min(2)
2393     .set_description("Number of striping periods to zero head of MDS journal write position"),
2394
2395     // -- OSD --
2396     Option("osd_calc_pg_upmaps_aggressively", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2397     .set_default(true)
2398     .set_flag(Option::FLAG_RUNTIME)
2399     .set_description("try to calculate PG upmaps more aggressively, e.g., "
2400                      "by doing a fairly exhaustive search of existing PGs "
2401                      "that can be unmapped or upmapped"),
2402
2403     Option("osd_calc_pg_upmaps_local_fallback_retries", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2404     .set_default(100)
2405     .set_flag(Option::FLAG_RUNTIME)
2406     .set_description("Maximum number of PGs we can attempt to unmap or upmap "
2407                      "for a specific overfull or underfull osd per iteration "),
2408
2409     Option("osd_numa_prefer_iface", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2410     .set_default(true)
2411     .set_flag(Option::FLAG_STARTUP)
2412     .set_description("prefer IP on network interface on same numa node as storage")
2413     .add_see_also("osd_numa_auto_affinity"),
2414
2415     Option("osd_numa_auto_affinity", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2416     .set_default(true)
2417     .set_flag(Option::FLAG_STARTUP)
2418     .set_description("automatically set affinity to numa node when storage and network match"),
2419
2420     Option("osd_numa_node", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2421     .set_default(-1)
2422     .set_flag(Option::FLAG_STARTUP)
2423     .set_description("set affinity to a numa node (-1 for none)")
2424     .add_see_also("osd_numa_auto_affinity"),
2425
2426     Option("osd_smart_report_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2427     .set_default(5)
2428     .set_description("Timeout (in seconds) for smarctl to run, default is set to 5"),
2429
2430     Option("osd_check_max_object_name_len_on_startup", Option::TYPE_BOOL, Option::LEVEL_DEV)
2431     .set_default(true)
2432     .set_description(""),
2433
2434     Option("osd_max_backfills", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2435     .set_default(1)
2436     .set_description("Maximum number of concurrent local and remote backfills or recoveries per OSD ")
2437     .set_long_description("There can be osd_max_backfills local reservations AND the same remote reservations per OSD. So a value of 1 lets this OSD participate as 1 PG primary in recovery and 1 shard of another recovering PG."),
2438
2439     Option("osd_min_recovery_priority", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2440     .set_default(0)
2441     .set_description("Minimum priority below which recovery is not performed")
2442     .set_long_description("The purpose here is to prevent the cluster from doing *any* lower priority work (e.g., rebalancing) below this threshold and focus solely on higher priority work (e.g., replicating degraded objects)."),
2443
2444     Option("osd_backfill_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2445     .set_default(30.0)
2446     .set_description("how frequently to retry backfill reservations after being denied (e.g., due to a full OSD)"),
2447
2448     Option("osd_recovery_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2449     .set_default(30.0)
2450     .set_description("how frequently to retry recovery reservations after being denied (e.g., due to a full OSD)"),
2451
2452     Option("osd_agent_max_ops", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2453     .set_default(4)
2454     .set_description("maximum concurrent tiering operations for tiering agent"),
2455
2456     Option("osd_agent_max_low_ops", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2457     .set_default(2)
2458     .set_description("maximum concurrent low-priority tiering operations for tiering agent"),
2459
2460     Option("osd_agent_min_evict_effort", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2461     .set_default(.1)
2462     .set_min_max(0.0, .99)
2463     .set_description("minimum effort to expend evicting clean objects"),
2464
2465     Option("osd_agent_quantize_effort", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2466     .set_default(.1)
2467     .set_description("size of quantize unit for eviction effort"),
2468
2469     Option("osd_agent_delay_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2470     .set_default(5.0)
2471     .set_description("how long agent should sleep if it has no work to do"),
2472
2473     Option("osd_find_best_info_ignore_history_les", Option::TYPE_BOOL, Option::LEVEL_DEV)
2474     .set_default(false)
2475     .set_description("ignore last_epoch_started value when peering AND PROBABLY LOSE DATA")
2476     .set_long_description("THIS IS AN EXTREMELY DANGEROUS OPTION THAT SHOULD ONLY BE USED AT THE DIRECTION OF A DEVELOPER.  It makes peering ignore the last_epoch_started value when peering, which can allow the OSD to believe an OSD has an authoritative view of a PG's contents even when it is in fact old and stale, typically leading to data loss (by believing a stale PG is up to date)."),
2477
2478     Option("osd_agent_hist_halflife", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2479     .set_default(1000)
2480     .set_description("halflife of agent atime and temp histograms"),
2481
2482     Option("osd_agent_slop", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2483     .set_default(.02)
2484     .set_description("slop factor to avoid switching tiering flush and eviction mode"),
2485
2486     Option("osd_uuid", Option::TYPE_UUID, Option::LEVEL_ADVANCED)
2487     .set_default(uuid_d())
2488     .set_flag(Option::FLAG_CREATE)
2489     .set_description("uuid label for a new OSD"),
2490
2491     Option("osd_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2492     .set_default("/var/lib/ceph/osd/$cluster-$id")
2493     .set_flag(Option::FLAG_NO_MON_UPDATE)
2494     .set_description("path to OSD data"),
2495
2496     Option("osd_journal", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2497     .set_default("/var/lib/ceph/osd/$cluster-$id/journal")
2498     .set_flag(Option::FLAG_NO_MON_UPDATE)
2499     .set_description("path to OSD journal (when FileStore backend is in use)"),
2500
2501     Option("osd_journal_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2502     .set_default(5120)
2503     .set_flag(Option::FLAG_CREATE)
2504     .set_description("size of FileStore journal (in MiB)"),
2505
2506     Option("osd_journal_flush_on_shutdown", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2507     .set_default(true)
2508     .set_description("flush FileStore journal contents during clean OSD shutdown"),
2509
2510     Option("osd_os_flags", Option::TYPE_UINT, Option::LEVEL_DEV)
2511     .set_default(0)
2512     .set_description("flags to skip filestore omap or journal initialization"),
2513
2514     Option("osd_max_write_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2515     .set_min(4)
2516     .set_default(90)
2517     .set_description("Maximum size of a RADOS write operation in megabytes")
2518     .set_long_description("This setting prevents clients from doing "
2519         "very large writes to RADOS.  If you set this to a value "
2520         "below what clients expect, they will receive an error "
2521         "when attempting to write to the cluster."),
2522
2523     Option("osd_max_pgls", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2524     .set_default(1024)
2525     .set_description("maximum number of results when listing objects in a pool"),
2526
2527     Option("osd_client_message_size_cap", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2528     .set_default(500_M)
2529     .set_description("maximum memory to devote to in-flight client requests")
2530     .set_long_description("If this value is exceeded, the OSD will not read any new client data off of the network until memory is freed."),
2531
2532     Option("osd_client_message_cap", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2533     .set_default(0)
2534     .set_description("maximum number of in-flight client requests"),
2535
2536     Option("osd_crush_update_weight_set", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2537     .set_default(true)
2538     .set_description("update CRUSH weight-set weights when updating weights")
2539     .set_long_description("If this setting is true, we will update the weight-set weights when adjusting an item's weight, effectively making changes take effect immediately, and discarding any previous optimization in the weight-set value.  Setting this value to false will leave it to the balancer to (slowly, presumably) adjust weights to approach the new target value."),
2540
2541     Option("osd_crush_chooseleaf_type", Option::TYPE_INT, Option::LEVEL_DEV)
2542     .set_default(1)
2543     .set_flag(Option::FLAG_CLUSTER_CREATE)
2544     .set_description("default chooseleaf type for osdmaptool --create"),
2545
2546     Option("osd_pool_use_gmt_hitset", Option::TYPE_BOOL, Option::LEVEL_DEV)
2547     .set_default(true)
2548     .set_description("use UTC for hitset timestamps")
2549     .set_long_description("This setting only exists for compatibility with hammer (and older) clusters."),
2550
2551     Option("osd_crush_update_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2552     .set_default(true)
2553     .set_description("update OSD CRUSH location on startup"),
2554
2555     Option("osd_class_update_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2556     .set_default(true)
2557     .set_description("set OSD device class on startup"),
2558
2559     Option("osd_crush_initial_weight", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2560     .set_default(-1)
2561     .set_description("if >= 0, initial CRUSH weight for newly created OSDs")
2562     .set_long_description("If this value is negative, the size of the OSD in TiB is used."),
2563
2564     Option("osd_pool_default_ec_fast_read", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2565     .set_default(false)
2566     .set_description("set ec_fast_read for new erasure-coded pools")
2567     .add_service("mon"),
2568
2569     Option("osd_pool_default_crush_rule", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2570     .set_default(-1)
2571     .set_description("CRUSH rule for newly created pools")
2572     .add_service("mon"),
2573
2574     Option("osd_pool_erasure_code_stripe_unit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2575     .set_default(4_K)
2576     .set_description("the amount of data (in bytes) in a data chunk, per stripe")
2577     .add_service("mon"),
2578
2579     Option("osd_pool_default_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2580     .set_default(3)
2581     .set_min_max(0, 10)
2582     .set_flag(Option::FLAG_RUNTIME)
2583     .set_description("the number of copies of an object for new replicated pools")
2584     .add_service("mon"),
2585
2586     Option("osd_pool_default_min_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2587     .set_default(0)
2588     .set_min_max(0, 255)
2589     .set_flag(Option::FLAG_RUNTIME)
2590     .set_description("the minimal number of copies allowed to write to a degraded pool for new replicated pools")
2591     .set_long_description("0 means no specific default; ceph will use size-size/2")
2592     .add_see_also("osd_pool_default_size")
2593     .add_service("mon"),
2594
2595     Option("osd_pool_default_pg_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2596     .set_default(32)
2597     .set_description("number of PGs for new pools")
2598     .set_flag(Option::FLAG_RUNTIME)
2599     .add_service("mon"),
2600
2601     Option("osd_pool_default_pgp_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2602     .set_default(0)
2603     .set_description("number of PGs for placement purposes (0 to match pg_num)")
2604     .add_see_also("osd_pool_default_pg_num")
2605     .set_flag(Option::FLAG_RUNTIME)
2606     .add_service("mon"),
2607
2608     Option("osd_pool_default_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2609     .set_default("replicated")
2610     .set_enum_allowed({"replicated", "erasure"})
2611     .set_flag(Option::FLAG_RUNTIME)
2612     .set_description("default type of pool to create")
2613     .add_service("mon"),
2614
2615     Option("osd_pool_default_erasure_code_profile", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2616     .set_default("plugin=jerasure technique=reed_sol_van k=2 m=2")
2617     .set_flag(Option::FLAG_RUNTIME)
2618     .set_description("default erasure code profile for new erasure-coded pools")
2619     .add_service("mon"),
2620
2621     Option("osd_erasure_code_plugins", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2622     .set_default("jerasure lrc"
2623   #if defined(HAVE_BETTER_YASM_ELF64) || defined(HAVE_ARMV8_SIMD)
2624          " isa"
2625   #endif
2626         )
2627     .set_flag(Option::FLAG_STARTUP)
2628     .set_description("erasure code plugins to load")
2629     .add_service("mon")
2630     .add_service("osd"),
2631
2632     Option("osd_allow_recovery_below_min_size", Option::TYPE_BOOL, Option::LEVEL_DEV)
2633     .set_default(true)
2634     .set_description("allow replicated pools to recover with < min_size active members")
2635     .add_service("osd"),
2636
2637     Option("osd_pool_default_flags", Option::TYPE_INT, Option::LEVEL_DEV)
2638     .set_default(0)
2639     .set_description("(integer) flags to set on new pools")
2640     .add_service("mon"),
2641
2642     Option("osd_pool_default_flag_hashpspool", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2643     .set_default(true)
2644     .set_description("set hashpspool (better hashing scheme) flag on new pools")
2645     .add_service("mon"),
2646
2647     Option("osd_pool_default_flag_nodelete", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2648     .set_default(false)
2649     .set_description("set nodelete flag on new pools")
2650     .add_service("mon"),
2651
2652     Option("osd_pool_default_flag_nopgchange", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2653     .set_default(false)
2654     .set_description("set nopgchange flag on new pools")
2655     .add_service("mon"),
2656
2657     Option("osd_pool_default_flag_nosizechange", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2658     .set_default(false)
2659     .set_description("set nosizechange flag on new pools")
2660     .add_service("mon"),
2661
2662     Option("osd_pool_default_hit_set_bloom_fpp", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2663     .set_default(.05)
2664     .set_description("")
2665     .add_see_also("osd_tier_default_cache_hit_set_type")
2666     .add_service("mon"),
2667
2668     Option("osd_pool_default_cache_target_dirty_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2669     .set_default(.4)
2670     .set_description(""),
2671
2672     Option("osd_pool_default_cache_target_dirty_high_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2673     .set_default(.6)
2674     .set_description(""),
2675
2676     Option("osd_pool_default_cache_target_full_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2677     .set_default(.8)
2678     .set_description(""),
2679
2680     Option("osd_pool_default_cache_min_flush_age", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2681     .set_default(0)
2682     .set_description(""),
2683
2684     Option("osd_pool_default_cache_min_evict_age", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2685     .set_default(0)
2686     .set_description(""),
2687
2688     Option("osd_pool_default_cache_max_evict_check_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2689     .set_default(10)
2690     .set_description(""),
2691
2692     Option("osd_pool_default_pg_autoscale_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2693     .set_default("on")
2694     .set_flag(Option::FLAG_RUNTIME)
2695     .set_enum_allowed({"off", "warn", "on"})
2696     .set_description("Default PG autoscaling behavior for new pools"),
2697
2698     Option("osd_pool_default_read_lease_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2699     .set_default(.8)
2700     .set_flag(Option::FLAG_RUNTIME)
2701     .set_description("Default read_lease_ratio for a pool, as a multiple of osd_heartbeat_grace")
2702     .set_long_description("This should be <= 1.0 so that the read lease will have expired by the time we decide to mark a peer OSD down.")
2703     .add_see_also("osd_heartbeat_grace"),
2704
2705     Option("osd_hit_set_min_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2706     .set_default(1000)
2707     .set_description(""),
2708
2709     Option("osd_hit_set_max_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2710     .set_default(100000)
2711     .set_description(""),
2712
2713     Option("osd_hit_set_namespace", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2714     .set_default(".ceph-internal")
2715     .set_description(""),
2716
2717     Option("osd_tier_promote_max_objects_sec", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2718     .set_default(25)
2719     .set_description(""),
2720
2721     Option("osd_tier_promote_max_bytes_sec", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2722     .set_default(5_M)
2723     .set_description(""),
2724
2725     Option("osd_tier_default_cache_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2726     .set_default("writeback")
2727     .set_enum_allowed({"none", "writeback", "forward",
2728                        "readonly", "readforward", "readproxy", "proxy"})
2729     .set_flag(Option::FLAG_RUNTIME)
2730     .set_description(""),
2731
2732     Option("osd_tier_default_cache_hit_set_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2733     .set_default(4)
2734     .set_description(""),
2735
2736     Option("osd_tier_default_cache_hit_set_period", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2737     .set_default(1200)
2738     .set_description(""),
2739
2740     Option("osd_tier_default_cache_hit_set_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2741     .set_default("bloom")
2742     .set_enum_allowed({"bloom", "explicit_hash", "explicit_object"})
2743     .set_flag(Option::FLAG_RUNTIME)
2744     .set_description(""),
2745
2746     Option("osd_tier_default_cache_min_read_recency_for_promote", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2747     .set_default(1)
2748     .set_description("number of recent HitSets the object must appear in to be promoted (on read)"),
2749
2750     Option("osd_tier_default_cache_min_write_recency_for_promote", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2751     .set_default(1)
2752     .set_description("number of recent HitSets the object must appear in to be promoted (on write)"),
2753
2754     Option("osd_tier_default_cache_hit_set_grade_decay_rate", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2755     .set_default(20)
2756     .set_description(""),
2757
2758     Option("osd_tier_default_cache_hit_set_search_last_n", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2759     .set_default(1)
2760     .set_description(""),
2761
2762     Option("osd_objecter_finishers", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2763     .set_default(1)
2764     .set_flag(Option::FLAG_STARTUP)
2765     .set_description(""),
2766
2767     Option("osd_map_dedup", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2768     .set_default(true)
2769     .set_description(""),
2770
2771     Option("osd_map_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2772     .set_default(50)
2773     .set_description(""),
2774
2775     Option("osd_map_message_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2776     .set_default(40)
2777     .set_description("maximum number of OSDMaps to include in a single message"),
2778
2779     Option("osd_map_message_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2780     .set_default(10_M)
2781     .set_description("maximum number of bytes worth of OSDMaps to include in a single message"),
2782
2783     Option("osd_map_share_max_epochs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2784     .set_default(40)
2785     .set_description(""),
2786
2787     Option("osd_pg_epoch_max_lag_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2788     .set_default(2.0)
2789     .set_description("Max multiple of the map cache that PGs can lag before we throttle map injest")
2790     .add_see_also("osd_map_cache_size"),
2791
2792     Option("osd_inject_bad_map_crc_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
2793     .set_default(0)
2794     .set_description(""),
2795
2796     Option("osd_inject_failure_on_pg_removal", Option::TYPE_BOOL, Option::LEVEL_DEV)
2797     .set_default(false)
2798     .set_description(""),
2799
2800     Option("osd_max_markdown_period", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2801     .set_default(600)
2802     .set_description(""),
2803
2804     Option("osd_max_markdown_count", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2805     .set_default(5)
2806     .set_description(""),
2807
2808     Option("osd_op_pq_max_tokens_per_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2809     .set_default(4194304)
2810     .set_description(""),
2811
2812     Option("osd_op_pq_min_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2813     .set_default(65536)
2814     .set_description(""),
2815
2816     Option("osd_recover_clone_overlap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2817     .set_default(true)
2818     .set_description(""),
2819
2820     Option("osd_num_cache_shards", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
2821     .set_default(32)
2822     .set_flag(Option::FLAG_STARTUP)
2823     .set_description("The number of cache shards to use in the object store."),
2824
2825     Option("osd_op_num_threads_per_shard", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2826     .set_default(0)
2827     .set_flag(Option::FLAG_STARTUP)
2828     .set_description(""),
2829
2830     Option("osd_op_num_threads_per_shard_hdd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2831     .set_default(1)
2832     .set_flag(Option::FLAG_STARTUP)
2833     .set_description("")
2834     .add_see_also("osd_op_num_threads_per_shard"),
2835
2836     Option("osd_op_num_threads_per_shard_ssd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2837     .set_default(2)
2838     .set_flag(Option::FLAG_STARTUP)
2839     .set_description("")
2840     .add_see_also("osd_op_num_threads_per_shard"),
2841
2842     Option("osd_op_num_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2843     .set_default(0)
2844     .set_flag(Option::FLAG_STARTUP)
2845     .set_description(""),
2846
2847     Option("osd_op_num_shards_hdd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2848     .set_default(5)
2849     .set_flag(Option::FLAG_STARTUP)
2850     .set_description("")
2851     .add_see_also("osd_op_num_shards"),
2852
2853     Option("osd_op_num_shards_ssd", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2854     .set_default(8)
2855     .set_flag(Option::FLAG_STARTUP)
2856     .set_description("")
2857     .add_see_also("osd_op_num_shards"),
2858
2859     Option("osd_skip_data_digest", Option::TYPE_BOOL, Option::LEVEL_DEV)
2860     .set_default(false)
2861     .set_description("Do not store full-object checksums if the backend (bluestore) does its own checksums.  Only usable with all BlueStore OSDs."),
2862
2863     Option("osd_op_queue", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2864     .set_default("wpq")
2865     .set_enum_allowed( { "wpq", "prioritized",
2866           "mclock_opclass", "mclock_client", "mclock_scheduler",
2867           "debug_random" } )
2868     .set_description("which operation priority queue algorithm to use")
2869     .set_long_description("which operation priority queue algorithm to use; "
2870                           "mclock_opclass mclock_client, and "
2871                           "mclock_client_profile are currently experimental")
2872     .add_see_also("osd_op_queue_cut_off"),
2873
2874     Option("osd_op_queue_cut_off", Option::TYPE_STR, Option::LEVEL_ADVANCED)
2875     .set_default("high")
2876     .set_enum_allowed( { "low", "high", "debug_random" } )
2877     .set_description("the threshold between high priority ops and low priority ops")
2878     .set_long_description("the threshold between high priority ops that use strict priority ordering and low priority ops that use a fairness algorithm that may or may not incorporate priority")
2879     .add_see_also("osd_op_queue"),
2880
2881     Option("osd_mclock_scheduler_client_res", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2882     .set_default(1)
2883     .set_description("IO proportion reserved for each client (default)")
2884     .set_long_description("Only considered for osd_op_queue = mClockScheduler")
2885     .add_see_also("osd_op_queue"),
2886
2887     Option("osd_mclock_scheduler_client_wgt", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2888     .set_default(1)
2889     .set_description("IO share for each client (default) over reservation")
2890     .set_long_description("Only considered for osd_op_queue = mClockScheduler")
2891     .add_see_also("osd_op_queue"),
2892
2893     Option("osd_mclock_scheduler_client_lim", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2894     .set_default(999999)
2895     .set_description("IO limit for each client (default) over reservation")
2896     .set_long_description("Only considered for osd_op_queue = mClockScheduler")
2897     .add_see_also("osd_op_queue"),
2898
2899     Option("osd_mclock_scheduler_background_recovery_res", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2900     .set_default(1)
2901     .set_description("IO proportion reserved for background recovery (default)")
2902     .set_long_description("Only considered for osd_op_queue = mClockScheduler")
2903     .add_see_also("osd_op_queue"),
2904
2905     Option("osd_mclock_scheduler_background_recovery_wgt", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2906     .set_default(1)
2907     .set_description("IO share for each background recovery over reservation")
2908     .set_long_description("Only considered for osd_op_queue = mClockScheduler")
2909     .add_see_also("osd_op_queue"),
2910
2911     Option("osd_mclock_scheduler_background_recovery_lim", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2912     .set_default(999999)
2913     .set_description("IO limit for background recovery over reservation")
2914     .set_long_description("Only considered for osd_op_queue = mClockScheduler")
2915     .add_see_also("osd_op_queue"),
2916
2917     Option("osd_mclock_scheduler_background_best_effort_res", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2918     .set_default(1)
2919     .set_description("IO proportion reserved for background best_effort (default)")
2920     .set_long_description("Only considered for osd_op_queue = mClockScheduler")
2921     .add_see_also("osd_op_queue"),
2922
2923     Option("osd_mclock_scheduler_background_best_effort_wgt", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2924     .set_default(1)
2925     .set_description("IO share for each background best_effort over reservation")
2926     .set_long_description("Only considered for osd_op_queue = mClockScheduler")
2927     .add_see_also("osd_op_queue"),
2928
2929     Option("osd_mclock_scheduler_background_best_effort_lim", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2930     .set_default(999999)
2931     .set_description("IO limit for background best_effort over reservation")
2932     .set_long_description("Only considered for osd_op_queue = mClockScheduler")
2933     .add_see_also("osd_op_queue"),
2934
2935     Option("osd_mclock_scheduler_anticipation_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2936     .set_default(0.0)
2937     .set_description("mclock anticipation timeout in seconds")
2938     .set_long_description("the amount of time that mclock waits until the unused resource is forfeited"),
2939
2940     Option("osd_ignore_stale_divergent_priors", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2941     .set_default(false)
2942     .set_description(""),
2943
2944     Option("osd_read_ec_check_for_errors", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
2945     .set_default(false)
2946     .set_description(""),
2947
2948     // Only use clone_overlap for recovery if there are fewer than
2949     // osd_recover_clone_overlap_limit entries in the overlap set
2950     Option("osd_recover_clone_overlap_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
2951     .set_default(10)
2952     .set_description("")
2953     .set_flag(Option::FLAG_RUNTIME),
2954
2955     Option("osd_debug_feed_pullee", Option::TYPE_INT, Option::LEVEL_DEV)
2956     .set_default(-1)
2957     .set_description("Feed a pullee, and force primary to pull "
2958                      "a currently missing object from it"),
2959
2960     Option("osd_backfill_scan_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2961     .set_default(64)
2962     .set_description(""),
2963
2964     Option("osd_backfill_scan_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2965     .set_default(512)
2966     .set_description(""),
2967
2968     Option("osd_op_thread_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2969     .set_default(15)
2970     .set_description(""),
2971
2972     Option("osd_op_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
2973     .set_default(150)
2974     .set_description(""),
2975
2976     Option("osd_recovery_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2977     .set_default(0)
2978     .set_description("Time in seconds to sleep before next recovery or backfill op"),
2979
2980     Option("osd_recovery_sleep_hdd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2981     .set_default(0.1)
2982     .set_description("Time in seconds to sleep before next recovery or backfill op for HDDs"),
2983
2984     Option("osd_recovery_sleep_ssd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2985     .set_default(0)
2986     .set_description("Time in seconds to sleep before next recovery or backfill op for SSDs")
2987     .add_see_also("osd_recovery_sleep"),
2988
2989     Option("osd_recovery_sleep_hybrid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2990     .set_default(0.025)
2991     .set_description("Time in seconds to sleep before next recovery or backfill op when data is on HDD and journal is on SSD")
2992     .add_see_also("osd_recovery_sleep"),
2993
2994     Option("osd_snap_trim_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2995     .set_default(0)
2996     .set_description("Time in seconds to sleep before next snap trim (overrides values below)"),
2997
2998     Option("osd_snap_trim_sleep_hdd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
2999     .set_default(5)
3000     .set_description("Time in seconds to sleep before next snap trim for HDDs"),
3001
3002     Option("osd_snap_trim_sleep_ssd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3003     .set_default(0)
3004     .set_description("Time in seconds to sleep before next snap trim for SSDs"),
3005
3006     Option("osd_snap_trim_sleep_hybrid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3007     .set_default(2)
3008     .set_description("Time in seconds to sleep before next snap trim when data is on HDD and journal is on SSD"),
3009
3010     Option("osd_scrub_invalid_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3011     .set_default(true)
3012     .set_description(""),
3013
3014     Option("osd_command_thread_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3015     .set_default(10_min)
3016     .set_description(""),
3017
3018     Option("osd_command_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3019     .set_default(15_min)
3020     .set_description(""),
3021
3022     Option("osd_heartbeat_interval", Option::TYPE_INT, Option::LEVEL_DEV)
3023     .set_default(6)
3024     .set_min_max(1, 60)
3025     .set_description("Interval (in seconds) between peer pings"),
3026
3027     Option("osd_heartbeat_grace", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3028     .set_default(20)
3029     .set_description(""),
3030
3031     Option("osd_heartbeat_stale", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3032     .set_default(600)
3033     .set_description("Interval (in seconds) we mark an unresponsive heartbeat peer as stale.")
3034     .set_long_description("Automatically mark unresponsive heartbeat sessions as stale and tear them down. "
3035                           "The primary benefit is that OSD doesn't need to keep a flood of "
3036                           "blocked heartbeat messages around in memory."),
3037
3038     Option("osd_heartbeat_min_peers", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3039     .set_default(10)
3040     .set_description(""),
3041
3042     Option("osd_heartbeat_use_min_delay_socket", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3043     .set_default(false)
3044     .set_description(""),
3045
3046     Option("osd_heartbeat_min_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3047     .set_default(2000)
3048     .set_description("Minimum heartbeat packet size in bytes. Will add dummy payload if heartbeat packet is smaller than this."),
3049
3050     Option("osd_pg_max_concurrent_snap_trims", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3051     .set_default(2)
3052     .set_description(""),
3053
3054     Option("osd_max_trimming_pgs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3055     .set_default(2)
3056     .set_description(""),
3057
3058     Option("osd_heartbeat_min_healthy_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3059     .set_default(.33)
3060     .set_description(""),
3061
3062     Option("osd_mon_heartbeat_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3063     .set_default(30)
3064     .set_description(""),
3065
3066     Option("osd_mon_heartbeat_stat_stale", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3067     .set_default(1_hr)
3068     .set_description("Stop reporting on heartbeat ping times not updated for this many seconds.")
3069     .set_long_description("Stop reporting on old heartbeat information unless this is set to zero"),
3070
3071     Option("osd_mon_report_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3072     .set_default(5)
3073     .set_description("Frequency of OSD reports to mon for peer failures, fullness status changes"),
3074
3075     Option("osd_mon_report_max_in_flight", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3076     .set_default(2)
3077     .set_description(""),
3078
3079     Option("osd_beacon_report_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3080     .set_default(300)
3081     .set_description(""),
3082
3083     Option("osd_pg_stat_report_interval_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3084     .set_default(500)
3085     .set_description(""),
3086
3087     Option("osd_mon_ack_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3088     .set_default(30.0)
3089     .set_description(""),
3090
3091     Option("osd_stats_ack_timeout_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3092     .set_default(2.0)
3093     .set_description(""),
3094
3095     Option("osd_stats_ack_timeout_decay", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3096     .set_default(.9)
3097     .set_description(""),
3098
3099     Option("osd_max_snap_prune_intervals_per_epoch", Option::TYPE_UINT, Option::LEVEL_DEV)
3100     .set_default(512)
3101     .set_description("Max number of snap intervals to report to mgr in pg_stat_t"),
3102
3103     Option("osd_default_data_pool_replay_window", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3104     .set_default(45)
3105     .set_description(""),
3106
3107     Option("osd_auto_mark_unfound_lost", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3108     .set_default(false)
3109     .set_description(""),
3110
3111     Option("osd_recovery_delay_start", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3112     .set_default(0)
3113     .set_description(""),
3114
3115     Option("osd_recovery_max_active", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3116     .set_default(0)
3117     .set_description("Number of simultaneous active recovery operations per OSD (overrides _ssd and _hdd if non-zero)")
3118     .add_see_also("osd_recovery_max_active_hdd")
3119     .add_see_also("osd_recovery_max_active_ssd"),
3120
3121     Option("osd_recovery_max_active_hdd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3122     .set_default(3)
3123     .set_description("Number of simultaneous active recovery oeprations per OSD (for rotational devices)")
3124     .add_see_also("osd_recovery_max_active")
3125     .add_see_also("osd_recovery_max_active_ssd"),
3126
3127     Option("osd_recovery_max_active_ssd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3128     .set_default(10)
3129     .set_description("Number of simultaneous active recovery oeprations per OSD (for non-rotational solid state devices)")
3130     .add_see_also("osd_recovery_max_active")
3131     .add_see_also("osd_recovery_max_active_hdd"),
3132
3133     Option("osd_recovery_max_single_start", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3134     .set_default(1)
3135     .set_description(""),
3136
3137     Option("osd_recovery_max_chunk", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3138     .set_default(8_M)
3139     .set_description(""),
3140
3141     Option("osd_recovery_max_omap_entries_per_chunk", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3142     .set_default(8096)
3143     .set_description(""),
3144
3145     Option("osd_copyfrom_max_chunk", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3146     .set_default(8_M)
3147     .set_description(""),
3148
3149     Option("osd_push_per_object_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3150     .set_default(1000)
3151     .set_description(""),
3152
3153     Option("osd_max_push_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3154     .set_default(8<<20)
3155     .set_description(""),
3156
3157     Option("osd_max_push_objects", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3158     .set_default(10)
3159     .set_description(""),
3160
3161     Option("osd_max_scrubs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3162     .set_default(1)
3163     .set_description("Maximum concurrent scrubs on a single OSD"),
3164
3165     Option("osd_scrub_during_recovery", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3166     .set_default(false)
3167     .set_description("Allow scrubbing when PGs on the OSD are undergoing recovery"),
3168
3169     Option("osd_repair_during_recovery", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3170     .set_default(false)
3171     .set_description("Allow requested repairing when PGs on the OSD are undergoing recovery"),
3172
3173     Option("osd_scrub_begin_hour", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3174     .set_default(0)
3175     .set_description("Restrict scrubbing to this hour of the day or later")
3176     .add_see_also("osd_scrub_end_hour"),
3177
3178     Option("osd_scrub_end_hour", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3179     .set_default(24)
3180     .set_description("Restrict scrubbing to hours of the day earlier than this")
3181     .add_see_also("osd_scrub_begin_hour"),
3182
3183     Option("osd_scrub_begin_week_day", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3184     .set_default(0)
3185     .set_description("Restrict scrubbing to this day of the week or later")
3186     .set_long_description("0 or 7 = Sunday, 1 = Monday, etc.")
3187     .add_see_also("osd_scrub_end_week_day"),
3188
3189     Option("osd_scrub_end_week_day", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3190     .set_default(7)
3191     .set_description("Restrict scrubbing to days of the week earlier than this")
3192     .set_long_description("0 or 7 = Sunday, 1 = Monday, etc.")
3193     .add_see_also("osd_scrub_begin_week_day"),
3194
3195     Option("osd_scrub_load_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3196     .set_default(0.5)
3197     .set_description("Allow scrubbing when system load divided by number of CPUs is below this value"),
3198
3199     Option("osd_scrub_min_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3200     .set_default(1_day)
3201     .set_description("Scrub each PG no more often than this interval")
3202     .add_see_also("osd_scrub_max_interval"),
3203
3204     Option("osd_scrub_max_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3205     .set_default(7_day)
3206     .set_description("Scrub each PG no less often than this interval")
3207     .add_see_also("osd_scrub_min_interval"),
3208
3209     Option("osd_scrub_interval_randomize_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3210     .set_default(0.5)
3211     .set_description("Ratio of scrub interval to randomly vary")
3212     .set_long_description("This prevents a scrub 'stampede' by randomly varying the scrub intervals so that they are soon uniformly distributed over the week")
3213     .add_see_also("osd_scrub_min_interval"),
3214
3215     Option("osd_scrub_backoff_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3216     .set_default(.66)
3217     .set_long_description("This is the precentage of ticks that do NOT schedule scrubs, 66% means that 1 out of 3 ticks will schedule scrubs")
3218     .set_description("Backoff ratio for scheduling scrubs"),
3219
3220     Option("osd_scrub_chunk_min", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3221     .set_default(5)
3222     .set_description("Minimum number of objects to scrub in a single chunk")
3223     .add_see_also("osd_scrub_chunk_max"),
3224
3225     Option("osd_scrub_chunk_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3226     .set_default(25)
3227     .set_description("Maximum number of objects to scrub in a single chunk")
3228     .add_see_also("osd_scrub_chunk_min"),
3229
3230     Option("osd_scrub_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3231     .set_default(0)
3232     .set_description("Duration to inject a delay during scrubbing"),
3233
3234     Option("osd_scrub_extended_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3235     .set_default(0)
3236     .set_description("Duration to inject a delay during scrubbing out of scrubbing hours")
3237     .add_see_also("osd_scrub_begin_hour")
3238     .add_see_also("osd_scrub_end_hour")
3239     .add_see_also("osd_scrub_begin_week_day")
3240     .add_see_also("osd_scrub_end_week_day"),
3241
3242     Option("osd_scrub_auto_repair", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3243     .set_default(false)
3244     .set_description("Automatically repair damaged objects detected during scrub"),
3245
3246     Option("osd_scrub_auto_repair_num_errors", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3247     .set_default(5)
3248     .set_description("Maximum number of detected errors to automatically repair")
3249     .add_see_also("osd_scrub_auto_repair"),
3250
3251     Option("osd_scrub_max_preemptions", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3252     .set_default(5)
3253     .set_min_max(0, 30)
3254     .set_description("Set the maximum number of times we will preempt a deep scrub due to a client operation before blocking client IO to complete the scrub"),
3255
3256     Option("osd_deep_scrub_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3257     .set_default(7_day)
3258     .set_description("Deep scrub each PG (i.e., verify data checksums) at least this often"),
3259
3260     Option("osd_deep_scrub_randomize_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3261     .set_default(0.15)
3262     .set_description("Scrubs will randomly become deep scrubs at this rate (0.15 -> 15% of scrubs are deep)")
3263     .set_long_description("This prevents a deep scrub 'stampede' by spreading deep scrubs so they are uniformly distributed over the week"),
3264
3265     Option("osd_deep_scrub_stride", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3266     .set_default(512_K)
3267     .set_description("Number of bytes to read from an object at a time during deep scrub"),
3268
3269     Option("osd_deep_scrub_keys", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3270     .set_default(1024)
3271     .set_description("Number of keys to read from an object at a time during deep scrub"),
3272
3273     Option("osd_deep_scrub_update_digest_min_age", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3274     .set_default(2_hr)
3275     .set_description("Update overall object digest only if object was last modified longer ago than this"),
3276
3277     Option("osd_deep_scrub_large_omap_object_key_threshold", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3278     .set_default(200000)
3279     .set_description("Warn when we encounter an object with more omap keys than this")
3280     .add_service("osd")
3281     .add_see_also("osd_deep_scrub_large_omap_object_value_sum_threshold"),
3282
3283     Option("osd_deep_scrub_large_omap_object_value_sum_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3284     .set_default(1_G)
3285     .set_description("Warn when we encounter an object with more omap key bytes than this")
3286     .add_service("osd")
3287     .add_see_also("osd_deep_scrub_large_omap_object_key_threshold"),
3288
3289     Option("osd_class_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3290     .set_default(CEPH_LIBDIR "/rados-classes")
3291     .set_description(""),
3292
3293     Option("osd_open_classes_on_start", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3294     .set_default(true)
3295     .set_description(""),
3296
3297     Option("osd_class_load_list", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3298     .set_default("cephfs hello journal lock log numops " "otp rbd refcount rgw rgw_gc timeindex user version cas")
3299     .set_description(""),
3300
3301     Option("osd_class_default_list", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3302     .set_default("cephfs hello journal lock log numops " "otp rbd refcount rgw rgw_gc timeindex user version cas")
3303     .set_description(""),
3304
3305     Option("osd_check_for_log_corruption", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3306     .set_default(false)
3307     .set_description(""),
3308
3309     Option("osd_use_stale_snap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3310     .set_default(false)
3311     .set_description(""),
3312
3313     Option("osd_rollback_to_cluster_snap", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3314     .set_default("")
3315     .set_description(""),
3316
3317     Option("osd_default_notify_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3318     .set_default(30)
3319     .set_description(""),
3320
3321     Option("osd_kill_backfill_at", Option::TYPE_INT, Option::LEVEL_DEV)
3322     .set_default(0)
3323     .set_description(""),
3324
3325     Option("osd_pg_epoch_persisted_max_stale", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3326     .set_default(40)
3327     .set_description(""),
3328
3329     Option("osd_target_pg_log_entries_per_osd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3330     .set_default(3000 * 100)
3331     .set_description("target number of PG entries total on an OSD")
3332     .add_see_also("osd_max_pg_log_entries")
3333     .add_see_also("osd_min_pg_log_entries"),
3334
3335     Option("osd_min_pg_log_entries", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3336     .set_default(250)
3337     .set_description("minimum number of entries to maintain in the PG log")
3338     .add_service("osd")
3339     .add_see_also("osd_max_pg_log_entries")
3340     .add_see_also("osd_pg_log_dups_tracked"),
3341
3342     Option("osd_max_pg_log_entries", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3343     .set_default(10000)
3344     .set_description("maximum number of entries to maintain in the PG log when degraded before we trim")
3345     .add_service("osd")
3346     .add_see_also("osd_min_pg_log_entries")
3347     .add_see_also("osd_pg_log_dups_tracked"),
3348
3349     Option("osd_pg_log_dups_tracked", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3350     .set_default(3000)
3351     .set_description("how many versions back to track in order to detect duplicate ops; this is combined with both the regular pg log entries and additional minimal dup detection entries")
3352     .add_service("osd")
3353     .add_see_also("osd_min_pg_log_entries")
3354     .add_see_also("osd_max_pg_log_entries"),
3355
3356     Option("osd_object_clean_region_max_num_intervals", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3357     .set_default(10)
3358     .set_description("number of intervals in clean_offsets")
3359     .set_long_description("partial recovery uses multiple intervals to record the clean part of the object"
3360         "when the number of intervals is greater than osd_object_clean_region_max_num_intervals, minimum interval will be trimmed"
3361         "(0 will recovery the entire object data interval)")
3362     .add_service("osd"),
3363
3364     Option("osd_force_recovery_pg_log_entries_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3365     .set_default(1.3)
3366     .set_description(""),
3367
3368     Option("osd_pg_log_trim_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3369     .set_default(100)
3370     .set_description(""),
3371
3372     Option("osd_force_auth_primary_missing_objects", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3373     .set_default(100)
3374     .set_description("Approximate missing objects above which to force auth_log_shard to be primary temporarily"),
3375
3376     Option("osd_async_recovery_min_cost", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3377     .set_default(100)
3378     .set_description("A mixture measure of number of current log entries difference "
3379                      "and historical missing objects,  above which we switch to use "
3380                      "asynchronous recovery when appropriate"),
3381
3382     Option("osd_max_pg_per_osd_hard_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3383     .set_default(3)
3384     .set_min(1)
3385     .set_description("Maximum number of PG per OSD, a factor of 'mon_max_pg_per_osd'")
3386     .set_long_description("OSD will refuse to instantiate PG if the number of PG it serves exceeds this number.")
3387     .add_see_also("mon_max_pg_per_osd"),
3388
3389     Option("osd_pg_log_trim_max", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3390     .set_default(10000)
3391     .set_description("maximum number of entries to remove at once from the PG log")
3392     .add_service("osd")
3393     .add_see_also("osd_min_pg_log_entries")
3394     .add_see_also("osd_max_pg_log_entries"),
3395
3396     Option("osd_op_complaint_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3397     .set_default(30)
3398     .set_description(""),
3399
3400     Option("osd_command_max_records", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3401     .set_default(256)
3402     .set_description(""),
3403
3404     Option("osd_max_pg_blocked_by", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3405     .set_default(16)
3406     .set_description(""),
3407
3408     Option("osd_op_log_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3409     .set_default(5)
3410     .set_description(""),
3411
3412     Option("osd_backoff_on_unfound", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3413     .set_default(true)
3414     .set_description(""),
3415
3416     Option("osd_backoff_on_degraded", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3417     .set_default(false)
3418     .set_description(""),
3419
3420     Option("osd_backoff_on_peering", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3421     .set_default(false)
3422     .set_description(""),
3423
3424     Option("osd_debug_shutdown", Option::TYPE_BOOL, Option::LEVEL_DEV)
3425     .set_default(false)
3426     .set_description("Turn up debug levels during shutdown"),
3427
3428     Option("osd_debug_crash_on_ignored_backoff", Option::TYPE_BOOL, Option::LEVEL_DEV)
3429     .set_default(false)
3430     .set_description(""),
3431
3432     Option("osd_debug_inject_dispatch_delay_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3433     .set_default(0)
3434     .set_description(""),
3435
3436     Option("osd_debug_inject_dispatch_delay_duration", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3437     .set_default(.1)
3438     .set_description(""),
3439
3440     Option("osd_debug_drop_ping_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3441     .set_default(0)
3442     .set_description(""),
3443
3444     Option("osd_debug_drop_ping_duration", Option::TYPE_INT, Option::LEVEL_DEV)
3445     .set_default(0)
3446     .set_description(""),
3447
3448     Option("osd_debug_op_order", Option::TYPE_BOOL, Option::LEVEL_DEV)
3449     .set_default(false)
3450     .set_description(""),
3451
3452     Option("osd_debug_verify_missing_on_start", Option::TYPE_BOOL, Option::LEVEL_DEV)
3453     .set_default(false)
3454     .set_description(""),
3455
3456     Option("osd_debug_verify_snaps", Option::TYPE_BOOL, Option::LEVEL_DEV)
3457     .set_default(false)
3458     .set_description(""),
3459
3460     Option("osd_debug_verify_stray_on_activate", Option::TYPE_BOOL, Option::LEVEL_DEV)
3461     .set_default(false)
3462     .set_description(""),
3463
3464     Option("osd_debug_skip_full_check_in_backfill_reservation", Option::TYPE_BOOL, Option::LEVEL_DEV)
3465     .set_default(false)
3466     .set_description(""),
3467
3468     Option("osd_debug_reject_backfill_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3469     .set_default(0)
3470     .set_description(""),
3471
3472     Option("osd_debug_inject_copyfrom_error", Option::TYPE_BOOL, Option::LEVEL_DEV)
3473     .set_default(false)
3474     .set_description(""),
3475
3476     Option("osd_debug_misdirected_ops", Option::TYPE_BOOL, Option::LEVEL_DEV)
3477     .set_default(false)
3478     .set_description(""),
3479
3480     Option("osd_debug_skip_full_check_in_recovery", Option::TYPE_BOOL, Option::LEVEL_DEV)
3481     .set_default(false)
3482     .set_description(""),
3483
3484     Option("osd_debug_random_push_read_error", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3485     .set_default(0)
3486     .set_description(""),
3487
3488     Option("osd_debug_verify_cached_snaps", Option::TYPE_BOOL, Option::LEVEL_DEV)
3489     .set_default(false)
3490     .set_description(""),
3491
3492     Option("osd_debug_deep_scrub_sleep", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3493     .set_default(0)
3494     .set_description("Inject an expensive sleep during deep scrub IO to make it easier to induce preemption"),
3495
3496     Option("osd_debug_no_acting_change", Option::TYPE_BOOL, Option::LEVEL_DEV)
3497     .set_default(false),
3498     Option("osd_debug_no_purge_strays", Option::TYPE_BOOL, Option::LEVEL_DEV)
3499     .set_default(false),
3500
3501     Option("osd_debug_pretend_recovery_active", Option::TYPE_BOOL, Option::LEVEL_DEV)
3502     .set_default(false)
3503     .set_description(""),
3504
3505     Option("osd_enable_op_tracker", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3506     .set_default(true)
3507     .set_description(""),
3508
3509     Option("osd_num_op_tracker_shard", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3510     .set_default(32)
3511     .set_description(""),
3512
3513     Option("osd_op_history_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3514     .set_default(20)
3515     .set_description(""),
3516
3517     Option("osd_op_history_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3518     .set_default(600)
3519     .set_description(""),
3520
3521     Option("osd_op_history_slow_op_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3522     .set_default(20)
3523     .set_description(""),
3524
3525     Option("osd_op_history_slow_op_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3526     .set_default(10.0)
3527     .set_description(""),
3528
3529     Option("osd_target_transaction_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3530     .set_default(30)
3531     .set_description(""),
3532
3533     Option("osd_delete_sleep", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3534     .set_default(0)
3535     .set_description("Time in seconds to sleep before next removal transaction (overrides values below)"),
3536
3537     Option("osd_delete_sleep_hdd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3538     .set_default(5)
3539     .set_description("Time in seconds to sleep before next removal transaction for HDDs"),
3540
3541     Option("osd_delete_sleep_ssd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3542     .set_default(1)
3543     .set_description("Time in seconds to sleep before next removal transaction for SSDs"),
3544
3545     Option("osd_delete_sleep_hybrid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3546     .set_default(1)
3547     .set_description("Time in seconds to sleep before next removal transaction when data is on HDD and journal is on SSD"),
3548
3549     Option("osd_failsafe_full_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3550     .set_default(.97)
3551     .set_description(""),
3552
3553     Option("osd_fast_shutdown", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3554     .set_default(true)
3555     .set_description("Fast, immediate shutdown")
3556     .set_long_description("Setting this to false makes the OSD do a slower teardown of all state when it receives a SIGINT or SIGTERM or when shutting down for any other reason.  That slow shutdown is primarilyy useful for doing memory leak checking with valgrind."),
3557
3558     Option("osd_fast_fail_on_connection_refused", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3559     .set_default(true)
3560     .set_description(""),
3561
3562     Option("osd_pg_object_context_cache_count", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3563     .set_default(64)
3564     .set_description(""),
3565
3566     Option("osd_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3567     .set_default(false)
3568     .set_description(""),
3569
3570     Option("osd_function_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3571     .set_default(false)
3572     .set_description(""),
3573
3574     Option("osd_fast_info", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3575     .set_default(true)
3576     .set_description(""),
3577
3578     Option("osd_debug_pg_log_writeout", Option::TYPE_BOOL, Option::LEVEL_DEV)
3579     .set_default(false)
3580     .set_description(""),
3581
3582     Option("osd_loop_before_reset_tphandle", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3583     .set_default(64)
3584     .set_description(""),
3585
3586     Option("threadpool_default_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3587     .set_default(60)
3588     .set_description(""),
3589
3590     Option("threadpool_empty_queue_max_wait", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3591     .set_default(2)
3592     .set_description(""),
3593
3594     Option("leveldb_log_to_ceph_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3595     .set_default(true)
3596     .set_description(""),
3597
3598     Option("leveldb_write_buffer_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3599     .set_default(8_M)
3600     .set_description(""),
3601
3602     Option("leveldb_cache_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3603     .set_default(128_M)
3604     .set_description(""),
3605
3606     Option("leveldb_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3607     .set_default(0)
3608     .set_description(""),
3609
3610     Option("leveldb_bloom_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3611     .set_default(0)
3612     .set_description(""),
3613
3614     Option("leveldb_max_open_files", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3615     .set_default(0)
3616     .set_description(""),
3617
3618     Option("leveldb_compression", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3619     .set_default(true)
3620     .set_description(""),
3621
3622     Option("leveldb_paranoid", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3623     .set_default(false)
3624     .set_description(""),
3625
3626     Option("leveldb_log", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3627     .set_default("/dev/null")
3628     .set_description(""),
3629
3630     Option("leveldb_compact_on_mount", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3631     .set_default(false)
3632     .set_description(""),
3633
3634     Option("rocksdb_log_to_ceph_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3635     .set_default(true)
3636     .set_description(""),
3637
3638     Option("rocksdb_cache_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3639     .set_default(512_M)
3640     .set_flag(Option::FLAG_RUNTIME)
3641     .set_description(""),
3642
3643     Option("rocksdb_cache_row_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3644     .set_default(0)
3645     .set_description(""),
3646
3647     Option("rocksdb_cache_shard_bits", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3648     .set_default(4)
3649     .set_description(""),
3650
3651     Option("rocksdb_cache_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3652     .set_default("binned_lru")
3653     .set_description(""),
3654
3655     Option("rocksdb_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3656     .set_default(4_K)
3657     .set_description(""),
3658
3659     Option("rocksdb_perf", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3660     .set_default(false)
3661     .set_description(""),
3662
3663     Option("rocksdb_collect_compaction_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3664     .set_default(false)
3665     .set_description(""),
3666
3667     Option("rocksdb_collect_extended_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3668     .set_default(false)
3669     .set_description(""),
3670
3671     Option("rocksdb_collect_memory_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3672     .set_default(false)
3673     .set_description(""),
3674
3675     Option("rocksdb_delete_range_threshold", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3676     .set_default(1048576)
3677     .set_description("The number of keys required to invoke DeleteRange when deleting muliple keys."),
3678
3679     Option("rocksdb_bloom_bits_per_key", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3680     .set_default(20)
3681     .set_description("Number of bits per key to use for RocksDB's bloom filters.")
3682     .set_long_description("RocksDB bloom filters can be used to quickly answer the question of whether or not a key may exist or definitely does not exist in a given RocksDB SST file without having to read all keys into memory.  Using a higher bit value decreases the likelihood of false positives at the expense of additional disk space and memory consumption when the filter is loaded into RAM.  The current default value of 20 was found to provide significant performance gains when getattr calls are made (such as during new object creation in bluestore) without significant memory overhead or cache pollution when combined with rocksdb partitioned index filters.  See: https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters for more information."),
3683
3684     Option("rocksdb_cache_index_and_filter_blocks", Option::TYPE_BOOL, Option::LEVEL_DEV)
3685     .set_default(true)
3686     .set_description("Whether to cache indices and filters in block cache")
3687     .set_long_description("By default RocksDB will load an SST file's index and bloom filters into memory when it is opened and remove them from memory when an SST file is closed.  Thus, memory consumption by indices and bloom filters is directly tied to the number of concurrent SST files allowed to be kept open.  This option instead stores cached indicies and filters in the block cache where they directly compete with other cached data.  By default we set this option to true to better account for and bound rocksdb memory usage and keep filters in memory even when an SST file is closed."),
3688
3689     Option("rocksdb_cache_index_and_filter_blocks_with_high_priority", Option::TYPE_BOOL, Option::LEVEL_DEV)
3690     .set_default(true)
3691     .set_description("Whether to cache indices and filters in the block cache with high priority")
3692     .set_long_description("A downside of setting rocksdb_cache_index_and_filter_blocks to true is that regular data can push indices and filters out of memory.  Setting this option to true means they are cached with higher priority than other data and should typically stay in the block cache."),
3693
3694     Option("rocksdb_pin_l0_filter_and_index_blocks_in_cache", Option::TYPE_BOOL, Option::LEVEL_DEV)
3695     .set_default(false)
3696     .set_description("Whether to pin Level 0 indices and bloom filters in the block cache")
3697     .set_long_description("A downside of setting rocksdb_cache_index_and_filter_blocks to true is that regular data can push indices and filters out of memory.  Setting this option to true means that level 0 SST files will always have their indices and filters pinned in the block cache."),
3698
3699     Option("rocksdb_index_type", Option::TYPE_STR, Option::LEVEL_DEV)
3700     .set_default("binary_search")
3701     .set_description("Type of index for SST files: binary_search, hash_search, two_level")
3702     .set_long_description("This option controls the table index type.  binary_search is a space efficient index block that is optimized for block-search-based index. hash_search may improve prefix lookup performance at the expense of higher disk and memory usage and potentially slower compactions.  two_level is an experimental index type that uses two binary search indexes and works in conjunction with partition filters.  See: http://rocksdb.org/blog/2017/05/12/partitioned-index-filter.html"),
3703
3704     Option("rocksdb_partition_filters", Option::TYPE_BOOL, Option::LEVEL_DEV)
3705     .set_default(false)
3706     .set_description("(experimental) partition SST index/filters into smaller blocks")
3707     .set_long_description("This is an experimental option for rocksdb that works in conjunction with two_level indices to avoid having to keep the entire filter/index in cache when cache_index_and_filter_blocks is true.  The idea is to keep a much smaller top-level index in heap/cache and then opportunistically cache the lower level indices.  See: https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters"),
3708
3709     Option("rocksdb_metadata_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
3710     .set_default(4_K)
3711     .set_description("The block size for index partitions. (0 = rocksdb default)"),
3712
3713     Option("mon_rocksdb_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3714     .set_default("write_buffer_size=33554432,"
3715                  "compression=kNoCompression,"
3716                  "level_compaction_dynamic_level_bytes=true")
3717     .set_description(""),
3718
3719     Option("osd_client_op_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3720     .set_default(63)
3721     .set_description(""),
3722
3723     Option("osd_recovery_op_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3724     .set_default(3)
3725     .set_description("Priority to use for recovery operations if not specified for the pool"),
3726
3727     Option("osd_peering_op_priority", Option::TYPE_UINT, Option::LEVEL_DEV)
3728     .set_default(255)
3729     .set_description(""),
3730
3731     Option("osd_snap_trim_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3732     .set_default(5)
3733     .set_description(""),
3734
3735     Option("osd_snap_trim_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3736     .set_default(1<<20)
3737     .set_description(""),
3738
3739     Option("osd_pg_delete_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3740     .set_default(5)
3741     .set_description(""),
3742
3743     Option("osd_pg_delete_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3744     .set_default(1<<20)
3745     .set_description(""),
3746
3747     Option("osd_scrub_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3748     .set_default(5)
3749     .set_description("Priority for scrub operations in work queue"),
3750
3751     Option("osd_scrub_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3752     .set_default(50<<20)
3753     .set_description("Cost for scrub operations in work queue"),
3754
3755     Option("osd_requested_scrub_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3756     .set_default(120)
3757     .set_description(""),
3758
3759     Option("osd_recovery_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3760     .set_default(5)
3761     .set_description("Priority of recovery in the work queue")
3762     .set_long_description("Not related to a pool's recovery_priority"),
3763
3764     Option("osd_recovery_cost", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3765     .set_default(20<<20)
3766     .set_description(""),
3767
3768     Option("osd_recovery_op_warn_multiple", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3769     .set_default(16)
3770     .set_description(""),
3771
3772     Option("osd_mon_shutdown_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3773     .set_default(5)
3774     .set_description(""),
3775
3776     Option("osd_shutdown_pgref_assert", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3777     .set_default(false)
3778     .set_description(""),
3779
3780     Option("osd_max_object_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3781     .set_default(128_M)
3782     .set_description(""),
3783
3784     Option("osd_max_object_name_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3785     .set_default(2048)
3786     .set_description(""),
3787
3788     Option("osd_max_object_namespace_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3789     .set_default(256)
3790     .set_description(""),
3791
3792     Option("osd_max_attr_name_len", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3793     .set_default(100)
3794     .set_description(""),
3795
3796     Option("osd_max_attr_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3797     .set_default(0)
3798     .set_description(""),
3799
3800     Option("osd_max_omap_entries_per_request", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3801     .set_default(1024)
3802     .set_description(""),
3803
3804     Option("osd_max_omap_bytes_per_request", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3805     .set_default(1_G)
3806     .set_description(""),
3807
3808     Option("osd_max_write_op_reply_len", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3809     .set_default(32)
3810     .set_description("Max size of the per-op payload for requests with the RETURNVEC flag set")
3811     .set_long_description("This value caps the amount of data (per op; a request may have many ops) that will be sent back to the client and recorded in the PG log."),
3812
3813     Option("osd_objectstore", Option::TYPE_STR, Option::LEVEL_ADVANCED)
3814     .set_default("bluestore")
3815     .set_enum_allowed({"bluestore", "filestore", "memstore", "kstore"})
3816     .set_flag(Option::FLAG_CREATE)
3817     .set_description("backend type for an OSD (like filestore or bluestore)"),
3818
3819     Option("osd_objectstore_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3820     .set_default(false)
3821     .set_description(""),
3822
3823     Option("osd_objectstore_fuse", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3824     .set_default(false)
3825     .set_description(""),
3826
3827     Option("osd_bench_small_size_max_iops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3828     .set_default(100)
3829     .set_description(""),
3830
3831     Option("osd_bench_large_size_max_throughput", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3832     .set_default(100_M)
3833     .set_description(""),
3834
3835     Option("osd_bench_max_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3836     .set_default(64_M)
3837     .set_description(""),
3838
3839     Option("osd_bench_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
3840     .set_default(30)
3841     .set_description(""),
3842
3843     Option("osd_blkin_trace_all", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3844     .set_default(false)
3845     .set_description(""),
3846
3847     Option("osdc_blkin_trace_all", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3848     .set_default(false)
3849     .set_description(""),
3850
3851     Option("osd_discard_disconnected_ops", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3852     .set_default(true)
3853     .set_description(""),
3854
3855     Option("osd_memory_target", Option::TYPE_SIZE, Option::LEVEL_BASIC)
3856     .set_default(4_G)
3857     .set_min(896_M)
3858     .set_flag(Option::FLAG_RUNTIME)
3859     .add_see_also("bluestore_cache_autotune")
3860     .add_see_also("osd_memory_cache_min")
3861     .add_see_also("osd_memory_base")
3862     .set_description("When tcmalloc and cache autotuning is enabled, try to keep this many bytes mapped in memory.")
3863     .set_long_description("The minimum value must be at least equal to osd_memory_base + osd_memory_cache_min."),
3864
3865     Option("osd_memory_target_cgroup_limit_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3866     .set_default(0.8)
3867     .set_min_max(0.0, 1.0)
3868     .add_see_also("osd_memory_target")
3869     .set_description("Set the default value for osd_memory_target to the cgroup memory limit (if set) times this value")
3870     .set_long_description("A value of 0 disables this feature."),
3871
3872     Option("osd_memory_base", Option::TYPE_SIZE, Option::LEVEL_DEV)
3873     .set_default(768_M)
3874     .set_flag(Option::FLAG_RUNTIME)
3875     .add_see_also("bluestore_cache_autotune")
3876     .set_description("When tcmalloc and cache autotuning is enabled, estimate the minimum amount of memory in bytes the OSD will need."),
3877
3878     Option("osd_memory_expected_fragmentation", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3879     .set_default(0.15)
3880     .set_min_max(0.0, 1.0)
3881     .set_flag(Option::FLAG_RUNTIME)
3882     .add_see_also("bluestore_cache_autotune")
3883     .set_description("When tcmalloc and cache autotuning is enabled, estimate the percent of memory fragmentation."),
3884
3885     Option("osd_memory_cache_min", Option::TYPE_SIZE, Option::LEVEL_DEV)
3886     .set_default(128_M)
3887     .set_min(128_M)
3888     .set_flag(Option::FLAG_RUNTIME)
3889     .add_see_also("bluestore_cache_autotune")
3890     .set_description("When tcmalloc and cache autotuning is enabled, set the minimum amount of memory used for caches."),
3891
3892     Option("osd_memory_cache_resize_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3893     .set_default(1)
3894     .add_see_also("bluestore_cache_autotune")
3895     .set_description("When tcmalloc and cache autotuning is enabled, wait this many seconds between resizing caches."),
3896
3897     Option("memstore_device_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3898     .set_default(1_G)
3899     .set_description(""),
3900
3901     Option("memstore_page_set", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3902     .set_default(false)
3903     .set_description(""),
3904
3905     Option("memstore_page_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3906     .set_default(64_K)
3907     .set_description(""),
3908
3909     Option("memstore_debug_omit_block_device_write", Option::TYPE_BOOL, Option::LEVEL_DEV)
3910     .set_default(false)
3911     .add_see_also("bluestore_debug_omit_block_device_write")
3912     .set_description("write metadata only"),
3913
3914     Option("objectstore_blackhole", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3915     .set_default(false)
3916     .set_description(""),
3917
3918     // --------------------------
3919     // bluestore
3920
3921     Option("bdev_debug_inflight_ios", Option::TYPE_BOOL, Option::LEVEL_DEV)
3922     .set_default(false)
3923     .set_description(""),
3924
3925     Option("bdev_inject_crash", Option::TYPE_INT, Option::LEVEL_DEV)
3926     .set_default(0)
3927     .set_description(""),
3928
3929     Option("bdev_inject_crash_flush_delay", Option::TYPE_INT, Option::LEVEL_DEV)
3930     .set_default(2)
3931     .set_description(""),
3932
3933     Option("bdev_aio", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3934     .set_default(true)
3935     .set_description(""),
3936
3937     Option("bdev_aio_poll_ms", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3938     .set_default(250)
3939     .set_description(""),
3940
3941     Option("bdev_aio_max_queue_depth", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3942     .set_default(1024)
3943     .set_description(""),
3944
3945     Option("bdev_aio_reap_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3946     .set_default(16)
3947     .set_description(""),
3948
3949     Option("bdev_block_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3950     .set_default(4_K)
3951     .set_description(""),
3952
3953     Option("bdev_debug_aio", Option::TYPE_BOOL, Option::LEVEL_DEV)
3954     .set_default(false)
3955     .set_description(""),
3956
3957     Option("bdev_debug_aio_suicide_timeout", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3958     .set_default(60.0)
3959     .set_description(""),
3960
3961     Option("bdev_debug_aio_log_age", Option::TYPE_FLOAT, Option::LEVEL_DEV)
3962     .set_default(5.0)
3963     .set_description(""),
3964
3965     Option("bdev_nvme_unbind_from_kernel", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3966     .set_default(false)
3967     .set_description(""),
3968
3969     Option("bdev_nvme_retry_count", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3970     .set_default(-1)
3971     .set_description(""),
3972
3973     Option("bdev_enable_discard", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3974     .set_default(false)
3975     .set_description(""),
3976
3977     Option("bdev_async_discard", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
3978     .set_default(false)
3979     .set_description(""),
3980
3981     Option("bdev_flock_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
3982     .set_default(0.1)
3983     .set_description("interval to retry the flock"),
3984
3985     Option("bdev_flock_retry", Option::TYPE_INT, Option::LEVEL_ADVANCED)
3986     .set_default(3)
3987     .set_description("times to retry the flock"),
3988
3989     Option("bluefs_alloc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3990     .set_default(1_M)
3991     .set_description("Allocation unit size for DB and WAL devices"),
3992
3993     Option("bluefs_shared_alloc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3994     .set_default(64_K)
3995     .set_description("Allocation unit size for primary/shared device"),
3996
3997     Option("bluefs_max_prefetch", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
3998     .set_default(1_M)
3999     .set_description(""),
4000
4001     Option("bluefs_min_log_runway", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4002     .set_default(1_M)
4003     .set_description(""),
4004
4005     Option("bluefs_max_log_runway", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4006     .set_default(4194304)
4007     .set_description(""),
4008
4009     Option("bluefs_log_compact_min_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4010     .set_default(5.0)
4011     .set_description(""),
4012
4013     Option("bluefs_log_compact_min_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4014     .set_default(16_M)
4015     .set_description(""),
4016
4017     Option("bluefs_min_flush_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4018     .set_default(512_K)
4019     .set_description(""),
4020
4021     Option("bluefs_compact_log_sync", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4022     .set_default(false)
4023     .set_description(""),
4024
4025     Option("bluefs_buffered_io", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4026     .set_default(false)
4027     .set_description("Enabled buffered IO for bluefs reads.")
4028     .set_long_description("When this option is enabled, bluefs will in some cases perform buffered reads.  This allows the kernel page cache to act as a secondary cache for things like RocksDB compaction.  For example, if the rocksdb block cache isn't large enough to hold blocks from the compressed SST files itself, they can be read from page cache instead of from the disk.  This option previously was enabled by default, however in some test cases it appears to cause excessive swap utilization by the linux kernel and a large negative performance impact after several hours of run time.  Please exercise caution when enabling."),
4029
4030     Option("bluefs_sync_write", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4031     .set_default(false)
4032     .set_description(""),
4033
4034     Option("bluefs_allocator", Option::TYPE_STR, Option::LEVEL_DEV)
4035     .set_default("hybrid")
4036     .set_enum_allowed({"bitmap", "stupid", "avl", "hybrid"})
4037     .set_description(""),
4038
4039     Option("bluefs_log_replay_check_allocations", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4040       .set_default(true)
4041       .set_description("Enables checks for allocations consistency during log replay"),
4042
4043     Option("bluefs_replay_recovery", Option::TYPE_BOOL, Option::LEVEL_DEV)
4044     .set_default(false)
4045     .set_description("Attempt to read bluefs log so large that it became unreadable.")
4046     .set_long_description("If BlueFS log grows to extreme sizes (200GB+) it is likely that it becames unreadable. "
4047                           "This options enables heuristics that scans devices for missing data. "
4048                           "DO NOT ENABLE BY DEFAULT"),
4049
4050     Option("bluefs_replay_recovery_disable_compact", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4051     .set_default(false)
4052     .set_description(""),
4053
4054     Option("bluestore_bluefs", Option::TYPE_BOOL, Option::LEVEL_DEV)
4055     .set_default(true)
4056     .set_flag(Option::FLAG_CREATE)
4057     .set_description("Use BlueFS to back rocksdb")
4058     .set_long_description("BlueFS allows rocksdb to share the same physical device(s) as the rest of BlueStore.  It should be used in all cases unless testing/developing an alternative metadata database for BlueStore."),
4059
4060     Option("bluestore_bluefs_env_mirror", Option::TYPE_BOOL, Option::LEVEL_DEV)
4061     .set_default(false)
4062     .set_flag(Option::FLAG_CREATE)
4063     .set_description("Mirror bluefs data to file system for testing/validation"),
4064
4065     Option("bluestore_bluefs_min", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4066     .set_default(1_G)
4067     .set_description("minimum disk space allocated to BlueFS (e.g., at mkfs)"),
4068
4069     Option("bluestore_bluefs_min_free", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4070     .set_default(1_G)
4071     .set_description("minimum free space allocated to BlueFS"),
4072
4073     Option("bluestore_bluefs_max_free", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4074     .set_default(10_G)
4075     .set_description("Maximum free space allocated to BlueFS"),
4076
4077     Option("bluestore_bluefs_min_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4078     .set_default(.02)
4079     .set_description("Minimum fraction of free space devoted to BlueFS"),
4080
4081     Option("bluestore_bluefs_max_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4082     .set_default(.90)
4083     .set_description("Maximum fraction of free storage devoted to BlueFS"),
4084
4085     Option("bluestore_bluefs_gift_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4086     .set_default(.02)
4087     .set_description("Maximum fraction of free space to give to BlueFS at once"),
4088
4089     Option("bluestore_bluefs_reclaim_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4090     .set_default(.20)
4091     .set_description("Maximum fraction of free space to reclaim from BlueFS at once"),
4092
4093     Option("bluestore_bluefs_balance_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4094     .set_default(1)
4095     .set_description("How frequently (in seconds) to balance free space between BlueFS and BlueStore"),
4096
4097     Option("bluestore_bluefs_alloc_failure_dump_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4098     .set_default(0)
4099     .set_description("How frequently (in seconds) to dump allocator on"
4100       "BlueFS space allocation failure"),
4101
4102     Option("bluestore_bluefs_db_compatibility", Option::TYPE_BOOL, Option::LEVEL_DEV)
4103     .set_default(true)
4104     .set_description("Sync db with legacy bluefs extents info")
4105     .set_long_description("Enforces db sync with legacy bluefs extents information on close."
4106                           " Enables downgrades to pre-nautilus releases"),
4107
4108     Option("bluestore_spdk_mem", Option::TYPE_SIZE, Option::LEVEL_DEV)
4109     .set_default(512)
4110     .set_description("Amount of dpdk memory size in MB")
4111     .set_long_description("If running multiple SPDK instances per node, you must specify the amount of dpdk memory size in MB each instance will use, to make sure each instance uses its own dpdk memory"),
4112
4113     Option("bluestore_spdk_coremask", Option::TYPE_STR, Option::LEVEL_DEV)
4114     .set_default("0x1")
4115     .set_description("A hexadecimal bit mask of the cores to run on. Note the core numbering can change between platforms and should be determined beforehand"),
4116
4117     Option("bluestore_spdk_max_io_completion", Option::TYPE_UINT, Option::LEVEL_DEV)
4118     .set_default(0)
4119     .set_description("Maximal I/Os to be batched completed while checking queue pair completions, 0 means let spdk library determine it"),
4120
4121     Option("bluestore_spdk_io_sleep", Option::TYPE_UINT, Option::LEVEL_DEV)
4122     .set_default(5)
4123     .set_description("Time period to wait if there is no completed I/O from polling"),
4124
4125     Option("bluestore_block_path", Option::TYPE_STR, Option::LEVEL_DEV)
4126     .set_default("")
4127     .set_flag(Option::FLAG_CREATE)
4128     .set_description("Path to block device/file"),
4129
4130     Option("bluestore_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4131     .set_default(100_G)
4132     .set_flag(Option::FLAG_CREATE)
4133     .set_description("Size of file to create for backing bluestore"),
4134
4135     Option("bluestore_block_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
4136     .set_default(true)
4137     .set_flag(Option::FLAG_CREATE)
4138     .set_description("Create bluestore_block_path if it doesn't exist")
4139     .add_see_also("bluestore_block_path").add_see_also("bluestore_block_size"),
4140
4141     Option("bluestore_block_db_path", Option::TYPE_STR, Option::LEVEL_DEV)
4142     .set_default("")
4143     .set_flag(Option::FLAG_CREATE)
4144     .set_description("Path for db block device"),
4145
4146     Option("bluestore_block_db_size", Option::TYPE_UINT, Option::LEVEL_DEV)
4147     .set_default(0)
4148     .set_flag(Option::FLAG_CREATE)
4149     .set_description("Size of file to create for bluestore_block_db_path"),
4150
4151     Option("bluestore_block_db_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
4152     .set_default(false)
4153     .set_flag(Option::FLAG_CREATE)
4154     .set_description("Create bluestore_block_db_path if it doesn't exist")
4155     .add_see_also("bluestore_block_db_path")
4156     .add_see_also("bluestore_block_db_size"),
4157
4158     Option("bluestore_block_wal_path", Option::TYPE_STR, Option::LEVEL_DEV)
4159     .set_default("")
4160     .set_flag(Option::FLAG_CREATE)
4161     .set_description("Path to block device/file backing bluefs wal"),
4162
4163     Option("bluestore_block_wal_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4164     .set_default(96_M)
4165     .set_flag(Option::FLAG_CREATE)
4166     .set_description("Size of file to create for bluestore_block_wal_path"),
4167
4168     Option("bluestore_block_wal_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
4169     .set_default(false)
4170     .set_flag(Option::FLAG_CREATE)
4171     .set_description("Create bluestore_block_wal_path if it doesn't exist")
4172     .add_see_also("bluestore_block_wal_path")
4173     .add_see_also("bluestore_block_wal_size"),
4174
4175     Option("bluestore_block_preallocate_file", Option::TYPE_BOOL, Option::LEVEL_DEV)
4176     .set_default(false)
4177     .set_flag(Option::FLAG_CREATE)
4178     .set_description("Preallocate file created via bluestore_block*_create"),
4179
4180     Option("bluestore_ignore_data_csum", Option::TYPE_BOOL, Option::LEVEL_DEV)
4181     .set_default(false)
4182     .set_flag(Option::FLAG_RUNTIME)
4183     .set_description("Ignore checksum errors on read and do not generate an EIO error"),
4184
4185     Option("bluestore_csum_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4186     .set_default("crc32c")
4187     .set_enum_allowed({"none", "crc32c", "crc32c_16", "crc32c_8", "xxhash32", "xxhash64"})
4188     .set_flag(Option::FLAG_RUNTIME)
4189     .set_description("Default checksum algorithm to use")
4190     .set_long_description("crc32c, xxhash32, and xxhash64 are available.  The _16 and _8 variants use only a subset of the bits for more compact (but less reliable) checksumming."),
4191
4192     Option("bluestore_retry_disk_reads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4193     .set_default(3)
4194     .set_min_max(0, 255)
4195     .set_flag(Option::FLAG_RUNTIME)
4196     .set_description("Number of read retries on checksum validation error")
4197     .set_long_description("Retries to read data from the disk this many times when checksum validation fails to handle spurious read errors gracefully."),
4198
4199     Option("bluestore_min_alloc_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4200     .set_default(0)
4201     .set_flag(Option::FLAG_CREATE)
4202     .set_description("Minimum allocation size to allocate for an object")
4203     .set_long_description("A smaller allocation size generally means less data is read and then rewritten when a copy-on-write operation is triggered (e.g., when writing to something that was recently snapshotted).  Similarly, less data is journaled before performing an overwrite (writes smaller than min_alloc_size must first pass through the BlueStore journal).  Larger values of min_alloc_size reduce the amount of metadata required to describe the on-disk layout and reduce overall fragmentation."),
4204
4205     Option("bluestore_min_alloc_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4206     .set_default(64_K)
4207     .set_flag(Option::FLAG_CREATE)
4208     .set_description("Default min_alloc_size value for rotational media")
4209     .add_see_also("bluestore_min_alloc_size"),
4210
4211     Option("bluestore_min_alloc_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4212     .set_default(4_K)
4213     .set_flag(Option::FLAG_CREATE)
4214     .set_description("Default min_alloc_size value for non-rotational (solid state)  media")
4215     .add_see_also("bluestore_min_alloc_size"),
4216
4217     Option("bluestore_max_alloc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4218     .set_default(0)
4219     .set_flag(Option::FLAG_CREATE)
4220     .set_description("Maximum size of a single allocation (0 for no max)"),
4221
4222     Option("bluestore_prefer_deferred_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4223     .set_default(0)
4224     .set_flag(Option::FLAG_RUNTIME)
4225     .set_description("Writes smaller than this size will be written to the journal and then asynchronously written to the device.  This can be beneficial when using rotational media where seeks are expensive, and is helpful both with and without solid state journal/wal devices."),
4226
4227     Option("bluestore_prefer_deferred_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4228     .set_default(32768)
4229     .set_flag(Option::FLAG_RUNTIME)
4230     .set_description("Default bluestore_prefer_deferred_size for rotational media")
4231     .add_see_also("bluestore_prefer_deferred_size"),
4232
4233     Option("bluestore_prefer_deferred_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4234     .set_default(0)
4235     .set_flag(Option::FLAG_RUNTIME)
4236     .set_description("Default bluestore_prefer_deferred_size for non-rotational (solid state) media")
4237     .add_see_also("bluestore_prefer_deferred_size"),
4238
4239     Option("bluestore_compression_mode", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4240     .set_default("none")
4241     .set_enum_allowed({"none", "passive", "aggressive", "force"})
4242     .set_flag(Option::FLAG_RUNTIME)
4243     .set_description("Default policy for using compression when pool does not specify")
4244     .set_long_description("'none' means never use compression.  'passive' means use compression when clients hint that data is compressible.  'aggressive' means use compression unless clients hint that data is not compressible.  This option is used when the per-pool property for the compression mode is not present."),
4245
4246     Option("bluestore_compression_algorithm", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4247     .set_default("snappy")
4248     .set_enum_allowed({"", "snappy", "zlib", "zstd", "lz4"})
4249     .set_flag(Option::FLAG_RUNTIME)
4250     .set_description("Default compression algorithm to use when writing object data")
4251     .set_long_description("This controls the default compressor to use (if any) if the per-pool property is not set.  Note that zstd is *not* recommended for bluestore due to high CPU overhead when compressing small amounts of data."),
4252
4253     Option("bluestore_compression_min_blob_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4254     .set_default(0)
4255     .set_flag(Option::FLAG_RUNTIME)
4256     .set_description("Maximum chunk size to apply compression to when random access is expected for an object.")
4257     .set_long_description("Chunks larger than this are broken into smaller chunks before being compressed"),
4258
4259     Option("bluestore_compression_min_blob_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4260     .set_default(128_K)
4261     .set_flag(Option::FLAG_RUNTIME)
4262     .set_description("Default value of bluestore_compression_min_blob_size for rotational media")
4263     .add_see_also("bluestore_compression_min_blob_size"),
4264
4265     Option("bluestore_compression_min_blob_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4266     .set_default(8_K)
4267     .set_flag(Option::FLAG_RUNTIME)
4268     .set_description("Default value of bluestore_compression_min_blob_size for non-rotational (solid state) media")
4269     .add_see_also("bluestore_compression_min_blob_size"),
4270
4271     Option("bluestore_compression_max_blob_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4272     .set_default(0)
4273     .set_flag(Option::FLAG_RUNTIME)
4274     .set_description("Maximum chunk size to apply compression to when non-random access is expected for an object.")
4275     .set_long_description("Chunks larger than this are broken into smaller chunks before being compressed"),
4276
4277     Option("bluestore_compression_max_blob_size_hdd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4278     .set_default(512_K)
4279     .set_flag(Option::FLAG_RUNTIME)
4280     .set_description("Default value of bluestore_compression_max_blob_size for rotational media")
4281     .add_see_also("bluestore_compression_max_blob_size"),
4282
4283     Option("bluestore_compression_max_blob_size_ssd", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4284     .set_default(64_K)
4285     .set_flag(Option::FLAG_RUNTIME)
4286     .set_description("Default value of bluestore_compression_max_blob_size for non-rotational (solid state) media")
4287     .add_see_also("bluestore_compression_max_blob_size"),
4288
4289     Option("bluestore_gc_enable_blob_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
4290     .set_default(0)
4291     .set_flag(Option::FLAG_RUNTIME)
4292     .set_description(""),
4293
4294     Option("bluestore_gc_enable_total_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
4295     .set_default(0)
4296     .set_flag(Option::FLAG_RUNTIME)
4297     .set_description(""),
4298
4299     Option("bluestore_max_blob_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4300     .set_default(0)
4301     .set_flag(Option::FLAG_RUNTIME)
4302     .set_description("")
4303     .set_long_description("Bluestore blobs are collections of extents (ie on-disk data) originating from one or more objects.  Blobs can be compressed, typically have checksum data, may be overwritten, may be shared (with an extent ref map), or split.  This setting controls the maximum size a blob is allowed to be."),
4304
4305     Option("bluestore_max_blob_size_hdd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4306     .set_default(512_K)
4307     .set_flag(Option::FLAG_RUNTIME)
4308     .set_description("")
4309     .add_see_also("bluestore_max_blob_size"),
4310
4311     Option("bluestore_max_blob_size_ssd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4312     .set_default(64_K)
4313     .set_flag(Option::FLAG_RUNTIME)
4314     .set_description("")
4315     .add_see_also("bluestore_max_blob_size"),
4316
4317     Option("bluestore_compression_required_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4318     .set_default(.875)
4319     .set_flag(Option::FLAG_RUNTIME)
4320     .set_description("Compression ratio required to store compressed data")
4321     .set_long_description("If we compress data and get less than this we discard the result and store the original uncompressed data."),
4322
4323     Option("bluestore_extent_map_shard_max_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4324     .set_default(1200)
4325     .set_description("Max size (bytes) for a single extent map shard before splitting"),
4326
4327     Option("bluestore_extent_map_shard_target_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4328     .set_default(500)
4329     .set_description("Target size (bytes) for a single extent map shard"),
4330
4331     Option("bluestore_extent_map_shard_min_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4332     .set_default(150)
4333     .set_description("Min size (bytes) for a single extent map shard before merging"),
4334
4335     Option("bluestore_extent_map_shard_target_size_slop", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4336     .set_default(.2)
4337     .set_description("Ratio above/below target for a shard when trying to align to an existing extent or blob boundary"),
4338
4339     Option("bluestore_extent_map_inline_shard_prealloc_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4340     .set_default(256)
4341     .set_description("Preallocated buffer for inline shards"),
4342
4343     Option("bluestore_cache_trim_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4344     .set_default(.05)
4345     .set_description("How frequently we trim the bluestore cache"),
4346
4347     Option("bluestore_cache_trim_max_skip_pinned", Option::TYPE_UINT, Option::LEVEL_DEV)
4348     .set_default(64)
4349     .set_description("Max pinned cache entries we consider before giving up"),
4350
4351     Option("bluestore_cache_type", Option::TYPE_STR, Option::LEVEL_DEV)
4352     .set_default("2q")
4353     .set_enum_allowed({"2q", "lru"})
4354     .set_description("Cache replacement algorithm"),
4355
4356     Option("bluestore_2q_cache_kin_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4357     .set_default(.5)
4358     .set_description("2Q paper suggests .5"),
4359
4360     Option("bluestore_2q_cache_kout_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4361     .set_default(.5)
4362     .set_description("2Q paper suggests .5"),
4363
4364     Option("bluestore_cache_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4365     .set_default(0)
4366     .set_description("Cache size (in bytes) for BlueStore")
4367     .set_long_description("This includes data and metadata cached by BlueStore as well as memory devoted to rocksdb's cache(s)."),
4368
4369     Option("bluestore_cache_size_hdd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4370     .set_default(1_G)
4371     .set_description("Default bluestore_cache_size for rotational media")
4372     .add_see_also("bluestore_cache_size"),
4373
4374     Option("bluestore_cache_size_ssd", Option::TYPE_SIZE, Option::LEVEL_DEV)
4375     .set_default(3_G)
4376     .set_description("Default bluestore_cache_size for non-rotational (solid state) media")
4377     .add_see_also("bluestore_cache_size"),
4378
4379     Option("bluestore_cache_meta_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4380     .set_default(.4)
4381     .add_see_also("bluestore_cache_size")
4382     .set_description("Ratio of bluestore cache to devote to metadata"),
4383
4384     Option("bluestore_cache_kv_ratio", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4385     .set_default(.4)
4386     .add_see_also("bluestore_cache_size")
4387     .set_description("Ratio of bluestore cache to devote to kv database (rocksdb)"),
4388
4389     Option("bluestore_cache_autotune", Option::TYPE_BOOL, Option::LEVEL_DEV)
4390     .set_default(true)
4391     .add_see_also("bluestore_cache_size")
4392     .add_see_also("bluestore_cache_meta_ratio")
4393     .set_description("Automatically tune the ratio of caches while respecting min values."),
4394
4395     Option("bluestore_cache_autotune_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4396     .set_default(5)
4397     .add_see_also("bluestore_cache_autotune")
4398     .set_description("The number of seconds to wait between rebalances when cache autotune is enabled."),
4399
4400     Option("bluestore_alloc_stats_dump_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4401       .set_default(3600 * 24)
4402       .set_description("The period (in second) for logging allocation statistics."),
4403
4404     Option("bluestore_kvbackend", Option::TYPE_STR, Option::LEVEL_DEV)
4405     .set_default("rocksdb")
4406     .set_flag(Option::FLAG_CREATE)
4407     .set_description("Key value database to use for bluestore"),
4408
4409     Option("bluestore_allocator", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4410     .set_default("hybrid")
4411     .set_enum_allowed({"bitmap", "stupid", "avl", "hybrid"})
4412     .set_description("Allocator policy")
4413     .set_long_description("Allocator to use for bluestore.  Stupid should only be used for testing."),
4414
4415     Option("bluestore_freelist_blocks_per_key", Option::TYPE_SIZE, Option::LEVEL_DEV)
4416     .set_default(128)
4417     .set_description("Block (and bits) per database key"),
4418
4419     Option("bluestore_bitmapallocator_blocks_per_zone", Option::TYPE_SIZE, Option::LEVEL_DEV)
4420     .set_default(1024)
4421     .set_description(""),
4422
4423     Option("bluestore_bitmapallocator_span_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4424     .set_default(1024)
4425     .set_description(""),
4426
4427     Option("bluestore_max_deferred_txc", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4428     .set_default(32)
4429     .set_description("Max transactions with deferred writes that can accumulate before we force flush deferred writes"),
4430
4431     Option("bluestore_max_defer_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4432     .set_default(3)
4433     .set_description("max duration to force deferred submit"),
4434
4435     Option("bluestore_rocksdb_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4436     .set_default("compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2")
4437     .set_description("Rocksdb options"),
4438
4439     Option("bluestore_rocksdb_cf", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4440     .set_default(false)
4441     .set_description("Enable use of rocksdb column families for bluestore metadata"),
4442
4443     Option("bluestore_rocksdb_cfs", Option::TYPE_STR, Option::LEVEL_DEV)
4444     .set_default("M= P= L=")
4445     .set_description("List of whitespace-separate key/value pairs where key is CF name and value is CF options"),
4446
4447     Option("bluestore_fsck_on_mount", Option::TYPE_BOOL, Option::LEVEL_DEV)
4448     .set_default(false)
4449     .set_description("Run fsck at mount"),
4450
4451     Option("bluestore_fsck_on_mount_deep", Option::TYPE_BOOL, Option::LEVEL_DEV)
4452     .set_default(false)
4453     .set_description("Run deep fsck at mount when bluestore_fsck_on_mount is set to true"),
4454
4455     Option("bluestore_fsck_quick_fix_on_mount", Option::TYPE_BOOL, Option::LEVEL_DEV)
4456       .set_default(true)
4457       .set_description("Do quick-fix for the store at mount"),
4458
4459     Option("bluestore_fsck_on_umount", Option::TYPE_BOOL, Option::LEVEL_DEV)
4460     .set_default(false)
4461     .set_description("Run fsck at umount"),
4462
4463     Option("bluestore_fsck_on_umount_deep", Option::TYPE_BOOL, Option::LEVEL_DEV)
4464     .set_default(false)
4465     .set_description("Run deep fsck at umount when bluestore_fsck_on_umount is set to true"),
4466
4467     Option("bluestore_fsck_on_mkfs", Option::TYPE_BOOL, Option::LEVEL_DEV)
4468     .set_default(true)
4469     .set_description("Run fsck after mkfs"),
4470
4471     Option("bluestore_fsck_on_mkfs_deep", Option::TYPE_BOOL, Option::LEVEL_DEV)
4472     .set_default(false)
4473     .set_description("Run deep fsck after mkfs"),
4474
4475     Option("bluestore_sync_submit_transaction", Option::TYPE_BOOL, Option::LEVEL_DEV)
4476     .set_default(false)
4477     .set_description("Try to submit metadata transaction to rocksdb in queuing thread context"),
4478
4479     Option("bluestore_fsck_read_bytes_cap", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4480     .set_default(64_M)
4481     .set_flag(Option::FLAG_RUNTIME)
4482     .set_description("Maximum bytes read at once by deep fsck"),
4483
4484     Option("bluestore_fsck_quick_fix_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4485       .set_default(2)
4486       .set_description("Number of additional threads to perform quick-fix (shallow fsck) command"),
4487
4488     Option("bluestore_throttle_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4489     .set_default(64_M)
4490     .set_flag(Option::FLAG_RUNTIME)
4491     .set_description("Maximum bytes in flight before we throttle IO submission"),
4492
4493     Option("bluestore_throttle_deferred_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4494     .set_default(128_M)
4495     .set_flag(Option::FLAG_RUNTIME)
4496     .set_description("Maximum bytes for deferred writes before we throttle IO submission"),
4497
4498     Option("bluestore_throttle_cost_per_io", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4499     .set_default(0)
4500     .set_flag(Option::FLAG_RUNTIME)
4501     .set_description("Overhead added to transaction cost (in bytes) for each IO"),
4502
4503   Option("bluestore_throttle_cost_per_io_hdd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4504     .set_default(670000)
4505     .set_flag(Option::FLAG_RUNTIME)
4506     .set_description("Default bluestore_throttle_cost_per_io for rotational media")
4507     .add_see_also("bluestore_throttle_cost_per_io"),
4508
4509     Option("bluestore_throttle_cost_per_io_ssd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4510     .set_default(4000)
4511     .set_flag(Option::FLAG_RUNTIME)
4512     .set_description("Default bluestore_throttle_cost_per_io for non-rotation (solid state) media")
4513     .add_see_also("bluestore_throttle_cost_per_io"),
4514
4515     Option("bluestore_deferred_batch_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4516     .set_default(0)
4517     .set_flag(Option::FLAG_RUNTIME)
4518     .set_description("Max number of deferred writes before we flush the deferred write queue"),
4519
4520     Option("bluestore_deferred_batch_ops_hdd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4521     .set_default(64)
4522     .set_flag(Option::FLAG_RUNTIME)
4523     .set_description("Default bluestore_deferred_batch_ops for rotational media")
4524     .add_see_also("bluestore_deferred_batch_ops"),
4525
4526     Option("bluestore_deferred_batch_ops_ssd", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4527     .set_default(16)
4528     .set_flag(Option::FLAG_RUNTIME)
4529     .set_description("Default bluestore_deferred_batch_ops for non-rotational (solid state) media")
4530     .add_see_also("bluestore_deferred_batch_ops"),
4531
4532     Option("bluestore_nid_prealloc", Option::TYPE_INT, Option::LEVEL_DEV)
4533     .set_default(1024)
4534     .set_description("Number of unique object ids to preallocate at a time"),
4535
4536     Option("bluestore_blobid_prealloc", Option::TYPE_UINT, Option::LEVEL_DEV)
4537     .set_default(10240)
4538     .set_description("Number of unique blob ids to preallocate at a time"),
4539
4540     Option("bluestore_clone_cow", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4541     .set_default(true)
4542     .set_flag(Option::FLAG_RUNTIME)
4543     .set_description("Use copy-on-write when cloning objects (versus reading and rewriting them at clone time)"),
4544
4545     Option("bluestore_default_buffered_read", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4546     .set_default(true)
4547     .set_flag(Option::FLAG_RUNTIME)
4548     .set_description("Cache read results by default (unless hinted NOCACHE or WONTNEED)"),
4549
4550     Option("bluestore_default_buffered_write", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4551     .set_default(false)
4552     .set_flag(Option::FLAG_RUNTIME)
4553     .set_description("Cache writes by default (unless hinted NOCACHE or WONTNEED)"),
4554
4555     Option("bluestore_debug_misc", Option::TYPE_BOOL, Option::LEVEL_DEV)
4556     .set_default(false)
4557     .set_description(""),
4558
4559     Option("bluestore_debug_no_reuse_blocks", Option::TYPE_BOOL, Option::LEVEL_DEV)
4560     .set_default(false)
4561     .set_description(""),
4562
4563     Option("bluestore_debug_small_allocations", Option::TYPE_INT, Option::LEVEL_DEV)
4564     .set_default(0)
4565     .set_description(""),
4566
4567     Option("bluestore_debug_too_many_blobs_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
4568     .set_default(24*1024)
4569     .set_description(""),
4570
4571     Option("bluestore_debug_freelist", Option::TYPE_BOOL, Option::LEVEL_DEV)
4572     .set_default(false)
4573     .set_description(""),
4574
4575     Option("bluestore_debug_prefill", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4576     .set_default(0)
4577     .set_description("simulate fragmentation"),
4578
4579     Option("bluestore_debug_prefragment_max", Option::TYPE_SIZE, Option::LEVEL_DEV)
4580     .set_default(1_M)
4581     .set_description(""),
4582
4583     Option("bluestore_debug_inject_read_err", Option::TYPE_BOOL, Option::LEVEL_DEV)
4584     .set_default(false)
4585     .set_description(""),
4586
4587     Option("bluestore_debug_randomize_serial_transaction", Option::TYPE_INT, Option::LEVEL_DEV)
4588     .set_default(0)
4589     .set_description(""),
4590
4591     Option("bluestore_debug_omit_block_device_write", Option::TYPE_BOOL, Option::LEVEL_DEV)
4592     .set_default(false)
4593     .set_description(""),
4594
4595     Option("bluestore_debug_fsck_abort", Option::TYPE_BOOL, Option::LEVEL_DEV)
4596     .set_default(false)
4597     .set_description(""),
4598
4599     Option("bluestore_debug_omit_kv_commit", Option::TYPE_BOOL, Option::LEVEL_DEV)
4600     .set_default(false)
4601     .set_description(""),
4602
4603     Option("bluestore_debug_permit_any_bdev_label", Option::TYPE_BOOL, Option::LEVEL_DEV)
4604     .set_default(false)
4605     .set_description(""),
4606
4607     Option("bluestore_debug_random_read_err", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4608     .set_default(0)
4609     .set_description(""),
4610
4611     Option("bluestore_debug_inject_bug21040", Option::TYPE_BOOL, Option::LEVEL_DEV)
4612     .set_default(false)
4613     .set_description(""),
4614
4615     Option("bluestore_debug_inject_csum_err_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4616     .set_default(0.0)
4617     .set_description("inject crc verification errors into bluestore device reads"),
4618
4619     Option("bluestore_fsck_error_on_no_per_pool_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4620     .set_default(false)
4621     .set_description("Make fsck error (instead of warn) when bluestore lacks per-pool stats, e.g., after an upgrade"),
4622
4623     Option("bluestore_warn_on_bluefs_spillover", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4624     .set_default(true)
4625     .set_description("Enable health indication on bluefs slow device usage"),
4626
4627     Option("bluestore_warn_on_legacy_statfs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4628     .set_default(true)
4629     .set_description("Enable health indication on lack of per-pool statfs reporting from bluestore"),
4630
4631     Option("bluestore_fsck_error_on_no_per_pool_omap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4632     .set_default(false)
4633     .set_description("Make fsck error (instead of warn) when objects without per-pool omap are found"),
4634
4635     Option("bluestore_warn_on_no_per_pool_omap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4636     .set_default(true)
4637     .set_description("Enable health indication on lack of per-pool omap"),
4638
4639     Option("bluestore_log_op_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4640     .set_default(5)
4641     .set_description("log operation if it's slower than this age (seconds)"),
4642
4643     Option("bluestore_log_omap_iterator_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4644     .set_default(5)
4645     .set_description("log omap iteration operation if it's slower than this age (seconds)"),
4646
4647     Option("bluestore_log_collection_list_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4648     .set_default(60)
4649     .set_description("log collection list operation if it's slower than this age (seconds)"),
4650
4651     Option("bluestore_debug_enforce_settings", Option::TYPE_STR, Option::LEVEL_DEV)
4652     .set_default("default")
4653     .set_enum_allowed({"default", "hdd", "ssd"})
4654     .set_description("Enforces specific hw profile settings")
4655     .set_long_description("'hdd' enforces settings intended for BlueStore above a rotational drive. 'ssd' enforces settings intended for BlueStore above a solid drive. 'default' - using settings for the actual hardware."),
4656
4657     Option("bluestore_avl_alloc_bf_threshold", Option::TYPE_UINT, Option::LEVEL_DEV)
4658     .set_default(131072)
4659     .set_description(""),
4660
4661     Option("bluestore_avl_alloc_bf_free_pct", Option::TYPE_UINT, Option::LEVEL_DEV)
4662     .set_default(4)
4663     .set_description(""),
4664
4665     Option("bluestore_hybrid_alloc_mem_cap", Option::TYPE_UINT, Option::LEVEL_DEV)
4666     .set_default(64_M)
4667     .set_description("Maximum RAM hybrid allocator should use before enabling bitmap supplement"),
4668
4669     Option("bluestore_volume_selection_policy", Option::TYPE_STR, Option::LEVEL_DEV)
4670     .set_default("use_some_extra")
4671     .set_enum_allowed({ "rocksdb_original", "use_some_extra" })
4672     .set_description("Determines bluefs volume selection policy")
4673     .set_long_description("Determines bluefs volume selection policy. 'use_some_extra' policy allows to override RocksDB level granularity and put high level's data to faster device even when the level doesn't completely fit there"),
4674
4675     Option("bluestore_volume_selection_reserved_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4676       .set_flag(Option::FLAG_STARTUP)
4677       .set_default(2.0)
4678       .set_description("DB level size multiplier. Determines amount of space at DB device to bar from the usage when 'use some extra' policy is in action. Reserved size is determined as sum(L_max_size[0], L_max_size[L-1]) + L_max_size[L] * this_factor"),
4679
4680     Option("bluestore_volume_selection_reserved", Option::TYPE_INT, Option::LEVEL_ADVANCED)
4681       .set_flag(Option::FLAG_STARTUP)
4682       .set_default(0)
4683       .set_description("Space reserved at DB device and not allowed for 'use some extra' policy usage. Overrides 'bluestore_volume_selection_reserved_factor' setting and introduces straightforward limit."),
4684
4685     Option("bluestore_ioring", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4686     .set_default(false)
4687     .set_description("Enables Linux io_uring API instead of libaio"),
4688
4689     Option("bdev_ioring_hipri", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4690     .set_default(false)
4691     .set_description("Enables Linux io_uring API Use polled IO completions"),
4692
4693     Option("bdev_ioring_sqthread_poll", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4694     .set_default(false)
4695     .set_description("Enables Linux io_uring API Offload submission/completion to kernel thread"),
4696
4697     Option("bluestore_kv_sync_util_logging_s", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4698     .set_default(10.0)
4699     .set_flag(Option::FLAG_RUNTIME)
4700     .set_description("KV sync thread utilization logging period")
4701     .set_long_description("How often (in seconds) to print KV sync thread utilization, "
4702       "not logged when set to 0 or when utilization is 0%"),
4703
4704
4705     // -----------------------------------------
4706     // kstore
4707
4708     Option("kstore_max_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4709     .set_default(512)
4710     .set_description(""),
4711
4712     Option("kstore_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4713     .set_default(64_M)
4714     .set_description(""),
4715
4716     Option("kstore_backend", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4717     .set_default("rocksdb")
4718     .set_description(""),
4719
4720     Option("kstore_rocksdb_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
4721     .set_default("compression=kNoCompression")
4722     .set_description("Options to pass through when RocksDB is used as the KeyValueDB for kstore."),
4723
4724     Option("kstore_fsck_on_mount", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4725     .set_default(false)
4726     .set_description("Whether or not to run fsck on mount for kstore."),
4727
4728     Option("kstore_fsck_on_mount_deep", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4729     .set_default(true)
4730     .set_description("Whether or not to run deep fsck on mount for kstore"),
4731
4732     Option("kstore_nid_prealloc", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4733     .set_default(1024)
4734     .set_description(""),
4735
4736     Option("kstore_sync_transaction", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4737     .set_default(false)
4738     .set_description(""),
4739
4740     Option("kstore_sync_submit_transaction", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4741     .set_default(false)
4742     .set_description(""),
4743
4744     Option("kstore_onode_map_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4745     .set_default(1024)
4746     .set_description(""),
4747
4748     Option("kstore_default_stripe_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4749     .set_default(65536)
4750     .set_description(""),
4751
4752     // ---------------------
4753     // filestore
4754
4755     Option("filestore_rocksdb_options", Option::TYPE_STR, Option::LEVEL_DEV)
4756     .set_default("max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression")
4757     .set_description("Options to pass through when RocksDB is used as the KeyValueDB for filestore."),
4758
4759     Option("filestore_omap_backend", Option::TYPE_STR, Option::LEVEL_DEV)
4760     .set_default("rocksdb")
4761     .set_enum_allowed({"leveldb", "rocksdb"})
4762     .set_description("The KeyValueDB to use for filestore metadata (ie omap)."),
4763
4764     Option("filestore_omap_backend_path", Option::TYPE_STR, Option::LEVEL_DEV)
4765     .set_default("")
4766     .set_description("The path where the filestore KeyValueDB should store it's database(s)."),
4767
4768     Option("filestore_wbthrottle_enable", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4769     .set_default(true)
4770     .set_description("Enabling throttling of operations to backing file system"),
4771
4772     Option("filestore_wbthrottle_btrfs_bytes_start_flusher", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4773     .set_default(41943040)
4774     .set_description("Start flushing (fsyncing) when this many bytes are written(btrfs)"),
4775
4776     Option("filestore_wbthrottle_btrfs_bytes_hard_limit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4777     .set_default(419430400)
4778     .set_description("Block writes when this many bytes haven't been flushed (fsynced) (btrfs)"),
4779
4780     Option("filestore_wbthrottle_btrfs_ios_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4781     .set_default(500)
4782     .set_description("Start flushing (fsyncing) when this many IOs are written (brtrfs)"),
4783
4784     Option("filestore_wbthrottle_btrfs_ios_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4785     .set_default(5000)
4786     .set_description("Block writes when this many IOs haven't been flushed (fsynced) (btrfs)"),
4787
4788     Option("filestore_wbthrottle_btrfs_inodes_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4789     .set_default(500)
4790     .set_description("Start flushing (fsyncing) when this many distinct inodes have been modified (btrfs)"),
4791
4792     Option("filestore_wbthrottle_xfs_bytes_start_flusher", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4793     .set_default(41943040)
4794     .set_description("Start flushing (fsyncing) when this many bytes are written(xfs)"),
4795
4796     Option("filestore_wbthrottle_xfs_bytes_hard_limit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4797     .set_default(419430400)
4798     .set_description("Block writes when this many bytes haven't been flushed (fsynced) (xfs)"),
4799
4800     Option("filestore_wbthrottle_xfs_ios_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4801     .set_default(500)
4802     .set_description("Start flushing (fsyncing) when this many IOs are written (xfs)"),
4803
4804     Option("filestore_wbthrottle_xfs_ios_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4805     .set_default(5000)
4806     .set_description("Block writes when this many IOs haven't been flushed (fsynced) (xfs)"),
4807
4808     Option("filestore_wbthrottle_xfs_inodes_start_flusher", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4809     .set_default(500)
4810     .set_description("Start flushing (fsyncing) when this many distinct inodes have been modified (xfs)"),
4811
4812     Option("filestore_wbthrottle_btrfs_inodes_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4813     .set_default(5000)
4814     .set_description("Block writing when this many inodes have outstanding writes (btrfs)"),
4815
4816     Option("filestore_wbthrottle_xfs_inodes_hard_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4817     .set_default(5000)
4818     .set_description("Block writing when this many inodes have outstanding writes (xfs)"),
4819
4820     Option("filestore_odsync_write", Option::TYPE_BOOL, Option::LEVEL_DEV)
4821     .set_default(false)
4822     .set_description("Write with O_DSYNC"),
4823
4824     Option("filestore_index_retry_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4825     .set_default(0)
4826     .set_description(""),
4827
4828     Option("filestore_debug_inject_read_err", Option::TYPE_BOOL, Option::LEVEL_DEV)
4829     .set_default(false)
4830     .set_description(""),
4831
4832     Option("filestore_debug_random_read_err", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4833     .set_default(0)
4834     .set_description(""),
4835
4836     Option("filestore_debug_omap_check", Option::TYPE_BOOL, Option::LEVEL_DEV)
4837     .set_default(false)
4838     .set_description(""),
4839
4840     Option("filestore_omap_header_cache_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4841     .set_default(1024)
4842     .set_description(""),
4843
4844     Option("filestore_max_inline_xattr_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4845     .set_default(0)
4846     .set_description(""),
4847
4848     Option("filestore_max_inline_xattr_size_xfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
4849     .set_default(65536)
4850     .set_description(""),
4851
4852     Option("filestore_max_inline_xattr_size_btrfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
4853     .set_default(2048)
4854     .set_description(""),
4855
4856     Option("filestore_max_inline_xattr_size_other", Option::TYPE_SIZE, Option::LEVEL_DEV)
4857     .set_default(512)
4858     .set_description(""),
4859
4860     Option("filestore_max_inline_xattrs", Option::TYPE_UINT, Option::LEVEL_DEV)
4861     .set_default(0)
4862     .set_description(""),
4863
4864     Option("filestore_max_inline_xattrs_xfs", Option::TYPE_UINT, Option::LEVEL_DEV)
4865     .set_default(10)
4866     .set_description(""),
4867
4868     Option("filestore_max_inline_xattrs_btrfs", Option::TYPE_UINT, Option::LEVEL_DEV)
4869     .set_default(10)
4870     .set_description(""),
4871
4872     Option("filestore_max_inline_xattrs_other", Option::TYPE_UINT, Option::LEVEL_DEV)
4873     .set_default(2)
4874     .set_description(""),
4875
4876     Option("filestore_max_xattr_value_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4877     .set_default(0)
4878     .set_description(""),
4879
4880     Option("filestore_max_xattr_value_size_xfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
4881     .set_default(64_K)
4882     .set_description(""),
4883
4884     Option("filestore_max_xattr_value_size_btrfs", Option::TYPE_SIZE, Option::LEVEL_DEV)
4885     .set_default(64_K)
4886     .set_description(""),
4887
4888     Option("filestore_max_xattr_value_size_other", Option::TYPE_SIZE, Option::LEVEL_DEV)
4889     .set_default(1_K)
4890     .set_description(""),
4891
4892     Option("filestore_sloppy_crc", Option::TYPE_BOOL, Option::LEVEL_DEV)
4893     .set_default(false)
4894     .set_description(""),
4895
4896     Option("filestore_sloppy_crc_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4897     .set_default(65536)
4898     .set_description(""),
4899
4900     Option("filestore_max_alloc_hint_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
4901     .set_default(1ULL << 20)
4902     .set_description(""),
4903
4904     Option("filestore_max_sync_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4905     .set_default(5)
4906     .set_description("Period between calls to syncfs(2) and journal trims (seconds)"),
4907
4908     Option("filestore_min_sync_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4909     .set_default(.01)
4910     .set_description("Minimum period between calls to syncfs(2)"),
4911
4912     Option("filestore_btrfs_snap", Option::TYPE_BOOL, Option::LEVEL_DEV)
4913     .set_default(true)
4914     .set_description(""),
4915
4916     Option("filestore_btrfs_clone_range", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4917     .set_default(true)
4918     .set_description("Use btrfs clone_range ioctl to efficiently duplicate objects"),
4919
4920     Option("filestore_zfs_snap", Option::TYPE_BOOL, Option::LEVEL_DEV)
4921     .set_default(false)
4922     .set_description(""),
4923
4924     Option("filestore_fsync_flushes_journal_data", Option::TYPE_BOOL, Option::LEVEL_DEV)
4925     .set_default(false)
4926     .set_description(""),
4927
4928     Option("filestore_fiemap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4929     .set_default(false)
4930     .set_description("Use fiemap ioctl(2) to determine which parts of objects are sparse"),
4931
4932     Option("filestore_punch_hole", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4933     .set_default(false)
4934     .set_description("Use fallocate(2) FALLOC_FL_PUNCH_HOLE to efficiently zero ranges of objects"),
4935
4936     Option("filestore_seek_data_hole", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4937     .set_default(false)
4938     .set_description("Use lseek(2) SEEK_HOLE and SEEK_DATA to determine which parts of objects are sparse"),
4939
4940     Option("filestore_splice", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4941     .set_default(false)
4942     .set_description("Use splice(2) to more efficiently copy data between files"),
4943
4944     Option("filestore_fadvise", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4945     .set_default(true)
4946     .set_description("Use posix_fadvise(2) to pass hints to file system"),
4947
4948     Option("filestore_collect_device_partition_information", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4949     .set_default(true)
4950     .set_description("Collect metadata about the backing file system on OSD startup"),
4951
4952     Option("filestore_xfs_extsize", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
4953     .set_default(false)
4954     .set_description("Use XFS extsize ioctl(2) to hint allocator about expected write sizes"),
4955
4956     Option("filestore_journal_parallel", Option::TYPE_BOOL, Option::LEVEL_DEV)
4957     .set_default(false)
4958     .set_description(""),
4959
4960     Option("filestore_journal_writeahead", Option::TYPE_BOOL, Option::LEVEL_DEV)
4961     .set_default(false)
4962     .set_description(""),
4963
4964     Option("filestore_journal_trailing", Option::TYPE_BOOL, Option::LEVEL_DEV)
4965     .set_default(false)
4966     .set_description(""),
4967
4968     Option("filestore_queue_max_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
4969     .set_default(50)
4970     .set_description("Max IO operations in flight"),
4971
4972     Option("filestore_queue_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
4973     .set_default(100_M)
4974     .set_description("Max (written) bytes in flight"),
4975
4976     Option("filestore_caller_concurrency", Option::TYPE_INT, Option::LEVEL_DEV)
4977     .set_default(10)
4978     .set_description(""),
4979
4980     Option("filestore_expected_throughput_bytes", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4981     .set_default(200_M)
4982     .set_description("Expected throughput of backend device (aids throttling calculations)"),
4983
4984     Option("filestore_expected_throughput_ops", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
4985     .set_default(200)
4986     .set_description("Expected through of backend device in IOPS (aids throttling calculations)"),
4987
4988     Option("filestore_queue_max_delay_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4989     .set_default(0)
4990     .set_description(""),
4991
4992     Option("filestore_queue_high_delay_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4993     .set_default(0)
4994     .set_description(""),
4995
4996     Option("filestore_queue_max_delay_multiple_bytes", Option::TYPE_FLOAT, Option::LEVEL_DEV)
4997     .set_default(0)
4998     .set_description(""),
4999
5000     Option("filestore_queue_high_delay_multiple_bytes", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5001     .set_default(0)
5002     .set_description(""),
5003
5004     Option("filestore_queue_max_delay_multiple_ops", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5005     .set_default(0)
5006     .set_description(""),
5007
5008     Option("filestore_queue_high_delay_multiple_ops", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5009     .set_default(0)
5010     .set_description(""),
5011
5012     Option("filestore_queue_low_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5013     .set_default(0.3)
5014     .set_description(""),
5015
5016     Option("filestore_queue_high_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5017     .set_default(0.9)
5018     .set_description(""),
5019
5020     Option("filestore_op_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5021     .set_default(2)
5022     .set_description("Threads used to apply changes to backing file system"),
5023
5024     Option("filestore_op_thread_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5025     .set_default(60)
5026     .set_description("Seconds before a worker thread is considered stalled"),
5027
5028     Option("filestore_op_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5029     .set_default(180)
5030     .set_description("Seconds before a worker thread is considered dead"),
5031
5032     Option("filestore_commit_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5033     .set_default(600)
5034     .set_description("Seconds before backing file system is considered hung"),
5035
5036     Option("filestore_fiemap_threshold", Option::TYPE_SIZE, Option::LEVEL_DEV)
5037     .set_default(4_K)
5038     .set_description(""),
5039
5040     Option("filestore_merge_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
5041     .set_default(-10)
5042     .set_description(""),
5043
5044     Option("filestore_split_multiple", Option::TYPE_INT, Option::LEVEL_DEV)
5045     .set_default(2)
5046     .set_description(""),
5047
5048     Option("filestore_split_rand_factor", Option::TYPE_UINT, Option::LEVEL_DEV)
5049     .set_default(20)
5050     .set_description(""),
5051
5052     Option("filestore_update_to", Option::TYPE_INT, Option::LEVEL_DEV)
5053     .set_default(1000)
5054     .set_description(""),
5055
5056     Option("filestore_blackhole", Option::TYPE_BOOL, Option::LEVEL_DEV)
5057     .set_default(false)
5058     .set_description(""),
5059
5060     Option("filestore_fd_cache_size", Option::TYPE_INT, Option::LEVEL_DEV)
5061     .set_default(128)
5062     .set_description(""),
5063
5064     Option("filestore_fd_cache_shards", Option::TYPE_INT, Option::LEVEL_DEV)
5065     .set_default(16)
5066     .set_description(""),
5067
5068     Option("filestore_ondisk_finisher_threads", Option::TYPE_INT, Option::LEVEL_DEV)
5069     .set_default(1)
5070     .set_description(""),
5071
5072     Option("filestore_apply_finisher_threads", Option::TYPE_INT, Option::LEVEL_DEV)
5073     .set_default(1)
5074     .set_description(""),
5075
5076     Option("filestore_dump_file", Option::TYPE_STR, Option::LEVEL_DEV)
5077     .set_default("")
5078     .set_description(""),
5079
5080     Option("filestore_kill_at", Option::TYPE_INT, Option::LEVEL_DEV)
5081     .set_default(0)
5082     .set_description(""),
5083
5084     Option("filestore_inject_stall", Option::TYPE_INT, Option::LEVEL_DEV)
5085     .set_default(0)
5086     .set_description(""),
5087
5088     Option("filestore_fail_eio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5089     .set_default(true)
5090     .set_description(""),
5091
5092     Option("filestore_debug_verify_split", Option::TYPE_BOOL, Option::LEVEL_DEV)
5093     .set_default(false)
5094     .set_description(""),
5095
5096     Option("journal_dio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5097     .set_default(true)
5098     .set_description(""),
5099
5100     Option("journal_aio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5101     .set_default(true)
5102     .set_description(""),
5103
5104     Option("journal_force_aio", Option::TYPE_BOOL, Option::LEVEL_DEV)
5105     .set_default(false)
5106     .set_description(""),
5107
5108     Option("journal_block_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5109     .set_default(4_K)
5110     .set_description(""),
5111
5112     Option("journal_block_align", Option::TYPE_BOOL, Option::LEVEL_DEV)
5113     .set_default(true)
5114     .set_description(""),
5115
5116     Option("journal_write_header_frequency", Option::TYPE_UINT, Option::LEVEL_DEV)
5117     .set_default(0)
5118     .set_description(""),
5119
5120     Option("journal_max_write_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5121     .set_default(10_M)
5122     .set_description("Max bytes in flight to journal"),
5123
5124     Option("journal_max_write_entries", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5125     .set_default(100)
5126     .set_description("Max IOs in flight to journal"),
5127
5128     Option("journal_throttle_low_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5129     .set_default(0.6)
5130     .set_description(""),
5131
5132     Option("journal_throttle_high_threshhold", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5133     .set_default(0.9)
5134     .set_description(""),
5135
5136     Option("journal_throttle_high_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5137     .set_default(0)
5138     .set_description(""),
5139
5140     Option("journal_throttle_max_multiple", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5141     .set_default(0)
5142     .set_description(""),
5143
5144     Option("journal_align_min_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
5145     .set_default(64_K)
5146     .set_description(""),
5147
5148     Option("journal_replay_from", Option::TYPE_INT, Option::LEVEL_DEV)
5149     .set_default(0)
5150     .set_description(""),
5151
5152     Option("mgr_stats_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5153     .set_default((int64_t)PerfCountersBuilder::PRIO_USEFUL)
5154     .set_description("Lowest perfcounter priority collected by mgr")
5155     .set_long_description("Daemons only set perf counter data to the manager "
5156                           "daemon if the counter has a priority higher than this.")
5157     .set_min_max((int64_t)PerfCountersBuilder::PRIO_DEBUGONLY,
5158                  (int64_t)PerfCountersBuilder::PRIO_CRITICAL + 1),
5159
5160     Option("journal_zero_on_create", Option::TYPE_BOOL, Option::LEVEL_DEV)
5161     .set_default(false)
5162     .set_description(""),
5163
5164     Option("journal_ignore_corruption", Option::TYPE_BOOL, Option::LEVEL_DEV)
5165     .set_default(false)
5166     .set_description(""),
5167
5168     Option("journal_discard", Option::TYPE_BOOL, Option::LEVEL_DEV)
5169     .set_default(false)
5170     .set_description(""),
5171
5172     Option("fio_dir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5173     .set_default("/tmp/fio")
5174     .set_description(""),
5175
5176     Option("rados_mon_op_timeout", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
5177     .set_default(0)
5178     .set_description("timeout for operations handled by monitors such as statfs (0 is unlimited)")
5179     .set_flag(Option::FLAG_RUNTIME)
5180     .set_min(0),
5181
5182     Option("rados_osd_op_timeout", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
5183     .set_default(0)
5184     .set_description("timeout for operations handled by osds such as write (0 is unlimited)")
5185     .set_flag(Option::FLAG_RUNTIME)
5186     .set_min(0),
5187
5188     Option("rados_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5189     .set_default(false)
5190     .set_description(""),
5191
5192     Option("cephadm_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5193     .set_default("/usr/sbin/cephadm")
5194     .add_service("mgr")
5195     .set_description("Path to cephadm utility"),
5196
5197     Option("mgr_module_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5198     .set_default(CEPH_DATADIR "/mgr")
5199     .add_service("mgr")
5200     .set_description("Filesystem path to manager modules."),
5201
5202     Option("mgr_initial_modules", Option::TYPE_STR, Option::LEVEL_BASIC)
5203     .set_default("restful iostat")
5204     .set_flag(Option::FLAG_NO_MON_UPDATE)
5205     .set_flag(Option::FLAG_CLUSTER_CREATE)
5206     .add_service("mon")
5207     .set_description("List of manager modules to enable when the cluster is "
5208                      "first started")
5209     .set_long_description("This list of module names is read by the monitor "
5210         "when the cluster is first started after installation, to populate "
5211         "the list of enabled manager modules.  Subsequent updates are done using "
5212         "the 'mgr module [enable|disable]' commands.  List may be comma "
5213         "or space separated."),
5214
5215     Option("mgr_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5216     .set_default("/var/lib/ceph/mgr/$cluster-$id")
5217     .set_flag(Option::FLAG_NO_MON_UPDATE)
5218     .add_service("mgr")
5219     .set_description("Filesystem path to the ceph-mgr data directory, used to "
5220                      "contain keyring."),
5221
5222     Option("mgr_tick_period", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
5223     .set_default(2)
5224     .add_service("mgr")
5225     .set_description("Period in seconds of beacon messages to monitor"),
5226
5227     Option("mgr_stats_period", Option::TYPE_INT, Option::LEVEL_BASIC)
5228     .set_default(5)
5229     .add_service("mgr")
5230     .set_description("Period in seconds of OSD/MDS stats reports to manager")
5231     .set_long_description("Use this setting to control the granularity of "
5232                           "time series data collection from daemons.  Adjust "
5233                           "upwards if the manager CPU load is too high, or "
5234                           "if you simply do not require the most up to date "
5235                           "performance counter data."),
5236
5237     Option("mgr_client_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5238     .set_default(128_M)
5239     .add_service("mgr"),
5240
5241     Option("mgr_client_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5242     .set_default(512)
5243     .add_service("mgr"),
5244
5245     Option("mgr_osd_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5246     .set_default(512_M)
5247     .add_service("mgr"),
5248
5249     Option("mgr_osd_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5250     .set_default(8192)
5251     .add_service("mgr"),
5252
5253     Option("mgr_mds_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5254     .set_default(128_M)
5255     .add_service("mgr"),
5256
5257     Option("mgr_mds_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5258     .set_default(128)
5259     .add_service("mgr"),
5260
5261     Option("mgr_mon_bytes", Option::TYPE_SIZE, Option::LEVEL_DEV)
5262     .set_default(128_M)
5263     .add_service("mgr"),
5264
5265     Option("mgr_mon_messages", Option::TYPE_UINT, Option::LEVEL_DEV)
5266     .set_default(128)
5267     .add_service("mgr"),
5268
5269     Option("mgr_connect_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5270     .set_default(1.0)
5271     .add_service("common"),
5272
5273     Option("mgr_service_beacon_grace", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5274     .set_default(60.0)
5275     .add_service("mgr")
5276     .set_description("Period in seconds from last beacon to manager dropping "
5277                      "state about a monitored service (RGW, rbd-mirror etc)"),
5278
5279     Option("mgr_client_service_daemon_unregister_timeout", Option::TYPE_FLOAT, Option::LEVEL_DEV)
5280     .set_default(1.0)
5281     .set_description("Time to wait during shutdown to deregister service with mgr"),
5282
5283     Option("mgr_debug_aggressive_pg_num_changes", Option::TYPE_BOOL, Option::LEVEL_DEV)
5284     .set_default(false)
5285     .set_description("Bypass most throttling and safety checks in pg[p]_num controller")
5286     .add_service("mgr"),
5287
5288     Option("mon_mgr_digest_period", Option::TYPE_INT, Option::LEVEL_DEV)
5289     .set_default(5)
5290     .add_service("mon")
5291     .set_description("Period in seconds between monitor-to-manager "
5292                      "health/status updates"),
5293
5294     Option("mon_mgr_beacon_grace", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
5295     .set_default(30)
5296     .add_service("mon")
5297     .set_description("Period in seconds from last beacon to monitor marking "
5298                      "a manager daemon as failed"),
5299
5300     Option("mon_mgr_inactive_grace", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5301     .set_default(60)
5302     .add_service("mon")
5303     .set_description("Period in seconds after cluster creation during which "
5304                      "cluster may have no active manager")
5305     .set_long_description("This grace period enables the cluster to come "
5306                           "up cleanly without raising spurious health check "
5307                           "failures about managers that aren't online yet"),
5308
5309     Option("mon_mgr_mkfs_grace", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5310     .set_default(120)
5311     .add_service("mon")
5312     .set_description("Period in seconds that the cluster may have no active "
5313                      "manager before this is reported as an ERR rather than "
5314                      "a WARN"),
5315
5316     Option("throttler_perf_counter", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5317     .set_default(true)
5318     .set_description(""),
5319
5320     Option("event_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5321     .set_default(false)
5322     .set_description(""),
5323
5324     Option("bluestore_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5325     .set_default(false)
5326     .set_description("Enable bluestore event tracing."),
5327
5328     Option("bluestore_throttle_trace_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
5329     .set_default(0)
5330     .set_description("Rate at which to sample bluestore transactions (per second)"),
5331
5332     Option("debug_deliberately_leak_memory", Option::TYPE_BOOL, Option::LEVEL_DEV)
5333     .set_default(false)
5334     .set_description(""),
5335
5336     Option("debug_asserts_on_shutdown", Option::TYPE_BOOL,Option::LEVEL_DEV)
5337     .set_default(false)
5338     .set_description("Enable certain asserts to check for refcounting bugs on shutdown; see http://tracker.ceph.com/issues/21738"),
5339
5340     Option("debug_asok_assert_abort", Option::TYPE_BOOL, Option::LEVEL_DEV)
5341     .set_default(false)
5342     .set_description("allow commands 'assert' and 'abort' via asok for testing crash dumps etc"),
5343
5344     Option("target_max_misplaced_ratio", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
5345     .set_default(.05)
5346     .set_description("Max ratio of misplaced objects to target when throttling data rebalancing activity"),
5347
5348     Option("device_failure_prediction_mode", Option::TYPE_STR, Option::LEVEL_BASIC)
5349     .set_default("none")
5350     .set_flag(Option::FLAG_RUNTIME)
5351     .set_enum_allowed({"none", "local", "cloud"})
5352     .set_description("Method used to predict device failures")
5353     .set_long_description("To disable prediction, use 'none',  'local' uses a prediction model that runs inside the mgr daemon.  'cloud' will share metrics with a cloud service and query the service for devicelife expectancy."),
5354
5355     /*  KRB Authentication. */
5356     Option("gss_ktab_client_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5357     .set_default("/var/lib/ceph/$name/gss_client_$name.ktab")
5358     .set_description("GSS/KRB5 Keytab file for client authentication")
5359     .add_service({"mon", "osd"})
5360     .set_long_description("This sets the full path for the GSS/Kerberos client keytab file location."),
5361
5362     Option("gss_target_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5363     .set_default("ceph")
5364     .set_description("")
5365     .add_service({"mon", "osd"})
5366     .set_long_description("This sets the gss target service name."),
5367
5368     Option("debug_disable_randomized_ping", Option::TYPE_BOOL, Option::LEVEL_DEV)
5369     .set_default(false)
5370     .set_description("Disable heartbeat ping randomization for testing purposes"),
5371
5372     Option("debug_heartbeat_testing_span", Option::TYPE_INT, Option::LEVEL_DEV)
5373     .set_default(0)
5374     .set_description("Override 60 second periods for testing only"),
5375
5376     // ----------------------------
5377     // Crimson specific options
5378
5379     Option("crimson_osd_obc_lru_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5380     .set_default(10)
5381     .set_description("Number of obcs to cache")
5382
5383   });
5384 }
5385
5386 std::vector<Option> get_rgw_options() {
5387   return std::vector<Option>({
5388     Option("rgw_acl_grants_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5389     .set_default(100)
5390     .set_description("Max number of ACL grants in a single request"),
5391
5392     Option("rgw_cors_rules_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5393     .set_default(100)
5394     .set_description("Max number of cors rules in a single request"),
5395
5396     Option("rgw_delete_multi_obj_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5397     .set_default(1000)
5398     .set_description("Max number of objects in a single multi-object delete request"),
5399
5400     Option("rgw_website_routing_rules_max_num", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5401     .set_default(50)
5402     .set_description("Max number of website routing rules in a single request"),
5403
5404     Option("rgw_rados_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5405     .set_default(false)
5406     .set_description("true if LTTng-UST tracepoints should be enabled"),
5407
5408     Option("rgw_op_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5409     .set_default(false)
5410     .set_description("true if LTTng-UST tracepoints should be enabled"),
5411
5412     Option("rgw_max_chunk_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5413     .set_default(4_M)
5414     .set_description("Set RGW max chunk size")
5415     .set_long_description(
5416         "The chunk size is the size of RADOS I/O requests that RGW sends when accessing "
5417         "data objects. RGW read and write operation will never request more than this amount "
5418         "in a single request. This also defines the rgw object head size, as head operations "
5419         "need to be atomic, and anything larger than this would require more than a single "
5420         "operation."),
5421
5422     Option("rgw_put_obj_min_window_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5423     .set_default(16_M)
5424     .set_description("The minimum RADOS write window size (in bytes).")
5425     .set_long_description(
5426         "The window size determines the total concurrent RADOS writes of a single rgw object. "
5427         "When writing an object RGW will send multiple chunks to RADOS. The total size of the "
5428         "writes does not exceed the window size. The window size can be automatically "
5429         "in order to better utilize the pipe.")
5430     .add_see_also({"rgw_put_obj_max_window_size", "rgw_max_chunk_size"}),
5431
5432     Option("rgw_put_obj_max_window_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5433     .set_default(64_M)
5434     .set_description("The maximum RADOS write window size (in bytes).")
5435     .set_long_description("The window size may be dynamically adjusted, but will not surpass this value.")
5436     .add_see_also({"rgw_put_obj_min_window_size", "rgw_max_chunk_size"}),
5437
5438     Option("rgw_max_put_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5439     .set_default(5_G)
5440     .set_description("Max size (in bytes) of regular (non multi-part) object upload.")
5441     .set_long_description(
5442         "Plain object upload is capped at this amount of data. In order to upload larger "
5443         "objects, a special upload mechanism is required. The S3 API provides the "
5444         "multi-part upload, and Swift provides DLO and SLO."),
5445
5446     Option("rgw_max_put_param_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5447     .set_default(1_M)
5448     .set_description("The maximum size (in bytes) of data input of certain RESTful requests."),
5449
5450     Option("rgw_max_attr_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5451     .set_default(0)
5452     .set_description("The maximum length of metadata value. 0 skips the check"),
5453
5454     Option("rgw_max_attr_name_len", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
5455     .set_default(0)
5456     .set_description("The maximum length of metadata name. 0 skips the check"),
5457
5458     Option("rgw_max_attrs_num_in_req", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5459     .set_default(0)
5460     .set_description("The maximum number of metadata items that can be put via single request"),
5461
5462     Option("rgw_override_bucket_index_max_shards", Option::TYPE_UINT, Option::LEVEL_DEV)
5463     .set_default(0)
5464     .set_description("The default number of bucket index shards for newly-created "
5465         "buckets. This value overrides bucket_index_max_shards stored in the zone. "
5466         "Setting this value in the zone is preferred, because it applies globally "
5467         "to all radosgw daemons running in the zone."),
5468
5469     Option("rgw_bucket_index_max_aio", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5470     .set_default(128)
5471     .set_description("Max number of concurrent RADOS requests when handling bucket shards."),
5472
5473     Option("rgw_enable_quota_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5474     .set_default(true)
5475     .set_description("Enables the quota maintenance thread.")
5476     .set_long_description(
5477         "The quota maintenance thread is responsible for quota related maintenance work. "
5478         "The thread itself can be disabled, but in order for quota to work correctly, at "
5479         "least one RGW in each zone needs to have this thread running. Having the thread "
5480         "enabled on multiple RGW processes within the same zone can spread "
5481         "some of the maintenance work between them.")
5482     .add_see_also({"rgw_enable_gc_threads", "rgw_enable_lc_threads"}),
5483
5484     Option("rgw_enable_gc_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5485     .set_default(true)
5486     .set_description("Enables the garbage collection maintenance thread.")
5487     .set_long_description(
5488         "The garbage collection maintenance thread is responsible for garbage collector "
5489         "maintenance work. The thread itself can be disabled, but in order for garbage "
5490         "collection to work correctly, at least one RGW in each zone needs to have this "
5491         "thread running.  Having the thread enabled on multiple RGW processes within the "
5492         "same zone can spread some of the maintenance work between them.")
5493     .add_see_also({"rgw_enable_quota_threads", "rgw_enable_lc_threads"}),
5494
5495     Option("rgw_enable_lc_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5496     .set_default(true)
5497     .set_description("Enables the lifecycle maintenance thread. This is required on at least one rgw for each zone.")
5498     .set_long_description(
5499         "The lifecycle maintenance thread is responsible for lifecycle related maintenance "
5500         "work. The thread itself can be disabled, but in order for lifecycle to work "
5501         "correctly, at least one RGW in each zone needs to have this thread running. Having"
5502         "the thread enabled on multiple RGW processes within the same zone can spread "
5503         "some of the maintenance work between them.")
5504     .add_see_also({"rgw_enable_gc_threads", "rgw_enable_quota_threads"}),
5505
5506     Option("rgw_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5507     .set_default("/var/lib/ceph/radosgw/$cluster-$id")
5508     .set_flag(Option::FLAG_NO_MON_UPDATE)
5509     .set_description("Alternative location for RGW configuration.")
5510     .set_long_description(
5511         "If this is set, the different Ceph system configurables (such as the keyring file "
5512         "will be located in the path that is specified here. "),
5513
5514     Option("rgw_enable_apis", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5515     .set_default("s3, s3website, swift, swift_auth, admin, sts, iam, pubsub")
5516     .set_description("A list of set of RESTful APIs that rgw handles."),
5517
5518     Option("rgw_cache_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5519     .set_default(true)
5520     .set_description("Enable RGW metadata cache.")
5521     .set_long_description(
5522         "The metadata cache holds metadata entries that RGW requires for processing "
5523         "requests. Metadata entries can be user info, bucket info, and bucket instance "
5524         "info. If not found in the cache, entries will be fetched from the backing "
5525         "RADOS store.")
5526     .add_see_also("rgw_cache_lru_size"),
5527
5528     Option("rgw_cache_lru_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5529     .set_default(10000)
5530     .set_description("Max number of items in RGW metadata cache.")
5531     .set_long_description(
5532         "When full, the RGW metadata cache evicts least recently used entries.")
5533     .add_see_also("rgw_cache_enabled"),
5534
5535     Option("rgw_socket_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5536     .set_default("")
5537     .set_description("RGW FastCGI socket path (for FastCGI over Unix domain sockets).")
5538     .add_see_also("rgw_fcgi_socket_backlog"),
5539
5540     Option("rgw_host", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5541     .set_default("")
5542     .set_description("RGW FastCGI host name (for FastCGI over TCP)")
5543     .add_see_also({"rgw_port", "rgw_fcgi_socket_backlog"}),
5544
5545     Option("rgw_port", Option::TYPE_STR, Option::LEVEL_BASIC)
5546     .set_default("")
5547     .set_description("RGW FastCGI port number (for FastCGI over TCP)")
5548     .add_see_also({"rgw_host", "rgw_fcgi_socket_backlog"}),
5549
5550     Option("rgw_dns_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5551     .set_default("")
5552     .set_description("The host name that RGW uses.")
5553     .set_long_description(
5554         "This is Needed for virtual hosting of buckets to work properly, unless configured "
5555         "via zonegroup configuration."),
5556
5557     Option("rgw_dns_s3website_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5558     .set_default("")
5559     .set_description("The host name that RGW uses for static websites (S3)")
5560     .set_long_description(
5561         "This is needed for virtual hosting of buckets, unless configured via zonegroup "
5562         "configuration."),
5563
5564     Option("rgw_numa_node", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5565     .set_default(-1)
5566     .set_flag(Option::FLAG_STARTUP)
5567     .set_description("set rgw's cpu affinity to a numa node (-1 for none)"),
5568
5569     Option("rgw_service_provider_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5570     .set_default("")
5571     .set_description("Service provider name which is contained in http response headers")
5572     .set_long_description(
5573         "As S3 or other cloud storage providers do, http response headers should contain the name of the provider. "
5574         "This name will be placed in http header 'Server'."),
5575
5576     Option("rgw_content_length_compat", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5577     .set_default(false)
5578     .set_description("Multiple content length headers compatibility")
5579     .set_long_description(
5580         "Try to handle requests with abiguous multiple content length headers "
5581         "(Content-Length, Http-Content-Length)."),
5582
5583     Option("rgw_relaxed_region_enforcement", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5584     .set_default(false)
5585     .set_description("Disable region constraint enforcement")
5586     .set_long_description(
5587         "Enable requests such as bucket creation to succeed irrespective of region restrictions (Jewel compat)."),
5588
5589     Option("rgw_lifecycle_work_time", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5590     .set_default("00:00-06:00")
5591     .set_description("Lifecycle allowed work time")
5592     .set_long_description("Local time window in which the lifecycle maintenance thread can work."),
5593
5594     Option("rgw_lc_lock_max_time", Option::TYPE_INT, Option::LEVEL_DEV)
5595     .set_default(90)
5596     .set_description(""),
5597
5598     Option("rgw_lc_thread_delay", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5599     .set_default(0)
5600     .set_description("Delay after processing of bucket listing chunks (i.e., per 1000 entries) in milliseconds"),
5601
5602     Option("rgw_lc_max_worker", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5603     .set_default(3)
5604     .set_description("Number of LCWorker tasks that will be run in parallel")
5605     .set_long_description(
5606       "Number of LCWorker tasks that will run in parallel--used to permit >1 "
5607       "bucket/index shards to be processed simultaneously"),
5608
5609     Option("rgw_lc_max_wp_worker", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5610     .set_default(3)
5611     .set_description("Number of workpool threads per LCWorker")
5612     .set_long_description(
5613       "Number of threads in per-LCWorker workpools--used to accelerate "
5614       "per-bucket processing"),
5615
5616     Option("rgw_lc_max_objs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5617     .set_default(32)
5618     .set_description("Number of lifecycle data shards")
5619     .set_long_description(
5620           "Number of RADOS objects to use for storing lifecycle index. This "
5621           "affects concurrency of lifecycle maintenance, as shards can be "
5622           "processed in parallel."),
5623
5624     Option("rgw_lc_max_rules", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5625     .set_default(1000)
5626     .set_description("Max number of lifecycle rules set on one bucket")
5627     .set_long_description("Number of lifecycle rules set on one bucket should be limited."),
5628
5629     Option("rgw_lc_debug_interval", Option::TYPE_INT, Option::LEVEL_DEV)
5630     .set_default(-1)
5631     .set_description(""),
5632
5633     Option("rgw_mp_lock_max_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5634     .set_default(600)
5635     .set_description("Multipart upload max completion time")
5636     .set_long_description(
5637         "Time length to allow completion of a multipart upload operation. This is done "
5638         "to prevent concurrent completions on the same object with the same upload id."),
5639
5640     Option("rgw_script_uri", Option::TYPE_STR, Option::LEVEL_DEV)
5641     .set_default("")
5642     .set_description(""),
5643
5644     Option("rgw_request_uri", Option::TYPE_STR, Option::LEVEL_DEV)
5645     .set_default("")
5646     .set_description(""),
5647
5648     Option("rgw_ignore_get_invalid_range", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5649     .set_default(false)
5650     .set_description("Treat invalid (e.g., negative) range request as full")
5651     .set_long_description("Treat invalid (e.g., negative) range request "
5652                           "as request for the full object (AWS compatibility)"),
5653
5654     Option("rgw_swift_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5655     .set_default("")
5656     .set_description("Swift-auth storage URL")
5657     .set_long_description(
5658         "Used in conjunction with rgw internal swift authentication. This affects the "
5659         "X-Storage-Url response header value.")
5660     .add_see_also("rgw_swift_auth_entry"),
5661
5662     Option("rgw_swift_url_prefix", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5663     .set_default("swift")
5664     .set_description("Swift URL prefix")
5665     .set_long_description("The URL path prefix for swift requests."),
5666
5667     Option("rgw_swift_auth_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5668     .set_default("")
5669     .set_description("Swift auth URL")
5670     .set_long_description(
5671         "Default url to which RGW connects and verifies tokens for v1 auth (if not using "
5672         "internal swift auth)."),
5673
5674     Option("rgw_swift_auth_entry", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5675     .set_default("auth")
5676     .set_description("Swift auth URL prefix")
5677     .set_long_description("URL path prefix for internal swift auth requests.")
5678     .add_see_also("rgw_swift_url"),
5679
5680     Option("rgw_swift_tenant_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5681     .set_default("")
5682     .set_description("Swift tenant name")
5683     .set_long_description("Tenant name that is used when constructing the swift path.")
5684     .add_see_also("rgw_swift_account_in_url"),
5685
5686     Option("rgw_swift_account_in_url", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5687     .set_default(false)
5688     .set_description("Swift account encoded in URL")
5689     .set_long_description("Whether the swift account is encoded in the uri path (AUTH_<account>).")
5690     .add_see_also("rgw_swift_tenant_name"),
5691
5692     Option("rgw_swift_enforce_content_length", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5693     .set_default(false)
5694     .set_description("Send content length when listing containers (Swift)")
5695     .set_long_description(
5696         "Whether content length header is needed when listing containers. When this is "
5697         "set to false, RGW will send extra info for each entry in the response."),
5698
5699     Option("rgw_keystone_url", Option::TYPE_STR, Option::LEVEL_BASIC)
5700     .set_default("")
5701     .set_description("The URL to the Keystone server."),
5702
5703     Option("rgw_keystone_admin_token", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5704     .set_default("")
5705     .set_description("DEPRECATED: The admin token (shared secret) that is used for the Keystone requests."),
5706
5707     Option("rgw_keystone_admin_token_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5708     .set_default("")
5709     .set_description("Path to a file containing the admin token (shared secret) that is used for the Keystone requests."),
5710
5711     Option("rgw_keystone_admin_user", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5712     .set_default("")
5713     .set_description("Keystone admin user."),
5714
5715     Option("rgw_keystone_admin_password", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5716     .set_default("")
5717     .set_description("DEPRECATED: Keystone admin password."),
5718
5719     Option("rgw_keystone_admin_password_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5720     .set_default("")
5721     .set_description("Path to a file containing the Keystone admin password."),
5722
5723     Option("rgw_keystone_admin_tenant", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5724     .set_default("")
5725     .set_description("Keystone admin user tenant."),
5726
5727     Option("rgw_keystone_admin_project", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5728     .set_default("")
5729     .set_description("Keystone admin user project (for Keystone v3)."),
5730
5731     Option("rgw_keystone_admin_domain", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5732     .set_default("")
5733     .set_description("Keystone admin user domain (for Keystone v3)."),
5734
5735     Option("rgw_keystone_barbican_user", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5736     .set_default("")
5737     .set_description("Keystone user to access barbican secrets."),
5738
5739     Option("rgw_keystone_barbican_password", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5740     .set_default("")
5741     .set_description("Keystone password for barbican user."),
5742
5743     Option("rgw_keystone_barbican_tenant", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5744     .set_default("")
5745     .set_description("Keystone barbican user tenant (Keystone v2.0)."),
5746
5747     Option("rgw_keystone_barbican_project", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5748     .set_default("")
5749     .set_description("Keystone barbican user project (Keystone v3)."),
5750
5751     Option("rgw_keystone_barbican_domain", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5752     .set_default("")
5753     .set_description("Keystone barbican user domain."),
5754
5755     Option("rgw_keystone_api_version", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5756     .set_default(2)
5757     .set_description("Version of Keystone API to use (2 or 3)."),
5758
5759     Option("rgw_keystone_accepted_roles", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5760     .set_default("Member, admin")
5761     .set_description("Only users with one of these roles will be served when doing Keystone authentication."),
5762
5763     Option("rgw_keystone_accepted_admin_roles", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5764     .set_default("")
5765     .set_description("List of roles allowing user to gain admin privileges (Keystone)."),
5766
5767     Option("rgw_keystone_token_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5768     .set_default(10000)
5769     .set_description("Keystone token cache size")
5770     .set_long_description(
5771         "Max number of Keystone tokens that will be cached. Token that is not cached "
5772         "requires RGW to access the Keystone server when authenticating."),
5773
5774     Option("rgw_keystone_verify_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5775     .set_default(true)
5776     .set_description("Should RGW verify the Keystone server SSL certificate."),
5777
5778     Option("rgw_keystone_implicit_tenants", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5779     .set_default("false")
5780     .set_enum_allowed( { "false", "true", "swift", "s3", "both", "0", "1", "none" } )
5781     .set_description("RGW Keystone implicit tenants creation")
5782     .set_long_description(
5783         "Implicitly create new users in their own tenant with the same name when "
5784         "authenticating via Keystone.  Can be limited to s3 or swift only."),
5785
5786     Option("rgw_cross_domain_policy", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5787     .set_default("<allow-access-from domain=\"*\" secure=\"false\" />")
5788     .set_description("RGW handle cross domain policy")
5789     .set_long_description("Returned cross domain policy when accessing the crossdomain.xml "
5790                           "resource (Swift compatiility)."),
5791
5792     Option("rgw_healthcheck_disabling_path", Option::TYPE_STR, Option::LEVEL_DEV)
5793     .set_default("")
5794     .set_description("Swift health check api can be disabled if a file can be accessed in this path."),
5795
5796     Option("rgw_s3_auth_use_rados", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5797     .set_default(true)
5798     .set_description("Should S3 authentication use credentials stored in RADOS backend."),
5799
5800     Option("rgw_s3_auth_use_keystone", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5801     .set_default(false)
5802     .set_description("Should S3 authentication use Keystone."),
5803
5804     Option("rgw_s3_auth_order", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5805      .set_default("sts, external, local")
5806      .set_description("Authentication strategy order to use for s3 authentication")
5807      .set_long_description(
5808           "Order of authentication strategies to try for s3 authentication, the allowed "
5809            "options are a comma separated list of engines external, local. The "
5810            "default order is to try all the externally configured engines before "
5811            "attempting local rados based authentication"),
5812
5813     Option("rgw_barbican_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5814     .set_default("")
5815     .set_description("URL to barbican server."),
5816
5817     Option("rgw_ldap_uri", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5818     .set_default("ldaps://<ldap.your.domain>")
5819     .set_description("Space-separated list of LDAP servers in URI format."),
5820
5821     Option("rgw_ldap_binddn", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5822     .set_default("uid=admin,cn=users,dc=example,dc=com")
5823     .set_description("LDAP entry RGW will bind with (user match)."),
5824
5825     Option("rgw_ldap_searchdn", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5826     .set_default("cn=users,cn=accounts,dc=example,dc=com")
5827     .set_description("LDAP search base (basedn)."),
5828
5829     Option("rgw_ldap_dnattr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5830     .set_default("uid")
5831     .set_description("LDAP attribute containing RGW user names (to form binddns)."),
5832
5833     Option("rgw_ldap_secret", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5834     .set_default("/etc/openldap/secret")
5835     .set_description("Path to file containing credentials for rgw_ldap_binddn."),
5836
5837     Option("rgw_s3_auth_use_ldap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5838     .set_default(false)
5839     .set_description("Should S3 authentication use LDAP."),
5840
5841     Option("rgw_ldap_searchfilter", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5842     .set_default("")
5843     .set_description("LDAP search filter."),
5844
5845     Option("rgw_opa_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5846     .set_default("")
5847     .set_description("URL to OPA server."),
5848
5849     Option("rgw_opa_token", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5850     .set_default("")
5851     .set_description("The Bearer token OPA uses to authenticate client requests."),
5852
5853     Option("rgw_opa_verify_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5854     .set_default(true)
5855     .set_description("Should RGW verify the OPA server SSL certificate."),
5856
5857     Option("rgw_use_opa_authz", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5858     .set_default(false)
5859     .set_description("Should OPA be used to authorize client requests."),
5860
5861     Option("rgw_admin_entry", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5862     .set_default("admin")
5863     .set_description("Path prefix to be used for accessing RGW RESTful admin API."),
5864
5865     Option("rgw_enforce_swift_acls", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5866     .set_default(true)
5867     .set_description("RGW enforce swift acls")
5868     .set_long_description(
5869         "Should RGW enforce special Swift-only ACLs. Swift has a special ACL that gives "
5870         "permission to access all objects in a container."),
5871
5872     Option("rgw_swift_token_expiration", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5873     .set_default(1_day)
5874     .set_description("Expiration time (in seconds) for token generated through RGW Swift auth."),
5875
5876     Option("rgw_print_continue", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5877     .set_default(true)
5878     .set_description("RGW support of 100-continue")
5879     .set_long_description(
5880         "Should RGW explicitly send 100 (continue) responses. This is mainly relevant when "
5881         "using FastCGI, as some FastCGI modules do not fully support this feature."),
5882
5883     Option("rgw_print_prohibited_content_length", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5884     .set_default(false)
5885     .set_description("RGW RFC-7230 compatibility")
5886     .set_long_description(
5887         "Specifies whether RGW violates RFC 7230 and sends Content-Length with 204 or 304 "
5888         "statuses."),
5889
5890     Option("rgw_remote_addr_param", Option::TYPE_STR, Option::LEVEL_ADVANCED)
5891     .set_default("REMOTE_ADDR")
5892     .set_description("HTTP header that holds the remote address in incoming requests.")
5893     .set_long_description(
5894         "RGW will use this header to extract requests origin. When RGW runs behind "
5895         "a reverse proxy, the remote address header will point at the proxy's address "
5896         "and not at the originator's address. Therefore it is sometimes possible to "
5897         "have the proxy add the originator's address in a separate HTTP header, which "
5898         "will allow RGW to log it correctly."
5899         )
5900     .add_see_also("rgw_enable_ops_log"),
5901
5902     Option("rgw_op_thread_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
5903     .set_default(10*60)
5904     .set_description("Timeout for async rados coroutine operations."),
5905
5906     Option("rgw_op_thread_suicide_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
5907     .set_default(0)
5908     .set_description(""),
5909
5910     Option("rgw_thread_pool_size", Option::TYPE_INT, Option::LEVEL_BASIC)
5911     .set_default(512)
5912     .set_description("RGW requests handling thread pool size.")
5913     .set_long_description(
5914         "This parameter determines the number of concurrent requests RGW can process "
5915         "when using either the civetweb, or the fastcgi frontends. The higher this "
5916         "number is, RGW will be able to deal with more concurrent requests at the "
5917         "cost of more resource utilization."),
5918
5919     Option("rgw_num_control_oids", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5920     .set_default(8)
5921     .set_description("Number of control objects used for cross-RGW communication.")
5922     .set_long_description(
5923         "RGW uses certain control objects to send messages between different RGW "
5924         "processes running on the same zone. These messages include metadata cache "
5925         "invalidation info that is being sent when metadata is modified (such as "
5926         "user or bucket information). A higher number of control objects allows "
5927         "better concurrency of these messages, at the cost of more resource "
5928         "utilization."),
5929
5930     Option("rgw_num_rados_handles", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
5931     .set_default(1)
5932     .set_description("Number of librados handles that RGW uses.")
5933     .set_long_description(
5934         "This param affects the number of separate librados handles it uses to "
5935         "connect to the RADOS backend, which directly affects the number of connections "
5936         "RGW will have to each OSD. A higher number affects resource utilization."),
5937
5938     Option("rgw_verify_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5939     .set_default(true)
5940     .set_description("Should RGW verify SSL when connecing to a remote HTTP server")
5941     .set_long_description(
5942         "RGW can send requests to other RGW servers (e.g., in multi-site sync work). "
5943         "This configurable selects whether RGW should verify the certificate for "
5944         "the remote peer and host.")
5945     .add_see_also("rgw_keystone_verify_ssl"),
5946
5947     Option("rgw_nfs_lru_lanes", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5948     .set_default(5)
5949     .set_description(""),
5950
5951     Option("rgw_nfs_lru_lane_hiwat", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5952     .set_default(911)
5953     .set_description(""),
5954
5955     Option("rgw_nfs_fhcache_partitions", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5956     .set_default(3)
5957     .set_description(""),
5958
5959     Option("rgw_nfs_fhcache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5960     .set_default(2017)
5961     .set_description(""),
5962
5963     Option("rgw_nfs_namespace_expire_secs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5964     .set_default(300)
5965     .set_min(1)
5966     .set_description(""),
5967
5968     Option("rgw_nfs_max_gc", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5969     .set_default(300)
5970     .set_min(1)
5971     .set_description(""),
5972
5973     Option("rgw_nfs_write_completion_interval_s", Option::TYPE_INT, Option::LEVEL_ADVANCED)
5974     .set_default(10)
5975     .set_description(""),
5976
5977     Option("rgw_nfs_s3_fast_attrs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5978     .set_default(false)
5979     .set_description("use fast S3 attrs from bucket index (immutable only)")
5980     .set_long_description("use fast S3 attrs from bucket index (assumes NFS "
5981                           "mounts are immutable)"),
5982
5983     Option("rgw_nfs_run_gc_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5984     .set_default(false)
5985     .set_description("run GC threads in librgw (default off)"),
5986
5987     Option("rgw_nfs_run_lc_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5988     .set_default(false)
5989     .set_description("run lifecycle threads in librgw (default off)"),
5990
5991     Option("rgw_nfs_run_quota_threads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5992     .set_default(false)
5993     .set_description("run quota threads in librgw (default off)"),
5994
5995     Option("rgw_nfs_run_sync_thread", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
5996     .set_default(false)
5997     .set_description("run sync thread in librgw (default off)"),
5998
5999     Option("rgw_rados_pool_autoscale_bias", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
6000     .set_default(4.0)
6001     .set_min_max(0.01, 100000.0)
6002     .set_description("pg_autoscale_bias value for RGW metadata (omap-heavy) pools"),
6003
6004     Option("rgw_rados_pool_pg_num_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6005     .set_default(8)
6006     .set_min_max(1, 1024)
6007     .set_description("pg_num_min value for RGW metadata (omap-heavy) pools"),
6008
6009     Option("rgw_rados_pool_recovery_priority", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6010     .set_default(5)
6011     .set_min_max(-10, 10)
6012     .set_description("recovery_priority value for RGW metadata (omap-heavy) pools"),
6013
6014     Option("rgw_zone", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6015     .set_default("")
6016     .set_description("Zone name")
6017     .add_see_also({"rgw_zonegroup", "rgw_realm"}),
6018
6019     Option("rgw_zone_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6020     .set_default(".rgw.root")
6021     .set_description("Zone root pool name")
6022     .set_long_description(
6023         "The zone root pool, is the pool where the RGW zone configuration located."
6024     )
6025     .add_see_also({"rgw_zonegroup_root_pool", "rgw_realm_root_pool", "rgw_period_root_pool"}),
6026
6027     Option("rgw_default_zone_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6028     .set_default("default.zone")
6029     .set_description("Default zone info object id")
6030     .set_long_description(
6031         "Name of the RADOS object that holds the default zone information."
6032     ),
6033
6034     Option("rgw_region", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6035     .set_default("")
6036     .set_description("Region name")
6037     .set_long_description(
6038         "Obsolete config option. The rgw_zonegroup option should be used instead.")
6039     .add_see_also("rgw_zonegroup"),
6040
6041     Option("rgw_region_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6042     .set_default(".rgw.root")
6043     .set_description("Region root pool")
6044     .set_long_description(
6045         "Obsolete config option. The rgw_zonegroup_root_pool should be used instead.")
6046     .add_see_also("rgw_zonegroup_root_pool"),
6047
6048     Option("rgw_default_region_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6049     .set_default("default.region")
6050     .set_description("Default region info object id")
6051     .set_long_description(
6052         "Obsolete config option. The rgw_default_zonegroup_info_oid should be used instead.")
6053     .add_see_also("rgw_default_zonegroup_info_oid"),
6054
6055     Option("rgw_zonegroup", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6056     .set_default("")
6057     .set_description("Zonegroup name")
6058     .add_see_also({"rgw_zone", "rgw_realm"}),
6059
6060     Option("rgw_zonegroup_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6061     .set_default(".rgw.root")
6062     .set_description("Zonegroup root pool")
6063     .set_long_description(
6064         "The zonegroup root pool, is the pool where the RGW zonegroup configuration located."
6065     )
6066     .add_see_also({"rgw_zone_root_pool", "rgw_realm_root_pool", "rgw_period_root_pool"}),
6067
6068     Option("rgw_default_zonegroup_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6069     .set_default("default.zonegroup")
6070     .set_description(""),
6071
6072     Option("rgw_realm", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6073     .set_default("")
6074     .set_description(""),
6075
6076     Option("rgw_realm_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6077     .set_default(".rgw.root")
6078     .set_description("Realm root pool")
6079     .set_long_description(
6080         "The realm root pool, is the pool where the RGW realm configuration located."
6081     )
6082     .add_see_also({"rgw_zonegroup_root_pool", "rgw_zone_root_pool", "rgw_period_root_pool"}),
6083
6084     Option("rgw_default_realm_info_oid", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6085     .set_default("default.realm")
6086     .set_description(""),
6087
6088     Option("rgw_period_root_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6089     .set_default(".rgw.root")
6090     .set_description("Period root pool")
6091     .set_long_description(
6092         "The period root pool, is the pool where the RGW period configuration located."
6093     )
6094     .add_see_also({"rgw_zonegroup_root_pool", "rgw_zone_root_pool", "rgw_realm_root_pool"}),
6095
6096     Option("rgw_period_latest_epoch_info_oid", Option::TYPE_STR, Option::LEVEL_DEV)
6097     .set_default(".latest_epoch")
6098     .set_description(""),
6099
6100     Option("rgw_log_nonexistent_bucket", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6101     .set_default(false)
6102     .set_description("Should RGW log operations on bucket that does not exist")
6103     .set_long_description(
6104         "This config option applies to the ops log. When this option is set, the ops log "
6105         "will log operations that are sent to non existing buckets. These operations "
6106         "inherently fail, and do not correspond to a specific user.")
6107     .add_see_also("rgw_enable_ops_log"),
6108
6109     Option("rgw_log_object_name", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6110     .set_default("%Y-%m-%d-%H-%i-%n")
6111     .set_description("Ops log object name format")
6112     .set_long_description(
6113         "Defines the format of the RADOS objects names that ops log uses to store ops "
6114         "log data")
6115     .add_see_also("rgw_enable_ops_log"),
6116
6117     Option("rgw_log_object_name_utc", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6118     .set_default(false)
6119     .set_description("Should ops log object name based on UTC")
6120     .set_long_description(
6121         "If set, the names of the RADOS objects that hold the ops log data will be based "
6122         "on UTC time zone. If not set, it will use the local time zone.")
6123     .add_see_also({"rgw_enable_ops_log", "rgw_log_object_name"}),
6124
6125     Option("rgw_usage_max_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6126     .set_default(32)
6127     .set_description("Number of shards for usage log.")
6128     .set_long_description(
6129         "The number of RADOS objects that RGW will use in order to store the usage log "
6130         "data.")
6131     .add_see_also("rgw_enable_usage_log"),
6132
6133     Option("rgw_usage_max_user_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6134     .set_default(1)
6135     .set_min(1)
6136     .set_description("Number of shards for single user in usage log")
6137     .set_long_description(
6138         "The number of shards that a single user will span over in the usage log.")
6139     .add_see_also("rgw_enable_usage_log"),
6140
6141     Option("rgw_enable_ops_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6142     .set_default(false)
6143     .set_description("Enable ops log")
6144     .add_see_also({"rgw_log_nonexistent_bucket", "rgw_log_object_name", "rgw_ops_log_rados",
6145                "rgw_ops_log_socket_path"}),
6146
6147     Option("rgw_enable_usage_log", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6148     .set_default(false)
6149     .set_description("Enable usage log")
6150     .add_see_also("rgw_usage_max_shards"),
6151
6152     Option("rgw_ops_log_rados", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6153     .set_default(true)
6154     .set_description("Use RADOS for ops log")
6155     .set_long_description(
6156        "If set, RGW will store ops log information in RADOS.")
6157     .add_see_also({"rgw_enable_ops_log"}),
6158
6159     Option("rgw_ops_log_socket_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6160     .set_default("")
6161     .set_description("Unix domain socket path for ops log.")
6162     .set_long_description(
6163         "Path to unix domain socket that RGW will listen for connection on. When connected, "
6164         "RGW will send ops log data through it.")
6165     .add_see_also({"rgw_enable_ops_log", "rgw_ops_log_data_backlog"}),
6166
6167     Option("rgw_ops_log_data_backlog", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6168     .set_default(5 << 20)
6169     .set_description("Ops log socket backlog")
6170     .set_long_description(
6171         "Maximum amount of data backlog that RGW can keep when ops log is configured to "
6172         "send info through unix domain socket. When data backlog is higher than this, "
6173         "ops log entries will be lost. In order to avoid ops log information loss, the "
6174         "listener needs to clear data (by reading it) quickly enough.")
6175     .add_see_also({"rgw_enable_ops_log", "rgw_ops_log_socket_path"}),
6176
6177     Option("rgw_fcgi_socket_backlog", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6178     .set_default(1024)
6179     .set_description("FastCGI socket connection backlog")
6180     .set_long_description(
6181         "Size of FastCGI connection backlog. This reflects the maximum number of new "
6182         "connection requests that RGW can handle concurrently without dropping any. ")
6183     .add_see_also({"rgw_host", "rgw_socket_path"}),
6184
6185     Option("rgw_usage_log_flush_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6186     .set_default(1024)
6187     .set_description("Number of entries in usage log before flushing")
6188     .set_long_description(
6189         "This is the max number of entries that will be held in the usage log, before it "
6190         "will be flushed to the backend. Note that the usage log is periodically flushed, "
6191         "even if number of entries does not reach this threshold. A usage log entry "
6192         "corresponds to one or more operations on a single bucket.i")
6193     .add_see_also({"rgw_enable_usage_log", "rgw_usage_log_tick_interval"}),
6194
6195     Option("rgw_usage_log_tick_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6196     .set_default(30)
6197     .set_description("Number of seconds between usage log flush cycles")
6198     .set_long_description(
6199         "The number of seconds between consecutive usage log flushes. The usage log will "
6200         "also flush itself to the backend if the number of pending entries reaches a "
6201         "certain threshold.")
6202     .add_see_also({"rgw_enable_usage_log", "rgw_usage_log_flush_threshold"}),
6203
6204     Option("rgw_init_timeout", Option::TYPE_INT, Option::LEVEL_BASIC)
6205     .set_default(300)
6206     .set_description("Initialization timeout")
6207     .set_long_description(
6208         "The time length (in seconds) that RGW will allow for its initialization. RGW "
6209         "process will give up and quit if initialization is not complete after this amount "
6210         "of time."),
6211
6212     Option("rgw_mime_types_file", Option::TYPE_STR, Option::LEVEL_BASIC)
6213     .set_default("/etc/mime.types")
6214     .set_description("Path to local mime types file")
6215     .set_long_description(
6216         "The mime types file is needed in Swift when uploading an object. If object's "
6217         "content type is not specified, RGW will use data from this file to assign "
6218         "a content type to the object."),
6219
6220     Option("rgw_gc_max_objs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6221     .set_default(32)
6222     .set_description("Number of shards for garbage collector data")
6223     .set_long_description(
6224         "The number of garbage collector data shards, is the number of RADOS objects that "
6225         "RGW will use to store the garbage collection information on.")
6226     .add_see_also({"rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_processor_period", "rgw_gc_max_concurrent_io"}),
6227
6228     Option("rgw_gc_obj_min_wait", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6229     .set_default(2_hr)
6230     .set_description("Garbage collection object expiration time")
6231     .set_long_description(
6232        "The length of time (in seconds) that the RGW collector will wait before purging "
6233        "a deleted object's data. RGW will not remove object immediately, as object could "
6234        "still have readers. A mechanism exists to increase the object's expiration time "
6235        "when it's being read. The recommended value of its lower limit is 30 minutes")
6236     .add_see_also({"rgw_gc_max_objs", "rgw_gc_processor_max_time", "rgw_gc_processor_period", "rgw_gc_max_concurrent_io"}),
6237
6238     Option("rgw_gc_processor_max_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6239     .set_default(1_hr)
6240     .set_description("Length of time GC processor can lease shard")
6241     .set_long_description(
6242         "Garbage collection thread in RGW process holds a lease on its data shards. These "
6243         "objects contain the information about the objects that need to be removed. RGW "
6244         "takes a lease in order to prevent multiple RGW processes from handling the same "
6245         "objects concurrently. This time signifies that maximum amount of time (in seconds) that RGW "
6246         "is allowed to hold that lease. In the case where RGW goes down uncleanly, this "
6247         "is the amount of time where processing of that data shard will be blocked.")
6248     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_period", "rgw_gc_max_concurrent_io"}),
6249
6250     Option("rgw_gc_processor_period", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6251     .set_default(1_hr)
6252     .set_description("Garbage collector cycle run time")
6253     .set_long_description(
6254         "The amount of time between the start of consecutive runs of the garbage collector "
6255         "threads. If garbage collector runs takes more than this period, it will not wait "
6256         "before running again.")
6257     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_max_concurrent_io", "rgw_gc_max_trim_chunk"}),
6258
6259     Option("rgw_gc_max_concurrent_io", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6260     .set_default(10)
6261     .set_description("Max concurrent RADOS IO operations for garbage collection")
6262     .set_long_description(
6263         "The maximum number of concurrent IO operations that the RGW garbage collection "
6264         "thread will use when purging old data.")
6265     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_max_trim_chunk"}),
6266
6267     Option("rgw_gc_max_trim_chunk", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6268     .set_default(16)
6269     .set_description("Max number of keys to remove from garbage collector log in a single operation")
6270     .add_see_also({"rgw_gc_max_objs", "rgw_gc_obj_min_wait", "rgw_gc_processor_max_time", "rgw_gc_max_concurrent_io"}),
6271
6272     Option("rgw_gc_max_deferred_entries_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6273     .set_default(3072)
6274     .set_description("maximum allowed size of deferred entries in queue head for gc"),
6275
6276     Option("rgw_gc_max_queue_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6277     .set_default(134213632)
6278     .set_description("Maximum allowed queue size for gc")
6279     .set_long_description(
6280         "The maximum allowed size of each gc queue, and its value should not "
6281         "be greater than (osd_max_object_size - rgw_gc_max_deferred_entries_size - 1K).")
6282     .add_see_also({"osd_max_object_size", "rgw_gc_max_deferred_entries_size"}),
6283
6284     Option("rgw_gc_max_deferred", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6285     .set_default(50)
6286     .set_description("Number of maximum deferred data entries to be stored in queue for gc"),
6287
6288     Option("rgw_s3_success_create_obj_status", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6289     .set_default(0)
6290     .set_description("HTTP return code override for object creation")
6291     .set_long_description(
6292         "If not zero, this is the HTTP return code that will be returned on a successful S3 "
6293         "object creation."),
6294
6295     Option("rgw_resolve_cname", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6296     .set_default(false)
6297     .set_description("Support vanity domain names via CNAME")
6298     .set_long_description(
6299         "If true, RGW will query DNS when detecting that it's serving a request that was "
6300         "sent to a host in another domain. If a CNAME record is configured for that domain "
6301         "it will use it instead. This gives user to have the ability of creating a unique "
6302         "domain of their own to point at data in their bucket."),
6303
6304     Option("rgw_obj_stripe_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6305     .set_default(4_M)
6306     .set_description("RGW object stripe size")
6307     .set_long_description(
6308         "The size of an object stripe for RGW objects. This is the maximum size a backing "
6309         "RADOS object will have. RGW objects that are larger than this will span over "
6310         "multiple objects."),
6311
6312     Option("rgw_extended_http_attrs", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6313     .set_default("")
6314     .set_description("RGW support extended HTTP attrs")
6315     .set_long_description(
6316         "Add new set of attributes that could be set on an object. These extra attributes "
6317         "can be set through HTTP header fields when putting the objects. If set, these "
6318         "attributes will return as HTTP fields when doing GET/HEAD on the object."),
6319
6320     Option("rgw_exit_timeout_secs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6321     .set_default(120)
6322     .set_description("RGW shutdown timeout")
6323     .set_long_description("Number of seconds to wait for a process before exiting unconditionally."),
6324
6325     Option("rgw_get_obj_window_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6326     .set_default(16_M)
6327     .set_description("RGW object read window size")
6328     .set_long_description("The window size in bytes for a single object read request"),
6329
6330     Option("rgw_get_obj_max_req_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6331     .set_default(4_M)
6332     .set_description("RGW object read chunk size")
6333     .set_long_description(
6334         "The maximum request size of a single object read operation sent to RADOS"),
6335
6336     Option("rgw_relaxed_s3_bucket_names", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6337     .set_default(false)
6338     .set_description("RGW enable relaxed S3 bucket names")
6339     .set_long_description("RGW enable relaxed S3 bucket name rules for US region buckets."),
6340
6341     Option("rgw_defer_to_bucket_acls", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6342     .set_default("")
6343     .set_description("Bucket ACLs override object ACLs")
6344     .set_long_description(
6345         "If not empty, a string that selects that mode of operation. 'recurse' will use "
6346         "bucket's ACL for the authorizaton. 'full-control' will allow users that users "
6347         "that have full control permission on the bucket have access to the object."),
6348
6349     Option("rgw_list_buckets_max_chunk", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6350     .set_default(1000)
6351     .set_description("Max number of buckets to retrieve in a single listing operation")
6352     .set_long_description(
6353         "When RGW fetches lists of user's buckets from the backend, this is the max number "
6354         "of entries it will try to retrieve in a single operation. Note that the backend "
6355         "may choose to return a smaller number of entries."),
6356
6357     Option("rgw_md_log_max_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6358     .set_default(64)
6359     .set_description("RGW number of metadata log shards")
6360     .set_long_description(
6361         "The number of shards the RGW metadata log entries will reside in. This affects "
6362         "the metadata sync parallelism as a shard can only be processed by a single "
6363         "RGW at a time"),
6364
6365     Option("rgw_curl_wait_timeout_ms", Option::TYPE_INT, Option::LEVEL_DEV)
6366     .set_default(1000)
6367     .set_description(""),
6368
6369     Option("rgw_curl_low_speed_limit", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6370     .set_default(1024)
6371     .set_long_description(
6372         "It contains the average transfer speed in bytes per second that the "
6373         "transfer should be below during rgw_curl_low_speed_time seconds for libcurl "
6374         "to consider it to be too slow and abort. Set it zero to disable this."),
6375
6376     Option("rgw_curl_low_speed_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6377     .set_default(300)
6378     .set_long_description(
6379         "It contains the time in number seconds that the transfer speed should be below "
6380         "the rgw_curl_low_speed_limit for the library to consider it too slow and abort. "
6381         "Set it zero to disable this."),
6382
6383     Option("rgw_copy_obj_progress", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6384     .set_default(true)
6385     .set_description("Send progress report through copy operation")
6386     .set_long_description(
6387         "If true, RGW will send progress information when copy operation is executed. "),
6388
6389     Option("rgw_copy_obj_progress_every_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6390     .set_default(1_M)
6391     .set_description("Send copy-object progress info after these many bytes"),
6392
6393     Option("rgw_sync_obj_etag_verify", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6394     .set_default(false)
6395     .set_description("Verify if the object copied from remote is identical to its source")
6396     .set_long_description(
6397         "If true, this option computes the MD5 checksum of the data which is written at the "
6398         "destination and checks if it is identical to the ETAG stored in the source. "
6399         "It ensures integrity of the objects fetched from a remote server over HTTP including "
6400         "multisite sync."),
6401
6402     Option("rgw_obj_tombstone_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6403     .set_default(1000)
6404     .set_description("Max number of entries to keep in tombstone cache")
6405     .set_long_description(
6406         "The tombstone cache is used when doing a multi-zone data sync. RGW keeps "
6407         "there information about removed objects which is needed in order to prevent "
6408         "re-syncing of objects that were already removed."),
6409
6410     Option("rgw_data_log_window", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6411     .set_default(30)
6412     .set_description("Data log time window")
6413     .set_long_description(
6414         "The data log keeps information about buckets that have objectst that were "
6415         "modified within a specific timeframe. The sync process then knows which buckets "
6416         "are needed to be scanned for data sync."),
6417
6418     Option("rgw_data_log_changes_size", Option::TYPE_INT, Option::LEVEL_DEV)
6419     .set_default(1000)
6420     .set_description("Max size of pending changes in data log")
6421     .set_long_description(
6422         "RGW will trigger update to the data log if the number of pending entries reached "
6423         "this number."),
6424
6425     Option("rgw_data_log_num_shards", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6426     .set_default(128)
6427     .set_description("Number of data log shards")
6428     .set_long_description(
6429         "The number of shards the RGW data log entries will reside in. This affects the "
6430         "data sync parallelism as a shard can only be processed by a single RGW at a time."),
6431
6432     Option("rgw_data_log_obj_prefix", Option::TYPE_STR, Option::LEVEL_DEV)
6433     .set_default("data_log")
6434     .set_description(""),
6435
6436     Option("rgw_bucket_quota_ttl", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6437     .set_default(600)
6438     .set_description("Bucket quota stats cache TTL")
6439     .set_long_description(
6440         "Length of time for bucket stats to be cached within RGW instance."),
6441
6442     Option("rgw_bucket_quota_soft_threshold", Option::TYPE_FLOAT, Option::LEVEL_BASIC)
6443     .set_default(0.95)
6444     .set_description("RGW quota soft threshold")
6445     .set_long_description(
6446         "Threshold from which RGW doesn't rely on cached info for quota "
6447         "decisions. This is done for higher accuracy of the quota mechanism at "
6448         "cost of performance, when getting close to the quota limit. The value "
6449         "configured here is the ratio between the data usage to the max usage "
6450         "as specified by the quota."),
6451
6452     Option("rgw_bucket_quota_cache_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6453     .set_default(10000)
6454     .set_description("RGW quota stats cache size")
6455     .set_long_description(
6456         "Maximum number of entries in the quota stats cache."),
6457
6458     Option("rgw_bucket_default_quota_max_objects", Option::TYPE_INT, Option::LEVEL_BASIC)
6459     .set_default(-1)
6460     .set_description("Default quota for max objects in a bucket")
6461     .set_long_description(
6462         "The default quota configuration for max number of objects in a bucket. A "
6463         "negative number means 'unlimited'."),
6464
6465     Option("rgw_bucket_default_quota_max_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6466     .set_default(-1)
6467     .set_description("Default quota for total size in a bucket")
6468     .set_long_description(
6469         "The default quota configuration for total size of objects in a bucket. A "
6470         "negative number means 'unlimited'."),
6471
6472     Option("rgw_expose_bucket", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6473     .set_default(false)
6474     .set_description("Send Bucket HTTP header with the response")
6475     .set_long_description(
6476         "If true, RGW will send a Bucket HTTP header with the responses. The header will "
6477         "contain the name of the bucket the operation happened on."),
6478
6479     Option("rgw_frontends", Option::TYPE_STR, Option::LEVEL_BASIC)
6480     .set_default("beast port=7480")
6481     .set_description("RGW frontends configuration")
6482     .set_long_description(
6483         "A comma delimited list of frontends configuration. Each configuration contains "
6484         "the type of the frontend followed by an optional space delimited set of "
6485         "key=value config parameters."),
6486
6487     Option("rgw_frontend_defaults", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6488     .set_default("beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key")
6489     .set_description("RGW frontends default configuration")
6490     .set_long_description(
6491         "A comma delimited list of default frontends configuration."),
6492
6493     Option("rgw_user_quota_bucket_sync_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6494     .set_default(180)
6495     .set_description("User quota bucket sync interval")
6496     .set_long_description(
6497         "Time period for accumulating modified buckets before syncing these stats."),
6498
6499     Option("rgw_user_quota_sync_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6500     .set_default(1_day)
6501     .set_description("User quota sync interval")
6502     .set_long_description(
6503         "Time period for accumulating modified buckets before syncing entire user stats."),
6504
6505     Option("rgw_user_quota_sync_idle_users", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6506     .set_default(false)
6507     .set_description("Should sync idle users quota")
6508     .set_long_description(
6509         "Whether stats for idle users be fully synced."),
6510
6511     Option("rgw_user_quota_sync_wait_time", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6512     .set_default(1_day)
6513     .set_description("User quota full-sync wait time")
6514     .set_long_description(
6515         "Minimum time between two full stats sync for non-idle users."),
6516
6517     Option("rgw_user_default_quota_max_objects", Option::TYPE_INT, Option::LEVEL_BASIC)
6518     .set_default(-1)
6519     .set_description("User quota max objects")
6520     .set_long_description(
6521         "The default quota configuration for total number of objects for a single user. A "
6522         "negative number means 'unlimited'."),
6523
6524     Option("rgw_user_default_quota_max_size", Option::TYPE_INT, Option::LEVEL_BASIC)
6525     .set_default(-1)
6526     .set_description("User quota max size")
6527     .set_long_description(
6528         "The default quota configuration for total size of objects for a single user. A "
6529         "negative number means 'unlimited'."),
6530
6531     Option("rgw_multipart_min_part_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6532     .set_default(5_M)
6533     .set_description("Minimum S3 multipart-upload part size")
6534     .set_long_description(
6535         "When doing a multipart upload, each part (other than the last part) should be "
6536         "at least this size."),
6537
6538     Option("rgw_multipart_part_upload_limit", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6539     .set_default(10000)
6540     .set_description("Max number of parts in multipart upload"),
6541
6542     Option("rgw_max_slo_entries", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6543     .set_default(1000)
6544     .set_description("Max number of entries in Swift Static Large Object manifest"),
6545
6546     Option("rgw_olh_pending_timeout_sec", Option::TYPE_INT, Option::LEVEL_DEV)
6547     .set_default(1_hr)
6548     .set_description("Max time for pending OLH change to complete")
6549     .set_long_description(
6550         "OLH is a versioned object's logical head. Operations on it are journaled and "
6551         "as pending before completion. If an operation doesn't complete with this amount "
6552         "of seconds, we remove the operation from the journal."),
6553
6554     Option("rgw_user_max_buckets", Option::TYPE_INT, Option::LEVEL_BASIC)
6555     .set_default(1000)
6556     .set_description("Max number of buckets per user")
6557     .set_long_description(
6558       "A user can create at most this number of buckets. Zero means "
6559       "no limit; a negative value means users cannot create any new "
6560       "buckets, although users will retain buckets already created."),
6561
6562     Option("rgw_objexp_gc_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6563     .set_default(10_min)
6564     .set_description("Swift objects expirer garbage collector interval"),
6565
6566     Option("rgw_objexp_hints_num_shards", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6567     .set_default(127)
6568     .set_description("Number of object expirer data shards")
6569     .set_long_description(
6570         "The number of shards the (Swift) object expirer will store its data on."),
6571
6572     Option("rgw_objexp_chunk_size", Option::TYPE_UINT, Option::LEVEL_DEV)
6573     .set_default(100)
6574     .set_description(""),
6575
6576     Option("rgw_enable_static_website", Option::TYPE_BOOL, Option::LEVEL_BASIC)
6577     .set_default(false)
6578     .set_description("Enable static website APIs")
6579     .set_long_description(
6580         "This configurable controls whether RGW handles the website control APIs. RGW can "
6581         "server static websites if s3website hostnames are configured, and unrelated to "
6582         "this configurable."),
6583
6584      Option("rgw_user_unique_email", Option::TYPE_BOOL, Option::LEVEL_BASIC)
6585     .set_default(true)
6586     .set_description("Require local RGW users to have unique email addresses")
6587     .set_long_description(
6588         "Enforce builtin user accounts to have unique email addresses.  This "
6589         "setting is historical.  In future, non-enforcement of email address "
6590         "uniqueness is likely to become the default."),
6591
6592     Option("rgw_log_http_headers", Option::TYPE_STR, Option::LEVEL_BASIC)
6593     .set_default("")
6594     .set_description("List of HTTP headers to log")
6595     .set_long_description(
6596         "A comma delimited list of HTTP headers to log when seen, ignores case (e.g., "
6597         "http_x_forwarded_for)."),
6598
6599     Option("rgw_num_async_rados_threads", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6600     .set_default(32)
6601     .set_description("Number of concurrent RADOS operations in multisite sync")
6602     .set_long_description(
6603         "The number of concurrent RADOS IO operations that will be triggered for handling "
6604         "multisite sync operations. This includes control related work, and not the actual "
6605         "sync operations."),
6606
6607     Option("rgw_md_notify_interval_msec", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6608     .set_default(200)
6609     .set_description("Length of time to aggregate metadata changes")
6610     .set_long_description(
6611         "Length of time (in milliseconds) in which the master zone aggregates all the "
6612         "metadata changes that occurred, before sending notifications to all the other "
6613         "zones."),
6614
6615     Option("rgw_run_sync_thread", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6616     .set_default(true)
6617     .set_description("Should run sync thread"),
6618
6619     Option("rgw_sync_lease_period", Option::TYPE_INT, Option::LEVEL_DEV)
6620     .set_default(120)
6621     .set_description(""),
6622
6623     Option("rgw_sync_log_trim_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6624     .set_default(1200)
6625     .set_description("Sync log trim interval")
6626     .set_long_description(
6627         "Time in seconds between attempts to trim sync logs."),
6628
6629     Option("rgw_sync_log_trim_max_buckets", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6630     .set_default(16)
6631     .set_description("Maximum number of buckets to trim per interval")
6632     .set_long_description("The maximum number of buckets to consider for bucket index log trimming each trim interval, regardless of the number of bucket index shards. Priority is given to buckets with the most sync activity over the last trim interval.")
6633     .add_see_also("rgw_sync_log_trim_interval")
6634     .add_see_also("rgw_sync_log_trim_min_cold_buckets")
6635     .add_see_also("rgw_sync_log_trim_concurrent_buckets"),
6636
6637     Option("rgw_sync_log_trim_min_cold_buckets", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6638     .set_default(4)
6639     .set_description("Minimum number of cold buckets to trim per interval")
6640     .set_long_description("Of the `rgw_sync_log_trim_max_buckets` selected for bucket index log trimming each trim interval, at least this many of them must be 'cold' buckets. These buckets are selected in order from the list of all bucket instances, to guarantee that all buckets will be visited eventually.")
6641     .add_see_also("rgw_sync_log_trim_interval")
6642     .add_see_also("rgw_sync_log_trim_max_buckets")
6643     .add_see_also("rgw_sync_log_trim_concurrent_buckets"),
6644
6645     Option("rgw_sync_log_trim_concurrent_buckets", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6646     .set_default(4)
6647     .set_description("Maximum number of buckets to trim in parallel")
6648     .add_see_also("rgw_sync_log_trim_interval")
6649     .add_see_also("rgw_sync_log_trim_max_buckets")
6650     .add_see_also("rgw_sync_log_trim_min_cold_buckets"),
6651
6652     Option("rgw_sync_data_inject_err_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
6653     .set_default(0)
6654     .set_description(""),
6655
6656     Option("rgw_sync_meta_inject_err_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
6657     .set_default(0)
6658     .set_description(""),
6659
6660     Option("rgw_sync_trace_history_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6661     .set_default(4096)
6662     .set_description("Sync trace history size")
6663     .set_long_description(
6664       "Maximum number of complete sync trace entries to keep."),
6665
6666     Option("rgw_sync_trace_per_node_log_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6667     .set_default(32)
6668     .set_description("Sync trace per-node log size")
6669     .set_long_description(
6670         "The number of log entries to keep per sync-trace node."),
6671
6672     Option("rgw_sync_trace_servicemap_update_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6673     .set_default(10)
6674     .set_description("Sync-trace service-map update interval")
6675     .set_long_description(
6676         "Number of seconds between service-map updates of sync-trace events."),
6677
6678     Option("rgw_period_push_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
6679     .set_default(2)
6680     .set_description("Period push interval")
6681     .set_long_description(
6682         "Number of seconds to wait before retrying 'period push' operation."),
6683
6684     Option("rgw_period_push_interval_max", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
6685     .set_default(30)
6686     .set_description("Period push maximum interval")
6687     .set_long_description(
6688         "The max number of seconds to wait before retrying 'period push' after exponential "
6689         "backoff."),
6690
6691     Option("rgw_safe_max_objects_per_shard", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6692     .set_default(100*1024)
6693     .set_description("Safe number of objects per shard")
6694     .set_long_description(
6695         "This is the max number of objects per bucket index shard that RGW considers "
6696         "safe. RGW will warn if it identifies a bucket where its per-shard count is "
6697         "higher than a percentage of this number.")
6698     .add_see_also("rgw_shard_warning_threshold"),
6699
6700     Option("rgw_shard_warning_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
6701     .set_default(90)
6702     .set_description("Warn about max objects per shard")
6703     .set_long_description(
6704         "Warn if number of objects per shard in a specific bucket passed this percentage "
6705         "of the safe number.")
6706     .add_see_also("rgw_safe_max_objects_per_shard"),
6707
6708     Option("rgw_swift_versioning_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6709     .set_default(false)
6710     .set_description("Enable Swift versioning"),
6711
6712     Option("rgw_swift_custom_header", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6713     .set_default("")
6714     .set_description("Enable swift custom header")
6715     .set_long_description(
6716         "If not empty, specifies a name of HTTP header that can include custom data. When "
6717         "uploading an object, if this header is passed RGW will store this header info "
6718         "and it will be available when listing the bucket."),
6719
6720     Option("rgw_swift_need_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6721     .set_default(true)
6722     .set_description("Enable stats on bucket listing in Swift"),
6723
6724     Option("rgw_reshard_num_logs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6725     .set_default(16)
6726     .set_min(1)
6727     .set_description("")
6728     .add_service("rgw"),
6729
6730     Option("rgw_reshard_bucket_lock_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6731     .set_default(360)
6732     .set_min(30)
6733     .set_description("Number of seconds the timeout on the reshard locks (bucket reshard lock and reshard log lock) are set to. As a reshard proceeds these locks can be renewed/extended. If too short, reshards cannot complete and will fail, causing a future reshard attempt. If too long a hung or crashed reshard attempt will keep the bucket locked for an extended period, not allowing RGW to detect the failed reshard attempt and recover.")
6734     .add_tag("performance")
6735     .add_service("rgw"),
6736
6737     Option("rgw_reshard_batch_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6738     .set_default(64)
6739     .set_min(8)
6740     .set_description("Number of reshard entries to batch together before sending the operations to the CLS back-end")
6741     .add_tag("performance")
6742     .add_service("rgw"),
6743
6744     Option("rgw_reshard_max_aio", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6745     .set_default(128)
6746     .set_min(16)
6747     .set_description("Maximum number of outstanding asynchronous I/O operations to allow at a time during resharding")
6748     .add_tag("performance")
6749     .add_service("rgw"),
6750
6751     Option("rgw_trust_forwarded_https", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6752     .set_default(false)
6753     .set_description("Trust Forwarded and X-Forwarded-Proto headers")
6754     .set_long_description(
6755         "When a proxy in front of radosgw is used for ssl termination, radosgw "
6756         "does not know whether incoming http connections are secure. Enable "
6757         "this option to trust the Forwarded and X-Forwarded-Proto headers sent "
6758         "by the proxy when determining whether the connection is secure. This "
6759         "is required for some features, such as server side encryption.")
6760     .add_see_also("rgw_crypt_require_ssl"),
6761
6762     Option("rgw_crypt_require_ssl", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6763     .set_default(true)
6764     .set_description("Requests including encryption key headers must be sent over ssl"),
6765
6766     Option("rgw_crypt_default_encryption_key", Option::TYPE_STR, Option::LEVEL_DEV)
6767     .set_default("")
6768     .set_description(""),
6769
6770     Option("rgw_crypt_s3_kms_backend", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6771     .set_default("barbican")
6772     .set_enum_allowed({"barbican", "vault", "testing"})
6773     .set_description(
6774         "Where the SSE-KMS encryption keys are stored. Supported KMS "
6775         "systems are OpenStack Barbican ('barbican', the default) and HashiCorp "
6776         "Vault ('vault')."),
6777
6778     Option("rgw_crypt_s3_kms_encryption_keys", Option::TYPE_STR, Option::LEVEL_DEV)
6779     .set_default("")
6780     .set_description(""),
6781
6782     Option("rgw_crypt_vault_auth", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6783     .set_default("token")
6784     .set_enum_allowed({"token", "agent"})
6785     .set_description(
6786         "Type of authentication method to be used with Vault. ")
6787     .add_see_also({
6788         "rgw_crypt_s3_kms_backend",
6789         "rgw_crypt_vault_addr",
6790         "rgw_crypt_vault_token_file"}),
6791
6792     Option("rgw_crypt_vault_token_file", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6793     .set_default("")
6794     .set_description(
6795         "If authentication method is 'token', provide a path to the token file, "
6796         "which for security reasons should readable only by Rados Gateway.")
6797     .add_see_also({
6798       "rgw_crypt_s3_kms_backend",
6799       "rgw_crypt_vault_auth",
6800       "rgw_crypt_vault_addr"}),
6801
6802     Option("rgw_crypt_vault_addr", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6803     .set_default("")
6804     .set_description("Vault server base address.")
6805     .add_see_also({
6806       "rgw_crypt_s3_kms_backend",
6807       "rgw_crypt_vault_auth",
6808       "rgw_crypt_vault_prefix"}),
6809
6810     Option("rgw_crypt_vault_prefix", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6811     .set_default("")
6812     .set_description("Vault secret URL prefix, which can be used to restrict "
6813                      "access to a particular subset of the Vault secret space.")
6814     .add_see_also({
6815       "rgw_crypt_s3_kms_backend",
6816       "rgw_crypt_vault_addr",
6817       "rgw_crypt_vault_auth"}),
6818
6819
6820     Option("rgw_crypt_vault_secret_engine", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6821     .set_enum_allowed({"kv", "transit"})
6822     .set_default("transit")
6823     .set_description(
6824         "Vault Secret Engine to be used to retrieve encryption keys.")
6825     .add_see_also({
6826       "rgw_crypt_s3_kms_backend",
6827       "rgw_crypt_vault_auth",
6828       "rgw_crypt_vault_addr"}),
6829
6830     Option("rgw_crypt_vault_namespace", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6831     .set_default("")
6832     .set_description("Vault Namespace to be used to select your tenant")
6833     .add_see_also({
6834       "rgw_crypt_s3_kms_backend",
6835       "rgw_crypt_vault_auth",
6836       "rgw_crypt_vault_addr"}),
6837
6838     Option("rgw_crypt_suppress_logs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6839     .set_default(true)
6840     .set_description("Suppress logs that might print client key"),
6841
6842     Option("rgw_list_bucket_min_readahead", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6843     .set_default(1000)
6844     .set_description("Minimum number of entries to request from rados for bucket listing"),
6845
6846     Option("rgw_rest_getusage_op_compat", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6847     .set_default(false)
6848     .set_description("REST GetUsage request backward compatibility"),
6849
6850     Option("rgw_torrent_flag", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6851     .set_default(false)
6852     .set_description("When true, uploaded objects will calculate and store "
6853                      "a SHA256 hash of object data so the object can be "
6854                      "retrieved as a torrent file"),
6855
6856     Option("rgw_torrent_tracker", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6857     .set_default("")
6858     .set_description("Torrent field announce and announce list"),
6859
6860     Option("rgw_torrent_createby", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6861     .set_default("")
6862     .set_description("torrent field created by"),
6863
6864     Option("rgw_torrent_comment", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6865     .set_default("")
6866     .set_description("Torrent field comment"),
6867
6868     Option("rgw_torrent_encoding", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6869     .set_default("")
6870     .set_description("torrent field encoding"),
6871
6872     Option("rgw_data_notify_interval_msec", Option::TYPE_INT, Option::LEVEL_ADVANCED)
6873     .set_default(200)
6874     .set_description("data changes notification interval to followers"),
6875
6876     Option("rgw_torrent_origin", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6877     .set_default("")
6878     .set_description("Torrent origin"),
6879
6880     Option("rgw_torrent_sha_unit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
6881     .set_default(512*1024)
6882     .set_description(""),
6883
6884     Option("rgw_dynamic_resharding", Option::TYPE_BOOL, Option::LEVEL_BASIC)
6885     .set_default(true)
6886     .set_description("Enable dynamic resharding")
6887     .set_long_description(
6888         "If true, RGW will dynamicall increase the number of shards in buckets that have "
6889         "a high number of objects per shard.")
6890     .add_see_also("rgw_max_objs_per_shard")
6891     .add_see_also("rgw_max_dynamic_shards"),
6892
6893     Option("rgw_max_objs_per_shard", Option::TYPE_UINT, Option::LEVEL_BASIC)
6894     .set_default(100000)
6895     .set_description("Max objects per shard for dynamic resharding")
6896     .set_long_description(
6897         "This is the max number of objects per bucket index shard that RGW will "
6898         "allow with dynamic resharding. RGW will trigger an automatic reshard operation "
6899         "on the bucket if it exceeds this number.")
6900     .add_see_also("rgw_dynamic_resharding")
6901     .add_see_also("rgw_max_dynamic_shards"),
6902
6903     Option("rgw_max_dynamic_shards", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6904     .set_default(1999)
6905     .set_min(1)
6906     .set_description("Max shards that dynamic resharding can create")
6907     .set_long_description(
6908         "This is the maximum number of bucket index shards that dynamic "
6909         "sharding is able to create on its own. This does not limit user "
6910         "requested resharding. Ideally this value is a prime number.")
6911     .add_see_also("rgw_dynamic_resharding")
6912     .add_see_also("rgw_max_objs_per_shard"),
6913
6914     Option("rgw_reshard_thread_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6915     .set_default(10_min)
6916     .set_min(10)
6917     .set_description("Number of seconds between processing of reshard log entries"),
6918
6919     Option("rgw_cache_expiry_interval", Option::TYPE_UINT,
6920            Option::LEVEL_ADVANCED)
6921     .set_default(15_min)
6922     .set_description("Number of seconds before entries in the cache are "
6923                      "assumed stale and re-fetched. Zero is never.")
6924     .add_tag("performance")
6925     .add_service("rgw")
6926     .set_long_description("The Rados Gateway stores metadata and objects in "
6927                           "an internal cache. This should be kept consistent "
6928                           "by the OSD's relaying notify events between "
6929                           "multiple watching RGW processes. In the event "
6930                           "that this notification protocol fails, bounding "
6931                           "the length of time that any data in the cache will "
6932                           "be assumed valid will ensure that any RGW instance "
6933                           "that falls out of sync will eventually recover. "
6934                           "This seems to be an issue mostly for large numbers "
6935                           "of RGW instances under heavy use. If you would like "
6936                           "to turn off cache expiry, set this value to zero."),
6937
6938     Option("rgw_inject_notify_timeout_probability", Option::TYPE_FLOAT,
6939            Option::LEVEL_DEV)
6940     .set_default(0)
6941     .add_tag("fault injection")
6942     .add_tag("testing")
6943     .add_service("rgw")
6944     .set_min_max(0.0, 1.0)
6945     .set_description("Likelihood of ignoring a notify")
6946     .set_long_description("This is the probability that the RGW cache will "
6947                           "ignore a cache notify message. It exists to help "
6948                           "with the development and testing of cache "
6949                           "consistency and recovery improvements. Please "
6950                           "do not set it in a production cluster, as it "
6951                           "actively causes failures. Set this to a floating "
6952                           "point value between 0 and 1."),
6953     Option("rgw_max_notify_retries", Option::TYPE_UINT,
6954            Option::LEVEL_ADVANCED)
6955     .set_default(3)
6956     .add_tag("error recovery")
6957     .add_service("rgw")
6958     .set_description("Number of attempts to notify peers before giving up.")
6959     .set_long_description("The number of times we will attempt to update "
6960                           "a peer's cache in the event of error before giving "
6961                           "up. This is unlikely to be an issue unless your "
6962                           "cluster is very heavily loaded. Beware that "
6963                           "increasing this value may cause some operations to "
6964                           "take longer in exceptional cases and thus may, "
6965                           "rarely, cause clients to time out."),
6966     Option("rgw_sts_entry", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6967     .set_default("sts")
6968     .set_description("STS URL prefix")
6969     .set_long_description("URL path prefix for internal STS requests."),
6970
6971     Option("rgw_sts_key", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6972     .set_default("sts")
6973     .set_description("STS Key")
6974     .set_long_description("Key used for encrypting/ decrypting session token."),
6975
6976     Option("rgw_s3_auth_use_sts", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
6977     .set_default(false)
6978     .set_description("Should S3 authentication use STS."),
6979
6980     Option("rgw_sts_max_session_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
6981     .set_default(43200)
6982     .set_description("Session token max duration")
6983     .set_long_description("Max duration in seconds for which the session token is valid."),
6984
6985     Option("rgw_max_listing_results", Option::TYPE_UINT,
6986            Option::LEVEL_ADVANCED)
6987     .set_default(1000)
6988     .set_min_max(1, 100000)
6989     .add_service("rgw")
6990     .set_description("Upper bound on results in listing operations, ListBucket max-keys")
6991     .set_long_description("This caps the maximum permitted value for listing-like operations in RGW S3. "
6992                           "Affects ListBucket(max-keys), "
6993                           "ListBucketVersions(max-keys), "
6994                           "ListBucketMultipartUploads(max-uploads), "
6995                           "ListMultipartUploadParts(max-parts)"),
6996
6997     Option("rgw_sts_token_introspection_url", Option::TYPE_STR, Option::LEVEL_ADVANCED)
6998     .set_default("")
6999     .set_description("STS Web Token introspection URL")
7000     .set_long_description("URL for introspecting an STS Web Token."),
7001
7002     Option("rgw_sts_client_id", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7003     .set_default("")
7004     .set_description("Client Id")
7005     .set_long_description("Client Id needed for introspecting a Web Token."),
7006
7007     Option("rgw_sts_client_secret", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7008     .set_default("")
7009     .set_description("Client Secret")
7010     .set_long_description("Client Secret needed for introspecting a Web Token."),
7011
7012     Option("rgw_max_concurrent_requests", Option::TYPE_INT, Option::LEVEL_BASIC)
7013     .set_default(1024)
7014     .set_description("Maximum number of concurrent HTTP requests.")
7015     .set_long_description(
7016         "Maximum number of concurrent HTTP requests that the beast frontend "
7017         "will process. Tuning this can help to limit memory usage under heavy "
7018         "load.")
7019     .add_tag("performance")
7020     .add_see_also("rgw_frontends"),
7021
7022     Option("rgw_scheduler_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7023     .set_default("throttler")
7024     .set_description("Set the type of dmclock scheduler, defaults to throttler "
7025                      "Other valid values are dmclock which is experimental"),
7026
7027     Option("rgw_dmclock_admin_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7028     .set_default(100.0)
7029     .set_description("mclock reservation for admin requests")
7030     .add_see_also("rgw_dmclock_admin_wgt")
7031     .add_see_also("rgw_dmclock_admin_lim"),
7032
7033     Option("rgw_dmclock_admin_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7034     .set_default(100.0)
7035     .set_description("mclock weight for admin requests")
7036     .add_see_also("rgw_dmclock_admin_res")
7037     .add_see_also("rgw_dmclock_admin_lim"),
7038
7039     Option("rgw_dmclock_admin_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7040     .set_default(0.0)
7041     .set_description("mclock limit for admin requests")
7042     .add_see_also("rgw_dmclock_admin_res")
7043     .add_see_also("rgw_dmclock_admin_wgt"),
7044
7045     Option("rgw_dmclock_auth_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7046     .set_default(200.0)
7047     .set_description("mclock reservation for object data requests")
7048     .add_see_also("rgw_dmclock_auth_wgt")
7049     .add_see_also("rgw_dmclock_auth_lim"),
7050
7051     Option("rgw_dmclock_auth_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7052     .set_default(100.0)
7053     .set_description("mclock weight for object data requests")
7054     .add_see_also("rgw_dmclock_auth_res")
7055     .add_see_also("rgw_dmclock_auth_lim"),
7056
7057     Option("rgw_dmclock_auth_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7058     .set_default(0.0)
7059     .set_description("mclock limit for object data requests")
7060     .add_see_also("rgw_dmclock_auth_res")
7061     .add_see_also("rgw_dmclock_auth_wgt"),
7062
7063     Option("rgw_dmclock_data_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7064     .set_default(500.0)
7065     .set_description("mclock reservation for object data requests")
7066     .add_see_also("rgw_dmclock_data_wgt")
7067     .add_see_also("rgw_dmclock_data_lim"),
7068
7069     Option("rgw_dmclock_data_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7070     .set_default(500.0)
7071     .set_description("mclock weight for object data requests")
7072     .add_see_also("rgw_dmclock_data_res")
7073     .add_see_also("rgw_dmclock_data_lim"),
7074
7075     Option("rgw_dmclock_data_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7076     .set_default(0.0)
7077     .set_description("mclock limit for object data requests")
7078     .add_see_also("rgw_dmclock_data_res")
7079     .add_see_also("rgw_dmclock_data_wgt"),
7080
7081     Option("rgw_dmclock_metadata_res", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7082     .set_default(500.0)
7083     .set_description("mclock reservation for metadata requests")
7084     .add_see_also("rgw_dmclock_metadata_wgt")
7085     .add_see_also("rgw_dmclock_metadata_lim"),
7086
7087     Option("rgw_dmclock_metadata_wgt", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7088     .set_default(500.0)
7089     .set_description("mclock weight for metadata requests")
7090     .add_see_also("rgw_dmclock_metadata_res")
7091     .add_see_also("rgw_dmclock_metadata_lim"),
7092
7093     Option("rgw_dmclock_metadata_lim", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7094     .set_default(0.0)
7095     .set_description("mclock limit for metadata requests")
7096     .add_see_also("rgw_dmclock_metadata_res")
7097     .add_see_also("rgw_dmclock_metadata_wgt"),
7098   });
7099 }
7100
7101 static std::vector<Option> get_rbd_options() {
7102   return std::vector<Option>({
7103     Option("rbd_default_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7104     .set_default("rbd")
7105     .set_description("default pool for storing new images")
7106     .set_validator([](std::string *value, std::string *error_message){
7107       std::regex pattern("^[^@/]+$");
7108       if (!std::regex_match (*value, pattern)) {
7109         *value = "rbd";
7110         *error_message = "invalid RBD default pool, resetting to 'rbd'";
7111       }
7112       return 0;
7113     }),
7114
7115     Option("rbd_default_data_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7116     .set_default("")
7117     .set_description("default pool for storing data blocks for new images")
7118     .set_validator([](std::string *value, std::string *error_message){
7119       std::regex pattern("^[^@/]*$");
7120       if (!std::regex_match (*value, pattern)) {
7121         *value = "";
7122         *error_message = "ignoring invalid RBD data pool";
7123       }
7124       return 0;
7125     }),
7126
7127     Option("rbd_default_features", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7128     .set_default("layering,exclusive-lock,object-map,fast-diff,deep-flatten")
7129     .set_description("default v2 image features for new images")
7130     .set_long_description(
7131         "RBD features are only applicable for v2 images. This setting accepts "
7132         "either an integer bitmask value or comma-delimited string of RBD "
7133         "feature names. This setting is always internally stored as an integer "
7134         "bitmask value. The mapping between feature bitmask value and feature "
7135         "name is as follows: +1 -> layering, +2 -> striping, "
7136         "+4 -> exclusive-lock, +8 -> object-map, +16 -> fast-diff, "
7137         "+32 -> deep-flatten, +64 -> journaling, +128 -> data-pool")
7138     .set_flag(Option::FLAG_RUNTIME)
7139     .set_validator([](std::string *value, std::string *error_message) {
7140         ostringstream ss;
7141         uint64_t features = librbd::rbd_features_from_string(*value, &ss);
7142         // Leave this in integer form to avoid breaking Cinder.  Someday
7143         // we would like to present this in string form instead...
7144         *value = stringify(features);
7145         if (ss.str().size()) {
7146           return -EINVAL;
7147         }
7148         return 0;
7149       }),
7150
7151     Option("rbd_op_threads", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7152     .set_default(1)
7153     .set_description("number of threads to utilize for internal processing"),
7154
7155     Option("rbd_op_thread_timeout", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7156     .set_default(60)
7157     .set_description("time in seconds for detecting a hung thread"),
7158
7159     Option("rbd_disable_zero_copy_writes", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7160     .set_default(true)
7161     .set_description("Disable the use of zero-copy writes to ensure unstable "
7162                      "writes from clients cannot cause a CRC mismatch"),
7163
7164     Option("rbd_non_blocking_aio", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7165     .set_default(true)
7166     .set_description("process AIO ops from a dispatch thread to prevent blocking"),
7167
7168     Option("rbd_cache", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7169     .set_default(true)
7170     .set_description("whether to enable caching (writeback unless rbd_cache_max_dirty is 0)"),
7171
7172     Option("rbd_cache_policy", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7173     .set_enum_allowed({"writethrough", "writeback", "writearound"})
7174     .set_default("writearound")
7175     .set_description("cache policy for handling writes."),
7176
7177     Option("rbd_cache_writethrough_until_flush", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7178     .set_default(true)
7179     .set_description("whether to make writeback caching writethrough until "
7180                      "flush is called, to be sure the user of librbd will send "
7181                      "flushes so that writeback is safe"),
7182
7183     Option("rbd_cache_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7184     .set_default(32_M)
7185     .set_description("cache size in bytes"),
7186
7187     Option("rbd_cache_max_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7188     .set_default(24_M)
7189     .set_description("dirty limit in bytes - set to 0 for write-through caching"),
7190
7191     Option("rbd_cache_target_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7192     .set_default(16_M)
7193     .set_description("target dirty limit in bytes"),
7194
7195     Option("rbd_cache_max_dirty_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7196     .set_default(1.0)
7197     .set_description("seconds in cache before writeback starts"),
7198
7199     Option("rbd_cache_max_dirty_object", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7200     .set_default(0)
7201     .set_description("dirty limit for objects - set to 0 for auto calculate from rbd_cache_size"),
7202
7203     Option("rbd_cache_block_writes_upfront", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7204     .set_default(false)
7205     .set_description("whether to block writes to the cache before the aio_write call completes"),
7206
7207     Option("rbd_parent_cache_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7208     .set_default(false)
7209     .set_description("whether to enable rbd shared ro cache"),
7210
7211     Option("rbd_concurrent_management_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7212     .set_default(10)
7213     .set_min(1)
7214     .set_description("how many operations can be in flight for a management operation like deleting or resizing an image"),
7215
7216     Option("rbd_balance_snap_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7217     .set_default(false)
7218     .set_description("distribute snap read requests to random OSD"),
7219
7220     Option("rbd_localize_snap_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7221     .set_default(false)
7222     .set_description("localize snap read requests to closest OSD"),
7223
7224     Option("rbd_balance_parent_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7225     .set_default(false)
7226     .set_description("distribute parent read requests to random OSD"),
7227
7228     Option("rbd_localize_parent_reads", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7229     .set_default(false)
7230     .set_description("localize parent requests to closest OSD"),
7231
7232     Option("rbd_sparse_read_threshold_bytes", Option::TYPE_SIZE,
7233            Option::LEVEL_ADVANCED)
7234     .set_default(64_K)
7235     .set_description("threshold for issuing a sparse-read")
7236     .set_long_description("minimum number of sequential bytes to read against "
7237                           "an object before issuing a sparse-read request to "
7238                           "the cluster. 0 implies it must be a full object read "
7239                           "to issue a sparse-read, 1 implies always use "
7240                           "sparse-read, and any value larger than the maximum "
7241                           "object size will disable sparse-read for all "
7242                           "requests"),
7243
7244     Option("rbd_readahead_trigger_requests", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7245     .set_default(10)
7246     .set_description("number of sequential requests necessary to trigger readahead"),
7247
7248     Option("rbd_readahead_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7249     .set_default(512_K)
7250     .set_description("set to 0 to disable readahead"),
7251
7252     Option("rbd_readahead_disable_after_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7253     .set_default(50_M)
7254     .set_description("how many bytes are read in total before readahead is disabled"),
7255
7256     Option("rbd_clone_copy_on_read", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7257     .set_default(false)
7258     .set_description("copy-up parent image blocks to clone upon read request"),
7259
7260     Option("rbd_blacklist_on_break_lock", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7261     .set_default(true)
7262     .set_description("whether to blacklist clients whose lock was broken"),
7263
7264     Option("rbd_blacklist_expire_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7265     .set_default(0)
7266     .set_description("number of seconds to blacklist - set to 0 for OSD default"),
7267
7268     Option("rbd_request_timed_out_seconds", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7269     .set_default(30)
7270     .set_description("number of seconds before maintenance request times out"),
7271
7272     Option("rbd_skip_partial_discard", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7273     .set_default(true)
7274     .set_description("skip discard (zero) of unaligned extents within an object"),
7275
7276     Option("rbd_discard_granularity_bytes", Option::TYPE_UINT,
7277            Option::LEVEL_ADVANCED)
7278     .set_default(64_K)
7279     .set_min_max(4_K, 32_M)
7280     .set_validator([](std::string *value, std::string *error_message){
7281         uint64_t f = strict_si_cast<uint64_t>(value->c_str(), error_message);
7282         if (!error_message->empty()) {
7283           return -EINVAL;
7284         } else if (!isp2(f)) {
7285           *error_message = "value must be a power of two";
7286           return -EINVAL;
7287         }
7288         return 0;
7289       })
7290     .set_description("minimum aligned size of discard operations"),
7291
7292     Option("rbd_enable_alloc_hint", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7293     .set_default(true)
7294     .set_description("when writing a object, it will issue a hint to osd backend to indicate the expected size object need"),
7295
7296     Option("rbd_compression_hint", Option::TYPE_STR, Option::LEVEL_BASIC)
7297     .set_enum_allowed({"none", "compressible", "incompressible"})
7298     .set_default("none")
7299     .set_description("Compression hint to send to the OSDs during writes")
7300     .set_flag(Option::FLAG_RUNTIME),
7301
7302     Option("rbd_read_from_replica_policy", Option::TYPE_STR, Option::LEVEL_BASIC)
7303     .set_enum_allowed({"default", "balance", "localize"})
7304     .set_default("default")
7305     .set_description("Read replica policy send to the OSDS during reads")
7306     .set_flag(Option::FLAG_RUNTIME),
7307
7308     Option("rbd_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7309     .set_default(false)
7310     .set_description("true if LTTng-UST tracepoints should be enabled"),
7311
7312     Option("rbd_blkin_trace_all", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7313     .set_default(false)
7314     .set_description("create a blkin trace for all RBD requests"),
7315
7316     Option("rbd_validate_pool", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7317     .set_default(true)
7318     .set_description("validate empty pools for RBD compatibility"),
7319
7320     Option("rbd_validate_names", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7321     .set_default(true)
7322     .set_description("validate new image names for RBD compatibility"),
7323
7324     Option("rbd_auto_exclusive_lock_until_manual_request", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7325     .set_default(true)
7326     .set_description("automatically acquire/release exclusive lock until it is explicitly requested"),
7327
7328     Option("rbd_move_to_trash_on_remove", Option::TYPE_BOOL, Option::LEVEL_BASIC)
7329     .set_default(false)
7330     .set_description("automatically move images to the trash when deleted"),
7331
7332     Option("rbd_move_to_trash_on_remove_expire_seconds", Option::TYPE_UINT, Option::LEVEL_BASIC)
7333     .set_default(0)
7334     .set_description("default number of seconds to protect deleted images in the trash"),
7335
7336     Option("rbd_move_parent_to_trash_on_remove", Option::TYPE_BOOL, Option::LEVEL_BASIC)
7337     .set_default(false)
7338     .set_description("move parent with clone format v2 children to the trash when deleted"),
7339
7340     Option("rbd_mirroring_resync_after_disconnect", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7341     .set_default(false)
7342     .set_description("automatically start image resync after mirroring is disconnected due to being laggy"),
7343
7344     Option("rbd_mirroring_delete_delay", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7345     .set_default(0)
7346     .set_description("time-delay in seconds for rbd-mirror delete propagation"),
7347
7348     Option("rbd_mirroring_replay_delay", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7349     .set_default(0)
7350     .set_description("time-delay in seconds for rbd-mirror asynchronous replication"),
7351
7352     Option("rbd_mirroring_max_mirroring_snapshots", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7353     .set_default(3)
7354     .set_min(3)
7355     .set_description("mirroring snapshots limit"),
7356
7357     Option("rbd_default_format", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7358     .set_default(2)
7359     .set_description("default image format for new images"),
7360
7361     Option("rbd_default_order", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7362     .set_default(22)
7363     .set_description("default order (data block object size) for new images"),
7364
7365     Option("rbd_default_stripe_count", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7366     .set_default(0)
7367     .set_description("default stripe count for new images"),
7368
7369     Option("rbd_default_stripe_unit", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7370     .set_default(0)
7371     .set_description("default stripe width for new images"),
7372
7373     Option("rbd_default_map_options", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7374     .set_default("")
7375     .set_description("default krbd map options"),
7376
7377     Option("rbd_default_clone_format", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7378     .set_enum_allowed({"1", "2", "auto"})
7379     .set_default("auto")
7380     .set_description("default internal format for handling clones")
7381     .set_long_description("This sets the internal format for tracking cloned "
7382                           "images. The setting of '1' requires attaching to "
7383                           "protected snapshots that cannot be removed until "
7384                           "the clone is removed/flattened. The setting of '2' "
7385                           "will allow clones to be attached to any snapshot "
7386                           "and permits removing in-use parent snapshots but "
7387                           "requires Mimic or later clients. The default "
7388                           "setting of 'auto' will use the v2 format if the "
7389                           "cluster is configured to require mimic or later "
7390                           "clients.")
7391     .set_flag(Option::FLAG_RUNTIME),
7392
7393     Option("rbd_journal_order", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7394     .set_min_max(12, 26)
7395     .set_default(24)
7396     .set_description("default order (object size) for journal data objects"),
7397
7398     Option("rbd_journal_splay_width", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7399     .set_default(4)
7400     .set_description("number of active journal objects"),
7401
7402     Option("rbd_journal_commit_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7403     .set_default(5)
7404     .set_description("commit time interval, seconds"),
7405
7406     Option("rbd_journal_object_writethrough_until_flush", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7407     .set_default(true)
7408     .set_description("when enabled, the rbd_journal_object_flush* configuration "
7409                      "options are ignored until the first flush so that batched "
7410                      "journal IO is known to be safe for consistency"),
7411
7412     Option("rbd_journal_object_flush_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7413     .set_default(0)
7414     .set_description("maximum number of pending commits per journal object"),
7415
7416     Option("rbd_journal_object_flush_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7417     .set_default(1_M)
7418     .set_description("maximum number of pending bytes per journal object"),
7419
7420     Option("rbd_journal_object_flush_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7421     .set_default(0)
7422     .set_description("maximum age (in seconds) for pending commits"),
7423
7424     Option("rbd_journal_object_max_in_flight_appends", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7425     .set_default(0)
7426     .set_description("maximum number of in-flight appends per journal object"),
7427
7428     Option("rbd_journal_pool", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7429     .set_default("")
7430     .set_description("pool for journal objects"),
7431
7432     Option("rbd_journal_max_payload_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7433     .set_default(16384)
7434     .set_description("maximum journal payload size before splitting"),
7435
7436     Option("rbd_journal_max_concurrent_object_sets", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7437     .set_default(0)
7438     .set_description("maximum number of object sets a journal client can be behind before it is automatically unregistered"),
7439
7440     Option("rbd_qos_iops_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7441     .set_default(0)
7442     .set_description("the desired limit of IO operations per second"),
7443
7444     Option("rbd_qos_bps_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7445     .set_default(0)
7446     .set_description("the desired limit of IO bytes per second"),
7447
7448     Option("rbd_qos_read_iops_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7449     .set_default(0)
7450     .set_description("the desired limit of read operations per second"),
7451
7452     Option("rbd_qos_write_iops_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7453     .set_default(0)
7454     .set_description("the desired limit of write operations per second"),
7455
7456     Option("rbd_qos_read_bps_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7457     .set_default(0)
7458     .set_description("the desired limit of read bytes per second"),
7459
7460     Option("rbd_qos_write_bps_limit", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7461     .set_default(0)
7462     .set_description("the desired limit of write bytes per second"),
7463
7464     Option("rbd_qos_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7465     .set_default(0)
7466     .set_description("the desired burst limit of IO operations"),
7467
7468     Option("rbd_qos_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7469     .set_default(0)
7470     .set_description("the desired burst limit of IO bytes"),
7471
7472     Option("rbd_qos_read_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7473     .set_default(0)
7474     .set_description("the desired burst limit of read operations"),
7475
7476     Option("rbd_qos_write_iops_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7477     .set_default(0)
7478     .set_description("the desired burst limit of write operations"),
7479
7480     Option("rbd_qos_read_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7481     .set_default(0)
7482     .set_description("the desired burst limit of read bytes"),
7483
7484     Option("rbd_qos_write_bps_burst", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7485     .set_default(0)
7486     .set_description("the desired burst limit of write bytes"),
7487
7488     Option("rbd_qos_schedule_tick_min", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7489     .set_default(50)
7490     .set_min(1)
7491     .set_description("minimum schedule tick (in milliseconds) for QoS"),
7492
7493     Option("rbd_discard_on_zeroed_write_same", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7494     .set_default(true)
7495     .set_description("discard data on zeroed write same instead of writing zero"),
7496
7497     Option("rbd_mtime_update_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7498     .set_default(60)
7499     .set_min(0)
7500     .set_description("RBD Image modify timestamp refresh interval. Set to 0 to disable modify timestamp update."),
7501
7502     Option("rbd_atime_update_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7503     .set_default(60)
7504     .set_min(0)
7505     .set_description("RBD Image access timestamp refresh interval. Set to 0 to disable access timestamp update."),
7506
7507     Option("rbd_io_scheduler", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7508     .set_default("simple")
7509     .set_enum_allowed({"none", "simple"})
7510     .set_description("RBD IO scheduler"),
7511
7512     Option("rbd_io_scheduler_simple_max_delay", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7513     .set_default(0)
7514     .set_min(0)
7515     .set_description("maximum io delay (in milliseconds) for simple io scheduler (if set to 0 dalay is calculated based on latency stats)"),
7516
7517     Option("rbd_rwl_enabled", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7518     .set_default(false)
7519     .set_description("enable persistent write back cache for this volume"),
7520
7521     Option("rbd_rwl_log_periodic_stats", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7522     .set_default(false)
7523     .set_description("emit periodic perf stats to debug log"),
7524
7525     Option("rbd_rwl_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7526     .set_default(1073741824)
7527     .set_min(1073741824)
7528     .set_description("size of the persistent write back cache for this volume"),
7529
7530     Option("rbd_rwl_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7531     .set_default("/tmp")
7532     .set_description("location of the persistent write back cache in a DAX-enabled filesystem on persistent memory"),
7533
7534     Option("rbd_config_pool_override_update_timestamp", Option::TYPE_UINT,
7535            Option::LEVEL_DEV)
7536     .set_default(0)
7537     .set_description("timestamp of last update to pool-level config overrides"),
7538
7539   });
7540 }
7541
7542 static std::vector<Option> get_rbd_mirror_options() {
7543   return std::vector<Option>({
7544     Option("rbd_mirror_journal_commit_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7545     .set_default(5)
7546     .set_description("commit time interval, seconds"),
7547
7548     Option("rbd_mirror_journal_poll_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7549     .set_default(5)
7550     .set_description("maximum age (in seconds) between successive journal polls"),
7551
7552     Option("rbd_mirror_sync_point_update_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7553     .set_default(30)
7554     .set_description("number of seconds between each update of the image sync point object number"),
7555
7556     Option("rbd_mirror_concurrent_image_syncs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7557     .set_default(5)
7558     .set_description("maximum number of image syncs in parallel"),
7559
7560     Option("rbd_mirror_pool_replayers_refresh_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7561     .set_default(30)
7562     .set_description("interval to refresh peers in rbd-mirror daemon"),
7563
7564     Option("rbd_mirror_concurrent_image_deletions", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7565     .set_default(1)
7566     .set_min(1)
7567     .set_description("maximum number of image deletions in parallel"),
7568
7569     Option("rbd_mirror_delete_retry_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7570     .set_default(30)
7571     .set_description("interval to check and retry the failed deletion requests"),
7572
7573     Option("rbd_mirror_image_state_check_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7574     .set_default(30)
7575     .set_min(1)
7576     .set_description("interval to get images from pool watcher and set sources in replayer"),
7577
7578     Option("rbd_mirror_leader_heartbeat_interval", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7579     .set_default(5)
7580     .set_min(1)
7581     .set_description("interval (in seconds) between mirror leader heartbeats"),
7582
7583     Option("rbd_mirror_leader_max_missed_heartbeats", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7584     .set_default(2)
7585     .set_description("number of missed heartbeats for non-lock owner to attempt to acquire lock"),
7586
7587     Option("rbd_mirror_leader_max_acquire_attempts_before_break", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7588     .set_default(3)
7589     .set_description("number of failed attempts to acquire lock after missing heartbeats before breaking lock"),
7590
7591     Option("rbd_mirror_image_policy_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7592     .set_default("simple")
7593     .set_enum_allowed({"none", "simple"})
7594     .set_description("active/active policy type for mapping images to instances"),
7595
7596     Option("rbd_mirror_image_policy_migration_throttle", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7597     .set_default(300)
7598     .set_description("number of seconds after which an image can be reshuffled (migrated) again"),
7599
7600     Option("rbd_mirror_image_policy_update_throttle_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7601     .set_default(1)
7602     .set_min(1)
7603     .set_description("interval (in seconds) to throttle images for mirror daemon peer updates"),
7604
7605     Option("rbd_mirror_image_policy_rebalance_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7606     .set_default(0)
7607     .set_description("number of seconds policy should be idle before trigerring reshuffle (rebalance) of images"),
7608
7609     Option("rbd_mirror_perf_stats_prio", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7610     .set_default((int64_t)PerfCountersBuilder::PRIO_USEFUL)
7611     .set_description("Priority level for mirror daemon replication perf counters")
7612     .set_long_description("The daemon will send perf counter data to the "
7613                           "manager daemon if the priority is not lower than "
7614                           "mgr_stats_threshold.")
7615     .set_min_max((int64_t)PerfCountersBuilder::PRIO_DEBUGONLY,
7616                  (int64_t)PerfCountersBuilder::PRIO_CRITICAL + 1),
7617
7618     Option("rbd_mirror_image_perf_stats_prio", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7619     .set_default((int64_t)PerfCountersBuilder::PRIO_USEFUL)
7620     .set_description("Priority level for mirror daemon per-image replication perf counters")
7621     .set_long_description("The daemon will send per-image perf counter data to the "
7622                           "manager daemon if the priority is not lower than "
7623                           "mgr_stats_threshold.")
7624     .set_min_max((int64_t)PerfCountersBuilder::PRIO_DEBUGONLY,
7625                  (int64_t)PerfCountersBuilder::PRIO_CRITICAL + 1),
7626
7627     Option("rbd_mirror_memory_autotune", Option::TYPE_BOOL, Option::LEVEL_DEV)
7628     .set_default(true)
7629     .add_see_also("rbd_mirror_memory_target")
7630     .set_description("Automatically tune the ratio of caches while respecting min values."),
7631
7632     Option("rbd_mirror_memory_target", Option::TYPE_SIZE, Option::LEVEL_BASIC)
7633     .set_default(4_G)
7634     .add_see_also("rbd_mirror_memory_autotune")
7635     .set_description("When tcmalloc and cache autotuning is enabled, try to keep this many bytes mapped in memory."),
7636
7637     Option("rbd_mirror_memory_base", Option::TYPE_SIZE, Option::LEVEL_DEV)
7638     .set_default(768_M)
7639     .add_see_also("rbd_mirror_memory_autotune")
7640     .set_description("When tcmalloc and cache autotuning is enabled, estimate the minimum amount of memory in bytes the rbd-mirror daemon will need."),
7641
7642     Option("rbd_mirror_memory_expected_fragmentation", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7643     .set_default(0.15)
7644     .set_min_max(0.0, 1.0)
7645     .add_see_also("rbd_mirror_memory_autotune")
7646     .set_description("When tcmalloc and cache autotuning is enabled, estimate the percent of memory fragmentation."),
7647
7648     Option("rbd_mirror_memory_cache_min", Option::TYPE_SIZE, Option::LEVEL_DEV)
7649     .set_default(128_M)
7650     .add_see_also("rbd_mirror_memory_autotune")
7651     .set_description("When tcmalloc and cache autotuning is enabled, set the minimum amount of memory used for cache."),
7652
7653     Option("rbd_mirror_memory_cache_resize_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7654     .set_default(5)
7655     .add_see_also("rbd_mirror_memory_autotune")
7656     .set_description("When tcmalloc and cache autotuning is enabled, wait this many seconds between resizing caches."),
7657
7658     Option("rbd_mirror_memory_cache_autotune_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7659     .set_default(30)
7660     .add_see_also("rbd_mirror_memory_autotune")
7661     .set_description("The number of seconds to wait between rebalances when cache autotune is enabled."),
7662   });
7663 }
7664
7665 static std::vector<Option> get_immutable_object_cache_options() {
7666   return std::vector<Option>({
7667     Option("immutable_object_cache_path", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7668     .set_default("/tmp/ceph_immutable_object_cache")
7669     .set_description("immutable object cache data dir"),
7670
7671     Option("immutable_object_cache_sock", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7672     .set_default("/var/run/ceph/immutable_object_cache_sock")
7673     .set_description("immutable object cache domain socket"),
7674
7675     Option("immutable_object_cache_max_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7676     .set_default(1_G)
7677     .set_description("max immutable object cache data size"),
7678
7679     Option("immutable_object_cache_max_inflight_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7680     .set_default(128)
7681     .set_description("max inflight promoting requests for immutable object cache daemon"),
7682
7683     Option("immutable_object_cache_client_dedicated_thread_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7684     .set_default(2)
7685     .set_description("immutable object cache client dedicated thread number"),
7686
7687     Option("immutable_object_cache_watermark", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7688     .set_default(0.1)
7689     .set_description("immutable object cache water mark"),
7690   });
7691 }
7692
7693 std::vector<Option> get_mds_options() {
7694   return std::vector<Option>({
7695     Option("mds_numa_node", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7696     .set_default(-1)
7697     .set_flag(Option::FLAG_STARTUP)
7698     .set_description("set mds's cpu affinity to a numa node (-1 for none)"),
7699
7700     Option("mds_data", Option::TYPE_STR, Option::LEVEL_ADVANCED)
7701     .set_default("/var/lib/ceph/mds/$cluster-$id")
7702     .set_flag(Option::FLAG_NO_MON_UPDATE)
7703     .set_description("path to MDS data and keyring"),
7704
7705     Option("mds_join_fs", Option::TYPE_STR, Option::LEVEL_BASIC)
7706     .set_default("")
7707     .set_description("file system MDS prefers to join")
7708     .set_long_description("This setting indicates which file system name the MDS should prefer to join (affinity). The monitors will try to have the MDS cluster safely reach a state where all MDS have strong affinity, even via failovers to a standby.")
7709     .set_flag(Option::FLAG_RUNTIME),
7710
7711     Option("mds_max_xattr_pairs_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7712     .set_default(64_K)
7713     .set_description("maximum aggregate size of extended attributes on a file"),
7714
7715     Option("mds_cache_trim_interval", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
7716     .set_default(1)
7717     .set_description("interval in seconds between cache trimming")
7718     .set_flag(Option::FLAG_RUNTIME),
7719
7720     Option("mds_cache_release_free_interval", Option::TYPE_SECS, Option::LEVEL_DEV)
7721     .set_default(10)
7722     .set_description("interval in seconds between heap releases")
7723     .set_flag(Option::FLAG_RUNTIME),
7724
7725     Option("mds_cache_memory_limit", Option::TYPE_SIZE, Option::LEVEL_BASIC)
7726     .set_default(4_G)
7727     .set_description("target maximum memory usage of MDS cache")
7728     .set_flag(Option::FLAG_RUNTIME)
7729     .set_long_description("This sets a target maximum memory usage of the MDS cache and is the primary tunable to limit the MDS memory usage. The MDS will try to stay under a reservation of this limit (by default 95%; 1 - mds_cache_reservation) by trimming unused metadata in its cache and recalling cached items in the client caches. It is possible for the MDS to exceed this limit due to slow recall from clients. The mds_health_cache_threshold (150%) sets a cache full threshold for when the MDS signals a cluster health warning."),
7730
7731     Option("mds_cache_reservation", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7732     .set_default(.05)
7733     .set_description("amount of memory to reserve for future cached objects")
7734     .set_flag(Option::FLAG_RUNTIME),
7735
7736     Option("mds_health_cache_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7737     .set_default(1.5)
7738     .set_description("threshold for cache size to generate health warning"),
7739
7740     Option("mds_cache_mid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7741     .set_default(.7)
7742     .set_description("midpoint for MDS cache LRU"),
7743
7744     Option("mds_cache_trim_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7745     .set_default(1)
7746     .set_description("decay rate for trimming MDS cache throttle")
7747     .set_flag(Option::FLAG_RUNTIME),
7748
7749     Option("mds_cache_trim_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7750     .set_default(64_K)
7751     .set_description("threshold for number of dentries that can be trimmed")
7752     .set_flag(Option::FLAG_RUNTIME),
7753
7754     Option("mds_max_file_recover", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7755     .set_default(32)
7756     .set_description("maximum number of files to recover file sizes in parallel"),
7757
7758     Option("mds_dir_max_commit_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7759     .set_default(10)
7760     .set_description("maximum size in megabytes for a RADOS write to a directory"),
7761
7762     Option("mds_dir_keys_per_op", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7763     .set_default(16384)
7764     .set_description("number of directory entries to read in one RADOS operation"),
7765
7766     Option("mds_decay_halflife", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7767     .set_default(5)
7768     .set_description("rate of decay for temperature counters on each directory for balancing"),
7769
7770     Option("mds_beacon_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7771     .set_default(4)
7772     .set_description("interval in seconds between MDS beacons to monitors"),
7773
7774     Option("mds_beacon_grace", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7775     .set_default(15)
7776     .set_description("tolerance in seconds for missed MDS beacons to monitors"),
7777
7778     Option("mds_heartbeat_grace", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7779     .set_default(15)
7780     .set_description("tolerance in seconds for MDS internal heartbeat"),
7781
7782     Option("mds_enforce_unique_name", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7783     .set_default(true)
7784     .set_description("require MDS name is unique in the cluster"),
7785
7786     Option("mds_session_blacklist_on_timeout", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7787     .set_default(true)
7788     .set_description("blacklist clients whose sessions have become stale"),
7789
7790     Option("mds_session_blacklist_on_evict", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7791     .set_default(true)
7792     .set_description("blacklist clients that have been evicted"),
7793
7794     Option("mds_sessionmap_keys_per_op", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7795     .set_default(1024)
7796     .set_description("number of omap keys to read from the SessionMap in one operation"),
7797
7798     Option("mds_recall_max_caps", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7799     .set_default(5000)
7800     .set_description("maximum number of caps to recall from client session in single recall")
7801     .set_flag(Option::FLAG_RUNTIME),
7802
7803     Option("mds_recall_max_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7804     .set_default(2.5)
7805     .set_description("decay rate for throttle on recalled caps on a session")
7806     .set_flag(Option::FLAG_RUNTIME),
7807
7808     Option("mds_recall_max_decay_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7809     .set_default(16_K)
7810     .set_description("decay threshold for throttle on recalled caps on a session")
7811     .set_flag(Option::FLAG_RUNTIME),
7812
7813     Option("mds_recall_global_max_decay_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7814     .set_default(64_K)
7815     .set_description("decay threshold for throttle on recalled caps globally")
7816     .set_flag(Option::FLAG_RUNTIME),
7817
7818     Option("mds_recall_warning_threshold", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7819     .set_default(32_K)
7820     .set_description("decay threshold for warning on slow session cap recall")
7821     .set_flag(Option::FLAG_RUNTIME),
7822
7823     Option("mds_recall_warning_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7824     .set_default(60.0)
7825     .set_description("decay rate for warning on slow session cap recall")
7826     .set_flag(Option::FLAG_RUNTIME),
7827
7828     Option("mds_session_cache_liveness_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7829     .add_see_also("mds_session_cache_liveness_magnitude")
7830     .set_default(5_min)
7831     .set_description("decay rate for session liveness leading to preemptive cap recall")
7832     .set_flag(Option::FLAG_RUNTIME)
7833     .set_long_description("This determines how long a session needs to be quiescent before the MDS begins preemptively recalling capabilities. The default of 5 minutes will cause 10 halvings of the decay counter after 1 hour, or 1/1024. The default magnitude of 10 (1^10 or 1024) is chosen so that the MDS considers a previously chatty session (approximately) to be quiescent after 1 hour."),
7834
7835     Option("mds_session_cache_liveness_magnitude", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7836     .add_see_also("mds_session_cache_liveness_decay_rate")
7837     .set_default(10)
7838     .set_description("decay magnitude for preemptively recalling caps on quiet client")
7839     .set_flag(Option::FLAG_RUNTIME)
7840     .set_long_description("This is the order of magnitude difference (in base 2) of the internal liveness decay counter and the number of capabilities the session holds. When this difference occurs, the MDS treats the session as quiescent and begins recalling capabilities."),
7841
7842     Option("mds_session_cap_acquisition_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7843     .set_default(10)
7844     .set_description("decay rate for session readdir caps leading to readdir throttle")
7845     .set_flag(Option::FLAG_RUNTIME)
7846     .set_long_description("The half-life for the session cap acquisition counter of caps acquired by readdir. This is used for throttling readdir requests from clients slow to release caps."),
7847
7848     Option("mds_session_cap_acquisition_throttle", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7849     .set_default(500000)
7850     .set_description("throttle point for cap acquisition decay counter"),
7851
7852     Option("mds_session_max_caps_throttle_ratio", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7853     .set_default(1.1)
7854     .set_description("ratio of mds_max_maps_per_client that client must exceed before readdir may be throttled by cap acquisition throttle"),
7855
7856     Option("mds_cap_acquisition_throttle_retry_request_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7857     .set_default(0.5)
7858     .set_description("timeout in seconds after which a client request is retried due to cap acquisition throttling"),
7859
7860     Option("mds_freeze_tree_timeout", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7861     .set_default(30)
7862     .set_description(""),
7863
7864     Option("mds_health_summarize_threshold", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7865     .set_default(10)
7866     .set_description("threshold of number of clients to summarize late client recall"),
7867
7868     Option("mds_reconnect_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7869     .set_default(45)
7870     .set_description("timeout in seconds to wait for clients to reconnect during MDS reconnect recovery state"),
7871
7872     Option("mds_tick_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7873     .set_default(5)
7874     .set_description("time in seconds between upkeep tasks"),
7875
7876     Option("mds_dirstat_min_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
7877     .set_default(1)
7878     .set_description(""),
7879
7880     Option("mds_scatter_nudge_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7881     .set_default(5)
7882     .set_description("minimum interval between scatter lock updates"),
7883
7884     Option("mds_client_prealloc_inos", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7885     .set_default(1000)
7886     .set_description("number of unused inodes to pre-allocate to clients for file creation"),
7887
7888     Option("mds_client_delegate_inos_pct", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7889     .set_default(50)
7890     .set_flag(Option::FLAG_RUNTIME)
7891     .set_description("percentage of preallocated inos to delegate to client"),
7892
7893     Option("mds_early_reply", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7894     .set_default(true)
7895     .set_description("additional reply to clients that metadata requests are complete but not yet durable"),
7896
7897     Option("mds_replay_unsafe_with_closed_session", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7898     .set_default(false)
7899     .set_flag(Option::FLAG_STARTUP)
7900     .set_description("complete all the replay request when mds is restarted, no matter the session is closed or not"),
7901
7902     Option("mds_default_dir_hash", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7903     .set_default(CEPH_STR_HASH_RJENKINS)
7904     .set_description("hash function to select directory fragment for dentry name"),
7905
7906     Option("mds_log_pause", Option::TYPE_BOOL, Option::LEVEL_DEV)
7907     .set_default(false)
7908     .set_description(""),
7909
7910     Option("mds_log_skip_corrupt_events", Option::TYPE_BOOL, Option::LEVEL_DEV)
7911     .set_default(false)
7912     .set_description(""),
7913
7914     Option("mds_log_max_events", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7915     .set_default(-1)
7916     .set_description("maximum number of events in the MDS journal (-1 is unlimited)"),
7917
7918     Option("mds_log_events_per_segment", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7919     .set_default(1024)
7920     .set_description("maximum number of events in an MDS journal segment"),
7921
7922     Option("mds_log_segment_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
7923     .set_default(0)
7924     .set_description("size in bytes of each MDS log segment"),
7925
7926     Option("mds_log_max_segments", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
7927     .set_default(128)
7928     .set_description("maximum number of segments which may be untrimmed"),
7929
7930     Option("mds_log_warn_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7931     .set_default(2.0)
7932     .set_min(1.0)
7933     .set_flag(Option::FLAG_RUNTIME)
7934     .set_description("trigger MDS_HEALTH_TRIM warning when the mds log is longer than mds_log_max_segments * mds_log_warn_factor"),
7935
7936     Option("mds_bal_export_pin", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7937     .set_default(true)
7938     .set_description("allow setting directory export pins to particular ranks"),
7939
7940     Option("mds_export_ephemeral_random", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7941     .set_default(false)
7942     .set_flag(Option::FLAG_RUNTIME)
7943     .set_description("allow ephemeral random pinning of the loaded subtrees")
7944     .set_long_description("probabilistically pin the loaded directory inode and the subtree beneath it to an MDS based on the consistent hash of the inode number. The higher this value the more likely the loaded subtrees get pinned"),
7945
7946     Option("mds_export_ephemeral_random_max", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7947     .set_default(0.01)
7948     .set_flag(Option::FLAG_RUNTIME)
7949     .set_description("the maximum percent permitted for random ephemeral pin policy")
7950     .set_min_max(0.0, 1.0)
7951     .add_see_also("mds_export_ephemeral_random"),
7952
7953     Option("mds_export_ephemeral_distributed", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
7954     .set_default(false)
7955     .set_flag(Option::FLAG_RUNTIME)
7956     .set_description("allow ephemeral distributed pinning of the loaded subtrees")
7957     .set_long_description("pin the immediate child directories of the loaded directory inode based on the consistent hash of the child's inode number. "),
7958
7959     Option("mds_bal_sample_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7960     .set_default(3.0)
7961     .set_description("interval in seconds between balancer ticks"),
7962
7963     Option("mds_bal_replicate_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7964     .set_default(8000)
7965     .set_description("hot popularity threshold to replicate a subtree"),
7966
7967     Option("mds_bal_unreplicate_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7968     .set_default(0)
7969     .set_description("cold popularity threshold to merge subtrees"),
7970
7971     Option("mds_bal_split_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7972     .set_default(10000)
7973     .set_description("minimum size of directory fragment before splitting"),
7974
7975     Option("mds_bal_split_rd", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7976     .set_default(25000)
7977     .set_description("hot read popularity threshold for splitting a directory fragment"),
7978
7979     Option("mds_bal_split_wr", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
7980     .set_default(10000)
7981     .set_description("hot write popularity threshold for splitting a directory fragment"),
7982
7983     Option("mds_bal_split_bits", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7984     .set_default(3)
7985     .set_min_max(1, 24)
7986     .set_description("power of two child fragments for a fragment on split"),
7987
7988     Option("mds_bal_merge_size", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7989     .set_default(50)
7990     .set_description("size of fragments where merging should occur"),
7991
7992     Option("mds_bal_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7993     .set_default(10)
7994     .set_description("interval between MDS balancer cycles"),
7995
7996     Option("mds_bal_fragment_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
7997     .set_default(5)
7998     .set_description("delay in seconds before interrupting client IO to perform splits"),
7999
8000     Option("mds_bal_fragment_size_max", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8001     .set_default(10000*10)
8002     .set_description("maximum size of a directory fragment before new creat/links fail"),
8003
8004     Option("mds_bal_fragment_fast_factor", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8005     .set_default(1.5)
8006     .set_description("ratio of mds_bal_split_size at which fast fragment splitting occurs"),
8007
8008     Option("mds_bal_fragment_dirs", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8009     .set_default(true)
8010     .set_description("enable directory fragmentation")
8011     .set_long_description("Directory fragmentation is a standard feature of CephFS that allows sharding directories across multiple objects for performance and stability. Additionally, this allows fragments to be distributed across multiple active MDSs to increase throughput. Disabling (new) fragmentation should only be done in exceptional circumstances and may lead to performance issues."),
8012
8013     Option("mds_bal_idle_threshold", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8014     .set_default(0)
8015     .set_description("idle metadata popularity threshold before rebalancing"),
8016
8017     Option("mds_bal_max", Option::TYPE_INT, Option::LEVEL_DEV)
8018     .set_default(-1)
8019     .set_description(""),
8020
8021     Option("mds_bal_max_until", Option::TYPE_INT, Option::LEVEL_DEV)
8022     .set_default(-1)
8023     .set_description(""),
8024
8025     Option("mds_bal_mode", Option::TYPE_INT, Option::LEVEL_DEV)
8026     .set_default(0)
8027     .set_description(""),
8028
8029     Option("mds_bal_min_rebalance", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8030     .set_default(.1)
8031     .set_description("amount overloaded over internal target before balancer begins offloading"),
8032
8033     Option("mds_bal_min_start", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8034     .set_default(.2)
8035     .set_description(""),
8036
8037     Option("mds_bal_need_min", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8038     .set_default(.8)
8039     .set_description(""),
8040
8041     Option("mds_bal_need_max", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8042     .set_default(1.2)
8043     .set_description(""),
8044
8045     Option("mds_bal_midchunk", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8046     .set_default(.3)
8047     .set_description(""),
8048
8049     Option("mds_bal_minchunk", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8050     .set_default(.001)
8051     .set_description(""),
8052
8053     Option("mds_bal_target_decay", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8054     .set_default(10.0)
8055     .set_description("rate of decay for export targets communicated to clients"),
8056
8057     Option("mds_oft_prefetch_dirfrags", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8058     .set_default(true)
8059     .set_description("prefetch dirfrags recorded in open file table on startup")
8060     .set_flag(Option::FLAG_STARTUP),
8061
8062     Option("mds_replay_interval", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8063     .set_default(1.0)
8064     .set_description("time in seconds between replay of updates to journal by standby replay MDS"),
8065
8066     Option("mds_shutdown_check", Option::TYPE_INT, Option::LEVEL_DEV)
8067     .set_default(0)
8068     .set_description(""),
8069
8070     Option("mds_thrash_exports", Option::TYPE_INT, Option::LEVEL_DEV)
8071     .set_default(0)
8072     .set_description(""),
8073
8074     Option("mds_thrash_fragments", Option::TYPE_INT, Option::LEVEL_DEV)
8075     .set_default(0)
8076     .set_description(""),
8077
8078     Option("mds_dump_cache_on_map", Option::TYPE_BOOL, Option::LEVEL_DEV)
8079     .set_default(false)
8080     .set_description(""),
8081
8082     Option("mds_dump_cache_after_rejoin", Option::TYPE_BOOL, Option::LEVEL_DEV)
8083     .set_default(false)
8084     .set_description(""),
8085
8086     Option("mds_verify_scatter", Option::TYPE_BOOL, Option::LEVEL_DEV)
8087     .set_default(false)
8088     .set_description(""),
8089
8090     Option("mds_debug_scatterstat", Option::TYPE_BOOL, Option::LEVEL_DEV)
8091     .set_default(false)
8092     .set_description(""),
8093
8094     Option("mds_debug_frag", Option::TYPE_BOOL, Option::LEVEL_DEV)
8095     .set_default(false)
8096     .set_description(""),
8097
8098     Option("mds_debug_auth_pins", Option::TYPE_BOOL, Option::LEVEL_DEV)
8099     .set_default(false)
8100     .set_description(""),
8101
8102     Option("mds_debug_subtrees", Option::TYPE_BOOL, Option::LEVEL_DEV)
8103     .set_default(false)
8104     .set_description(""),
8105
8106     Option("mds_kill_mdstable_at", Option::TYPE_INT, Option::LEVEL_DEV)
8107     .set_default(0)
8108     .set_description(""),
8109
8110     Option("mds_max_export_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
8111     .set_default(20_M)
8112     .set_description(""),
8113
8114     Option("mds_kill_export_at", Option::TYPE_INT, Option::LEVEL_DEV)
8115     .set_default(0)
8116     .set_description(""),
8117
8118     Option("mds_kill_import_at", Option::TYPE_INT, Option::LEVEL_DEV)
8119     .set_default(0)
8120     .set_description(""),
8121
8122     Option("mds_kill_link_at", Option::TYPE_INT, Option::LEVEL_DEV)
8123     .set_default(0)
8124     .set_description(""),
8125
8126     Option("mds_kill_rename_at", Option::TYPE_INT, Option::LEVEL_DEV)
8127     .set_default(0)
8128     .set_description(""),
8129
8130     Option("mds_kill_openc_at", Option::TYPE_INT, Option::LEVEL_DEV)
8131     .set_default(0)
8132     .set_description(""),
8133
8134     Option("mds_kill_journal_at", Option::TYPE_INT, Option::LEVEL_DEV)
8135     .set_default(0)
8136     .set_description(""),
8137
8138     Option("mds_kill_journal_expire_at", Option::TYPE_INT, Option::LEVEL_DEV)
8139     .set_default(0)
8140     .set_description(""),
8141
8142     Option("mds_kill_journal_replay_at", Option::TYPE_INT, Option::LEVEL_DEV)
8143     .set_default(0)
8144     .set_description(""),
8145
8146     Option("mds_journal_format", Option::TYPE_UINT, Option::LEVEL_DEV)
8147     .set_default(1)
8148     .set_description(""),
8149
8150     Option("mds_kill_create_at", Option::TYPE_INT, Option::LEVEL_DEV)
8151     .set_default(0)
8152     .set_description(""),
8153
8154     Option("mds_inject_traceless_reply_probability", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8155     .set_default(0)
8156     .set_description(""),
8157
8158     Option("mds_wipe_sessions", Option::TYPE_BOOL, Option::LEVEL_DEV)
8159     .set_default(0)
8160     .set_description(""),
8161
8162     Option("mds_wipe_ino_prealloc", Option::TYPE_BOOL, Option::LEVEL_DEV)
8163     .set_default(0)
8164     .set_description(""),
8165
8166     Option("mds_skip_ino", Option::TYPE_INT, Option::LEVEL_DEV)
8167     .set_default(0)
8168     .set_description(""),
8169
8170     Option("mds_enable_op_tracker", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8171     .set_default(true)
8172     .set_description("track remote operation progression and statistics"),
8173
8174     Option("mds_op_history_size", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8175     .set_default(20)
8176     .set_description("maximum size for list of historical operations"),
8177
8178     Option("mds_op_history_duration", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8179     .set_default(600)
8180     .set_description("expiration time in seconds of historical operations"),
8181
8182     Option("mds_op_complaint_time", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8183     .set_default(30)
8184     .set_description("time in seconds to consider an operation blocked after no updates"),
8185
8186     Option("mds_op_log_threshold", Option::TYPE_INT, Option::LEVEL_DEV)
8187     .set_default(5)
8188     .set_description(""),
8189
8190     Option("mds_snap_min_uid", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8191     .set_default(0)
8192     .set_description("minimum uid of client to perform snapshots"),
8193
8194     Option("mds_snap_max_uid", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8195     .set_default(4294967294)
8196     .set_description("maximum uid of client to perform snapshots"),
8197
8198     Option("mds_snap_rstat", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8199     .set_default(false)
8200     .set_description("enabled nested rstat for snapshots"),
8201
8202     Option("mds_verify_backtrace", Option::TYPE_UINT, Option::LEVEL_DEV)
8203     .set_default(1)
8204     .set_description(""),
8205
8206     Option("mds_max_completed_flushes", Option::TYPE_UINT, Option::LEVEL_DEV)
8207     .set_default(100000)
8208     .set_description(""),
8209
8210     Option("mds_max_completed_requests", Option::TYPE_UINT, Option::LEVEL_DEV)
8211     .set_default(100000)
8212     .set_description(""),
8213
8214     Option("mds_action_on_write_error", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8215     .set_default(1)
8216     .set_description("action to take when MDS cannot write to RADOS (0:ignore, 1:read-only, 2:suicide)"),
8217
8218     Option("mds_mon_shutdown_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8219     .set_default(5)
8220     .set_description("time to wait for mon to receive damaged MDS rank notification"),
8221
8222     Option("mds_max_purge_files", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8223     .set_default(64)
8224     .set_description("maximum number of deleted files to purge in parallel"),
8225
8226     Option("mds_max_purge_ops", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8227     .set_default(8192)
8228     .set_description("maximum number of purge operations performed in parallel"),
8229
8230     Option("mds_max_purge_ops_per_pg", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8231     .set_default(0.5)
8232     .set_description("number of parallel purge operations performed per PG"),
8233
8234     Option("mds_purge_queue_busy_flush_period", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8235     .set_default(1.0)
8236     .set_description(""),
8237
8238     Option("mds_root_ino_uid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8239     .set_default(0)
8240     .set_description("default uid for new root directory"),
8241
8242     Option("mds_root_ino_gid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8243     .set_default(0)
8244     .set_description("default gid for new root directory"),
8245
8246     Option("mds_max_scrub_ops_in_progress", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8247     .set_default(5)
8248     .set_description("maximum number of scrub operations performed in parallel"),
8249
8250     Option("mds_forward_all_requests_to_auth", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8251     .set_default(false)
8252     .set_flag(Option::FLAG_RUNTIME)
8253     .set_description("always process op on auth mds"),
8254
8255     Option("mds_damage_table_max_entries", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8256     .set_default(10000)
8257     .set_description("maximum number of damage table entries"),
8258
8259     Option("mds_client_writeable_range_max_inc_objs", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8260     .set_default(1024)
8261     .set_description("maximum number of objects in writeable range of a file for a client"),
8262
8263     Option("mds_min_caps_per_client", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8264     .set_default(100)
8265     .set_description("minimum number of capabilities a client may hold"),
8266
8267     Option("mds_min_caps_working_set", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8268     .set_default(10000)
8269     .set_flag(Option::FLAG_RUNTIME)
8270     .set_description("number of capabilities a client may hold without cache pressure warnings generated"),
8271
8272     Option("mds_max_caps_per_client", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8273     .set_default(1_M)
8274     .set_description("maximum number of capabilities a client may hold"),
8275
8276     Option("mds_hack_allow_loading_invalid_metadata", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8277      .set_default(0)
8278      .set_description("INTENTIONALLY CAUSE DATA LOSS by bypasing checks for invalid metadata on disk. Allows testing repair tools."),
8279
8280     Option("mds_defer_session_stale", Option::TYPE_BOOL, Option::LEVEL_DEV)
8281      .set_default(true),
8282
8283     Option("mds_inject_migrator_session_race", Option::TYPE_BOOL, Option::LEVEL_DEV)
8284      .set_default(false),
8285
8286     Option("mds_request_load_average_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8287     .set_default(60)
8288     .set_description("rate of decay in seconds for calculating request load average"),
8289
8290     Option("mds_cap_revoke_eviction_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8291      .set_default(0)
8292      .set_description("number of seconds after which clients which have not responded to cap revoke messages by the MDS are evicted."),
8293
8294     Option("mds_max_retries_on_remount_failure", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8295      .set_default(5)
8296      .set_description("number of consecutive failed remount attempts for invalidating kernel dcache after which client would abort."),
8297
8298     Option("mds_dump_cache_threshold_formatter", Option::TYPE_SIZE, Option::LEVEL_DEV)
8299      .set_default(1_G)
8300      .set_description("threshold for cache usage to disallow \"dump cache\" operation to formatter")
8301      .set_long_description("Disallow MDS from dumping caches to formatter via \"dump cache\" command if cache usage exceeds this threshold."),
8302
8303     Option("mds_dump_cache_threshold_file", Option::TYPE_SIZE, Option::LEVEL_DEV)
8304      .set_default(0)
8305      .set_description("threshold for cache usage to disallow \"dump cache\" operation to file")
8306      .set_long_description("Disallow MDS from dumping caches to file via \"dump cache\" command if cache usage exceeds this threshold."),
8307
8308     Option("mds_task_status_update_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8309      .set_default(2.0)
8310      .set_description("task status update interval to manager")
8311      .set_long_description("interval (in seconds) for sending mds task status to ceph manager"),
8312
8313     Option("mds_max_snaps_per_dir", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
8314      .set_default(100)
8315      .set_min_max(0, 4096)
8316      .set_flag(Option::FLAG_RUNTIME)
8317      .set_description("max snapshots per directory")
8318      .set_long_description("maximum number of snapshots that can be created per directory"),
8319   });
8320 }
8321
8322 std::vector<Option> get_mds_client_options() {
8323   return std::vector<Option>({
8324     Option("client_cache_size", Option::TYPE_SIZE, Option::LEVEL_BASIC)
8325     .set_default(16384)
8326     .set_description("soft maximum number of directory entries in client cache"),
8327
8328     Option("client_cache_mid", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8329     .set_default(.75)
8330     .set_description("mid-point of client cache LRU"),
8331
8332     Option("client_use_random_mds", Option::TYPE_BOOL, Option::LEVEL_DEV)
8333     .set_default(false)
8334     .set_description("issue new requests to a random active MDS"),
8335
8336     Option("client_mount_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8337     .set_default(300.0)
8338     .set_description("timeout for mounting CephFS (seconds)"),
8339
8340     Option("client_tick_interval", Option::TYPE_FLOAT, Option::LEVEL_DEV)
8341     .set_default(1.0)
8342     .set_description("seconds between client upkeep ticks"),
8343
8344     Option("client_trace", Option::TYPE_STR, Option::LEVEL_DEV)
8345     .set_default("")
8346     .set_description("file containing trace of client operations"),
8347
8348     Option("client_readahead_min", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8349     .set_default(128*1024)
8350     .set_description("minimum bytes to readahead in a file"),
8351
8352     Option("client_readahead_max_bytes", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8353     .set_default(0)
8354     .set_description("maximum bytes to readahead in a file (zero is unlimited)"),
8355
8356     Option("client_readahead_max_periods", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8357     .set_default(4)
8358     .set_description("maximum stripe periods to readahead in a file"),
8359
8360     Option("client_reconnect_stale", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8361     .set_default(false)
8362     .set_description("reconnect when the session becomes stale"),
8363
8364     Option("client_snapdir", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8365     .set_default(".snap")
8366     .set_description("pseudo directory for snapshot access to a directory"),
8367
8368     Option("client_mountpoint", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8369     .set_default("/")
8370     .set_description("default mount-point"),
8371
8372     Option("client_mount_uid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8373     .set_default(-1)
8374     .set_description("uid to mount as"),
8375
8376     Option("client_mount_gid", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8377     .set_default(-1)
8378     .set_description("gid to mount as"),
8379
8380     /* RADOS client option */
8381     Option("client_notify_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
8382     .set_default(10)
8383     .set_description(""),
8384
8385     /* RADOS client option */
8386     Option("osd_client_watch_timeout", Option::TYPE_INT, Option::LEVEL_DEV)
8387     .set_default(30)
8388     .set_description(""),
8389
8390     Option("client_caps_release_delay", Option::TYPE_INT, Option::LEVEL_DEV)
8391     .set_default(5)
8392     .set_description(""),
8393
8394     Option("client_quota_df", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8395     .set_default(true)
8396     .set_description("show quota usage for statfs (df)"),
8397
8398     Option("client_oc", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8399     .set_default(true)
8400     .set_description("enable object caching"),
8401
8402     Option("client_oc_size", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8403     .set_default(200_M)
8404     .set_description("maximum size of object cache"),
8405
8406     Option("client_oc_max_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8407     .set_default(100_M)
8408     .set_description("maximum size of dirty pages in object cache"),
8409
8410     Option("client_oc_target_dirty", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8411     .set_default(8_M)
8412     .set_description("target size of dirty pages object cache"),
8413
8414     Option("client_oc_max_dirty_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
8415     .set_default(5.0)
8416     .set_description("maximum age of dirty pages in object cache (seconds)"),
8417
8418     Option("client_oc_max_objects", Option::TYPE_INT, Option::LEVEL_ADVANCED)
8419     .set_default(1000)
8420     .set_description("maximum number of objects in cache"),
8421
8422     Option("client_debug_getattr_caps", Option::TYPE_BOOL, Option::LEVEL_DEV)
8423     .set_default(false)
8424     .set_description(""),
8425
8426     Option("client_debug_force_sync_read", Option::TYPE_BOOL, Option::LEVEL_DEV)
8427     .set_default(false)
8428     .set_description(""),
8429
8430     Option("client_debug_inject_tick_delay", Option::TYPE_INT, Option::LEVEL_DEV)
8431     .set_default(0)
8432     .set_description(""),
8433
8434     Option("client_max_inline_size", Option::TYPE_SIZE, Option::LEVEL_DEV)
8435     .set_default(4_K)
8436     .set_description(""),
8437
8438     Option("client_inject_release_failure", Option::TYPE_BOOL, Option::LEVEL_DEV)
8439     .set_default(false)
8440     .set_description(""),
8441
8442     Option("client_inject_fixed_oldest_tid", Option::TYPE_BOOL, Option::LEVEL_DEV)
8443     .set_default(false)
8444     .set_description(""),
8445
8446     Option("client_metadata", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8447     .set_default("")
8448     .set_description("metadata key=value comma-delimited pairs appended to session metadata"),
8449
8450     Option("client_acl_type", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8451     .set_default("")
8452     .set_description("ACL type to enforce (none or \"posix_acl\")"),
8453
8454     Option("client_permissions", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8455     .set_default(true)
8456     .set_description("client-enforced permission checking"),
8457
8458     Option("client_dirsize_rbytes", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8459     .set_default(true)
8460     .set_description("set the directory size as the number of file bytes recursively used")
8461     .set_long_description("This option enables a CephFS feature that stores the recursive directory size (the bytes used by files in the directory and its descendents) in the st_size field of the stat structure."),
8462
8463     Option("client_force_lazyio", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8464     .set_default(false)
8465     .set_description(""),
8466
8467     // note: the max amount of "in flight" dirty data is roughly (max - target)
8468     Option("fuse_use_invalidate_cb", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8469     .set_default(true)
8470     .set_description("use fuse 2.8+ invalidate callback to keep page cache consistent"),
8471
8472     Option("fuse_disable_pagecache", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8473     .set_default(false)
8474     .set_description("disable page caching in the kernel for this FUSE mount"),
8475
8476     Option("fuse_allow_other", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8477     .set_default(true)
8478     .set_description("pass allow_other to FUSE on mount"),
8479
8480     Option("fuse_default_permissions", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8481     .set_default(false)
8482     .set_description("pass default_permisions to FUSE on mount")
8483     .set_flag(Option::FLAG_STARTUP),
8484
8485     Option("fuse_big_writes", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8486     .set_default(true)
8487     .set_description("big_writes is deprecated in libfuse 3.0.0"),
8488
8489     Option("fuse_max_write", Option::TYPE_SIZE, Option::LEVEL_ADVANCED)
8490     .set_default(0)
8491     .set_description("set the maximum number of bytes in a single write operation")
8492     .set_long_description("Set the maximum number of bytes in a single write operation that may pass atomically through FUSE. The FUSE default is 128kB and may be indicated by setting this option to 0."),
8493
8494     Option("fuse_atomic_o_trunc", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8495     .set_default(true)
8496     .set_description("pass atomic_o_trunc flag to FUSE on mount"),
8497
8498     Option("fuse_debug", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8499     .set_default(false)
8500     .set_flag(Option::FLAG_STARTUP)
8501     .set_flag(Option::FLAG_NO_MON_UPDATE)
8502     .set_description("enable debugging for the libfuse"),
8503
8504     Option("fuse_multithreaded", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8505     .set_default(true)
8506     .set_description("allow parallel processing through FUSE library"),
8507
8508     Option("fuse_require_active_mds", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8509     .set_default(true)
8510     .set_description("require active MDSs in the file system when mounting"),
8511
8512     Option("fuse_syncfs_on_mksnap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8513     .set_default(true)
8514     .set_description("synchronize all local metadata/file changes after snapshot"),
8515
8516     Option("fuse_set_user_groups", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8517     .set_default(true)
8518     .set_description("check for ceph-fuse to consider supplementary groups for permissions"),
8519
8520     Option("client_try_dentry_invalidate", Option::TYPE_BOOL, Option::LEVEL_DEV)
8521     .set_default(false)
8522     .set_description(""),
8523
8524     Option("client_die_on_failed_remount", Option::TYPE_BOOL, Option::LEVEL_DEV)
8525     .set_default(false)
8526     .set_description(""),
8527
8528     Option("client_die_on_failed_dentry_invalidate", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8529     .set_default(true)
8530     .set_description("kill the client when no dentry invalidation options are available")
8531     .set_long_description("The CephFS client requires a mechanism to invalidate dentries in the caller (e.g. the kernel for ceph-fuse) when capabilities must be recalled. If the client cannot do this then the MDS cache cannot shrink which can cause the MDS to fail."),
8532
8533     Option("client_check_pool_perm", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
8534     .set_default(true)
8535     .set_description("confirm access to inode's data pool/namespace described in file layout"),
8536
8537     Option("client_use_faked_inos", Option::TYPE_BOOL, Option::LEVEL_DEV)
8538     .set_default(false)
8539     .set_description(""),
8540
8541     Option("client_fs", Option::TYPE_STR, Option::LEVEL_ADVANCED)
8542     .set_flag(Option::FLAG_STARTUP)
8543     .set_default("")
8544     .set_description("CephFS file system name to mount")
8545     .set_long_description("Use this with ceph-fuse, or with any process "
8546         "that uses libcephfs.  Programs using libcephfs may also pass "
8547         "the filesystem name into mount(), which will override this setting. "
8548         "If no filesystem name is given in mount() or this setting, the default "
8549         "filesystem will be mounted (usually the first created)."),
8550
8551     /* Alias for client_fs. Deprecated */
8552     Option("client_mds_namespace", Option::TYPE_STR, Option::LEVEL_DEV)
8553     .set_flag(Option::FLAG_STARTUP)
8554     .set_default(""),
8555
8556     Option("fake_statfs_for_testing", Option::TYPE_INT, Option::LEVEL_DEV)
8557     .set_default(0)
8558     .set_description("Set a value for kb and compute kb_used from total of num_bytes"),
8559
8560     Option("debug_allow_any_pool_priority", Option::TYPE_BOOL, Option::LEVEL_DEV)
8561     .set_default(false)
8562     .set_description("Allow any pool priority to be set to test conversion to new range"),
8563
8564     Option("client_shutdown_timeout", Option::TYPE_SECS, Option::LEVEL_ADVANCED)
8565     .set_flag(Option::FLAG_RUNTIME)
8566     .set_default(30)
8567     .set_min(0)
8568     .set_description("timeout for shutting down CephFS")
8569     .set_long_description("Timeout for shutting down CephFS via unmount or shutdown.")
8570     .add_tag("client")
8571   });
8572 }
8573
8574
8575 std::vector<Option> get_cephfs_shell_options() {
8576   return std::vector<Option>({
8577     Option("allow_ansi", Option::TYPE_STR, Option::LEVEL_BASIC)
8578     .set_default("Terminal")
8579     .set_description("Allow ANSI escape sequences in output. Values: "
8580                      "Terminal, Always, Never"),
8581
8582     Option("colors", Option::TYPE_STR, Option::LEVEL_BASIC)
8583     .set_default("Terminal")
8584     .set_description("Colouring CephFS shell input and output. Values: "
8585                      "Terminal, Always, Never"),
8586
8587     Option("continuation_prompt", Option::TYPE_STR, Option::LEVEL_BASIC)
8588     .set_default(">")
8589     .set_description("Prompt string when a command continue to second line"),
8590
8591     Option("debug_shell", Option::TYPE_BOOL, Option::LEVEL_BASIC)
8592     .set_default(false)
8593     .set_description("Allow tracebacks on error for CephFS Shell"),
8594
8595     Option("echo", Option::TYPE_BOOL, Option::LEVEL_BASIC)
8596     .set_default(false)
8597     .set_description("Print command issued on prompt before execution"),
8598
8599     Option("editor", Option::TYPE_STR, Option::LEVEL_BASIC)
8600     .set_default("vim")
8601     .set_description("Default text editor for shell"),
8602
8603     Option("feedback_to_output", Option::TYPE_BOOL, Option::LEVEL_BASIC)
8604     .set_default(false)
8605     .set_description("include '|' and '>' in result"),
8606
8607     Option("max_completion_items", Option::TYPE_INT, Option::LEVEL_BASIC)
8608     .set_default(50)
8609     .set_description("Maximum number of items to be displayed by tab "
8610                      "completion"),
8611
8612     Option("prompt", Option::TYPE_STR, Option::LEVEL_BASIC)
8613     .set_default("\x1b[01;33mCephFS:~\x1b[96m/\x1b[0m\x1b[01;33m>>>\x1b[00m ")
8614     .set_description("Whether non-essential feedback should be printed."),
8615
8616     Option("quiet", Option::TYPE_BOOL, Option::LEVEL_BASIC)
8617     .set_default(false)
8618     .set_description("Whether non-essential feedback should be printed."),
8619
8620     Option("timing", Option::TYPE_BOOL, Option::LEVEL_BASIC)
8621     .set_default(false)
8622     .set_description("Whether execution time should be reported"),
8623   });
8624 }
8625
8626 static std::vector<Option> build_options()
8627 {
8628   std::vector<Option> result = get_global_options();
8629
8630   auto ingest = [&result](std::vector<Option>&& options, const char* svc) {
8631     for (auto &o : options) {
8632       o.add_service(svc);
8633       result.push_back(std::move(o));
8634     }
8635   };
8636
8637   ingest(get_rgw_options(), "rgw");
8638   ingest(get_rbd_options(), "rbd");
8639   ingest(get_rbd_mirror_options(), "rbd-mirror");
8640   ingest(get_immutable_object_cache_options(), "immutable-objet-cache");
8641   ingest(get_mds_options(), "mds");
8642   ingest(get_mds_client_options(), "mds_client");
8643   ingest(get_cephfs_shell_options(), "cephfs-shell");
8644
8645   return result;
8646 }
8647
8648 const std::vector<Option> ceph_options = build_options();