ceph/src/rgw/rgw_appmain.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 /*
   5  * Ceph - scalable distributed file system
   6  *
   7  * Copyright (C) 2022 Red Hat, Inc
   8  *
   9  * This is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public
  11  * License version 2.1, as published by the Free Software
  12  * Foundation.  See file COPYING.
  13  *
  14  */
  15
  16 #include <boost/intrusive/list.hpp>
  17 #include "global/global_init.h"
  18 #include "global/signal_handler.h"
  19 #include "common/config.h"
  20 #include "common/errno.h"
  21 #include "common/Timer.h"
  22 #include "common/TracepointProvider.h"
  23 #include "common/openssl_opts_handler.h"
  24 #include "common/numa.h"
  25 #include "include/compat.h"
  26 #include "include/str_list.h"
  27 #include "include/stringify.h"
  28 #include "rgw_main.h"
  29 #include "rgw_common.h"
  30 #include "rgw_sal_rados.h"
  31 #include "rgw_period_pusher.h"
  32 #include "rgw_realm_reloader.h"
  33 #include "rgw_rest.h"
  34 #include "rgw_rest_s3.h"
  35 #include "rgw_rest_swift.h"
  36 #include "rgw_rest_admin.h"
  37 #include "rgw_rest_info.h"
  38 #include "rgw_rest_usage.h"
  39 #include "rgw_rest_bucket.h"
  40 #include "rgw_rest_metadata.h"
  41 #include "rgw_rest_log.h"
  42 #include "rgw_rest_config.h"
  43 #include "rgw_rest_realm.h"
  44 #include "rgw_rest_ratelimit.h"
  45 #include "rgw_swift_auth.h"
  46 #include "rgw_log.h"
  47 #include "rgw_lib.h"
  48 #include "rgw_frontend.h"
  49 #include "rgw_lib_frontend.h"
  50 #include "rgw_tools.h"
  51 #include "rgw_resolve.h"
  52 #include "rgw_process.h"
  53 #include "rgw_frontend.h"
  54 #include "rgw_http_client_curl.h"
  55 #include "rgw_kmip_client.h"
  56 #include "rgw_kmip_client_impl.h"
  57 #include "rgw_perf_counters.h"
  58 #include "rgw_signal.h"
  59 #ifdef WITH_RADOSGW_AMQP_ENDPOINT
  60 #include "rgw_amqp.h"
  61 #endif
  62 #ifdef WITH_RADOSGW_KAFKA_ENDPOINT
  63 #include "rgw_kafka.h"
  64 #endif
  65 #ifdef WITH_ARROW_FLIGHT
  66 #include "rgw_flight_frontend.h"
  67 #endif
  68 #include "rgw_asio_frontend.h"
  69 #include "rgw_dmclock_scheduler_ctx.h"
  70 #include "rgw_lua.h"
  71 #ifdef WITH_RADOSGW_DBSTORE
  72 #include "rgw_sal_dbstore.h"
  73 #endif
  74 #include "rgw_lua_background.h"
  75 #include "services/svc_zone.h"
  76
  77 #ifdef HAVE_SYS_PRCTL_H
  78 #include <sys/prctl.h>
  79 #endif
  80
  81 #define dout_subsys ceph_subsys_rgw
  82
  83 using namespace std;
  84
  85 namespace {
  86   TracepointProvider::Traits rgw_op_tracepoint_traits(
  87     "librgw_op_tp.so", "rgw_op_tracing");
  88   TracepointProvider::Traits rgw_rados_tracepoint_traits(
  89     "librgw_rados_tp.so", "rgw_rados_tracing");
  90 }
  91
  92 OpsLogFile* rgw::AppMain::ops_log_file;
  93
  94 void rgw::AppMain::init_frontends1(bool nfs)
  95 {
  96   this->nfs = nfs;
  97   std::string fe_key = (nfs) ? "rgw_nfs_frontends" : "rgw_frontends";
  98   std::vector<std::string> frontends;
  99   std::string rgw_frontends_str = g_conf().get_val<string>(fe_key);
 100   g_conf().early_expand_meta(rgw_frontends_str, &cerr);
 101   get_str_vec(rgw_frontends_str, ",", frontends);
 102
 103   /* default frontends */
 104   if (nfs) {
 105     const auto is_rgw_nfs = [](const auto& s){return s == "rgw-nfs";};
 106     if (std::find_if(frontends.begin(), frontends.end(), is_rgw_nfs) == frontends.end()) {
 107       frontends.push_back("rgw-nfs");
 108     }
 109   } else {
 110     if (frontends.empty()) {
 111       frontends.push_back("beast");
 112     }
 113   }
 114
 115   for (auto &f : frontends) {
 116     if (f.find("beast") != string::npos) {
 117       have_http_frontend = true;
 118       if (f.find("port") != string::npos) {
 119         // check for the most common ws problems
 120         if ((f.find("port=") == string::npos) ||
 121             (f.find("port= ") != string::npos)) {
 122           derr <<
 123     R"(WARNING: radosgw frontend config found unexpected spacing around 'port'
 124     (ensure frontend port parameter has the form 'port=80' with no spaces
 125     before or after '='))"
 126                << dendl;
 127         }
 128       }
 129     } else {
 130       if (f.find("civetweb") != string::npos) {
 131         have_http_frontend = true;
 132       }
 133     } /* fe !beast */
 134
 135     RGWFrontendConfig *config = new RGWFrontendConfig(f);
 136     int r = config->init();
 137     if (r < 0) {
 138       delete config;
 139       cerr << "ERROR: failed to init config: " << f << std::endl;
 140       continue;
 141     }
 142
 143     fe_configs.push_back(config);
 144     fe_map.insert(
 145         pair<string, RGWFrontendConfig *>(config->get_framework(), config));
 146   } /* for each frontend */
 147
 148   // maintain existing region root pool for new multisite objects
 149   if (!g_conf()->rgw_region_root_pool.empty()) {
 150     const char *root_pool = g_conf()->rgw_region_root_pool.c_str();
 151     if (g_conf()->rgw_zonegroup_root_pool.empty()) {
 152       g_conf().set_val_or_die("rgw_zonegroup_root_pool", root_pool);
 153     }
 154     if (g_conf()->rgw_period_root_pool.empty()) {
 155       g_conf().set_val_or_die("rgw_period_root_pool", root_pool);
 156     }
 157     if (g_conf()->rgw_realm_root_pool.empty()) {
 158       g_conf().set_val_or_die("rgw_realm_root_pool", root_pool);
 159     }
 160   }
 161
 162   // for region -> zonegroup conversion (must happen before
 163   // common_init_finish())
 164   if (!g_conf()->rgw_region.empty() && g_conf()->rgw_zonegroup.empty()) {
 165     g_conf().set_val_or_die("rgw_zonegroup", g_conf()->rgw_region.c_str());
 166   }
 167
 168   ceph::crypto::init_openssl_engine_once();
 169 } /* init_frontends1 */
 170
 171 void rgw::AppMain::init_numa()
 172 {
 173   if (nfs) {
 174     return;
 175   }
 176
 177   int numa_node = g_conf().get_val<int64_t>("rgw_numa_node");
 178   size_t numa_cpu_set_size = 0;
 179   cpu_set_t numa_cpu_set;
 180
 181   if (numa_node >= 0) {
 182     int r = get_numa_node_cpu_set(numa_node, &numa_cpu_set_size, &numa_cpu_set);
 183     if (r < 0) {
 184       dout(1) << __func__ << " unable to determine rgw numa node " << numa_node
 185               << " CPUs" << dendl;
 186       numa_node = -1;
 187     } else {
 188       r = set_cpu_affinity_all_threads(numa_cpu_set_size, &numa_cpu_set);
 189       if (r < 0) {
 190         derr << __func__ << " failed to set numa affinity: " << cpp_strerror(r)
 191         << dendl;
 192       }
 193     }
 194   } else {
 195     dout(1) << __func__ << " not setting numa affinity" << dendl;
 196   }
 197 } /* init_numa */
 198
 199 void rgw::AppMain::init_storage()
 200 {
 201     auto run_gc =
 202     (g_conf()->rgw_enable_gc_threads &&
 203       ((!nfs) || (nfs && g_conf()->rgw_nfs_run_gc_threads)));
 204
 205   auto run_lc =
 206     (g_conf()->rgw_enable_lc_threads &&
 207       ((!nfs) || (nfs && g_conf()->rgw_nfs_run_lc_threads)));
 208
 209   auto run_quota =
 210     (g_conf()->rgw_enable_quota_threads &&
 211       ((!nfs) || (nfs && g_conf()->rgw_nfs_run_quota_threads)));
 212
 213   auto run_sync =
 214     (g_conf()->rgw_run_sync_thread &&
 215       ((!nfs) || (nfs && g_conf()->rgw_nfs_run_sync_thread)));
 216
 217   DriverManager::Config cfg = DriverManager::get_config(false, g_ceph_context);
 218   env.driver = DriverManager::get_storage(dpp, dpp->get_cct(),
 219           cfg,
 220           run_gc,
 221           run_lc,
 222           run_quota,
 223           run_sync,
 224           g_conf().get_val<bool>("rgw_dynamic_resharding"),
 225           g_conf()->rgw_cache_enabled);
 226
 227 } /* init_storage */
 228
 229 void rgw::AppMain::init_perfcounters()
 230 {
 231   (void) rgw_perf_start(dpp->get_cct());
 232 } /* init_perfcounters */
 233
 234 void rgw::AppMain::init_http_clients()
 235 {
 236   rgw_init_resolver();
 237   rgw::curl::setup_curl(fe_map);
 238   rgw_http_client_init(dpp->get_cct());
 239   rgw_kmip_client_init(*new RGWKMIPManagerImpl(dpp->get_cct()));
 240 } /* init_http_clients */
 241
 242 void rgw::AppMain::cond_init_apis()
 243 {
 244    rgw_rest_init(g_ceph_context, env.driver->get_zone()->get_zonegroup());
 245
 246   if (have_http_frontend) {
 247     std::vector<std::string> apis;
 248     get_str_vec(g_conf()->rgw_enable_apis, apis);
 249
 250     std::map<std::string, bool> apis_map;
 251     for (auto &api : apis) {
 252       apis_map[api] = true;
 253     }
 254
 255     /* warn about insecure keystone secret config options */
 256     if (!(g_ceph_context->_conf->rgw_keystone_admin_token.empty() ||
 257           g_ceph_context->_conf->rgw_keystone_admin_password.empty())) {
 258       dout(0)
 259           << "WARNING: rgw_keystone_admin_token and "
 260              "rgw_keystone_admin_password should be avoided as they can "
 261              "expose secrets.  Prefer the new rgw_keystone_admin_token_path "
 262              "and rgw_keystone_admin_password_path options, which read their "
 263              "secrets from files."
 264           << dendl;
 265     }
 266
 267     // S3 website mode is a specialization of S3
 268     const bool s3website_enabled = apis_map.count("s3website") > 0;
 269     const bool sts_enabled = apis_map.count("sts") > 0;
 270     const bool iam_enabled = apis_map.count("iam") > 0;
 271     const bool pubsub_enabled =
 272         apis_map.count("pubsub") > 0 || apis_map.count("notifications") > 0;
 273     // Swift API entrypoint could placed in the root instead of S3
 274     const bool swift_at_root = g_conf()->rgw_swift_url_prefix == "/";
 275     if (apis_map.count("s3") > 0 || s3website_enabled) {
 276       if (!swift_at_root) {
 277         rest.register_default_mgr(set_logging(
 278             rest_filter(env.driver, RGW_REST_S3,
 279                         new RGWRESTMgr_S3(s3website_enabled, sts_enabled,
 280                                           iam_enabled, pubsub_enabled))));
 281       } else {
 282         derr << "Cannot have the S3 or S3 Website enabled together with "
 283              << "Swift API placed in the root of hierarchy" << dendl;
 284       }
 285     }
 286
 287     if (apis_map.count("swift") > 0) {
 288       RGWRESTMgr_SWIFT* const swift_resource = new RGWRESTMgr_SWIFT;
 289
 290       if (! g_conf()->rgw_cross_domain_policy.empty()) {
 291         swift_resource->register_resource("crossdomain.xml",
 292                             set_logging(new RGWRESTMgr_SWIFT_CrossDomain));
 293       }
 294
 295       swift_resource->register_resource("healthcheck",
 296                             set_logging(new RGWRESTMgr_SWIFT_HealthCheck));
 297
 298       swift_resource->register_resource("info",
 299                             set_logging(new RGWRESTMgr_SWIFT_Info));
 300
 301       if (! swift_at_root) {
 302         rest.register_resource(g_conf()->rgw_swift_url_prefix,
 303                             set_logging(rest_filter(env.driver, RGW_REST_SWIFT,
 304                                                     swift_resource)));
 305       } else {
 306         if (env.driver->get_zone()->get_zonegroup().get_zone_count() > 1) {
 307           derr << "Placing Swift API in the root of URL hierarchy while running"
 308               << " multi-site configuration requires another instance of RadosGW"
 309               << " with S3 API enabled!" << dendl;
 310         }
 311
 312         rest.register_default_mgr(set_logging(swift_resource));
 313       }
 314     }
 315
 316     if (apis_map.count("swift_auth") > 0) {
 317       rest.register_resource(g_conf()->rgw_swift_auth_entry,
 318                 set_logging(new RGWRESTMgr_SWIFT_Auth));
 319     }
 320
 321     if (apis_map.count("admin") > 0) {
 322       RGWRESTMgr_Admin *admin_resource = new RGWRESTMgr_Admin;
 323       admin_resource->register_resource("info", new RGWRESTMgr_Info);
 324       admin_resource->register_resource("usage", new RGWRESTMgr_Usage);
 325       /* Register driver-specific admin APIs */
 326       env.driver->register_admin_apis(admin_resource);
 327       rest.register_resource(g_conf()->rgw_admin_entry, admin_resource);
 328     }
 329   } /* have_http_frontend */
 330 } /* init_apis */
 331
 332 void rgw::AppMain::init_ldap()
 333 {
 334   CephContext* cct = env.driver->ctx();
 335   const string &ldap_uri = cct->_conf->rgw_ldap_uri;
 336   const string &ldap_binddn = cct->_conf->rgw_ldap_binddn;
 337   const string &ldap_searchdn = cct->_conf->rgw_ldap_searchdn;
 338   const string &ldap_searchfilter = cct->_conf->rgw_ldap_searchfilter;
 339   const string &ldap_dnattr = cct->_conf->rgw_ldap_dnattr;
 340   std::string ldap_bindpw = parse_rgw_ldap_bindpw(cct);
 341
 342   ldh.reset(new rgw::LDAPHelper(ldap_uri, ldap_binddn,
 343             ldap_bindpw.c_str(), ldap_searchdn, ldap_searchfilter, ldap_dnattr));
 344   ldh->init();
 345   ldh->bind();
 346 } /* init_ldap */
 347
 348 void rgw::AppMain::init_opslog()
 349 {
 350   rgw_log_usage_init(dpp->get_cct(), env.driver);
 351
 352   OpsLogManifold *olog_manifold = new OpsLogManifold();
 353   if (!g_conf()->rgw_ops_log_socket_path.empty()) {
 354     OpsLogSocket *olog_socket =
 355         new OpsLogSocket(g_ceph_context, g_conf()->rgw_ops_log_data_backlog);
 356     olog_socket->init(g_conf()->rgw_ops_log_socket_path);
 357     olog_manifold->add_sink(olog_socket);
 358   }
 359   if (!g_conf()->rgw_ops_log_file_path.empty()) {
 360     ops_log_file =
 361         new OpsLogFile(g_ceph_context, g_conf()->rgw_ops_log_file_path,
 362                        g_conf()->rgw_ops_log_data_backlog);
 363     ops_log_file->start();
 364     olog_manifold->add_sink(ops_log_file);
 365   }
 366   olog_manifold->add_sink(new OpsLogRados(env.driver));
 367   olog = olog_manifold;
 368 } /* init_opslog */
 369
 370 int rgw::AppMain::init_frontends2(RGWLib* rgwlib)
 371 {
 372   int r{0};
 373   vector<string> frontends_def;
 374   std::string frontend_defs_str =
 375     g_conf().get_val<string>("rgw_frontend_defaults");
 376   get_str_vec(frontend_defs_str, ",", frontends_def);
 377
 378   service_map_meta["pid"] = stringify(getpid());
 379
 380   std::map<std::string, std::unique_ptr<RGWFrontendConfig> > fe_def_map;
 381   for (auto& f : frontends_def) {
 382     RGWFrontendConfig *config = new RGWFrontendConfig(f);
 383     int r = config->init();
 384     if (r < 0) {
 385       delete config;
 386       cerr << "ERROR: failed to init default config: " << f << std::endl;
 387       continue;
 388     }
 389     fe_def_map[config->get_framework()].reset(config);
 390   }
 391
 392   /* Initialize the registry of auth strategies which will coordinate
 393    * the dynamic reconfiguration. */
 394   implicit_tenant_context.reset(new rgw::auth::ImplicitTenants{g_conf()});
 395   g_conf().add_observer(implicit_tenant_context.get());
 396
 397   /* allocate a mime table (you'd never guess that from the name) */
 398   rgw_tools_init(dpp, dpp->get_cct());
 399
 400   /* Header custom behavior */
 401   rest.register_x_headers(g_conf()->rgw_log_http_headers);
 402
 403   sched_ctx.reset(new rgw::dmclock::SchedulerCtx{dpp->get_cct()});
 404   ratelimiter.reset(new ActiveRateLimiter{dpp->get_cct()});
 405   ratelimiter->start();
 406
 407   // initialize RGWProcessEnv
 408   env.rest = &rest;
 409   env.olog = olog;
 410   env.auth_registry = rgw::auth::StrategyRegistry::create(
 411       dpp->get_cct(), *implicit_tenant_context, env.driver);
 412   env.ratelimiting = ratelimiter.get();
 413
 414   int fe_count = 0;
 415   for (multimap<string, RGWFrontendConfig *>::iterator fiter = fe_map.begin();
 416        fiter != fe_map.end(); ++fiter, ++fe_count) {
 417     RGWFrontendConfig *config = fiter->second;
 418     string framework = config->get_framework();
 419
 420     auto def_iter = fe_def_map.find(framework);
 421     if (def_iter != fe_def_map.end()) {
 422       config->set_default_config(*def_iter->second);
 423     }
 424
 425     RGWFrontend* fe = nullptr;
 426
 427     if (framework == "loadgen") {
 428       fe = new RGWLoadGenFrontend(env, config);
 429     }
 430     else if (framework == "beast") {
 431       fe = new RGWAsioFrontend(env, config, *sched_ctx);
 432     }
 433     else if (framework == "rgw-nfs") {
 434       fe = new RGWLibFrontend(env, config);
 435       if (rgwlib) {
 436         rgwlib->set_fe(static_cast<RGWLibFrontend*>(fe));
 437       }
 438     }
 439     else if (framework == "arrow_flight") {
 440 #ifdef WITH_ARROW_FLIGHT
 441       int port;
 442       config->get_val("port", 8077, &port);
 443       fe = new rgw::flight::FlightFrontend(env, config, port);
 444 #else
 445       derr << "WARNING: arrow_flight frontend requested, but not included in build; skipping" << dendl;
 446       continue;
 447 #endif
 448     }
 449
 450     service_map_meta["frontend_type#" + stringify(fe_count)] = framework;
 451     service_map_meta["frontend_config#" + stringify(fe_count)] = config->get_config();
 452
 453     if (! fe) {
 454       dout(0) << "WARNING: skipping unknown framework: " << framework << dendl;
 455       continue;
 456     }
 457
 458     dout(0) << "starting handler: " << fiter->first << dendl;
 459     int r = fe->init();
 460     if (r < 0) {
 461       derr << "ERROR: failed initializing frontend" << dendl;
 462       return -r;
 463     }
 464     r = fe->run();
 465     if (r < 0) {
 466       derr << "ERROR: failed run" << dendl;
 467       return -r;
 468     }
 469
 470     fes.push_back(fe);
 471   }
 472
 473   std::string daemon_type = (nfs) ? "rgw-nfs" : "rgw";
 474   r = env.driver->register_to_service_map(dpp, daemon_type, service_map_meta);
 475   if (r < 0) {
 476     derr << "ERROR: failed to register to service map: " << cpp_strerror(-r) << dendl;
 477     /* ignore error */
 478   }
 479
 480   if (env.driver->get_name() == "rados") {
 481     // add a watcher to respond to realm configuration changes
 482     pusher = std::make_unique<RGWPeriodPusher>(dpp, env.driver, null_yield);
 483     fe_pauser = std::make_unique<RGWFrontendPauser>(fes, pusher.get());
 484     rgw_pauser = std::make_unique<RGWPauser>();
 485     rgw_pauser->add_pauser(fe_pauser.get());
 486     if (env.lua.background) {
 487       rgw_pauser->add_pauser(env.lua.background);
 488     }
 489     reloader = std::make_unique<RGWRealmReloader>(
 490         env, *implicit_tenant_context, service_map_meta, rgw_pauser.get());
 491     realm_watcher = std::make_unique<RGWRealmWatcher>(dpp, g_ceph_context,
 492                                   static_cast<rgw::sal::RadosStore*>(env.driver)->svc()->zone->get_realm());
 493     realm_watcher->add_watcher(RGWRealmNotify::Reload, *reloader);
 494     realm_watcher->add_watcher(RGWRealmNotify::ZonesNeedPeriod, *pusher.get());
 495   }
 496
 497   return r;
 498 } /* init_frontends2 */
 499
 500 void rgw::AppMain::init_tracepoints()
 501 {
 502   TracepointProvider::initialize<rgw_rados_tracepoint_traits>(dpp->get_cct());
 503   TracepointProvider::initialize<rgw_op_tracepoint_traits>(dpp->get_cct());
 504   tracing::rgw::tracer.init("rgw");
 505 } /* init_tracepoints() */
 506
 507 void rgw::AppMain::init_notification_endpoints()
 508 {
 509 #ifdef WITH_RADOSGW_AMQP_ENDPOINT
 510   if (!rgw::amqp::init(dpp->get_cct())) {
 511     derr << "ERROR: failed to initialize AMQP manager" << dendl;
 512   }
 513 #endif
 514 #ifdef WITH_RADOSGW_KAFKA_ENDPOINT
 515   if (!rgw::kafka::init(dpp->get_cct())) {
 516     derr << "ERROR: failed to initialize Kafka manager" << dendl;
 517   }
 518 #endif
 519 } /* init_notification_endpoints */
 520
 521 void rgw::AppMain::init_lua()
 522 {
 523   rgw::sal::Driver* driver = env.driver;
 524   int r{0};
 525   std::string path = g_conf().get_val<std::string>("rgw_luarocks_location");
 526   if (!path.empty()) {
 527     path += "/" + g_conf()->name.to_str();
 528   }
 529   env.lua.luarocks_path = path;
 530
 531 #ifdef WITH_RADOSGW_LUA_PACKAGES
 532   rgw::lua::packages_t failed_packages;
 533   std::string output;
 534   r = rgw::lua::install_packages(dpp, driver, null_yield, path,
 535                                  failed_packages, output);
 536   if (r < 0) {
 537     dout(1) << "WARNING: failed to install lua packages from allowlist"
 538             << dendl;
 539   }
 540   if (!output.empty()) {
 541     dout(10) << "INFO: lua packages installation output: \n" << output << dendl;
 542   }
 543   for (const auto &p : failed_packages) {
 544     dout(5) << "WARNING: failed to install lua package: " << p
 545             << " from allowlist" << dendl;
 546   }
 547 #endif
 548
 549   env.lua.manager = env.driver->get_lua_manager();
 550
 551   if (driver->get_name() == "rados") { /* Supported for only RadosStore */
 552     lua_background = std::make_unique<
 553       rgw::lua::Background>(driver, dpp->get_cct(), path);
 554     lua_background->start();
 555     env.lua.background = lua_background.get();
 556   }
 557 } /* init_lua */
 558
 559 void rgw::AppMain::shutdown(std::function<void(void)> finalize_async_signals)
 560 {
 561   if (env.driver->get_name() == "rados") {
 562     reloader.reset(); // stop the realm reloader
 563   }
 564
 565   for (auto& fe : fes) {
 566     fe->stop();
 567   }
 568
 569   for (auto& fe : fes) {
 570     fe->join();
 571     delete fe;
 572   }
 573
 574   for (auto& fec : fe_configs) {
 575     delete fec;
 576   }
 577
 578   ldh.reset(nullptr); // deletes
 579   finalize_async_signals(); // callback
 580   rgw_log_usage_finalize();
 581
 582   delete olog;
 583
 584   if (lua_background) {
 585     lua_background->shutdown();
 586   }
 587
 588   DriverManager::close_storage(env.driver);
 589
 590   rgw_tools_cleanup();
 591   rgw_shutdown_resolver();
 592   rgw_http_client_cleanup();
 593   rgw_kmip_client_cleanup();
 594   rgw::curl::cleanup_curl();
 595   g_conf().remove_observer(implicit_tenant_context.get());
 596   implicit_tenant_context.reset(); // deletes
 597 #ifdef WITH_RADOSGW_AMQP_ENDPOINT
 598   rgw::amqp::shutdown();
 599 #endif
 600 #ifdef WITH_RADOSGW_KAFKA_ENDPOINT
 601   rgw::kafka::shutdown();
 602 #endif
 603   rgw_perf_stop(g_ceph_context);
 604   ratelimiter.reset(); // deletes--ensure this happens before we destruct
 605 } /* AppMain::shutdown */