1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
4 #include "common/ceph_argparse.h"
5 #include "global/global_init.h"
6 #include "global/signal_handler.h"
7 #include "common/config.h"
8 #include "common/errno.h"
9 #include "common/Timer.h"
10 #include "common/safe_io.h"
11 #include "common/TracepointProvider.h"
12 #include "common/numa.h"
13 #include "include/compat.h"
14 #include "include/str_list.h"
15 #include "include/stringify.h"
16 #include "rgw_common.h"
17 #include "rgw_rados.h"
18 #include "rgw_period_pusher.h"
19 #include "rgw_realm_reloader.h"
21 #include "rgw_rest_s3.h"
22 #include "rgw_rest_swift.h"
23 #include "rgw_rest_admin.h"
24 #include "rgw_rest_usage.h"
25 #include "rgw_rest_user.h"
26 #include "rgw_rest_bucket.h"
27 #include "rgw_rest_metadata.h"
28 #include "rgw_rest_log.h"
29 #include "rgw_rest_config.h"
30 #include "rgw_rest_realm.h"
31 #include "rgw_rest_sts.h"
32 #include "rgw_swift_auth.h"
34 #include "rgw_tools.h"
35 #include "rgw_resolve.h"
36 #include "rgw_request.h"
37 #include "rgw_process.h"
38 #include "rgw_frontend.h"
39 #include "rgw_http_client_curl.h"
40 #include "rgw_perf_counters.h"
41 #ifdef WITH_RADOSGW_AMQP_ENDPOINT
44 #ifdef WITH_RADOSGW_KAFKA_ENDPOINT
45 #include "rgw_kafka.h"
47 #if defined(WITH_RADOSGW_BEAST_FRONTEND)
48 #include "rgw_asio_frontend.h"
49 #endif /* WITH_RADOSGW_BEAST_FRONTEND */
50 #include "rgw_dmclock_scheduler_ctx.h"
52 #include "services/svc_zone.h"
54 #ifdef HAVE_SYS_PRCTL_H
55 #include <sys/prctl.h>
58 #define dout_subsys ceph_subsys_rgw
61 TracepointProvider::Traits
rgw_op_tracepoint_traits("librgw_op_tp.so",
63 TracepointProvider::Traits
rgw_rados_tracepoint_traits("librgw_rados_tp.so",
67 static sig_t sighandler_alrm
;
71 static int signal_fd
[2] = {0, 0};
73 void signal_shutdown()
76 int ret
= write(signal_fd
[0], (char *)&val
, sizeof(val
));
78 derr
<< "ERROR: " << __func__
<< ": write() returned "
79 << cpp_strerror(errno
) << dendl
;
83 static void wait_shutdown()
86 int r
= safe_read_exact(signal_fd
[1], &val
, sizeof(val
));
88 derr
<< "safe_read_exact returned with error" << dendl
;
92 static int signal_fd_init()
94 return socketpair(AF_UNIX
, SOCK_STREAM
, 0, signal_fd
);
97 static void signal_fd_finalize()
103 static void handle_sigterm(int signum
)
105 dout(1) << __func__
<< dendl
;
106 #if defined(WITH_RADOSGW_FCGI_FRONTEND)
107 FCGX_ShutdownPending();
110 // send a signal to make fcgi's accept(2) wake up. unfortunately the
111 // initial signal often isn't sufficient because we race with accept's
112 // check of the flag wet by ShutdownPending() above.
113 if (signum
!= SIGUSR1
) {
116 // safety net in case we get stuck doing an orderly shutdown.
117 uint64_t secs
= g_ceph_context
->_conf
->rgw_exit_timeout_secs
;
120 dout(1) << __func__
<< " set alarm for " << secs
<< dendl
;
125 static void godown_alarm(int signum
)
131 class C_InitTimeout
: public Context
{
134 void finish(int r
) override
{
135 derr
<< "Initialization timeout, failed to initialize" << dendl
;
142 cout
<< "usage: radosgw [options...]" << std::endl
;
143 cout
<< "options:\n";
144 cout
<< " --rgw-region=<region> region in which radosgw runs\n";
145 cout
<< " --rgw-zone=<zone> zone in which radosgw runs\n";
146 cout
<< " --rgw-socket-path=<path> specify a unix domain socket path\n";
147 cout
<< " -m monaddress[:port] connect to specified monitor\n";
148 cout
<< " --keyring=<path> path to radosgw keyring\n";
149 cout
<< " --logfile=<logfile> file to log debug output\n";
150 cout
<< " --debug-rgw=<log-level>/<memory-level> set radosgw debug level\n";
151 generic_server_usage();
156 static RGWRESTMgr
*set_logging(RGWRESTMgr
*mgr
)
158 mgr
->set_logging(true);
162 static RGWRESTMgr
*rest_filter(RGWRados
*store
, int dialect
, RGWRESTMgr
*orig
)
164 RGWSyncModuleInstanceRef sync_module
= store
->get_sync_module();
166 return sync_module
->get_rest_filter(dialect
, orig
);
173 * start up the RADOS connection and then handle HTTP messages as they come in
175 int radosgw_Main(int argc
, const char **argv
)
177 // dout() messages will be sent to stderr, but FCGX wants messages on stdout
178 // Redirect stderr to stdout.
179 TEMP_FAILURE_RETRY(close(STDERR_FILENO
));
180 if (TEMP_FAILURE_RETRY(dup2(STDOUT_FILENO
, STDERR_FILENO
)) < 0) {
182 cout
<< "failed to redirect stderr to stdout: " << cpp_strerror(err
)
187 /* alternative default for module */
188 map
<string
,string
> defaults
= {
189 { "debug_rgw", "1/5" },
190 { "keyring", "$rgw_data/keyring" },
191 { "objecter_inflight_ops", "24576" }
194 vector
<const char*> args
;
195 argv_to_vec(argc
, argv
, args
);
197 cerr
<< argv
[0] << ": -h or --help for usage" << std::endl
;
200 if (ceph_argparse_need_usage(args
)) {
205 int flags
= CINIT_FLAG_UNPRIVILEGED_DAEMON_DEFAULTS
;
206 // Prevent global_init() from dropping permissions until frontends can bind
208 flags
|= CINIT_FLAG_DEFER_DROP_PRIVILEGES
;
210 auto cct
= global_init(&defaults
, args
, CEPH_ENTITY_TYPE_CLIENT
,
211 CODE_ENVIRONMENT_DAEMON
,
214 // First, let's determine which frontends are configured.
215 list
<string
> frontends
;
216 string rgw_frontends_str
= g_conf().get_val
<string
>("rgw_frontends");
217 g_conf().early_expand_meta(rgw_frontends_str
, &cerr
);
218 get_str_list(rgw_frontends_str
, ",", frontends
);
219 multimap
<string
, RGWFrontendConfig
*> fe_map
;
220 list
<RGWFrontendConfig
*> configs
;
221 if (frontends
.empty()) {
222 frontends
.push_back("civetweb");
224 for (list
<string
>::iterator iter
= frontends
.begin(); iter
!= frontends
.end(); ++iter
) {
227 if (f
.find("civetweb") != string::npos
|| f
.find("beast") != string::npos
) {
228 if (f
.find("port") != string::npos
) {
229 // check for the most common ws problems
230 if ((f
.find("port=") == string::npos
) ||
231 (f
.find("port= ") != string::npos
)) {
232 derr
<< "WARNING: radosgw frontend config found unexpected spacing around 'port' "
233 << "(ensure frontend port parameter has the form 'port=80' with no spaces "
234 << "before or after '=')" << dendl
;
239 RGWFrontendConfig
*config
= new RGWFrontendConfig(f
);
240 int r
= config
->init();
243 cerr
<< "ERROR: failed to init config: " << f
<< std::endl
;
247 configs
.push_back(config
);
249 string framework
= config
->get_framework();
250 fe_map
.insert(pair
<string
, RGWFrontendConfig
*>(framework
, config
));
253 int numa_node
= g_conf().get_val
<int64_t>("rgw_numa_node");
254 size_t numa_cpu_set_size
= 0;
255 cpu_set_t numa_cpu_set
;
257 if (numa_node
>= 0) {
258 int r
= get_numa_node_cpu_set(numa_node
, &numa_cpu_set_size
, &numa_cpu_set
);
260 dout(1) << __func__
<< " unable to determine rgw numa node " << numa_node
264 r
= set_cpu_affinity_all_threads(numa_cpu_set_size
, &numa_cpu_set
);
266 derr
<< __func__
<< " failed to set numa affinity: " << cpp_strerror(r
)
271 dout(1) << __func__
<< " not setting numa affinity" << dendl
;
274 // maintain existing region root pool for new multisite objects
275 if (!g_conf()->rgw_region_root_pool
.empty()) {
276 const char *root_pool
= g_conf()->rgw_region_root_pool
.c_str();
277 if (g_conf()->rgw_zonegroup_root_pool
.empty()) {
278 g_conf().set_val_or_die("rgw_zonegroup_root_pool", root_pool
);
280 if (g_conf()->rgw_period_root_pool
.empty()) {
281 g_conf().set_val_or_die("rgw_period_root_pool", root_pool
);
283 if (g_conf()->rgw_realm_root_pool
.empty()) {
284 g_conf().set_val_or_die("rgw_realm_root_pool", root_pool
);
288 // for region -> zonegroup conversion (must happen before common_init_finish())
289 if (!g_conf()->rgw_region
.empty() && g_conf()->rgw_zonegroup
.empty()) {
290 g_conf().set_val_or_die("rgw_zonegroup", g_conf()->rgw_region
.c_str());
293 if (g_conf()->daemonize
) {
294 global_init_daemonize(g_ceph_context
);
296 ceph::mutex mutex
= ceph::make_mutex("main");
297 SafeTimer
init_timer(g_ceph_context
, mutex
);
300 init_timer
.add_event_after(g_conf()->rgw_init_timeout
, new C_InitTimeout
);
303 common_init_finish(g_ceph_context
);
305 init_async_signal_handler();
306 register_async_signal_handler(SIGHUP
, sighup_handler
);
308 TracepointProvider::initialize
<rgw_rados_tracepoint_traits
>(g_ceph_context
);
309 TracepointProvider::initialize
<rgw_op_tracepoint_traits
>(g_ceph_context
);
311 int r
= rgw_tools_init(g_ceph_context
);
313 derr
<< "ERROR: unable to initialize rgw tools" << dendl
;
318 rgw::curl::setup_curl(fe_map
);
319 rgw_http_client_init(g_ceph_context
);
321 #if defined(WITH_RADOSGW_FCGI_FRONTEND)
325 rgw::sal::RGWRadosStore
*store
=
326 RGWStoreManager::get_storage(g_ceph_context
,
327 g_conf()->rgw_enable_gc_threads
,
328 g_conf()->rgw_enable_lc_threads
,
329 g_conf()->rgw_enable_quota_threads
,
330 g_conf()->rgw_run_sync_thread
,
331 g_conf().get_val
<bool>("rgw_dynamic_resharding"),
332 g_conf()->rgw_cache_enabled
);
335 init_timer
.cancel_all_events();
336 init_timer
.shutdown();
339 derr
<< "Couldn't init storage provider (RADOS)" << dendl
;
342 r
= rgw_perf_start(g_ceph_context
);
344 derr
<< "ERROR: failed starting rgw perf" << dendl
;
348 rgw_rest_init(g_ceph_context
, store
->svc()->zone
->get_zonegroup());
351 init_timer
.cancel_all_events();
352 init_timer
.shutdown();
355 rgw_log_usage_init(g_ceph_context
, store
->getRados());
361 get_str_list(g_conf()->rgw_enable_apis
, apis
);
363 map
<string
, bool> apis_map
;
364 for (list
<string
>::iterator li
= apis
.begin(); li
!= apis
.end(); ++li
) {
365 apis_map
[*li
] = true;
368 /* warn about insecure keystone secret config options */
369 if (!(g_ceph_context
->_conf
->rgw_keystone_admin_token
.empty() ||
370 g_ceph_context
->_conf
->rgw_keystone_admin_password
.empty())) {
371 dout(0) << "WARNING: rgw_keystone_admin_token and rgw_keystone_admin_password should be avoided as they can expose secrets. Prefer the new rgw_keystone_admin_token_path and rgw_keystone_admin_password_path options, which read their secrets from files." << dendl
;
374 // S3 website mode is a specialization of S3
375 const bool s3website_enabled
= apis_map
.count("s3website") > 0;
376 const bool sts_enabled
= apis_map
.count("sts") > 0;
377 const bool iam_enabled
= apis_map
.count("iam") > 0;
378 const bool pubsub_enabled
= apis_map
.count("pubsub") > 0;
379 // Swift API entrypoint could placed in the root instead of S3
380 const bool swift_at_root
= g_conf()->rgw_swift_url_prefix
== "/";
381 if (apis_map
.count("s3") > 0 || s3website_enabled
) {
382 if (! swift_at_root
) {
383 rest
.register_default_mgr(set_logging(rest_filter(store
->getRados(), RGW_REST_S3
,
384 new RGWRESTMgr_S3(s3website_enabled
, sts_enabled
, iam_enabled
, pubsub_enabled
))));
386 derr
<< "Cannot have the S3 or S3 Website enabled together with "
387 << "Swift API placed in the root of hierarchy" << dendl
;
392 if (pubsub_enabled
) {
393 #ifdef WITH_RADOSGW_AMQP_ENDPOINT
394 if (!rgw::amqp::init(cct
.get())) {
395 dout(1) << "ERROR: failed to initialize AMQP manager" << dendl
;
398 #ifdef WITH_RADOSGW_KAFKA_ENDPOINT
399 if (!rgw::kafka::init(cct
.get())) {
400 dout(1) << "ERROR: failed to initialize Kafka manager" << dendl
;
405 if (apis_map
.count("swift") > 0) {
406 RGWRESTMgr_SWIFT
* const swift_resource
= new RGWRESTMgr_SWIFT
;
408 if (! g_conf()->rgw_cross_domain_policy
.empty()) {
409 swift_resource
->register_resource("crossdomain.xml",
410 set_logging(new RGWRESTMgr_SWIFT_CrossDomain
));
413 swift_resource
->register_resource("healthcheck",
414 set_logging(new RGWRESTMgr_SWIFT_HealthCheck
));
416 swift_resource
->register_resource("info",
417 set_logging(new RGWRESTMgr_SWIFT_Info
));
419 if (! swift_at_root
) {
420 rest
.register_resource(g_conf()->rgw_swift_url_prefix
,
421 set_logging(rest_filter(store
->getRados(), RGW_REST_SWIFT
,
424 if (store
->svc()->zone
->get_zonegroup().zones
.size() > 1) {
425 derr
<< "Placing Swift API in the root of URL hierarchy while running"
426 << " multi-site configuration requires another instance of RadosGW"
427 << " with S3 API enabled!" << dendl
;
430 rest
.register_default_mgr(set_logging(swift_resource
));
434 if (apis_map
.count("swift_auth") > 0) {
435 rest
.register_resource(g_conf()->rgw_swift_auth_entry
,
436 set_logging(new RGWRESTMgr_SWIFT_Auth
));
439 if (apis_map
.count("admin") > 0) {
440 RGWRESTMgr_Admin
*admin_resource
= new RGWRESTMgr_Admin
;
441 admin_resource
->register_resource("usage", new RGWRESTMgr_Usage
);
442 admin_resource
->register_resource("user", new RGWRESTMgr_User
);
443 admin_resource
->register_resource("bucket", new RGWRESTMgr_Bucket
);
445 /*Registering resource for /admin/metadata */
446 admin_resource
->register_resource("metadata", new RGWRESTMgr_Metadata
);
447 admin_resource
->register_resource("log", new RGWRESTMgr_Log
);
448 admin_resource
->register_resource("config", new RGWRESTMgr_Config
);
449 admin_resource
->register_resource("realm", new RGWRESTMgr_Realm
);
450 rest
.register_resource(g_conf()->rgw_admin_entry
, admin_resource
);
453 /* Initialize the registry of auth strategies which will coordinate
454 * the dynamic reconfiguration. */
455 rgw::auth::ImplicitTenants implicit_tenant_context
{g_conf()};
456 g_conf().add_observer(&implicit_tenant_context
);
457 auto auth_registry
= \
458 rgw::auth::StrategyRegistry::create(g_ceph_context
, implicit_tenant_context
, store
->getRados()->pctl
);
460 /* Header custom behavior */
461 rest
.register_x_headers(g_conf()->rgw_log_http_headers
);
463 if (cct
->_conf
.get_val
<std::string
>("rgw_scheduler_type") == "dmclock" &&
464 !cct
->check_experimental_feature_enabled("dmclock")){
465 derr
<< "dmclock scheduler type is experimental and needs to be"
466 << "set in the option enable experimental data corrupting features"
471 rgw::dmclock::SchedulerCtx sched_ctx
{cct
.get()};
473 OpsLogSocket
*olog
= NULL
;
475 if (!g_conf()->rgw_ops_log_socket_path
.empty()) {
476 olog
= new OpsLogSocket(g_ceph_context
, g_conf()->rgw_ops_log_data_backlog
);
477 olog
->init(g_conf()->rgw_ops_log_socket_path
);
480 r
= signal_fd_init();
482 derr
<< "ERROR: unable to initialize signal fds" << dendl
;
486 register_async_signal_handler(SIGTERM
, handle_sigterm
);
487 register_async_signal_handler(SIGINT
, handle_sigterm
);
488 register_async_signal_handler(SIGUSR1
, handle_sigterm
);
489 sighandler_alrm
= signal(SIGALRM
, godown_alarm
);
491 map
<string
, string
> service_map_meta
;
492 service_map_meta
["pid"] = stringify(getpid());
494 list
<RGWFrontend
*> fes
;
496 string frontend_defs_str
= g_conf().get_val
<string
>("rgw_frontend_defaults");
498 list
<string
> frontends_def
;
499 get_str_list(frontend_defs_str
, ",", frontends_def
);
501 map
<string
, std::unique_ptr
<RGWFrontendConfig
> > fe_def_map
;
502 for (auto& f
: frontends_def
) {
503 RGWFrontendConfig
*config
= new RGWFrontendConfig(f
);
504 int r
= config
->init();
507 cerr
<< "ERROR: failed to init default config: " << f
<< std::endl
;
511 fe_def_map
[config
->get_framework()].reset(config
);
516 for (multimap
<string
, RGWFrontendConfig
*>::iterator fiter
= fe_map
.begin();
517 fiter
!= fe_map
.end(); ++fiter
, ++fe_count
) {
518 RGWFrontendConfig
*config
= fiter
->second
;
519 string framework
= config
->get_framework();
521 auto def_iter
= fe_def_map
.find(framework
);
522 if (def_iter
!= fe_def_map
.end()) {
523 config
->set_default_config(*def_iter
->second
);
526 RGWFrontend
*fe
= NULL
;
528 if (framework
== "civetweb" || framework
== "mongoose") {
529 framework
= "civetweb";
530 std::string uri_prefix
;
531 config
->get_val("prefix", "", &uri_prefix
);
533 RGWProcessEnv env
= { store
, &rest
, olog
, 0, uri_prefix
, auth_registry
};
534 //TODO: move all of scheduler initializations to frontends?
536 fe
= new RGWCivetWebFrontend(env
, config
, sched_ctx
);
538 else if (framework
== "loadgen") {
540 config
->get_val("port", 80, &port
);
541 std::string uri_prefix
;
542 config
->get_val("prefix", "", &uri_prefix
);
544 RGWProcessEnv env
= { store
, &rest
, olog
, port
, uri_prefix
, auth_registry
};
546 fe
= new RGWLoadGenFrontend(env
, config
);
548 #if defined(WITH_RADOSGW_BEAST_FRONTEND)
549 else if (framework
== "beast") {
551 config
->get_val("port", 80, &port
);
552 std::string uri_prefix
;
553 config
->get_val("prefix", "", &uri_prefix
);
554 RGWProcessEnv env
{ store
, &rest
, olog
, port
, uri_prefix
, auth_registry
};
555 fe
= new RGWAsioFrontend(env
, config
, sched_ctx
);
557 #endif /* WITH_RADOSGW_BEAST_FRONTEND */
558 #if defined(WITH_RADOSGW_FCGI_FRONTEND)
559 else if (framework
== "fastcgi" || framework
== "fcgi") {
560 framework
= "fastcgi";
561 std::string uri_prefix
;
562 config
->get_val("prefix", "", &uri_prefix
);
563 RGWProcessEnv fcgi_pe
= { store
, &rest
, olog
, 0, uri_prefix
, auth_registry
};
565 fe
= new RGWFCGXFrontend(fcgi_pe
, config
);
567 #endif /* WITH_RADOSGW_FCGI_FRONTEND */
569 service_map_meta
["frontend_type#" + stringify(fe_count
)] = framework
;
570 service_map_meta
["frontend_config#" + stringify(fe_count
)] = config
->get_config();
573 dout(0) << "WARNING: skipping unknown framework: " << framework
<< dendl
;
577 dout(0) << "starting handler: " << fiter
->first
<< dendl
;
580 derr
<< "ERROR: failed initializing frontend" << dendl
;
585 derr
<< "ERROR: failed run" << dendl
;
592 r
= store
->getRados()->register_to_service_map("rgw", service_map_meta
);
594 derr
<< "ERROR: failed to register to service map: " << cpp_strerror(-r
) << dendl
;
600 // add a watcher to respond to realm configuration changes
601 RGWPeriodPusher
pusher(store
);
602 RGWFrontendPauser
pauser(fes
, implicit_tenant_context
, &pusher
);
603 auto reloader
= std::make_unique
<RGWRealmReloader
>(store
,
604 service_map_meta
, &pauser
);
606 RGWRealmWatcher
realm_watcher(g_ceph_context
, store
->svc()->zone
->get_realm());
607 realm_watcher
.add_watcher(RGWRealmNotify::Reload
, *reloader
);
608 realm_watcher
.add_watcher(RGWRealmNotify::ZonesNeedPeriod
, pusher
);
610 #if defined(HAVE_SYS_PRCTL_H)
611 if (prctl(PR_SET_DUMPABLE
, 1) == -1) {
612 cerr
<< "warning: unable to set dumpable flag: " << cpp_strerror(errno
) << std::endl
;
618 derr
<< "shutting down" << dendl
;
620 reloader
.reset(); // stop the realm reloader
622 for (list
<RGWFrontend
*>::iterator liter
= fes
.begin(); liter
!= fes
.end();
624 RGWFrontend
*fe
= *liter
;
628 for (list
<RGWFrontend
*>::iterator liter
= fes
.begin(); liter
!= fes
.end();
630 RGWFrontend
*fe
= *liter
;
635 for (list
<RGWFrontendConfig
*>::iterator liter
= configs
.begin();
636 liter
!= configs
.end(); ++liter
) {
637 RGWFrontendConfig
*fec
= *liter
;
641 unregister_async_signal_handler(SIGHUP
, sighup_handler
);
642 unregister_async_signal_handler(SIGTERM
, handle_sigterm
);
643 unregister_async_signal_handler(SIGINT
, handle_sigterm
);
644 unregister_async_signal_handler(SIGUSR1
, handle_sigterm
);
645 shutdown_async_signal_handler();
647 rgw_log_usage_finalize();
651 RGWStoreManager::close_storage(store
);
652 rgw::auth::s3::LDAPEngine::shutdown();
654 rgw_shutdown_resolver();
655 rgw_http_client_cleanup();
656 rgw::curl::cleanup_curl();
657 g_conf().remove_observer(&implicit_tenant_context
);
658 #ifdef WITH_RADOSGW_AMQP_ENDPOINT
659 rgw::amqp::shutdown();
661 #ifdef WITH_RADOSGW_KAFKA_ENDPOINT
662 rgw::kafka::shutdown();
665 rgw_perf_stop(g_ceph_context
);
667 dout(1) << "final shutdown" << dendl
;
669 signal_fd_finalize();
676 int radosgw_main(int argc
, const char** argv
)
678 return radosgw_Main(argc
, argv
);