]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
9f95a23c | 2 | // vim: ts=8 sw=2 smarttab ft=cpp |
7c673cae FG |
3 | |
4 | #include "rgw_realm_reloader.h" | |
7c673cae FG |
5 | |
6 | #include "rgw_bucket.h" | |
7 | #include "rgw_log.h" | |
8 | #include "rgw_rest.h" | |
9 | #include "rgw_user.h" | |
9f95a23c | 10 | #include "rgw_sal.h" |
f67539c2 | 11 | #include "rgw_sal_rados.h" |
7c673cae | 12 | |
11fdf7f2 TL |
13 | #include "services/svc_zone.h" |
14 | ||
224ce89b WB |
15 | #include "common/errno.h" |
16 | ||
7c673cae FG |
17 | #define dout_subsys ceph_subsys_rgw |
18 | ||
19 | #undef dout_prefix | |
20 | #define dout_prefix (*_dout << "rgw realm reloader: ") | |
21 | ||
22 | ||
23 | // safe callbacks from SafeTimer are unneccessary. reload() can take a long | |
24 | // time, so we don't want to hold the mutex and block handle_notify() for the | |
25 | // duration | |
26 | static constexpr bool USE_SAFE_TIMER_CALLBACKS = false; | |
27 | ||
28 | ||
20effc67 | 29 | RGWRealmReloader::RGWRealmReloader(rgw::sal::Store*& store, std::map<std::string, std::string>& service_map_meta, |
224ce89b | 30 | Pauser* frontends) |
7c673cae | 31 | : store(store), |
224ce89b | 32 | service_map_meta(service_map_meta), |
7c673cae FG |
33 | frontends(frontends), |
34 | timer(store->ctx(), mutex, USE_SAFE_TIMER_CALLBACKS), | |
9f95a23c | 35 | mutex(ceph::make_mutex("RGWRealmReloader")), |
7c673cae FG |
36 | reload_scheduled(nullptr) |
37 | { | |
38 | timer.init(); | |
39 | } | |
40 | ||
41 | RGWRealmReloader::~RGWRealmReloader() | |
42 | { | |
9f95a23c | 43 | std::lock_guard lock{mutex}; |
7c673cae FG |
44 | timer.shutdown(); |
45 | } | |
46 | ||
47 | class RGWRealmReloader::C_Reload : public Context { | |
48 | RGWRealmReloader* reloader; | |
49 | public: | |
11fdf7f2 | 50 | explicit C_Reload(RGWRealmReloader* reloader) : reloader(reloader) {} |
7c673cae FG |
51 | void finish(int r) override { reloader->reload(); } |
52 | }; | |
53 | ||
54 | void RGWRealmReloader::handle_notify(RGWRealmNotify type, | |
11fdf7f2 | 55 | bufferlist::const_iterator& p) |
7c673cae FG |
56 | { |
57 | if (!store) { | |
58 | /* we're in the middle of reload */ | |
59 | return; | |
60 | } | |
61 | ||
62 | CephContext *const cct = store->ctx(); | |
63 | ||
9f95a23c | 64 | std::lock_guard lock{mutex}; |
7c673cae FG |
65 | if (reload_scheduled) { |
66 | ldout(cct, 4) << "Notification on realm, reconfiguration " | |
67 | "already scheduled" << dendl; | |
68 | return; | |
69 | } | |
70 | ||
71 | reload_scheduled = new C_Reload(this); | |
9f95a23c | 72 | cond.notify_one(); // wake reload() if it blocked on a bad configuration |
7c673cae FG |
73 | |
74 | // schedule reload() without delay | |
75 | timer.add_event_after(0, reload_scheduled); | |
76 | ||
77 | ldout(cct, 4) << "Notification on realm, reconfiguration scheduled" << dendl; | |
78 | } | |
79 | ||
80 | void RGWRealmReloader::reload() | |
81 | { | |
82 | CephContext *const cct = store->ctx(); | |
b3b6e05e TL |
83 | const DoutPrefix dp(cct, dout_subsys, "rgw realm reloader: "); |
84 | ldpp_dout(&dp, 1) << "Pausing frontends for realm update..." << dendl; | |
7c673cae FG |
85 | |
86 | frontends->pause(); | |
87 | ||
b3b6e05e | 88 | ldpp_dout(&dp, 1) << "Frontends paused" << dendl; |
7c673cae FG |
89 | |
90 | // TODO: make RGWRados responsible for rgw_log_usage lifetime | |
91 | rgw_log_usage_finalize(); | |
92 | ||
93 | // destroy the existing store | |
20effc67 | 94 | StoreManager::close_storage(store); |
7c673cae FG |
95 | store = nullptr; |
96 | ||
b3b6e05e | 97 | ldpp_dout(&dp, 1) << "Store closed" << dendl; |
7c673cae FG |
98 | { |
99 | // allow a new notify to reschedule us. it's important that we do this | |
100 | // before we start loading the new realm, or we could miss some updates | |
9f95a23c | 101 | std::lock_guard lock{mutex}; |
7c673cae FG |
102 | reload_scheduled = nullptr; |
103 | } | |
104 | ||
b3b6e05e | 105 | |
7c673cae FG |
106 | while (!store) { |
107 | // recreate and initialize a new store | |
11fdf7f2 | 108 | store = |
20effc67 TL |
109 | StoreManager::get_storage(&dp, cct, |
110 | "rados", | |
11fdf7f2 TL |
111 | cct->_conf->rgw_enable_gc_threads, |
112 | cct->_conf->rgw_enable_lc_threads, | |
113 | cct->_conf->rgw_enable_quota_threads, | |
114 | cct->_conf->rgw_run_sync_thread, | |
115 | cct->_conf.get_val<bool>("rgw_dynamic_resharding"), | |
116 | cct->_conf->rgw_cache_enabled); | |
7c673cae | 117 | |
b3b6e05e | 118 | ldpp_dout(&dp, 1) << "Creating new store" << dendl; |
7c673cae | 119 | |
20effc67 | 120 | rgw::sal::Store* store_cleanup = nullptr; |
7c673cae | 121 | { |
9f95a23c | 122 | std::unique_lock lock{mutex}; |
7c673cae FG |
123 | |
124 | // failure to recreate RGWRados is not a recoverable error, but we | |
125 | // don't want to assert or abort the entire cluster. instead, just | |
126 | // sleep until we get another notification, and retry until we get | |
127 | // a working configuration | |
128 | if (store == nullptr) { | |
b3b6e05e | 129 | ldpp_dout(&dp, -1) << "Failed to reinitialize RGWRados after a realm " |
7c673cae FG |
130 | "configuration update. Waiting for a new update." << dendl; |
131 | ||
132 | // sleep until another event is scheduled | |
9f95a23c | 133 | cond.wait(lock, [this] { return reload_scheduled; }); |
7c673cae FG |
134 | ldout(cct, 1) << "Woke up with a new configuration, retrying " |
135 | "RGWRados initialization." << dendl; | |
136 | } | |
137 | ||
138 | if (reload_scheduled) { | |
139 | // cancel the event; we'll handle it now | |
140 | timer.cancel_event(reload_scheduled); | |
141 | reload_scheduled = nullptr; | |
142 | ||
143 | // if we successfully created a store, clean it up outside of the lock, | |
144 | // then continue to loop and recreate another | |
145 | std::swap(store, store_cleanup); | |
146 | } | |
147 | } | |
148 | ||
149 | if (store_cleanup) { | |
b3b6e05e | 150 | ldpp_dout(&dp, 4) << "Got another notification, restarting RGWRados " |
7c673cae FG |
151 | "initialization." << dendl; |
152 | ||
20effc67 | 153 | StoreManager::close_storage(store_cleanup); |
7c673cae FG |
154 | } |
155 | } | |
156 | ||
20effc67 | 157 | int r = store->register_to_service_map(&dp, "rgw", service_map_meta); |
224ce89b | 158 | if (r < 0) { |
b3b6e05e | 159 | ldpp_dout(&dp, -1) << "ERROR: failed to register to service map: " << cpp_strerror(-r) << dendl; |
224ce89b WB |
160 | |
161 | /* ignore error */ | |
162 | } | |
163 | ||
b3b6e05e | 164 | ldpp_dout(&dp, 1) << "Finishing initialization of new store" << dendl; |
7c673cae | 165 | // finish initializing the new store |
b3b6e05e | 166 | ldpp_dout(&dp, 1) << " - REST subsystem init" << dendl; |
20effc67 | 167 | rgw_rest_init(cct, store->get_zone()->get_zonegroup()); |
b3b6e05e | 168 | ldpp_dout(&dp, 1) << " - usage subsystem init" << dendl; |
20effc67 | 169 | rgw_log_usage_init(cct, store); |
7c673cae | 170 | |
b3b6e05e | 171 | ldpp_dout(&dp, 1) << "Resuming frontends with new realm configuration." << dendl; |
7c673cae FG |
172 | |
173 | frontends->resume(store); | |
174 | } |