]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
4 | #include "ClusterWatcher.h" | |
11fdf7f2 TL |
5 | #include "include/stringify.h" |
6 | #include "common/ceph_json.h" | |
7c673cae FG |
7 | #include "common/debug.h" |
8 | #include "common/errno.h" | |
224ce89b | 9 | #include "cls/rbd/cls_rbd_client.h" |
7c673cae FG |
10 | #include "librbd/internal.h" |
11 | #include "librbd/api/Mirror.h" | |
c07f9fc5 | 12 | #include "tools/rbd_mirror/ServiceDaemon.h" |
9f95a23c | 13 | #include "json_spirit/json_spirit.h" |
7c673cae FG |
14 | |
15 | #define dout_context g_ceph_context | |
16 | #define dout_subsys ceph_subsys_rbd_mirror | |
17 | #undef dout_prefix | |
18 | #define dout_prefix *_dout << "rbd::mirror::ClusterWatcher:" << this << " " \ | |
19 | << __func__ << ": " | |
20 | ||
21 | using std::list; | |
22 | using std::map; | |
23 | using std::set; | |
24 | using std::string; | |
25 | using std::unique_ptr; | |
26 | using std::vector; | |
27 | ||
28 | using librados::Rados; | |
29 | using librados::IoCtx; | |
30 | ||
31 | namespace rbd { | |
32 | namespace mirror { | |
33 | ||
9f95a23c | 34 | ClusterWatcher::ClusterWatcher(RadosRef cluster, ceph::mutex &lock, |
c07f9fc5 FG |
35 | ServiceDaemon<librbd::ImageCtx>* service_daemon) |
36 | : m_cluster(cluster), m_lock(lock), m_service_daemon(service_daemon) | |
7c673cae FG |
37 | { |
38 | } | |
39 | ||
40 | const ClusterWatcher::PoolPeers& ClusterWatcher::get_pool_peers() const | |
41 | { | |
9f95a23c | 42 | ceph_assert(ceph_mutex_is_locked(m_lock)); |
7c673cae FG |
43 | return m_pool_peers; |
44 | } | |
45 | ||
9f95a23c TL |
46 | std::string ClusterWatcher::get_site_name() const { |
47 | ceph_assert(ceph_mutex_is_locked(m_lock)); | |
48 | return m_site_name; | |
49 | } | |
50 | ||
7c673cae FG |
51 | void ClusterWatcher::refresh_pools() |
52 | { | |
53 | dout(20) << "enter" << dendl; | |
54 | ||
55 | PoolPeers pool_peers; | |
11fdf7f2 | 56 | read_pool_peers(&pool_peers); |
7c673cae | 57 | |
9f95a23c TL |
58 | std::string site_name; |
59 | int r = read_site_name(&site_name); | |
60 | ||
61 | std::lock_guard l{m_lock}; | |
7c673cae | 62 | m_pool_peers = pool_peers; |
9f95a23c TL |
63 | |
64 | if (r >= 0) { | |
65 | m_site_name = site_name; | |
66 | } | |
67 | ||
7c673cae FG |
68 | // TODO: perhaps use a workqueue instead, once we get notifications |
69 | // about config changes for existing pools | |
70 | } | |
71 | ||
11fdf7f2 | 72 | void ClusterWatcher::read_pool_peers(PoolPeers *pool_peers) |
7c673cae | 73 | { |
b32b8144 FG |
74 | int r = m_cluster->wait_for_latest_osdmap(); |
75 | if (r < 0) { | |
76 | derr << "error waiting for OSD map: " << cpp_strerror(r) << dendl; | |
77 | return; | |
78 | } | |
79 | ||
7c673cae | 80 | list<pair<int64_t, string> > pools; |
b32b8144 | 81 | r = m_cluster->pool_list2(pools); |
7c673cae FG |
82 | if (r < 0) { |
83 | derr << "error listing pools: " << cpp_strerror(r) << dendl; | |
84 | return; | |
85 | } | |
86 | ||
c07f9fc5 FG |
87 | std::set<int64_t> service_pool_ids; |
88 | for (auto& kv : pools) { | |
7c673cae | 89 | int64_t pool_id = kv.first; |
c07f9fc5 | 90 | auto& pool_name = kv.second; |
7c673cae FG |
91 | int64_t base_tier; |
92 | r = m_cluster->pool_get_base_tier(pool_id, &base_tier); | |
93 | if (r == -ENOENT) { | |
94 | dout(10) << "pool " << pool_name << " no longer exists" << dendl; | |
95 | continue; | |
96 | } else if (r < 0) { | |
97 | derr << "Error retrieving base tier for pool " << pool_name << dendl; | |
98 | continue; | |
99 | } | |
100 | if (pool_id != base_tier) { | |
101 | // pool is a cache; skip it | |
102 | continue; | |
103 | } | |
104 | ||
105 | IoCtx ioctx; | |
106 | r = m_cluster->ioctx_create2(pool_id, ioctx); | |
107 | if (r == -ENOENT) { | |
108 | dout(10) << "pool " << pool_id << " no longer exists" << dendl; | |
109 | continue; | |
110 | } else if (r < 0) { | |
111 | derr << "Error accessing pool " << pool_name << cpp_strerror(r) << dendl; | |
112 | continue; | |
113 | } | |
114 | ||
224ce89b WB |
115 | cls::rbd::MirrorMode mirror_mode_internal; |
116 | r = librbd::cls_client::mirror_mode_get(&ioctx, &mirror_mode_internal); | |
c07f9fc5 FG |
117 | if (r == 0 && mirror_mode_internal == cls::rbd::MIRROR_MODE_DISABLED) { |
118 | dout(10) << "mirroring is disabled for pool " << pool_name << dendl; | |
119 | continue; | |
120 | } | |
121 | ||
122 | service_pool_ids.insert(pool_id); | |
123 | if (m_service_pools.find(pool_id) == m_service_pools.end()) { | |
124 | m_service_pools[pool_id] = {}; | |
125 | m_service_daemon->add_pool(pool_id, pool_name); | |
126 | } | |
127 | ||
224ce89b WB |
128 | if (r == -EPERM) { |
129 | dout(10) << "access denied querying pool " << pool_name << dendl; | |
c07f9fc5 FG |
130 | m_service_pools[pool_id] = m_service_daemon->add_or_update_callout( |
131 | pool_id, m_service_pools[pool_id], | |
132 | service_daemon::CALLOUT_LEVEL_WARNING, "access denied"); | |
224ce89b WB |
133 | continue; |
134 | } else if (r < 0) { | |
7c673cae FG |
135 | derr << "could not tell whether mirroring was enabled for " << pool_name |
136 | << " : " << cpp_strerror(r) << dendl; | |
c07f9fc5 FG |
137 | m_service_pools[pool_id] = m_service_daemon->add_or_update_callout( |
138 | pool_id, m_service_pools[pool_id], | |
139 | service_daemon::CALLOUT_LEVEL_WARNING, "mirroring mode query failed"); | |
7c673cae FG |
140 | continue; |
141 | } | |
142 | ||
9f95a23c TL |
143 | vector<librbd::mirror_peer_site_t> configs; |
144 | r = librbd::api::Mirror<>::peer_site_list(ioctx, &configs); | |
7c673cae FG |
145 | if (r < 0) { |
146 | derr << "error reading mirroring config for pool " << pool_name | |
147 | << cpp_strerror(r) << dendl; | |
c07f9fc5 FG |
148 | m_service_pools[pool_id] = m_service_daemon->add_or_update_callout( |
149 | pool_id, m_service_pools[pool_id], | |
150 | service_daemon::CALLOUT_LEVEL_ERROR, "mirroring peer list failed"); | |
7c673cae FG |
151 | continue; |
152 | } | |
153 | ||
9f95a23c TL |
154 | std::vector<PeerSpec> peers; |
155 | peers.reserve(configs.size()); | |
156 | for (auto& peer : configs) { | |
157 | if (peer.direction != RBD_MIRROR_PEER_DIRECTION_TX) { | |
158 | peers.push_back(peer); | |
159 | } | |
160 | } | |
161 | ||
11fdf7f2 | 162 | for (auto& peer : peers) { |
9f95a23c | 163 | r = resolve_peer_site_config_keys(pool_id, pool_name, &peer); |
11fdf7f2 TL |
164 | if (r < 0) { |
165 | break; | |
166 | } | |
167 | } | |
168 | ||
c07f9fc5 FG |
169 | if (m_service_pools[pool_id] != service_daemon::CALLOUT_ID_NONE) { |
170 | m_service_daemon->remove_callout(pool_id, m_service_pools[pool_id]); | |
171 | m_service_pools[pool_id] = service_daemon::CALLOUT_ID_NONE; | |
172 | } | |
173 | ||
11fdf7f2 | 174 | pool_peers->emplace(pool_id, Peers{peers.begin(), peers.end()}); |
7c673cae | 175 | } |
c07f9fc5 FG |
176 | |
177 | for (auto it = m_service_pools.begin(); it != m_service_pools.end(); ) { | |
178 | auto current_it(it++); | |
179 | if (service_pool_ids.find(current_it->first) == service_pool_ids.end()) { | |
180 | m_service_daemon->remove_pool(current_it->first); | |
181 | m_service_pools.erase(current_it->first); | |
182 | } | |
183 | } | |
7c673cae FG |
184 | } |
185 | ||
9f95a23c TL |
186 | int ClusterWatcher::read_site_name(std::string* site_name) { |
187 | dout(10) << dendl; | |
188 | ||
189 | librbd::RBD rbd; | |
190 | return rbd.mirror_site_name_get(*m_cluster, site_name); | |
191 | } | |
192 | ||
193 | int ClusterWatcher::resolve_peer_site_config_keys(int64_t pool_id, | |
194 | const std::string& pool_name, | |
195 | PeerSpec* peer) { | |
11fdf7f2 TL |
196 | dout(10) << "retrieving config-key: pool_id=" << pool_id << ", " |
197 | << "pool_name=" << pool_name << ", " | |
198 | << "peer_uuid=" << peer->uuid << dendl; | |
199 | ||
200 | std::string cmd = | |
201 | "{" | |
202 | "\"prefix\": \"config-key get\", " | |
203 | "\"key\": \"" RBD_MIRROR_PEER_CONFIG_KEY_PREFIX + stringify(pool_id) + | |
204 | "/" + peer->uuid + "\"" | |
205 | "}"; | |
206 | ||
207 | bufferlist in_bl; | |
208 | bufferlist out_bl; | |
209 | int r = m_cluster->mon_command(cmd, in_bl, &out_bl, nullptr); | |
210 | if (r == -ENOENT || out_bl.length() == 0) { | |
211 | return 0; | |
212 | } else if (r < 0) { | |
213 | derr << "error reading mirroring peer config for pool " << pool_name << ": " | |
214 | << cpp_strerror(r) << dendl; | |
215 | m_service_pools[pool_id] = m_service_daemon->add_or_update_callout( | |
216 | pool_id, m_service_pools[pool_id], | |
217 | service_daemon::CALLOUT_LEVEL_WARNING, | |
218 | "mirroring peer config-key query failed"); | |
219 | return r; | |
220 | } | |
221 | ||
222 | bool json_valid = false; | |
223 | json_spirit::mValue json_root; | |
224 | if(json_spirit::read(out_bl.to_str(), json_root)) { | |
225 | try { | |
226 | auto& json_obj = json_root.get_obj(); | |
227 | if (json_obj.count("mon_host")) { | |
228 | peer->mon_host = json_obj["mon_host"].get_str(); | |
229 | } | |
230 | if (json_obj.count("key")) { | |
231 | peer->key = json_obj["key"].get_str(); | |
232 | } | |
233 | json_valid = true; | |
234 | } catch (std::runtime_error&) { | |
235 | } | |
236 | } | |
237 | ||
238 | if (!json_valid) { | |
239 | derr << "error parsing mirroring peer config for pool " << pool_name << ", " | |
240 | << "peer " << peer->uuid << dendl; | |
241 | m_service_pools[pool_id] = m_service_daemon->add_or_update_callout( | |
242 | pool_id, m_service_pools[pool_id], | |
243 | service_daemon::CALLOUT_LEVEL_WARNING, | |
244 | "mirroring peer config-key decode failed"); | |
245 | } | |
246 | ||
247 | return 0; | |
248 | } | |
249 | ||
7c673cae FG |
250 | } // namespace mirror |
251 | } // namespace rbd |