]>
Commit | Line | Data |
---|---|---|
11fdf7f2 | 1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
9f95a23c | 2 | // vim: ts=8 sw=2 smarttab ft=cpp |
11fdf7f2 | 3 | |
31f18b77 FG |
4 | #include "rgw_sync_module_es.h" |
5 | #include "rgw_sync_module_es_rest.h" | |
6 | #include "rgw_es_query.h" | |
7 | #include "rgw_op.h" | |
8 | #include "rgw_rest.h" | |
9 | #include "rgw_rest_s3.h" | |
f67539c2 | 10 | #include "rgw_sal_rados.h" |
31f18b77 FG |
11 | |
12 | #define dout_context g_ceph_context | |
13 | #define dout_subsys ceph_subsys_rgw | |
14 | ||
20effc67 TL |
15 | using namespace std; |
16 | ||
31f18b77 FG |
17 | struct es_index_obj_response { |
18 | string bucket; | |
19 | rgw_obj_key key; | |
20 | uint64_t versioned_epoch{0}; | |
21 | ACLOwner owner; | |
22 | set<string> read_permissions; | |
23 | ||
24 | struct { | |
25 | uint64_t size{0}; | |
26 | ceph::real_time mtime; | |
27 | string etag; | |
28 | string content_type; | |
a8e16298 | 29 | string storage_class; |
31f18b77 FG |
30 | map<string, string> custom_str; |
31 | map<string, int64_t> custom_int; | |
32 | map<string, string> custom_date; | |
33 | ||
34 | template <class T> | |
35 | struct _custom_entry { | |
36 | string name; | |
37 | T value; | |
38 | void decode_json(JSONObj *obj) { | |
39 | JSONDecoder::decode_json("name", name, obj); | |
40 | JSONDecoder::decode_json("value", value, obj); | |
41 | } | |
42 | }; | |
43 | ||
44 | void decode_json(JSONObj *obj) { | |
45 | JSONDecoder::decode_json("size", size, obj); | |
46 | string mtime_str; | |
47 | JSONDecoder::decode_json("mtime", mtime_str, obj); | |
48 | parse_time(mtime_str.c_str(), &mtime); | |
49 | JSONDecoder::decode_json("etag", etag, obj); | |
50 | JSONDecoder::decode_json("content_type", content_type, obj); | |
a8e16298 | 51 | JSONDecoder::decode_json("storage_class", storage_class, obj); |
31f18b77 FG |
52 | list<_custom_entry<string> > str_entries; |
53 | JSONDecoder::decode_json("custom-string", str_entries, obj); | |
54 | for (auto& e : str_entries) { | |
55 | custom_str[e.name] = e.value; | |
56 | } | |
57 | list<_custom_entry<int64_t> > int_entries; | |
58 | JSONDecoder::decode_json("custom-int", int_entries, obj); | |
59 | for (auto& e : int_entries) { | |
60 | custom_int[e.name] = e.value; | |
61 | } | |
62 | list<_custom_entry<string> > date_entries; | |
63 | JSONDecoder::decode_json("custom-date", date_entries, obj); | |
64 | for (auto& e : date_entries) { | |
65 | custom_date[e.name] = e.value; | |
66 | } | |
67 | } | |
68 | } meta; | |
69 | ||
70 | void decode_json(JSONObj *obj) { | |
71 | JSONDecoder::decode_json("bucket", bucket, obj); | |
72 | JSONDecoder::decode_json("name", key.name, obj); | |
73 | JSONDecoder::decode_json("instance", key.instance, obj); | |
74 | JSONDecoder::decode_json("versioned_epoch", versioned_epoch, obj); | |
75 | JSONDecoder::decode_json("permissions", read_permissions, obj); | |
76 | JSONDecoder::decode_json("owner", owner, obj); | |
77 | JSONDecoder::decode_json("meta", meta, obj); | |
78 | } | |
79 | }; | |
80 | ||
81 | struct es_search_response { | |
82 | uint32_t took; | |
83 | bool timed_out; | |
84 | struct { | |
85 | uint32_t total; | |
86 | uint32_t successful; | |
87 | uint32_t failed; | |
88 | void decode_json(JSONObj *obj) { | |
89 | JSONDecoder::decode_json("total", total, obj); | |
90 | JSONDecoder::decode_json("successful", successful, obj); | |
91 | JSONDecoder::decode_json("failed", failed, obj); | |
92 | } | |
93 | } shards; | |
94 | struct obj_hit { | |
95 | string index; | |
96 | string type; | |
97 | string id; | |
98 | // double score | |
99 | es_index_obj_response source; | |
100 | void decode_json(JSONObj *obj) { | |
101 | JSONDecoder::decode_json("_index", index, obj); | |
102 | JSONDecoder::decode_json("_type", type, obj); | |
103 | JSONDecoder::decode_json("_id", id, obj); | |
104 | JSONDecoder::decode_json("_source", source, obj); | |
105 | } | |
106 | }; | |
107 | struct { | |
108 | uint32_t total; | |
109 | // double max_score; | |
110 | list<obj_hit> hits; | |
111 | void decode_json(JSONObj *obj) { | |
112 | JSONDecoder::decode_json("total", total, obj); | |
113 | // JSONDecoder::decode_json("max_score", max_score, obj); | |
114 | JSONDecoder::decode_json("hits", hits, obj); | |
115 | } | |
116 | } hits; | |
117 | void decode_json(JSONObj *obj) { | |
118 | JSONDecoder::decode_json("took", took, obj); | |
119 | JSONDecoder::decode_json("timed_out", timed_out, obj); | |
120 | JSONDecoder::decode_json("_shards", shards, obj); | |
121 | JSONDecoder::decode_json("hits", hits, obj); | |
122 | } | |
123 | }; | |
124 | ||
125 | class RGWMetadataSearchOp : public RGWOp { | |
126 | RGWSyncModuleInstanceRef sync_module_ref; | |
127 | RGWElasticSyncModuleInstance *es_module; | |
128 | protected: | |
129 | string expression; | |
130 | string custom_prefix; | |
131 | #define MAX_KEYS_DEFAULT 100 | |
132 | uint64_t max_keys{MAX_KEYS_DEFAULT}; | |
133 | string marker_str; | |
134 | uint64_t marker{0}; | |
135 | string next_marker; | |
136 | bool is_truncated{false}; | |
137 | string err; | |
138 | ||
139 | es_search_response response; | |
140 | ||
141 | public: | |
142 | RGWMetadataSearchOp(const RGWSyncModuleInstanceRef& sync_module) : sync_module_ref(sync_module) { | |
143 | es_module = static_cast<RGWElasticSyncModuleInstance *>(sync_module_ref.get()); | |
144 | } | |
145 | ||
f67539c2 | 146 | int verify_permission(optional_yield) override { |
31f18b77 FG |
147 | return 0; |
148 | } | |
149 | virtual int get_params() = 0; | |
11fdf7f2 | 150 | void pre_exec() override; |
f67539c2 | 151 | void execute(optional_yield y) override; |
31f18b77 | 152 | |
11fdf7f2 TL |
153 | const char* name() const override { return "metadata_search"; } |
154 | virtual RGWOpType get_type() override { return RGW_OP_METADATA_SEARCH; } | |
155 | virtual uint32_t op_mask() override { return RGW_OP_TYPE_READ; } | |
31f18b77 FG |
156 | }; |
157 | ||
158 | void RGWMetadataSearchOp::pre_exec() | |
159 | { | |
160 | rgw_bucket_object_pre_exec(s); | |
161 | } | |
162 | ||
f67539c2 | 163 | void RGWMetadataSearchOp::execute(optional_yield y) |
31f18b77 FG |
164 | { |
165 | op_ret = get_params(); | |
166 | if (op_ret < 0) | |
167 | return; | |
168 | ||
169 | list<pair<string, string> > conds; | |
170 | ||
9f95a23c TL |
171 | if (!s->user->get_info().system) { |
172 | conds.push_back(make_pair("permissions", s->user->get_id().to_str())); | |
31f18b77 FG |
173 | } |
174 | ||
175 | if (!s->bucket_name.empty()) { | |
176 | conds.push_back(make_pair("bucket", s->bucket_name)); | |
177 | } | |
178 | ||
179 | ESQueryCompiler es_query(expression, &conds, custom_prefix); | |
180 | ||
181 | static map<string, string, ltstr_nocase> aliases = { | |
182 | { "bucket", "bucket" }, /* forces lowercase */ | |
183 | { "name", "name" }, | |
184 | { "key", "name" }, | |
185 | { "instance", "instance" }, | |
186 | { "etag", "meta.etag" }, | |
187 | { "size", "meta.size" }, | |
188 | { "mtime", "meta.mtime" }, | |
189 | { "lastmodified", "meta.mtime" }, | |
a8e16298 TL |
190 | { "last_modified", "meta.mtime" }, |
191 | { "contenttype", "meta.content_type" }, | |
192 | { "content_type", "meta.content_type" }, | |
193 | { "storageclass", "meta.storage_class" }, | |
194 | { "storage_class", "meta.storage_class" }, | |
31f18b77 FG |
195 | }; |
196 | es_query.set_field_aliases(&aliases); | |
197 | ||
198 | static map<string, ESEntityTypeMap::EntityType> generic_map = { {"bucket", ESEntityTypeMap::ES_ENTITY_STR}, | |
199 | {"name", ESEntityTypeMap::ES_ENTITY_STR}, | |
200 | {"instance", ESEntityTypeMap::ES_ENTITY_STR}, | |
201 | {"permissions", ESEntityTypeMap::ES_ENTITY_STR}, | |
202 | {"meta.etag", ESEntityTypeMap::ES_ENTITY_STR}, | |
a8e16298 | 203 | {"meta.content_type", ESEntityTypeMap::ES_ENTITY_STR}, |
31f18b77 | 204 | {"meta.mtime", ESEntityTypeMap::ES_ENTITY_DATE}, |
a8e16298 TL |
205 | {"meta.size", ESEntityTypeMap::ES_ENTITY_INT}, |
206 | {"meta.storage_class", ESEntityTypeMap::ES_ENTITY_STR} }; | |
31f18b77 FG |
207 | ESEntityTypeMap gm(generic_map); |
208 | es_query.set_generic_type_map(&gm); | |
209 | ||
210 | static set<string> restricted_fields = { {"permissions"} }; | |
211 | es_query.set_restricted_fields(&restricted_fields); | |
212 | ||
213 | map<string, ESEntityTypeMap::EntityType> custom_map; | |
f67539c2 | 214 | for (auto& i : s->bucket->get_info().mdsearch_config) { |
31f18b77 FG |
215 | custom_map[i.first] = (ESEntityTypeMap::EntityType)i.second; |
216 | } | |
217 | ||
218 | ESEntityTypeMap em(custom_map); | |
219 | es_query.set_custom_type_map(&em); | |
220 | ||
221 | bool valid = es_query.compile(&err); | |
222 | if (!valid) { | |
b3b6e05e | 223 | ldpp_dout(this, 10) << "invalid query, failed generating request json" << dendl; |
31f18b77 FG |
224 | op_ret = -EINVAL; |
225 | return; | |
226 | } | |
227 | ||
228 | JSONFormatter f; | |
229 | encode_json("root", es_query, &f); | |
230 | ||
231 | RGWRESTConn *conn = es_module->get_rest_conn(); | |
232 | ||
233 | bufferlist in; | |
234 | bufferlist out; | |
235 | ||
236 | stringstream ss; | |
237 | ||
238 | f.flush(ss); | |
239 | in.append(ss.str()); | |
240 | ||
241 | string resource = es_module->get_index_path() + "/_search"; | |
242 | param_vec_t params; | |
243 | static constexpr int BUFSIZE = 32; | |
244 | char buf[BUFSIZE]; | |
245 | snprintf(buf, sizeof(buf), "%lld", (long long)max_keys); | |
246 | params.push_back(param_pair_t("size", buf)); | |
247 | if (marker > 0) { | |
248 | params.push_back(param_pair_t("from", marker_str.c_str())); | |
249 | } | |
b3b6e05e | 250 | ldpp_dout(this, 20) << "sending request to elasticsearch, payload=" << string(in.c_str(), in.length()) << dendl; |
a8e16298 | 251 | auto& extra_headers = es_module->get_request_headers(); |
b3b6e05e | 252 | op_ret = conn->get_resource(s, resource, ¶ms, &extra_headers, |
f67539c2 | 253 | out, &in, nullptr, y); |
31f18b77 | 254 | if (op_ret < 0) { |
b3b6e05e | 255 | ldpp_dout(this, 0) << "ERROR: failed to fetch resource (r=" << resource << ", ret=" << op_ret << ")" << dendl; |
31f18b77 FG |
256 | return; |
257 | } | |
258 | ||
b3b6e05e | 259 | ldpp_dout(this, 20) << "response: " << string(out.c_str(), out.length()) << dendl; |
31f18b77 FG |
260 | |
261 | JSONParser jparser; | |
262 | if (!jparser.parse(out.c_str(), out.length())) { | |
b3b6e05e | 263 | ldpp_dout(this, 0) << "ERROR: failed to parse elasticsearch response" << dendl; |
31f18b77 FG |
264 | op_ret = -EINVAL; |
265 | return; | |
266 | } | |
267 | ||
268 | try { | |
269 | decode_json_obj(response, &jparser); | |
9f95a23c | 270 | } catch (const JSONDecoder::err& e) { |
b3b6e05e | 271 | ldpp_dout(this, 0) << "ERROR: failed to decode JSON input: " << e.what() << dendl; |
31f18b77 FG |
272 | op_ret = -EINVAL; |
273 | return; | |
274 | } | |
275 | ||
276 | } | |
277 | ||
278 | class RGWMetadataSearch_ObjStore_S3 : public RGWMetadataSearchOp { | |
279 | public: | |
11fdf7f2 | 280 | explicit RGWMetadataSearch_ObjStore_S3(const RGWSyncModuleInstanceRef& _sync_module) : RGWMetadataSearchOp(_sync_module) { |
31f18b77 FG |
281 | custom_prefix = "x-amz-meta-"; |
282 | } | |
283 | ||
284 | int get_params() override { | |
285 | expression = s->info.args.get("query"); | |
286 | bool exists; | |
287 | string max_keys_str = s->info.args.get("max-keys", &exists); | |
288 | #define MAX_KEYS_MAX 10000 | |
289 | if (exists) { | |
290 | string err; | |
291 | max_keys = strict_strtoll(max_keys_str.c_str(), 10, &err); | |
292 | if (!err.empty()) { | |
293 | return -EINVAL; | |
294 | } | |
295 | if (max_keys > MAX_KEYS_MAX) { | |
296 | max_keys = MAX_KEYS_MAX; | |
297 | } | |
298 | } | |
299 | marker_str = s->info.args.get("marker", &exists); | |
300 | if (exists) { | |
301 | string err; | |
302 | marker = strict_strtoll(marker_str.c_str(), 10, &err); | |
303 | if (!err.empty()) { | |
304 | return -EINVAL; | |
305 | } | |
306 | } | |
307 | uint64_t nm = marker + max_keys; | |
308 | static constexpr int BUFSIZE = 32; | |
309 | char buf[BUFSIZE]; | |
310 | snprintf(buf, sizeof(buf), "%lld", (long long)nm); | |
311 | next_marker = buf; | |
312 | return 0; | |
313 | } | |
314 | void send_response() override { | |
315 | if (op_ret) { | |
316 | s->err.message = err; | |
317 | set_req_state_err(s, op_ret); | |
318 | } | |
319 | dump_errno(s); | |
320 | end_header(s, this, "application/xml"); | |
321 | ||
322 | if (op_ret < 0) { | |
323 | return; | |
324 | } | |
325 | ||
326 | is_truncated = (response.hits.hits.size() >= max_keys); | |
327 | ||
328 | s->formatter->open_object_section("SearchMetadataResponse"); | |
329 | s->formatter->dump_string("Marker", marker_str); | |
330 | s->formatter->dump_string("IsTruncated", (is_truncated ? "true" : "false")); | |
331 | if (is_truncated) { | |
332 | s->formatter->dump_string("NextMarker", next_marker); | |
333 | } | |
1e59de90 | 334 | if (s->format == RGWFormat::JSON) { |
31f18b77 FG |
335 | s->formatter->open_array_section("Objects"); |
336 | } | |
337 | for (auto& i : response.hits.hits) { | |
338 | s->formatter->open_object_section("Contents"); | |
339 | es_index_obj_response& e = i.source; | |
340 | s->formatter->dump_string("Bucket", e.bucket); | |
341 | s->formatter->dump_string("Key", e.key.name); | |
342 | string instance = (!e.key.instance.empty() ? e.key.instance : "null"); | |
343 | s->formatter->dump_string("Instance", instance.c_str()); | |
344 | s->formatter->dump_int("VersionedEpoch", e.versioned_epoch); | |
20effc67 | 345 | dump_time(s, "LastModified", e.meta.mtime); |
31f18b77 FG |
346 | s->formatter->dump_int("Size", e.meta.size); |
347 | s->formatter->dump_format("ETag", "\"%s\"", e.meta.etag.c_str()); | |
348 | s->formatter->dump_string("ContentType", e.meta.content_type.c_str()); | |
a8e16298 | 349 | s->formatter->dump_string("StorageClass", e.meta.storage_class.c_str()); |
31f18b77 FG |
350 | dump_owner(s, e.owner.get_id(), e.owner.get_display_name()); |
351 | s->formatter->open_array_section("CustomMetadata"); | |
352 | for (auto& m : e.meta.custom_str) { | |
353 | s->formatter->open_object_section("Entry"); | |
354 | s->formatter->dump_string("Name", m.first.c_str()); | |
355 | s->formatter->dump_string("Value", m.second); | |
356 | s->formatter->close_section(); | |
357 | } | |
358 | for (auto& m : e.meta.custom_int) { | |
359 | s->formatter->open_object_section("Entry"); | |
360 | s->formatter->dump_string("Name", m.first.c_str()); | |
361 | s->formatter->dump_int("Value", m.second); | |
362 | s->formatter->close_section(); | |
363 | } | |
364 | for (auto& m : e.meta.custom_date) { | |
365 | s->formatter->open_object_section("Entry"); | |
366 | s->formatter->dump_string("Name", m.first.c_str()); | |
367 | s->formatter->dump_string("Value", m.second); | |
368 | s->formatter->close_section(); | |
369 | } | |
370 | s->formatter->close_section(); | |
371 | rgw_flush_formatter(s, s->formatter); | |
372 | s->formatter->close_section(); | |
373 | }; | |
1e59de90 | 374 | if (s->format == RGWFormat::JSON) { |
31f18b77 FG |
375 | s->formatter->close_section(); |
376 | } | |
377 | s->formatter->close_section(); | |
378 | rgw_flush_formatter_and_reset(s, s->formatter); | |
379 | } | |
380 | }; | |
381 | ||
382 | class RGWHandler_REST_MDSearch_S3 : public RGWHandler_REST_S3 { | |
383 | protected: | |
11fdf7f2 | 384 | RGWOp *op_get() override { |
31f18b77 | 385 | if (s->info.args.exists("query")) { |
1e59de90 | 386 | return new RGWMetadataSearch_ObjStore_S3(driver->get_sync_module()); |
31f18b77 FG |
387 | } |
388 | if (!s->init_state.url_bucket.empty() && | |
389 | s->info.args.exists("mdsearch")) { | |
390 | return new RGWGetBucketMetaSearch_ObjStore_S3; | |
391 | } | |
392 | return nullptr; | |
393 | } | |
11fdf7f2 | 394 | RGWOp *op_head() override { |
31f18b77 FG |
395 | return nullptr; |
396 | } | |
11fdf7f2 | 397 | RGWOp *op_post() override { |
31f18b77 FG |
398 | return nullptr; |
399 | } | |
400 | public: | |
11fdf7f2 | 401 | explicit RGWHandler_REST_MDSearch_S3(const rgw::auth::StrategyRegistry& auth_registry) : RGWHandler_REST_S3(auth_registry) {} |
31f18b77 FG |
402 | virtual ~RGWHandler_REST_MDSearch_S3() {} |
403 | }; | |
404 | ||
405 | ||
1e59de90 TL |
406 | RGWHandler_REST* RGWRESTMgr_MDSearch_S3::get_handler(rgw::sal::Driver* driver, |
407 | req_state* const s, | |
31f18b77 FG |
408 | const rgw::auth::StrategyRegistry& auth_registry, |
409 | const std::string& frontend_prefix) | |
410 | { | |
411 | int ret = | |
1e59de90 TL |
412 | RGWHandler_REST_S3::init_from_header(driver, s, |
413 | RGWFormat::XML, true); | |
31f18b77 FG |
414 | if (ret < 0) { |
415 | return nullptr; | |
416 | } | |
417 | ||
f67539c2 | 418 | if (!s->object->empty()) { |
31f18b77 FG |
419 | return nullptr; |
420 | } | |
421 | ||
422 | RGWHandler_REST *handler = new RGWHandler_REST_MDSearch_S3(auth_registry); | |
423 | ||
b3b6e05e | 424 | ldpp_dout(s, 20) << __func__ << " handler=" << typeid(*handler).name() |
31f18b77 FG |
425 | << dendl; |
426 | return handler; | |
427 | } | |
428 |