]> git.proxmox.com Git - ceph.git/blame - ceph/src/rgw/rgw_multi.cc
update sources to ceph Nautilus 14.2.1
[ceph.git] / ceph / src / rgw / rgw_multi.cc
CommitLineData
7c673cae
FG
1// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2// vim: ts=8 sw=2 smarttab
3
4#include <string.h>
5
6#include <iostream>
7#include <map>
8
9#include "include/types.h"
10
11#include "rgw_xml.h"
12#include "rgw_multi.h"
224ce89b 13#include "rgw_op.h"
7c673cae 14
11fdf7f2
TL
15#include "services/svc_sys_obj.h"
16
7c673cae
FG
17#define dout_subsys ceph_subsys_rgw
18
11fdf7f2
TL
19
20
21bool MultipartMetaFilter::filter(const string& name, string& key) {
22 // the length of the suffix so we can skip past it
23 static const size_t MP_META_SUFFIX_LEN = MP_META_SUFFIX.length();
24
25 size_t len = name.size();
26
27 // make sure there's room for suffix plus at least one more
28 // character
29 if (len <= MP_META_SUFFIX_LEN)
30 return false;
31
32 size_t pos = name.find(MP_META_SUFFIX, len - MP_META_SUFFIX_LEN);
33 if (pos == string::npos)
34 return false;
35
36 pos = name.rfind('.', pos - 1);
37 if (pos == string::npos)
38 return false;
39
40 key = name.substr(0, pos);
41
42 return true;
43}
7c673cae
FG
44
45
46bool RGWMultiPart::xml_end(const char *el)
47{
48 RGWMultiPartNumber *num_obj = static_cast<RGWMultiPartNumber *>(find_first("PartNumber"));
49 RGWMultiETag *etag_obj = static_cast<RGWMultiETag *>(find_first("ETag"));
50
51 if (!num_obj || !etag_obj)
52 return false;
53
54 string s = num_obj->get_data();
55 if (s.empty())
56 return false;
57
58 num = atoi(s.c_str());
59
60 s = etag_obj->get_data();
61 etag = s;
62
63 return true;
64}
65
66bool RGWMultiCompleteUpload::xml_end(const char *el) {
67 XMLObjIter iter = find("Part");
68 RGWMultiPart *part = static_cast<RGWMultiPart *>(iter.get_next());
69 while (part) {
70 int num = part->get_num();
71 string etag = part->get_etag();
72 parts[num] = etag;
73 part = static_cast<RGWMultiPart *>(iter.get_next());
74 }
75 return true;
76}
77
78
79XMLObj *RGWMultiXMLParser::alloc_obj(const char *el) {
80 XMLObj *obj = NULL;
81 if (strcmp(el, "CompleteMultipartUpload") == 0 ||
82 strcmp(el, "MultipartUpload") == 0) {
83 obj = new RGWMultiCompleteUpload();
84 } else if (strcmp(el, "Part") == 0) {
85 obj = new RGWMultiPart();
86 } else if (strcmp(el, "PartNumber") == 0) {
87 obj = new RGWMultiPartNumber();
88 } else if (strcmp(el, "ETag") == 0) {
89 obj = new RGWMultiETag();
90 }
91
92 return obj;
93}
94
95bool is_v2_upload_id(const string& upload_id)
96{
97 const char *uid = upload_id.c_str();
98
99 return (strncmp(uid, MULTIPART_UPLOAD_ID_PREFIX, sizeof(MULTIPART_UPLOAD_ID_PREFIX) - 1) == 0) ||
100 (strncmp(uid, MULTIPART_UPLOAD_ID_PREFIX_LEGACY, sizeof(MULTIPART_UPLOAD_ID_PREFIX_LEGACY) - 1) == 0);
101}
102
11fdf7f2
TL
103int list_multipart_parts(RGWRados *store, RGWBucketInfo& bucket_info,
104 CephContext *cct,
105 const string& upload_id,
106 const string& meta_oid, int num_parts,
107 int marker, map<uint32_t, RGWUploadPartInfo>& parts,
108 int *next_marker, bool *truncated,
109 bool assume_unsorted)
7c673cae
FG
110{
111 map<string, bufferlist> parts_map;
112 map<string, bufferlist>::iterator iter;
7c673cae
FG
113
114 rgw_obj obj;
115 obj.init_ns(bucket_info.bucket, meta_oid, RGW_OBJ_NS_MULTIPART);
116 obj.set_in_extra_data(true);
117
118 rgw_raw_obj raw_obj;
119 store->obj_to_raw(bucket_info.placement_rule, obj, &raw_obj);
120
121 bool sorted_omap = is_v2_upload_id(upload_id) && !assume_unsorted;
122
123 int ret;
124
125 parts.clear();
126
11fdf7f2
TL
127
128 auto obj_ctx = store->svc.sysobj->init_obj_ctx();
129 auto sysobj = obj_ctx.get_obj(raw_obj);
130
7c673cae
FG
131 if (sorted_omap) {
132 string p;
133 p = "part.";
134 char buf[32];
135
136 snprintf(buf, sizeof(buf), "%08d", marker);
137 p.append(buf);
138
11fdf7f2 139 ret = sysobj.omap().get_vals(p, num_parts + 1, &parts_map, nullptr);
7c673cae 140 } else {
11fdf7f2 141 ret = sysobj.omap().get_all(&parts_map);
7c673cae
FG
142 }
143 if (ret < 0)
144 return ret;
145
146 int i;
147 int last_num = 0;
148
149 uint32_t expected_next = marker + 1;
150
151 for (i = 0, iter = parts_map.begin(); (i < num_parts || !sorted_omap) && iter != parts_map.end(); ++iter, ++i) {
152 bufferlist& bl = iter->second;
11fdf7f2 153 auto bli = bl.cbegin();
7c673cae
FG
154 RGWUploadPartInfo info;
155 try {
11fdf7f2 156 decode(info, bli);
7c673cae
FG
157 } catch (buffer::error& err) {
158 ldout(cct, 0) << "ERROR: could not part info, caught buffer::error" << dendl;
159 return -EIO;
160 }
161 if (sorted_omap) {
162 if (info.num != expected_next) {
163 /* ouch, we expected a specific part num here, but we got a different one. Either
164 * a part is missing, or it could be a case of mixed rgw versions working on the same
165 * upload, where one gateway doesn't support correctly sorted omap keys for multipart
166 * upload just assume data is unsorted.
167 */
168 return list_multipart_parts(store, bucket_info, cct, upload_id, meta_oid, num_parts, marker, parts, next_marker, truncated, true);
169 }
170 expected_next++;
171 }
172 if (sorted_omap ||
173 (int)info.num > marker) {
174 parts[info.num] = info;
175 last_num = info.num;
176 }
177 }
178
179 if (sorted_omap) {
180 if (truncated)
181 *truncated = (iter != parts_map.end());
182 } else {
183 /* rebuild a map with only num_parts entries */
184
185 map<uint32_t, RGWUploadPartInfo> new_parts;
186 map<uint32_t, RGWUploadPartInfo>::iterator piter;
187
188 for (i = 0, piter = parts.begin(); i < num_parts && piter != parts.end(); ++i, ++piter) {
189 new_parts[piter->first] = piter->second;
190 last_num = piter->first;
191 }
192
193 if (truncated)
194 *truncated = (piter != parts.end());
195
196 parts.swap(new_parts);
197 }
198
199 if (next_marker) {
200 *next_marker = last_num;
201 }
202
203 return 0;
204}
205
206int list_multipart_parts(RGWRados *store, struct req_state *s,
11fdf7f2
TL
207 const string& upload_id,
208 const string& meta_oid, int num_parts,
209 int marker, map<uint32_t, RGWUploadPartInfo>& parts,
210 int *next_marker, bool *truncated,
211 bool assume_unsorted)
7c673cae
FG
212{
213 return list_multipart_parts(store, s->bucket_info, s->cct, upload_id, meta_oid, num_parts, marker, parts, next_marker, truncated, assume_unsorted);
214}
215
216int abort_multipart_upload(RGWRados *store, CephContext *cct, RGWObjectCtx *obj_ctx, RGWBucketInfo& bucket_info, RGWMPObj& mp_obj)
217{
218 rgw_obj meta_obj;
219 meta_obj.init_ns(bucket_info.bucket, mp_obj.get_meta(), RGW_OBJ_NS_MULTIPART);
220 meta_obj.set_in_extra_data(true);
221 meta_obj.index_hash_source = mp_obj.get_key();
222 cls_rgw_obj_chain chain;
223 list<rgw_obj_index_key> remove_objs;
224 map<uint32_t, RGWUploadPartInfo> obj_parts;
225 bool truncated;
226 int marker = 0;
227 int ret;
228
229 do {
230 ret = list_multipart_parts(store, bucket_info, cct, mp_obj.get_upload_id(), mp_obj.get_meta(), 1000,
231 marker, obj_parts, &marker, &truncated);
232 if (ret < 0)
11fdf7f2 233 return (ret == -ENOENT) ? -ERR_NO_SUCH_UPLOAD : ret;
7c673cae
FG
234 for (auto obj_iter = obj_parts.begin(); obj_iter != obj_parts.end(); ++obj_iter) {
235 RGWUploadPartInfo& obj_part = obj_iter->second;
236 rgw_obj obj;
237 if (obj_part.manifest.empty()) {
238 string oid = mp_obj.get_part(obj_iter->second.num);
239 obj.init_ns(bucket_info.bucket, oid, RGW_OBJ_NS_MULTIPART);
240 obj.index_hash_source = mp_obj.get_key();
241 ret = store->delete_obj(*obj_ctx, bucket_info, obj, 0);
242 if (ret < 0 && ret != -ENOENT)
243 return ret;
244 } else {
245 store->update_gc_chain(meta_obj, obj_part.manifest, &chain);
246 RGWObjManifest::obj_iterator oiter = obj_part.manifest.obj_begin();
247 if (oiter != obj_part.manifest.obj_end()) {
248 rgw_obj head;
249 rgw_raw_obj raw_head = oiter.get_location().get_raw_obj(store);
250 rgw_raw_obj_to_obj(bucket_info.bucket, raw_head, &head);
251
252 rgw_obj_index_key key;
253 head.key.get_index_key(&key);
254 remove_objs.push_back(key);
255 }
256 }
257 }
258 } while (truncated);
259 /* use upload id as tag */
260 ret = store->send_chain_to_gc(chain, mp_obj.get_upload_id() , false); // do it async
261 if (ret < 0) {
262 ldout(cct, 5) << "gc->send_chain() returned " << ret << dendl;
11fdf7f2 263 return (ret == -ENOENT) ? -ERR_NO_SUCH_UPLOAD : ret;
7c673cae
FG
264 }
265 RGWRados::Object del_target(store, bucket_info, *obj_ctx, meta_obj);
266 RGWRados::Object::Delete del_op(&del_target);
267
268 del_op.params.bucket_owner = bucket_info.owner;
269 del_op.params.versioning_status = 0;
270 if (!remove_objs.empty()) {
271 del_op.params.remove_objs = &remove_objs;
272 }
273
274 // and also remove the metadata obj
275 ret = del_op.delete_obj();
11fdf7f2 276 return (ret == -ENOENT) ? -ERR_NO_SUCH_UPLOAD : ret;
7c673cae
FG
277}
278
224ce89b
WB
279int list_bucket_multiparts(RGWRados *store, RGWBucketInfo& bucket_info,
280 string& prefix, string& marker, string& delim,
281 int& max_uploads, vector<rgw_bucket_dir_entry> *objs,
282 map<string, bool> *common_prefixes, bool *is_truncated)
283{
284 RGWRados::Bucket target(store, bucket_info);
285 RGWRados::Bucket::List list_op(&target);
286 MultipartMetaFilter mp_filter;
287
288 list_op.params.prefix = prefix;
289 list_op.params.delim = delim;
290 list_op.params.marker = marker;
291 list_op.params.ns = RGW_OBJ_NS_MULTIPART;
292 list_op.params.filter = &mp_filter;
293
294 return(list_op.list_objects(max_uploads, objs, common_prefixes,is_truncated));
295}
296
297int abort_bucket_multiparts(RGWRados *store, CephContext *cct, RGWBucketInfo& bucket_info,
298 string& prefix, string& delim)
299{
300 int ret, max = 1000, num_deleted = 0;
301 vector<rgw_bucket_dir_entry> objs;
302 RGWObjectCtx obj_ctx(store);
303 string marker;
224ce89b
WB
304 bool is_truncated;
305
306 do {
307 ret = list_bucket_multiparts(store, bucket_info, prefix, marker, delim,
11fdf7f2 308 max, &objs, nullptr, &is_truncated);
224ce89b
WB
309 if (ret < 0) {
310 return ret;
311 }
312 if (!objs.empty()) {
11fdf7f2 313 RGWMPObj mp;
224ce89b
WB
314 for (const auto& obj : objs) {
315 rgw_obj_key key(obj.key);
11fdf7f2 316 if (!mp.from_meta(key.name))
224ce89b 317 continue;
11fdf7f2 318 ret = abort_multipart_upload(store, cct, &obj_ctx, bucket_info, mp);
91327a77 319 if (ret < 0 && ret != -ENOENT && ret != -ERR_NO_SUCH_UPLOAD) {
224ce89b
WB
320 return ret;
321 }
322 num_deleted++;
323 }
324 if (num_deleted) {
325 ldout(store->ctx(),0) << "WARNING : aborted " << num_deleted << " incomplete multipart uploads" << dendl;
326 }
327 }
328 } while (is_truncated);
329
330 return ret;
331}