]> git.proxmox.com Git - ceph.git/blame - ceph/src/tools/rados/RadosImport.cc
import 15.2.5
[ceph.git] / ceph / src / tools / rados / RadosImport.cc
CommitLineData
7c673cae
FG
1// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2// vim: ts=8 sw=2 smarttab
3/*
4 * Ceph - scalable distributed file system
5 *
6 * Copyright (C) 2015 Red Hat
7 *
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
12 *
13 */
14
15
16#include "common/errno.h"
17
18#include "osd/PGLog.h"
19#include "RadosImport.h"
20
21#define dout_context g_ceph_context
22#define dout_subsys ceph_subsys_rados
23
24int RadosImport::import(std::string pool, bool no_overwrite)
25{
26 librados::IoCtx ioctx;
27 librados::Rados cluster;
28
29 char *id = getenv("CEPH_CLIENT_ID");
30 if (id) cerr << "Client id is: " << id << std::endl;
31 int ret = cluster.init(id);
32 if (ret) {
33 cerr << "Error " << ret << " in cluster.init" << std::endl;
34 return ret;
35 }
36 ret = cluster.conf_read_file(NULL);
37 if (ret) {
38 cerr << "Error " << ret << " in cluster.conf_read_file" << std::endl;
39 return ret;
40 }
41 ret = cluster.conf_parse_env(NULL);
42 if (ret) {
43 cerr << "Error " << ret << " in cluster.conf_read_env" << std::endl;
44 return ret;
45 }
46 ret = cluster.connect();
47 if (ret) {
48 cerr << "Error " << ret << " in cluster.connect" << std::endl;
49 return ret;
50 }
51
52 ret = cluster.ioctx_create(pool.c_str(), ioctx);
53 if (ret < 0) {
54 cerr << "ioctx_create " << pool << " failed with " << ret << std::endl;
55 return ret;
56 }
57
58 return import(ioctx, no_overwrite);
59}
60
61int RadosImport::import(librados::IoCtx &io_ctx, bool no_overwrite)
62{
63 bufferlist ebl;
64 pg_info_t info;
65 PGLog::IndexedLog log;
66
67 int ret = read_super();
68 if (ret)
69 return ret;
70
71 if (sh.magic != super_header::super_magic) {
72 cerr << "Invalid magic number: 0x"
73 << std::hex << sh.magic << " vs. 0x" << super_header::super_magic
74 << std::dec << std::endl;
75 return -EFAULT;
76 }
77
78 if (sh.version > super_header::super_ver) {
79 cerr << "Can't handle export format version=" << sh.version << std::endl;
80 return -EINVAL;
81 }
82
83 //First section must be TYPE_PG_BEGIN
84 sectiontype_t type;
85 ret = read_section(&type, &ebl);
86 if (ret)
87 return ret;
88
89 bool pool_mode = false;
90 if (type == TYPE_POOL_BEGIN) {
91 pool_mode = true;
92 cout << "Importing pool" << std::endl;
93 } else if (type == TYPE_PG_BEGIN) {
11fdf7f2 94 auto ebliter = ebl.cbegin();
7c673cae
FG
95 pg_begin pgb;
96 pgb.decode(ebliter);
97 spg_t pgid = pgb.pgid;;
98 if (!pgid.is_no_shard()) {
99 cerr << "Importing Erasure Coded shard is not supported" << std::endl;
100 return -EOPNOTSUPP;
101 }
102 dout(10) << "Exported features: " << pgb.superblock.compat_features << dendl;
103 cout << "Importing from pgid " << pgid << std::endl;
104 } else {
105 cerr << "Invalid initial section code " << type << std::endl;
106 return -EFAULT;
107 }
108
109 // XXX: How to check export features?
110#if 0
111 if (sb.compat_features.compare(pgb.superblock.compat_features) == -1) {
112 cerr << "Export has incompatible features set "
113 << pgb.superblock.compat_features << std::endl;
114 return -EINVAL;
115 }
116#endif
117
118#if defined(__linux__)
119 if (file_fd != STDIN_FILENO)
120 posix_fadvise(file_fd, 0, 0, POSIX_FADV_SEQUENTIAL);
121#endif
122
123 bool done = false;
124 bool found_metadata = false;
125 while(!done) {
126 ret = read_section(&type, &ebl);
127 if (ret)
128 return ret;
129
130 //cout << "do_import: Section type " << hex << type << dec << std::endl;
131 if (type >= END_OF_TYPES) {
132 cout << "Skipping unknown section type" << std::endl;
133 continue;
134 }
135 switch(type) {
136 case TYPE_OBJECT_BEGIN:
137 ret = get_object_rados(io_ctx, ebl, no_overwrite);
138 if (ret) {
139 cerr << "Error inserting object: " << ret << std::endl;
140 return ret;
141 }
142 break;
143 case TYPE_PG_METADATA:
144 dout(10) << "Don't care about the old metadata" << dendl;
145 found_metadata = true;
146 break;
147 case TYPE_PG_END:
148 done = true;
149 break;
150 case TYPE_POOL_END:
151 done = true;
152 break;
153 default:
154 return -EFAULT;
155 }
156 }
157
158 if (!(pool_mode || found_metadata)) {
159 cerr << "Missing metadata section!" << std::endl;
160 }
161
162#if defined(__linux__)
163 if (file_fd != STDIN_FILENO)
164 posix_fadvise(file_fd, 0, 0, POSIX_FADV_DONTNEED);
165#endif
166 return 0;
167}
168
169int RadosImport::get_object_rados(librados::IoCtx &ioctx, bufferlist &bl, bool no_overwrite)
170{
11fdf7f2 171 auto ebliter = bl.cbegin();
7c673cae
FG
172 object_begin ob;
173 ob.decode(ebliter);
174 map<string,bufferlist>::iterator i;
175 bufferlist abl;
176 bool skipping;
177
178 data_section ds;
179 attr_section as;
180 omap_hdr_section oh;
181 omap_section os;
182
11fdf7f2 183 ceph_assert(g_ceph_context);
7c673cae
FG
184 if (ob.hoid.hobj.nspace == g_ceph_context->_conf->osd_hit_set_namespace) {
185 cout << "Skipping internal object " << ob.hoid << std::endl;
186 skip_object(bl);
187 return 0;
188 }
189
190 if (!ob.hoid.hobj.is_head()) {
191 cout << "Skipping non-head for " << ob.hoid << std::endl;
192 skip_object(bl);
193 return 0;
194 }
195
196 ioctx.set_namespace(ob.hoid.hobj.get_namespace());
f6b5b4d7 197 ioctx.locator_set_key(ob.hoid.hobj.get_key());
7c673cae
FG
198
199 string msg("Write");
200 skipping = false;
201 if (dry_run) {
202 uint64_t psize;
203 time_t pmtime;
204 int ret = ioctx.stat(ob.hoid.hobj.oid.name, &psize, &pmtime);
205 if (ret == 0) {
206 if (no_overwrite)
207 // Could set skipping, but dry-run doesn't change anything either
208 msg = "Skipping existing";
209 else
210 msg = "***Overwrite***";
211 }
212 } else {
213 int ret = ioctx.create(ob.hoid.hobj.oid.name, true);
214 if (ret && ret != -EEXIST) {
215 cerr << "create failed: " << cpp_strerror(ret) << std::endl;
216 return ret;
217 }
218 if (ret == -EEXIST) {
219 if (no_overwrite) {
220 msg = "Skipping existing";
221 skipping = true;
222 } else {
223 msg = "***Overwrite***";
224 ret = ioctx.remove(ob.hoid.hobj.oid.name);
225 if (ret < 0) {
226 cerr << "remove failed: " << cpp_strerror(ret) << std::endl;
227 return ret;
228 }
229 ret = ioctx.create(ob.hoid.hobj.oid.name, true);
230 // If object re-appeared after removal, let's just skip it
231 if (ret == -EEXIST) {
232 skipping = true;
233 msg = "Skipping in-use object";
234 ret = 0;
235 }
236 if (ret < 0) {
237 cerr << "create failed: " << cpp_strerror(ret) << std::endl;
238 return ret;
239 }
240 }
241 }
242 }
243
244 cout << msg << " " << ob.hoid << std::endl;
245
246 bool need_align = false;
247 uint64_t alignment = 0;
248 if (align) {
249 need_align = true;
250 alignment = align;
251 } else {
252 int ret = ioctx.pool_requires_alignment2(&need_align);
253 if (ret < 0) {
254 cerr << "pool_requires_alignment2 failed: " << cpp_strerror(ret)
255 << std::endl;
256 return ret;
257 }
258
259 if (need_align) {
260 ret = ioctx.pool_required_alignment2(&alignment);
261 if (ret < 0) {
262 cerr << "pool_required_alignment2 failed: " << cpp_strerror(ret)
263 << std::endl;
264 return ret;
265 }
11fdf7f2 266 ceph_assert(alignment != 0);
7c673cae
FG
267 }
268 }
269
270 if (need_align) {
271 dout(10) << "alignment = " << alignment << dendl;
272 }
273
274 bufferlist ebl, databl;
275 uint64_t in_offset = 0, out_offset = 0;
276 bool done = false;
277 while(!done) {
278 sectiontype_t type;
279 int ret = read_section(&type, &ebl);
280 if (ret) {
281 cerr << "Error reading section: " << ret << std::endl;
282 return ret;
283 }
284
11fdf7f2 285 ebliter = ebl.cbegin();
7c673cae
FG
286 //cout << "\tdo_object: Section type " << hex << type << dec << std::endl;
287 //cout << "\t\tsection size " << ebl.length() << std::endl;
288 if (type >= END_OF_TYPES) {
289 cout << "Skipping unknown object section type" << std::endl;
290 continue;
291 }
292 switch(type) {
293 case TYPE_DATA:
294 ds.decode(ebliter);
295 dout(10) << "\tdata: offset " << ds.offset << " len " << ds.len << dendl;
296 if (need_align) {
297 if (ds.offset != in_offset) {
298 cerr << "Discontiguous object data in export" << std::endl;
299 return -EFAULT;
300 }
11fdf7f2 301 ceph_assert(ds.databl.length() == ds.len);
7c673cae
FG
302 databl.claim_append(ds.databl);
303 in_offset += ds.len;
304 if (databl.length() >= alignment) {
305 uint64_t rndlen = uint64_t(databl.length() / alignment) * alignment;
306 dout(10) << "write offset=" << out_offset << " len=" << rndlen << dendl;
307 if (!dry_run && !skipping) {
308 ret = ioctx.write(ob.hoid.hobj.oid.name, databl, rndlen, out_offset);
309 if (ret) {
310 cerr << "write failed: " << cpp_strerror(ret) << std::endl;
311 return ret;
312 }
313 }
314 out_offset += rndlen;
315 bufferlist n;
316 if (databl.length() > rndlen) {
11fdf7f2 317 ceph_assert(databl.length() - rndlen < alignment);
7c673cae
FG
318 n.substr_of(databl, rndlen, databl.length() - rndlen);
319 }
320 databl = n;
321 }
322 break;
323 }
324 if (!dry_run && !skipping) {
325 ret = ioctx.write(ob.hoid.hobj.oid.name, ds.databl, ds.len, ds.offset);
326 if (ret) {
327 cerr << "write failed: " << cpp_strerror(ret) << std::endl;
328 return ret;
329 }
330 }
331 break;
332 case TYPE_ATTRS:
333 as.decode(ebliter);
334
335 dout(10) << "\tattrs: len " << as.data.size() << dendl;
336 if (dry_run || skipping)
337 break;
338 for (std::map<string,bufferlist>::iterator i = as.data.begin();
339 i != as.data.end(); ++i) {
340 // The user xattrs that we want all begin with "_" with length > 1.
341 // Drop key "_" and all attributes that do not start with '_'
342 if (i->first == "_" || i->first[0] != '_')
343 continue;
344 ret = ioctx.setxattr(ob.hoid.hobj.oid.name, i->first.substr(1).c_str(), i->second);
345 if (ret) {
346 cerr << "setxattr failed: " << cpp_strerror(ret) << std::endl;
347 if (ret != -EOPNOTSUPP)
348 return ret;
349 }
350 }
351 break;
352 case TYPE_OMAP_HDR:
353 oh.decode(ebliter);
354
355 dout(10) << "\tomap header: " << string(oh.hdr.c_str(), oh.hdr.length())
356 << dendl;
357 if (dry_run || skipping)
358 break;
359 ret = ioctx.omap_set_header(ob.hoid.hobj.oid.name, oh.hdr);
360 if (ret) {
361 cerr << "omap_set_header failed: " << cpp_strerror(ret) << std::endl;
362 if (ret != -EOPNOTSUPP)
363 return ret;
364 }
365 break;
366 case TYPE_OMAP:
367 os.decode(ebliter);
368
369 dout(10) << "\tomap: size " << os.omap.size() << dendl;
370 if (dry_run || skipping)
371 break;
372 ret = ioctx.omap_set(ob.hoid.hobj.oid.name, os.omap);
373 if (ret) {
374 cerr << "omap_set failed: " << cpp_strerror(ret) << std::endl;
375 if (ret != -EOPNOTSUPP)
376 return ret;
377 }
378 break;
379 case TYPE_OBJECT_END:
380 done = true;
381 if (need_align && databl.length() > 0) {
11fdf7f2 382 ceph_assert(databl.length() < alignment);
7c673cae
FG
383 dout(10) << "END write offset=" << out_offset << " len=" << databl.length() << dendl;
384 if (dry_run || skipping)
385 break;
386 ret = ioctx.write(ob.hoid.hobj.oid.name, databl, databl.length(), out_offset);
387 if (ret) {
388 cerr << "write failed: " << cpp_strerror(ret) << std::endl;
389 return ret;
390 }
391 }
392 break;
393 default:
394 cerr << "Unexpected section type " << type << std::endl;
395 return -EFAULT;
396 }
397 }
398 return 0;
399}