]>
git.proxmox.com Git - ceph.git/blob - ceph/src/tools/rados/RadosImport.cc
1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2015 Red Hat
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
16 #include "common/errno.h"
18 #include "osd/PGLog.h"
19 #include "RadosImport.h"
21 #define dout_context g_ceph_context
22 #define dout_subsys ceph_subsys_rados
24 int RadosImport::import(std::string pool
, bool no_overwrite
)
26 librados::IoCtx ioctx
;
27 librados::Rados cluster
;
29 char *id
= getenv("CEPH_CLIENT_ID");
30 if (id
) cerr
<< "Client id is: " << id
<< std::endl
;
31 int ret
= cluster
.init(id
);
33 cerr
<< "Error " << ret
<< " in cluster.init" << std::endl
;
36 ret
= cluster
.conf_read_file(NULL
);
38 cerr
<< "Error " << ret
<< " in cluster.conf_read_file" << std::endl
;
41 ret
= cluster
.conf_parse_env(NULL
);
43 cerr
<< "Error " << ret
<< " in cluster.conf_read_env" << std::endl
;
46 ret
= cluster
.connect();
48 cerr
<< "Error " << ret
<< " in cluster.connect" << std::endl
;
52 ret
= cluster
.ioctx_create(pool
.c_str(), ioctx
);
54 cerr
<< "ioctx_create " << pool
<< " failed with " << ret
<< std::endl
;
58 return import(ioctx
, no_overwrite
);
61 int RadosImport::import(librados::IoCtx
&io_ctx
, bool no_overwrite
)
65 PGLog::IndexedLog log
;
67 int ret
= read_super();
71 if (sh
.magic
!= super_header::super_magic
) {
72 cerr
<< "Invalid magic number: 0x"
73 << std::hex
<< sh
.magic
<< " vs. 0x" << super_header::super_magic
74 << std::dec
<< std::endl
;
78 if (sh
.version
> super_header::super_ver
) {
79 cerr
<< "Can't handle export format version=" << sh
.version
<< std::endl
;
83 //First section must be TYPE_PG_BEGIN
85 ret
= read_section(&type
, &ebl
);
89 bool pool_mode
= false;
90 if (type
== TYPE_POOL_BEGIN
) {
92 cout
<< "Importing pool" << std::endl
;
93 } else if (type
== TYPE_PG_BEGIN
) {
94 auto ebliter
= ebl
.cbegin();
97 spg_t pgid
= pgb
.pgid
;;
98 if (!pgid
.is_no_shard()) {
99 cerr
<< "Importing Erasure Coded shard is not supported" << std::endl
;
102 dout(10) << "Exported features: " << pgb
.superblock
.compat_features
<< dendl
;
103 cout
<< "Importing from pgid " << pgid
<< std::endl
;
105 cerr
<< "Invalid initial section code " << type
<< std::endl
;
109 // XXX: How to check export features?
111 if (sb
.compat_features
.compare(pgb
.superblock
.compat_features
) == -1) {
112 cerr
<< "Export has incompatible features set "
113 << pgb
.superblock
.compat_features
<< std::endl
;
118 #if defined(__linux__)
119 if (file_fd
!= STDIN_FILENO
)
120 posix_fadvise(file_fd
, 0, 0, POSIX_FADV_SEQUENTIAL
);
124 bool found_metadata
= false;
126 ret
= read_section(&type
, &ebl
);
130 //cout << "do_import: Section type " << hex << type << dec << std::endl;
131 if (type
>= END_OF_TYPES
) {
132 cout
<< "Skipping unknown section type" << std::endl
;
136 case TYPE_OBJECT_BEGIN
:
137 ret
= get_object_rados(io_ctx
, ebl
, no_overwrite
);
139 cerr
<< "Error inserting object: " << ret
<< std::endl
;
143 case TYPE_PG_METADATA
:
144 dout(10) << "Don't care about the old metadata" << dendl
;
145 found_metadata
= true;
158 if (!(pool_mode
|| found_metadata
)) {
159 cerr
<< "Missing metadata section!" << std::endl
;
162 #if defined(__linux__)
163 if (file_fd
!= STDIN_FILENO
)
164 posix_fadvise(file_fd
, 0, 0, POSIX_FADV_DONTNEED
);
169 int RadosImport::get_object_rados(librados::IoCtx
&ioctx
, bufferlist
&bl
, bool no_overwrite
)
171 auto ebliter
= bl
.cbegin();
174 map
<string
,bufferlist
>::iterator i
;
183 ceph_assert(g_ceph_context
);
184 if (ob
.hoid
.hobj
.nspace
== g_ceph_context
->_conf
->osd_hit_set_namespace
) {
185 cout
<< "Skipping internal object " << ob
.hoid
<< std::endl
;
190 if (!ob
.hoid
.hobj
.is_head()) {
191 cout
<< "Skipping non-head for " << ob
.hoid
<< std::endl
;
196 ioctx
.set_namespace(ob
.hoid
.hobj
.get_namespace());
203 int ret
= ioctx
.stat(ob
.hoid
.hobj
.oid
.name
, &psize
, &pmtime
);
206 // Could set skipping, but dry-run doesn't change anything either
207 msg
= "Skipping existing";
209 msg
= "***Overwrite***";
212 int ret
= ioctx
.create(ob
.hoid
.hobj
.oid
.name
, true);
213 if (ret
&& ret
!= -EEXIST
) {
214 cerr
<< "create failed: " << cpp_strerror(ret
) << std::endl
;
217 if (ret
== -EEXIST
) {
219 msg
= "Skipping existing";
222 msg
= "***Overwrite***";
223 ret
= ioctx
.remove(ob
.hoid
.hobj
.oid
.name
);
225 cerr
<< "remove failed: " << cpp_strerror(ret
) << std::endl
;
228 ret
= ioctx
.create(ob
.hoid
.hobj
.oid
.name
, true);
229 // If object re-appeared after removal, let's just skip it
230 if (ret
== -EEXIST
) {
232 msg
= "Skipping in-use object";
236 cerr
<< "create failed: " << cpp_strerror(ret
) << std::endl
;
243 cout
<< msg
<< " " << ob
.hoid
<< std::endl
;
245 bool need_align
= false;
246 uint64_t alignment
= 0;
251 int ret
= ioctx
.pool_requires_alignment2(&need_align
);
253 cerr
<< "pool_requires_alignment2 failed: " << cpp_strerror(ret
)
259 ret
= ioctx
.pool_required_alignment2(&alignment
);
261 cerr
<< "pool_required_alignment2 failed: " << cpp_strerror(ret
)
265 ceph_assert(alignment
!= 0);
270 dout(10) << "alignment = " << alignment
<< dendl
;
273 bufferlist ebl
, databl
;
274 uint64_t in_offset
= 0, out_offset
= 0;
278 int ret
= read_section(&type
, &ebl
);
280 cerr
<< "Error reading section: " << ret
<< std::endl
;
284 ebliter
= ebl
.cbegin();
285 //cout << "\tdo_object: Section type " << hex << type << dec << std::endl;
286 //cout << "\t\tsection size " << ebl.length() << std::endl;
287 if (type
>= END_OF_TYPES
) {
288 cout
<< "Skipping unknown object section type" << std::endl
;
294 dout(10) << "\tdata: offset " << ds
.offset
<< " len " << ds
.len
<< dendl
;
296 if (ds
.offset
!= in_offset
) {
297 cerr
<< "Discontiguous object data in export" << std::endl
;
300 ceph_assert(ds
.databl
.length() == ds
.len
);
301 databl
.claim_append(ds
.databl
);
303 if (databl
.length() >= alignment
) {
304 uint64_t rndlen
= uint64_t(databl
.length() / alignment
) * alignment
;
305 dout(10) << "write offset=" << out_offset
<< " len=" << rndlen
<< dendl
;
306 if (!dry_run
&& !skipping
) {
307 ret
= ioctx
.write(ob
.hoid
.hobj
.oid
.name
, databl
, rndlen
, out_offset
);
309 cerr
<< "write failed: " << cpp_strerror(ret
) << std::endl
;
313 out_offset
+= rndlen
;
315 if (databl
.length() > rndlen
) {
316 ceph_assert(databl
.length() - rndlen
< alignment
);
317 n
.substr_of(databl
, rndlen
, databl
.length() - rndlen
);
323 if (!dry_run
&& !skipping
) {
324 ret
= ioctx
.write(ob
.hoid
.hobj
.oid
.name
, ds
.databl
, ds
.len
, ds
.offset
);
326 cerr
<< "write failed: " << cpp_strerror(ret
) << std::endl
;
334 dout(10) << "\tattrs: len " << as
.data
.size() << dendl
;
335 if (dry_run
|| skipping
)
337 for (std::map
<string
,bufferlist
>::iterator i
= as
.data
.begin();
338 i
!= as
.data
.end(); ++i
) {
339 // The user xattrs that we want all begin with "_" with length > 1.
340 // Drop key "_" and all attributes that do not start with '_'
341 if (i
->first
== "_" || i
->first
[0] != '_')
343 ret
= ioctx
.setxattr(ob
.hoid
.hobj
.oid
.name
, i
->first
.substr(1).c_str(), i
->second
);
345 cerr
<< "setxattr failed: " << cpp_strerror(ret
) << std::endl
;
346 if (ret
!= -EOPNOTSUPP
)
354 dout(10) << "\tomap header: " << string(oh
.hdr
.c_str(), oh
.hdr
.length())
356 if (dry_run
|| skipping
)
358 ret
= ioctx
.omap_set_header(ob
.hoid
.hobj
.oid
.name
, oh
.hdr
);
360 cerr
<< "omap_set_header failed: " << cpp_strerror(ret
) << std::endl
;
361 if (ret
!= -EOPNOTSUPP
)
368 dout(10) << "\tomap: size " << os
.omap
.size() << dendl
;
369 if (dry_run
|| skipping
)
371 ret
= ioctx
.omap_set(ob
.hoid
.hobj
.oid
.name
, os
.omap
);
373 cerr
<< "omap_set failed: " << cpp_strerror(ret
) << std::endl
;
374 if (ret
!= -EOPNOTSUPP
)
378 case TYPE_OBJECT_END
:
380 if (need_align
&& databl
.length() > 0) {
381 ceph_assert(databl
.length() < alignment
);
382 dout(10) << "END write offset=" << out_offset
<< " len=" << databl
.length() << dendl
;
383 if (dry_run
|| skipping
)
385 ret
= ioctx
.write(ob
.hoid
.hobj
.oid
.name
, databl
, databl
.length(), out_offset
);
387 cerr
<< "write failed: " << cpp_strerror(ret
) << std::endl
;
393 cerr
<< "Unexpected section type " << type
<< std::endl
;