1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "include/buffer.h"
5 #include "include/stringify.h"
6 #include "common/ceph_argparse.h"
7 #include "common/config_proxy.h"
8 #include "common/errno.h"
9 #include "erasure-code/ErasureCode.h"
10 #include "erasure-code/ErasureCodePlugin.h"
11 #include "global/global_context.h"
12 #include "global/global_init.h"
13 #include "osd/ECUtil.h"
20 #include <boost/algorithm/string.hpp>
24 std::vector
<std::string
> display_params
= {
25 "chunk_count", "data_chunk_count", "coding_chunk_count"
28 void usage(const std::string message
, ostream
&out
) {
29 if (!message
.empty()) {
30 out
<< message
<< std::endl
;
31 out
<< "" << std::endl
;
33 out
<< "usage: ceph-erasure-code-tool test-plugin-exists <plugin>" << std::endl
;
34 out
<< " ceph-erasure-code-tool validate-profile <profile> [<display-param> ...]" << std::endl
;
35 out
<< " ceph-erasure-code-tool calc-chunk-size <profile> <object_size>" << std::endl
;
36 out
<< " ceph-erasure-code-tool encode <profile> <stripe_unit> <want_to_encode> <fname>" << std::endl
;
37 out
<< " ceph-erasure-code-tool decode <profile> <stripe_unit> <want_to_decode> <fname>" << std::endl
;
38 out
<< "" << std::endl
;
39 out
<< " plugin - plugin name" << std::endl
;
40 out
<< " profile - comma separated list of erasure-code profile settings" << std::endl
;
41 out
<< " example: plugin=jerasure,technique=reed_sol_van,k=3,m=2" << std::endl
;
42 out
<< " display-param - parameter to display (display all if empty)" << std::endl
;
43 out
<< " may be: " << display_params
<< std::endl
;
44 out
<< " object_size - object size" << std::endl
;
45 out
<< " stripe_unit - stripe unit" << std::endl
;
46 out
<< " want_to_encode - comma separated list of shards to encode" << std::endl
;
47 out
<< " want_to_decode - comma separated list of shards to decode" << std::endl
;
48 out
<< " fname - name for input/output files" << std::endl
;
49 out
<< " when encoding input is read form {fname} file," << std::endl
;
50 out
<< " result is stored in {fname}.{shard} files" << std::endl
;
51 out
<< " when decoding input is read form {fname}.{shard} files," << std::endl
;
52 out
<< " result is stored in {fname} file" << std::endl
;
55 int ec_init(const std::string
&profile_str
,
56 const std::string
&stripe_unit_str
,
57 ceph::ErasureCodeInterfaceRef
*ec_impl
,
58 std::unique_ptr
<ECUtil::stripe_info_t
> *sinfo
) {
59 ceph::ErasureCodeProfile profile
;
60 std::vector
<std::string
> opts
;
61 boost::split(opts
, profile_str
, boost::is_any_of(", "));
62 for (auto &opt_str
: opts
) {
63 std::vector
<std::string
> opt
;
64 boost::split(opt
, opt_str
, boost::is_any_of("="));
65 if (opt
.size() <= 1) {
66 usage("invalid profile", std::cerr
);
69 profile
[opt
[0]] = opt
[1];
71 auto plugin
= profile
.find("plugin");
72 if (plugin
== profile
.end()) {
73 usage("invalid profile: plugin not specified", std::cerr
);
78 ceph::ErasureCodePluginRegistry::instance().factory(
79 plugin
->second
, g_conf().get_val
<std::string
>("erasure_code_dir"),
80 profile
, ec_impl
, &ss
);
82 usage("invalid profile: " + ss
.str(), std::cerr
);
86 if (sinfo
== nullptr) {
90 uint64_t stripe_unit
= atoi(stripe_unit_str
.c_str());
91 if (stripe_unit
<= 0) {
92 usage("invalid stripe unit", std::cerr
);
96 uint64_t stripe_size
= atoi(profile
["k"].c_str());
97 ceph_assert(stripe_size
> 0);
98 uint64_t stripe_width
= stripe_size
* stripe_unit
;
99 sinfo
->reset(new ECUtil::stripe_info_t(stripe_size
, stripe_width
));
104 int do_test_plugin_exists(const std::vector
<const char*> &args
) {
105 if (args
.size() < 1) {
106 usage("not enought arguments", std::cerr
);
110 ErasureCodePluginRegistry
&instance
= ErasureCodePluginRegistry::instance();
111 ErasureCodePlugin
*plugin
;
114 std::lock_guard l
{instance
.lock
};
115 int r
= instance
.load(
116 args
[0], g_conf().get_val
<std::string
>("erasure_code_dir"), &plugin
, &ss
);
117 std::cerr
<< ss
.str() << endl
;
121 int do_validate_profile(const std::vector
<const char*> &args
) {
122 if (args
.size() < 1) {
123 usage("not enought arguments", std::cerr
);
127 ceph::ErasureCodeInterfaceRef ec_impl
;
128 int r
= ec_init(args
[0], {}, &ec_impl
, nullptr);
133 if (args
.size() > 1) {
134 std::set
<std::string
> valid_params(display_params
.begin(),
135 display_params
.end());
136 display_params
.clear();
137 for (size_t i
= 1; i
< args
.size(); i
++) {
138 if (!valid_params
.count(args
[i
])) {
139 usage("invalid display param: " + std::string(args
[i
]), std::cerr
);
142 display_params
.push_back(args
[i
]);
146 for (auto ¶m
: display_params
) {
147 if (display_params
.size() > 1) {
148 std::cout
<< param
<< ": ";
150 if (param
== "chunk_count") {
151 std::cout
<< ec_impl
->get_chunk_count() << std::endl
;
152 } else if (param
== "data_chunk_count") {
153 std::cout
<< ec_impl
->get_data_chunk_count() << std::endl
;
154 } else if (param
== "coding_chunk_count") {
155 std::cout
<< ec_impl
->get_coding_chunk_count() << std::endl
;
157 ceph_abort_msgf("unknown display_param: %s", param
.c_str());
164 int do_calc_chunk_size(const std::vector
<const char*> &args
) {
165 if (args
.size() < 2) {
166 usage("not enought arguments", std::cerr
);
170 ceph::ErasureCodeInterfaceRef ec_impl
;
171 int r
= ec_init(args
[0], {}, &ec_impl
, nullptr);
176 uint64_t object_size
= atoi(args
[1]);
177 if (object_size
<= 0) {
178 usage("invalid object size", std::cerr
);
182 std::cout
<< ec_impl
->get_chunk_size(object_size
) << std::endl
;
186 int do_encode(const std::vector
<const char*> &args
) {
187 if (args
.size() < 4) {
188 usage("not enought arguments", std::cerr
);
192 ceph::ErasureCodeInterfaceRef ec_impl
;
193 std::unique_ptr
<ECUtil::stripe_info_t
> sinfo
;
194 int r
= ec_init(args
[0], args
[1], &ec_impl
, &sinfo
);
200 std::vector
<std::string
> shards
;
201 boost::split(shards
, args
[2], boost::is_any_of(","));
202 for (auto &shard
: shards
) {
203 want
.insert(atoi(shard
.c_str()));
205 ceph::bufferlist decoded_data
;
206 std::string fname
= args
[3];
209 r
= decoded_data
.read_file(fname
.c_str(), &error
);
211 std::cerr
<< "failed to read " << fname
<< ": " << error
<< std::endl
;
215 uint64_t stripe_width
= sinfo
->get_stripe_width();
216 if (decoded_data
.length() % stripe_width
!= 0) {
217 uint64_t pad
= stripe_width
- decoded_data
.length() % stripe_width
;
218 decoded_data
.append_zero(pad
);
221 std::map
<int, ceph::bufferlist
> encoded_data
;
222 r
= ECUtil::encode(*sinfo
, ec_impl
, decoded_data
, want
, &encoded_data
);
224 std::cerr
<< "failed to encode: " << cpp_strerror(r
) << std::endl
;
228 for (auto &[shard
, bl
] : encoded_data
) {
229 std::string name
= fname
+ "." + stringify(shard
);
230 r
= bl
.write_file(name
.c_str());
232 std::cerr
<< "failed to write " << name
<< ": " << cpp_strerror(r
)
241 int do_decode(const std::vector
<const char*> &args
) {
242 if (args
.size() < 4) {
243 usage("not enought arguments", std::cerr
);
247 ceph::ErasureCodeInterfaceRef ec_impl
;
248 std::unique_ptr
<ECUtil::stripe_info_t
> sinfo
;
249 int r
= ec_init(args
[0], args
[1], &ec_impl
, &sinfo
);
254 std::map
<int, ceph::bufferlist
> encoded_data
;
255 std::vector
<std::string
> shards
;
256 boost::split(shards
, args
[2], boost::is_any_of(","));
257 for (auto &shard
: shards
) {
258 encoded_data
[atoi(shard
.c_str())] = {};
260 ceph::bufferlist decoded_data
;
261 std::string fname
= args
[3];
263 for (auto &[shard
, bl
] : encoded_data
) {
264 std::string name
= fname
+ "." + stringify(shard
);
266 r
= bl
.read_file(name
.c_str(), &error
);
268 std::cerr
<< "failed to read " << name
<< ": " << error
<< std::endl
;
273 r
= ECUtil::decode(*sinfo
, ec_impl
, encoded_data
, &decoded_data
);
275 std::cerr
<< "failed to decode: " << cpp_strerror(r
) << std::endl
;
279 r
= decoded_data
.write_file(fname
.c_str());
281 std::cerr
<< "failed to write " << fname
<< ": " << cpp_strerror(r
)
289 int main(int argc
, const char **argv
) {
290 auto args
= argv_to_vec(argc
, argv
);
291 auto cct
= global_init(nullptr, args
, CEPH_ENTITY_TYPE_CLIENT
,
292 CODE_ENVIRONMENT_UTILITY
,
293 CINIT_FLAG_NO_MON_CONFIG
);
295 if (args
.empty() || args
[0] == std::string("-h") ||
296 args
[0] == std::string("--help")) {
297 usage("", std::cout
);
301 if (args
.size() < 1) {
302 usage("not enought arguments", std::cerr
);
306 std::string cmd
= args
[0];
307 std::vector
<const char*> cmd_args(args
.begin() + 1, args
.end());
309 if (cmd
== "test-plugin-exists") {
310 return do_test_plugin_exists(cmd_args
);
311 } else if (cmd
== "validate-profile") {
312 return do_validate_profile(cmd_args
);
313 } else if (cmd
== "calc-chunk-size") {
314 return do_calc_chunk_size(cmd_args
);
315 } else if (cmd
== "encode") {
316 return do_encode(cmd_args
);
317 } else if (cmd
== "decode") {
318 return do_decode(cmd_args
);
321 usage("invalid command: " + cmd
, std::cerr
);