ceph/src/tools/erasure-code/ceph-erasure-code-tool.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "include/buffer.h"
   5 #include "include/stringify.h"
   6 #include "common/ceph_argparse.h"
   7 #include "common/config_proxy.h"
   8 #include "common/errno.h"
   9 #include "erasure-code/ErasureCode.h"
  10 #include "erasure-code/ErasureCodePlugin.h"
  11 #include "global/global_context.h"
  12 #include "global/global_init.h"
  13 #include "osd/ECUtil.h"
  14
  15 #include <iostream>
  16 #include <map>
  17 #include <string>
  18 #include <vector>
  19
  20 #include <boost/algorithm/string.hpp>
  21
  22 std::vector<std::string> display_params = {
  23   "chunk_count", "data_chunk_count", "coding_chunk_count"
  24 };
  25
  26 void usage(const std::string message, ostream &out) {
  27   if (!message.empty()) {
  28     out << message << std::endl;
  29     out << "" << std::endl;
  30   }
  31   out << "usage: ceph-erasure-code-tool test-plugin-exists <plugin>" << std::endl;
  32   out << "       ceph-erasure-code-tool validate-profile <profile> [<display-param> ...]" << std::endl;
  33   out << "       ceph-erasure-code-tool calc-chunk-size <profile> <object_size>" << std::endl;
  34   out << "       ceph-erasure-code-tool encode <profile> <stripe_unit> <want_to_encode> <fname>" << std::endl;
  35   out << "       ceph-erasure-code-tool decode <profile> <stripe_unit> <want_to_decode> <fname>" << std::endl;
  36   out << "" << std::endl;
  37   out << "  plugin          - plugin name" << std::endl;
  38   out << "  profile         - comma separated list of erasure-code profile settings" << std::endl;
  39   out << "                    example: plugin=jerasure,technique=reed_sol_van,k=3,m=2" << std::endl;
  40   out << "  display-param   - parameter to display (display all if empty)" << std::endl;
  41   out << "                    may be: " << display_params << std::endl;
  42   out << "  object_size     - object size" << std::endl;
  43   out << "  stripe_unit     - stripe unit" << std::endl;
  44   out << "  want_to_encode  - comma separated list of shards to encode" << std::endl;
  45   out << "  want_to_decode  - comma separated list of shards to decode" << std::endl;
  46   out << "  fname           - name for input/output files" << std::endl;
  47   out << "                    when encoding input is read form {fname} file," << std::endl;
  48   out << "                                  result is stored in {fname}.{shard} files" << std::endl;
  49   out << "                    when decoding input is read form {fname}.{shard} files," << std::endl;
  50   out << "                                  result is stored in {fname} file" << std::endl;
  51 }
  52
  53 int ec_init(const std::string &profile_str,
  54             const std::string &stripe_unit_str,
  55             ceph::ErasureCodeInterfaceRef *ec_impl,
  56             std::unique_ptr<ECUtil::stripe_info_t> *sinfo) {
  57   ceph::ErasureCodeProfile profile;
  58   std::vector<std::string> opts;
  59   boost::split(opts, profile_str, boost::is_any_of(", "));
  60   for (auto &opt_str : opts) {
  61     std::vector<std::string> opt;
  62     boost::split(opt, opt_str, boost::is_any_of("="));
  63     if (opt.size() <= 1) {
  64       usage("invalid profile", std::cerr);
  65       return 1;
  66     }
  67     profile[opt[0]] = opt[1];
  68   }
  69   auto plugin = profile.find("plugin");
  70   if (plugin == profile.end()) {
  71       usage("invalid profile: plugin not specified", std::cerr);
  72       return 1;
  73   }
  74
  75   stringstream ss;
  76   ceph::ErasureCodePluginRegistry::instance().factory(
  77     plugin->second, g_conf().get_val<std::string>("erasure_code_dir"),
  78     profile, ec_impl, &ss);
  79   if (!*ec_impl) {
  80     usage("invalid profile: " + ss.str(), std::cerr);
  81     return 1;
  82   }
  83
  84   if (sinfo == nullptr) {
  85     return 0;
  86   }
  87
  88   uint64_t stripe_unit = atoi(stripe_unit_str.c_str());
  89   if (stripe_unit <= 0) {
  90     usage("invalid stripe unit", std::cerr);
  91     return 1;
  92   }
  93
  94   uint64_t stripe_size = atoi(profile["k"].c_str());
  95   ceph_assert(stripe_size > 0);
  96   uint64_t stripe_width = stripe_size * stripe_unit;
  97   sinfo->reset(new ECUtil::stripe_info_t(stripe_size, stripe_width));
  98
  99   return 0;
 100 }
 101
 102 int do_test_plugin_exists(const std::vector<const char*> &args) {
 103   if (args.size() < 1) {
 104     usage("not enought arguments", std::cerr);
 105     return 1;
 106   }
 107
 108   ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
 109   ErasureCodePlugin *plugin;
 110   stringstream ss;
 111
 112   std::lock_guard l{instance.lock};
 113   int r = instance.load(
 114     args[0], g_conf().get_val<std::string>("erasure_code_dir"), &plugin, &ss);
 115   std::cerr << ss.str() << endl;
 116   return r;
 117 }
 118
 119 int do_validate_profile(const std::vector<const char*> &args) {
 120   if (args.size() < 1) {
 121     usage("not enought arguments", std::cerr);
 122     return 1;
 123   }
 124
 125   ceph::ErasureCodeInterfaceRef ec_impl;
 126   int r = ec_init(args[0], {}, &ec_impl, nullptr);
 127   if (r < 0) {
 128     return r;
 129   }
 130
 131   if (args.size() > 1) {
 132     std::set<std::string> valid_params(display_params.begin(),
 133                                        display_params.end());
 134     display_params.clear();
 135     for (size_t i = 1; i < args.size(); i++) {
 136       if (!valid_params.count(args[i])) {
 137         usage("invalid display param: " + std::string(args[i]), std::cerr);
 138         return 1;
 139       }
 140       display_params.push_back(args[i]);
 141     }
 142   }
 143
 144   for (auto &param : display_params) {
 145     if (display_params.size() > 1) {
 146       std::cout << param << ": ";
 147     }
 148     if (param == "chunk_count") {
 149       std::cout << ec_impl->get_chunk_count() << std::endl;
 150     } else if (param == "data_chunk_count") {
 151       std::cout << ec_impl->get_data_chunk_count() << std::endl;
 152     } else if (param == "coding_chunk_count") {
 153       std::cout << ec_impl->get_coding_chunk_count() << std::endl;
 154     } else {
 155       ceph_abort_msgf("unknown display_param: %s", param.c_str());
 156     }
 157   }
 158
 159   return 0;
 160 }
 161
 162 int do_calc_chunk_size(const std::vector<const char*> &args) {
 163   if (args.size() < 2) {
 164     usage("not enought arguments", std::cerr);
 165     return 1;
 166   }
 167
 168   ceph::ErasureCodeInterfaceRef ec_impl;
 169   int r = ec_init(args[0], {}, &ec_impl, nullptr);
 170   if (r < 0) {
 171     return r;
 172   }
 173
 174   uint64_t object_size = atoi(args[1]);
 175   if (object_size <= 0) {
 176     usage("invalid object size", std::cerr);
 177     return 1;
 178   }
 179
 180   std::cout << ec_impl->get_chunk_size(object_size) << std::endl;
 181   return 0;
 182 }
 183
 184 int do_encode(const std::vector<const char*> &args) {
 185   if (args.size() < 4) {
 186     usage("not enought arguments", std::cerr);
 187     return 1;
 188   }
 189
 190   ceph::ErasureCodeInterfaceRef ec_impl;
 191   std::unique_ptr<ECUtil::stripe_info_t> sinfo;
 192   int r = ec_init(args[0], args[1], &ec_impl, &sinfo);
 193   if (r < 0) {
 194     return r;
 195   }
 196
 197   std::set<int> want;
 198   std::vector<std::string> shards;
 199   boost::split(shards, args[2], boost::is_any_of(","));
 200   for (auto &shard : shards) {
 201     want.insert(atoi(shard.c_str()));
 202   }
 203   ceph::bufferlist decoded_data;
 204   std::string fname = args[3];
 205
 206   std::string error;
 207   r = decoded_data.read_file(fname.c_str(), &error);
 208   if (r < 0) {
 209     std::cerr << "failed to read " << fname << ": " << error << std::endl;
 210     return 1;
 211   }
 212
 213   uint64_t stripe_width = sinfo->get_stripe_width();
 214   if (decoded_data.length() % stripe_width != 0) {
 215     uint64_t pad = stripe_width - decoded_data.length() % stripe_width;
 216     decoded_data.append_zero(pad);
 217   }
 218
 219   std::map<int, ceph::bufferlist> encoded_data;
 220   r = ECUtil::encode(*sinfo, ec_impl, decoded_data, want, &encoded_data);
 221   if (r < 0) {
 222     std::cerr << "failed to encode: " << cpp_strerror(r) << std::endl;
 223     return 1;
 224   }
 225
 226   for (auto &[shard, bl] : encoded_data) {
 227     std::string name = fname + "." + stringify(shard);
 228     r = bl.write_file(name.c_str());
 229     if (r < 0) {
 230       std::cerr << "failed to write " << name << ": " << cpp_strerror(r)
 231                 << std::endl;
 232       return 1;
 233     }
 234   }
 235
 236   return 0;
 237 }
 238
 239 int do_decode(const std::vector<const char*> &args) {
 240   if (args.size() < 4) {
 241     usage("not enought arguments", std::cerr);
 242     return 1;
 243   }
 244
 245   ceph::ErasureCodeInterfaceRef ec_impl;
 246   std::unique_ptr<ECUtil::stripe_info_t> sinfo;
 247   int r = ec_init(args[0], args[1], &ec_impl, &sinfo);
 248   if (r < 0) {
 249     return r;
 250   }
 251
 252   std::map<int, ceph::bufferlist> encoded_data;
 253   std::vector<std::string> shards;
 254   boost::split(shards, args[2], boost::is_any_of(","));
 255   for (auto &shard : shards) {
 256     encoded_data[atoi(shard.c_str())] = {};
 257   }
 258   ceph::bufferlist decoded_data;
 259   std::string fname = args[3];
 260
 261   for (auto &[shard, bl] : encoded_data) {
 262     std::string name = fname + "." + stringify(shard);
 263     std::string error;
 264     r = bl.read_file(name.c_str(), &error);
 265     if (r < 0) {
 266       std::cerr << "failed to read " << name << ": " << error << std::endl;
 267       return 1;
 268     }
 269   }
 270
 271   r = ECUtil::decode(*sinfo, ec_impl, encoded_data, &decoded_data);
 272   if (r < 0) {
 273     std::cerr << "failed to decode: " << cpp_strerror(r) << std::endl;
 274     return 1;
 275   }
 276
 277   r = decoded_data.write_file(fname.c_str());
 278   if (r < 0) {
 279     std::cerr << "failed to write " << fname << ": " << cpp_strerror(r)
 280               << std::endl;
 281     return 1;
 282   }
 283
 284   return 0;
 285 }
 286
 287 int main(int argc, const char **argv) {
 288   std::vector<const char*> args;
 289   argv_to_vec(argc, argv, args);
 290   auto cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT,
 291                          CODE_ENVIRONMENT_UTILITY,
 292                          CINIT_FLAG_NO_MON_CONFIG);
 293
 294   if (args.empty() || args[0] == std::string("-h") ||
 295       args[0] == std::string("--help")) {
 296     usage("", std::cout);
 297     return 0;
 298   }
 299
 300   if (args.size() < 1) {
 301     usage("not enought arguments", std::cerr);
 302     return 1;
 303   }
 304
 305   std::string cmd = args[0];
 306   std::vector<const char*> cmd_args(args.begin() + 1, args.end());
 307
 308   if (cmd == "test-plugin-exists") {
 309     return do_test_plugin_exists(cmd_args);
 310   } else if (cmd == "validate-profile") {
 311     return do_validate_profile(cmd_args);
 312   } else if (cmd == "calc-chunk-size") {
 313     return do_calc_chunk_size(cmd_args);
 314   } else if (cmd == "encode") {
 315     return do_encode(cmd_args);
 316   } else if (cmd == "decode") {
 317     return do_decode(cmd_args);
 318   }
 319
 320   usage("invalid command: " + cmd, std::cerr);
 321   return 1;
 322 }