ceph/src/test/encoding/ceph_dencoder.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2015 Red Hat
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14
  15
  16 #include <errno.h>
  17 #include "include/types.h"
  18 #include "ceph_ver.h"
  19 #include "include/encoding.h"
  20 #include "include/ceph_features.h"
  21 #include "common/ceph_argparse.h"
  22 #include "common/Formatter.h"
  23 #include "common/errno.h"
  24 #include "msg/Message.h"
  25 #include "include/assert.h"
  26
  27 #define TYPE(t)
  28 #define TYPE_STRAYDATA(t)
  29 #define TYPE_NONDETERMINISTIC(t)
  30 #define TYPE_FEATUREFUL(t)
  31 #define TYPE_FEATUREFUL_STRAYDATA(t)
  32 #define TYPE_FEATUREFUL_NONDETERMINISTIC(t)
  33 #define TYPE_FEATUREFUL_NOCOPY(t)
  34 #define TYPE_NOCOPY(t)
  35 #define MESSAGE(t)
  36 #include "types.h"
  37 #undef TYPE
  38 #undef TYPE_STRAYDATA
  39 #undef TYPE_NONDETERMINISTIC
  40 #undef TYPE_NOCOPY
  41 #undef TYPE_FEATUREFUL
  42 #undef TYPE_FEATUREFUL_STRAYDATA
  43 #undef TYPE_FEATUREFUL_NONDETERMINISTIC
  44 #undef TYPE_FEATUREFUL_NOCOPY
  45 #undef MESSAGE
  46
  47 #define MB(m) ((m) * 1024 * 1024)
  48
  49 void usage(ostream &out)
  50 {
  51   out << "usage: ceph-dencoder [commands ...]" << std::endl;
  52   out << "\n";
  53   out << "  version             print version string (to stdout)\n";
  54   out << "\n";
  55   out << "  import <encfile>    read encoded data from encfile\n";
  56   out << "  export <outfile>    write encoded data to outfile\n";
  57   out << "\n";
  58   out << "  set_features <num>  set feature bits used for encoding\n";
  59   out << "  get_features        print feature bits (int) to stdout\n";
  60   out << "\n";
  61   out << "  list_types          list supported types\n";
  62   out << "  type <classname>    select in-memory type\n";
  63   out << "  skip <num>          skip <num> leading bytes before decoding\n";
  64   out << "  decode              decode into in-memory object\n";
  65   out << "  encode              encode in-memory object\n";
  66   out << "  dump_json           dump in-memory object as json (to stdout)\n";
  67   out << "  hexdump             print encoded data in hex\n";
  68   out << "\n";
  69   out << "  copy                copy object (via operator=)\n";
  70   out << "  copy_ctor           copy object (via copy ctor)\n";
  71   out << "\n";
  72   out << "  count_tests         print number of generated test objects (to stdout)\n";
  73   out << "  select_test <n>     select generated test object as in-memory object\n";
  74   out << "  is_deterministic    exit w/ success if type encodes deterministically\n";
  75 }
  76 struct Dencoder {
  77   virtual ~Dencoder() {}
  78   virtual string decode(bufferlist bl, uint64_t seek) = 0;
  79   virtual void encode(bufferlist& out, uint64_t features) = 0;
  80   virtual void dump(ceph::Formatter *f) = 0;
  81   virtual void copy() {
  82     cerr << "copy operator= not supported" << std::endl;
  83   }
  84   virtual void copy_ctor() {
  85     cerr << "copy ctor not supported" << std::endl;
  86   }
  87   virtual void generate() = 0;
  88   virtual int num_generated() = 0;
  89   virtual string select_generated(unsigned n) = 0;
  90   virtual bool is_deterministic() = 0;
  91   //virtual void print(ostream& out) = 0;
  92 };
  93
  94 template<class T>
  95 class DencoderBase : public Dencoder {
  96 protected:
  97   T* m_object;
  98   list<T*> m_list;
  99   bool stray_okay;
 100   bool nondeterministic;
 101
 102 public:
 103   DencoderBase(bool stray_okay, bool nondeterministic)
 104     : m_object(new T),
 105       stray_okay(stray_okay),
 106       nondeterministic(nondeterministic) {}
 107   ~DencoderBase() override {
 108     delete m_object;
 109   }
 110
 111   string decode(bufferlist bl, uint64_t seek) override {
 112     bufferlist::iterator p = bl.begin();
 113     p.seek(seek);
 114     try {
 115       ::decode(*m_object, p);
 116     }
 117     catch (buffer::error& e) {
 118       return e.what();
 119     }
 120     if (!stray_okay && !p.end()) {
 121       ostringstream ss;
 122       ss << "stray data at end of buffer, offset " << p.get_off();
 123       return ss.str();
 124     }
 125     return string();
 126   }
 127
 128   void encode(bufferlist& out, uint64_t features) override = 0;
 129
 130   void dump(ceph::Formatter *f) override {
 131     m_object->dump(f);
 132   }
 133   void generate() override {
 134     T::generate_test_instances(m_list);
 135   }
 136   int num_generated() override {
 137     return m_list.size();
 138   }
 139   string select_generated(unsigned i) override {
 140     // allow 0- or 1-based (by wrapping)
 141     if (i == 0)
 142       i = m_list.size();
 143     if ((i == 0) || (i > m_list.size()))
 144       return "invalid id for generated object";
 145     typename list<T*>::iterator p = m_list.begin();
 146     for (i--; i > 0 && p != m_list.end(); ++p, --i) ;
 147     m_object = *p;
 148     return string();
 149   }
 150
 151   bool is_deterministic() override {
 152     return !nondeterministic;
 153   }
 154 };
 155
 156 template<class T>
 157 class DencoderImplNoFeatureNoCopy : public DencoderBase<T> {
 158 public:
 159   DencoderImplNoFeatureNoCopy(bool stray_ok, bool nondeterministic)
 160     : DencoderBase<T>(stray_ok, nondeterministic) {}
 161   void encode(bufferlist& out, uint64_t features) override {
 162     out.clear();
 163     ::encode(*this->m_object, out);
 164   }
 165 };
 166
 167 template<class T>
 168 class DencoderImplNoFeature : public DencoderImplNoFeatureNoCopy<T> {
 169 public:
 170   DencoderImplNoFeature(bool stray_ok, bool nondeterministic)
 171     : DencoderImplNoFeatureNoCopy<T>(stray_ok, nondeterministic) {}
 172   void copy() override {
 173     T *n = new T;
 174     *n = *this->m_object;
 175     delete this->m_object;
 176     this->m_object = n;
 177   }
 178   void copy_ctor() override {
 179     T *n = new T(*this->m_object);
 180     delete this->m_object;
 181     this->m_object = n;
 182   }
 183 };
 184
 185 template<class T>
 186 class DencoderImplFeaturefulNoCopy : public DencoderBase<T> {
 187 public:
 188   DencoderImplFeaturefulNoCopy(bool stray_ok, bool nondeterministic)
 189     : DencoderBase<T>(stray_ok, nondeterministic) {}
 190   void encode(bufferlist& out, uint64_t features) override {
 191     out.clear();
 192     ::encode(*(this->m_object), out, features);
 193   }
 194 };
 195
 196 template<class T>
 197 class DencoderImplFeatureful : public DencoderImplFeaturefulNoCopy<T> {
 198 public:
 199   DencoderImplFeatureful(bool stray_ok, bool nondeterministic)
 200     : DencoderImplFeaturefulNoCopy<T>(stray_ok, nondeterministic) {}
 201   void copy() override {
 202     T *n = new T;
 203     *n = *this->m_object;
 204     delete this->m_object;
 205     this->m_object = n;
 206   }
 207   void copy_ctor() override {
 208     T *n = new T(*this->m_object);
 209     delete this->m_object;
 210     this->m_object = n;
 211   }
 212 };
 213
 214 template<class T>
 215 class MessageDencoderImpl : public Dencoder {
 216   T *m_object;
 217   list<T*> m_list;
 218
 219 public:
 220   MessageDencoderImpl() {
 221     m_object = new T;
 222   }
 223   ~MessageDencoderImpl() override {
 224     m_object->put();
 225   }
 226
 227   string decode(bufferlist bl, uint64_t seek) override {
 228     bufferlist::iterator p = bl.begin();
 229     p.seek(seek);
 230     try {
 231       Message *n = decode_message(g_ceph_context, 0, p);
 232       if (!n)
 233         throw std::runtime_error("failed to decode");
 234       if (n->get_type() != m_object->get_type()) {
 235         stringstream ss;
 236         ss << "decoded type " << n->get_type() << " instead of expected " << m_object->get_type();
 237         throw std::runtime_error(ss.str());
 238       }
 239       m_object->put();
 240       m_object = static_cast<T *>(n);
 241     }
 242     catch (buffer::error& e) {
 243       return e.what();
 244     }
 245     if (!p.end()) {
 246       ostringstream ss;
 247       ss << "stray data at end of buffer, offset " << p.get_off();
 248       return ss.str();
 249     }
 250     return string();
 251   }
 252
 253   void encode(bufferlist& out, uint64_t features) override {
 254     out.clear();
 255     encode_message(m_object, features, out);
 256   }
 257
 258   void dump(ceph::Formatter *f) override {
 259     m_object->dump(f);
 260   }
 261   void generate() override {
 262     //T::generate_test_instances(m_list);
 263   }
 264   int num_generated() override {
 265     return m_list.size();
 266   }
 267   string select_generated(unsigned i) override {
 268     // allow 0- or 1-based (by wrapping)
 269     if (i == 0)
 270       i = m_list.size();
 271     if ((i == 0) || (i > m_list.size()))
 272       return "invalid id for generated object";
 273     typename list<T*>::iterator p = m_list.begin();
 274     for (i--; i > 0 && p != m_list.end(); ++p, --i) ;
 275     m_object->put();
 276     m_object = *p;
 277     return string();
 278   }
 279   bool is_deterministic() override {
 280     return true;
 281   }
 282
 283   //void print(ostream& out) {
 284   //out << m_object << std::endl;
 285   //}
 286 };
 287
 288
 289
 290 int main(int argc, const char **argv)
 291 {
 292   // dencoders
 293   map<string,Dencoder*> dencoders;
 294
 295 #define T_STR(x) #x
 296 #define T_STRINGIFY(x) T_STR(x)
 297 #define TYPE(t) dencoders[T_STRINGIFY(t)] = new DencoderImplNoFeature<t>(false, false);
 298 #define TYPE_STRAYDATA(t) dencoders[T_STRINGIFY(t)] = new DencoderImplNoFeature<t>(true, false);
 299 #define TYPE_NONDETERMINISTIC(t) dencoders[T_STRINGIFY(t)] = new DencoderImplNoFeature<t>(false, true);
 300 #define TYPE_FEATUREFUL(t) dencoders[T_STRINGIFY(t)] = new DencoderImplFeatureful<t>(false, false);
 301 #define TYPE_FEATUREFUL_STRAYDATA(t) dencoders[T_STRINGIFY(t)] = new DencoderImplFeatureful<t>(true, false);
 302 #define TYPE_FEATUREFUL_NONDETERMINISTIC(t) dencoders[T_STRINGIFY(t)] = new DencoderImplFeatureful<t>(false, true);
 303 #define TYPE_FEATUREFUL_NOCOPY(t) dencoders[T_STRINGIFY(t)] = new DencoderImplFeaturefulNoCopy<t>(false, false);
 304 #define TYPE_NOCOPY(t) dencoders[T_STRINGIFY(t)] = new DencoderImplNoFeatureNoCopy<t>(false, false);
 305 #define MESSAGE(t) dencoders[T_STRINGIFY(t)] = new MessageDencoderImpl<t>;
 306 #include "types.h"
 307 #undef TYPE
 308 #undef TYPE_STRAYDATA
 309 #undef TYPE_NONDETERMINISTIC
 310 #undef TYPE_NOCOPY
 311 #undef TYPE_FEATUREFUL
 312 #undef TYPE_FEATUREFUL_STRAYDATA
 313 #undef TYPE_FEATUREFUL_NONDETERMINISTIC
 314 #undef TYPE_FEATUREFUL_NOCOPY
 315 #undef T_STR
 316 #undef T_STRINGIFY
 317
 318   vector<const char*> args;
 319   argv_to_vec(argc, argv, args);
 320   env_to_vec(args);
 321
 322   Dencoder *den = NULL;
 323   uint64_t features = CEPH_FEATURES_SUPPORTED_DEFAULT;
 324   bufferlist encbl;
 325   uint64_t skip = 0;
 326
 327   if (args.empty()) {
 328     cerr << "-h for help" << std::endl;
 329     exit(1);
 330   }
 331   for (std::vector<const char*>::iterator i = args.begin(); i != args.end(); ++i) {
 332     string err;
 333
 334     if (*i == string("help") || *i == string("-h") || *i == string("--help")) {
 335       usage(cout);
 336       exit(0);
 337     } else if (*i == string("version")) {
 338       cout << CEPH_GIT_NICE_VER << std::endl;
 339     } else if (*i == string("list_types")) {
 340       for (map<string,Dencoder*>::iterator p = dencoders.begin();
 341            p != dencoders.end();
 342            ++p)
 343         cout << p->first << std::endl;
 344       exit(0);
 345     } else if (*i == string("type")) {
 346       ++i;
 347       if (i == args.end()) {
 348         cerr << "expecting type" << std::endl;
 349         exit(1);
 350       }
 351       string cname = *i;
 352       if (!dencoders.count(cname)) {
 353         cerr << "class '" << cname << "' unknown" << std::endl;
 354         exit(1);
 355       }
 356       den = dencoders[cname];
 357       den->generate();
 358     } else if (*i == string("skip")) {
 359       ++i;
 360       if (i == args.end()) {
 361         cerr << "expecting byte count" << std::endl;
 362         exit(1);
 363       }
 364       skip = atoi(*i);
 365     } else if (*i == string("get_features")) {
 366       cout << CEPH_FEATURES_SUPPORTED_DEFAULT << std::endl;
 367       exit(0);
 368     } else if (*i == string("set_features")) {
 369       ++i;
 370       if (i == args.end()) {
 371         cerr << "expecting features" << std::endl;
 372         exit(1);
 373       }
 374       features = atoll(*i);
 375     } else if (*i == string("encode")) {
 376       if (!den) {
 377         cerr << "must first select type with 'type <name>'" << std::endl;
 378         exit(1);
 379       }
 380       den->encode(encbl, features | CEPH_FEATURE_RESERVED); // hack for OSDMap
 381     } else if (*i == string("decode")) {
 382       if (!den) {
 383         cerr << "must first select type with 'type <name>'" << std::endl;
 384         exit(1);
 385       }
 386       err = den->decode(encbl, skip);
 387     } else if (*i == string("copy_ctor")) {
 388       if (!den) {
 389         cerr << "must first select type with 'type <name>'" << std::endl;
 390         exit(1);
 391       }
 392       den->copy_ctor();
 393     } else if (*i == string("copy")) {
 394       if (!den) {
 395         cerr << "must first select type with 'type <name>'" << std::endl;
 396         exit(1);
 397       }
 398       den->copy();
 399     } else if (*i == string("dump_json")) {
 400       if (!den) {
 401         cerr << "must first select type with 'type <name>'" << std::endl;
 402         exit(1);
 403       }
 404       JSONFormatter jf(true);
 405       jf.open_object_section("object");
 406       den->dump(&jf);
 407       jf.close_section();
 408       jf.flush(cout);
 409       cout << std::endl;
 410
 411     } else if (*i == string("hexdump")) {
 412       encbl.hexdump(cout);
 413     } else if (*i == string("import")) {
 414       ++i;
 415       if (i == args.end()) {
 416         cerr << "expecting filename" << std::endl;
 417         exit(1);
 418       }
 419       int r;
 420       if (*i == string("-")) {
 421         *i = "stdin";
 422         // Read up to 1mb if stdin specified
 423         r = encbl.read_fd(STDIN_FILENO, MB(1));
 424       } else {
 425         r = encbl.read_file(*i, &err);
 426       }
 427       if (r < 0) {
 428         cerr << "error reading " << *i << ": " << err << std::endl;
 429         exit(1);
 430       }
 431
 432     } else if (*i == string("export")) {
 433       ++i;
 434       if (i == args.end()) {
 435         cerr << "expecting filename" << std::endl;
 436         exit(1);
 437       }
 438       int fd = ::open(*i, O_WRONLY|O_CREAT|O_TRUNC, 0644);
 439       if (fd < 0) {
 440         cerr << "error opening " << *i << " for write: " << cpp_strerror(errno) << std::endl;
 441         exit(1);
 442       }
 443       int r = encbl.write_fd(fd);
 444       if (r < 0) {
 445         cerr << "error writing " << *i << ": " << cpp_strerror(errno) << std::endl;
 446         exit(1);
 447       }
 448       ::close(fd);
 449
 450     } else if (*i == string("count_tests")) {
 451       if (!den) {
 452         cerr << "must first select type with 'type <name>'" << std::endl;
 453         exit(1);
 454       }
 455       cout << den->num_generated() << std::endl;
 456     } else if (*i == string("select_test")) {
 457       if (!den) {
 458         cerr << "must first select type with 'type <name>'" << std::endl;
 459         exit(1);
 460       }
 461       ++i;
 462       if (i == args.end()) {
 463         cerr << "expecting instance number" << std::endl;
 464         exit(1);
 465       }
 466       int n = atoi(*i);
 467       err = den->select_generated(n);
 468     } else if (*i == string("is_deterministic")) {
 469       if (!den) {
 470         cerr << "must first select type with 'type <name>'" << std::endl;
 471         exit(1);
 472       }
 473       if (den->is_deterministic())
 474         exit(0);
 475       else
 476         exit(1);
 477     } else {
 478       cerr << "unknown option '" << *i << "'" << std::endl;
 479       exit(1);
 480     }
 481     if (err.length()) {
 482       cerr << "error: " << err << std::endl;
 483       exit(1);
 484     }
 485   }
 486   return 0;
 487 }