ceph/src/rocksdb/utilities/document/json_document.cc

   1 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5 #ifndef ROCKSDB_LITE
   6
   7 #include "rocksdb/utilities/json_document.h"
   8
   9 #ifndef __STDC_FORMAT_MACROS
  10 #define __STDC_FORMAT_MACROS
  11 #endif
  12
  13 #include <assert.h>
  14 #include <inttypes.h>
  15 #include <string.h>
  16
  17 #include <functional>
  18 #include <limits>
  19 #include <map>
  20 #include <memory>
  21 #include <string>
  22 #include <vector>
  23
  24
  25 #include "third-party/fbson/FbsonDocument.h"
  26 #include "third-party/fbson/FbsonJsonParser.h"
  27 #include "third-party/fbson/FbsonUtil.h"
  28 #include "util/coding.h"
  29
  30 using std::placeholders::_1;
  31
  32 namespace {
  33
  34 size_t ObjectNumElem(const fbson::ObjectVal& objectVal) {
  35   size_t size = 0;
  36   for (auto keyValuePair : objectVal) {
  37     (void)keyValuePair;
  38     ++size;
  39   }
  40   return size;
  41 }
  42
  43 template <typename Func>
  44 void InitJSONDocument(std::unique_ptr<char[]>* data,
  45                       fbson::FbsonValue** value,
  46                       Func f) {
  47   // TODO(stash): maybe add function to FbsonDocument to avoid creating array?
  48   fbson::FbsonWriter writer;
  49   bool res __attribute__((__unused__)) = writer.writeStartArray();
  50   assert(res);
  51   uint32_t bytesWritten __attribute__((__unused__));
  52   bytesWritten = f(writer);
  53   assert(bytesWritten != 0);
  54   res = writer.writeEndArray();
  55   assert(res);
  56   char* buf = new char[writer.getOutput()->getSize()];
  57   memcpy(buf, writer.getOutput()->getBuffer(), writer.getOutput()->getSize());
  58
  59   *value = ((fbson::FbsonDocument *)buf)->getValue();
  60   assert((*value)->isArray());
  61   assert(((fbson::ArrayVal*)*value)->numElem() == 1);
  62   *value = ((fbson::ArrayVal*)*value)->get(0);
  63   data->reset(buf);
  64 }
  65
  66 void InitString(std::unique_ptr<char[]>* data,
  67                 fbson::FbsonValue** value,
  68                 const std::string& s) {
  69   InitJSONDocument(data, value, std::bind(
  70       [](fbson::FbsonWriter& writer, const std::string& str) -> uint32_t {
  71         bool res __attribute__((__unused__)) = writer.writeStartString();
  72         assert(res);
  73         auto bytesWritten = writer.writeString(str.c_str(),
  74                             static_cast<uint32_t>(str.length()));
  75         res = writer.writeEndString();
  76         assert(res);
  77         // If the string is empty, then bytesWritten == 0, and assert in
  78         // InitJsonDocument will fail.
  79         return bytesWritten + static_cast<uint32_t>(str.empty());
  80       },
  81   _1, s));
  82 }
  83
  84 bool IsNumeric(fbson::FbsonValue* value) {
  85   return value->isInt8() || value->isInt16() ||
  86          value->isInt32() ||  value->isInt64();
  87 }
  88
  89 int64_t GetInt64ValFromFbsonNumericType(fbson::FbsonValue* value) {
  90   switch (value->type()) {
  91     case fbson::FbsonType::T_Int8:
  92       return reinterpret_cast<fbson::Int8Val*>(value)->val();
  93     case fbson::FbsonType::T_Int16:
  94       return reinterpret_cast<fbson::Int16Val*>(value)->val();
  95     case fbson::FbsonType::T_Int32:
  96       return reinterpret_cast<fbson::Int32Val*>(value)->val();
  97     case fbson::FbsonType::T_Int64:
  98       return reinterpret_cast<fbson::Int64Val*>(value)->val();
  99     default:
 100       assert(false);
 101   }
 102   return 0;
 103 }
 104
 105 bool IsComparable(fbson::FbsonValue* left, fbson::FbsonValue* right) {
 106   if (left->type() == right->type()) {
 107     return true;
 108   }
 109   if (IsNumeric(left) && IsNumeric(right)) {
 110     return true;
 111   }
 112   return false;
 113 }
 114
 115 void CreateArray(std::unique_ptr<char[]>* data, fbson::FbsonValue** value) {
 116   fbson::FbsonWriter writer;
 117   bool res __attribute__((__unused__)) = writer.writeStartArray();
 118   assert(res);
 119   res = writer.writeEndArray();
 120   assert(res);
 121   data->reset(new char[writer.getOutput()->getSize()]);
 122   memcpy(data->get(),
 123          writer.getOutput()->getBuffer(),
 124          writer.getOutput()->getSize());
 125   *value = reinterpret_cast<fbson::FbsonDocument*>(data->get())->getValue();
 126 }
 127
 128 void CreateObject(std::unique_ptr<char[]>* data, fbson::FbsonValue** value) {
 129   fbson::FbsonWriter writer;
 130   bool res __attribute__((__unused__)) = writer.writeStartObject();
 131   assert(res);
 132   res = writer.writeEndObject();
 133   assert(res);
 134   data->reset(new char[writer.getOutput()->getSize()]);
 135   memcpy(data->get(),
 136          writer.getOutput()->getBuffer(),
 137          writer.getOutput()->getSize());
 138   *value = reinterpret_cast<fbson::FbsonDocument*>(data->get())->getValue();
 139 }
 140
 141 }  // namespace
 142
 143 namespace rocksdb {
 144
 145
 146 // TODO(stash): find smth easier
 147 JSONDocument::JSONDocument() {
 148   InitJSONDocument(&data_,
 149                    &value_,
 150                    std::bind(&fbson::FbsonWriter::writeNull, _1));
 151 }
 152
 153 JSONDocument::JSONDocument(bool b) {
 154   InitJSONDocument(&data_,
 155                    &value_,
 156                    std::bind(&fbson::FbsonWriter::writeBool, _1, b));
 157 }
 158
 159 JSONDocument::JSONDocument(double d) {
 160   InitJSONDocument(&data_,
 161                    &value_,
 162                    std::bind(&fbson::FbsonWriter::writeDouble, _1, d));
 163 }
 164
 165 JSONDocument::JSONDocument(int8_t i) {
 166   InitJSONDocument(&data_,
 167                    &value_,
 168                    std::bind(&fbson::FbsonWriter::writeInt8, _1, i));
 169 }
 170
 171 JSONDocument::JSONDocument(int16_t i) {
 172   InitJSONDocument(&data_,
 173                    &value_,
 174                    std::bind(&fbson::FbsonWriter::writeInt16, _1, i));
 175 }
 176
 177 JSONDocument::JSONDocument(int32_t i) {
 178   InitJSONDocument(&data_,
 179                    &value_,
 180                    std::bind(&fbson::FbsonWriter::writeInt32, _1, i));
 181 }
 182
 183 JSONDocument::JSONDocument(int64_t i) {
 184   InitJSONDocument(&data_,
 185                    &value_,
 186                    std::bind(&fbson::FbsonWriter::writeInt64, _1, i));
 187 }
 188
 189 JSONDocument::JSONDocument(const std::string& s) {
 190   InitString(&data_, &value_, s);
 191 }
 192
 193 JSONDocument::JSONDocument(const char* s) : JSONDocument(std::string(s)) {
 194 }
 195
 196 void JSONDocument::InitFromValue(const fbson::FbsonValue* val) {
 197   data_.reset(new char[val->numPackedBytes()]);
 198   memcpy(data_.get(), val, val->numPackedBytes());
 199   value_ = reinterpret_cast<fbson::FbsonValue*>(data_.get());
 200 }
 201
 202 // Private constructor
 203 JSONDocument::JSONDocument(fbson::FbsonValue* val, bool makeCopy) {
 204   if (makeCopy) {
 205     InitFromValue(val);
 206   } else {
 207     value_ = val;
 208   }
 209 }
 210
 211 JSONDocument::JSONDocument(Type _type) {
 212   // TODO(icanadi) make all of this better by using templates
 213   switch (_type) {
 214     case kNull:
 215       InitJSONDocument(&data_, &value_,
 216                        std::bind(&fbson::FbsonWriter::writeNull, _1));
 217       break;
 218     case kObject:
 219       CreateObject(&data_, &value_);
 220       break;
 221     case kBool:
 222       InitJSONDocument(&data_, &value_,
 223                        std::bind(&fbson::FbsonWriter::writeBool, _1, false));
 224       break;
 225     case kDouble:
 226       InitJSONDocument(&data_, &value_,
 227                        std::bind(&fbson::FbsonWriter::writeDouble, _1, 0.));
 228       break;
 229     case kArray:
 230       CreateArray(&data_, &value_);
 231       break;
 232     case kInt64:
 233       InitJSONDocument(&data_, &value_,
 234                        std::bind(&fbson::FbsonWriter::writeInt64, _1, 0));
 235       break;
 236     case kString:
 237       InitString(&data_, &value_, "");
 238       break;
 239     default:
 240       assert(false);
 241   }
 242 }
 243
 244 JSONDocument::JSONDocument(const JSONDocument& jsonDocument) {
 245   if (jsonDocument.IsOwner()) {
 246     InitFromValue(jsonDocument.value_);
 247   } else {
 248     value_ = jsonDocument.value_;
 249   }
 250 }
 251
 252 JSONDocument::JSONDocument(JSONDocument&& jsonDocument) {
 253   value_ = jsonDocument.value_;
 254   data_.swap(jsonDocument.data_);
 255 }
 256
 257 JSONDocument& JSONDocument::operator=(JSONDocument jsonDocument) {
 258   value_ = jsonDocument.value_;
 259   data_.swap(jsonDocument.data_);
 260   return *this;
 261 }
 262
 263 JSONDocument::Type JSONDocument::type() const {
 264   switch (value_->type()) {
 265     case fbson::FbsonType::T_Null:
 266       return JSONDocument::kNull;
 267
 268     case fbson::FbsonType::T_True:
 269     case fbson::FbsonType::T_False:
 270       return JSONDocument::kBool;
 271
 272     case fbson::FbsonType::T_Int8:
 273     case fbson::FbsonType::T_Int16:
 274     case fbson::FbsonType::T_Int32:
 275     case fbson::FbsonType::T_Int64:
 276       return JSONDocument::kInt64;
 277
 278     case fbson::FbsonType::T_Double:
 279       return JSONDocument::kDouble;
 280
 281     case fbson::FbsonType::T_String:
 282       return JSONDocument::kString;
 283
 284     case fbson::FbsonType::T_Object:
 285       return JSONDocument::kObject;
 286
 287     case fbson::FbsonType::T_Array:
 288       return JSONDocument::kArray;
 289
 290     case fbson::FbsonType::T_Binary:
 291     default:
 292       assert(false);
 293   }
 294   return JSONDocument::kNull;
 295 }
 296
 297 bool JSONDocument::Contains(const std::string& key) const {
 298   assert(IsObject());
 299   auto objectVal = reinterpret_cast<fbson::ObjectVal*>(value_);
 300   return objectVal->find(key.c_str()) != nullptr;
 301 }
 302
 303 JSONDocument JSONDocument::operator[](const std::string& key) const {
 304   assert(IsObject());
 305   auto objectVal = reinterpret_cast<fbson::ObjectVal*>(value_);
 306   auto foundValue = objectVal->find(key.c_str());
 307   assert(foundValue != nullptr);
 308   // No need to save paths in const objects
 309   JSONDocument ans(foundValue, false);
 310   return ans;
 311 }
 312
 313 size_t JSONDocument::Count() const {
 314   assert(IsObject() || IsArray());
 315   if (IsObject()) {
 316     // TODO(stash): add to fbson?
 317     const fbson::ObjectVal& objectVal =
 318           *reinterpret_cast<fbson::ObjectVal*>(value_);
 319     return ObjectNumElem(objectVal);
 320   } else if (IsArray()) {
 321     auto arrayVal = reinterpret_cast<fbson::ArrayVal*>(value_);
 322     return arrayVal->numElem();
 323   }
 324   assert(false);
 325   return 0;
 326 }
 327
 328 JSONDocument JSONDocument::operator[](size_t i) const {
 329   assert(IsArray());
 330   auto arrayVal = reinterpret_cast<fbson::ArrayVal*>(value_);
 331   auto foundValue = arrayVal->get(static_cast<int>(i));
 332   JSONDocument ans(foundValue, false);
 333   return ans;
 334 }
 335
 336 bool JSONDocument::IsNull() const {
 337   return value_->isNull();
 338 }
 339
 340 bool JSONDocument::IsArray() const {
 341   return value_->isArray();
 342 }
 343
 344 bool JSONDocument::IsBool() const {
 345   return value_->isTrue() || value_->isFalse();
 346 }
 347
 348 bool JSONDocument::IsDouble() const {
 349   return value_->isDouble();
 350 }
 351
 352 bool JSONDocument::IsInt64() const {
 353   return value_->isInt8() || value_->isInt16() ||
 354          value_->isInt32() || value_->isInt64();
 355 }
 356
 357 bool JSONDocument::IsObject() const {
 358   return value_->isObject();
 359 }
 360
 361 bool JSONDocument::IsString() const {
 362   return value_->isString();
 363 }
 364
 365 bool JSONDocument::GetBool() const {
 366   assert(IsBool());
 367   return value_->isTrue();
 368 }
 369
 370 double JSONDocument::GetDouble() const {
 371   assert(IsDouble());
 372   return ((fbson::DoubleVal*)value_)->val();
 373 }
 374
 375 int64_t JSONDocument::GetInt64() const {
 376   assert(IsInt64());
 377   return GetInt64ValFromFbsonNumericType(value_);
 378 }
 379
 380 std::string JSONDocument::GetString() const {
 381   assert(IsString());
 382   fbson::StringVal* stringVal = (fbson::StringVal*)value_;
 383   return std::string(stringVal->getBlob(), stringVal->getBlobLen());
 384 }
 385
 386 namespace {
 387
 388 // FbsonValue can be int8, int16, int32, int64
 389 bool CompareNumeric(fbson::FbsonValue* left, fbson::FbsonValue* right) {
 390   assert(IsNumeric(left) && IsNumeric(right));
 391   return GetInt64ValFromFbsonNumericType(left) ==
 392          GetInt64ValFromFbsonNumericType(right);
 393 }
 394
 395 bool CompareSimpleTypes(fbson::FbsonValue* left, fbson::FbsonValue* right) {
 396   if (IsNumeric(left)) {
 397     return CompareNumeric(left, right);
 398   }
 399   if (left->numPackedBytes() != right->numPackedBytes()) {
 400     return false;
 401   }
 402   return memcmp(left, right, left->numPackedBytes()) == 0;
 403 }
 404
 405 bool CompareFbsonValue(fbson::FbsonValue* left, fbson::FbsonValue* right) {
 406   if (!IsComparable(left, right)) {
 407     return false;
 408   }
 409
 410   switch (left->type()) {
 411     case fbson::FbsonType::T_True:
 412     case fbson::FbsonType::T_False:
 413     case fbson::FbsonType::T_Null:
 414       return true;
 415     case fbson::FbsonType::T_Int8:
 416     case fbson::FbsonType::T_Int16:
 417     case fbson::FbsonType::T_Int32:
 418     case fbson::FbsonType::T_Int64:
 419       return CompareNumeric(left, right);
 420     case fbson::FbsonType::T_String:
 421     case fbson::FbsonType::T_Double:
 422       return CompareSimpleTypes(left, right);
 423     case fbson::FbsonType::T_Object:
 424     {
 425       auto leftObject = reinterpret_cast<fbson::ObjectVal*>(left);
 426       auto rightObject = reinterpret_cast<fbson::ObjectVal*>(right);
 427       if (ObjectNumElem(*leftObject) != ObjectNumElem(*rightObject)) {
 428         return false;
 429       }
 430       for (auto && keyValue : *leftObject) {
 431         std::string str(keyValue.getKeyStr(), keyValue.klen());
 432         if (rightObject->find(str.c_str()) == nullptr) {
 433           return false;
 434         }
 435         if (!CompareFbsonValue(keyValue.value(),
 436                                rightObject->find(str.c_str()))) {
 437           return false;
 438         }
 439       }
 440       return true;
 441     }
 442     case fbson::FbsonType::T_Array:
 443     {
 444       auto leftArr = reinterpret_cast<fbson::ArrayVal*>(left);
 445       auto rightArr = reinterpret_cast<fbson::ArrayVal*>(right);
 446       if (leftArr->numElem() != rightArr->numElem()) {
 447         return false;
 448       }
 449       for (int i = 0; i < static_cast<int>(leftArr->numElem()); ++i) {
 450         if (!CompareFbsonValue(leftArr->get(i), rightArr->get(i))) {
 451           return false;
 452         }
 453       }
 454       return true;
 455     }
 456     default:
 457       assert(false);
 458   }
 459   return false;
 460 }
 461
 462 }  // namespace
 463
 464 bool JSONDocument::operator==(const JSONDocument& rhs) const {
 465   return CompareFbsonValue(value_, rhs.value_);
 466 }
 467
 468 bool JSONDocument::operator!=(const JSONDocument& rhs) const {
 469   return !(*this == rhs);
 470 }
 471
 472 JSONDocument JSONDocument::Copy() const {
 473   return JSONDocument(value_, true);
 474 }
 475
 476 bool JSONDocument::IsOwner() const {
 477   return data_.get() != nullptr;
 478 }
 479
 480 std::string JSONDocument::DebugString() const {
 481   fbson::FbsonToJson fbsonToJson;
 482   return fbsonToJson.json(value_);
 483 }
 484
 485 JSONDocument::ItemsIteratorGenerator JSONDocument::Items() const {
 486   assert(IsObject());
 487   return ItemsIteratorGenerator(*(reinterpret_cast<fbson::ObjectVal*>(value_)));
 488 }
 489
 490 // TODO(icanadi) (perf) allocate objects with arena
 491 JSONDocument* JSONDocument::ParseJSON(const char* json) {
 492   fbson::FbsonJsonParser parser;
 493   if (!parser.parse(json)) {
 494     return nullptr;
 495   }
 496
 497   auto fbsonVal = fbson::FbsonDocument::createValue(
 498                     parser.getWriter().getOutput()->getBuffer(),
 499               static_cast<uint32_t>(parser.getWriter().getOutput()->getSize()));
 500
 501   if (fbsonVal == nullptr) {
 502     return nullptr;
 503   }
 504
 505   return new JSONDocument(fbsonVal, true);
 506 }
 507
 508 void JSONDocument::Serialize(std::string* dst) const {
 509   // first byte is reserved for header
 510   // currently, header is only version number. that will help us provide
 511   // backwards compatility. we might also store more information here if
 512   // necessary
 513   dst->push_back(kSerializationFormatVersion);
 514   dst->push_back(FBSON_VER);
 515   dst->append(reinterpret_cast<char*>(value_), value_->numPackedBytes());
 516 }
 517
 518 const char JSONDocument::kSerializationFormatVersion = 2;
 519
 520 JSONDocument* JSONDocument::Deserialize(const Slice& src) {
 521   Slice input(src);
 522   if (src.size() == 0) {
 523     return nullptr;
 524   }
 525   char header = input[0];
 526   if (header == 1) {
 527     assert(false);
 528   }
 529   input.remove_prefix(1);
 530   auto value = fbson::FbsonDocument::createValue(input.data(),
 531                 static_cast<uint32_t>(input.size()));
 532   if (value == nullptr) {
 533     return nullptr;
 534   }
 535
 536   return new JSONDocument(value, true);
 537 }
 538
 539 class JSONDocument::const_item_iterator::Impl {
 540  public:
 541   typedef fbson::ObjectVal::const_iterator It;
 542
 543   explicit Impl(It it) : it_(it) {}
 544
 545   const char* getKeyStr() const {
 546     return it_->getKeyStr();
 547   }
 548
 549   uint8_t klen() const {
 550     return it_->klen();
 551   }
 552
 553   It& operator++() {
 554     return ++it_;
 555   }
 556
 557   bool operator!=(const Impl& other) {
 558     return it_ != other.it_;
 559   }
 560
 561   fbson::FbsonValue* value() const {
 562     return it_->value();
 563   }
 564
 565  private:
 566   It it_;
 567 };
 568
 569 JSONDocument::const_item_iterator::const_item_iterator(Impl* impl)
 570 : it_(impl) {}
 571
 572 JSONDocument::const_item_iterator::const_item_iterator(const_item_iterator&& a)
 573 : it_(std::move(a.it_)) {}
 574
 575 JSONDocument::const_item_iterator&
 576   JSONDocument::const_item_iterator::operator++() {
 577   ++(*it_);
 578   return *this;
 579 }
 580
 581 bool JSONDocument::const_item_iterator::operator!=(
 582                                   const const_item_iterator& other) {
 583   return *it_ != *(other.it_);
 584 }
 585
 586 JSONDocument::const_item_iterator::~const_item_iterator() {
 587 }
 588
 589 JSONDocument::const_item_iterator::value_type
 590   JSONDocument::const_item_iterator::operator*() {
 591   return JSONDocument::const_item_iterator::value_type(std::string(it_->getKeyStr(), it_->klen()),
 592     JSONDocument(it_->value(), false));
 593 }
 594
 595 JSONDocument::ItemsIteratorGenerator::ItemsIteratorGenerator(
 596                                       const fbson::ObjectVal& object)
 597   : object_(object) {}
 598
 599 JSONDocument::const_item_iterator
 600       JSONDocument::ItemsIteratorGenerator::begin() const {
 601   return const_item_iterator(new const_item_iterator::Impl(object_.begin()));
 602 }
 603
 604 JSONDocument::const_item_iterator
 605       JSONDocument::ItemsIteratorGenerator::end() const {
 606   return const_item_iterator(new const_item_iterator::Impl(object_.end()));
 607 }
 608
 609 }  // namespace rocksdb
 610 #endif  // ROCKSDB_LITE