ceph/src/rocksdb/fuzz/sst_file_writer_fuzzer.cc

   1 // Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
   2 // This source code is licensed under both the GPLv2 (found in the
   3 // COPYING file in the root directory) and Apache 2.0 License
   4 // (found in the LICENSE.Apache file in the root directory).
   5
   6 #include <algorithm>
   7 #include <iostream>
   8 #include <memory>
   9 #include <string>
  10
  11 #include "proto/gen/db_operation.pb.h"
  12 #include "rocksdb/file_system.h"
  13 #include "rocksdb/sst_file_writer.h"
  14 #include "src/libfuzzer/libfuzzer_macro.h"
  15 #include "table/table_builder.h"
  16 #include "table/table_reader.h"
  17 #include "util.h"
  18
  19 using ROCKSDB_NAMESPACE::BytewiseComparator;
  20 using ROCKSDB_NAMESPACE::Comparator;
  21 using ROCKSDB_NAMESPACE::EnvOptions;
  22 using ROCKSDB_NAMESPACE::ExternalSstFileInfo;
  23 using ROCKSDB_NAMESPACE::FileOptions;
  24 using ROCKSDB_NAMESPACE::FileSystem;
  25 using ROCKSDB_NAMESPACE::ImmutableCFOptions;
  26 using ROCKSDB_NAMESPACE::ImmutableOptions;
  27 using ROCKSDB_NAMESPACE::InternalIterator;
  28 using ROCKSDB_NAMESPACE::IOOptions;
  29 using ROCKSDB_NAMESPACE::kMaxSequenceNumber;
  30 using ROCKSDB_NAMESPACE::Options;
  31 using ROCKSDB_NAMESPACE::ParsedInternalKey;
  32 using ROCKSDB_NAMESPACE::ParseInternalKey;
  33 using ROCKSDB_NAMESPACE::RandomAccessFileReader;
  34 using ROCKSDB_NAMESPACE::ReadOptions;
  35 using ROCKSDB_NAMESPACE::SstFileWriter;
  36 using ROCKSDB_NAMESPACE::Status;
  37 using ROCKSDB_NAMESPACE::TableReader;
  38 using ROCKSDB_NAMESPACE::TableReaderCaller;
  39 using ROCKSDB_NAMESPACE::TableReaderOptions;
  40 using ROCKSDB_NAMESPACE::ValueType;
  41
  42 // Keys in SST file writer operations must be unique and in ascending order.
  43 // For each DBOperation generated by the fuzzer, this function is called on
  44 // it to deduplicate and sort the keys in the DBOperations.
  45 protobuf_mutator::libfuzzer::PostProcessorRegistration<DBOperations> reg = {
  46     [](DBOperations* input, unsigned int /* seed */) {
  47       const Comparator* comparator = BytewiseComparator();
  48       auto ops = input->mutable_operations();
  49
  50       // Make sure begin <= end for DELETE_RANGE.
  51       for (DBOperation& op : *ops) {
  52         if (op.type() == OpType::DELETE_RANGE) {
  53           auto begin = op.key();
  54           auto end = op.value();
  55           if (comparator->Compare(begin, end) > 0) {
  56             std::swap(begin, end);
  57             op.set_key(begin);
  58             op.set_value(end);
  59           }
  60         }
  61       }
  62
  63       std::sort(ops->begin(), ops->end(),
  64                 [&comparator](const DBOperation& a, const DBOperation& b) {
  65                   return comparator->Compare(a.key(), b.key()) < 0;
  66                 });
  67
  68       auto last = std::unique(
  69           ops->begin(), ops->end(),
  70           [&comparator](const DBOperation& a, const DBOperation& b) {
  71             return comparator->Compare(a.key(), b.key()) == 0;
  72           });
  73       ops->erase(last, ops->end());
  74     }};
  75
  76 TableReader* NewTableReader(const std::string& sst_file_path,
  77                             const Options& options,
  78                             const EnvOptions& env_options,
  79                             const ImmutableCFOptions& cf_ioptions) {
  80   // This code block is similar to SstFileReader::Open.
  81
  82   uint64_t file_size = 0;
  83   std::unique_ptr<RandomAccessFileReader> file_reader;
  84   std::unique_ptr<TableReader> table_reader;
  85   const auto& fs = options.env->GetFileSystem();
  86   FileOptions fopts(env_options);
  87   Status s = options.env->GetFileSize(sst_file_path, &file_size);
  88   if (s.ok()) {
  89     s = RandomAccessFileReader::Create(fs, sst_file_path, fopts, &file_reader,
  90                                        nullptr);
  91   }
  92   if (s.ok()) {
  93     ImmutableOptions iopts(options, cf_ioptions);
  94     TableReaderOptions t_opt(iopts, /*prefix_extractor=*/nullptr, env_options,
  95                              cf_ioptions.internal_comparator);
  96     t_opt.largest_seqno = kMaxSequenceNumber;
  97     s = options.table_factory->NewTableReader(t_opt, std::move(file_reader),
  98                                               file_size, &table_reader,
  99                                               /*prefetch=*/false);
 100   }
 101   if (!s.ok()) {
 102     std::cerr << "Failed to create TableReader for " << sst_file_path << ": "
 103               << s.ToString() << std::endl;
 104     abort();
 105   }
 106   return table_reader.release();
 107 }
 108
 109 ValueType ToValueType(OpType op_type) {
 110   switch (op_type) {
 111     case OpType::PUT:
 112       return ValueType::kTypeValue;
 113     case OpType::MERGE:
 114       return ValueType::kTypeMerge;
 115     case OpType::DELETE:
 116       return ValueType::kTypeDeletion;
 117     case OpType::DELETE_RANGE:
 118       return ValueType::kTypeRangeDeletion;
 119     default:
 120       std::cerr << "Unknown operation type " << static_cast<int>(op_type)
 121                 << std::endl;
 122       abort();
 123   }
 124 }
 125
 126 // Fuzzes DB operations as input, let SstFileWriter generate a SST file
 127 // according to the operations, then let TableReader read and check all the
 128 // key-value pairs from the generated SST file.
 129 DEFINE_PROTO_FUZZER(DBOperations& input) {
 130   if (input.operations().empty()) {
 131     return;
 132   }
 133
 134   std::string sstfile;
 135   {
 136     auto fs = FileSystem::Default();
 137     std::string dir;
 138     IOOptions opt;
 139     CHECK_OK(fs->GetTestDirectory(opt, &dir, nullptr));
 140     sstfile = dir + "/SstFileWriterFuzzer.sst";
 141   }
 142
 143   Options options;
 144   EnvOptions env_options(options);
 145   ImmutableCFOptions cf_ioptions(options);
 146
 147   // Generate sst file.
 148   SstFileWriter writer(env_options, options);
 149   CHECK_OK(writer.Open(sstfile));
 150   for (const DBOperation& op : input.operations()) {
 151     switch (op.type()) {
 152       case OpType::PUT: {
 153         CHECK_OK(writer.Put(op.key(), op.value()));
 154         break;
 155       }
 156       case OpType::MERGE: {
 157         CHECK_OK(writer.Merge(op.key(), op.value()));
 158         break;
 159       }
 160       case OpType::DELETE: {
 161         CHECK_OK(writer.Delete(op.key()));
 162         break;
 163       }
 164       case OpType::DELETE_RANGE: {
 165         CHECK_OK(writer.DeleteRange(op.key(), op.value()));
 166         break;
 167       }
 168       default: {
 169         std::cerr << "Unsupported operation" << static_cast<int>(op.type())
 170                   << std::endl;
 171         abort();
 172       }
 173     }
 174   }
 175   ExternalSstFileInfo info;
 176   CHECK_OK(writer.Finish(&info));
 177
 178   // Iterate and verify key-value pairs.
 179   std::unique_ptr<TableReader> table_reader(
 180       ::NewTableReader(sstfile, options, env_options, cf_ioptions));
 181   ReadOptions roptions;
 182   CHECK_OK(table_reader->VerifyChecksum(roptions,
 183                                         TableReaderCaller::kUncategorized));
 184   std::unique_ptr<InternalIterator> it(
 185       table_reader->NewIterator(roptions, /*prefix_extractor=*/nullptr,
 186                                 /*arena=*/nullptr, /*skip_filters=*/true,
 187                                 TableReaderCaller::kUncategorized));
 188   it->SeekToFirst();
 189   for (const DBOperation& op : input.operations()) {
 190     if (op.type() == OpType::DELETE_RANGE) {
 191       // InternalIterator cannot iterate over DELETE_RANGE entries.
 192       continue;
 193     }
 194     CHECK_TRUE(it->Valid());
 195     ParsedInternalKey ikey;
 196     CHECK_OK(ParseInternalKey(it->key(), &ikey, /*log_err_key=*/true));
 197     CHECK_EQ(ikey.user_key.ToString(), op.key());
 198     CHECK_EQ(ikey.sequence, 0);
 199     CHECK_EQ(ikey.type, ToValueType(op.type()));
 200     if (op.type() != OpType::DELETE) {
 201       CHECK_EQ(op.value(), it->value().ToString());
 202     }
 203     it->Next();
 204   }
 205   CHECK_TRUE(!it->Valid());
 206
 207   // Delete sst file.
 208   remove(sstfile.c_str());
 209 }