]> git.proxmox.com Git - ceph.git/blame - ceph/src/rocksdb/db/range_del_aggregator_bench.cc
bump version to 18.2.2-pve1
[ceph.git] / ceph / src / rocksdb / db / range_del_aggregator_bench.cc
CommitLineData
11fdf7f2
TL
1// Copyright (c) 2018-present, Facebook, Inc. All rights reserved.
2// This source code is licensed under both the GPLv2 (found in the
3// COPYING file in the root directory) and Apache 2.0 License
4// (found in the LICENSE.Apache file in the root directory).
5
6#ifndef GFLAGS
7#include <cstdio>
8int main() {
9 fprintf(stderr, "Please install gflags to run rocksdb tools\n");
10 return 1;
11}
12#else
13
11fdf7f2 14#include <iomanip>
1e59de90 15#include <iostream>
11fdf7f2
TL
16#include <memory>
17#include <random>
18#include <set>
19#include <string>
20#include <vector>
21
1e59de90 22#include "db/dbformat.h"
11fdf7f2 23#include "db/range_del_aggregator.h"
494da23a 24#include "db/range_tombstone_fragmenter.h"
11fdf7f2 25#include "rocksdb/comparator.h"
1e59de90 26#include "rocksdb/system_clock.h"
11fdf7f2 27#include "util/coding.h"
1e59de90 28#include "util/gflags_compat.h"
11fdf7f2
TL
29#include "util/random.h"
30#include "util/stop_watch.h"
1e59de90 31#include "util/vector_iterator.h"
11fdf7f2
TL
32
33using GFLAGS_NAMESPACE::ParseCommandLineFlags;
34
35DEFINE_int32(num_range_tombstones, 1000, "number of range tombstones created");
36
494da23a 37DEFINE_int32(num_runs, 1000, "number of test runs");
11fdf7f2
TL
38
39DEFINE_int32(tombstone_start_upper_bound, 1000,
40 "exclusive upper bound on range tombstone start keys");
41
42DEFINE_int32(should_delete_upper_bound, 1000,
43 "exclusive upper bound on keys passed to ShouldDelete");
44
45DEFINE_double(tombstone_width_mean, 100.0, "average range tombstone width");
46
47DEFINE_double(tombstone_width_stddev, 0.0,
48 "standard deviation of range tombstone width");
49
11fdf7f2
TL
50DEFINE_int32(seed, 0, "random number generator seed");
51
52DEFINE_int32(should_deletes_per_run, 1, "number of ShouldDelete calls per run");
53
54DEFINE_int32(add_tombstones_per_run, 1,
55 "number of AddTombstones calls per run");
56
1e59de90
TL
57DEFINE_bool(use_compaction_range_del_aggregator, false,
58 "Whether to use CompactionRangeDelAggregator. Default is to use "
59 "ReadRangeDelAggregator.");
60
11fdf7f2
TL
61namespace {
62
63struct Stats {
64 uint64_t time_add_tombstones = 0;
65 uint64_t time_first_should_delete = 0;
66 uint64_t time_rest_should_delete = 0;
1e59de90 67 uint64_t time_fragment_tombstones = 0;
11fdf7f2
TL
68};
69
70std::ostream& operator<<(std::ostream& os, const Stats& s) {
71 std::ios fmt_holder(nullptr);
72 fmt_holder.copyfmt(os);
73
74 os << std::left;
1e59de90
TL
75 os << std::setw(25) << "Fragment Tombstones: "
76 << s.time_fragment_tombstones /
77 (FLAGS_add_tombstones_per_run * FLAGS_num_runs * 1.0e3)
78 << " us\n";
11fdf7f2
TL
79 os << std::setw(25) << "AddTombstones: "
80 << s.time_add_tombstones /
81 (FLAGS_add_tombstones_per_run * FLAGS_num_runs * 1.0e3)
82 << " us\n";
83 os << std::setw(25) << "ShouldDelete (first): "
84 << s.time_first_should_delete / (FLAGS_num_runs * 1.0e3) << " us\n";
85 if (FLAGS_should_deletes_per_run > 1) {
86 os << std::setw(25) << "ShouldDelete (rest): "
87 << s.time_rest_should_delete /
88 ((FLAGS_should_deletes_per_run - 1) * FLAGS_num_runs * 1.0e3)
89 << " us\n";
90 }
91
92 os.copyfmt(fmt_holder);
93 return os;
94}
95
f67539c2
TL
96auto icmp = ROCKSDB_NAMESPACE::InternalKeyComparator(
97 ROCKSDB_NAMESPACE::BytewiseComparator());
494da23a 98
11fdf7f2
TL
99} // anonymous namespace
100
f67539c2 101namespace ROCKSDB_NAMESPACE {
11fdf7f2
TL
102
103namespace {
104
105// A wrapper around RangeTombstones and the underlying data of its start and end
106// keys.
107struct PersistentRangeTombstone {
108 std::string start_key;
109 std::string end_key;
110 RangeTombstone tombstone;
111
112 PersistentRangeTombstone(std::string start, std::string end,
113 SequenceNumber seq)
114 : start_key(std::move(start)), end_key(std::move(end)) {
115 tombstone = RangeTombstone(start_key, end_key, seq);
116 }
117
118 PersistentRangeTombstone() = default;
119
120 PersistentRangeTombstone(const PersistentRangeTombstone& t) { *this = t; }
121
122 PersistentRangeTombstone& operator=(const PersistentRangeTombstone& t) {
123 start_key = t.start_key;
124 end_key = t.end_key;
125 tombstone = RangeTombstone(start_key, end_key, t.tombstone.seq_);
126
127 return *this;
128 }
129
130 PersistentRangeTombstone(PersistentRangeTombstone&& t) noexcept { *this = t; }
131
132 PersistentRangeTombstone& operator=(PersistentRangeTombstone&& t) {
133 start_key = std::move(t.start_key);
134 end_key = std::move(t.end_key);
135 tombstone = RangeTombstone(start_key, end_key, t.tombstone.seq_);
136
137 return *this;
138 }
139};
140
141struct TombstoneStartKeyComparator {
142 explicit TombstoneStartKeyComparator(const Comparator* c) : cmp(c) {}
143
144 bool operator()(const RangeTombstone& a, const RangeTombstone& b) const {
145 return cmp->Compare(a.start_key_, b.start_key_) < 0;
146 }
147
148 const Comparator* cmp;
149};
150
151std::unique_ptr<InternalIterator> MakeRangeDelIterator(
152 const std::vector<PersistentRangeTombstone>& range_dels) {
153 std::vector<std::string> keys, values;
154 for (const auto& range_del : range_dels) {
155 auto key_and_value = range_del.tombstone.Serialize();
156 keys.push_back(key_and_value.first.Encode().ToString());
157 values.push_back(key_and_value.second.ToString());
158 }
1e59de90
TL
159 return std::unique_ptr<VectorIterator>(
160 new VectorIterator(keys, values, &icmp));
11fdf7f2
TL
161}
162
163// convert long to a big-endian slice key
164static std::string Key(int64_t val) {
165 std::string little_endian_key;
166 std::string big_endian_key;
167 PutFixed64(&little_endian_key, val);
168 assert(little_endian_key.size() == sizeof(val));
169 big_endian_key.resize(sizeof(val));
170 for (size_t i = 0; i < sizeof(val); ++i) {
171 big_endian_key[i] = little_endian_key[sizeof(val) - 1 - i];
172 }
173 return big_endian_key;
174}
175
176} // anonymous namespace
177
f67539c2 178} // namespace ROCKSDB_NAMESPACE
11fdf7f2
TL
179
180int main(int argc, char** argv) {
181 ParseCommandLineFlags(&argc, &argv, true);
182
183 Stats stats;
1e59de90
TL
184 ROCKSDB_NAMESPACE::SystemClock* clock =
185 ROCKSDB_NAMESPACE::SystemClock::Default().get();
f67539c2 186 ROCKSDB_NAMESPACE::Random64 rnd(FLAGS_seed);
11fdf7f2
TL
187 std::default_random_engine random_gen(FLAGS_seed);
188 std::normal_distribution<double> normal_dist(FLAGS_tombstone_width_mean,
189 FLAGS_tombstone_width_stddev);
f67539c2 190 std::vector<std::vector<ROCKSDB_NAMESPACE::PersistentRangeTombstone> >
11fdf7f2
TL
191 all_persistent_range_tombstones(FLAGS_add_tombstones_per_run);
192 for (int i = 0; i < FLAGS_add_tombstones_per_run; i++) {
193 all_persistent_range_tombstones[i] =
f67539c2 194 std::vector<ROCKSDB_NAMESPACE::PersistentRangeTombstone>(
11fdf7f2
TL
195 FLAGS_num_range_tombstones);
196 }
f67539c2 197 auto mode = ROCKSDB_NAMESPACE::RangeDelPositioningMode::kForwardTraversal;
1e59de90 198 std::vector<ROCKSDB_NAMESPACE::SequenceNumber> snapshots{0};
11fdf7f2 199 for (int i = 0; i < FLAGS_num_runs; i++) {
1e59de90
TL
200 std::unique_ptr<ROCKSDB_NAMESPACE::RangeDelAggregator> range_del_agg =
201 nullptr;
202 if (FLAGS_use_compaction_range_del_aggregator) {
203 range_del_agg.reset(new ROCKSDB_NAMESPACE::CompactionRangeDelAggregator(
204 &icmp, snapshots));
205 } else {
206 range_del_agg.reset(new ROCKSDB_NAMESPACE::ReadRangeDelAggregator(
207 &icmp, ROCKSDB_NAMESPACE::kMaxSequenceNumber /* upper_bound */));
208 }
494da23a 209
f67539c2
TL
210 std::vector<
211 std::unique_ptr<ROCKSDB_NAMESPACE::FragmentedRangeTombstoneList> >
494da23a 212 fragmented_range_tombstone_lists(FLAGS_add_tombstones_per_run);
11fdf7f2
TL
213
214 for (auto& persistent_range_tombstones : all_persistent_range_tombstones) {
215 // TODO(abhimadan): consider whether creating the range tombstones right
216 // before AddTombstones is artificially warming the cache compared to
217 // real workloads.
218 for (int j = 0; j < FLAGS_num_range_tombstones; j++) {
219 uint64_t start = rnd.Uniform(FLAGS_tombstone_start_upper_bound);
494da23a
TL
220 uint64_t end = static_cast<uint64_t>(
221 std::round(start + std::max(1.0, normal_dist(random_gen))));
f67539c2
TL
222 persistent_range_tombstones[j] =
223 ROCKSDB_NAMESPACE::PersistentRangeTombstone(
224 ROCKSDB_NAMESPACE::Key(start), ROCKSDB_NAMESPACE::Key(end), j);
11fdf7f2 225 }
1e59de90 226 auto iter =
f67539c2 227 ROCKSDB_NAMESPACE::MakeRangeDelIterator(persistent_range_tombstones);
1e59de90
TL
228 ROCKSDB_NAMESPACE::StopWatchNano stop_watch_fragment_tombstones(
229 clock, true /* auto_start */);
494da23a 230 fragmented_range_tombstone_lists.emplace_back(
f67539c2 231 new ROCKSDB_NAMESPACE::FragmentedRangeTombstoneList(
1e59de90
TL
232 std::move(iter), icmp, FLAGS_use_compaction_range_del_aggregator,
233 snapshots));
234 stats.time_fragment_tombstones +=
235 stop_watch_fragment_tombstones.ElapsedNanos();
f67539c2 236 std::unique_ptr<ROCKSDB_NAMESPACE::FragmentedRangeTombstoneIterator>
494da23a 237 fragmented_range_del_iter(
f67539c2 238 new ROCKSDB_NAMESPACE::FragmentedRangeTombstoneIterator(
494da23a 239 fragmented_range_tombstone_lists.back().get(), icmp,
f67539c2 240 ROCKSDB_NAMESPACE::kMaxSequenceNumber));
494da23a 241
f67539c2 242 ROCKSDB_NAMESPACE::StopWatchNano stop_watch_add_tombstones(
1e59de90
TL
243 clock, true /* auto_start */);
244 range_del_agg->AddTombstones(std::move(fragmented_range_del_iter));
11fdf7f2
TL
245 stats.time_add_tombstones += stop_watch_add_tombstones.ElapsedNanos();
246 }
247
f67539c2 248 ROCKSDB_NAMESPACE::ParsedInternalKey parsed_key;
11fdf7f2 249 parsed_key.sequence = FLAGS_num_range_tombstones / 2;
f67539c2 250 parsed_key.type = ROCKSDB_NAMESPACE::kTypeValue;
11fdf7f2
TL
251
252 uint64_t first_key = rnd.Uniform(FLAGS_should_delete_upper_bound -
253 FLAGS_should_deletes_per_run + 1);
254
255 for (int j = 0; j < FLAGS_should_deletes_per_run; j++) {
f67539c2 256 std::string key_string = ROCKSDB_NAMESPACE::Key(first_key + j);
11fdf7f2
TL
257 parsed_key.user_key = key_string;
258
f67539c2 259 ROCKSDB_NAMESPACE::StopWatchNano stop_watch_should_delete(
1e59de90
TL
260 clock, true /* auto_start */);
261 range_del_agg->ShouldDelete(parsed_key, mode);
11fdf7f2
TL
262 uint64_t call_time = stop_watch_should_delete.ElapsedNanos();
263
264 if (j == 0) {
265 stats.time_first_should_delete += call_time;
266 } else {
267 stats.time_rest_should_delete += call_time;
268 }
269 }
270 }
271
272 std::cout << "=========================\n"
273 << "Results:\n"
274 << "=========================\n"
275 << stats;
276
277 return 0;
278}
279
280#endif // GFLAGS