]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | // Copyright (c) 2018-present, Facebook, Inc. All rights reserved. |
2 | // This source code is licensed under both the GPLv2 (found in the | |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
5 | ||
6 | #ifndef GFLAGS | |
7 | #include <cstdio> | |
8 | int main() { | |
9 | fprintf(stderr, "Please install gflags to run rocksdb tools\n"); | |
10 | return 1; | |
11 | } | |
12 | #else | |
13 | ||
14 | #include <iostream> | |
15 | #include <iomanip> | |
16 | #include <memory> | |
17 | #include <random> | |
18 | #include <set> | |
19 | #include <string> | |
20 | #include <vector> | |
21 | ||
22 | #include "db/range_del_aggregator.h" | |
23 | #include "rocksdb/comparator.h" | |
24 | #include "rocksdb/env.h" | |
25 | #include "util/coding.h" | |
26 | #include "util/random.h" | |
27 | #include "util/stop_watch.h" | |
28 | #include "util/testutil.h" | |
29 | ||
30 | #include "util/gflags_compat.h" | |
31 | ||
32 | using GFLAGS_NAMESPACE::ParseCommandLineFlags; | |
33 | ||
34 | DEFINE_int32(num_range_tombstones, 1000, "number of range tombstones created"); | |
35 | ||
36 | DEFINE_int32(num_runs, 10000, "number of test runs"); | |
37 | ||
38 | DEFINE_int32(tombstone_start_upper_bound, 1000, | |
39 | "exclusive upper bound on range tombstone start keys"); | |
40 | ||
41 | DEFINE_int32(should_delete_upper_bound, 1000, | |
42 | "exclusive upper bound on keys passed to ShouldDelete"); | |
43 | ||
44 | DEFINE_double(tombstone_width_mean, 100.0, "average range tombstone width"); | |
45 | ||
46 | DEFINE_double(tombstone_width_stddev, 0.0, | |
47 | "standard deviation of range tombstone width"); | |
48 | ||
49 | DEFINE_bool(use_collapsed, true, "use the collapsed range tombstone map"); | |
50 | ||
51 | DEFINE_int32(seed, 0, "random number generator seed"); | |
52 | ||
53 | DEFINE_int32(should_deletes_per_run, 1, "number of ShouldDelete calls per run"); | |
54 | ||
55 | DEFINE_int32(add_tombstones_per_run, 1, | |
56 | "number of AddTombstones calls per run"); | |
57 | ||
58 | namespace { | |
59 | ||
60 | struct Stats { | |
61 | uint64_t time_add_tombstones = 0; | |
62 | uint64_t time_first_should_delete = 0; | |
63 | uint64_t time_rest_should_delete = 0; | |
64 | }; | |
65 | ||
66 | std::ostream& operator<<(std::ostream& os, const Stats& s) { | |
67 | std::ios fmt_holder(nullptr); | |
68 | fmt_holder.copyfmt(os); | |
69 | ||
70 | os << std::left; | |
71 | os << std::setw(25) << "AddTombstones: " | |
72 | << s.time_add_tombstones / | |
73 | (FLAGS_add_tombstones_per_run * FLAGS_num_runs * 1.0e3) | |
74 | << " us\n"; | |
75 | os << std::setw(25) << "ShouldDelete (first): " | |
76 | << s.time_first_should_delete / (FLAGS_num_runs * 1.0e3) << " us\n"; | |
77 | if (FLAGS_should_deletes_per_run > 1) { | |
78 | os << std::setw(25) << "ShouldDelete (rest): " | |
79 | << s.time_rest_should_delete / | |
80 | ((FLAGS_should_deletes_per_run - 1) * FLAGS_num_runs * 1.0e3) | |
81 | << " us\n"; | |
82 | } | |
83 | ||
84 | os.copyfmt(fmt_holder); | |
85 | return os; | |
86 | } | |
87 | ||
88 | } // anonymous namespace | |
89 | ||
90 | namespace rocksdb { | |
91 | ||
92 | namespace { | |
93 | ||
94 | // A wrapper around RangeTombstones and the underlying data of its start and end | |
95 | // keys. | |
96 | struct PersistentRangeTombstone { | |
97 | std::string start_key; | |
98 | std::string end_key; | |
99 | RangeTombstone tombstone; | |
100 | ||
101 | PersistentRangeTombstone(std::string start, std::string end, | |
102 | SequenceNumber seq) | |
103 | : start_key(std::move(start)), end_key(std::move(end)) { | |
104 | tombstone = RangeTombstone(start_key, end_key, seq); | |
105 | } | |
106 | ||
107 | PersistentRangeTombstone() = default; | |
108 | ||
109 | PersistentRangeTombstone(const PersistentRangeTombstone& t) { *this = t; } | |
110 | ||
111 | PersistentRangeTombstone& operator=(const PersistentRangeTombstone& t) { | |
112 | start_key = t.start_key; | |
113 | end_key = t.end_key; | |
114 | tombstone = RangeTombstone(start_key, end_key, t.tombstone.seq_); | |
115 | ||
116 | return *this; | |
117 | } | |
118 | ||
119 | PersistentRangeTombstone(PersistentRangeTombstone&& t) noexcept { *this = t; } | |
120 | ||
121 | PersistentRangeTombstone& operator=(PersistentRangeTombstone&& t) { | |
122 | start_key = std::move(t.start_key); | |
123 | end_key = std::move(t.end_key); | |
124 | tombstone = RangeTombstone(start_key, end_key, t.tombstone.seq_); | |
125 | ||
126 | return *this; | |
127 | } | |
128 | }; | |
129 | ||
130 | struct TombstoneStartKeyComparator { | |
131 | explicit TombstoneStartKeyComparator(const Comparator* c) : cmp(c) {} | |
132 | ||
133 | bool operator()(const RangeTombstone& a, const RangeTombstone& b) const { | |
134 | return cmp->Compare(a.start_key_, b.start_key_) < 0; | |
135 | } | |
136 | ||
137 | const Comparator* cmp; | |
138 | }; | |
139 | ||
140 | std::unique_ptr<InternalIterator> MakeRangeDelIterator( | |
141 | const std::vector<PersistentRangeTombstone>& range_dels) { | |
142 | std::vector<std::string> keys, values; | |
143 | for (const auto& range_del : range_dels) { | |
144 | auto key_and_value = range_del.tombstone.Serialize(); | |
145 | keys.push_back(key_and_value.first.Encode().ToString()); | |
146 | values.push_back(key_and_value.second.ToString()); | |
147 | } | |
148 | return std::unique_ptr<test::VectorIterator>( | |
149 | new test::VectorIterator(keys, values)); | |
150 | } | |
151 | ||
152 | // convert long to a big-endian slice key | |
153 | static std::string Key(int64_t val) { | |
154 | std::string little_endian_key; | |
155 | std::string big_endian_key; | |
156 | PutFixed64(&little_endian_key, val); | |
157 | assert(little_endian_key.size() == sizeof(val)); | |
158 | big_endian_key.resize(sizeof(val)); | |
159 | for (size_t i = 0; i < sizeof(val); ++i) { | |
160 | big_endian_key[i] = little_endian_key[sizeof(val) - 1 - i]; | |
161 | } | |
162 | return big_endian_key; | |
163 | } | |
164 | ||
165 | } // anonymous namespace | |
166 | ||
167 | } // namespace rocksdb | |
168 | ||
169 | int main(int argc, char** argv) { | |
170 | ParseCommandLineFlags(&argc, &argv, true); | |
171 | ||
172 | Stats stats; | |
173 | rocksdb::Random64 rnd(FLAGS_seed); | |
174 | std::default_random_engine random_gen(FLAGS_seed); | |
175 | std::normal_distribution<double> normal_dist(FLAGS_tombstone_width_mean, | |
176 | FLAGS_tombstone_width_stddev); | |
177 | std::vector<std::vector<rocksdb::PersistentRangeTombstone> > | |
178 | all_persistent_range_tombstones(FLAGS_add_tombstones_per_run); | |
179 | for (int i = 0; i < FLAGS_add_tombstones_per_run; i++) { | |
180 | all_persistent_range_tombstones[i] = | |
181 | std::vector<rocksdb::PersistentRangeTombstone>( | |
182 | FLAGS_num_range_tombstones); | |
183 | } | |
184 | auto mode = FLAGS_use_collapsed | |
185 | ? rocksdb::RangeDelPositioningMode::kForwardTraversal | |
186 | : rocksdb::RangeDelPositioningMode::kFullScan; | |
187 | ||
188 | for (int i = 0; i < FLAGS_num_runs; i++) { | |
189 | auto icmp = rocksdb::InternalKeyComparator(rocksdb::BytewiseComparator()); | |
190 | rocksdb::RangeDelAggregator range_del_agg(icmp, {} /* snapshots */, | |
191 | FLAGS_use_collapsed); | |
192 | ||
193 | for (auto& persistent_range_tombstones : all_persistent_range_tombstones) { | |
194 | // TODO(abhimadan): consider whether creating the range tombstones right | |
195 | // before AddTombstones is artificially warming the cache compared to | |
196 | // real workloads. | |
197 | for (int j = 0; j < FLAGS_num_range_tombstones; j++) { | |
198 | uint64_t start = rnd.Uniform(FLAGS_tombstone_start_upper_bound); | |
199 | uint64_t end = start + std::max(1.0, normal_dist(random_gen)); | |
200 | persistent_range_tombstones[j] = rocksdb::PersistentRangeTombstone( | |
201 | rocksdb::Key(start), rocksdb::Key(end), j); | |
202 | } | |
203 | ||
204 | auto range_del_iter = | |
205 | rocksdb::MakeRangeDelIterator(persistent_range_tombstones); | |
206 | rocksdb::StopWatchNano stop_watch_add_tombstones(rocksdb::Env::Default(), | |
207 | true /* auto_start */); | |
208 | range_del_agg.AddTombstones(std::move(range_del_iter)); | |
209 | stats.time_add_tombstones += stop_watch_add_tombstones.ElapsedNanos(); | |
210 | } | |
211 | ||
212 | rocksdb::ParsedInternalKey parsed_key; | |
213 | parsed_key.sequence = FLAGS_num_range_tombstones / 2; | |
214 | parsed_key.type = rocksdb::kTypeValue; | |
215 | ||
216 | uint64_t first_key = rnd.Uniform(FLAGS_should_delete_upper_bound - | |
217 | FLAGS_should_deletes_per_run + 1); | |
218 | ||
219 | for (int j = 0; j < FLAGS_should_deletes_per_run; j++) { | |
220 | std::string key_string = rocksdb::Key(first_key + j); | |
221 | parsed_key.user_key = key_string; | |
222 | ||
223 | rocksdb::StopWatchNano stop_watch_should_delete(rocksdb::Env::Default(), | |
224 | true /* auto_start */); | |
225 | range_del_agg.ShouldDelete(parsed_key, mode); | |
226 | uint64_t call_time = stop_watch_should_delete.ElapsedNanos(); | |
227 | ||
228 | if (j == 0) { | |
229 | stats.time_first_should_delete += call_time; | |
230 | } else { | |
231 | stats.time_rest_should_delete += call_time; | |
232 | } | |
233 | } | |
234 | } | |
235 | ||
236 | std::cout << "=========================\n" | |
237 | << "Results:\n" | |
238 | << "=========================\n" | |
239 | << stats; | |
240 | ||
241 | return 0; | |
242 | } | |
243 | ||
244 | #endif // GFLAGS |