]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | |
6 | #ifndef GFLAGS | |
7 | #include <cstdio> | |
8 | int main() { | |
9 | fprintf(stderr, "Please install gflags to run rocksdb tools\n"); | |
10 | return 1; | |
11 | } | |
12 | #else | |
13 | ||
f67539c2 | 14 | #include "db/db_impl/db_impl.h" |
7c673cae | 15 | #include "db/dbformat.h" |
f67539c2 TL |
16 | #include "env/composite_env_wrapper.h" |
17 | #include "file/random_access_file_reader.h" | |
7c673cae FG |
18 | #include "monitoring/histogram.h" |
19 | #include "rocksdb/db.h" | |
20 | #include "rocksdb/slice_transform.h" | |
21 | #include "rocksdb/table.h" | |
f67539c2 | 22 | #include "table/block_based/block_based_table_factory.h" |
7c673cae FG |
23 | #include "table/get_context.h" |
24 | #include "table/internal_iterator.h" | |
f67539c2 | 25 | #include "table/plain/plain_table_factory.h" |
7c673cae | 26 | #include "table/table_builder.h" |
f67539c2 TL |
27 | #include "test_util/testharness.h" |
28 | #include "test_util/testutil.h" | |
11fdf7f2 | 29 | #include "util/gflags_compat.h" |
7c673cae | 30 | |
11fdf7f2 TL |
31 | using GFLAGS_NAMESPACE::ParseCommandLineFlags; |
32 | using GFLAGS_NAMESPACE::SetUsageMessage; | |
7c673cae | 33 | |
f67539c2 | 34 | namespace ROCKSDB_NAMESPACE { |
7c673cae FG |
35 | |
36 | namespace { | |
37 | // Make a key that i determines the first 4 characters and j determines the | |
38 | // last 4 characters. | |
39 | static std::string MakeKey(int i, int j, bool through_db) { | |
40 | char buf[100]; | |
41 | snprintf(buf, sizeof(buf), "%04d__key___%04d", i, j); | |
42 | if (through_db) { | |
43 | return std::string(buf); | |
44 | } | |
45 | // If we directly query table, which operates on internal keys | |
46 | // instead of user keys, we need to add 8 bytes of internal | |
47 | // information (row type etc) to user key to make an internal | |
48 | // key. | |
49 | InternalKey key(std::string(buf), 0, ValueType::kTypeValue); | |
50 | return key.Encode().ToString(); | |
51 | } | |
52 | ||
53 | uint64_t Now(Env* env, bool measured_by_nanosecond) { | |
54 | return measured_by_nanosecond ? env->NowNanos() : env->NowMicros(); | |
55 | } | |
56 | } // namespace | |
57 | ||
58 | // A very simple benchmark that. | |
59 | // Create a table with roughly numKey1 * numKey2 keys, | |
60 | // where there are numKey1 prefixes of the key, each has numKey2 number of | |
61 | // distinguished key, differing in the suffix part. | |
62 | // If if_query_empty_keys = false, query the existing keys numKey1 * numKey2 | |
63 | // times randomly. | |
64 | // If if_query_empty_keys = true, query numKey1 * numKey2 random empty keys. | |
65 | // Print out the total time. | |
66 | // If through_db=true, a full DB will be created and queries will be against | |
67 | // it. Otherwise, operations will be directly through table level. | |
68 | // | |
69 | // If for_terator=true, instead of just query one key each time, it queries | |
70 | // a range sharing the same prefix. | |
71 | namespace { | |
72 | void TableReaderBenchmark(Options& opts, EnvOptions& env_options, | |
73 | ReadOptions& read_options, int num_keys1, | |
11fdf7f2 | 74 | int num_keys2, int num_iter, int /*prefix_len*/, |
7c673cae FG |
75 | bool if_query_empty_keys, bool for_iterator, |
76 | bool through_db, bool measured_by_nanosecond) { | |
f67539c2 | 77 | ROCKSDB_NAMESPACE::InternalKeyComparator ikc(opts.comparator); |
7c673cae | 78 | |
11fdf7f2 TL |
79 | std::string file_name = |
80 | test::PerThreadDBPath("rocksdb_table_reader_benchmark"); | |
81 | std::string dbname = test::PerThreadDBPath("rocksdb_table_reader_bench_db"); | |
7c673cae FG |
82 | WriteOptions wo; |
83 | Env* env = Env::Default(); | |
84 | TableBuilder* tb = nullptr; | |
85 | DB* db = nullptr; | |
86 | Status s; | |
87 | const ImmutableCFOptions ioptions(opts); | |
11fdf7f2 TL |
88 | const ColumnFamilyOptions cfo(opts); |
89 | const MutableCFOptions moptions(cfo); | |
494da23a | 90 | std::unique_ptr<WritableFileWriter> file_writer; |
7c673cae | 91 | if (!through_db) { |
494da23a | 92 | std::unique_ptr<WritableFile> file; |
7c673cae FG |
93 | env->NewWritableFile(file_name, &file, env_options); |
94 | ||
95 | std::vector<std::unique_ptr<IntTblPropCollectorFactory> > | |
96 | int_tbl_prop_collector_factories; | |
97 | ||
f67539c2 TL |
98 | file_writer.reset(new WritableFileWriter( |
99 | NewLegacyWritableFileWrapper(std::move(file)), file_name, env_options)); | |
7c673cae FG |
100 | int unknown_level = -1; |
101 | tb = opts.table_factory->NewTableBuilder( | |
11fdf7f2 TL |
102 | TableBuilderOptions( |
103 | ioptions, moptions, ikc, &int_tbl_prop_collector_factories, | |
494da23a TL |
104 | CompressionType::kNoCompression, 0 /* sample_for_compression */, |
105 | CompressionOptions(), false /* skip_filters */, | |
11fdf7f2 | 106 | kDefaultColumnFamilyName, unknown_level), |
7c673cae FG |
107 | 0 /* column_family_id */, file_writer.get()); |
108 | } else { | |
109 | s = DB::Open(opts, dbname, &db); | |
110 | ASSERT_OK(s); | |
111 | ASSERT_TRUE(db != nullptr); | |
112 | } | |
113 | // Populate slightly more than 1M keys | |
114 | for (int i = 0; i < num_keys1; i++) { | |
115 | for (int j = 0; j < num_keys2; j++) { | |
116 | std::string key = MakeKey(i * 2, j, through_db); | |
117 | if (!through_db) { | |
118 | tb->Add(key, key); | |
119 | } else { | |
120 | db->Put(wo, key, key); | |
121 | } | |
122 | } | |
123 | } | |
124 | if (!through_db) { | |
125 | tb->Finish(); | |
126 | file_writer->Close(); | |
127 | } else { | |
128 | db->Flush(FlushOptions()); | |
129 | } | |
130 | ||
494da23a | 131 | std::unique_ptr<TableReader> table_reader; |
7c673cae | 132 | if (!through_db) { |
494da23a | 133 | std::unique_ptr<RandomAccessFile> raf; |
7c673cae FG |
134 | s = env->NewRandomAccessFile(file_name, &raf, env_options); |
135 | if (!s.ok()) { | |
136 | fprintf(stderr, "Create File Error: %s\n", s.ToString().c_str()); | |
137 | exit(1); | |
138 | } | |
139 | uint64_t file_size; | |
140 | env->GetFileSize(file_name, &file_size); | |
494da23a | 141 | std::unique_ptr<RandomAccessFileReader> file_reader( |
f67539c2 TL |
142 | new RandomAccessFileReader(NewLegacyRandomAccessFileWrapper(raf), |
143 | file_name)); | |
7c673cae | 144 | s = opts.table_factory->NewTableReader( |
11fdf7f2 TL |
145 | TableReaderOptions(ioptions, moptions.prefix_extractor.get(), |
146 | env_options, ikc), | |
147 | std::move(file_reader), file_size, &table_reader); | |
7c673cae FG |
148 | if (!s.ok()) { |
149 | fprintf(stderr, "Open Table Error: %s\n", s.ToString().c_str()); | |
150 | exit(1); | |
151 | } | |
152 | } | |
153 | ||
154 | Random rnd(301); | |
155 | std::string result; | |
156 | HistogramImpl hist; | |
157 | ||
158 | for (int it = 0; it < num_iter; it++) { | |
159 | for (int i = 0; i < num_keys1; i++) { | |
160 | for (int j = 0; j < num_keys2; j++) { | |
161 | int r1 = rnd.Uniform(num_keys1) * 2; | |
162 | int r2 = rnd.Uniform(num_keys2); | |
163 | if (if_query_empty_keys) { | |
164 | r1++; | |
165 | r2 = num_keys2 * 2 - r2; | |
166 | } | |
167 | ||
168 | if (!for_iterator) { | |
169 | // Query one existing key; | |
170 | std::string key = MakeKey(r1, r2, through_db); | |
171 | uint64_t start_time = Now(env, measured_by_nanosecond); | |
172 | if (!through_db) { | |
173 | PinnableSlice value; | |
174 | MergeContext merge_context; | |
494da23a | 175 | SequenceNumber max_covering_tombstone_seq = 0; |
7c673cae FG |
176 | GetContext get_context(ioptions.user_comparator, |
177 | ioptions.merge_operator, ioptions.info_log, | |
178 | ioptions.statistics, GetContext::kNotFound, | |
179 | Slice(key), &value, nullptr, &merge_context, | |
f67539c2 | 180 | true, &max_covering_tombstone_seq, env); |
11fdf7f2 | 181 | s = table_reader->Get(read_options, key, &get_context, nullptr); |
7c673cae FG |
182 | } else { |
183 | s = db->Get(read_options, key, &result); | |
184 | } | |
185 | hist.Add(Now(env, measured_by_nanosecond) - start_time); | |
186 | } else { | |
187 | int r2_len; | |
188 | if (if_query_empty_keys) { | |
189 | r2_len = 0; | |
190 | } else { | |
191 | r2_len = rnd.Uniform(num_keys2) + 1; | |
192 | if (r2_len + r2 > num_keys2) { | |
193 | r2_len = num_keys2 - r2; | |
194 | } | |
195 | } | |
196 | std::string start_key = MakeKey(r1, r2, through_db); | |
197 | std::string end_key = MakeKey(r1, r2 + r2_len, through_db); | |
198 | uint64_t total_time = 0; | |
199 | uint64_t start_time = Now(env, measured_by_nanosecond); | |
200 | Iterator* iter = nullptr; | |
201 | InternalIterator* iiter = nullptr; | |
202 | if (!through_db) { | |
f67539c2 TL |
203 | iiter = table_reader->NewIterator( |
204 | read_options, /*prefix_extractor=*/nullptr, /*arena=*/nullptr, | |
205 | /*skip_filters=*/false, TableReaderCaller::kUncategorized); | |
7c673cae FG |
206 | } else { |
207 | iter = db->NewIterator(read_options); | |
208 | } | |
209 | int count = 0; | |
210 | for (through_db ? iter->Seek(start_key) : iiter->Seek(start_key); | |
211 | through_db ? iter->Valid() : iiter->Valid(); | |
212 | through_db ? iter->Next() : iiter->Next()) { | |
213 | if (if_query_empty_keys) { | |
214 | break; | |
215 | } | |
216 | // verify key; | |
217 | total_time += Now(env, measured_by_nanosecond) - start_time; | |
218 | assert(Slice(MakeKey(r1, r2 + count, through_db)) == | |
219 | (through_db ? iter->key() : iiter->key())); | |
220 | start_time = Now(env, measured_by_nanosecond); | |
221 | if (++count >= r2_len) { | |
222 | break; | |
223 | } | |
224 | } | |
225 | if (count != r2_len) { | |
226 | fprintf( | |
227 | stderr, "Iterator cannot iterate expected number of entries. " | |
228 | "Expected %d but got %d\n", r2_len, count); | |
229 | assert(false); | |
230 | } | |
231 | delete iter; | |
232 | total_time += Now(env, measured_by_nanosecond) - start_time; | |
233 | hist.Add(total_time); | |
234 | } | |
235 | } | |
236 | } | |
237 | } | |
238 | ||
239 | fprintf( | |
240 | stderr, | |
241 | "===================================================" | |
242 | "====================================================\n" | |
243 | "InMemoryTableSimpleBenchmark: %20s num_key1: %5d " | |
244 | "num_key2: %5d %10s\n" | |
245 | "===================================================" | |
246 | "====================================================" | |
247 | "\nHistogram (unit: %s): \n%s", | |
248 | opts.table_factory->Name(), num_keys1, num_keys2, | |
249 | for_iterator ? "iterator" : (if_query_empty_keys ? "empty" : "non_empty"), | |
250 | measured_by_nanosecond ? "nanosecond" : "microsecond", | |
251 | hist.ToString().c_str()); | |
252 | if (!through_db) { | |
253 | env->DeleteFile(file_name); | |
254 | } else { | |
255 | delete db; | |
256 | db = nullptr; | |
257 | DestroyDB(dbname, opts); | |
258 | } | |
259 | } | |
260 | } // namespace | |
f67539c2 | 261 | } // namespace ROCKSDB_NAMESPACE |
7c673cae FG |
262 | |
263 | DEFINE_bool(query_empty, false, "query non-existing keys instead of existing " | |
264 | "ones."); | |
265 | DEFINE_int32(num_keys1, 4096, "number of distinguish prefix of keys"); | |
266 | DEFINE_int32(num_keys2, 512, "number of distinguish keys for each prefix"); | |
267 | DEFINE_int32(iter, 3, "query non-existing keys instead of existing ones"); | |
268 | DEFINE_int32(prefix_len, 16, "Prefix length used for iterators and indexes"); | |
269 | DEFINE_bool(iterator, false, "For test iterator"); | |
270 | DEFINE_bool(through_db, false, "If enable, a DB instance will be created and " | |
271 | "the query will be against DB. Otherwise, will be directly against " | |
272 | "a table reader."); | |
273 | DEFINE_bool(mmap_read, true, "Whether use mmap read"); | |
274 | DEFINE_string(table_factory, "block_based", | |
275 | "Table factory to use: `block_based` (default), `plain_table` or " | |
276 | "`cuckoo_hash`."); | |
277 | DEFINE_string(time_unit, "microsecond", | |
278 | "The time unit used for measuring performance. User can specify " | |
279 | "`microsecond` (default) or `nanosecond`"); | |
280 | ||
281 | int main(int argc, char** argv) { | |
282 | SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) + | |
283 | " [OPTIONS]..."); | |
284 | ParseCommandLineFlags(&argc, &argv, true); | |
285 | ||
f67539c2 TL |
286 | std::shared_ptr<ROCKSDB_NAMESPACE::TableFactory> tf; |
287 | ROCKSDB_NAMESPACE::Options options; | |
7c673cae | 288 | if (FLAGS_prefix_len < 16) { |
f67539c2 TL |
289 | options.prefix_extractor.reset( |
290 | ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_len)); | |
7c673cae | 291 | } |
f67539c2 TL |
292 | ROCKSDB_NAMESPACE::ReadOptions ro; |
293 | ROCKSDB_NAMESPACE::EnvOptions env_options; | |
7c673cae | 294 | options.create_if_missing = true; |
f67539c2 | 295 | options.compression = ROCKSDB_NAMESPACE::CompressionType::kNoCompression; |
7c673cae FG |
296 | |
297 | if (FLAGS_table_factory == "cuckoo_hash") { | |
298 | #ifndef ROCKSDB_LITE | |
299 | options.allow_mmap_reads = FLAGS_mmap_read; | |
300 | env_options.use_mmap_reads = FLAGS_mmap_read; | |
f67539c2 | 301 | ROCKSDB_NAMESPACE::CuckooTableOptions table_options; |
7c673cae | 302 | table_options.hash_table_ratio = 0.75; |
f67539c2 | 303 | tf.reset(ROCKSDB_NAMESPACE::NewCuckooTableFactory(table_options)); |
7c673cae FG |
304 | #else |
305 | fprintf(stderr, "Plain table is not supported in lite mode\n"); | |
306 | exit(1); | |
307 | #endif // ROCKSDB_LITE | |
308 | } else if (FLAGS_table_factory == "plain_table") { | |
309 | #ifndef ROCKSDB_LITE | |
310 | options.allow_mmap_reads = FLAGS_mmap_read; | |
311 | env_options.use_mmap_reads = FLAGS_mmap_read; | |
312 | ||
f67539c2 | 313 | ROCKSDB_NAMESPACE::PlainTableOptions plain_table_options; |
7c673cae FG |
314 | plain_table_options.user_key_len = 16; |
315 | plain_table_options.bloom_bits_per_key = (FLAGS_prefix_len == 16) ? 0 : 8; | |
316 | plain_table_options.hash_table_ratio = 0.75; | |
317 | ||
f67539c2 TL |
318 | tf.reset(new ROCKSDB_NAMESPACE::PlainTableFactory(plain_table_options)); |
319 | options.prefix_extractor.reset( | |
320 | ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_len)); | |
7c673cae FG |
321 | #else |
322 | fprintf(stderr, "Cuckoo table is not supported in lite mode\n"); | |
323 | exit(1); | |
324 | #endif // ROCKSDB_LITE | |
325 | } else if (FLAGS_table_factory == "block_based") { | |
f67539c2 | 326 | tf.reset(new ROCKSDB_NAMESPACE::BlockBasedTableFactory()); |
7c673cae FG |
327 | } else { |
328 | fprintf(stderr, "Invalid table type %s\n", FLAGS_table_factory.c_str()); | |
329 | } | |
330 | ||
331 | if (tf) { | |
332 | // if user provides invalid options, just fall back to microsecond. | |
333 | bool measured_by_nanosecond = FLAGS_time_unit == "nanosecond"; | |
334 | ||
335 | options.table_factory = tf; | |
f67539c2 TL |
336 | ROCKSDB_NAMESPACE::TableReaderBenchmark( |
337 | options, env_options, ro, FLAGS_num_keys1, FLAGS_num_keys2, FLAGS_iter, | |
338 | FLAGS_prefix_len, FLAGS_query_empty, FLAGS_iterator, FLAGS_through_db, | |
339 | measured_by_nanosecond); | |
7c673cae FG |
340 | } else { |
341 | return 1; | |
342 | } | |
343 | ||
344 | return 0; | |
345 | } | |
346 | ||
347 | #endif // GFLAGS |