]>
Commit | Line | Data |
---|---|---|
7c673cae | 1 | // Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
11fdf7f2 TL |
2 | // This source code is licensed under both the GPLv2 (found in the |
3 | // COPYING file in the root directory) and Apache 2.0 License | |
4 | // (found in the LICENSE.Apache file in the root directory). | |
7c673cae FG |
5 | |
6 | #pragma once | |
7 | ||
8 | #include <string> | |
9 | ||
10 | #include "rocksdb/slice.h" | |
11 | ||
12 | #include "port/port.h" | |
494da23a | 13 | #include "util/hash.h" |
7c673cae FG |
14 | |
15 | #include <atomic> | |
16 | #include <memory> | |
17 | ||
18 | namespace rocksdb { | |
19 | ||
20 | class Slice; | |
21 | class Allocator; | |
22 | class Logger; | |
23 | ||
24 | class DynamicBloom { | |
25 | public: | |
26 | // allocator: pass allocator to bloom filter, hence trace the usage of memory | |
27 | // total_bits: fixed total bits for the bloom | |
28 | // num_probes: number of hash probes for a single key | |
29 | // locality: If positive, optimize for cache line locality, 0 otherwise. | |
30 | // hash_func: customized hash function | |
31 | // huge_page_tlb_size: if >0, try to allocate bloom bytes from huge page TLB | |
11fdf7f2 | 32 | // within this page size. Need to reserve huge pages for |
7c673cae FG |
33 | // it to be allocated, like: |
34 | // sysctl -w vm.nr_hugepages=20 | |
35 | // See linux doc Documentation/vm/hugetlbpage.txt | |
36 | explicit DynamicBloom(Allocator* allocator, | |
37 | uint32_t total_bits, uint32_t locality = 0, | |
38 | uint32_t num_probes = 6, | |
7c673cae FG |
39 | size_t huge_page_tlb_size = 0, |
40 | Logger* logger = nullptr); | |
41 | ||
494da23a | 42 | explicit DynamicBloom(uint32_t num_probes = 6); |
7c673cae FG |
43 | |
44 | void SetTotalBits(Allocator* allocator, uint32_t total_bits, | |
45 | uint32_t locality, size_t huge_page_tlb_size, | |
46 | Logger* logger); | |
47 | ||
48 | ~DynamicBloom() {} | |
49 | ||
50 | // Assuming single threaded access to this function. | |
51 | void Add(const Slice& key); | |
52 | ||
53 | // Like Add, but may be called concurrent with other functions. | |
54 | void AddConcurrently(const Slice& key); | |
55 | ||
56 | // Assuming single threaded access to this function. | |
57 | void AddHash(uint32_t hash); | |
58 | ||
59 | // Like AddHash, but may be called concurrent with other functions. | |
60 | void AddHashConcurrently(uint32_t hash); | |
61 | ||
62 | // Multithreaded access to this function is OK | |
63 | bool MayContain(const Slice& key) const; | |
64 | ||
65 | // Multithreaded access to this function is OK | |
66 | bool MayContainHash(uint32_t hash) const; | |
67 | ||
68 | void Prefetch(uint32_t h); | |
69 | ||
70 | uint32_t GetNumBlocks() const { return kNumBlocks; } | |
71 | ||
72 | Slice GetRawData() const { | |
73 | return Slice(reinterpret_cast<char*>(data_), GetTotalBits() / 8); | |
74 | } | |
75 | ||
76 | void SetRawData(unsigned char* raw_data, uint32_t total_bits, | |
77 | uint32_t num_blocks = 0); | |
78 | ||
79 | uint32_t GetTotalBits() const { return kTotalBits; } | |
80 | ||
81 | bool IsInitialized() const { return kNumBlocks > 0 || kTotalBits > 0; } | |
82 | ||
83 | private: | |
84 | uint32_t kTotalBits; | |
85 | uint32_t kNumBlocks; | |
86 | const uint32_t kNumProbes; | |
87 | ||
7c673cae FG |
88 | std::atomic<uint8_t>* data_; |
89 | ||
90 | // or_func(ptr, mask) should effect *ptr |= mask with the appropriate | |
91 | // concurrency safety, working with bytes. | |
92 | template <typename OrFunc> | |
93 | void AddHash(uint32_t hash, const OrFunc& or_func); | |
94 | }; | |
95 | ||
494da23a | 96 | inline void DynamicBloom::Add(const Slice& key) { AddHash(BloomHash(key)); } |
7c673cae FG |
97 | |
98 | inline void DynamicBloom::AddConcurrently(const Slice& key) { | |
494da23a | 99 | AddHashConcurrently(BloomHash(key)); |
7c673cae FG |
100 | } |
101 | ||
102 | inline void DynamicBloom::AddHash(uint32_t hash) { | |
103 | AddHash(hash, [](std::atomic<uint8_t>* ptr, uint8_t mask) { | |
104 | ptr->store(ptr->load(std::memory_order_relaxed) | mask, | |
105 | std::memory_order_relaxed); | |
106 | }); | |
107 | } | |
108 | ||
109 | inline void DynamicBloom::AddHashConcurrently(uint32_t hash) { | |
110 | AddHash(hash, [](std::atomic<uint8_t>* ptr, uint8_t mask) { | |
111 | // Happens-before between AddHash and MaybeContains is handled by | |
112 | // access to versions_->LastSequence(), so all we have to do here is | |
113 | // avoid races (so we don't give the compiler a license to mess up | |
114 | // our code) and not lose bits. std::memory_order_relaxed is enough | |
115 | // for that. | |
116 | if ((mask & ptr->load(std::memory_order_relaxed)) != mask) { | |
117 | ptr->fetch_or(mask, std::memory_order_relaxed); | |
118 | } | |
119 | }); | |
120 | } | |
121 | ||
122 | inline bool DynamicBloom::MayContain(const Slice& key) const { | |
494da23a | 123 | return (MayContainHash(BloomHash(key))); |
7c673cae FG |
124 | } |
125 | ||
11fdf7f2 TL |
126 | #if defined(_MSC_VER) |
127 | #pragma warning(push) | |
128 | // local variable is initialized but not referenced | |
129 | #pragma warning(disable : 4189) | |
130 | #endif | |
7c673cae FG |
131 | inline void DynamicBloom::Prefetch(uint32_t h) { |
132 | if (kNumBlocks != 0) { | |
133 | uint32_t b = ((h >> 11 | (h << 21)) % kNumBlocks) * (CACHE_LINE_SIZE * 8); | |
134 | PREFETCH(&(data_[b / 8]), 0, 3); | |
135 | } | |
136 | } | |
11fdf7f2 TL |
137 | #if defined(_MSC_VER) |
138 | #pragma warning(pop) | |
139 | #endif | |
7c673cae FG |
140 | |
141 | inline bool DynamicBloom::MayContainHash(uint32_t h) const { | |
142 | assert(IsInitialized()); | |
143 | const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits | |
144 | if (kNumBlocks != 0) { | |
145 | uint32_t b = ((h >> 11 | (h << 21)) % kNumBlocks) * (CACHE_LINE_SIZE * 8); | |
146 | for (uint32_t i = 0; i < kNumProbes; ++i) { | |
147 | // Since CACHE_LINE_SIZE is defined as 2^n, this line will be optimized | |
148 | // to a simple and operation by compiler. | |
149 | const uint32_t bitpos = b + (h % (CACHE_LINE_SIZE * 8)); | |
150 | uint8_t byteval = data_[bitpos / 8].load(std::memory_order_relaxed); | |
151 | if ((byteval & (1 << (bitpos % 8))) == 0) { | |
152 | return false; | |
153 | } | |
154 | // Rotate h so that we don't reuse the same bytes. | |
155 | h = h / (CACHE_LINE_SIZE * 8) + | |
156 | (h % (CACHE_LINE_SIZE * 8)) * (0x20000000U / CACHE_LINE_SIZE); | |
157 | h += delta; | |
158 | } | |
159 | } else { | |
160 | for (uint32_t i = 0; i < kNumProbes; ++i) { | |
161 | const uint32_t bitpos = h % kTotalBits; | |
162 | uint8_t byteval = data_[bitpos / 8].load(std::memory_order_relaxed); | |
163 | if ((byteval & (1 << (bitpos % 8))) == 0) { | |
164 | return false; | |
165 | } | |
166 | h += delta; | |
167 | } | |
168 | } | |
169 | return true; | |
170 | } | |
171 | ||
172 | template <typename OrFunc> | |
173 | inline void DynamicBloom::AddHash(uint32_t h, const OrFunc& or_func) { | |
174 | assert(IsInitialized()); | |
175 | const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits | |
176 | if (kNumBlocks != 0) { | |
177 | uint32_t b = ((h >> 11 | (h << 21)) % kNumBlocks) * (CACHE_LINE_SIZE * 8); | |
178 | for (uint32_t i = 0; i < kNumProbes; ++i) { | |
179 | // Since CACHE_LINE_SIZE is defined as 2^n, this line will be optimized | |
180 | // to a simple and operation by compiler. | |
181 | const uint32_t bitpos = b + (h % (CACHE_LINE_SIZE * 8)); | |
182 | or_func(&data_[bitpos / 8], (1 << (bitpos % 8))); | |
183 | // Rotate h so that we don't reuse the same bytes. | |
184 | h = h / (CACHE_LINE_SIZE * 8) + | |
185 | (h % (CACHE_LINE_SIZE * 8)) * (0x20000000U / CACHE_LINE_SIZE); | |
186 | h += delta; | |
187 | } | |
188 | } else { | |
189 | for (uint32_t i = 0; i < kNumProbes; ++i) { | |
190 | const uint32_t bitpos = h % kTotalBits; | |
191 | or_func(&data_[bitpos / 8], (1 << (bitpos % 8))); | |
192 | h += delta; | |
193 | } | |
194 | } | |
195 | } | |
196 | ||
197 | } // rocksdb |