]>
git.proxmox.com Git - ceph.git/blob - ceph/src/rocksdb/util/arena.cc
1 // Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
6 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
7 // Use of this source code is governed by a BSD-style license that can be
8 // found in the LICENSE file. See the AUTHORS file for names of contributors.
10 #include "util/arena.h"
11 #ifdef ROCKSDB_MALLOC_USABLE_SIZE
13 #include <malloc_np.h>
22 #include "port/port.h"
23 #include "rocksdb/env.h"
24 #include "util/logging.h"
25 #include "util/sync_point.h"
29 // MSVC complains that it is already defined since it is static in the header.
31 const size_t Arena::kInlineSize
;
34 const size_t Arena::kMinBlockSize
= 4096;
35 const size_t Arena::kMaxBlockSize
= 2u << 30;
36 static const int kAlignUnit
= alignof(max_align_t
);
38 size_t OptimizeBlockSize(size_t block_size
) {
39 // Make sure block_size is in optimal range
40 block_size
= std::max(Arena::kMinBlockSize
, block_size
);
41 block_size
= std::min(Arena::kMaxBlockSize
, block_size
);
43 // make sure block_size is the multiple of kAlignUnit
44 if (block_size
% kAlignUnit
!= 0) {
45 block_size
= (1 + block_size
/ kAlignUnit
) * kAlignUnit
;
51 Arena::Arena(size_t block_size
, AllocTracker
* tracker
, size_t huge_page_size
)
52 : kBlockSize(OptimizeBlockSize(block_size
)), tracker_(tracker
) {
53 assert(kBlockSize
>= kMinBlockSize
&& kBlockSize
<= kMaxBlockSize
&&
54 kBlockSize
% kAlignUnit
== 0);
55 TEST_SYNC_POINT_CALLBACK("Arena::Arena:0", const_cast<size_t*>(&kBlockSize
));
56 alloc_bytes_remaining_
= sizeof(inline_block_
);
57 blocks_memory_
+= alloc_bytes_remaining_
;
58 aligned_alloc_ptr_
= inline_block_
;
59 unaligned_alloc_ptr_
= inline_block_
+ alloc_bytes_remaining_
;
61 hugetlb_size_
= huge_page_size
;
62 if (hugetlb_size_
&& kBlockSize
> hugetlb_size_
) {
63 hugetlb_size_
= ((kBlockSize
- 1U) / hugetlb_size_
+ 1U) * hugetlb_size_
;
68 if (tracker_
!= nullptr) {
69 tracker_
->Allocate(kInlineSize
);
74 if (tracker_
!= nullptr) {
75 assert(tracker_
->is_freed());
78 for (const auto& block
: blocks_
) {
83 for (const auto& mmap_info
: huge_blocks_
) {
84 if (mmap_info
.addr_
== nullptr) {
87 auto ret
= munmap(mmap_info
.addr_
, mmap_info
.length_
);
89 // TODO(sdong): Better handling
95 char* Arena::AllocateFallback(size_t bytes
, bool aligned
) {
96 if (bytes
> kBlockSize
/ 4) {
97 ++irregular_block_num
;
98 // Object is more than a quarter of our block size. Allocate it separately
99 // to avoid wasting too much space in leftover bytes.
100 return AllocateNewBlock(bytes
);
103 // We waste the remaining space in the current block.
105 char* block_head
= nullptr;
108 size
= hugetlb_size_
;
109 block_head
= AllocateFromHugePage(size
);
114 block_head
= AllocateNewBlock(size
);
116 alloc_bytes_remaining_
= size
- bytes
;
119 aligned_alloc_ptr_
= block_head
+ bytes
;
120 unaligned_alloc_ptr_
= block_head
+ size
;
123 aligned_alloc_ptr_
= block_head
;
124 unaligned_alloc_ptr_
= block_head
+ size
- bytes
;
125 return unaligned_alloc_ptr_
;
129 char* Arena::AllocateFromHugePage(size_t bytes
) {
131 if (hugetlb_size_
== 0) {
134 // Reserve space in `huge_blocks_` before calling `mmap`.
135 // Use `emplace_back()` instead of `reserve()` to let std::vector manage its
136 // own memory and do fewer reallocations.
138 // - If `emplace_back` throws, no memory leaks because we haven't called
140 // - If `mmap` throws, no memory leaks because the vector will be cleaned up
142 huge_blocks_
.emplace_back(nullptr /* addr */, 0 /* length */);
144 void* addr
= mmap(nullptr, bytes
, (PROT_READ
| PROT_WRITE
),
145 (MAP_PRIVATE
| MAP_ANONYMOUS
| MAP_HUGETLB
), -1, 0);
147 if (addr
== MAP_FAILED
) {
150 huge_blocks_
.back() = MmapInfo(addr
, bytes
);
151 blocks_memory_
+= bytes
;
152 if (tracker_
!= nullptr) {
153 tracker_
->Allocate(bytes
);
155 return reinterpret_cast<char*>(addr
);
162 char* Arena::AllocateAligned(size_t bytes
, size_t huge_page_size
,
164 assert((kAlignUnit
& (kAlignUnit
- 1)) ==
165 0); // Pointer size should be a power of 2
168 if (huge_page_size
> 0 && bytes
> 0) {
169 // Allocate from a huge page TBL table.
170 assert(logger
!= nullptr); // logger need to be passed in.
171 size_t reserved_size
=
172 ((bytes
- 1U) / huge_page_size
+ 1U) * huge_page_size
;
173 assert(reserved_size
>= bytes
);
175 char* addr
= AllocateFromHugePage(reserved_size
);
176 if (addr
== nullptr) {
177 ROCKS_LOG_WARN(logger
,
178 "AllocateAligned fail to allocate huge TLB pages: %s",
180 // fail back to malloc
186 (void)huge_page_size
;
191 reinterpret_cast<uintptr_t>(aligned_alloc_ptr_
) & (kAlignUnit
- 1);
192 size_t slop
= (current_mod
== 0 ? 0 : kAlignUnit
- current_mod
);
193 size_t needed
= bytes
+ slop
;
195 if (needed
<= alloc_bytes_remaining_
) {
196 result
= aligned_alloc_ptr_
+ slop
;
197 aligned_alloc_ptr_
+= needed
;
198 alloc_bytes_remaining_
-= needed
;
200 // AllocateFallback always returns aligned memory
201 result
= AllocateFallback(bytes
, true /* aligned */);
203 assert((reinterpret_cast<uintptr_t>(result
) & (kAlignUnit
- 1)) == 0);
207 char* Arena::AllocateNewBlock(size_t block_bytes
) {
208 // Reserve space in `blocks_` before allocating memory via new.
209 // Use `emplace_back()` instead of `reserve()` to let std::vector manage its
210 // own memory and do fewer reallocations.
212 // - If `emplace_back` throws, no memory leaks because we haven't called `new`
214 // - If `new` throws, no memory leaks because the vector will be cleaned up
216 blocks_
.emplace_back(nullptr);
218 char* block
= new char[block_bytes
];
219 size_t allocated_size
;
220 #ifdef ROCKSDB_MALLOC_USABLE_SIZE
221 allocated_size
= malloc_usable_size(block
);
223 // It's hard to predict what malloc_usable_size() returns.
224 // A callback can allow users to change the costed size.
225 std::pair
<size_t*, size_t*> pair(&allocated_size
, &block_bytes
);
226 TEST_SYNC_POINT_CALLBACK("Arena::AllocateNewBlock:0", &pair
);
229 allocated_size
= block_bytes
;
230 #endif // ROCKSDB_MALLOC_USABLE_SIZE
231 blocks_memory_
+= allocated_size
;
232 if (tracker_
!= nullptr) {
233 tracker_
->Allocate(allocated_size
);
235 blocks_
.back() = block
;
239 } // namespace rocksdb