]>
git.proxmox.com Git - ceph.git/blob - ceph/src/include/mempool.h
1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2016 Allen Samuels <allen.samuels@sandisk.com>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #ifndef _CEPH_INCLUDE_MEMPOOL_H
16 #define _CEPH_INCLUDE_MEMPOOL_H
20 #include <unordered_map>
26 #include <boost/container/flat_set.hpp>
27 #include <boost/container/flat_map.hpp>
29 #include "common/Formatter.h"
30 #include "common/ceph_atomic.h"
31 #include "include/ceph_assert.h"
32 #include "include/compact_map.h"
33 #include "include/compact_set.h"
41 A memory pool is a method for accounting the consumption of memory of
44 Memory pools are statically declared (see pool_index_t).
46 Each memory pool tracks the number of bytes and items it contains.
48 Allocators can be declared and associated with a type so that they are
49 tracked independently of the pool total. This additional accounting
50 is optional and only incurs an overhead if the debugging is enabled at
51 runtime. This allows developers to see what types are consuming the
58 Using memory pools is very easy.
60 To create a new memory pool, simply add a new name into the list of
61 memory pools that's defined in "DEFINE_MEMORY_POOLS_HELPER". That's
64 For each memory pool that's created a C++ namespace is also
65 automatically created (name is same as in DEFINE_MEMORY_POOLS_HELPER).
66 That namespace contains a set of common STL containers that are predefined
67 with the appropriate allocators.
69 Thus for mempool "osd" we have automatically available to us:
72 mempool::osd::multimap
74 mempool::osd::multiset
77 mempool::osd::unordered_map
80 Putting objects in a mempool
81 ----------------------------
83 In order to use a memory pool with a particular type, a few additional
84 declarations are needed.
89 MEMPOOL_CLASS_HELPERS();
93 Then, in an appropriate .cc file,
95 MEMPOOL_DEFINE_OBJECT_FACTORY(Foo, foo, osd);
97 The second argument can generally be identical to the first, except
98 when the type contains a nested scope. For example, for
99 BlueStore::Onode, we need to do
101 MEMPOOL_DEFINE_OBJECT_FACTORY(BlueStore::Onode, bluestore_onode,
104 (This is just because we need to name some static variables and we
105 can't use :: in a variable name.)
107 XXX Note: the new operator hard-codes the allocation size to the size of the
108 object given in MEMPOOL_DEFINE_OBJECT_FACTORY. For this reason, you cannot
109 incorporate mempools into a base class without also defining a helper/factory
110 for the child class as well (as the base class is usually smaller than the
113 In order to use the STL containers, simply use the namespaced variant
114 of the container type. For example,
116 mempool::osd::map<int> myvec;
121 The simplest way to interrogate the process is with
126 This will dump information about *all* memory pools. When debug mode
127 is enabled, the runtime complexity of dump is O(num_shards *
128 num_types). When debug name is disabled it is O(num_shards).
130 You can also interrogate a specific pool programmatically with
132 size_t bytes = mempool::unittest_2::allocated_bytes();
133 size_t items = mempool::unittest_2::allocated_items();
135 The runtime complexity is O(num_shards).
137 Note that you cannot easily query per-type, primarily because debug
138 mode is optional and you should not rely on that information being
145 // --------------------------------------------------------------
146 // define memory pools
148 #define DEFINE_MEMORY_POOLS_HELPER(f) \
151 f(bluestore_cache_data) \
152 f(bluestore_cache_onode) \
153 f(bluestore_cache_meta) \
154 f(bluestore_cache_other) \
155 f(bluestore_Buffer) \
156 f(bluestore_Extent) \
158 f(bluestore_SharedBlob) \
159 f(bluestore_inline_bl) \
162 f(bluestore_writing_deferred) \
163 f(bluestore_writing) \
165 f(bluefs_file_reader) \
166 f(bluefs_file_writer) \
180 // give them integer ids
181 #define P(x) mempool_##x,
183 DEFINE_MEMORY_POOLS_HELPER(P
)
184 num_pools
// Must be last.
188 extern bool debug_mode
;
189 extern void set_debug_mode(bool d
);
191 // --------------------------------------------------------------
194 // we shard pool stats across many shard_t's to reduce the amount
195 // of cacheline ping pong.
200 num_shards
= 1 << num_shard_bits
203 // align shard to a cacheline
205 ceph::atomic
<size_t> bytes
= {0};
206 ceph::atomic
<size_t> items
= {0};
207 char __padding
[128 - sizeof(ceph::atomic
<size_t>)*2];
208 } __attribute__ ((aligned (128)));
210 static_assert(sizeof(shard_t
) == 128, "shard_t should be cacheline-sized");
215 void dump(ceph::Formatter
*f
) const {
216 f
->dump_int("items", items
);
217 f
->dump_int("bytes", bytes
);
220 stats_t
& operator+=(const stats_t
& o
) {
227 pool_t
& get_pool(pool_index_t ix
);
228 const char *get_pool_name(pool_index_t ix
);
231 const char *type_name
;
233 ceph::atomic
<ssize_t
> items
= {0}; // signed
236 struct type_info_hash
{
237 std::size_t operator()(const std::type_info
& k
) const {
238 return k
.hash_code();
243 shard_t shard
[num_shards
];
245 mutable std::mutex lock
; // only used for types list
246 std::unordered_map
<const char *, type_t
> type_map
;
250 // How much this pool consumes. O(<num_shards>)
252 size_t allocated_bytes() const;
253 size_t allocated_items() const;
255 void adjust_count(ssize_t items
, ssize_t bytes
);
257 shard_t
* pick_a_shard() {
259 // http://fossies.org/dox/glibc-2.24/pthread__self_8c_source.html
260 size_t me
= (size_t)pthread_self();
261 size_t i
= (me
>> 3) & ((1 << num_shard_bits
) - 1);
265 type_t
*get_type(const std::type_info
& ti
, size_t size
) {
266 std::lock_guard
<std::mutex
> l(lock
);
267 auto p
= type_map
.find(ti
.name());
268 if (p
!= type_map
.end()) {
271 type_t
&t
= type_map
[ti
.name()];
272 t
.type_name
= ti
.name();
277 // get pool stats. by_type is not populated if !debug
278 void get_stats(stats_t
*total
,
279 std::map
<std::string
, stats_t
> *by_type
) const;
281 void dump(ceph::Formatter
*f
, stats_t
*ptotal
=0) const;
284 void dump(ceph::Formatter
*f
);
287 // STL allocator for use with containers. All actual state
288 // is stored in the static pool_allocator_base_t, which saves us from
289 // passing the allocator to container constructors.
291 template<pool_index_t pool_ix
, typename T
>
292 class pool_allocator
{
294 type_t
*type
= nullptr;
297 typedef pool_allocator
<pool_ix
, T
> allocator_type
;
298 typedef T value_type
;
299 typedef value_type
*pointer
;
300 typedef const value_type
* const_pointer
;
301 typedef value_type
& reference
;
302 typedef const value_type
& const_reference
;
303 typedef std::size_t size_type
;
304 typedef std::ptrdiff_t difference_type
;
306 template<typename U
> struct rebind
{
307 typedef pool_allocator
<pool_ix
,U
> other
;
310 void init(bool force_register
) {
311 pool
= &get_pool(pool_ix
);
312 if (debug_mode
|| force_register
) {
313 type
= pool
->get_type(typeid(T
), sizeof(T
));
317 pool_allocator(bool force_register
=false) {
318 init(force_register
);
321 pool_allocator(const pool_allocator
<pool_ix
,U
>&) {
325 T
* allocate(size_t n
, void *p
= nullptr) {
326 size_t total
= sizeof(T
) * n
;
327 shard_t
*shard
= pool
->pick_a_shard();
328 shard
->bytes
+= total
;
333 T
* r
= reinterpret_cast<T
*>(new char[total
]);
337 void deallocate(T
* p
, size_t n
) {
338 size_t total
= sizeof(T
) * n
;
339 shard_t
*shard
= pool
->pick_a_shard();
340 shard
->bytes
-= total
;
345 delete[] reinterpret_cast<char*>(p
);
348 T
* allocate_aligned(size_t n
, size_t align
, void *p
= nullptr) {
349 size_t total
= sizeof(T
) * n
;
350 shard_t
*shard
= pool
->pick_a_shard();
351 shard
->bytes
+= total
;
357 int rc
= ::posix_memalign((void**)(void*)&ptr
, align
, total
);
359 throw std::bad_alloc();
360 T
* r
= reinterpret_cast<T
*>(ptr
);
364 void deallocate_aligned(T
* p
, size_t n
) {
365 size_t total
= sizeof(T
) * n
;
366 shard_t
*shard
= pool
->pick_a_shard();
367 shard
->bytes
-= total
;
384 void construct(T
* p
, const T
& val
) {
385 ::new ((void *)p
) T(val
);
388 template<class U
, class... Args
> void construct(U
* p
,Args
&&... args
) {
389 ::new((void *)p
) U(std::forward
<Args
>(args
)...);
392 bool operator==(const pool_allocator
&) const { return true; }
393 bool operator!=(const pool_allocator
&) const { return false; }
401 static const mempool::pool_index_t id = mempool::mempool_##x; \
402 template<typename v> \
403 using pool_allocator = mempool::pool_allocator<id,v>; \
405 using string = std::basic_string<char,std::char_traits<char>, \
406 pool_allocator<char>>; \
408 template<typename k,typename v, typename cmp = std::less<k> > \
409 using map = std::map<k, v, cmp, \
410 pool_allocator<std::pair<const k,v>>>; \
412 template<typename k,typename v, typename cmp = std::less<k> > \
413 using compact_map = compact_map<k, v, cmp, \
414 pool_allocator<std::pair<const k,v>>>; \
416 template<typename k,typename v, typename cmp = std::less<k> > \
417 using compact_multimap = compact_multimap<k, v, cmp, \
418 pool_allocator<std::pair<const k,v>>>; \
420 template<typename k, typename cmp = std::less<k> > \
421 using compact_set = compact_set<k, cmp, pool_allocator<k>>; \
423 template<typename k,typename v, typename cmp = std::less<k> > \
424 using multimap = std::multimap<k,v,cmp, \
425 pool_allocator<std::pair<const k, \
428 template<typename k, typename cmp = std::less<k> > \
429 using set = std::set<k,cmp,pool_allocator<k>>; \
431 template<typename k, typename cmp = std::less<k> > \
432 using flat_set = boost::container::flat_set<k,cmp,pool_allocator<k>>; \
434 template<typename k, typename v, typename cmp = std::less<k> > \
435 using flat_map = boost::container::flat_map<k,v,cmp, \
436 pool_allocator<std::pair<k,v>>>; \
438 template<typename v> \
439 using list = std::list<v,pool_allocator<v>>; \
441 template<typename v> \
442 using vector = std::vector<v,pool_allocator<v>>; \
444 template<typename k, typename v, \
445 typename h=std::hash<k>, \
446 typename eq = std::equal_to<k>> \
447 using unordered_map = \
448 std::unordered_map<k,v,h,eq,pool_allocator<std::pair<const k,v>>>;\
450 inline size_t allocated_bytes() { \
451 return mempool::get_pool(id).allocated_bytes(); \
453 inline size_t allocated_items() { \
454 return mempool::get_pool(id).allocated_items(); \
458 DEFINE_MEMORY_POOLS_HELPER(P
)
464 // the elements allocated by mempool is in the same memory space as the ones
465 // allocated by the default allocator. so compare them in an efficient way:
466 // libstdc++'s std::equal is specialized to use memcmp if T is integer or
467 // pointer. this is good enough for our usecase. use
468 // std::is_trivially_copyable<T> to expand the support to more types if
470 template<typename T
, mempool::pool_index_t pool_index
>
471 bool operator==(const std::vector
<T
, std::allocator
<T
>>& lhs
,
472 const std::vector
<T
, mempool::pool_allocator
<pool_index
, T
>>& rhs
)
474 return (lhs
.size() == rhs
.size() &&
475 std::equal(lhs
.begin(), lhs
.end(), rhs
.begin()));
478 template<typename T
, mempool::pool_index_t pool_index
>
479 bool operator!=(const std::vector
<T
, std::allocator
<T
>>& lhs
,
480 const std::vector
<T
, mempool::pool_allocator
<pool_index
, T
>>& rhs
)
482 return !(lhs
== rhs
);
485 template<typename T
, mempool::pool_index_t pool_index
>
486 bool operator==(const std::vector
<T
, mempool::pool_allocator
<pool_index
, T
>>& lhs
,
487 const std::vector
<T
, std::allocator
<T
>>& rhs
)
492 template<typename T
, mempool::pool_index_t pool_index
>
493 bool operator!=(const std::vector
<T
, mempool::pool_allocator
<pool_index
, T
>>& lhs
,
494 const std::vector
<T
, std::allocator
<T
>>& rhs
)
496 return !(lhs
== rhs
);
499 // Use this for any type that is contained by a container (unless it
500 // is a class you defined; see below).
501 #define MEMPOOL_DECLARE_FACTORY(obj, factoryname, pool) \
502 namespace mempool { \
504 extern pool_allocator<obj> alloc_##factoryname; \
508 #define MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool) \
509 namespace mempool { \
511 pool_allocator<obj> alloc_##factoryname = {true}; \
515 // Use this for each class that belongs to a mempool. For example,
518 // MEMPOOL_CLASS_HELPERS();
522 #define MEMPOOL_CLASS_HELPERS() \
523 void *operator new(size_t size); \
524 void *operator new[](size_t size) noexcept { \
525 ceph_abort_msg("no array new"); \
527 void operator delete(void *); \
528 void operator delete[](void *) { ceph_abort_msg("no array delete"); }
531 // Use this in some particular .cc file to match each class with a
532 // MEMPOOL_CLASS_HELPERS().
533 #define MEMPOOL_DEFINE_OBJECT_FACTORY(obj,factoryname,pool) \
534 MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool) \
535 void *obj::operator new(size_t size) { \
536 return mempool::pool::alloc_##factoryname.allocate(1); \
538 void obj::operator delete(void *p) { \
539 return mempool::pool::alloc_##factoryname.deallocate((obj*)p, 1); \