]>
git.proxmox.com Git - ceph.git/blob - ceph/src/include/mempool.h
3b94d87205e4420c477ba39b0d8824670755d39f
1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2016 Allen Samuels <allen.samuels@sandisk.com>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #ifndef _CEPH_INCLUDE_MEMPOOL_H
16 #define _CEPH_INCLUDE_MEMPOOL_H
20 #include <unordered_map>
28 #include <common/Formatter.h>
29 #include "include/assert.h"
30 #include "include/compact_map.h"
31 #include "include/compact_set.h"
39 A memory pool is a method for accounting the consumption of memory of
42 Memory pools are statically declared (see pool_index_t).
44 Each memory pool tracks the number of bytes and items it contains.
46 Allocators can be declared and associated with a type so that they are
47 tracked independently of the pool total. This additional accounting
48 is optional and only incurs an overhead if the debugging is enabled at
49 runtime. This allows developers to see what types are consuming the
56 Using memory pools is very easy.
58 To create a new memory pool, simply add a new name into the list of
59 memory pools that's defined in "DEFINE_MEMORY_POOLS_HELPER". That's
62 For each memory pool that's created a C++ namespace is also
63 automatically created (name is same as in DEFINE_MEMORY_POOLS_HELPER).
64 That namespace contains a set of common STL containers that are predefined
65 with the appropriate allocators.
67 Thus for mempool "osd" we have automatically available to us:
70 mempool::osd::multimap
72 mempool::osd::multiset
75 mempool::osd::unordered_map
78 Putting objects in a mempool
79 ----------------------------
81 In order to use a memory pool with a particular type, a few additional
82 declarations are needed.
87 MEMPOOL_CLASS_HELPERS();
91 Then, in an appropriate .cc file,
93 MEMPOOL_DEFINE_OBJECT_FACTORY(Foo, foo, osd);
95 The second argument can generally be identical to the first, except
96 when the type contains a nested scope. For example, for
97 BlueStore::Onode, we need to do
99 MEMPOOL_DEFINE_OBJECT_FACTORY(BlueStore::Onode, bluestore_onode,
102 (This is just because we need to name some static variables and we
103 can't use :: in a variable name.)
105 XXX Note: the new operator hard-codes the allocation size to the size of the
106 object given in MEMPOOL_DEFINE_OBJECT_FACTORY. For this reason, you cannot
107 incorporate mempools into a base class without also defining a helper/factory
108 for the child class as well (as the base class is usually smaller than the
111 In order to use the STL containers, simply use the namespaced variant
112 of the container type. For example,
114 mempool::osd::map<int> myvec;
119 The simplest way to interrogate the process is with
124 This will dump information about *all* memory pools. When debug mode
125 is enabled, the runtime complexity of dump is O(num_shards *
126 num_types). When debug name is disabled it is O(num_shards).
128 You can also interrogate a specific pool programmatically with
130 size_t bytes = mempool::unittest_2::allocated_bytes();
131 size_t items = mempool::unittest_2::allocated_items();
133 The runtime complexity is O(num_shards).
135 Note that you cannot easily query per-type, primarily because debug
136 mode is optional and you should not rely on that information being
143 // --------------------------------------------------------------
144 // define memory pools
146 #define DEFINE_MEMORY_POOLS_HELPER(f) \
149 f(bluestore_cache_data) \
150 f(bluestore_cache_onode) \
151 f(bluestore_cache_other) \
154 f(bluestore_writing_deferred) \
155 f(bluestore_writing) \
170 // give them integer ids
171 #define P(x) mempool_##x,
173 DEFINE_MEMORY_POOLS_HELPER(P
)
174 num_pools
// Must be last.
178 extern bool debug_mode
;
179 extern void set_debug_mode(bool d
);
181 // --------------------------------------------------------------
184 // we shard pool stats across many shard_t's to reduce the amount
185 // of cacheline ping pong.
190 num_shards
= 1 << num_shard_bits
193 // align shard to a cacheline
195 std::atomic
<size_t> bytes
= {0};
196 std::atomic
<size_t> items
= {0};
197 char __padding
[128 - sizeof(std::atomic
<size_t>)*2];
198 } __attribute__ ((aligned (128)));
200 static_assert(sizeof(shard_t
) == 128, "shard_t should be cacheline-sized");
205 void dump(ceph::Formatter
*f
) const {
206 f
->dump_int("items", items
);
207 f
->dump_int("bytes", bytes
);
210 stats_t
& operator+=(const stats_t
& o
) {
217 pool_t
& get_pool(pool_index_t ix
);
218 const char *get_pool_name(pool_index_t ix
);
221 const char *type_name
;
223 std::atomic
<ssize_t
> items
= {0}; // signed
226 struct type_info_hash
{
227 std::size_t operator()(const std::type_info
& k
) const {
228 return k
.hash_code();
233 shard_t shard
[num_shards
];
235 mutable std::mutex lock
; // only used for types list
236 std::unordered_map
<const char *, type_t
> type_map
;
240 // How much this pool consumes. O(<num_shards>)
242 size_t allocated_bytes() const;
243 size_t allocated_items() const;
245 void adjust_count(ssize_t items
, ssize_t bytes
);
247 shard_t
* pick_a_shard() {
249 // http://fossies.org/dox/glibc-2.24/pthread__self_8c_source.html
250 size_t me
= (size_t)pthread_self();
251 size_t i
= (me
>> 3) & ((1 << num_shard_bits
) - 1);
255 type_t
*get_type(const std::type_info
& ti
, size_t size
) {
256 std::lock_guard
<std::mutex
> l(lock
);
257 auto p
= type_map
.find(ti
.name());
258 if (p
!= type_map
.end()) {
261 type_t
&t
= type_map
[ti
.name()];
262 t
.type_name
= ti
.name();
267 // get pool stats. by_type is not populated if !debug
268 void get_stats(stats_t
*total
,
269 std::map
<std::string
, stats_t
> *by_type
) const;
271 void dump(ceph::Formatter
*f
, stats_t
*ptotal
=0) const;
274 void dump(ceph::Formatter
*f
);
277 // STL allocator for use with containers. All actual state
278 // is stored in the static pool_allocator_base_t, which saves us from
279 // passing the allocator to container constructors.
281 template<pool_index_t pool_ix
, typename T
>
282 class pool_allocator
{
284 type_t
*type
= nullptr;
287 typedef pool_allocator
<pool_ix
, T
> allocator_type
;
288 typedef T value_type
;
289 typedef value_type
*pointer
;
290 typedef const value_type
* const_pointer
;
291 typedef value_type
& reference
;
292 typedef const value_type
& const_reference
;
293 typedef std::size_t size_type
;
294 typedef std::ptrdiff_t difference_type
;
296 template<typename U
> struct rebind
{
297 typedef pool_allocator
<pool_ix
,U
> other
;
300 void init(bool force_register
) {
301 pool
= &get_pool(pool_ix
);
302 if (debug_mode
|| force_register
) {
303 type
= pool
->get_type(typeid(T
), sizeof(T
));
307 pool_allocator(bool force_register
=false) {
308 init(force_register
);
311 pool_allocator(const pool_allocator
<pool_ix
,U
>&) {
315 T
* allocate(size_t n
, void *p
= nullptr) {
316 size_t total
= sizeof(T
) * n
;
317 shard_t
*shard
= pool
->pick_a_shard();
318 shard
->bytes
+= total
;
323 T
* r
= reinterpret_cast<T
*>(new char[total
]);
327 void deallocate(T
* p
, size_t n
) {
328 size_t total
= sizeof(T
) * n
;
329 shard_t
*shard
= pool
->pick_a_shard();
330 shard
->bytes
-= total
;
335 delete[] reinterpret_cast<char*>(p
);
338 T
* allocate_aligned(size_t n
, size_t align
, void *p
= nullptr) {
339 size_t total
= sizeof(T
) * n
;
340 shard_t
*shard
= pool
->pick_a_shard();
341 shard
->bytes
+= total
;
347 int rc
= ::posix_memalign((void**)(void*)&ptr
, align
, total
);
349 throw std::bad_alloc();
350 T
* r
= reinterpret_cast<T
*>(ptr
);
354 void deallocate_aligned(T
* p
, size_t n
) {
355 size_t total
= sizeof(T
) * n
;
356 shard_t
*shard
= pool
->pick_a_shard();
357 shard
->bytes
-= total
;
374 void construct(T
* p
, const T
& val
) {
375 ::new ((void *)p
) T(val
);
378 template<class U
, class... Args
> void construct(U
* p
,Args
&&... args
) {
379 ::new((void *)p
) U(std::forward
<Args
>(args
)...);
382 bool operator==(const pool_allocator
&) const { return true; }
383 bool operator!=(const pool_allocator
&) const { return false; }
391 static const mempool::pool_index_t id = mempool::mempool_##x; \
392 template<typename v> \
393 using pool_allocator = mempool::pool_allocator<id,v>; \
395 using string = std::basic_string<char,std::char_traits<char>, \
396 pool_allocator<char>>; \
398 template<typename k,typename v, typename cmp = std::less<k> > \
399 using map = std::map<k, v, cmp, \
400 pool_allocator<std::pair<const k,v>>>; \
402 template<typename k,typename v, typename cmp = std::less<k> > \
403 using compact_map = compact_map<k, v, cmp, \
404 pool_allocator<std::pair<const k,v>>>; \
406 template<typename k, typename cmp = std::less<k> > \
407 using compact_set = compact_set<k, cmp, pool_allocator<k>>; \
409 template<typename k,typename v, typename cmp = std::less<k> > \
410 using multimap = std::multimap<k,v,cmp, \
411 pool_allocator<std::pair<const k, \
414 template<typename k, typename cmp = std::less<k> > \
415 using set = std::set<k,cmp,pool_allocator<k>>; \
417 template<typename v> \
418 using list = std::list<v,pool_allocator<v>>; \
420 template<typename v> \
421 using vector = std::vector<v,pool_allocator<v>>; \
423 template<typename k, typename v, \
424 typename h=std::hash<k>, \
425 typename eq = std::equal_to<k>> \
426 using unordered_map = \
427 std::unordered_map<k,v,h,eq,pool_allocator<std::pair<const k,v>>>;\
429 inline size_t allocated_bytes() { \
430 return mempool::get_pool(id).allocated_bytes(); \
432 inline size_t allocated_items() { \
433 return mempool::get_pool(id).allocated_items(); \
437 DEFINE_MEMORY_POOLS_HELPER(P
)
445 // Use this for any type that is contained by a container (unless it
446 // is a class you defined; see below).
447 #define MEMPOOL_DECLARE_FACTORY(obj, factoryname, pool) \
448 namespace mempool { \
450 extern pool_allocator<obj> alloc_##factoryname; \
454 #define MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool) \
455 namespace mempool { \
457 pool_allocator<obj> alloc_##factoryname = {true}; \
461 // Use this for each class that belongs to a mempool. For example,
464 // MEMPOOL_CLASS_HELPERS();
468 #define MEMPOOL_CLASS_HELPERS() \
469 void *operator new(size_t size); \
470 void *operator new[](size_t size) noexcept { \
471 assert(0 == "no array new"); \
473 void operator delete(void *); \
474 void operator delete[](void *) { assert(0 == "no array delete"); }
477 // Use this in some particular .cc file to match each class with a
478 // MEMPOOL_CLASS_HELPERS().
479 #define MEMPOOL_DEFINE_OBJECT_FACTORY(obj,factoryname,pool) \
480 MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool) \
481 void *obj::operator new(size_t size) { \
482 return mempool::pool::alloc_##factoryname.allocate(1); \
484 void obj::operator delete(void *p) { \
485 return mempool::pool::alloc_##factoryname.deallocate((obj*)p, 1); \