]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2016 Allen Samuels <allen.samuels@sandisk.com> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | ||
15 | #ifndef _CEPH_INCLUDE_MEMPOOL_H | |
16 | #define _CEPH_INCLUDE_MEMPOOL_H | |
17 | ||
18 | #include <cstddef> | |
19 | #include <map> | |
20 | #include <unordered_map> | |
21 | #include <set> | |
22 | #include <vector> | |
23 | #include <list> | |
24 | #include <mutex> | |
7c673cae | 25 | #include <typeinfo> |
11fdf7f2 TL |
26 | #include <boost/container/flat_set.hpp> |
27 | #include <boost/container/flat_map.hpp> | |
7c673cae | 28 | |
9f95a23c TL |
29 | #include "common/Formatter.h" |
30 | #include "common/ceph_atomic.h" | |
11fdf7f2 | 31 | #include "include/ceph_assert.h" |
94b18763 FG |
32 | #include "include/compact_map.h" |
33 | #include "include/compact_set.h" | |
f67539c2 | 34 | #include "include/compat.h" |
7c673cae FG |
35 | |
36 | ||
37 | /* | |
38 | ||
39 | Memory Pools | |
40 | ============ | |
41 | ||
42 | A memory pool is a method for accounting the consumption of memory of | |
43 | a set of containers. | |
44 | ||
45 | Memory pools are statically declared (see pool_index_t). | |
46 | ||
47 | Each memory pool tracks the number of bytes and items it contains. | |
48 | ||
49 | Allocators can be declared and associated with a type so that they are | |
50 | tracked independently of the pool total. This additional accounting | |
51 | is optional and only incurs an overhead if the debugging is enabled at | |
52 | runtime. This allows developers to see what types are consuming the | |
53 | pool resources. | |
54 | ||
55 | ||
56 | Declaring | |
57 | --------- | |
58 | ||
59 | Using memory pools is very easy. | |
60 | ||
61 | To create a new memory pool, simply add a new name into the list of | |
62 | memory pools that's defined in "DEFINE_MEMORY_POOLS_HELPER". That's | |
63 | it. :) | |
64 | ||
65 | For each memory pool that's created a C++ namespace is also | |
66 | automatically created (name is same as in DEFINE_MEMORY_POOLS_HELPER). | |
67 | That namespace contains a set of common STL containers that are predefined | |
68 | with the appropriate allocators. | |
69 | ||
31f18b77 | 70 | Thus for mempool "osd" we have automatically available to us: |
7c673cae | 71 | |
31f18b77 FG |
72 | mempool::osd::map |
73 | mempool::osd::multimap | |
74 | mempool::osd::set | |
75 | mempool::osd::multiset | |
76 | mempool::osd::list | |
77 | mempool::osd::vector | |
78 | mempool::osd::unordered_map | |
7c673cae FG |
79 | |
80 | ||
81 | Putting objects in a mempool | |
82 | ---------------------------- | |
83 | ||
84 | In order to use a memory pool with a particular type, a few additional | |
85 | declarations are needed. | |
86 | ||
87 | For a class: | |
88 | ||
89 | struct Foo { | |
90 | MEMPOOL_CLASS_HELPERS(); | |
91 | ... | |
92 | }; | |
93 | ||
94 | Then, in an appropriate .cc file, | |
95 | ||
31f18b77 | 96 | MEMPOOL_DEFINE_OBJECT_FACTORY(Foo, foo, osd); |
7c673cae FG |
97 | |
98 | The second argument can generally be identical to the first, except | |
99 | when the type contains a nested scope. For example, for | |
100 | BlueStore::Onode, we need to do | |
101 | ||
102 | MEMPOOL_DEFINE_OBJECT_FACTORY(BlueStore::Onode, bluestore_onode, | |
103 | bluestore_meta); | |
104 | ||
105 | (This is just because we need to name some static variables and we | |
106 | can't use :: in a variable name.) | |
107 | ||
181888fb FG |
108 | XXX Note: the new operator hard-codes the allocation size to the size of the |
109 | object given in MEMPOOL_DEFINE_OBJECT_FACTORY. For this reason, you cannot | |
110 | incorporate mempools into a base class without also defining a helper/factory | |
111 | for the child class as well (as the base class is usually smaller than the | |
112 | child class). | |
113 | ||
7c673cae FG |
114 | In order to use the STL containers, simply use the namespaced variant |
115 | of the container type. For example, | |
116 | ||
31f18b77 | 117 | mempool::osd::map<int> myvec; |
7c673cae FG |
118 | |
119 | Introspection | |
120 | ------------- | |
121 | ||
122 | The simplest way to interrogate the process is with | |
123 | ||
124 | Formater *f = ... | |
125 | mempool::dump(f); | |
126 | ||
127 | This will dump information about *all* memory pools. When debug mode | |
128 | is enabled, the runtime complexity of dump is O(num_shards * | |
129 | num_types). When debug name is disabled it is O(num_shards). | |
130 | ||
131 | You can also interrogate a specific pool programmatically with | |
132 | ||
133 | size_t bytes = mempool::unittest_2::allocated_bytes(); | |
134 | size_t items = mempool::unittest_2::allocated_items(); | |
135 | ||
136 | The runtime complexity is O(num_shards). | |
137 | ||
138 | Note that you cannot easily query per-type, primarily because debug | |
139 | mode is optional and you should not rely on that information being | |
140 | available. | |
141 | ||
142 | */ | |
143 | ||
144 | namespace mempool { | |
145 | ||
146 | // -------------------------------------------------------------- | |
147 | // define memory pools | |
148 | ||
149 | #define DEFINE_MEMORY_POOLS_HELPER(f) \ | |
150 | f(bloom_filter) \ | |
7c673cae | 151 | f(bluestore_alloc) \ |
31f18b77 FG |
152 | f(bluestore_cache_data) \ |
153 | f(bluestore_cache_onode) \ | |
f91f0fd5 | 154 | f(bluestore_cache_meta) \ |
31f18b77 | 155 | f(bluestore_cache_other) \ |
f91f0fd5 TL |
156 | f(bluestore_Buffer) \ |
157 | f(bluestore_Extent) \ | |
158 | f(bluestore_Blob) \ | |
159 | f(bluestore_SharedBlob) \ | |
160 | f(bluestore_inline_bl) \ | |
7c673cae | 161 | f(bluestore_fsck) \ |
31f18b77 | 162 | f(bluestore_txc) \ |
f91f0fd5 | 163 | f(bluestore_writing_deferred) \ |
31f18b77 | 164 | f(bluestore_writing) \ |
7c673cae | 165 | f(bluefs) \ |
f91f0fd5 TL |
166 | f(bluefs_file_reader) \ |
167 | f(bluefs_file_writer) \ | |
31f18b77 | 168 | f(buffer_anon) \ |
7c673cae | 169 | f(buffer_meta) \ |
7c673cae | 170 | f(osd) \ |
31f18b77 FG |
171 | f(osd_mapbl) \ |
172 | f(osd_pglog) \ | |
7c673cae FG |
173 | f(osdmap) \ |
174 | f(osdmap_mapping) \ | |
31f18b77 | 175 | f(pgmap) \ |
181888fb | 176 | f(mds_co) \ |
7c673cae FG |
177 | f(unittest_1) \ |
178 | f(unittest_2) | |
179 | ||
180 | ||
181 | // give them integer ids | |
182 | #define P(x) mempool_##x, | |
183 | enum pool_index_t { | |
184 | DEFINE_MEMORY_POOLS_HELPER(P) | |
185 | num_pools // Must be last. | |
186 | }; | |
187 | #undef P | |
188 | ||
189 | extern bool debug_mode; | |
190 | extern void set_debug_mode(bool d); | |
191 | ||
192 | // -------------------------------------------------------------- | |
193 | class pool_t; | |
194 | ||
195 | // we shard pool stats across many shard_t's to reduce the amount | |
196 | // of cacheline ping pong. | |
197 | enum { | |
198 | num_shard_bits = 5 | |
199 | }; | |
200 | enum { | |
201 | num_shards = 1 << num_shard_bits | |
202 | }; | |
203 | ||
204 | // align shard to a cacheline | |
205 | struct shard_t { | |
9f95a23c TL |
206 | ceph::atomic<size_t> bytes = {0}; |
207 | ceph::atomic<size_t> items = {0}; | |
208 | char __padding[128 - sizeof(ceph::atomic<size_t>)*2]; | |
7c673cae FG |
209 | } __attribute__ ((aligned (128))); |
210 | ||
211 | static_assert(sizeof(shard_t) == 128, "shard_t should be cacheline-sized"); | |
212 | ||
213 | struct stats_t { | |
214 | ssize_t items = 0; | |
215 | ssize_t bytes = 0; | |
216 | void dump(ceph::Formatter *f) const { | |
217 | f->dump_int("items", items); | |
218 | f->dump_int("bytes", bytes); | |
219 | } | |
31f18b77 FG |
220 | |
221 | stats_t& operator+=(const stats_t& o) { | |
222 | items += o.items; | |
223 | bytes += o.bytes; | |
224 | return *this; | |
225 | } | |
7c673cae FG |
226 | }; |
227 | ||
228 | pool_t& get_pool(pool_index_t ix); | |
229 | const char *get_pool_name(pool_index_t ix); | |
230 | ||
231 | struct type_t { | |
232 | const char *type_name; | |
233 | size_t item_size; | |
9f95a23c | 234 | ceph::atomic<ssize_t> items = {0}; // signed |
7c673cae FG |
235 | }; |
236 | ||
237 | struct type_info_hash { | |
238 | std::size_t operator()(const std::type_info& k) const { | |
239 | return k.hash_code(); | |
240 | } | |
241 | }; | |
242 | ||
243 | class pool_t { | |
244 | shard_t shard[num_shards]; | |
245 | ||
246 | mutable std::mutex lock; // only used for types list | |
247 | std::unordered_map<const char *, type_t> type_map; | |
248 | ||
249 | public: | |
250 | // | |
251 | // How much this pool consumes. O(<num_shards>) | |
252 | // | |
253 | size_t allocated_bytes() const; | |
254 | size_t allocated_items() const; | |
255 | ||
31f18b77 FG |
256 | void adjust_count(ssize_t items, ssize_t bytes); |
257 | ||
f67539c2 | 258 | static size_t pick_a_shard_int() { |
7c673cae | 259 | // Dirt cheap, see: |
f67539c2 | 260 | // https://fossies.org/dox/glibc-2.32/pthread__self_8c_source.html |
7c673cae | 261 | size_t me = (size_t)pthread_self(); |
f67539c2 TL |
262 | size_t i = (me >> 12) & ((1 << num_shard_bits) - 1); |
263 | return i; | |
264 | } | |
265 | ||
266 | shard_t* pick_a_shard() { | |
267 | size_t i = pick_a_shard_int(); | |
7c673cae FG |
268 | return &shard[i]; |
269 | } | |
270 | ||
271 | type_t *get_type(const std::type_info& ti, size_t size) { | |
272 | std::lock_guard<std::mutex> l(lock); | |
273 | auto p = type_map.find(ti.name()); | |
274 | if (p != type_map.end()) { | |
275 | return &p->second; | |
276 | } | |
277 | type_t &t = type_map[ti.name()]; | |
278 | t.type_name = ti.name(); | |
279 | t.item_size = size; | |
280 | return &t; | |
281 | } | |
282 | ||
283 | // get pool stats. by_type is not populated if !debug | |
284 | void get_stats(stats_t *total, | |
285 | std::map<std::string, stats_t> *by_type) const; | |
286 | ||
31f18b77 | 287 | void dump(ceph::Formatter *f, stats_t *ptotal=0) const; |
7c673cae FG |
288 | }; |
289 | ||
290 | void dump(ceph::Formatter *f); | |
291 | ||
292 | ||
293 | // STL allocator for use with containers. All actual state | |
294 | // is stored in the static pool_allocator_base_t, which saves us from | |
295 | // passing the allocator to container constructors. | |
296 | ||
297 | template<pool_index_t pool_ix, typename T> | |
298 | class pool_allocator { | |
299 | pool_t *pool; | |
300 | type_t *type = nullptr; | |
301 | ||
302 | public: | |
303 | typedef pool_allocator<pool_ix, T> allocator_type; | |
304 | typedef T value_type; | |
305 | typedef value_type *pointer; | |
306 | typedef const value_type * const_pointer; | |
307 | typedef value_type& reference; | |
308 | typedef const value_type& const_reference; | |
309 | typedef std::size_t size_type; | |
310 | typedef std::ptrdiff_t difference_type; | |
311 | ||
312 | template<typename U> struct rebind { | |
313 | typedef pool_allocator<pool_ix,U> other; | |
314 | }; | |
315 | ||
316 | void init(bool force_register) { | |
317 | pool = &get_pool(pool_ix); | |
318 | if (debug_mode || force_register) { | |
319 | type = pool->get_type(typeid(T), sizeof(T)); | |
320 | } | |
321 | } | |
322 | ||
323 | pool_allocator(bool force_register=false) { | |
324 | init(force_register); | |
325 | } | |
326 | template<typename U> | |
327 | pool_allocator(const pool_allocator<pool_ix,U>&) { | |
328 | init(false); | |
329 | } | |
330 | ||
331 | T* allocate(size_t n, void *p = nullptr) { | |
332 | size_t total = sizeof(T) * n; | |
333 | shard_t *shard = pool->pick_a_shard(); | |
334 | shard->bytes += total; | |
335 | shard->items += n; | |
336 | if (type) { | |
337 | type->items += n; | |
338 | } | |
339 | T* r = reinterpret_cast<T*>(new char[total]); | |
340 | return r; | |
341 | } | |
342 | ||
343 | void deallocate(T* p, size_t n) { | |
344 | size_t total = sizeof(T) * n; | |
345 | shard_t *shard = pool->pick_a_shard(); | |
346 | shard->bytes -= total; | |
347 | shard->items -= n; | |
348 | if (type) { | |
349 | type->items -= n; | |
350 | } | |
351 | delete[] reinterpret_cast<char*>(p); | |
352 | } | |
353 | ||
354 | T* allocate_aligned(size_t n, size_t align, void *p = nullptr) { | |
355 | size_t total = sizeof(T) * n; | |
356 | shard_t *shard = pool->pick_a_shard(); | |
357 | shard->bytes += total; | |
358 | shard->items += n; | |
359 | if (type) { | |
360 | type->items += n; | |
361 | } | |
362 | char *ptr; | |
363 | int rc = ::posix_memalign((void**)(void*)&ptr, align, total); | |
364 | if (rc) | |
365 | throw std::bad_alloc(); | |
366 | T* r = reinterpret_cast<T*>(ptr); | |
367 | return r; | |
368 | } | |
369 | ||
370 | void deallocate_aligned(T* p, size_t n) { | |
371 | size_t total = sizeof(T) * n; | |
372 | shard_t *shard = pool->pick_a_shard(); | |
373 | shard->bytes -= total; | |
374 | shard->items -= n; | |
375 | if (type) { | |
376 | type->items -= n; | |
377 | } | |
f67539c2 | 378 | aligned_free(p); |
7c673cae FG |
379 | } |
380 | ||
381 | void destroy(T* p) { | |
382 | p->~T(); | |
383 | } | |
384 | ||
385 | template<class U> | |
386 | void destroy(U *p) { | |
387 | p->~U(); | |
388 | } | |
389 | ||
390 | void construct(T* p, const T& val) { | |
391 | ::new ((void *)p) T(val); | |
392 | } | |
393 | ||
394 | template<class U, class... Args> void construct(U* p,Args&&... args) { | |
395 | ::new((void *)p) U(std::forward<Args>(args)...); | |
396 | } | |
397 | ||
398 | bool operator==(const pool_allocator&) const { return true; } | |
399 | bool operator!=(const pool_allocator&) const { return false; } | |
400 | }; | |
401 | ||
402 | ||
403 | // Namespace mempool | |
404 | ||
405 | #define P(x) \ | |
406 | namespace x { \ | |
407 | static const mempool::pool_index_t id = mempool::mempool_##x; \ | |
408 | template<typename v> \ | |
409 | using pool_allocator = mempool::pool_allocator<id,v>; \ | |
410 | \ | |
411 | using string = std::basic_string<char,std::char_traits<char>, \ | |
412 | pool_allocator<char>>; \ | |
413 | \ | |
414 | template<typename k,typename v, typename cmp = std::less<k> > \ | |
415 | using map = std::map<k, v, cmp, \ | |
416 | pool_allocator<std::pair<const k,v>>>; \ | |
417 | \ | |
94b18763 FG |
418 | template<typename k,typename v, typename cmp = std::less<k> > \ |
419 | using compact_map = compact_map<k, v, cmp, \ | |
420 | pool_allocator<std::pair<const k,v>>>; \ | |
421 | \ | |
11fdf7f2 TL |
422 | template<typename k,typename v, typename cmp = std::less<k> > \ |
423 | using compact_multimap = compact_multimap<k, v, cmp, \ | |
424 | pool_allocator<std::pair<const k,v>>>; \ | |
425 | \ | |
94b18763 FG |
426 | template<typename k, typename cmp = std::less<k> > \ |
427 | using compact_set = compact_set<k, cmp, pool_allocator<k>>; \ | |
428 | \ | |
7c673cae FG |
429 | template<typename k,typename v, typename cmp = std::less<k> > \ |
430 | using multimap = std::multimap<k,v,cmp, \ | |
431 | pool_allocator<std::pair<const k, \ | |
432 | v>>>; \ | |
433 | \ | |
434 | template<typename k, typename cmp = std::less<k> > \ | |
435 | using set = std::set<k,cmp,pool_allocator<k>>; \ | |
436 | \ | |
11fdf7f2 TL |
437 | template<typename k, typename cmp = std::less<k> > \ |
438 | using flat_set = boost::container::flat_set<k,cmp,pool_allocator<k>>; \ | |
439 | \ | |
440 | template<typename k, typename v, typename cmp = std::less<k> > \ | |
441 | using flat_map = boost::container::flat_map<k,v,cmp, \ | |
442 | pool_allocator<std::pair<k,v>>>; \ | |
443 | \ | |
7c673cae FG |
444 | template<typename v> \ |
445 | using list = std::list<v,pool_allocator<v>>; \ | |
446 | \ | |
447 | template<typename v> \ | |
448 | using vector = std::vector<v,pool_allocator<v>>; \ | |
449 | \ | |
450 | template<typename k, typename v, \ | |
451 | typename h=std::hash<k>, \ | |
452 | typename eq = std::equal_to<k>> \ | |
453 | using unordered_map = \ | |
454 | std::unordered_map<k,v,h,eq,pool_allocator<std::pair<const k,v>>>;\ | |
455 | \ | |
456 | inline size_t allocated_bytes() { \ | |
457 | return mempool::get_pool(id).allocated_bytes(); \ | |
458 | } \ | |
459 | inline size_t allocated_items() { \ | |
460 | return mempool::get_pool(id).allocated_items(); \ | |
461 | } \ | |
462 | }; | |
463 | ||
464 | DEFINE_MEMORY_POOLS_HELPER(P) | |
465 | ||
466 | #undef P | |
467 | ||
468 | }; | |
469 | ||
11fdf7f2 TL |
470 | // the elements allocated by mempool is in the same memory space as the ones |
471 | // allocated by the default allocator. so compare them in an efficient way: | |
472 | // libstdc++'s std::equal is specialized to use memcmp if T is integer or | |
473 | // pointer. this is good enough for our usecase. use | |
474 | // std::is_trivially_copyable<T> to expand the support to more types if | |
475 | // nececssary. | |
476 | template<typename T, mempool::pool_index_t pool_index> | |
477 | bool operator==(const std::vector<T, std::allocator<T>>& lhs, | |
478 | const std::vector<T, mempool::pool_allocator<pool_index, T>>& rhs) | |
479 | { | |
480 | return (lhs.size() == rhs.size() && | |
481 | std::equal(lhs.begin(), lhs.end(), rhs.begin())); | |
482 | } | |
483 | ||
484 | template<typename T, mempool::pool_index_t pool_index> | |
485 | bool operator!=(const std::vector<T, std::allocator<T>>& lhs, | |
486 | const std::vector<T, mempool::pool_allocator<pool_index, T>>& rhs) | |
487 | { | |
488 | return !(lhs == rhs); | |
489 | } | |
490 | ||
491 | template<typename T, mempool::pool_index_t pool_index> | |
492 | bool operator==(const std::vector<T, mempool::pool_allocator<pool_index, T>>& lhs, | |
493 | const std::vector<T, std::allocator<T>>& rhs) | |
494 | { | |
495 | return rhs == lhs; | |
496 | } | |
497 | ||
498 | template<typename T, mempool::pool_index_t pool_index> | |
499 | bool operator!=(const std::vector<T, mempool::pool_allocator<pool_index, T>>& lhs, | |
500 | const std::vector<T, std::allocator<T>>& rhs) | |
501 | { | |
502 | return !(lhs == rhs); | |
503 | } | |
7c673cae FG |
504 | |
505 | // Use this for any type that is contained by a container (unless it | |
506 | // is a class you defined; see below). | |
507 | #define MEMPOOL_DECLARE_FACTORY(obj, factoryname, pool) \ | |
508 | namespace mempool { \ | |
509 | namespace pool { \ | |
510 | extern pool_allocator<obj> alloc_##factoryname; \ | |
511 | } \ | |
512 | } | |
513 | ||
514 | #define MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool) \ | |
515 | namespace mempool { \ | |
516 | namespace pool { \ | |
517 | pool_allocator<obj> alloc_##factoryname = {true}; \ | |
518 | } \ | |
519 | } | |
520 | ||
521 | // Use this for each class that belongs to a mempool. For example, | |
522 | // | |
523 | // class T { | |
524 | // MEMPOOL_CLASS_HELPERS(); | |
525 | // ... | |
526 | // }; | |
527 | // | |
528 | #define MEMPOOL_CLASS_HELPERS() \ | |
529 | void *operator new(size_t size); \ | |
530 | void *operator new[](size_t size) noexcept { \ | |
11fdf7f2 | 531 | ceph_abort_msg("no array new"); \ |
7c673cae FG |
532 | return nullptr; } \ |
533 | void operator delete(void *); \ | |
11fdf7f2 | 534 | void operator delete[](void *) { ceph_abort_msg("no array delete"); } |
7c673cae FG |
535 | |
536 | ||
537 | // Use this in some particular .cc file to match each class with a | |
538 | // MEMPOOL_CLASS_HELPERS(). | |
539 | #define MEMPOOL_DEFINE_OBJECT_FACTORY(obj,factoryname,pool) \ | |
540 | MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool) \ | |
541 | void *obj::operator new(size_t size) { \ | |
542 | return mempool::pool::alloc_##factoryname.allocate(1); \ | |
543 | } \ | |
544 | void obj::operator delete(void *p) { \ | |
545 | return mempool::pool::alloc_##factoryname.deallocate((obj*)p, 1); \ | |
546 | } | |
547 | ||
548 | #endif |