[ceph.git] / ceph / src / include / mempool.h

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2016 Allen Samuels <allen.samuels@sandisk.com>
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#ifndef _CEPH_INCLUDE_MEMPOOL_H
#define _CEPH_INCLUDE_MEMPOOL_H

#include <cstddef>
#include <map>
#include <unordered_map>
#include <set>
#include <vector>
#include <list>
#include <mutex>
#include <atomic>
#include <typeinfo>

#include <common/Formatter.h>
#include "include/assert.h"


/*

Memory Pools
============

A memory pool is a method for accounting the consumption of memory of
a set of containers.

Memory pools are statically declared (see pool_index_t).

Each memory pool tracks the number of bytes and items it contains.

Allocators can be declared and associated with a type so that they are
tracked independently of the pool total.  This additional accounting
is optional and only incurs an overhead if the debugging is enabled at
runtime.  This allows developers to see what types are consuming the
pool resources.


Declaring
---------

Using memory pools is very easy.

To create a new memory pool, simply add a new name into the list of
memory pools that's defined in "DEFINE_MEMORY_POOLS_HELPER".  That's
it.  :)

For each memory pool that's created a C++ namespace is also
automatically created (name is same as in DEFINE_MEMORY_POOLS_HELPER).
That namespace contains a set of common STL containers that are predefined
with the appropriate allocators.

Thus for mempool "osd" we have automatically available to us:

   mempool::osd::map
   mempool::osd::multimap
   mempool::osd::set
   mempool::osd::multiset
   mempool::osd::list
   mempool::osd::vector
   mempool::osd::unordered_map


Putting objects in a mempool
----------------------------

In order to use a memory pool with a particular type, a few additional
declarations are needed.

For a class:

  struct Foo {
    MEMPOOL_CLASS_HELPERS();
    ...
  };

Then, in an appropriate .cc file,

  MEMPOOL_DEFINE_OBJECT_FACTORY(Foo, foo, osd);

The second argument can generally be identical to the first, except
when the type contains a nested scope.  For example, for
BlueStore::Onode, we need to do

  MEMPOOL_DEFINE_OBJECT_FACTORY(BlueStore::Onode, bluestore_onode,
                                bluestore_meta);

(This is just because we need to name some static variables and we
can't use :: in a variable name.)

In order to use the STL containers, simply use the namespaced variant
of the container type.  For example,

  mempool::osd::map<int> myvec;

Introspection
-------------

The simplest way to interrogate the process is with

  Formater *f = ...
  mempool::dump(f);

This will dump information about *all* memory pools.  When debug mode
is enabled, the runtime complexity of dump is O(num_shards *
num_types).  When debug name is disabled it is O(num_shards).

You can also interrogate a specific pool programmatically with

  size_t bytes = mempool::unittest_2::allocated_bytes();
  size_t items = mempool::unittest_2::allocated_items();

The runtime complexity is O(num_shards).

Note that you cannot easily query per-type, primarily because debug
mode is optional and you should not rely on that information being
available.

*/

namespace mempool {

// --------------------------------------------------------------
// define memory pools

#define DEFINE_MEMORY_POOLS_HELPER(f) \
  f(bloom_filter)		      \
  f(bluestore_alloc)		      \
  f(bluestore_cache_data)	      \
  f(bluestore_cache_onode)	      \
  f(bluestore_cache_other)	      \
  f(bluestore_fsck)		      \
  f(bluestore_txc)		      \
  f(bluestore_writing_deferred)	      \
  f(bluestore_writing)		      \
  f(bluefs)			      \
  f(buffer_anon)		      \
  f(buffer_meta)		      \
  f(osd)			      \
  f(osd_mapbl)			      \
  f(osd_pglog)			      \
  f(osdmap)			      \
  f(osdmap_mapping)		      \
  f(pgmap)			      \
  f(unittest_1)			      \
  f(unittest_2)


// give them integer ids
#define P(x) mempool_##x,
enum pool_index_t {
  DEFINE_MEMORY_POOLS_HELPER(P)
  num_pools        // Must be last.
};
#undef P

extern bool debug_mode;
extern void set_debug_mode(bool d);

// --------------------------------------------------------------
class pool_t;

// we shard pool stats across many shard_t's to reduce the amount
// of cacheline ping pong.
enum {
  num_shard_bits = 5
};
enum {
  num_shards = 1 << num_shard_bits
};

// align shard to a cacheline
struct shard_t {
  std::atomic<size_t> bytes = {0};
  std::atomic<size_t> items = {0};
  char __padding[128 - sizeof(std::atomic<size_t>)*2];
} __attribute__ ((aligned (128)));

static_assert(sizeof(shard_t) == 128, "shard_t should be cacheline-sized");

struct stats_t {
  ssize_t items = 0;
  ssize_t bytes = 0;
  void dump(ceph::Formatter *f) const {
    f->dump_int("items", items);
    f->dump_int("bytes", bytes);
  }

  stats_t& operator+=(const stats_t& o) {
    items += o.items;
    bytes += o.bytes;
    return *this;
  }
};

pool_t& get_pool(pool_index_t ix);
const char *get_pool_name(pool_index_t ix);

struct type_t {
  const char *type_name;
  size_t item_size;
  std::atomic<ssize_t> items = {0};  // signed
};

struct type_info_hash {
  std::size_t operator()(const std::type_info& k) const {
    return k.hash_code();
  }
};

class pool_t {
  shard_t shard[num_shards];

  mutable std::mutex lock;  // only used for types list
  std::unordered_map<const char *, type_t> type_map;

public:
  //
  // How much this pool consumes. O(<num_shards>)
  //
  size_t allocated_bytes() const;
  size_t allocated_items() const;

  void adjust_count(ssize_t items, ssize_t bytes);

  shard_t* pick_a_shard() {
    // Dirt cheap, see:
    //   http://fossies.org/dox/glibc-2.24/pthread__self_8c_source.html
    size_t me = (size_t)pthread_self();
    size_t i = (me >> 3) & ((1 << num_shard_bits) - 1);
    return &shard[i];
  }

  type_t *get_type(const std::type_info& ti, size_t size) {
    std::lock_guard<std::mutex> l(lock);
    auto p = type_map.find(ti.name());
    if (p != type_map.end()) {
      return &p->second;
    }
    type_t &t = type_map[ti.name()];
    t.type_name = ti.name();
    t.item_size = size;
    return &t;
  }

  // get pool stats.  by_type is not populated if !debug
  void get_stats(stats_t *total,
		 std::map<std::string, stats_t> *by_type) const;

  void dump(ceph::Formatter *f, stats_t *ptotal=0) const;
};

void dump(ceph::Formatter *f);


// STL allocator for use with containers.  All actual state
// is stored in the static pool_allocator_base_t, which saves us from
// passing the allocator to container constructors.

template<pool_index_t pool_ix, typename T>
class pool_allocator {
  pool_t *pool;
  type_t *type = nullptr;

public:
  typedef pool_allocator<pool_ix, T> allocator_type;
  typedef T value_type;
  typedef value_type *pointer;
  typedef const value_type * const_pointer;
  typedef value_type& reference;
  typedef const value_type& const_reference;
  typedef std::size_t size_type;
  typedef std::ptrdiff_t difference_type;

  template<typename U> struct rebind {
    typedef pool_allocator<pool_ix,U> other;
  };

  void init(bool force_register) {
    pool = &get_pool(pool_ix);
    if (debug_mode || force_register) {
      type = pool->get_type(typeid(T), sizeof(T));
    }
  }

  pool_allocator(bool force_register=false) {
    init(force_register);
  }
  template<typename U>
  pool_allocator(const pool_allocator<pool_ix,U>&) {
    init(false);
  }

  T* allocate(size_t n, void *p = nullptr) {
    size_t total = sizeof(T) * n;
    shard_t *shard = pool->pick_a_shard();
    shard->bytes += total;
    shard->items += n;
    if (type) {
      type->items += n;
    }
    T* r = reinterpret_cast<T*>(new char[total]);
    return r;
  }

  void deallocate(T* p, size_t n) {
    size_t total = sizeof(T) * n;
    shard_t *shard = pool->pick_a_shard();
    shard->bytes -= total;
    shard->items -= n;
    if (type) {
      type->items -= n;
    }
    delete[] reinterpret_cast<char*>(p);
  }

  T* allocate_aligned(size_t n, size_t align, void *p = nullptr) {
    size_t total = sizeof(T) * n;
    shard_t *shard = pool->pick_a_shard();
    shard->bytes += total;
    shard->items += n;
    if (type) {
      type->items += n;
    }
    char *ptr;
    int rc = ::posix_memalign((void**)(void*)&ptr, align, total);
    if (rc)
      throw std::bad_alloc();
    T* r = reinterpret_cast<T*>(ptr);
    return r;
  }

  void deallocate_aligned(T* p, size_t n) {
    size_t total = sizeof(T) * n;
    shard_t *shard = pool->pick_a_shard();
    shard->bytes -= total;
    shard->items -= n;
    if (type) {
      type->items -= n;
    }
    ::free(p);
  }

  void destroy(T* p) {
    p->~T();
  }

  template<class U>
  void destroy(U *p) {
    p->~U();
  }

  void construct(T* p, const T& val) {
    ::new ((void *)p) T(val);
  }

  template<class U, class... Args> void construct(U* p,Args&&... args) {
    ::new((void *)p) U(std::forward<Args>(args)...);
  }

  bool operator==(const pool_allocator&) const { return true; }
  bool operator!=(const pool_allocator&) const { return false; }
};


// Namespace mempool

#define P(x)								\
  namespace x {								\
    static const mempool::pool_index_t id = mempool::mempool_##x;	\
    template<typename v>						\
    using pool_allocator = mempool::pool_allocator<id,v>;		\
                                                                        \
    using string = std::basic_string<char,std::char_traits<char>,       \
                                     pool_allocator<char>>;             \
                                                                        \
    template<typename k,typename v, typename cmp = std::less<k> >	\
    using map = std::map<k, v, cmp,					\
			 pool_allocator<std::pair<const k,v>>>;		\
                                                                        \
    template<typename k,typename v, typename cmp = std::less<k> >	\
    using multimap = std::multimap<k,v,cmp,				\
				   pool_allocator<std::pair<const k,	\
							    v>>>;	\
                                                                        \
    template<typename k, typename cmp = std::less<k> >			\
    using set = std::set<k,cmp,pool_allocator<k>>;			\
                                                                        \
    template<typename v>						\
    using list = std::list<v,pool_allocator<v>>;			\
                                                                        \
    template<typename v>						\
    using vector = std::vector<v,pool_allocator<v>>;			\
                                                                        \
    template<typename k, typename v,					\
	     typename h=std::hash<k>,					\
	     typename eq = std::equal_to<k>>				\
    using unordered_map =						\
      std::unordered_map<k,v,h,eq,pool_allocator<std::pair<const k,v>>>;\
                                                                        \
    inline size_t allocated_bytes() {					\
      return mempool::get_pool(id).allocated_bytes();			\
    }									\
    inline size_t allocated_items() {					\
      return mempool::get_pool(id).allocated_items();			\
    }									\
  };

DEFINE_MEMORY_POOLS_HELPER(P)

#undef P

};


// Use this for any type that is contained by a container (unless it
// is a class you defined; see below).
#define MEMPOOL_DECLARE_FACTORY(obj, factoryname, pool)			\
  namespace mempool {							\
    namespace pool {							\
      extern pool_allocator<obj> alloc_##factoryname;			\
    }									\
  }

#define MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool)			\
  namespace mempool {							\
    namespace pool {							\
      pool_allocator<obj> alloc_##factoryname = {true};			\
    }									\
  }

// Use this for each class that belongs to a mempool.  For example,
//
//   class T {
//     MEMPOOL_CLASS_HELPERS();
//     ...
//   };
//
#define MEMPOOL_CLASS_HELPERS()						\
  void *operator new(size_t size);					\
  void *operator new[](size_t size) noexcept {				\
    assert(0 == "no array new");					\
    return nullptr; }							\
  void  operator delete(void *);					\
  void  operator delete[](void *) { assert(0 == "no array delete"); }


// Use this in some particular .cc file to match each class with a
// MEMPOOL_CLASS_HELPERS().
#define MEMPOOL_DEFINE_OBJECT_FACTORY(obj,factoryname,pool)		\
  MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool)			\
  void *obj::operator new(size_t size) {				\
    return mempool::pool::alloc_##factoryname.allocate(1); \
  }									\
  void obj::operator delete(void *p)  {					\
    return mempool::pool::alloc_##factoryname.deallocate((obj*)p, 1);	\
  }

#endif
Commit	Line	Data
7c673cae FG	1	// -- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t --
	2	// vim: ts=8 sw=2 smarttab
	3	/*
	4	* Ceph - scalable distributed file system
	5	*
	6	* Copyright (C) 2016 Allen Samuels <allen.samuels@sandisk.com>
	7	*
	8	* This is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU Lesser General Public
	10	* License version 2.1, as published by the Free Software
	11	* Foundation. See file COPYING.
	12	*
	13	*/
	14
	15	#ifndef _CEPH_INCLUDE_MEMPOOL_H
	16	#define _CEPH_INCLUDE_MEMPOOL_H
	17
	18	#include <cstddef>
	19	#include <map>
	20	#include <unordered_map>
	21	#include <set>
	22	#include <vector>
	23	#include <list>
	24	#include <mutex>
	25	#include <atomic>
	26	#include <typeinfo>
	27
	28	#include <common/Formatter.h>
	29	#include "include/assert.h"
	30
	31
	32	/*
	33
	34	Memory Pools
	35	============
	36
	37	A memory pool is a method for accounting the consumption of memory of
	38	a set of containers.
	39
	40	Memory pools are statically declared (see pool_index_t).
	41
	42	Each memory pool tracks the number of bytes and items it contains.
	43
	44	Allocators can be declared and associated with a type so that they are
	45	tracked independently of the pool total. This additional accounting
	46	is optional and only incurs an overhead if the debugging is enabled at
	47	runtime. This allows developers to see what types are consuming the
	48	pool resources.
	49
	50
	51	Declaring
	52	---------
	53
	54	Using memory pools is very easy.
	55
	56	To create a new memory pool, simply add a new name into the list of
	57	memory pools that's defined in "DEFINE_MEMORY_POOLS_HELPER". That's
	58	it. :)
	59
	60	For each memory pool that's created a C++ namespace is also
	61	automatically created (name is same as in DEFINE_MEMORY_POOLS_HELPER).
	62	That namespace contains a set of common STL containers that are predefined
	63	with the appropriate allocators.
	64
31f18b77	65	Thus for mempool "osd" we have automatically available to us:
7c673cae	66
31f18b77 FG	67	mempool::osd::map
	68	mempool::osd::multimap
	69	mempool::osd::set
	70	mempool::osd::multiset
	71	mempool::osd::list
	72	mempool::osd::vector
	73	mempool::osd::unordered_map
7c673cae FG	74
	75
	76	Putting objects in a mempool
	77	----------------------------
	78
	79	In order to use a memory pool with a particular type, a few additional
	80	declarations are needed.
	81
	82	For a class:
	83
	84	struct Foo {
	85	MEMPOOL_CLASS_HELPERS();
	86	...
	87	};
	88
	89	Then, in an appropriate .cc file,
	90
31f18b77	91	MEMPOOL_DEFINE_OBJECT_FACTORY(Foo, foo, osd);
7c673cae FG	92
	93	The second argument can generally be identical to the first, except
	94	when the type contains a nested scope. For example, for
	95	BlueStore::Onode, we need to do
	96
	97	MEMPOOL_DEFINE_OBJECT_FACTORY(BlueStore::Onode, bluestore_onode,
	98	bluestore_meta);
	99
	100	(This is just because we need to name some static variables and we
	101	can't use :: in a variable name.)
	102
	103	In order to use the STL containers, simply use the namespaced variant
	104	of the container type. For example,
	105
31f18b77	106	mempool::osd::map<int> myvec;
7c673cae FG	107
	108	Introspection
	109	-------------
	110
	111	The simplest way to interrogate the process is with
	112
	113	Formater *f = ...
	114	mempool::dump(f);
	115
	116	This will dump information about all memory pools. When debug mode
	117	is enabled, the runtime complexity of dump is O(num_shards *
	118	num_types). When debug name is disabled it is O(num_shards).
	119
	120	You can also interrogate a specific pool programmatically with
	121
	122	size_t bytes = mempool::unittest_2::allocated_bytes();
	123	size_t items = mempool::unittest_2::allocated_items();
	124
	125	The runtime complexity is O(num_shards).
	126
	127	Note that you cannot easily query per-type, primarily because debug
	128	mode is optional and you should not rely on that information being
	129	available.
	130
	131	*/
	132
	133	namespace mempool {
	134
	135	// --------------------------------------------------------------
	136	// define memory pools
	137
	138	#define DEFINE_MEMORY_POOLS_HELPER(f) \
	139	f(bloom_filter) \
7c673cae	140	f(bluestore_alloc) \
31f18b77 FG	141	f(bluestore_cache_data) \
	142	f(bluestore_cache_onode) \
	143	f(bluestore_cache_other) \
7c673cae	144	f(bluestore_fsck) \
31f18b77 FG	145	f(bluestore_txc) \
	146	f(bluestore_writing_deferred) \
	147	f(bluestore_writing) \
7c673cae	148	f(bluefs) \
31f18b77	149	f(buffer_anon) \
7c673cae	150	f(buffer_meta) \
7c673cae	151	f(osd) \
31f18b77 FG	152	f(osd_mapbl) \
31f18b77 FG	153	f(osd_pglog) \
7c673cae FG	154	f(osdmap) \
7c673cae FG	155	f(osdmap_mapping) \
31f18b77	156	f(pgmap) \
7c673cae FG	157	f(unittest_1) \
	158	f(unittest_2)
	159
	160
	161	// give them integer ids
	162	#define P(x) mempool_##x,
	163	enum pool_index_t {
	164	DEFINE_MEMORY_POOLS_HELPER(P)
	165	num_pools // Must be last.
	166	};
	167	#undef P
	168
	169	extern bool debug_mode;
	170	extern void set_debug_mode(bool d);
	171
	172	// --------------------------------------------------------------
	173	class pool_t;
	174
	175	// we shard pool stats across many shard_t's to reduce the amount
	176	// of cacheline ping pong.
	177	enum {
	178	num_shard_bits = 5
	179	};
	180	enum {
	181	num_shards = 1 << num_shard_bits
	182	};
	183
	184	// align shard to a cacheline
	185	struct shard_t {
	186	std::atomic<size_t> bytes = {0};
	187	std::atomic<size_t> items = {0};
	188	char __padding[128 - sizeof(std::atomic<size_t>)*2];
	189	} __attribute__ ((aligned (128)));
	190
	191	static_assert(sizeof(shard_t) == 128, "shard_t should be cacheline-sized");
	192
	193	struct stats_t {
	194	ssize_t items = 0;
	195	ssize_t bytes = 0;
	196	void dump(ceph::Formatter *f) const {
	197	f->dump_int("items", items);
	198	f->dump_int("bytes", bytes);
	199	}
31f18b77 FG	200
	201	stats_t& operator+=(const stats_t& o) {
	202	items += o.items;
	203	bytes += o.bytes;
	204	return *this;
	205	}
7c673cae FG	206	};
	207
	208	pool_t& get_pool(pool_index_t ix);
	209	const char *get_pool_name(pool_index_t ix);
	210
	211	struct type_t {
	212	const char *type_name;
	213	size_t item_size;
	214	std::atomic<ssize_t> items = {0}; // signed
	215	};
	216
	217	struct type_info_hash {
	218	std::size_t operator()(const std::type_info& k) const {
	219	return k.hash_code();
	220	}
	221	};
	222
	223	class pool_t {
	224	shard_t shard[num_shards];
	225
	226	mutable std::mutex lock; // only used for types list
	227	std::unordered_map<const char *, type_t> type_map;
	228
	229	public:
	230	//
	231	// How much this pool consumes. O(<num_shards>)
	232	//
	233	size_t allocated_bytes() const;
	234	size_t allocated_items() const;
	235
31f18b77 FG	236	void adjust_count(ssize_t items, ssize_t bytes);
31f18b77 FG	237
7c673cae FG	238	shard_t* pick_a_shard() {
	239	// Dirt cheap, see:
	240	// http://fossies.org/dox/glibc-2.24/pthread__self_8c_source.html
	241	size_t me = (size_t)pthread_self();
	242	size_t i = (me >> 3) & ((1 << num_shard_bits) - 1);
	243	return &shard[i];
	244	}
	245
	246	type_t *get_type(const std::type_info& ti, size_t size) {
	247	std::lock_guard<std::mutex> l(lock);
	248	auto p = type_map.find(ti.name());
	249	if (p != type_map.end()) {
	250	return &p->second;
	251	}
	252	type_t &t = type_map[ti.name()];
	253	t.type_name = ti.name();
	254	t.item_size = size;
	255	return &t;
	256	}
	257
	258	// get pool stats. by_type is not populated if !debug
	259	void get_stats(stats_t *total,
	260	std::map<std::string, stats_t> *by_type) const;
	261
31f18b77	262	void dump(ceph::Formatter f, stats_t ptotal=0) const;
7c673cae FG	263	};
	264
	265	void dump(ceph::Formatter *f);
	266
	267
	268	// STL allocator for use with containers. All actual state
	269	// is stored in the static pool_allocator_base_t, which saves us from
	270	// passing the allocator to container constructors.
	271
	272	template<pool_index_t pool_ix, typename T>
	273	class pool_allocator {
	274	pool_t *pool;
	275	type_t *type = nullptr;
	276
	277	public:
	278	typedef pool_allocator<pool_ix, T> allocator_type;
	279	typedef T value_type;
	280	typedef value_type *pointer;
	281	typedef const value_type * const_pointer;
	282	typedef value_type& reference;
	283	typedef const value_type& const_reference;
	284	typedef std::size_t size_type;
	285	typedef std::ptrdiff_t difference_type;
	286
	287	template<typename U> struct rebind {
	288	typedef pool_allocator<pool_ix,U> other;
	289	};
	290
	291	void init(bool force_register) {
	292	pool = &get_pool(pool_ix);
	293	if (debug_mode \|\| force_register) {
	294	type = pool->get_type(typeid(T), sizeof(T));
	295	}
	296	}
	297
	298	pool_allocator(bool force_register=false) {
	299	init(force_register);
	300	}
	301	template<typename U>
	302	pool_allocator(const pool_allocator<pool_ix,U>&) {
	303	init(false);
	304	}
	305
	306	T* allocate(size_t n, void *p = nullptr) {
	307	size_t total = sizeof(T) * n;
	308	shard_t *shard = pool->pick_a_shard();
	309	shard->bytes += total;
	310	shard->items += n;
	311	if (type) {
	312	type->items += n;
	313	}
	314	T* r = reinterpret_cast<T*>(new char[total]);
	315	return r;
	316	}
	317
	318	void deallocate(T* p, size_t n) {
	319	size_t total = sizeof(T) * n;
	320	shard_t *shard = pool->pick_a_shard();
	321	shard->bytes -= total;
	322	shard->items -= n;
	323	if (type) {
	324	type->items -= n;
	325	}
	326	delete[] reinterpret_cast<char*>(p);
327	}
328
329	T* allocate_aligned(size_t n, size_t align, void *p = nullptr) {
330	size_t total = sizeof(T) * n;
331	shard_t *shard = pool->pick_a_shard();
332	shard->bytes += total;
333	shard->items += n;
334	if (type) {
335	type->items += n;
336	}
337	char *ptr;
338	int rc = ::posix_memalign((void*)(void)&ptr, align, total);
339	if (rc)
340	throw std::bad_alloc();
341	T* r = reinterpret_cast<T*>(ptr);
342	return r;
343	}
344
345	void deallocate_aligned(T* p, size_t n) {
346	size_t total = sizeof(T) * n;
347	shard_t *shard = pool->pick_a_shard();
348	shard->bytes -= total;
349	shard->items -= n;
350	if (type) {
351	type->items -= n;
352	}
353	::free(p);
354	}
355
356	void destroy(T* p) {
357	p->~T();
358	}
359
360	template<class U>
361	void destroy(U *p) {
362	p->~U();
363	}
364
365	void construct(T* p, const T& val) {
366	::new ((void *)p) T(val);
367	}
368
369	template<class U, class... Args> void construct(U* p,Args&&... args) {
370	::new((void *)p) U(std::forward<Args>(args)...);
371	}
372
373	bool operator==(const pool_allocator&) const { return true; }
374	bool operator!=(const pool_allocator&) const { return false; }
375	};
376
377
378	// Namespace mempool
379
380	#define P(x) \
381	namespace x { \
382	static const mempool::pool_index_t id = mempool::mempool_##x; \
383	template<typename v> \
384	using pool_allocator = mempool::pool_allocator<id,v>; \
385	\
386	using string = std::basic_string<char,std::char_traits<char>, \
387	pool_allocator<char>>; \
388	\
389	template<typename k,typename v, typename cmp = std::less<k> > \
390	using map = std::map<k, v, cmp, \
391	pool_allocator<std::pair<const k,v>>>; \
392	\
393	template<typename k,typename v, typename cmp = std::less<k> > \
394	using multimap = std::multimap<k,v,cmp, \
395	pool_allocator<std::pair<const k, \
396	v>>>; \
397	\
398	template<typename k, typename cmp = std::less<k> > \
399	using set = std::set<k,cmp,pool_allocator<k>>; \
400	\
401	template<typename v> \
402	using list = std::list<v,pool_allocator<v>>; \
403	\
404	template<typename v> \
405	using vector = std::vector<v,pool_allocator<v>>; \
406	\
407	template<typename k, typename v, \
408	typename h=std::hash<k>, \
409	typename eq = std::equal_to<k>> \
410	using unordered_map = \
411	std::unordered_map<k,v,h,eq,pool_allocator<std::pair<const k,v>>>;\
412	\
413	inline size_t allocated_bytes() { \
414	return mempool::get_pool(id).allocated_bytes(); \
415	} \
416	inline size_t allocated_items() { \
417	return mempool::get_pool(id).allocated_items(); \
418	} \
419	};
420
421	DEFINE_MEMORY_POOLS_HELPER(P)
422
423	#undef P
424
425	};
426
427
428
429	// Use this for any type that is contained by a container (unless it
430	// is a class you defined; see below).
431	#define MEMPOOL_DECLARE_FACTORY(obj, factoryname, pool) \
432	namespace mempool { \
433	namespace pool { \
434	extern pool_allocator<obj> alloc_##factoryname; \
435	} \
436	}
437
438	#define MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool) \
439	namespace mempool { \
440	namespace pool { \
441	pool_allocator<obj> alloc_##factoryname = {true}; \
442	} \
443	}
444
445	// Use this for each class that belongs to a mempool. For example,
446	//
447	// class T {
448	// MEMPOOL_CLASS_HELPERS();
449	// ...
450	// };
451	//
452	#define MEMPOOL_CLASS_HELPERS() \
453	void *operator new(size_t size); \
454	void *operator new[](size_t size) noexcept { \
455	assert(0 == "no array new"); \
456	return nullptr; } \
457	void operator delete(void *); \
458	void operator delete[](void *) { assert(0 == "no array delete"); }
459
460
461	// Use this in some particular .cc file to match each class with a
462	// MEMPOOL_CLASS_HELPERS().
463	#define MEMPOOL_DEFINE_OBJECT_FACTORY(obj,factoryname,pool) \
464	MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool) \
465	void *obj::operator new(size_t size) { \
466	return mempool::pool::alloc_##factoryname.allocate(1); \
467	} \
468	void obj::operator delete(void *p) { \
469	return mempool::pool::alloc_##factoryname.deallocate((obj*)p, 1); \
470	}
471
472	#endif