[ceph.git] / ceph / src / test / test_mempool.cc

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph distributed storage system
 *
 * Copyright (C) 2016 Western Digital Corporation
 *
 * Author: Allen Samuels <allen.samuels@sandisk.com>
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2.1 of the License, or (at your option) any later version.
 *
 */

#include <stdio.h>

#include "global/global_init.h"
#include "common/ceph_argparse.h"
#include "global/global_context.h"
#include "gtest/gtest.h"
#include "include/mempool.h"

void check_usage(mempool::pool_index_t ix)
{
  mempool::pool_t *pool = &mempool::get_pool(ix);
  mempool::stats_t total;
  map<std::string,mempool::stats_t> m;
  pool->get_stats(&total, &m);
  size_t usage = pool->allocated_bytes();
  size_t sum = 0;
  for (auto& p : m) {
    sum += p.second.bytes;
  }
  if (sum != usage) {
    ceph::TableFormatter jf;
    pool->dump(&jf);
    jf.flush(std::cout);
  }
  EXPECT_EQ(sum, usage);
}

template<typename A, typename B>
void eq_elements(const A& a, const B& b)
{
  auto lhs = a.begin();
  auto rhs = b.begin();
  while (lhs != a.end()) {
    EXPECT_EQ(*lhs,*rhs);
    lhs++;
    rhs++;
  }
  EXPECT_EQ(rhs,b.end());
}

template<typename A, typename B>
void eq_pairs(const A& a, const B& b)
{
  auto lhs = a.begin();
  auto rhs = b.begin();
  while (lhs != a.end()) {
    EXPECT_EQ(lhs->first,rhs->first);
    EXPECT_EQ(lhs->second,rhs->second);
    lhs++;
    rhs++;
  }
  EXPECT_EQ(rhs,b.end());
}

#define MAKE_INSERTER(inserter)				\
  template<typename A,typename B>			\
void do_##inserter(A& a, B& b, int count, int base) {	\
  for (int i = 0; i < count; ++i) {			\
    a.inserter(base + i);				\
    b.inserter(base + i);				\
  }							\
}

MAKE_INSERTER(push_back);
MAKE_INSERTER(insert);

template<typename A,typename B>
void do_insert_key(A& a, B& b, int count, int base)
{
  for (int i = 0; i < count; ++i) {
    a.insert(make_pair(base+i,base+i));
    b.insert(make_pair(base+i,base+i));
    check_usage(mempool::osd::id);
  }
}

TEST(mempool, vector_context)
{
  check_usage(mempool::osd::id);
  EXPECT_EQ(mempool::osd::allocated_bytes(), 0u);
  EXPECT_EQ(mempool::osd::allocated_items(), 0u);
  for (unsigned i = 0; i < 10; ++i) {
    vector<int> a;
    mempool::osd::vector<int> b,c;
    eq_elements(a,b);
    do_push_back(a,b,i,i);
    eq_elements(a,b);
    check_usage(mempool::osd::id);

    mempool::stats_t total;
    map<std::string,mempool::stats_t> by_type;
    mempool::get_pool(mempool::osd::id).get_stats(&total, &by_type);
    EXPECT_GE(mempool::osd::allocated_bytes(), i * 4u);
    EXPECT_GE(mempool::osd::allocated_items(), i);

    c.swap(b);
    eq_elements(a,c);
    check_usage(mempool::osd::id);
    a.clear();
    b.clear();
    c.clear();
  }
}

TEST(mempool, list_context)
{
  for (unsigned i = 1; i < 10; ++i) {
    list<int> a;
    mempool::osd::list<int> b,c;
    eq_elements(a,b);
    do_push_back(a,b,i,i);
    eq_elements(a,b);
    c.swap(b);
    eq_elements(a,c);
    a.erase(a.begin());
    c.erase(c.begin());
    eq_elements(a,c);
    a.clear();
    b.clear();
    c.clear();
    do_push_back(a,b,i,i);
    c.splice(c.begin(),b,b.begin(),b.end());

    mempool::stats_t total;
    map<std::string,mempool::stats_t> by_type;
    mempool::get_pool(mempool::osd::id).get_stats(&total, &by_type);
    EXPECT_GE(mempool::osd::allocated_bytes(), i * 4u);
    EXPECT_EQ(mempool::osd::allocated_items(), i);

    eq_elements(a,c);
    check_usage(mempool::osd::id);
  }
}

TEST(mempool, set_context)
{
  for (int i = 0; i < 10; ++i) {
    set<int> a;
    mempool::osd::set<int> b;
    do_insert(a,b,i,i);
    eq_elements(a,b);
    check_usage(mempool::osd::id);
  }

  for (int i = 1; i < 10; ++i) {
    set<int> a;
    mempool::osd::set<int> b;
    do_insert(a,b,i,0);
    EXPECT_NE(a.find(i/2),a.end());
    EXPECT_NE(b.find(i/2),b.end());
    a.erase(a.find(i/2));
    b.erase(b.find(i/2));
    eq_elements(a,b);
    check_usage(mempool::osd::id);
  }
}

struct obj {
  MEMPOOL_CLASS_HELPERS();
  int a;
  int b;
  obj() : a(1), b(1) {}
  obj(int _a) : a(_a), b(2) {}
  obj(int _a,int _b) : a(_a), b(_b) {}
  friend inline bool operator<(const obj& l, const obj& r) {
    return l.a < r.a;
  }
};
MEMPOOL_DEFINE_OBJECT_FACTORY(obj, obj, osdmap);

TEST(mempool, test_factory)
{
   obj *o1 = new obj();
   obj *o2 = new obj(10);
   obj *o3 = new obj(20,30);
   check_usage(mempool::osdmap::id);
   EXPECT_NE(o1,nullptr);
   EXPECT_EQ(o1->a,1);
   EXPECT_EQ(o1->b,1);
   EXPECT_EQ(o2->a,10);
   EXPECT_EQ(o2->b,2);
   EXPECT_EQ(o3->a,20);
   EXPECT_EQ(o3->b,30);

   delete o1;
   delete o2;
   delete o3;
   check_usage(mempool::osdmap::id);
}

TEST(mempool, vector)
{
  {
    mempool::osd::vector<int> v;
    v.push_back(1);
    v.push_back(2);
  }
  {
    mempool::osdmap::vector<obj> v;
    v.push_back(obj());
    v.push_back(obj(1));
  }
}

TEST(mempool, set)
{
  mempool::osd::set<int> set_int;
  set_int.insert(1);
  set_int.insert(2);
  mempool::osdmap::set<obj> set_obj;
  set_obj.insert(obj());
  set_obj.insert(obj(1));
  set_obj.insert(obj(1, 2));
}

TEST(mempool, map)
{
  {
    mempool::osd::map<int,int> v;
    v[1] = 2;
    v[3] = 4;
  }
  {
    mempool::osdmap::map<int,obj> v;
    v[1] = obj();
    v[2] = obj(2);
    v[3] = obj(2, 3);
  }
}

TEST(mempool, list)
{
  {
    mempool::osd::list<int> v;
    v.push_back(1);
    v.push_back(2);
  }
  {
    mempool::osdmap::list<obj> v;
    v.push_back(obj());
    v.push_back(obj(1));
  }
}

TEST(mempool, unordered_map)
{
  mempool::osdmap::unordered_map<int,obj> h;
  h[1] = obj();
  h[2] = obj(1);
}

TEST(mempool, string_test)
{
  mempool::osdmap::string s;
  s.reserve(100);
  EXPECT_GE(mempool::osdmap::allocated_items(), s.capacity() + 1u); // +1 for zero-byte termination :
  for (size_t i = 0; i < 10; ++i) {
    s += '1';
    s.append(s);
    EXPECT_GE(mempool::osdmap::allocated_items(), s.capacity() + 1u);
  }
}

TEST(mempool, bufferlist)
{
  bufferlist bl;
  int len = 1048576;
  size_t before = mempool::buffer_anon::allocated_bytes();
  cout << "before " << before << std::endl;
  bl.append(buffer::create_aligned(len, 4096));
  size_t after = mempool::buffer_anon::allocated_bytes();
  cout << "after " << after << std::endl;
  ASSERT_GE(after, before + len);
}

TEST(mempool, bufferlist_reassign)
{
  bufferlist bl;
  size_t items_before = mempool::buffer_anon::allocated_items();
  size_t bytes_before = mempool::buffer_anon::allocated_bytes();
  bl.append("fooo");
  ASSERT_EQ(items_before + 1, mempool::buffer_anon::allocated_items());
  ASSERT_LT(bytes_before, mempool::buffer_anon::allocated_bytes());

  // move existing bl
  bl.reassign_to_mempool(mempool::mempool_osd);
  ASSERT_EQ(items_before, mempool::buffer_anon::allocated_items());
  ASSERT_EQ(bytes_before, mempool::buffer_anon::allocated_bytes());

  // additional appends should go to the same pool
  items_before = mempool::osd::allocated_items();
  bytes_before = mempool::osd::allocated_bytes();
  cout << "anon b " << mempool::buffer_anon::allocated_bytes() << std::endl;
  for (unsigned i = 0; i < 1000; ++i) {
    bl.append("asdfddddddddddddddddddddddasfdasdfasdfasdfasdfasdf");
  }
  cout << "anon a " << mempool::buffer_anon::allocated_bytes() << std::endl;
  ASSERT_LT(items_before, mempool::osd::allocated_items());
  ASSERT_LT(bytes_before, mempool::osd::allocated_bytes());

  // try_.. won't
  items_before = mempool::osd::allocated_items();
  bytes_before = mempool::osd::allocated_bytes();
  bl.try_assign_to_mempool(mempool::mempool_bloom_filter);
  ASSERT_EQ(items_before, mempool::osd::allocated_items());
  ASSERT_EQ(bytes_before, mempool::osd::allocated_bytes());
}

int main(int argc, char **argv)
{
  vector<const char*> args;
  argv_to_vec(argc, (const char **)argv, args);

  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
			 CODE_ENVIRONMENT_UTILITY, 0);
  common_init_finish(g_ceph_context);

  // enable debug mode for the tests
  mempool::set_debug_mode(true);

  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}


/*
 * Local Variables:
 * compile-command: "cd ../../build ; make -j4 &&
 *   make unittest_mempool &&
 *   valgrind --tool=memcheck ./unittest_mempool --gtest_filter=*.*"
 * End:
 */
Commit	Line	Data
7c673cae FG	1	// -- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t --
	2	// vim: ts=8 sw=2 smarttab
	3	/*
	4	* Ceph distributed storage system
	5	*
	6	* Copyright (C) 2016 Western Digital Corporation
	7	*
	8	* Author: Allen Samuels <allen.samuels@sandisk.com>
	9	*
	10	* This library is free software; you can redistribute it and/or
	11	* modify it under the terms of the GNU Lesser General Public
	12	* License as published by the Free Software Foundation; either
	13	* version 2.1 of the License, or (at your option) any later version.
	14	*
	15	*/
	16
	17	#include <stdio.h>
	18
	19	#include "global/global_init.h"
	20	#include "common/ceph_argparse.h"
	21	#include "global/global_context.h"
	22	#include "gtest/gtest.h"
	23	#include "include/mempool.h"
	24
	25	void check_usage(mempool::pool_index_t ix)
	26	{
	27	mempool::pool_t *pool = &mempool::get_pool(ix);
	28	mempool::stats_t total;
	29	map<std::string,mempool::stats_t> m;
	30	pool->get_stats(&total, &m);
	31	size_t usage = pool->allocated_bytes();
	32	size_t sum = 0;
	33	for (auto& p : m) {
	34	sum += p.second.bytes;
	35	}
	36	if (sum != usage) {
	37	ceph::TableFormatter jf;
	38	pool->dump(&jf);
	39	jf.flush(std::cout);
	40	}
	41	EXPECT_EQ(sum, usage);
	42	}
	43
	44	template<typename A, typename B>
	45	void eq_elements(const A& a, const B& b)
	46	{
	47	auto lhs = a.begin();
	48	auto rhs = b.begin();
	49	while (lhs != a.end()) {
	50	EXPECT_EQ(lhs,rhs);
	51	lhs++;
	52	rhs++;
	53	}
	54	EXPECT_EQ(rhs,b.end());
	55	}
	56
	57	template<typename A, typename B>
	58	void eq_pairs(const A& a, const B& b)
	59	{
	60	auto lhs = a.begin();
	61	auto rhs = b.begin();
	62	while (lhs != a.end()) {
	63	EXPECT_EQ(lhs->first,rhs->first);
	64	EXPECT_EQ(lhs->second,rhs->second);
65	lhs++;
66	rhs++;
67	}
68	EXPECT_EQ(rhs,b.end());
69	}
70
71	#define MAKE_INSERTER(inserter) \
72	template<typename A,typename B> \
73	void do_##inserter(A& a, B& b, int count, int base) { \
74	for (int i = 0; i < count; ++i) { \
75	a.inserter(base + i); \
76	b.inserter(base + i); \
77	} \
78	}
79
80	MAKE_INSERTER(push_back);
81	MAKE_INSERTER(insert);
82
83	template<typename A,typename B>
84	void do_insert_key(A& a, B& b, int count, int base)
85	{
86	for (int i = 0; i < count; ++i) {
87	a.insert(make_pair(base+i,base+i));
88	b.insert(make_pair(base+i,base+i));
31f18b77	89	check_usage(mempool::osd::id);
7c673cae FG	90	}
	91	}
	92
	93	TEST(mempool, vector_context)
	94	{
31f18b77 FG	95	check_usage(mempool::osd::id);
	96	EXPECT_EQ(mempool::osd::allocated_bytes(), 0u);
	97	EXPECT_EQ(mempool::osd::allocated_items(), 0u);
7c673cae FG	98	for (unsigned i = 0; i < 10; ++i) {
7c673cae FG	99	vector<int> a;
31f18b77	100	mempool::osd::vector<int> b,c;
7c673cae FG	101	eq_elements(a,b);
	102	do_push_back(a,b,i,i);
	103	eq_elements(a,b);
31f18b77	104	check_usage(mempool::osd::id);
7c673cae FG	105
	106	mempool::stats_t total;
	107	map<std::string,mempool::stats_t> by_type;
31f18b77 FG	108	mempool::get_pool(mempool::osd::id).get_stats(&total, &by_type);
	109	EXPECT_GE(mempool::osd::allocated_bytes(), i * 4u);
	110	EXPECT_GE(mempool::osd::allocated_items(), i);
7c673cae FG	111
	112	c.swap(b);
	113	eq_elements(a,c);
31f18b77	114	check_usage(mempool::osd::id);
7c673cae FG	115	a.clear();
	116	b.clear();
	117	c.clear();
	118	}
	119	}
	120
	121	TEST(mempool, list_context)
	122	{
	123	for (unsigned i = 1; i < 10; ++i) {
	124	list<int> a;
31f18b77	125	mempool::osd::list<int> b,c;
7c673cae FG	126	eq_elements(a,b);
	127	do_push_back(a,b,i,i);
	128	eq_elements(a,b);
	129	c.swap(b);
	130	eq_elements(a,c);
	131	a.erase(a.begin());
	132	c.erase(c.begin());
	133	eq_elements(a,c);
	134	a.clear();
	135	b.clear();
	136	c.clear();
	137	do_push_back(a,b,i,i);
	138	c.splice(c.begin(),b,b.begin(),b.end());
	139
	140	mempool::stats_t total;
	141	map<std::string,mempool::stats_t> by_type;
31f18b77 FG	142	mempool::get_pool(mempool::osd::id).get_stats(&total, &by_type);
	143	EXPECT_GE(mempool::osd::allocated_bytes(), i * 4u);
	144	EXPECT_EQ(mempool::osd::allocated_items(), i);
7c673cae FG	145
7c673cae FG	146	eq_elements(a,c);
31f18b77	147	check_usage(mempool::osd::id);
7c673cae FG	148	}
	149	}
	150
	151	TEST(mempool, set_context)
	152	{
	153	for (int i = 0; i < 10; ++i) {
	154	set<int> a;
31f18b77	155	mempool::osd::set<int> b;
7c673cae FG	156	do_insert(a,b,i,i);
7c673cae FG	157	eq_elements(a,b);
31f18b77	158	check_usage(mempool::osd::id);
7c673cae FG	159	}
	160
	161	for (int i = 1; i < 10; ++i) {
	162	set<int> a;
31f18b77	163	mempool::osd::set<int> b;
7c673cae FG	164	do_insert(a,b,i,0);
	165	EXPECT_NE(a.find(i/2),a.end());
	166	EXPECT_NE(b.find(i/2),b.end());
	167	a.erase(a.find(i/2));
	168	b.erase(b.find(i/2));
	169	eq_elements(a,b);
31f18b77	170	check_usage(mempool::osd::id);
7c673cae FG	171	}
	172	}
	173
	174	struct obj {
	175	MEMPOOL_CLASS_HELPERS();
	176	int a;
	177	int b;
	178	obj() : a(1), b(1) {}
	179	obj(int _a) : a(_a), b(2) {}
	180	obj(int _a,int _b) : a(_a), b(_b) {}
	181	friend inline bool operator<(const obj& l, const obj& r) {
	182	return l.a < r.a;
	183	}
	184	};
31f18b77	185	MEMPOOL_DEFINE_OBJECT_FACTORY(obj, obj, osdmap);
7c673cae FG	186
	187	TEST(mempool, test_factory)
	188	{
	189	obj *o1 = new obj();
	190	obj *o2 = new obj(10);
	191	obj *o3 = new obj(20,30);
31f18b77	192	check_usage(mempool::osdmap::id);
7c673cae FG	193	EXPECT_NE(o1,nullptr);
	194	EXPECT_EQ(o1->a,1);
	195	EXPECT_EQ(o1->b,1);
	196	EXPECT_EQ(o2->a,10);
	197	EXPECT_EQ(o2->b,2);
	198	EXPECT_EQ(o3->a,20);
	199	EXPECT_EQ(o3->b,30);
	200
	201	delete o1;
	202	delete o2;
	203	delete o3;
31f18b77	204	check_usage(mempool::osdmap::id);
7c673cae FG	205	}
	206
	207	TEST(mempool, vector)
	208	{
	209	{
31f18b77	210	mempool::osd::vector<int> v;
7c673cae FG	211	v.push_back(1);
	212	v.push_back(2);
	213	}
	214	{
31f18b77	215	mempool::osdmap::vector<obj> v;
7c673cae FG	216	v.push_back(obj());
	217	v.push_back(obj(1));
	218	}
	219	}
	220
	221	TEST(mempool, set)
	222	{
31f18b77	223	mempool::osd::set<int> set_int;
7c673cae FG	224	set_int.insert(1);
7c673cae FG	225	set_int.insert(2);
31f18b77	226	mempool::osdmap::set<obj> set_obj;
7c673cae FG	227	set_obj.insert(obj());
	228	set_obj.insert(obj(1));
	229	set_obj.insert(obj(1, 2));
	230	}
	231
	232	TEST(mempool, map)
	233	{
	234	{
31f18b77	235	mempool::osd::map<int,int> v;
7c673cae FG	236	v[1] = 2;
	237	v[3] = 4;
	238	}
	239	{
31f18b77	240	mempool::osdmap::map<int,obj> v;
7c673cae FG	241	v[1] = obj();
	242	v[2] = obj(2);
	243	v[3] = obj(2, 3);
	244	}
	245	}
	246
	247	TEST(mempool, list)
	248	{
	249	{
31f18b77	250	mempool::osd::list<int> v;
7c673cae FG	251	v.push_back(1);
	252	v.push_back(2);
	253	}
	254	{
31f18b77	255	mempool::osdmap::list<obj> v;
7c673cae FG	256	v.push_back(obj());
	257	v.push_back(obj(1));
	258	}
	259	}
	260
	261	TEST(mempool, unordered_map)
	262	{
31f18b77	263	mempool::osdmap::unordered_map<int,obj> h;
7c673cae FG	264	h[1] = obj();
	265	h[2] = obj(1);
	266	}
	267
31f18b77 FG	268	TEST(mempool, string_test)
	269	{
	270	mempool::osdmap::string s;
	271	s.reserve(100);
	272	EXPECT_GE(mempool::osdmap::allocated_items(), s.capacity() + 1u); // +1 for zero-byte termination :
	273	for (size_t i = 0; i < 10; ++i) {
	274	s += '1';
	275	s.append(s);
	276	EXPECT_GE(mempool::osdmap::allocated_items(), s.capacity() + 1u);
	277	}
	278	}
	279
7c673cae FG	280	TEST(mempool, bufferlist)
	281	{
	282	bufferlist bl;
	283	int len = 1048576;
31f18b77	284	size_t before = mempool::buffer_anon::allocated_bytes();
7c673cae FG	285	cout << "before " << before << std::endl;
7c673cae FG	286	bl.append(buffer::create_aligned(len, 4096));
31f18b77	287	size_t after = mempool::buffer_anon::allocated_bytes();
7c673cae FG	288	cout << "after " << after << std::endl;
	289	ASSERT_GE(after, before + len);
	290	}
	291
31f18b77	292	TEST(mempool, bufferlist_reassign)
7c673cae	293	{
31f18b77 FG	294	bufferlist bl;
	295	size_t items_before = mempool::buffer_anon::allocated_items();
	296	size_t bytes_before = mempool::buffer_anon::allocated_bytes();
	297	bl.append("fooo");
	298	ASSERT_EQ(items_before + 1, mempool::buffer_anon::allocated_items());
	299	ASSERT_LT(bytes_before, mempool::buffer_anon::allocated_bytes());
	300
	301	// move existing bl
	302	bl.reassign_to_mempool(mempool::mempool_osd);
	303	ASSERT_EQ(items_before, mempool::buffer_anon::allocated_items());
	304	ASSERT_EQ(bytes_before, mempool::buffer_anon::allocated_bytes());
	305
	306	// additional appends should go to the same pool
	307	items_before = mempool::osd::allocated_items();
	308	bytes_before = mempool::osd::allocated_bytes();
	309	cout << "anon b " << mempool::buffer_anon::allocated_bytes() << std::endl;
	310	for (unsigned i = 0; i < 1000; ++i) {
	311	bl.append("asdfddddddddddddddddddddddasfdasdfasdfasdfasdfasdf");
7c673cae	312	}
31f18b77 FG	313	cout << "anon a " << mempool::buffer_anon::allocated_bytes() << std::endl;
	314	ASSERT_LT(items_before, mempool::osd::allocated_items());
	315	ASSERT_LT(bytes_before, mempool::osd::allocated_bytes());
	316
	317	// try_.. won't
	318	items_before = mempool::osd::allocated_items();
	319	bytes_before = mempool::osd::allocated_bytes();
	320	bl.try_assign_to_mempool(mempool::mempool_bloom_filter);
	321	ASSERT_EQ(items_before, mempool::osd::allocated_items());
	322	ASSERT_EQ(bytes_before, mempool::osd::allocated_bytes());
7c673cae FG	323	}
	324
	325	int main(int argc, char **argv)
	326	{
	327	vector<const char*> args;
	328	argv_to_vec(argc, (const char **)argv, args);
	329
	330	auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
	331	CODE_ENVIRONMENT_UTILITY, 0);
	332	common_init_finish(g_ceph_context);
	333
	334	// enable debug mode for the tests
	335	mempool::set_debug_mode(true);
	336
	337	::testing::InitGoogleTest(&argc, argv);
	338	return RUN_ALL_TESTS();
	339	}
	340
	341
	342	/*
	343	* Local Variables:
	344	* compile-command: "cd ../../build ; make -j4 &&
	345	* make unittest_mempool &&
	346	* valgrind --tool=memcheck ./unittest_mempool --gtest_filter=."
	347	* End:
	348	*/