[ceph.git] / ceph / src / test / erasure-code / TestErasureCodeExample.cc

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
// vim: ts=8 sw=2 smarttab
/*
 * Ceph distributed storage system
 *
 * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
 *
 * Author: Loic Dachary <loic@dachary.org>
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2.1 of the License, or (at your option) any later version.
 *
 */
#include <stdlib.h>

#include "include/stringify.h"
#include "ErasureCodeExample.h"
#include "global/global_context.h"
#include "gtest/gtest.h"

using namespace std;

TEST(ErasureCodeExample, chunk_size)
{
  ErasureCodeExample example;
  EXPECT_EQ(3u, example.get_chunk_count());
  EXPECT_EQ(11u, example.get_chunk_size(20));
}

TEST(ErasureCodeExample, minimum_to_decode)
{
  ErasureCodeExample example;
  set<int> available_chunks;
  set<int> want_to_read;
  want_to_read.insert(1);
  {
    set<int> minimum;
    EXPECT_EQ(-EIO, example._minimum_to_decode(want_to_read,
                                              available_chunks,
                                              &minimum));
  }
  available_chunks.insert(0);
  available_chunks.insert(2);
  {
    set<int> minimum;
    EXPECT_EQ(0, example._minimum_to_decode(want_to_read,
					    available_chunks,
					    &minimum));
    EXPECT_EQ(available_chunks, minimum);
    EXPECT_EQ(2u, minimum.size());
    EXPECT_EQ(1u, minimum.count(0));
    EXPECT_EQ(1u, minimum.count(2));
  }
  {
    set<int> minimum;
    available_chunks.insert(1);
    EXPECT_EQ(0, example._minimum_to_decode(want_to_read,
					    available_chunks,
					    &minimum));
    EXPECT_EQ(1u, minimum.size());
    EXPECT_EQ(1u, minimum.count(1));
  }
}

TEST(ErasureCodeExample, minimum_to_decode_with_cost)
{
  ErasureCodeExample example;
  map<int,int> available;
  set<int> want_to_read;
  want_to_read.insert(1);
  {
    set<int> minimum;
    EXPECT_EQ(-EIO, example.minimum_to_decode_with_cost(want_to_read,
							available,
							&minimum));
  }
  available[0] = 1;
  available[2] = 1;
  {
    set<int> minimum;
    EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
						     available,
						     &minimum));
    EXPECT_EQ(2u, minimum.size());
    EXPECT_EQ(1u, minimum.count(0));
    EXPECT_EQ(1u, minimum.count(2));
  }
  {
    set<int> minimum;
    available[1] = 1;
    EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
						     available,
						     &minimum));
    EXPECT_EQ(1u, minimum.size());
    EXPECT_EQ(1u, minimum.count(1));
  }
  {
    set<int> minimum;
    available[1] = 2;
    EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
						     available,
						     &minimum));
    EXPECT_EQ(2u, minimum.size());
    EXPECT_EQ(1u, minimum.count(0));
    EXPECT_EQ(1u, minimum.count(2));
  }
}

TEST(ErasureCodeExample, encode_decode)
{
  ErasureCodeExample example;

  bufferlist in;
  in.append("ABCDE");
  set<int> want_to_encode;
  for(unsigned int i = 0; i < example.get_chunk_count(); i++)
    want_to_encode.insert(i);
  map<int, bufferlist> encoded;
  EXPECT_EQ(0, example.encode(want_to_encode, in, &encoded));
  EXPECT_EQ(example.get_chunk_count(), encoded.size());
  EXPECT_EQ(example.get_chunk_size(in.length()), encoded[0].length());
  EXPECT_EQ('A', encoded[0][0]);
  EXPECT_EQ('B', encoded[0][1]);
  EXPECT_EQ('C', encoded[0][2]);
  EXPECT_EQ('D', encoded[1][0]);
  EXPECT_EQ('E', encoded[1][1]);
  EXPECT_EQ('A'^'D', encoded[2][0]);
  EXPECT_EQ('B'^'E', encoded[2][1]);
  EXPECT_EQ('C'^0, encoded[2][2]);

  // all chunks are available
  {
    int want_to_decode[] = { 0, 1 };
    map<int, bufferlist> decoded;
    EXPECT_EQ(0, example._decode(set<int>(want_to_decode, want_to_decode+2),
				 encoded,
				 &decoded));
    EXPECT_EQ(2u, decoded.size());
    EXPECT_EQ(3u, decoded[0].length());
    EXPECT_EQ('A', decoded[0][0]);
    EXPECT_EQ('B', decoded[0][1]);
    EXPECT_EQ('C', decoded[0][2]);
    EXPECT_EQ('D', decoded[1][0]);
    EXPECT_EQ('E', decoded[1][1]);
  }

  // one chunk is missing 
  {
    map<int, bufferlist> degraded = encoded;
    degraded.erase(0);
    EXPECT_EQ(2u, degraded.size());
    int want_to_decode[] = { 0, 1 };
    map<int, bufferlist> decoded;
    EXPECT_EQ(0, example._decode(set<int>(want_to_decode, want_to_decode+2),
				 degraded,
				 &decoded));
    EXPECT_EQ(2u, decoded.size());
    EXPECT_EQ(3u, decoded[0].length());
    EXPECT_EQ('A', decoded[0][0]);
    EXPECT_EQ('B', decoded[0][1]);
    EXPECT_EQ('C', decoded[0][2]);
    EXPECT_EQ('D', decoded[1][0]);
    EXPECT_EQ('E', decoded[1][1]);
  }
}

TEST(ErasureCodeExample, decode)
{
  ErasureCodeExample example;

#define LARGE_ENOUGH 2048
  bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
  in_ptr.zero();
  in_ptr.set_length(0);
  const char *payload =
    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
  in_ptr.append(payload, strlen(payload));
  bufferlist in;
  in.push_back(in_ptr);
  int want_to_encode[] = { 0, 1, 2 };
  map<int, bufferlist> encoded;
  EXPECT_EQ(0, example.encode(set<int>(want_to_encode, want_to_encode+3),
                              in,
                              &encoded));
  EXPECT_EQ(3u, encoded.size());

  // successfull decode
  bufferlist out;
  EXPECT_EQ(0, example.decode_concat(encoded, &out));
  bufferlist usable;
  usable.substr_of(out, 0, in.length());
  EXPECT_TRUE(usable == in);

  // cannot recover
  map<int, bufferlist> degraded;  
  degraded[0] = encoded[0];
  EXPECT_EQ(-ERANGE, example.decode_concat(degraded, &out));
}

TEST(ErasureCodeExample, create_rule)
{
  std::unique_ptr<CrushWrapper> c = std::make_unique<CrushWrapper>();
  c->create();
  c->set_type_name(2, "root");
  c->set_type_name(1, "host");
  c->set_type_name(0, "osd");

  int rootno;
  c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
		5, 0, NULL, NULL, &rootno);
  c->set_item_name(rootno, "default");

  map<string,string> loc;
  loc["root"] = "default";

  int num_host = 2;
  int num_osd = 5;
  int osd = 0;
  for (int h=0; h<num_host; ++h) {
    loc["host"] = string("host-") + stringify(h);
    for (int o=0; o<num_osd; ++o, ++osd) {
      c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
    }
  }

  stringstream ss;
  ErasureCodeExample example;
  EXPECT_EQ(0, example.create_rule("myrule", *c, &ss));
}

/*
 * Local Variables:
 * compile-command: "cd ../.. ; 
 *   make -j4 && 
 *   make unittest_erasure_code_example && 
 *   valgrind  --leak-check=full --tool=memcheck \
 *      ./unittest_erasure_code_example --gtest_filter=*.* \
 *      --log-to-stderr=true --debug-osd=20
 * "
 * End:
 */
Commit	Line	Data
1e59de90	1	// -- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t --
7c673cae FG	2	// vim: ts=8 sw=2 smarttab
	3	/*
	4	* Ceph distributed storage system
	5	*
	6	* Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
	7	*
	8	* Author: Loic Dachary <loic@dachary.org>
	9	*
	10	* This library is free software; you can redistribute it and/or
	11	* modify it under the terms of the GNU Lesser General Public
	12	* License as published by the Free Software Foundation; either
	13	* version 2.1 of the License, or (at your option) any later version.
1e59de90	14	*
7c673cae FG	15	*/
	16	#include <stdlib.h>
	17
	18	#include "include/stringify.h"
	19	#include "ErasureCodeExample.h"
	20	#include "global/global_context.h"
	21	#include "gtest/gtest.h"
	22
1e59de90 TL	23	using namespace std;
1e59de90 TL	24
7c673cae FG	25	TEST(ErasureCodeExample, chunk_size)
	26	{
	27	ErasureCodeExample example;
	28	EXPECT_EQ(3u, example.get_chunk_count());
	29	EXPECT_EQ(11u, example.get_chunk_size(20));
	30	}
	31
	32	TEST(ErasureCodeExample, minimum_to_decode)
	33	{
	34	ErasureCodeExample example;
	35	set<int> available_chunks;
	36	set<int> want_to_read;
	37	want_to_read.insert(1);
	38	{
	39	set<int> minimum;
11fdf7f2	40	EXPECT_EQ(-EIO, example._minimum_to_decode(want_to_read,
7c673cae FG	41	available_chunks,
	42	&minimum));
	43	}
	44	available_chunks.insert(0);
	45	available_chunks.insert(2);
	46	{
	47	set<int> minimum;
11fdf7f2 TL	48	EXPECT_EQ(0, example._minimum_to_decode(want_to_read,
	49	available_chunks,
	50	&minimum));
7c673cae FG	51	EXPECT_EQ(available_chunks, minimum);
	52	EXPECT_EQ(2u, minimum.size());
	53	EXPECT_EQ(1u, minimum.count(0));
	54	EXPECT_EQ(1u, minimum.count(2));
	55	}
	56	{
	57	set<int> minimum;
	58	available_chunks.insert(1);
11fdf7f2 TL	59	EXPECT_EQ(0, example._minimum_to_decode(want_to_read,
	60	available_chunks,
	61	&minimum));
7c673cae FG	62	EXPECT_EQ(1u, minimum.size());
	63	EXPECT_EQ(1u, minimum.count(1));
	64	}
	65	}
	66
	67	TEST(ErasureCodeExample, minimum_to_decode_with_cost)
	68	{
	69	ErasureCodeExample example;
	70	map<int,int> available;
	71	set<int> want_to_read;
	72	want_to_read.insert(1);
	73	{
	74	set<int> minimum;
	75	EXPECT_EQ(-EIO, example.minimum_to_decode_with_cost(want_to_read,
	76	available,
	77	&minimum));
	78	}
	79	available[0] = 1;
	80	available[2] = 1;
	81	{
	82	set<int> minimum;
	83	EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
	84	available,
	85	&minimum));
	86	EXPECT_EQ(2u, minimum.size());
	87	EXPECT_EQ(1u, minimum.count(0));
	88	EXPECT_EQ(1u, minimum.count(2));
	89	}
	90	{
	91	set<int> minimum;
	92	available[1] = 1;
	93	EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
	94	available,
	95	&minimum));
	96	EXPECT_EQ(1u, minimum.size());
	97	EXPECT_EQ(1u, minimum.count(1));
	98	}
	99	{
	100	set<int> minimum;
	101	available[1] = 2;
	102	EXPECT_EQ(0, example.minimum_to_decode_with_cost(want_to_read,
	103	available,
	104	&minimum));
	105	EXPECT_EQ(2u, minimum.size());
	106	EXPECT_EQ(1u, minimum.count(0));
	107	EXPECT_EQ(1u, minimum.count(2));
	108	}
	109	}
	110
	111	TEST(ErasureCodeExample, encode_decode)
	112	{
	113	ErasureCodeExample example;
	114
	115	bufferlist in;
	116	in.append("ABCDE");
	117	set<int> want_to_encode;
	118	for(unsigned int i = 0; i < example.get_chunk_count(); i++)
	119	want_to_encode.insert(i);
	120	map<int, bufferlist> encoded;
	121	EXPECT_EQ(0, example.encode(want_to_encode, in, &encoded));
	122	EXPECT_EQ(example.get_chunk_count(), encoded.size());
	123	EXPECT_EQ(example.get_chunk_size(in.length()), encoded[0].length());
	124	EXPECT_EQ('A', encoded[0][0]);
	125	EXPECT_EQ('B', encoded[0][1]);
126	EXPECT_EQ('C', encoded[0][2]);
127	EXPECT_EQ('D', encoded[1][0]);
128	EXPECT_EQ('E', encoded[1][1]);
129	EXPECT_EQ('A'^'D', encoded[2][0]);
130	EXPECT_EQ('B'^'E', encoded[2][1]);
131	EXPECT_EQ('C'^0, encoded[2][2]);
132
133	// all chunks are available
134	{
135	int want_to_decode[] = { 0, 1 };
136	map<int, bufferlist> decoded;
11fdf7f2 TL	137	EXPECT_EQ(0, example._decode(set<int>(want_to_decode, want_to_decode+2),
	138	encoded,
	139	&decoded));
7c673cae FG	140	EXPECT_EQ(2u, decoded.size());
	141	EXPECT_EQ(3u, decoded[0].length());
	142	EXPECT_EQ('A', decoded[0][0]);
	143	EXPECT_EQ('B', decoded[0][1]);
	144	EXPECT_EQ('C', decoded[0][2]);
	145	EXPECT_EQ('D', decoded[1][0]);
	146	EXPECT_EQ('E', decoded[1][1]);
	147	}
	148
	149	// one chunk is missing
	150	{
	151	map<int, bufferlist> degraded = encoded;
	152	degraded.erase(0);
	153	EXPECT_EQ(2u, degraded.size());
	154	int want_to_decode[] = { 0, 1 };
	155	map<int, bufferlist> decoded;
11fdf7f2 TL	156	EXPECT_EQ(0, example._decode(set<int>(want_to_decode, want_to_decode+2),
	157	degraded,
	158	&decoded));
7c673cae FG	159	EXPECT_EQ(2u, decoded.size());
	160	EXPECT_EQ(3u, decoded[0].length());
	161	EXPECT_EQ('A', decoded[0][0]);
	162	EXPECT_EQ('B', decoded[0][1]);
	163	EXPECT_EQ('C', decoded[0][2]);
	164	EXPECT_EQ('D', decoded[1][0]);
	165	EXPECT_EQ('E', decoded[1][1]);
	166	}
	167	}
	168
	169	TEST(ErasureCodeExample, decode)
	170	{
	171	ErasureCodeExample example;
	172
	173	#define LARGE_ENOUGH 2048
	174	bufferptr in_ptr(buffer::create_page_aligned(LARGE_ENOUGH));
	175	in_ptr.zero();
	176	in_ptr.set_length(0);
	177	const char *payload =
	178	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
	179	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
	180	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
	181	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
	182	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
	183	in_ptr.append(payload, strlen(payload));
	184	bufferlist in;
11fdf7f2	185	in.push_back(in_ptr);
7c673cae FG	186	int want_to_encode[] = { 0, 1, 2 };
	187	map<int, bufferlist> encoded;
	188	EXPECT_EQ(0, example.encode(set<int>(want_to_encode, want_to_encode+3),
	189	in,
	190	&encoded));
	191	EXPECT_EQ(3u, encoded.size());
	192
	193	// successfull decode
	194	bufferlist out;
	195	EXPECT_EQ(0, example.decode_concat(encoded, &out));
	196	bufferlist usable;
	197	usable.substr_of(out, 0, in.length());
	198	EXPECT_TRUE(usable == in);
	199
	200	// cannot recover
	201	map<int, bufferlist> degraded;
	202	degraded[0] = encoded[0];
	203	EXPECT_EQ(-ERANGE, example.decode_concat(degraded, &out));
	204	}
	205
224ce89b	206	TEST(ErasureCodeExample, create_rule)
7c673cae	207	{
11fdf7f2	208	std::unique_ptr<CrushWrapper> c = std::make_unique<CrushWrapper>();
7c673cae FG	209	c->create();
	210	c->set_type_name(2, "root");
	211	c->set_type_name(1, "host");
	212	c->set_type_name(0, "osd");
	213
	214	int rootno;
	215	c->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1,
	216	5, 0, NULL, NULL, &rootno);
	217	c->set_item_name(rootno, "default");
	218
	219	map<string,string> loc;
	220	loc["root"] = "default";
	221
	222	int num_host = 2;
	223	int num_osd = 5;
	224	int osd = 0;
	225	for (int h=0; h<num_host; ++h) {
	226	loc["host"] = string("host-") + stringify(h);
	227	for (int o=0; o<num_osd; ++o, ++osd) {
	228	c->insert_item(g_ceph_context, osd, 1.0, string("osd.") + stringify(osd), loc);
	229	}
	230	}
	231
	232	stringstream ss;
	233	ErasureCodeExample example;
224ce89b	234	EXPECT_EQ(0, example.create_rule("myrule", *c, &ss));
7c673cae FG	235	}
	236
	237	/*
	238	* Local Variables:
	239	* compile-command: "cd ../.. ;
	240	* make -j4 &&
	241	* make unittest_erasure_code_example &&
	242	* valgrind --leak-check=full --tool=memcheck \
	243	* ./unittest_erasure_code_example --gtest_filter=. \
	244	* --log-to-stderr=true --debug-osd=20
	245	* "
	246	* End:
	247	*/
	248