[ceph.git] / ceph / src / test / erasure-code / TestErasureCodeShec_all.cc

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2014,2015 FUJITSU LIMITED
 *
 * Author: Shotaro Kawaguchi <kawaguchi.s@jp.fujitsu.com>
 * Author: Takanori Nakao <nakao.takanori@jp.fujitsu.com>
 * Author: Takeshi Miyamae <miyamae.takeshi@jp.fujitsu.com>
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2.1 of the License, or (at your option) any later version.
 *
 */

// SUMMARY: TestErasureCodeShec combination of k,m,c by 301 patterns

#include <errno.h>
#include <stdlib.h>

#include "crush/CrushWrapper.h"
#include "osd/osd_types.h"
#include "include/stringify.h"
#include "global/global_init.h"
#include "erasure-code/shec/ErasureCodeShec.h"
#include "erasure-code/ErasureCodePlugin.h"
#include "common/ceph_argparse.h"
#include "global/global_context.h"
#include "gtest/gtest.h"

struct Param_d {
  char* k;
  char* m;
  char* c;
  int ch_size;
  char sk[16];
  char sm[16];
  char sc[16];
};
struct Param_d param[301];

unsigned int g_recover = 0;
unsigned int g_cannot_recover = 0;
struct Recover_d {
  int k;
  int m;
  int c;
  set<int> want;
  set<int> avail;
};
struct std::vector<Recover_d> cannot_recover;

class ParameterTest : public ::testing::TestWithParam<struct Param_d> {

};

TEST_P(ParameterTest, parameter_all)
{
  int result;
  //get parameters
  char* k = GetParam().k;
  char* m = GetParam().m;
  char* c = GetParam().c;
  unsigned c_size = GetParam().ch_size;
  int i_k = atoi(k);
  int i_m = atoi(m);
  int i_c = atoi(c);

  //init
  ErasureCodeShecTableCache tcache;
  ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
				  tcache,
				  ErasureCodeShec::MULTIPLE);
  ErasureCodeProfile *profile = new ErasureCodeProfile();
  (*profile)["plugin"] = "shec";
  (*profile)["technique"] = "";
  (*profile)["crush-failure-domain"] = "osd";
  (*profile)["k"] = k;
  (*profile)["m"] = m;
  (*profile)["c"] = c;

  result = shec->init(*profile, &cerr);

  //check profile
  EXPECT_EQ(i_k, shec->k);
  EXPECT_EQ(i_m, shec->m);
  EXPECT_EQ(i_c, shec->c);
  EXPECT_EQ(8, shec->w);
  EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique);
  EXPECT_STREQ("default", shec->rule_root.c_str());
  EXPECT_STREQ("osd", shec->rule_failure_domain.c_str());
  EXPECT_TRUE(shec->matrix != NULL);
  EXPECT_EQ(0, result);

  //minimum_to_decode
  //want_to_decode will be a combination that chooses 1~c from k+m
  set<int> want_to_decode, available_chunks, minimum_chunks;
  int array_want_to_decode[shec->get_chunk_count()];
  struct Recover_d comb;

  for (int w = 1; w <= i_c; w++) {
    const unsigned int r = w;		// combination(k+m,r)

    for (unsigned int i = 0; i < r; ++i) {
      array_want_to_decode[i] = 1;
    }
    for (unsigned int i = r; i < shec->get_chunk_count(); ++i) {
      array_want_to_decode[i] = 0;
    }

    do {
      for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
	available_chunks.insert(i);
      }
      for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
	if (array_want_to_decode[i]) {
	  want_to_decode.insert(i);
	  available_chunks.erase(i);
	}
      }

      result = shec->minimum_to_decode(want_to_decode, available_chunks,
				       &minimum_chunks);

      if (result == 0){
	EXPECT_EQ(0, result);
	EXPECT_TRUE(minimum_chunks.size());
	g_recover++;
      } else {
	EXPECT_EQ(-EIO, result);
	EXPECT_EQ(0u, minimum_chunks.size());
	g_cannot_recover++;
	comb.k = shec->k;
	comb.m = shec->m;
	comb.c = shec->c;
	comb.want = want_to_decode;
	comb.avail = available_chunks;
	cannot_recover.push_back(comb);
      }

      want_to_decode.clear();
      available_chunks.clear();
      minimum_chunks.clear();
    } while (std::prev_permutation(
		 array_want_to_decode,
		 array_want_to_decode + shec->get_chunk_count()));
  }

  //minimum_to_decode_with_cost
  set<int> want_to_decode_with_cost, minimum_chunks_with_cost;
  map<int, int> available_chunks_with_cost;

  for (unsigned int i = 0; i < 1; i++) {
    want_to_decode_with_cost.insert(i);
  }
  for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
    available_chunks_with_cost[i] = i;
  }

  result = shec->minimum_to_decode_with_cost(
      want_to_decode_with_cost,
      available_chunks_with_cost,
      &minimum_chunks_with_cost);
  EXPECT_EQ(0, result);
  EXPECT_TRUE(minimum_chunks_with_cost.size());

  //encode
  bufferlist in;
  set<int> want_to_encode;
  map<int, bufferlist> encoded;

  in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
	    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
	    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
	    "012345"//192
  );
  for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
    want_to_encode.insert(i);
  }

  result = shec->encode(want_to_encode, in, &encoded);
  EXPECT_EQ(0, result);
  EXPECT_EQ(i_k+i_m, (int)encoded.size());
  EXPECT_EQ(c_size, encoded[0].length());

  //decode
  int want_to_decode2[i_k + i_m];
  map<int, bufferlist> decoded;

  for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
    want_to_decode2[i] = i;
  }

  result = shec->decode(set<int>(want_to_decode2, want_to_decode2 + 2),
			encoded, &decoded);
  EXPECT_EQ(0, result);
  EXPECT_EQ(2u, decoded.size());
  EXPECT_EQ(c_size, decoded[0].length());

  //check encoded,decoded
  bufferlist out1, out2, usable;

  //out1 is "encoded"
  for (unsigned int i = 0; i < encoded.size(); i++) {
    out1.append(encoded[i]);
  }

  //out2 is "decoded"
  shec->decode_concat(encoded, &out2);
  usable.substr_of(out2, 0, in.length());

  EXPECT_FALSE(out1 == in);
  EXPECT_TRUE(usable == in);

  //create_rule
  stringstream ss;
  CrushWrapper *crush = new CrushWrapper;
  crush->create();
  crush->set_type_name(2, "root");
  crush->set_type_name(1, "host");
  crush->set_type_name(0, "osd");

  int rootno;
  crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL,
		    NULL, &rootno);
  crush->set_item_name(rootno, "default");

  map < string, string > loc;
  loc["root"] = "default";

  int num_host = 2;
  int num_osd = 5;
  int osd = 0;
  for (int h = 0; h < num_host; ++h) {
    loc["host"] = string("host-") + stringify(h);
    for (int o = 0; o < num_osd; ++o, ++osd) {
      crush->insert_item(g_ceph_context, osd, 1.0,
			 string("osd.") + stringify(osd), loc);
    }
  }

  result = shec->create_rule("myrule", *crush, &ss);
  EXPECT_EQ(0, result);
  EXPECT_STREQ("myrule", crush->rule_name_map[0].c_str());

  //get_chunk_count
  EXPECT_EQ(i_k+i_m, (int)shec->get_chunk_count());

  //get_data_chunk_count
  EXPECT_EQ(i_k, (int)shec->get_data_chunk_count());

  //get_chunk_size
  EXPECT_EQ(c_size, shec->get_chunk_size(192));

  delete shec;
  delete profile;
  delete crush;
}

INSTANTIATE_TEST_CASE_P(Test, ParameterTest, ::testing::ValuesIn(param));

int main(int argc, char **argv)
{
  int i = 0;
  int r;
  const int kObjectSize = 192;
  unsigned alignment, tail, padded_length;
  float recovery_percentage;

  //make_kmc
  for (unsigned int k = 1; k <= 12; k++) {
    for (unsigned int m = 1; (m <= k) && (k + m <= 20); m++) {
      for (unsigned int c = 1; c <= m; c++) {
	sprintf(param[i].sk, "%u", k);
	sprintf(param[i].sm, "%u", m);
	sprintf(param[i].sc, "%u", c);

	param[i].k = param[i].sk;
	param[i].m = param[i].sm;
	param[i].c = param[i].sc;

	alignment = k * 8 * sizeof(int);
	tail = kObjectSize % alignment;
	padded_length = kObjectSize + (tail ? (alignment - tail) : 0);
	param[i].ch_size = padded_length / k;
	i++;
      }
    }
  }

  vector<const char*> args;
  argv_to_vec(argc, (const char **) argv, args);

  auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
			 CODE_ENVIRONMENT_UTILITY, 0);
  common_init_finish(g_ceph_context);

  const char* env = getenv("CEPH_LIB");
  string directory(env ? env : ".libs");
  g_conf->set_val("erasure_code_dir", directory, false);

  ::testing::InitGoogleTest(&argc, argv);

  r = RUN_ALL_TESTS();

  std::cout << "minimum_to_decode:recover_num = " << g_recover << std::endl;
  std::cout << "minimum_to_decode:cannot_recover_num = " << g_cannot_recover
      << std::endl;
  recovery_percentage = 100.0
      - (float) (100.0 * g_cannot_recover / (g_recover + g_cannot_recover));
  printf("recovery_percentage:%f\n",recovery_percentage);
  if (recovery_percentage > 99.0) {
    std::cout << "[       OK ] Recovery percentage is more than 99.0%"
	<< std::endl;
  } else {
    std::cout << "[       NG ] Recovery percentage is less than 99.0%"
	<< std::endl;
  }
  std::cout << "cannot recovery patterns:" << std::endl;
  for (std::vector<Recover_d>::const_iterator i = cannot_recover.begin();
       i != cannot_recover.end(); ++i) {
    std::cout << "---" << std::endl;
    std::cout << "k = " << i->k << ", m = " << i->m << ", c = " << i->c
	<< std::endl;
    std::cout << "want_to_decode  :" << i->want << std::endl;
    std::cout << "available_chunks:" << i->avail << std::endl;
  }
  std::cout << "---" << std::endl;

  return r;
}
Commit	Line	Data
7c673cae FG	1	// -- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t --
	2	// vim: ts=8 sw=2 smarttab
	3	/*
	4	* Ceph - scalable distributed file system
	5	*
	6	* Copyright (C) 2014,2015 FUJITSU LIMITED
	7	*
	8	* Author: Shotaro Kawaguchi <kawaguchi.s@jp.fujitsu.com>
	9	* Author: Takanori Nakao <nakao.takanori@jp.fujitsu.com>
	10	* Author: Takeshi Miyamae <miyamae.takeshi@jp.fujitsu.com>
	11	*
	12	* This library is free software; you can redistribute it and/or
	13	* modify it under the terms of the GNU Lesser General Public
	14	* License as published by the Free Software Foundation; either
	15	* version 2.1 of the License, or (at your option) any later version.
	16	*
	17	*/
	18
	19	// SUMMARY: TestErasureCodeShec combination of k,m,c by 301 patterns
	20
	21	#include <errno.h>
	22	#include <stdlib.h>
	23
	24	#include "crush/CrushWrapper.h"
	25	#include "osd/osd_types.h"
	26	#include "include/stringify.h"
	27	#include "global/global_init.h"
	28	#include "erasure-code/shec/ErasureCodeShec.h"
	29	#include "erasure-code/ErasureCodePlugin.h"
	30	#include "common/ceph_argparse.h"
	31	#include "global/global_context.h"
	32	#include "gtest/gtest.h"
	33
	34	struct Param_d {
	35	char* k;
	36	char* m;
	37	char* c;
	38	int ch_size;
	39	char sk[16];
	40	char sm[16];
	41	char sc[16];
	42	};
	43	struct Param_d param[301];
	44
	45	unsigned int g_recover = 0;
	46	unsigned int g_cannot_recover = 0;
	47	struct Recover_d {
	48	int k;
	49	int m;
	50	int c;
	51	set<int> want;
	52	set<int> avail;
	53	};
	54	struct std::vector<Recover_d> cannot_recover;
	55
	56	class ParameterTest : public ::testing::TestWithParam<struct Param_d> {
	57
	58	};
	59
	60	TEST_P(ParameterTest, parameter_all)
	61	{
	62	int result;
	63	//get parameters
	64	char* k = GetParam().k;
65	char* m = GetParam().m;
66	char* c = GetParam().c;
67	unsigned c_size = GetParam().ch_size;
68	int i_k = atoi(k);
69	int i_m = atoi(m);
70	int i_c = atoi(c);
71
72	//init
73	ErasureCodeShecTableCache tcache;
74	ErasureCodeShec* shec = new ErasureCodeShecReedSolomonVandermonde(
75	tcache,
76	ErasureCodeShec::MULTIPLE);
77	ErasureCodeProfile *profile = new ErasureCodeProfile();
78	(*profile)["plugin"] = "shec";
79	(*profile)["technique"] = "";
224ce89b	80	(*profile)["crush-failure-domain"] = "osd";
7c673cae FG	81	(*profile)["k"] = k;
	82	(*profile)["m"] = m;
	83	(*profile)["c"] = c;
	84
	85	result = shec->init(*profile, &cerr);
	86
	87	//check profile
	88	EXPECT_EQ(i_k, shec->k);
	89	EXPECT_EQ(i_m, shec->m);
	90	EXPECT_EQ(i_c, shec->c);
	91	EXPECT_EQ(8, shec->w);
	92	EXPECT_EQ(ErasureCodeShec::MULTIPLE, shec->technique);
224ce89b WB	93	EXPECT_STREQ("default", shec->rule_root.c_str());
224ce89b WB	94	EXPECT_STREQ("osd", shec->rule_failure_domain.c_str());
7c673cae FG	95	EXPECT_TRUE(shec->matrix != NULL);
	96	EXPECT_EQ(0, result);
	97
	98	//minimum_to_decode
	99	//want_to_decode will be a combination that chooses 1~c from k+m
	100	set<int> want_to_decode, available_chunks, minimum_chunks;
	101	int array_want_to_decode[shec->get_chunk_count()];
	102	struct Recover_d comb;
	103
	104	for (int w = 1; w <= i_c; w++) {
	105	const unsigned int r = w; // combination(k+m,r)
	106
	107	for (unsigned int i = 0; i < r; ++i) {
	108	array_want_to_decode[i] = 1;
	109	}
	110	for (unsigned int i = r; i < shec->get_chunk_count(); ++i) {
	111	array_want_to_decode[i] = 0;
	112	}
	113
	114	do {
	115	for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
	116	available_chunks.insert(i);
	117	}
	118	for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
	119	if (array_want_to_decode[i]) {
	120	want_to_decode.insert(i);
	121	available_chunks.erase(i);
	122	}
	123	}
	124
	125	result = shec->minimum_to_decode(want_to_decode, available_chunks,
	126	&minimum_chunks);
	127
	128	if (result == 0){
	129	EXPECT_EQ(0, result);
	130	EXPECT_TRUE(minimum_chunks.size());
	131	g_recover++;
	132	} else {
	133	EXPECT_EQ(-EIO, result);
	134	EXPECT_EQ(0u, minimum_chunks.size());
	135	g_cannot_recover++;
	136	comb.k = shec->k;
	137	comb.m = shec->m;
	138	comb.c = shec->c;
	139	comb.want = want_to_decode;
	140	comb.avail = available_chunks;
	141	cannot_recover.push_back(comb);
	142	}
	143
	144	want_to_decode.clear();
	145	available_chunks.clear();
	146	minimum_chunks.clear();
	147	} while (std::prev_permutation(
	148	array_want_to_decode,
	149	array_want_to_decode + shec->get_chunk_count()));
	150	}
	151
	152	//minimum_to_decode_with_cost
	153	set<int> want_to_decode_with_cost, minimum_chunks_with_cost;
	154	map<int, int> available_chunks_with_cost;
	155
	156	for (unsigned int i = 0; i < 1; i++) {
	157	want_to_decode_with_cost.insert(i);
	158	}
159	for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
160	available_chunks_with_cost[i] = i;
161	}
162
163	result = shec->minimum_to_decode_with_cost(
164	want_to_decode_with_cost,
165	available_chunks_with_cost,
166	&minimum_chunks_with_cost);
167	EXPECT_EQ(0, result);
168	EXPECT_TRUE(minimum_chunks_with_cost.size());
169
170	//encode
171	bufferlist in;
172	set<int> want_to_encode;
173	map<int, bufferlist> encoded;
174
175	in.append("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//length = 62
176	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//124
177	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"//186
178	"012345"//192
179	);
180	for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
181	want_to_encode.insert(i);
182	}
183
184	result = shec->encode(want_to_encode, in, &encoded);
185	EXPECT_EQ(0, result);
186	EXPECT_EQ(i_k+i_m, (int)encoded.size());
187	EXPECT_EQ(c_size, encoded[0].length());
188
189	//decode
190	int want_to_decode2[i_k + i_m];
191	map<int, bufferlist> decoded;
192
193	for (unsigned int i = 0; i < shec->get_chunk_count(); i++) {
194	want_to_decode2[i] = i;
195	}
196
197	result = shec->decode(set<int>(want_to_decode2, want_to_decode2 + 2),
198	encoded, &decoded);
199	EXPECT_EQ(0, result);
200	EXPECT_EQ(2u, decoded.size());
201	EXPECT_EQ(c_size, decoded[0].length());
202
203	//check encoded,decoded
204	bufferlist out1, out2, usable;
205
206	//out1 is "encoded"
207	for (unsigned int i = 0; i < encoded.size(); i++) {
208	out1.append(encoded[i]);
209	}
210
211	//out2 is "decoded"
212	shec->decode_concat(encoded, &out2);
213	usable.substr_of(out2, 0, in.length());
214
215	EXPECT_FALSE(out1 == in);
216	EXPECT_TRUE(usable == in);
217
224ce89b	218	//create_rule
7c673cae FG	219	stringstream ss;
	220	CrushWrapper *crush = new CrushWrapper;
	221	crush->create();
	222	crush->set_type_name(2, "root");
	223	crush->set_type_name(1, "host");
	224	crush->set_type_name(0, "osd");
	225
	226	int rootno;
	227	crush->add_bucket(0, CRUSH_BUCKET_STRAW, CRUSH_HASH_RJENKINS1, 2, 0, NULL,
	228	NULL, &rootno);
	229	crush->set_item_name(rootno, "default");
	230
	231	map < string, string > loc;
	232	loc["root"] = "default";
	233
	234	int num_host = 2;
	235	int num_osd = 5;
	236	int osd = 0;
	237	for (int h = 0; h < num_host; ++h) {
	238	loc["host"] = string("host-") + stringify(h);
	239	for (int o = 0; o < num_osd; ++o, ++osd) {
	240	crush->insert_item(g_ceph_context, osd, 1.0,
	241	string("osd.") + stringify(osd), loc);
	242	}
	243	}
	244
224ce89b	245	result = shec->create_rule("myrule", *crush, &ss);
7c673cae FG	246	EXPECT_EQ(0, result);
	247	EXPECT_STREQ("myrule", crush->rule_name_map[0].c_str());
	248
	249	//get_chunk_count
	250	EXPECT_EQ(i_k+i_m, (int)shec->get_chunk_count());
	251
	252	//get_data_chunk_count
	253	EXPECT_EQ(i_k, (int)shec->get_data_chunk_count());
	254
	255	//get_chunk_size
	256	EXPECT_EQ(c_size, shec->get_chunk_size(192));
	257
	258	delete shec;
	259	delete profile;
	260	delete crush;
	261	}
	262
	263	INSTANTIATE_TEST_CASE_P(Test, ParameterTest, ::testing::ValuesIn(param));
	264
	265	int main(int argc, char **argv)
	266	{
	267	int i = 0;
	268	int r;
	269	const int kObjectSize = 192;
	270	unsigned alignment, tail, padded_length;
	271	float recovery_percentage;
	272
	273	//make_kmc
	274	for (unsigned int k = 1; k <= 12; k++) {
	275	for (unsigned int m = 1; (m <= k) && (k + m <= 20); m++) {
	276	for (unsigned int c = 1; c <= m; c++) {
	277	sprintf(param[i].sk, "%u", k);
	278	sprintf(param[i].sm, "%u", m);
	279	sprintf(param[i].sc, "%u", c);
	280
	281	param[i].k = param[i].sk;
	282	param[i].m = param[i].sm;
	283	param[i].c = param[i].sc;
	284
	285	alignment = k * 8 * sizeof(int);
	286	tail = kObjectSize % alignment;
	287	padded_length = kObjectSize + (tail ? (alignment - tail) : 0);
	288	param[i].ch_size = padded_length / k;
	289	i++;
	290	}
	291	}
	292	}
	293
	294	vector<const char*> args;
	295	argv_to_vec(argc, (const char **) argv, args);
	296
	297	auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
	298	CODE_ENVIRONMENT_UTILITY, 0);
	299	common_init_finish(g_ceph_context);
	300
	301	const char* env = getenv("CEPH_LIB");
	302	string directory(env ? env : ".libs");
	303	g_conf->set_val("erasure_code_dir", directory, false);
	304
	305	::testing::InitGoogleTest(&argc, argv);
	306
	307	r = RUN_ALL_TESTS();
	308
	309	std::cout << "minimum_to_decode:recover_num = " << g_recover << std::endl;
310	std::cout << "minimum_to_decode:cannot_recover_num = " << g_cannot_recover
311	<< std::endl;
312	recovery_percentage = 100.0
313	- (float) (100.0 * g_cannot_recover / (g_recover + g_cannot_recover));
314	printf("recovery_percentage:%f\n",recovery_percentage);
315	if (recovery_percentage > 99.0) {
316	std::cout << "[ OK ] Recovery percentage is more than 99.0%"
317	<< std::endl;
318	} else {
319	std::cout << "[ NG ] Recovery percentage is less than 99.0%"
320	<< std::endl;
321	}
322	std::cout << "cannot recovery patterns:" << std::endl;
323	for (std::vector<Recover_d>::const_iterator i = cannot_recover.begin();
324	i != cannot_recover.end(); ++i) {
325	std::cout << "---" << std::endl;
326	std::cout << "k = " << i->k << ", m = " << i->m << ", c = " << i->c
327	<< std::endl;
328	std::cout << "want_to_decode :" << i->want << std::endl;
329	std::cout << "available_chunks:" << i->avail << std::endl;
330	}
331	std::cout << "---" << std::endl;
332
333	return r;
334	}