[ceph.git] / ceph / src / test / objectstore / FileStoreDiff.cc

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2012 New Dream Network
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
*/
#include <stdio.h>
#include <stdlib.h>
#include <map>
#include <boost/scoped_ptr.hpp>
#include "common/debug.h"
#include "os/filestore/FileStore.h"
#include "common/config.h"

#include "FileStoreDiff.h"

#define dout_context g_ceph_context
#define dout_subsys ceph_subsys_filestore
#undef dout_prefix
#define dout_prefix *_dout << "filestore_diff "

FileStoreDiff::FileStoreDiff(FileStore *a, FileStore *b)
    : a_store(a), b_store(b)
{
  int err;
  err = a_store->mount();
  ceph_assert(err == 0);

  err = b_store->mount();
  ceph_assert(err == 0);
}

FileStoreDiff::~FileStoreDiff()
{
  a_store->umount();
  b_store->umount();
}


bool FileStoreDiff::diff_attrs(std::map<std::string,bufferptr>& b,
    std::map<std::string,bufferptr>& a)
{
  bool ret = false;
  std::map<std::string, bufferptr>::iterator b_it = b.begin();
  std::map<std::string, bufferptr>::iterator a_it = a.begin();
  for (; b_it != b.end(); ++b_it, ++a_it) {
    if (b_it->first != a_it->first) {
      cout << "diff_attrs name mismatch (verify: " << b_it->first
          << ", store: " << a_it->first << ")" << std::endl;
      ret = true;
      continue;
    }

    if (!b_it->second.cmp(a_it->second)) {
      cout << "diff_attrs contents mismatch on attr " << b_it->first << std::endl;
      ret = true;
      continue;
    }
  }
  return ret;
}

static bool diff_omap(std::map<std::string,bufferlist>& b,
		      std::map<std::string,bufferlist>& a)
{
  bool ret = false;
  std::map<std::string, bufferlist>::iterator b_it = b.begin();
  std::map<std::string, bufferlist>::iterator a_it = a.begin();
  for (; b_it != b.end(); ++b_it, ++a_it) {
    if (a_it == a.end()) {
      cout << __func__ << " a reached end before b, a missing " << b_it->first
	   << std::endl;
      ret = true;
      break;
    }
    if (b_it->first != a_it->first) {
      cout << "diff_attrs name mismatch (verify: " << b_it->first
          << ", store: " << a_it->first << ")" << std::endl;
      ret = true;
      continue;
    }

    if (!(b_it->second == a_it->second)) {
      cout << "diff_attrs contents mismatch on attr " << b_it->first << std::endl;
      ret = true;
      continue;
    }
  }
  return ret;
}

bool FileStoreDiff::diff_objects_stat(struct stat& a, struct stat& b)
{
  bool ret = false;

  if (a.st_uid != b.st_uid) {
    cout << "diff_objects_stat uid mismatch (A: "
        << a.st_uid << " != B: " << b.st_uid << ")" << std::endl;
    ret = true;
  }

  if (a.st_gid != b.st_gid) {
    cout << "diff_objects_stat gid mismatch (A: "
        << a.st_gid << " != B: " << b.st_gid << ")" << std::endl;
    ret = true;
  }

  if (a.st_mode != b.st_mode) {
    cout << "diff_objects_stat mode mismatch (A: "
        << a.st_mode << " != B: " << b.st_mode << ")" << std::endl;
    ret = true;
  }

  if (a.st_nlink != b.st_nlink) {
    cout << "diff_objects_stat nlink mismatch (A: "
        << a.st_nlink << " != B: " << b.st_nlink << ")" << std::endl;
    ret = true;
  }

  if (a.st_size != b.st_size) {
    cout << "diff_objects_stat size mismatch (A: "
        << a.st_size << " != B: " << b.st_size << ")" << std::endl;
    ret = true;
  }
  return ret;
}

bool FileStoreDiff::diff_objects(FileStore *a_store, FileStore *b_store, coll_t coll)
{
  bool ret = false;

  int err;
  std::vector<ghobject_t> b_objects, a_objects;
  err = b_store->collection_list(coll, ghobject_t(), ghobject_t::get_max(),
				 INT_MAX, &b_objects, NULL);
  if (err < 0) {
    cout << "diff_objects list on verify coll " << coll.to_str()
	    << " returns " << err << std::endl;
    return true;
  }
  err = a_store->collection_list(coll, ghobject_t(), ghobject_t::get_max(),
				 INT_MAX, &a_objects, NULL);
  if (err < 0) {
    cout << "diff_objects list on store coll " << coll.to_str()
              << " returns " << err << std::endl;
    return true;
  }

  if (b_objects.size() != a_objects.size()) {
    cout << "diff_objects " << coll << " num objs mismatch (A: " << a_objects.size()
        << ", B: " << b_objects.size() << ")" << std::endl;
    ret = true;
    cout << "a: " << a_objects << std::endl;
    cout << "b: " << b_objects << std::endl;
  }

  auto a_ch = a_store->open_collection(coll);
  auto b_ch = b_store->open_collection(coll);
  std::vector<ghobject_t>::iterator b_it = b_objects.begin();
  std::vector<ghobject_t>::iterator a_it = b_objects.begin();
  for (; b_it != b_objects.end(); ++b_it, ++a_it) {
    ghobject_t b_obj = *b_it, a_obj = *a_it;
    if (b_obj.hobj.oid.name != a_obj.hobj.oid.name) {
      cout << "diff_objects name mismatch on A object "
          << coll << "/" << a_obj << " and B object "
          << coll << "/" << b_obj << std::endl;
      ret = true;
      continue;
    }

    struct stat b_stat, a_stat;
    err = b_store->stat(b_ch, b_obj, &b_stat);
    if (err < 0) {
      cout << "diff_objects error stating B object "
	      << coll.to_str() << "/" << b_obj.hobj.oid.name << std::endl;
      ret = true;
    }
    err = a_store->stat(a_ch, a_obj, &a_stat);
    if (err < 0) {
      cout << "diff_objects error stating A object "
          << coll << "/" << a_obj << std::endl;
      ret = true;
    }

    if (diff_objects_stat(a_stat, b_stat)) {
      cout << "diff_objects stat mismatch on "
          << coll << "/" << b_obj << std::endl;
      ret = true;
    }

    bufferlist a_obj_bl, b_obj_bl;
    b_store->read(b_ch, b_obj, 0, b_stat.st_size, b_obj_bl);
    a_store->read(a_ch, a_obj, 0, a_stat.st_size, a_obj_bl);

    if (!a_obj_bl.contents_equal(b_obj_bl)) {
      cout << "diff_objects content mismatch on "
          << coll << "/" << b_obj << std::endl;
      ret = true;
    }

    std::map<std::string, bufferptr> a_obj_attrs_map, b_obj_attrs_map;
    err = a_store->getattrs(a_ch, a_obj, a_obj_attrs_map);
    if (err < 0) {
      cout << "diff_objects getattrs on A object " << coll << "/" << a_obj
              << " returns " << err << std::endl;
      ret = true;
    }
    err = b_store->getattrs(b_ch, b_obj, b_obj_attrs_map);
    if (err < 0) {
      cout << "diff_objects getattrs on B object " << coll << "/" << b_obj
              << "returns " << err << std::endl;
      ret = true;
    }

    if (diff_attrs(b_obj_attrs_map, a_obj_attrs_map)) {
      cout << "diff_objects attrs mismatch on A object "
          << coll << "/" << a_obj << " and B object "
          << coll << "/" << b_obj << std::endl;
      ret = true;
    }

    std::map<std::string, bufferlist> a_obj_omap, b_obj_omap;
    std::set<std::string> a_omap_keys, b_omap_keys;
    err = a_store->omap_get_keys(a_ch, a_obj, &a_omap_keys);
    if (err < 0) {
      cout << "diff_objects getomap on A object " << coll << "/" << a_obj
              << " returns " << err << std::endl;
      ret = true;
    }
    err = a_store->omap_get_values(a_ch, a_obj, a_omap_keys, &a_obj_omap);
    if (err < 0) {
      cout << "diff_objects getomap on A object " << coll << "/" << a_obj
              << " returns " << err << std::endl;
      ret = true;
    }
    err = b_store->omap_get_keys(b_ch, b_obj, &b_omap_keys);
    if (err < 0) {
      cout << "diff_objects getomap on A object " << coll << "/" << b_obj
              << " returns " << err << std::endl;
      ret = true;
    }
    err = b_store->omap_get_values(b_ch, b_obj, b_omap_keys, &b_obj_omap);
    if (err < 0) {
      cout << "diff_objects getomap on A object " << coll << "/" << b_obj
              << " returns " << err << std::endl;
      ret = true;
    }
    if (diff_omap(a_obj_omap, b_obj_omap)) {
      cout << "diff_objects omap mismatch on A object "
	      << coll << "/" << a_obj << " and B object "
	      << coll << "/" << b_obj << std::endl;
      cout << "a: " << a_obj_omap << std::endl;
      cout << "b: " << b_obj_omap << std::endl;
      ret = true;
    }
  }

  return ret;
}

bool FileStoreDiff::diff()
{
  bool ret = false;

  std::vector<coll_t> a_coll_list, b_coll_list;
  a_store->list_collections(a_coll_list);
  b_store->list_collections(b_coll_list);

  std::vector<coll_t>::iterator it = b_coll_list.begin();
  for (; it != b_coll_list.end(); ++it) {
    coll_t b_coll = *it;
    if (!a_store->collection_exists(b_coll)) {
      cout << "diff B coll " << b_coll.to_str() << " DNE on A" << std::endl;
      ret = true;
      continue;
    }
    for (std::vector<coll_t>::iterator j = a_coll_list.begin();
	 j != a_coll_list.end(); ++j) {
      if (*j == *it) {
	a_coll_list.erase(j);
	break;
      }
    }

    if (diff_objects(a_store, b_store, b_coll))
      ret = true;
  }
  for (std::vector<coll_t>::iterator it = a_coll_list.begin();
       it != a_coll_list.end(); ++it) {
    cout << "diff A coll " << *it << " DNE on B" << std::endl;
    ret = true;
  }

  return ret;
}
Commit	Line	Data
7c673cae FG	1	// -- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t --
	2	// vim: ts=8 sw=2 smarttab
	3	/*
	4	* Ceph - scalable distributed file system
	5	*
	6	* Copyright (C) 2012 New Dream Network
	7	*
	8	* This is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU Lesser General Public
	10	* License version 2.1, as published by the Free Software
	11	* Foundation. See file COPYING.
	12	*/
	13	#include <stdio.h>
	14	#include <stdlib.h>
	15	#include <map>
	16	#include <boost/scoped_ptr.hpp>
	17	#include "common/debug.h"
	18	#include "os/filestore/FileStore.h"
	19	#include "common/config.h"
	20
	21	#include "FileStoreDiff.h"
	22
	23	#define dout_context g_ceph_context
	24	#define dout_subsys ceph_subsys_filestore
	25	#undef dout_prefix
	26	#define dout_prefix *_dout << "filestore_diff "
	27
	28	FileStoreDiff::FileStoreDiff(FileStore a, FileStore b)
	29	: a_store(a), b_store(b)
	30	{
	31	int err;
	32	err = a_store->mount();
	33	ceph_assert(err == 0);
	34
	35	err = b_store->mount();
	36	ceph_assert(err == 0);
	37	}
	38
	39	FileStoreDiff::~FileStoreDiff()
	40	{
	41	a_store->umount();
	42	b_store->umount();
	43	}
	44
	45
	46	bool FileStoreDiff::diff_attrs(std::map<std::string,bufferptr>& b,
	47	std::map<std::string,bufferptr>& a)
	48	{
	49	bool ret = false;
	50	std::map<std::string, bufferptr>::iterator b_it = b.begin();
	51	std::map<std::string, bufferptr>::iterator a_it = a.begin();
	52	for (; b_it != b.end(); ++b_it, ++a_it) {
	53	if (b_it->first != a_it->first) {
28e407b8 AA	54	cout << "diff_attrs name mismatch (verify: " << b_it->first
28e407b8 AA	55	<< ", store: " << a_it->first << ")" << std::endl;
7c673cae FG	56	ret = true;
	57	continue;
	58	}
	59
	60	if (!b_it->second.cmp(a_it->second)) {
28e407b8	61	cout << "diff_attrs contents mismatch on attr " << b_it->first << std::endl;
7c673cae FG	62	ret = true;
	63	continue;
	64	}
	65	}
	66	return ret;
	67	}
	68
	69	static bool diff_omap(std::map<std::string,bufferlist>& b,
	70	std::map<std::string,bufferlist>& a)
	71	{
	72	bool ret = false;
	73	std::map<std::string, bufferlist>::iterator b_it = b.begin();
	74	std::map<std::string, bufferlist>::iterator a_it = a.begin();
	75	for (; b_it != b.end(); ++b_it, ++a_it) {
28e407b8 AA	76	if (a_it == a.end()) {
	77	cout << __func__ << " a reached end before b, a missing " << b_it->first
	78	<< std::endl;
	79	ret = true;
	80	break;
	81	}
7c673cae	82	if (b_it->first != a_it->first) {
28e407b8 AA	83	cout << "diff_attrs name mismatch (verify: " << b_it->first
28e407b8 AA	84	<< ", store: " << a_it->first << ")" << std::endl;
7c673cae FG	85	ret = true;
	86	continue;
	87	}
	88
	89	if (!(b_it->second == a_it->second)) {
28e407b8	90	cout << "diff_attrs contents mismatch on attr " << b_it->first << std::endl;
7c673cae FG	91	ret = true;
	92	continue;
	93	}
	94	}
	95	return ret;
	96	}
	97
	98	bool FileStoreDiff::diff_objects_stat(struct stat& a, struct stat& b)
	99	{
	100	bool ret = false;
	101
	102	if (a.st_uid != b.st_uid) {
28e407b8 AA	103	cout << "diff_objects_stat uid mismatch (A: "
28e407b8 AA	104	<< a.st_uid << " != B: " << b.st_uid << ")" << std::endl;
7c673cae FG	105	ret = true;
	106	}
	107
	108	if (a.st_gid != b.st_gid) {
28e407b8 AA	109	cout << "diff_objects_stat gid mismatch (A: "
28e407b8 AA	110	<< a.st_gid << " != B: " << b.st_gid << ")" << std::endl;
7c673cae FG	111	ret = true;
	112	}
	113
	114	if (a.st_mode != b.st_mode) {
28e407b8 AA	115	cout << "diff_objects_stat mode mismatch (A: "
28e407b8 AA	116	<< a.st_mode << " != B: " << b.st_mode << ")" << std::endl;
7c673cae FG	117	ret = true;
	118	}
	119
	120	if (a.st_nlink != b.st_nlink) {
28e407b8 AA	121	cout << "diff_objects_stat nlink mismatch (A: "
28e407b8 AA	122	<< a.st_nlink << " != B: " << b.st_nlink << ")" << std::endl;
7c673cae FG	123	ret = true;
	124	}
	125
	126	if (a.st_size != b.st_size) {
28e407b8 AA	127	cout << "diff_objects_stat size mismatch (A: "
28e407b8 AA	128	<< a.st_size << " != B: " << b.st_size << ")" << std::endl;
7c673cae FG	129	ret = true;
	130	}
	131	return ret;
	132	}
	133
	134	bool FileStoreDiff::diff_objects(FileStore a_store, FileStore b_store, coll_t coll)
	135	{
7c673cae FG	136	bool ret = false;
	137
	138	int err;
	139	std::vector<ghobject_t> b_objects, a_objects;
	140	err = b_store->collection_list(coll, ghobject_t(), ghobject_t::get_max(),
	141	INT_MAX, &b_objects, NULL);
	142	if (err < 0) {
28e407b8 AA	143	cout << "diff_objects list on verify coll " << coll.to_str()
28e407b8 AA	144	<< " returns " << err << std::endl;
7c673cae FG	145	return true;
	146	}
	147	err = a_store->collection_list(coll, ghobject_t(), ghobject_t::get_max(),
	148	INT_MAX, &a_objects, NULL);
	149	if (err < 0) {
28e407b8 AA	150	cout << "diff_objects list on store coll " << coll.to_str()
28e407b8 AA	151	<< " returns " << err << std::endl;
7c673cae FG	152	return true;
	153	}
	154
	155	if (b_objects.size() != a_objects.size()) {
11fdf7f2	156	cout << "diff_objects " << coll << " num objs mismatch (A: " << a_objects.size()
28e407b8	157	<< ", B: " << b_objects.size() << ")" << std::endl;
7c673cae	158	ret = true;
11fdf7f2 TL	159	cout << "a: " << a_objects << std::endl;
11fdf7f2 TL	160	cout << "b: " << b_objects << std::endl;
7c673cae FG	161	}
7c673cae FG	162
11fdf7f2 TL	163	auto a_ch = a_store->open_collection(coll);
11fdf7f2 TL	164	auto b_ch = b_store->open_collection(coll);
7c673cae FG	165	std::vector<ghobject_t>::iterator b_it = b_objects.begin();
	166	std::vector<ghobject_t>::iterator a_it = b_objects.begin();
	167	for (; b_it != b_objects.end(); ++b_it, ++a_it) {
	168	ghobject_t b_obj = b_it, a_obj = a_it;
	169	if (b_obj.hobj.oid.name != a_obj.hobj.oid.name) {
28e407b8	170	cout << "diff_objects name mismatch on A object "
7c673cae	171	<< coll << "/" << a_obj << " and B object "
28e407b8	172	<< coll << "/" << b_obj << std::endl;
7c673cae FG	173	ret = true;
	174	continue;
	175	}
	176
	177	struct stat b_stat, a_stat;
11fdf7f2	178	err = b_store->stat(b_ch, b_obj, &b_stat);
7c673cae	179	if (err < 0) {
28e407b8 AA	180	cout << "diff_objects error stating B object "
28e407b8 AA	181	<< coll.to_str() << "/" << b_obj.hobj.oid.name << std::endl;
7c673cae FG	182	ret = true;
7c673cae FG	183	}
11fdf7f2	184	err = a_store->stat(a_ch, a_obj, &a_stat);
7c673cae	185	if (err < 0) {
28e407b8 AA	186	cout << "diff_objects error stating A object "
28e407b8 AA	187	<< coll << "/" << a_obj << std::endl;
7c673cae FG	188	ret = true;
	189	}
	190
	191	if (diff_objects_stat(a_stat, b_stat)) {
28e407b8 AA	192	cout << "diff_objects stat mismatch on "
28e407b8 AA	193	<< coll << "/" << b_obj << std::endl;
7c673cae FG	194	ret = true;
	195	}
	196
	197	bufferlist a_obj_bl, b_obj_bl;
11fdf7f2 TL	198	b_store->read(b_ch, b_obj, 0, b_stat.st_size, b_obj_bl);
11fdf7f2 TL	199	a_store->read(a_ch, a_obj, 0, a_stat.st_size, a_obj_bl);
7c673cae FG	200
7c673cae FG	201	if (!a_obj_bl.contents_equal(b_obj_bl)) {
28e407b8 AA	202	cout << "diff_objects content mismatch on "
28e407b8 AA	203	<< coll << "/" << b_obj << std::endl;
7c673cae FG	204	ret = true;
	205	}
	206
	207	std::map<std::string, bufferptr> a_obj_attrs_map, b_obj_attrs_map;
11fdf7f2	208	err = a_store->getattrs(a_ch, a_obj, a_obj_attrs_map);
7c673cae	209	if (err < 0) {
28e407b8 AA	210	cout << "diff_objects getattrs on A object " << coll << "/" << a_obj
28e407b8 AA	211	<< " returns " << err << std::endl;
7c673cae FG	212	ret = true;
7c673cae FG	213	}
11fdf7f2	214	err = b_store->getattrs(b_ch, b_obj, b_obj_attrs_map);
7c673cae	215	if (err < 0) {
28e407b8 AA	216	cout << "diff_objects getattrs on B object " << coll << "/" << b_obj
28e407b8 AA	217	<< "returns " << err << std::endl;
7c673cae FG	218	ret = true;
	219	}
	220
	221	if (diff_attrs(b_obj_attrs_map, a_obj_attrs_map)) {
28e407b8	222	cout << "diff_objects attrs mismatch on A object "
7c673cae	223	<< coll << "/" << a_obj << " and B object "
28e407b8	224	<< coll << "/" << b_obj << std::endl;
7c673cae FG	225	ret = true;
	226	}
	227
	228	std::map<std::string, bufferlist> a_obj_omap, b_obj_omap;
	229	std::set<std::string> a_omap_keys, b_omap_keys;
11fdf7f2	230	err = a_store->omap_get_keys(a_ch, a_obj, &a_omap_keys);
7c673cae	231	if (err < 0) {
28e407b8 AA	232	cout << "diff_objects getomap on A object " << coll << "/" << a_obj
28e407b8 AA	233	<< " returns " << err << std::endl;
7c673cae FG	234	ret = true;
7c673cae FG	235	}
11fdf7f2	236	err = a_store->omap_get_values(a_ch, a_obj, a_omap_keys, &a_obj_omap);
7c673cae	237	if (err < 0) {
28e407b8 AA	238	cout << "diff_objects getomap on A object " << coll << "/" << a_obj
28e407b8 AA	239	<< " returns " << err << std::endl;
7c673cae FG	240	ret = true;
7c673cae FG	241	}
11fdf7f2	242	err = b_store->omap_get_keys(b_ch, b_obj, &b_omap_keys);
7c673cae	243	if (err < 0) {
28e407b8 AA	244	cout << "diff_objects getomap on A object " << coll << "/" << b_obj
28e407b8 AA	245	<< " returns " << err << std::endl;
7c673cae FG	246	ret = true;
7c673cae FG	247	}
11fdf7f2	248	err = b_store->omap_get_values(b_ch, b_obj, b_omap_keys, &b_obj_omap);
7c673cae	249	if (err < 0) {
28e407b8 AA	250	cout << "diff_objects getomap on A object " << coll << "/" << b_obj
28e407b8 AA	251	<< " returns " << err << std::endl;
7c673cae FG	252	ret = true;
	253	}
	254	if (diff_omap(a_obj_omap, b_obj_omap)) {
28e407b8	255	cout << "diff_objects omap mismatch on A object "
7c673cae	256	<< coll << "/" << a_obj << " and B object "
28e407b8 AA	257	<< coll << "/" << b_obj << std::endl;
	258	cout << "a: " << a_obj_omap << std::endl;
	259	cout << "b: " << b_obj_omap << std::endl;
7c673cae FG	260	ret = true;
	261	}
	262	}
	263
	264	return ret;
	265	}
	266
	267	bool FileStoreDiff::diff()
	268	{
	269	bool ret = false;
	270
	271	std::vector<coll_t> a_coll_list, b_coll_list;
	272	a_store->list_collections(a_coll_list);
	273	b_store->list_collections(b_coll_list);
	274
	275	std::vector<coll_t>::iterator it = b_coll_list.begin();
	276	for (; it != b_coll_list.end(); ++it) {
	277	coll_t b_coll = *it;
	278	if (!a_store->collection_exists(b_coll)) {
28e407b8	279	cout << "diff B coll " << b_coll.to_str() << " DNE on A" << std::endl;
7c673cae FG	280	ret = true;
	281	continue;
	282	}
	283	for (std::vector<coll_t>::iterator j = a_coll_list.begin();
	284	j != a_coll_list.end(); ++j) {
	285	if (j == it) {
	286	a_coll_list.erase(j);
	287	break;
	288	}
	289	}
	290
	291	if (diff_objects(a_store, b_store, b_coll))
	292	ret = true;
	293	}
	294	for (std::vector<coll_t>::iterator it = a_coll_list.begin();
	295	it != a_coll_list.end(); ++it) {
28e407b8	296	cout << "diff A coll " << *it << " DNE on B" << std::endl;
7c673cae FG	297	ret = true;
	298	}
	299
	300	return ret;
	301	}