[ceph.git] / ceph / src / test / librgw_file_marker.cc

// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
 * Ceph - scalable distributed file system
 *
 * Copyright (C) 2015 Red Hat, Inc.
 *
 * This is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License version 2.1, as published by the Free Software
 * Foundation.  See file COPYING.
 *
 */

#include <stdint.h>
#include <tuple>
#include <iostream>
#include <fstream>
#include <stack>

#include "include/rados/librgw.h"
#include "include/rados/rgw_file.h"
#include "rgw/rgw_file.h"
#include "rgw/rgw_lib_frontend.h" // direct requests

#include "gtest/gtest.h"
#include "common/ceph_argparse.h"
#include "common/debug.h"
#include "global/global_init.h"
#include "include/ceph_assert.h"

#define dout_subsys ceph_subsys_rgw

using namespace std;

namespace {

  using namespace rgw;
  using std::get;
  using std::string;

  librgw_t rgw_h = nullptr;
  string userid("testuser");
  string access_key("");
  string secret_key("");
  struct rgw_fs *fs = nullptr;
  CephContext* cct = nullptr;

  uint32_t owner_uid = 867;
  uint32_t owner_gid = 5309;

  uint32_t create_mask = RGW_SETATTR_UID | RGW_SETATTR_GID | RGW_SETATTR_MODE;

  string bucket_name("dmarker");

  class obj_rec
  {
  public:
    string name;
    struct rgw_file_handle* fh;
    struct rgw_file_handle* parent_fh;
    RGWFileHandle* rgw_fh; // alias into fh

    struct state {
      bool readdir;
      state() : readdir(false) {}
    } state;

    obj_rec(string _name, struct rgw_file_handle* _fh,
	    struct rgw_file_handle* _parent_fh, RGWFileHandle* _rgw_fh)
      : name(std::move(_name)), fh(_fh), parent_fh(_parent_fh),
	rgw_fh(_rgw_fh) {}

    void clear() {
      fh = nullptr;
      rgw_fh = nullptr;
    }

    void sync() {
      if (fh)
	rgw_fh = get_rgwfh(fh);
    }

    friend ostream& operator<<(ostream& os, const obj_rec& rec);
  };

  /* Unused
  ostream& operator<<(ostream& os, const obj_rec& rec)
  {
    RGWFileHandle* rgw_fh = rec.rgw_fh;
    if (rgw_fh) {
      const char* type = rgw_fh->is_dir() ? "DIR " : "FILE ";
      os << rec.rgw_fh->full_object_name()
	 << " (" << rec.rgw_fh->object_name() << "): "
	 << type;
    }
    return os;
  }
  */

  std::stack<obj_rec> obj_stack;
  std::deque<obj_rec> cleanup_queue;

  typedef std::vector<obj_rec> obj_vec;
  typedef std::tuple<obj_rec, obj_vec> dirs1_rec;
  typedef std::vector<dirs1_rec> dirs1_vec;

  dirs1_vec dirs_vec;

  struct obj_rec_st
  {
    const obj_rec& obj;
    const struct stat& st;

    obj_rec_st(const obj_rec& _obj, const struct stat& _st)
      : obj(_obj), st(_st) {}
  };

  /* Unused
  ostream& operator<<(ostream& os, const obj_rec_st& rec)
  {
    RGWFileHandle* rgw_fh = rec.obj.rgw_fh;
    if (rgw_fh) {
      const char* type = rgw_fh->is_dir() ? "DIR " : "FILE ";
      os << rgw_fh->full_object_name()
	 << " (" << rgw_fh->object_name() << "): "
	 << type;
      const struct stat& st = rec.st;
      switch(uint8_t(rgw_fh->is_dir())) {
      case 1:
	os << " mode: " << st.st_mode;
	os << " nlinks: " << st.st_nlink;
	break;
      case 0:
      default:
	os << " mode: " << st.st_mode;
	os << " size: " << st.st_size;
	// xxx
	break;
      }
    }
    return os;
  }
  */

  bool do_marker1 = false;
  bool do_marker2 = true;
  bool do_create = false;
  bool do_delete = false;
  bool verbose = false;

  string marker_dir("nfs_marker");
  struct rgw_file_handle *bucket_fh = nullptr;
  struct rgw_file_handle *marker_fh;
  uint32_t marker_nobjs = 2*1024;
  std::deque<obj_rec> marker_objs;

  using dirent_t = std::tuple<std::string, uint64_t>;
  struct dirent_vec
  {
    std::vector<dirent_t> obj_names;
    uint32_t count;
    dirent_vec() : count(0) {}
  };

  struct {
    int argc;
    char **argv;
  } saved_args;
}

TEST(LibRGW, TVAR) {
  typedef boost::variant<uint64_t*, const char*> readdir_offset;

  uint64_t i1{64001};
  std::string s1{"blunderbuss"};

  readdir_offset v1{&i1};
  readdir_offset v2{s1.c_str()};
  readdir_offset v3{static_cast<const char*>(nullptr)};

  uint64_t* pi1 = get<uint64_t*>(v1);
  ASSERT_NE(pi1, nullptr);
  std::cout << "read i1: " << *pi1 << std::endl;

  const char* ps1 = get<const char*>(v2);
  ASSERT_NE(ps1, nullptr);
  std::cout << "read s1: " << ps1 << std::endl;

  const char* ps3 = get<const char*>(v3);
  ASSERT_EQ(ps3, nullptr);
  std::cout << "read s3: " << ps3 << std::endl;
}

TEST(LibRGW, INIT) {
  int ret = librgw_create(&rgw_h, saved_args.argc, saved_args.argv);
  ASSERT_EQ(ret, 0);
  ASSERT_NE(rgw_h, nullptr);
}

TEST(LibRGW, MOUNT) {
  int ret = rgw_mount2(rgw_h, userid.c_str(), access_key.c_str(),
                       secret_key.c_str(), "/", &fs, RGW_MOUNT_FLAG_NONE);
  ASSERT_EQ(ret, 0);
  ASSERT_NE(fs, nullptr);

  cct = static_cast<RGWLibFS*>(fs->fs_private)->get_context();
}

TEST(LibRGW, MARKER1_SETUP_BUCKET) {
  /* "large" directory enumeration test.  this one deals only with
   * file objects */
  struct stat st;
  int ret;

  st.st_uid = owner_uid;
  st.st_gid = owner_gid;
  st.st_mode = 755;

  (void) rgw_lookup(fs, fs->root_fh, bucket_name.c_str(), &bucket_fh,
		    nullptr, 0, RGW_LOOKUP_FLAG_NONE);
  if (! bucket_fh) {
    if (do_create) {
      struct stat st;

      st.st_uid = owner_uid;
      st.st_gid = owner_gid;
      st.st_mode = 755;

      ret = rgw_mkdir(fs, fs->root_fh, bucket_name.c_str(), &st, create_mask,
		      &bucket_fh, RGW_MKDIR_FLAG_NONE);
      ASSERT_EQ(ret, 0);
    }
  }

  ASSERT_NE(bucket_fh, nullptr);

  (void) rgw_lookup(fs, bucket_fh, marker_dir.c_str(), &marker_fh,
		    nullptr, 0, RGW_LOOKUP_FLAG_NONE);
  if (! marker_fh) {
    if (do_create) {
      ret = rgw_mkdir(fs, bucket_fh, marker_dir.c_str(), &st, create_mask,
		      &marker_fh, RGW_MKDIR_FLAG_NONE);
      ASSERT_EQ(ret, 0);
    }
  }

  ASSERT_NE(marker_fh, nullptr);
}

TEST(LibRGW, MARKER1_SETUP_OBJECTS)
{
  /* "large" directory enumeration test.  this one deals only with
   * file objects */

  if (do_create) {
    int ret;

    for (uint32_t ix = 0; ix < marker_nobjs; ++ix) {
      std::string object_name("f_");
      object_name += to_string(ix);
      obj_rec obj{object_name, nullptr, marker_fh, nullptr};
      // lookup object--all operations are by handle
      ret = rgw_lookup(fs, marker_fh, obj.name.c_str(), &obj.fh,
		       nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
      ASSERT_EQ(ret, 0);
      obj.rgw_fh = get_rgwfh(obj.fh);
      // open object--open transaction
      ret = rgw_open(fs, obj.fh, 0 /* posix flags */, RGW_OPEN_FLAG_NONE);
      ASSERT_EQ(ret, 0);
      ASSERT_TRUE(obj.rgw_fh->is_open());
      // unstable write data
      size_t nbytes;
      string data("data for ");
      data += object_name;
      int ret = rgw_write(fs, obj.fh, 0, data.length(), &nbytes,
			  (void*) data.c_str(), RGW_WRITE_FLAG_NONE);
      ASSERT_EQ(ret, 0);
      ASSERT_EQ(nbytes, data.length());
      // commit transaction (write on close)
      ret = rgw_close(fs, obj.fh, 0 /* flags */);
      ASSERT_EQ(ret, 0);
      if (verbose) {
	/* XXX std:cout fragged...did it get /0 in the stream
	 * somewhere? */
	printf("created: %s:%s\n", bucket_name.c_str(), obj.name.c_str());
      }
      // save for cleanup
      marker_objs.push_back(obj);
    }
  }
}

extern "C" {
  static bool r2_cb(const char* name, void *arg, uint64_t offset,
		    struct stat* st, uint32_t st_mask,
		    uint32_t flags) {
    dirent_vec& dvec =
      *(static_cast<dirent_vec*>(arg));

    printf("%s bucket=%s dir=%s iv count=%d called back name=%s flags=%d\n",
	   __func__,
	   bucket_name.c_str(),
	   marker_dir.c_str(),
	   dvec.count,
	   name,
	   flags);

    string name_str{name};
    if (! ((name_str == ".") ||
	   (name_str == ".."))) {
      dvec.obj_names.push_back(dirent_t{std::move(name_str), offset});
    }
    return true; /* XXX */
  }
}

TEST(LibRGW, MARKER1_READDIR)
{
  if (do_marker1) {
    using std::get;

    dirent_vec dvec;
    uint64_t offset = 0;
    bool eof = false;

    /* because RGWReaddirRequest::default_max is 1000 (XXX make
     * configurable?) and marker_nobjs is 5*1024, the number
     * of required rgw_readdir operations N should be
     * marker_nobjs/1000 < N < marker_nobjs/1000+1, i.e., 6 when
     * marker_nobjs==5*1024 */
    uint32_t max_iterations = marker_nobjs/1000+1;

    do {
      ASSERT_TRUE(dvec.count <= max_iterations);
      int ret = rgw_readdir(fs, marker_fh, &offset, r2_cb, &dvec, &eof,
			    RGW_READDIR_FLAG_DOTDOT);
      ASSERT_EQ(ret, 0);
      ASSERT_GE(dvec.obj_names.size(), 0);
      ASSERT_EQ(offset, get<1>(dvec.obj_names.back())); // cookie check
      ++dvec.count;
    } while(!eof);
    std::cout << "Read " << dvec.obj_names.size() << " objects in "
	      << marker_dir.c_str() << std::endl;
  }
}

TEST(LibRGW, MARKER2_READDIR)
{
  if (do_marker2) {
    using std::get;

    dirent_vec dvec;
    std::string marker{""};
    bool eof = false;

    /* because RGWReaddirRequest::default_max is 1000 (XXX make
     * configurable?) and marker_nobjs is 5*1024, the number
     * of required rgw_readdir operations N should be
     * marker_nobjs/1000 < N < marker_nobjs/1000+1, i.e., 6 when
     * marker_nobjs==5*1024 */
    uint32_t max_iterations = marker_nobjs/1000+1;

    do {
      ASSERT_TRUE(dvec.count <= max_iterations);
      int ret = rgw_readdir2(fs, marker_fh,
			     (marker.length() > 0) ? marker.c_str() : nullptr,
			     r2_cb, &dvec, &eof,
			     RGW_READDIR_FLAG_NONE);
      ASSERT_EQ(ret, 0);
      ASSERT_GE(dvec.obj_names.size(), 0);
      marker = get<0>(dvec.obj_names.back());
      ++dvec.count;
    } while((!eof) && dvec.count < 4);
    std::cout << "Read " << dvec.obj_names.size() << " objects in "
	      << marker_dir.c_str() << std::endl;
  }
}

TEST(LibRGW, MARKER1_OBJ_CLEANUP)
{
  int rc;
  for (auto& obj : marker_objs) {
    if (obj.fh) {
      if (do_delete) {
	if (verbose) {
	  std::cout << "unlinking: " << bucket_name << ":" << obj.name
		    << std::endl;
	}
	rc = rgw_unlink(fs, marker_fh, obj.name.c_str(), RGW_UNLINK_FLAG_NONE);
      }
      rc = rgw_fh_rele(fs, obj.fh, 0 /* flags */);
      ASSERT_EQ(rc, 0);
    }
  }
  marker_objs.clear();
}

TEST(LibRGW, CLEANUP) {
  int rc;

  if (do_marker1) {
    cleanup_queue.push_back(
      obj_rec{bucket_name, bucket_fh, fs->root_fh, get_rgwfh(fs->root_fh)});
  }

  for (auto& elt : cleanup_queue) {
    if (elt.fh) {
      rc = rgw_fh_rele(fs, elt.fh, 0 /* flags */);
      ASSERT_EQ(rc, 0);
    }
  }
  cleanup_queue.clear();
}

TEST(LibRGW, UMOUNT) {
  if (! fs)
    return;

  int ret = rgw_umount(fs, RGW_UMOUNT_FLAG_NONE);
  ASSERT_EQ(ret, 0);
}

TEST(LibRGW, SHUTDOWN) {
  librgw_shutdown(rgw_h);
}

int main(int argc, char *argv[])
{
  auto args = argv_to_vec(argc, argv);
  env_to_vec(args);

  char* v = getenv("AWS_ACCESS_KEY_ID");
  if (v) {
    access_key = v;
  }

  v = getenv("AWS_SECRET_ACCESS_KEY");
  if (v) {
    secret_key = v;
  }

  string val;
  for (auto arg_iter = args.begin(); arg_iter != args.end();) {
    if (ceph_argparse_witharg(args, arg_iter, &val, "--access",
			      (char*) nullptr)) {
      access_key = val;
    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--secret",
				     (char*) nullptr)) {
      secret_key = val;
    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--userid",
				     (char*) nullptr)) {
      userid = val;
    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--bn",
				     (char*) nullptr)) {
      bucket_name = val;
    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--uid",
				     (char*) nullptr)) {
      owner_uid = std::stoi(val);
    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--gid",
				     (char*) nullptr)) {
      owner_gid = std::stoi(val);
    } else if (ceph_argparse_witharg(args, arg_iter, &val, "--nobjs",
				     (char*) nullptr)) {
      marker_nobjs = std::stoi(val);
    } else if (ceph_argparse_flag(args, arg_iter, "--marker1",
					    (char*) nullptr)) {
      do_marker1 = true;
    } else if (ceph_argparse_flag(args, arg_iter, "--create",
					    (char*) nullptr)) {
      do_create = true;
    } else if (ceph_argparse_flag(args, arg_iter, "--delete",
					    (char*) nullptr)) {
      do_delete = true;
    } else if (ceph_argparse_flag(args, arg_iter, "--verbose",
					    (char*) nullptr)) {
      verbose = true;
    } else {
      ++arg_iter;
    }
  }

  /* don't accidentally run as anonymous */
  if ((access_key == "") ||
      (secret_key == "")) {
    std::cout << argv[0] << " no AWS credentials, exiting" << std::endl;
    return EPERM;
  }

  saved_args.argc = argc;
  saved_args.argv = argv;

  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
}
Commit	Line	Data
3efd9988 FG	1	// -- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t --
	2	// vim: ts=8 sw=2 smarttab
	3	/*
	4	* Ceph - scalable distributed file system
	5	*
	6	* Copyright (C) 2015 Red Hat, Inc.
	7	*
	8	* This is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU Lesser General Public
	10	* License version 2.1, as published by the Free Software
	11	* Foundation. See file COPYING.
	12	*
	13	*/
	14
	15	#include <stdint.h>
	16	#include <tuple>
	17	#include <iostream>
	18	#include <fstream>
	19	#include <stack>
	20
	21	#include "include/rados/librgw.h"
	22	#include "include/rados/rgw_file.h"
	23	#include "rgw/rgw_file.h"
	24	#include "rgw/rgw_lib_frontend.h" // direct requests
	25
	26	#include "gtest/gtest.h"
3efd9988 FG	27	#include "common/ceph_argparse.h"
	28	#include "common/debug.h"
	29	#include "global/global_init.h"
11fdf7f2	30	#include "include/ceph_assert.h"
3efd9988 FG	31
	32	#define dout_subsys ceph_subsys_rgw
	33
20effc67 TL	34	using namespace std;
20effc67 TL	35
3efd9988 FG	36	namespace {
	37
	38	using namespace rgw;
	39	using std::get;
	40	using std::string;
	41
	42	librgw_t rgw_h = nullptr;
	43	string userid("testuser");
	44	string access_key("");
	45	string secret_key("");
	46	struct rgw_fs *fs = nullptr;
	47	CephContext* cct = nullptr;
	48
	49	uint32_t owner_uid = 867;
	50	uint32_t owner_gid = 5309;
	51
	52	uint32_t create_mask = RGW_SETATTR_UID \| RGW_SETATTR_GID \| RGW_SETATTR_MODE;
	53
f67539c2	54	string bucket_name("dmarker");
3efd9988 FG	55
	56	class obj_rec
	57	{
	58	public:
	59	string name;
	60	struct rgw_file_handle* fh;
	61	struct rgw_file_handle* parent_fh;
	62	RGWFileHandle* rgw_fh; // alias into fh
	63
	64	struct state {
	65	bool readdir;
	66	state() : readdir(false) {}
	67	} state;
	68
	69	obj_rec(string _name, struct rgw_file_handle* _fh,
	70	struct rgw_file_handle* _parent_fh, RGWFileHandle* _rgw_fh)
	71	: name(std::move(_name)), fh(_fh), parent_fh(_parent_fh),
	72	rgw_fh(_rgw_fh) {}
	73
	74	void clear() {
	75	fh = nullptr;
	76	rgw_fh = nullptr;
	77	}
	78
	79	void sync() {
	80	if (fh)
	81	rgw_fh = get_rgwfh(fh);
	82	}
	83
	84	friend ostream& operator<<(ostream& os, const obj_rec& rec);
	85	};
	86
11fdf7f2	87	/* Unused
3efd9988 FG	88	ostream& operator<<(ostream& os, const obj_rec& rec)
	89	{
	90	RGWFileHandle* rgw_fh = rec.rgw_fh;
	91	if (rgw_fh) {
	92	const char* type = rgw_fh->is_dir() ? "DIR " : "FILE ";
	93	os << rec.rgw_fh->full_object_name()
	94	<< " (" << rec.rgw_fh->object_name() << "): "
	95	<< type;
	96	}
	97	return os;
	98	}
11fdf7f2 TL	99	*/
11fdf7f2 TL	100
3efd9988 FG	101	std::stack<obj_rec> obj_stack;
	102	std::deque<obj_rec> cleanup_queue;
	103
	104	typedef std::vector<obj_rec> obj_vec;
	105	typedef std::tuple<obj_rec, obj_vec> dirs1_rec;
	106	typedef std::vector<dirs1_rec> dirs1_vec;
	107
	108	dirs1_vec dirs_vec;
	109
	110	struct obj_rec_st
	111	{
	112	const obj_rec& obj;
	113	const struct stat& st;
	114
	115	obj_rec_st(const obj_rec& _obj, const struct stat& _st)
	116	: obj(_obj), st(_st) {}
	117	};
	118
11fdf7f2	119	/* Unused
3efd9988 FG	120	ostream& operator<<(ostream& os, const obj_rec_st& rec)
	121	{
	122	RGWFileHandle* rgw_fh = rec.obj.rgw_fh;
	123	if (rgw_fh) {
	124	const char* type = rgw_fh->is_dir() ? "DIR " : "FILE ";
	125	os << rgw_fh->full_object_name()
	126	<< " (" << rgw_fh->object_name() << "): "
	127	<< type;
	128	const struct stat& st = rec.st;
	129	switch(uint8_t(rgw_fh->is_dir())) {
	130	case 1:
	131	os << " mode: " << st.st_mode;
	132	os << " nlinks: " << st.st_nlink;
	133	break;
	134	case 0:
	135	default:
	136	os << " mode: " << st.st_mode;
	137	os << " size: " << st.st_size;
	138	// xxx
	139	break;
	140	}
	141	}
	142	return os;
	143	}
11fdf7f2	144	*/
3efd9988 FG	145
	146	bool do_marker1 = false;
	147	bool do_marker2 = true;
	148	bool do_create = false;
	149	bool do_delete = false;
	150	bool verbose = false;
	151
	152	string marker_dir("nfs_marker");
	153	struct rgw_file_handle *bucket_fh = nullptr;
	154	struct rgw_file_handle *marker_fh;
f67539c2	155	uint32_t marker_nobjs = 2*1024;
3efd9988 FG	156	std::deque<obj_rec> marker_objs;
	157
	158	using dirent_t = std::tuple<std::string, uint64_t>;
	159	struct dirent_vec
	160	{
	161	std::vector<dirent_t> obj_names;
	162	uint32_t count;
	163	dirent_vec() : count(0) {}
	164	};
	165
	166	struct {
	167	int argc;
	168	char **argv;
	169	} saved_args;
	170	}
	171
	172	TEST(LibRGW, TVAR) {
	173	typedef boost::variant<uint64_t, const char> readdir_offset;
	174
	175	uint64_t i1{64001};
	176	std::string s1{"blunderbuss"};
	177
	178	readdir_offset v1{&i1};
	179	readdir_offset v2{s1.c_str()};
	180	readdir_offset v3{static_cast<const char*>(nullptr)};
	181
	182	uint64_t* pi1 = get<uint64_t*>(v1);
	183	ASSERT_NE(pi1, nullptr);
	184	std::cout << "read i1: " << *pi1 << std::endl;
	185
	186	const char* ps1 = get<const char*>(v2);
	187	ASSERT_NE(ps1, nullptr);
	188	std::cout << "read s1: " << ps1 << std::endl;
	189
	190	const char* ps3 = get<const char*>(v3);
	191	ASSERT_EQ(ps3, nullptr);
	192	std::cout << "read s3: " << ps3 << std::endl;
	193	}
	194
	195	TEST(LibRGW, INIT) {
	196	int ret = librgw_create(&rgw_h, saved_args.argc, saved_args.argv);
	197	ASSERT_EQ(ret, 0);
	198	ASSERT_NE(rgw_h, nullptr);
	199	}
	200
	201	TEST(LibRGW, MOUNT) {
	202	int ret = rgw_mount2(rgw_h, userid.c_str(), access_key.c_str(),
	203	secret_key.c_str(), "/", &fs, RGW_MOUNT_FLAG_NONE);
	204	ASSERT_EQ(ret, 0);
	205	ASSERT_NE(fs, nullptr);
	206
	207	cct = static_cast<RGWLibFS*>(fs->fs_private)->get_context();
	208	}
	209
	210	TEST(LibRGW, MARKER1_SETUP_BUCKET) {
	211	/* "large" directory enumeration test. this one deals only with
	212	* file objects */
	213	struct stat st;
	214	int ret;
	215
	216	st.st_uid = owner_uid;
	217	st.st_gid = owner_gid;
	218	st.st_mode = 755;
	219
220	(void) rgw_lookup(fs, fs->root_fh, bucket_name.c_str(), &bucket_fh,
eafe8130	221	nullptr, 0, RGW_LOOKUP_FLAG_NONE);
3efd9988 FG	222	if (! bucket_fh) {
	223	if (do_create) {
	224	struct stat st;
	225
	226	st.st_uid = owner_uid;
	227	st.st_gid = owner_gid;
	228	st.st_mode = 755;
	229
	230	ret = rgw_mkdir(fs, fs->root_fh, bucket_name.c_str(), &st, create_mask,
	231	&bucket_fh, RGW_MKDIR_FLAG_NONE);
	232	ASSERT_EQ(ret, 0);
	233	}
	234	}
	235
	236	ASSERT_NE(bucket_fh, nullptr);
	237
	238	(void) rgw_lookup(fs, bucket_fh, marker_dir.c_str(), &marker_fh,
eafe8130	239	nullptr, 0, RGW_LOOKUP_FLAG_NONE);
3efd9988 FG	240	if (! marker_fh) {
	241	if (do_create) {
	242	ret = rgw_mkdir(fs, bucket_fh, marker_dir.c_str(), &st, create_mask,
	243	&marker_fh, RGW_MKDIR_FLAG_NONE);
	244	ASSERT_EQ(ret, 0);
	245	}
	246	}
	247
	248	ASSERT_NE(marker_fh, nullptr);
	249	}
	250
	251	TEST(LibRGW, MARKER1_SETUP_OBJECTS)
	252	{
	253	/* "large" directory enumeration test. this one deals only with
	254	* file objects */
f67539c2	255
3efd9988 FG	256	if (do_create) {
	257	int ret;
	258
f67539c2	259	for (uint32_t ix = 0; ix < marker_nobjs; ++ix) {
3efd9988 FG	260	std::string object_name("f_");
	261	object_name += to_string(ix);
	262	obj_rec obj{object_name, nullptr, marker_fh, nullptr};
	263	// lookup object--all operations are by handle
	264	ret = rgw_lookup(fs, marker_fh, obj.name.c_str(), &obj.fh,
eafe8130	265	nullptr, 0, RGW_LOOKUP_FLAG_CREATE);
3efd9988 FG	266	ASSERT_EQ(ret, 0);
	267	obj.rgw_fh = get_rgwfh(obj.fh);
	268	// open object--open transaction
	269	ret = rgw_open(fs, obj.fh, 0 /* posix flags */, RGW_OPEN_FLAG_NONE);
	270	ASSERT_EQ(ret, 0);
	271	ASSERT_TRUE(obj.rgw_fh->is_open());
	272	// unstable write data
	273	size_t nbytes;
	274	string data("data for ");
	275	data += object_name;
	276	int ret = rgw_write(fs, obj.fh, 0, data.length(), &nbytes,
	277	(void*) data.c_str(), RGW_WRITE_FLAG_NONE);
	278	ASSERT_EQ(ret, 0);
	279	ASSERT_EQ(nbytes, data.length());
	280	// commit transaction (write on close)
	281	ret = rgw_close(fs, obj.fh, 0 /* flags */);
	282	ASSERT_EQ(ret, 0);
f67539c2 TL	283	if (verbose) {
	284	/* XXX std:cout fragged...did it get /0 in the stream
	285	* somewhere? */
	286	printf("created: %s:%s\n", bucket_name.c_str(), obj.name.c_str());
	287	}
3efd9988 FG	288	// save for cleanup
	289	marker_objs.push_back(obj);
	290	}
	291	}
	292	}
	293
	294	extern "C" {
	295	static bool r2_cb(const char* name, void *arg, uint64_t offset,
eafe8130	296	struct stat* st, uint32_t st_mask,
3efd9988 FG	297	uint32_t flags) {
	298	dirent_vec& dvec =
	299	(static_cast<dirent_vec>(arg));
f67539c2 TL	300
	301	printf("%s bucket=%s dir=%s iv count=%d called back name=%s flags=%d\n",
	302	__func__,
	303	bucket_name.c_str(),
	304	marker_dir.c_str(),
	305	dvec.count,
	306	name,
	307	flags);
3efd9988 FG	308
	309	string name_str{name};
	310	if (! ((name_str == ".") \|\|
	311	(name_str == ".."))) {
	312	dvec.obj_names.push_back(dirent_t{std::move(name_str), offset});
	313	}
	314	return true; /* XXX */
	315	}
	316	}
	317
	318	TEST(LibRGW, MARKER1_READDIR)
	319	{
	320	if (do_marker1) {
	321	using std::get;
	322
	323	dirent_vec dvec;
	324	uint64_t offset = 0;
	325	bool eof = false;
	326
	327	/* because RGWReaddirRequest::default_max is 1000 (XXX make
	328	* configurable?) and marker_nobjs is 5*1024, the number
	329	* of required rgw_readdir operations N should be
	330	* marker_nobjs/1000 < N < marker_nobjs/1000+1, i.e., 6 when
	331	* marker_nobjs==51024 /
	332	uint32_t max_iterations = marker_nobjs/1000+1;
	333
	334	do {
	335	ASSERT_TRUE(dvec.count <= max_iterations);
	336	int ret = rgw_readdir(fs, marker_fh, &offset, r2_cb, &dvec, &eof,
	337	RGW_READDIR_FLAG_DOTDOT);
	338	ASSERT_EQ(ret, 0);
f67539c2	339	ASSERT_GE(dvec.obj_names.size(), 0);
3efd9988 FG	340	ASSERT_EQ(offset, get<1>(dvec.obj_names.back())); // cookie check
	341	++dvec.count;
	342	} while(!eof);
	343	std::cout << "Read " << dvec.obj_names.size() << " objects in "
	344	<< marker_dir.c_str() << std::endl;
	345	}
	346	}
	347
	348	TEST(LibRGW, MARKER2_READDIR)
	349	{
	350	if (do_marker2) {
	351	using std::get;
	352
	353	dirent_vec dvec;
	354	std::string marker{""};
	355	bool eof = false;
	356
	357	/* because RGWReaddirRequest::default_max is 1000 (XXX make
	358	* configurable?) and marker_nobjs is 5*1024, the number
	359	* of required rgw_readdir operations N should be
	360	* marker_nobjs/1000 < N < marker_nobjs/1000+1, i.e., 6 when
	361	* marker_nobjs==51024 /
	362	uint32_t max_iterations = marker_nobjs/1000+1;
	363
	364	do {
	365	ASSERT_TRUE(dvec.count <= max_iterations);
	366	int ret = rgw_readdir2(fs, marker_fh,
	367	(marker.length() > 0) ? marker.c_str() : nullptr,
	368	r2_cb, &dvec, &eof,
f67539c2	369	RGW_READDIR_FLAG_NONE);
3efd9988	370	ASSERT_EQ(ret, 0);
f67539c2	371	ASSERT_GE(dvec.obj_names.size(), 0);
3efd9988 FG	372	marker = get<0>(dvec.obj_names.back());
	373	++dvec.count;
	374	} while((!eof) && dvec.count < 4);
	375	std::cout << "Read " << dvec.obj_names.size() << " objects in "
	376	<< marker_dir.c_str() << std::endl;
	377	}
	378	}
	379
	380	TEST(LibRGW, MARKER1_OBJ_CLEANUP)
	381	{
	382	int rc;
	383	for (auto& obj : marker_objs) {
	384	if (obj.fh) {
	385	if (do_delete) {
	386	if (verbose) {
	387	std::cout << "unlinking: " << bucket_name << ":" << obj.name
	388	<< std::endl;
	389	}
	390	rc = rgw_unlink(fs, marker_fh, obj.name.c_str(), RGW_UNLINK_FLAG_NONE);
	391	}
	392	rc = rgw_fh_rele(fs, obj.fh, 0 /* flags */);
	393	ASSERT_EQ(rc, 0);
	394	}
	395	}
	396	marker_objs.clear();
	397	}
	398
	399	TEST(LibRGW, CLEANUP) {
	400	int rc;
	401
	402	if (do_marker1) {
	403	cleanup_queue.push_back(
	404	obj_rec{bucket_name, bucket_fh, fs->root_fh, get_rgwfh(fs->root_fh)});
	405	}
	406
	407	for (auto& elt : cleanup_queue) {
	408	if (elt.fh) {
	409	rc = rgw_fh_rele(fs, elt.fh, 0 /* flags */);
	410	ASSERT_EQ(rc, 0);
	411	}
	412	}
	413	cleanup_queue.clear();
	414	}
	415
	416	TEST(LibRGW, UMOUNT) {
	417	if (! fs)
	418	return;
	419
	420	int ret = rgw_umount(fs, RGW_UMOUNT_FLAG_NONE);
	421	ASSERT_EQ(ret, 0);
	422	}
	423
	424	TEST(LibRGW, SHUTDOWN) {
	425	librgw_shutdown(rgw_h);
	426	}
	427
	428	int main(int argc, char *argv[])
	429	{
20effc67	430	auto args = argv_to_vec(argc, argv);
3efd9988 FG	431	env_to_vec(args);
3efd9988 FG	432
20effc67	433	char* v = getenv("AWS_ACCESS_KEY_ID");
3efd9988 FG	434	if (v) {
	435	access_key = v;
	436	}
	437
	438	v = getenv("AWS_SECRET_ACCESS_KEY");
	439	if (v) {
	440	secret_key = v;
	441	}
	442
20effc67	443	string val;
3efd9988 FG	444	for (auto arg_iter = args.begin(); arg_iter != args.end();) {
	445	if (ceph_argparse_witharg(args, arg_iter, &val, "--access",
	446	(char*) nullptr)) {
	447	access_key = val;
	448	} else if (ceph_argparse_witharg(args, arg_iter, &val, "--secret",
	449	(char*) nullptr)) {
	450	secret_key = val;
	451	} else if (ceph_argparse_witharg(args, arg_iter, &val, "--userid",
	452	(char*) nullptr)) {
	453	userid = val;
	454	} else if (ceph_argparse_witharg(args, arg_iter, &val, "--bn",
	455	(char*) nullptr)) {
	456	bucket_name = val;
	457	} else if (ceph_argparse_witharg(args, arg_iter, &val, "--uid",
	458	(char*) nullptr)) {
	459	owner_uid = std::stoi(val);
	460	} else if (ceph_argparse_witharg(args, arg_iter, &val, "--gid",
	461	(char*) nullptr)) {
	462	owner_gid = std::stoi(val);
f67539c2 TL	463	} else if (ceph_argparse_witharg(args, arg_iter, &val, "--nobjs",
	464	(char*) nullptr)) {
	465	marker_nobjs = std::stoi(val);
3efd9988 FG	466	} else if (ceph_argparse_flag(args, arg_iter, "--marker1",
	467	(char*) nullptr)) {
	468	do_marker1 = true;
	469	} else if (ceph_argparse_flag(args, arg_iter, "--create",
	470	(char*) nullptr)) {
	471	do_create = true;
	472	} else if (ceph_argparse_flag(args, arg_iter, "--delete",
	473	(char*) nullptr)) {
	474	do_delete = true;
	475	} else if (ceph_argparse_flag(args, arg_iter, "--verbose",
	476	(char*) nullptr)) {
	477	verbose = true;
	478	} else {
	479	++arg_iter;
	480	}
	481	}
	482
11fdf7f2	483	/* don't accidentally run as anonymous */
3efd9988 FG	484	if ((access_key == "") \|\|
	485	(secret_key == "")) {
	486	std::cout << argv[0] << " no AWS credentials, exiting" << std::endl;
	487	return EPERM;
	488	}
	489
	490	saved_args.argc = argc;
	491	saved_args.argv = argv;
	492
	493	::testing::InitGoogleTest(&argc, argv);
	494	return RUN_ALL_TESTS();
	495	}