]>
git.proxmox.com Git - ceph.git/blob - ceph/src/test/librgw_file_marker.cc
1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2015 Red Hat, Inc.
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
21 #include "include/rados/librgw.h"
22 #include "include/rados/rgw_file.h"
23 #include "rgw/rgw_file.h"
24 #include "rgw/rgw_lib_frontend.h" // direct requests
26 #include "gtest/gtest.h"
27 #include "common/backport14.h"
28 #include "common/ceph_argparse.h"
29 #include "common/debug.h"
30 #include "global/global_init.h"
31 #include "include/assert.h"
33 #define dout_subsys ceph_subsys_rgw
41 librgw_t rgw_h
= nullptr;
42 string
userid("testuser");
43 string
access_key("");
44 string
secret_key("");
45 struct rgw_fs
*fs
= nullptr;
46 CephContext
* cct
= nullptr;
48 uint32_t owner_uid
= 867;
49 uint32_t owner_gid
= 5309;
51 uint32_t create_mask
= RGW_SETATTR_UID
| RGW_SETATTR_GID
| RGW_SETATTR_MODE
;
53 string
bucket_name("nfsroot");
59 struct rgw_file_handle
* fh
;
60 struct rgw_file_handle
* parent_fh
;
61 RGWFileHandle
* rgw_fh
; // alias into fh
65 state() : readdir(false) {}
68 obj_rec(string _name
, struct rgw_file_handle
* _fh
,
69 struct rgw_file_handle
* _parent_fh
, RGWFileHandle
* _rgw_fh
)
70 : name(std::move(_name
)), fh(_fh
), parent_fh(_parent_fh
),
80 rgw_fh
= get_rgwfh(fh
);
83 friend ostream
& operator<<(ostream
& os
, const obj_rec
& rec
);
86 ostream
& operator<<(ostream
& os
, const obj_rec
& rec
)
88 RGWFileHandle
* rgw_fh
= rec
.rgw_fh
;
90 const char* type
= rgw_fh
->is_dir() ? "DIR " : "FILE ";
91 os
<< rec
.rgw_fh
->full_object_name()
92 << " (" << rec
.rgw_fh
->object_name() << "): "
98 std::stack
<obj_rec
> obj_stack
;
99 std::deque
<obj_rec
> cleanup_queue
;
101 typedef std::vector
<obj_rec
> obj_vec
;
102 typedef std::tuple
<obj_rec
, obj_vec
> dirs1_rec
;
103 typedef std::vector
<dirs1_rec
> dirs1_vec
;
110 const struct stat
& st
;
112 obj_rec_st(const obj_rec
& _obj
, const struct stat
& _st
)
113 : obj(_obj
), st(_st
) {}
116 ostream
& operator<<(ostream
& os
, const obj_rec_st
& rec
)
118 RGWFileHandle
* rgw_fh
= rec
.obj
.rgw_fh
;
120 const char* type
= rgw_fh
->is_dir() ? "DIR " : "FILE ";
121 os
<< rgw_fh
->full_object_name()
122 << " (" << rgw_fh
->object_name() << "): "
124 const struct stat
& st
= rec
.st
;
125 switch(uint8_t(rgw_fh
->is_dir())) {
127 os
<< " mode: " << st
.st_mode
;
128 os
<< " nlinks: " << st
.st_nlink
;
132 os
<< " mode: " << st
.st_mode
;
133 os
<< " size: " << st
.st_size
;
141 bool do_marker1
= false;
142 bool do_marker2
= true;
143 bool do_create
= false;
144 bool do_delete
= false;
145 bool verbose
= false;
147 string
marker_dir("nfs_marker");
148 struct rgw_file_handle
*bucket_fh
= nullptr;
149 struct rgw_file_handle
*marker_fh
;
150 static constexpr int marker_nobjs
= 2*1024;
151 std::deque
<obj_rec
> marker_objs
;
153 using dirent_t
= std::tuple
<std::string
, uint64_t>;
156 std::vector
<dirent_t
> obj_names
;
158 dirent_vec() : count(0) {}
168 typedef boost::variant
<uint64_t*, const char*> readdir_offset
;
171 std::string s1
{"blunderbuss"};
173 readdir_offset v1
{&i1
};
174 readdir_offset v2
{s1
.c_str()};
175 readdir_offset v3
{static_cast<const char*>(nullptr)};
177 uint64_t* pi1
= get
<uint64_t*>(v1
);
178 ASSERT_NE(pi1
, nullptr);
179 std::cout
<< "read i1: " << *pi1
<< std::endl
;
181 const char* ps1
= get
<const char*>(v2
);
182 ASSERT_NE(ps1
, nullptr);
183 std::cout
<< "read s1: " << ps1
<< std::endl
;
185 const char* ps3
= get
<const char*>(v3
);
186 ASSERT_EQ(ps3
, nullptr);
187 std::cout
<< "read s3: " << ps3
<< std::endl
;
191 int ret
= librgw_create(&rgw_h
, saved_args
.argc
, saved_args
.argv
);
193 ASSERT_NE(rgw_h
, nullptr);
196 TEST(LibRGW
, MOUNT
) {
197 int ret
= rgw_mount2(rgw_h
, userid
.c_str(), access_key
.c_str(),
198 secret_key
.c_str(), "/", &fs
, RGW_MOUNT_FLAG_NONE
);
200 ASSERT_NE(fs
, nullptr);
202 cct
= static_cast<RGWLibFS
*>(fs
->fs_private
)->get_context();
205 TEST(LibRGW
, MARKER1_SETUP_BUCKET
) {
206 /* "large" directory enumeration test. this one deals only with
211 st
.st_uid
= owner_uid
;
212 st
.st_gid
= owner_gid
;
215 (void) rgw_lookup(fs
, fs
->root_fh
, bucket_name
.c_str(), &bucket_fh
,
216 RGW_LOOKUP_FLAG_NONE
);
221 st
.st_uid
= owner_uid
;
222 st
.st_gid
= owner_gid
;
225 ret
= rgw_mkdir(fs
, fs
->root_fh
, bucket_name
.c_str(), &st
, create_mask
,
226 &bucket_fh
, RGW_MKDIR_FLAG_NONE
);
231 ASSERT_NE(bucket_fh
, nullptr);
233 (void) rgw_lookup(fs
, bucket_fh
, marker_dir
.c_str(), &marker_fh
,
234 RGW_LOOKUP_FLAG_NONE
);
237 ret
= rgw_mkdir(fs
, bucket_fh
, marker_dir
.c_str(), &st
, create_mask
,
238 &marker_fh
, RGW_MKDIR_FLAG_NONE
);
243 ASSERT_NE(marker_fh
, nullptr);
246 TEST(LibRGW
, MARKER1_SETUP_OBJECTS
)
248 /* "large" directory enumeration test. this one deals only with
253 for (int ix
= 0; ix
< marker_nobjs
; ++ix
) {
254 std::string
object_name("f_");
255 object_name
+= to_string(ix
);
256 obj_rec obj
{object_name
, nullptr, marker_fh
, nullptr};
257 // lookup object--all operations are by handle
258 ret
= rgw_lookup(fs
, marker_fh
, obj
.name
.c_str(), &obj
.fh
,
259 RGW_LOOKUP_FLAG_CREATE
);
261 obj
.rgw_fh
= get_rgwfh(obj
.fh
);
262 // open object--open transaction
263 ret
= rgw_open(fs
, obj
.fh
, 0 /* posix flags */, RGW_OPEN_FLAG_NONE
);
265 ASSERT_TRUE(obj
.rgw_fh
->is_open());
266 // unstable write data
268 string
data("data for ");
270 int ret
= rgw_write(fs
, obj
.fh
, 0, data
.length(), &nbytes
,
271 (void*) data
.c_str(), RGW_WRITE_FLAG_NONE
);
273 ASSERT_EQ(nbytes
, data
.length());
274 // commit transaction (write on close)
275 ret
= rgw_close(fs
, obj
.fh
, 0 /* flags */);
278 marker_objs
.push_back(obj
);
284 static bool r2_cb(const char* name
, void *arg
, uint64_t offset
,
287 *(static_cast<dirent_vec
*>(arg
));
288 lsubdout(cct
, rgw
, 10) << __func__
289 << " bucket=" << bucket_name
290 << " dir=" << marker_dir
291 << " iv count=" << dvec
.count
292 << " called back name=" << name
293 << " flags=" << flags
296 std::cout
<< __func__
297 << " bucket=" << bucket_name
298 << " dir=" << marker_dir
299 << " iv count=" << dvec
.count
300 << " called back name=" << name
301 << " flags=" << flags
304 string name_str
{name
};
305 if (! ((name_str
== ".") ||
306 (name_str
== ".."))) {
307 dvec
.obj_names
.push_back(dirent_t
{std::move(name_str
), offset
});
309 return true; /* XXX */
313 TEST(LibRGW
, MARKER1_READDIR
)
322 /* because RGWReaddirRequest::default_max is 1000 (XXX make
323 * configurable?) and marker_nobjs is 5*1024, the number
324 * of required rgw_readdir operations N should be
325 * marker_nobjs/1000 < N < marker_nobjs/1000+1, i.e., 6 when
326 * marker_nobjs==5*1024 */
327 uint32_t max_iterations
= marker_nobjs
/1000+1;
330 ASSERT_TRUE(dvec
.count
<= max_iterations
);
331 int ret
= rgw_readdir(fs
, marker_fh
, &offset
, r2_cb
, &dvec
, &eof
,
332 RGW_READDIR_FLAG_DOTDOT
);
334 ASSERT_EQ(offset
, get
<1>(dvec
.obj_names
.back())); // cookie check
337 std::cout
<< "Read " << dvec
.obj_names
.size() << " objects in "
338 << marker_dir
.c_str() << std::endl
;
342 TEST(LibRGW
, MARKER2_READDIR
)
348 std::string marker
{""};
351 /* because RGWReaddirRequest::default_max is 1000 (XXX make
352 * configurable?) and marker_nobjs is 5*1024, the number
353 * of required rgw_readdir operations N should be
354 * marker_nobjs/1000 < N < marker_nobjs/1000+1, i.e., 6 when
355 * marker_nobjs==5*1024 */
356 uint32_t max_iterations
= marker_nobjs
/1000+1;
359 ASSERT_TRUE(dvec
.count
<= max_iterations
);
360 int ret
= rgw_readdir2(fs
, marker_fh
,
361 (marker
.length() > 0) ? marker
.c_str() : nullptr,
363 RGW_READDIR_FLAG_DOTDOT
);
365 marker
= get
<0>(dvec
.obj_names
.back());
367 } while((!eof
) && dvec
.count
< 4);
368 std::cout
<< "Read " << dvec
.obj_names
.size() << " objects in "
369 << marker_dir
.c_str() << std::endl
;
373 TEST(LibRGW
, MARKER1_OBJ_CLEANUP
)
376 for (auto& obj
: marker_objs
) {
380 std::cout
<< "unlinking: " << bucket_name
<< ":" << obj
.name
383 rc
= rgw_unlink(fs
, marker_fh
, obj
.name
.c_str(), RGW_UNLINK_FLAG_NONE
);
385 rc
= rgw_fh_rele(fs
, obj
.fh
, 0 /* flags */);
392 TEST(LibRGW
, CLEANUP
) {
396 cleanup_queue
.push_back(
397 obj_rec
{bucket_name
, bucket_fh
, fs
->root_fh
, get_rgwfh(fs
->root_fh
)});
400 for (auto& elt
: cleanup_queue
) {
402 rc
= rgw_fh_rele(fs
, elt
.fh
, 0 /* flags */);
406 cleanup_queue
.clear();
409 TEST(LibRGW
, UMOUNT
) {
413 int ret
= rgw_umount(fs
, RGW_UMOUNT_FLAG_NONE
);
417 TEST(LibRGW
, SHUTDOWN
) {
418 librgw_shutdown(rgw_h
);
421 int main(int argc
, char *argv
[])
425 vector
<const char*> args
;
427 argv_to_vec(argc
, const_cast<const char**>(argv
), args
);
430 v
= getenv("AWS_ACCESS_KEY_ID");
435 v
= getenv("AWS_SECRET_ACCESS_KEY");
440 for (auto arg_iter
= args
.begin(); arg_iter
!= args
.end();) {
441 if (ceph_argparse_witharg(args
, arg_iter
, &val
, "--access",
444 } else if (ceph_argparse_witharg(args
, arg_iter
, &val
, "--secret",
447 } else if (ceph_argparse_witharg(args
, arg_iter
, &val
, "--userid",
450 } else if (ceph_argparse_witharg(args
, arg_iter
, &val
, "--bn",
453 } else if (ceph_argparse_witharg(args
, arg_iter
, &val
, "--uid",
455 owner_uid
= std::stoi(val
);
456 } else if (ceph_argparse_witharg(args
, arg_iter
, &val
, "--gid",
458 owner_gid
= std::stoi(val
);
459 } else if (ceph_argparse_flag(args
, arg_iter
, "--marker1",
462 } else if (ceph_argparse_flag(args
, arg_iter
, "--create",
465 } else if (ceph_argparse_flag(args
, arg_iter
, "--delete",
468 } else if (ceph_argparse_flag(args
, arg_iter
, "--verbose",
476 /* dont accidentally run as anonymous */
477 if ((access_key
== "") ||
478 (secret_key
== "")) {
479 std::cout
<< argv
[0] << " no AWS credentials, exiting" << std::endl
;
483 saved_args
.argc
= argc
;
484 saved_args
.argv
= argv
;
486 ::testing::InitGoogleTest(&argc
, argv
);
487 return RUN_ALL_TESTS();