ceph/src/test/librgw_file_aw.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2015 Red Hat, Inc.
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14
  15 #include <stdint.h>
  16 #include <tuple>
  17 #include <iostream>
  18 #include <vector>
  19 #include <map>
  20 #include <random>
  21 #include "xxhash.h"
  22
  23 #include "include/rados/librgw.h"
  24 #include "include/rados/rgw_file.h"
  25
  26 #include "gtest/gtest.h"
  27 #include "common/ceph_argparse.h"
  28 #include "common/debug.h"
  29 #include "global/global_init.h"
  30
  31 #define dout_context g_ceph_context
  32 #define dout_subsys ceph_subsys_rgw
  33
  34 namespace {
  35   librgw_t rgw = nullptr;
  36   string userid("testuser");
  37   string access_key("");
  38   string secret_key("");
  39   struct rgw_fs *fs = nullptr;
  40
  41   uint32_t owner_uid = 867;
  42   uint32_t owner_gid = 5309;
  43   uint32_t create_mask = RGW_SETATTR_UID | RGW_SETATTR_GID | RGW_SETATTR_MODE;
  44
  45   bool do_create = false;
  46   bool do_delete = false;
  47   bool do_verify = false;
  48   bool do_hexdump = false;
  49
  50   string bucket_name = "sorry_dave";
  51   string object_name = "jocaml";
  52
  53   struct rgw_file_handle *bucket_fh = nullptr;
  54   struct rgw_file_handle *object_fh = nullptr;
  55
  56   typedef std::tuple<string,uint64_t, struct rgw_file_handle*> fid_type;
  57   std::vector<fid_type> fids;
  58
  59   std::uniform_int_distribution<uint8_t> uint_dist;
  60   std::mt19937 rng;
  61
  62   constexpr int iovcnt = 16;
  63   constexpr int page_size = 65536;
  64   constexpr int seed = 8675309;
  65
  66   struct ZPage
  67   {
  68     char data[page_size];
  69     uint64_t cksum;
  70   }; /* ZPage */
  71
  72   struct ZPageSet
  73   {
  74     std::vector<ZPage*> pages;
  75     struct iovec* iovs;
  76
  77     ZPageSet(int n) {
  78       pages.reserve(n);
  79       iovs = (struct iovec*) calloc(n, sizeof(struct iovec));
  80       for (int page_ix = 0; page_ix < n; ++page_ix) {
  81         ZPage* p = new ZPage();
  82         for (int data_ix = 0; data_ix < page_size; ++data_ix) {
  83           p->data[data_ix] = uint_dist(rng);
  84         } // data_ix
  85         p->cksum = XXH64(p->data, page_size, seed);
  86         pages.emplace_back(p);
  87         // and iovs
  88         struct iovec* iov = &iovs[page_ix];
  89         iov->iov_base = p->data;
  90         iov->iov_len = page_size;
  91       } // page_ix
  92     }
  93
  94     int size() { return pages.size(); }
  95
  96     struct iovec* get_iovs() { return iovs; }
  97
  98     bool operator==(const ZPageSet& rhs) {
  99       int n = size();
 100       for (int page_ix = 0; page_ix < n; ++page_ix) {
 101         ZPage* p1 = pages[page_ix];
 102         ZPage* p2 = rhs.pages[page_ix];
 103         if (p1->cksum != p2->cksum)
 104           return false;
 105       }
 106       return true;
 107     }
 108
 109     bool operator==(const rgw_uio* uio) {
 110       uint64_t cksum;
 111       int vix = 0, off = 0;
 112       rgw_vio* vio = &uio->uio_vio[vix];
 113       int vio_len = vio->vio_len;
 114       char *data;
 115
 116       for (int ix = 0; ix < iovcnt; ++ix) {
 117         ZPage* p1 = pages[ix];
 118         data = static_cast<char*>(vio->vio_base) + off;
 119         cksum = XXH64(data, page_size, seed);
 120
 121         if (p1->cksum != cksum) {
 122           int r = memcmp(data, p1->data, page_size);
 123           std::cout << "problem at ix " << ix << " r " << r<< std::endl;
 124           return false;
 125         }
 126
 127         off += page_size;
 128         if (off >= vio_len) {
 129           vio = &uio->uio_vio[++vix];
 130           vio_len = vio->vio_len;
 131           off = 0;
 132         }
 133       }
 134       return true;
 135     }
 136
 137     void cksum() {
 138       int n = size();
 139       for (int page_ix = 0; page_ix < n; ++page_ix) {
 140         ZPage* p = pages[page_ix];
 141         p->cksum = XXH64(p->data, page_size, seed);
 142       }
 143     }
 144
 145     void reset_iovs() { // VOP_READ and VOP_WRITE update
 146       int n = size();
 147       for (int page_ix = 0; page_ix < n; ++page_ix) {
 148         ZPage* p = pages[page_ix];
 149         struct iovec* iov = &iovs[page_ix];
 150         iov->iov_base = p->data;
 151         iov->iov_len = page_size;
 152       }
 153     }
 154
 155     ~ZPageSet() {
 156       for (unsigned int ix = 0; ix < pages.size(); ++ix)
 157         delete pages[ix];
 158       free(iovs);
 159     }
 160   }; /* ZPageSet */
 161
 162   ZPageSet zp_set1{iovcnt}; // 1M random data in 16 64K pages
 163
 164   struct {
 165     int argc;
 166     char **argv;
 167   } saved_args;
 168 }
 169
 170 TEST(LibRGW, INIT) {
 171   int ret = librgw_create(&rgw, saved_args.argc, saved_args.argv);
 172   ASSERT_EQ(ret, 0);
 173   ASSERT_NE(rgw, nullptr);
 174 }
 175
 176 TEST(LibRGW, MOUNT) {
 177   int ret = rgw_mount(rgw, userid.c_str(), access_key.c_str(),
 178                       secret_key.c_str(), &fs, RGW_MOUNT_FLAG_NONE);
 179   ASSERT_EQ(ret, 0);
 180   ASSERT_NE(fs, nullptr);
 181 }
 182
 183 TEST(LibRGW, CREATE_BUCKET) {
 184   if (do_create) {
 185     struct stat st;
 186     struct rgw_file_handle *fh;
 187
 188     st.st_uid = owner_uid;
 189     st.st_gid = owner_gid;
 190     st.st_mode = 755;
 191
 192     int ret = rgw_mkdir(fs, fs->root_fh, bucket_name.c_str(), &st, create_mask,
 193                         &fh, RGW_MKDIR_FLAG_NONE);
 194     ASSERT_EQ(ret, 0);
 195   }
 196 }
 197
 198 TEST(LibRGW, LOOKUP_BUCKET) {
 199   int ret = rgw_lookup(fs, fs->root_fh, bucket_name.c_str(), &bucket_fh,
 200                       RGW_LOOKUP_FLAG_NONE);
 201   ASSERT_EQ(ret, 0);
 202 }
 203
 204 TEST(LibRGW, LOOKUP_OBJECT) {
 205   int ret = rgw_lookup(fs, bucket_fh, object_name.c_str(), &object_fh,
 206                        RGW_LOOKUP_FLAG_CREATE);
 207   ASSERT_EQ(ret, 0);
 208 }
 209
 210 TEST(LibRGW, OPEN1) {
 211   int ret = rgw_open(fs, object_fh, 0 /* posix flags */, RGW_OPEN_FLAG_NONE);
 212   ASSERT_EQ(ret, 0);
 213 }
 214
 215 TEST(LibRGW, PUT_OBJECT) {
 216   size_t nbytes;
 217   struct iovec *iovs = zp_set1.get_iovs();
 218   off_t offset = 0;
 219   for (int ix : {0, 1}) {
 220     struct iovec *iov = &iovs[ix];
 221     // quick check
 222     sprintf(static_cast<char*>(iov->iov_base), "::hi mom (%d)", ix);
 223     iov->iov_len = 14;
 224     int ret = rgw_write(fs, object_fh, offset, iov->iov_len, &nbytes,
 225                         iov->iov_base, RGW_WRITE_FLAG_NONE);
 226     offset += iov->iov_len;
 227     ASSERT_EQ(ret, 0);
 228     ASSERT_EQ(nbytes, iov->iov_len);
 229   }
 230 }
 231
 232 TEST(LibRGW, CLOSE1) {
 233   int ret = rgw_close(fs, object_fh, RGW_CLOSE_FLAG_NONE);
 234   ASSERT_EQ(ret, 0);
 235 }
 236
 237 TEST(LibRGW, OPEN2) {
 238   int ret = rgw_open(fs, object_fh, 0 /* posix flags */, RGW_OPEN_FLAG_NONE);
 239   ASSERT_EQ(ret, 0);
 240 }
 241
 242 TEST(LibRGW, GET_OBJECT) {
 243   size_t nread;
 244   off_t offset = 0;
 245   struct iovec *iovs = zp_set1.get_iovs();
 246   for (int ix : {2 , 3}) {
 247     struct iovec *iov = &iovs[ix];
 248     int ret = rgw_read(fs, object_fh, offset, iovs[ix-2].iov_len, &nread,
 249                        iov->iov_base, RGW_READ_FLAG_NONE);
 250     iov->iov_len = nread;
 251     offset += iov->iov_len;
 252     ASSERT_EQ(ret, 0);
 253     ASSERT_EQ(nread, iovs[ix-2].iov_len);
 254     std::cout << "read: " << static_cast<char*>(iov->iov_base) << std::endl;
 255   }
 256 }
 257
 258 TEST(LibRGW, CLOSE2) {
 259   int ret = rgw_close(fs, object_fh, RGW_CLOSE_FLAG_NONE);
 260   ASSERT_EQ(ret, 0);
 261 }
 262
 263 TEST(LibRGW, STAT_OBJECT) {
 264   struct stat st;
 265   int ret = rgw_getattr(fs, object_fh, &st, RGW_GETATTR_FLAG_NONE);
 266   ASSERT_EQ(ret, 0);
 267   dout(15) << "rgw_getattr on " << object_name << " size = "
 268            << st.st_size << dendl;
 269 }
 270
 271 TEST(LibRGW, DELETE_OBJECT) {
 272   if (do_delete) {
 273     int ret = rgw_unlink(fs, bucket_fh, object_name.c_str(),
 274                          RGW_UNLINK_FLAG_NONE);
 275     ASSERT_EQ(ret, 0);
 276   }
 277 }
 278
 279 TEST(LibRGW, CLEANUP) {
 280   int ret;
 281   if (object_fh) {
 282     ret = rgw_fh_rele(fs, object_fh, RGW_FH_RELE_FLAG_NONE);
 283     ASSERT_EQ(ret, 0);
 284   }
 285   ret = rgw_fh_rele(fs, bucket_fh, 0 /* flags */);
 286   ASSERT_EQ(ret, 0);
 287 }
 288
 289 TEST(LibRGW, UMOUNT) {
 290   if (! fs)
 291     return;
 292
 293   int ret = rgw_umount(fs, RGW_UMOUNT_FLAG_NONE);
 294   ASSERT_EQ(ret, 0);
 295 }
 296
 297 TEST(LibRGW, SHUTDOWN) {
 298   librgw_shutdown(rgw);
 299 }
 300
 301 int main(int argc, char *argv[])
 302 {
 303   char *v{nullptr};
 304   string val;
 305   vector<const char*> args;
 306
 307   argv_to_vec(argc, const_cast<const char**>(argv), args);
 308   env_to_vec(args);
 309
 310   v = getenv("AWS_ACCESS_KEY_ID");
 311   if (v) {
 312     access_key = v;
 313   }
 314
 315   v = getenv("AWS_SECRET_ACCESS_KEY");
 316   if (v) {
 317     secret_key = v;
 318   }
 319
 320   for (auto arg_iter = args.begin(); arg_iter != args.end();) {
 321     if (ceph_argparse_witharg(args, arg_iter, &val, "--access",
 322                               (char*) nullptr)) {
 323       access_key = val;
 324     } else if (ceph_argparse_witharg(args, arg_iter, &val, "--secret",
 325                                      (char*) nullptr)) {
 326       secret_key = val;
 327     } else if (ceph_argparse_witharg(args, arg_iter, &val, "--userid",
 328                                      (char*) nullptr)) {
 329       userid = val;
 330     } else if (ceph_argparse_witharg(args, arg_iter, &val, "--bn",
 331                                      (char*) nullptr)) {
 332       bucket_name = val;
 333     } else if (ceph_argparse_witharg(args, arg_iter, &val, "--uid",
 334                                      (char*) nullptr)) {
 335       owner_uid = std::stoi(val);
 336     } else if (ceph_argparse_witharg(args, arg_iter, &val, "--gid",
 337                                      (char*) nullptr)) {
 338       owner_gid = std::stoi(val);
 339     } else if (ceph_argparse_flag(args, arg_iter, "--verify",
 340                                             (char*) nullptr)) {
 341       do_verify = true;
 342     } else if (ceph_argparse_flag(args, arg_iter, "--create",
 343                                             (char*) nullptr)) {
 344       do_create = true;
 345     } else if (ceph_argparse_flag(args, arg_iter, "--delete",
 346                                             (char*) nullptr)) {
 347       do_delete = true;
 348     } else if (ceph_argparse_flag(args, arg_iter, "--hexdump",
 349                                             (char*) nullptr)) {
 350       do_hexdump = true;
 351     } else {
 352       ++arg_iter;
 353     }
 354   }
 355
 356   /* dont accidentally run as anonymous */
 357   if ((access_key == "") ||
 358       (secret_key == "")) {
 359     std::cout << argv[0] << " no AWS credentials, exiting" << std::endl;
 360     return EPERM;
 361   }
 362
 363   saved_args.argc = argc;
 364   saved_args.argv = argv;
 365
 366   ::testing::InitGoogleTest(&argc, argv);
 367   return RUN_ALL_TESTS();
 368 }