ceph/src/test/libcephfs/deleg.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Tests for Ceph delegation handling
   5  *
   6  * (c) 2017, Jeff Layton <jlayton@redhat.com>
   7  */
   8
   9 #include "gtest/gtest.h"
  10 #include "include/cephfs/libcephfs.h"
  11 #include "include/stat.h"
  12 #include <errno.h>
  13 #include <fcntl.h>
  14 #include <unistd.h>
  15 #include <sys/types.h>
  16 #include <sys/stat.h>
  17 #include <dirent.h>
  18 #include <sys/uio.h>
  19
  20 #ifdef __linux__
  21 #include <limits.h>
  22 #include <sys/xattr.h>
  23 #endif
  24
  25 #include <map>
  26 #include <vector>
  27 #include <thread>
  28 #include <atomic>
  29
  30 #include "include/ceph_assert.h"
  31
  32 /* in ms -- 1 minute */
  33 #define MAX_WAIT        (60 * 1000)
  34
  35 static void wait_for_atomic_bool(std::atomic_bool &recalled)
  36 {
  37   int i = 0;
  38
  39   while (!recalled.load()) {
  40     ASSERT_LT(i++, MAX_WAIT);
  41     usleep(1000);
  42   }
  43 }
  44
  45 static int ceph_ll_delegation_wait(struct ceph_mount_info *cmount, Fh *fh,
  46                                    unsigned cmd, ceph_deleg_cb_t cb, void *priv)
  47 {
  48   int ret, retry = 0;
  49
  50   /* Wait 10s at most */
  51   do {
  52     ret = ceph_ll_delegation(cmount, fh, cmd, cb, priv);
  53     usleep(10000);
  54   } while (ret == -EAGAIN && retry++ < 1000);
  55
  56   return ret;
  57 }
  58
  59 static int set_default_deleg_timeout(struct ceph_mount_info *cmount)
  60 {
  61   uint32_t session_timeout = ceph_get_cap_return_timeout(cmount);
  62   return ceph_set_deleg_timeout(cmount, session_timeout - 1);
  63 }
  64
  65 static void dummy_deleg_cb(Fh *fh, void *priv)
  66 {
  67   std::atomic_bool *recalled = (std::atomic_bool *)priv;
  68   recalled->store(true);
  69 }
  70
  71 static void open_breaker_func(struct ceph_mount_info *cmount, const char *filename, int flags, std::atomic_bool *opened)
  72 {
  73   bool do_shutdown = false;
  74
  75   if (!cmount) {
  76     ASSERT_EQ(ceph_create(&cmount, NULL), 0);
  77     ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
  78     ASSERT_EQ(ceph_conf_parse_env(cmount, NULL), 0);
  79     ASSERT_EQ(ceph_mount(cmount, "/"), 0);
  80     ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
  81     do_shutdown = true;
  82   }
  83
  84   Inode *root, *file;
  85   ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
  86
  87   Fh *fh;
  88   struct ceph_statx stx;
  89   UserPerm *perms = ceph_mount_perms(cmount);
  90
  91   ASSERT_EQ(ceph_ll_lookup(cmount, root, filename, &file, &stx, CEPH_STATX_ALL_STATS, 0, perms), 0);
  92   int ret, i = 0;
  93   for (;;) {
  94     ASSERT_EQ(ceph_ll_getattr(cmount, file, &stx, CEPH_STATX_ALL_STATS, 0, perms), 0);
  95     ret = ceph_ll_open(cmount, file, flags, &fh, perms);
  96     if (ret != -EAGAIN)
  97       break;
  98     ASSERT_LT(i++, MAX_WAIT);
  99     usleep(1000);
 100   }
 101   ASSERT_EQ(ret, 0);
 102   opened->store(true);
 103   ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
 104
 105   if (do_shutdown)
 106     ceph_shutdown(cmount);
 107 }
 108
 109 enum {
 110   DelegTestLink,
 111   DelegTestRename,
 112   DelegTestUnlink
 113 };
 114
 115 static void namespace_breaker_func(struct ceph_mount_info *cmount, int cmd, const char *oldname, const char *newname)
 116 {
 117   bool do_shutdown = false;
 118
 119   if (!cmount) {
 120     ASSERT_EQ(ceph_create(&cmount, NULL), 0);
 121     ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
 122     ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
 123     ASSERT_EQ(ceph_mount(cmount, "/"), 0);
 124     ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
 125     do_shutdown = true;
 126   }
 127
 128   Inode *root, *file = nullptr;
 129   ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
 130
 131   struct ceph_statx stx;
 132   UserPerm *perms = ceph_mount_perms(cmount);
 133
 134   int ret, i = 0;
 135   for (;;) {
 136     switch (cmd) {
 137     case DelegTestRename:
 138       ret = ceph_ll_rename(cmount, root, oldname, root, newname, perms);
 139       break;
 140     case DelegTestLink:
 141       if (!file) {
 142         ASSERT_EQ(ceph_ll_lookup(cmount, root, oldname, &file, &stx, 0, 0, perms), 0);
 143       }
 144       ret = ceph_ll_link(cmount, file, root, newname, perms);
 145       break;
 146     case DelegTestUnlink:
 147       ret = ceph_ll_unlink(cmount, root, oldname, perms);
 148       break;
 149     default:
 150       // Bad command
 151       ceph_abort();
 152     }
 153     if (ret != -EAGAIN)
 154       break;
 155     ASSERT_LT(i++, MAX_WAIT);
 156     usleep(1000);
 157   }
 158   ASSERT_EQ(ret, 0);
 159
 160   if (do_shutdown)
 161     ceph_shutdown(cmount);
 162 }
 163
 164 static void simple_deleg_test(struct ceph_mount_info *cmount, struct ceph_mount_info *tcmount)
 165 {
 166   Inode *root, *file;
 167
 168   ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
 169
 170   char filename[32];
 171
 172   Fh *fh;
 173   struct ceph_statx stx;
 174   UserPerm *perms = ceph_mount_perms(cmount);
 175
 176   std::atomic_bool recalled(false);
 177   std::atomic_bool opened(false);
 178
 179   // ensure r/w open breaks a r/w delegation
 180   sprintf(filename, "deleg.rwrw.%x", getpid());
 181   ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
 182                     O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
 183   ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
 184   std::thread breaker1(open_breaker_func, tcmount, filename, O_RDWR, &opened);
 185
 186   wait_for_atomic_bool(recalled);
 187   ASSERT_EQ(opened.load(), false);
 188   ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
 189   breaker1.join();
 190   ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
 191   ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
 192
 193   // ensure r/o open breaks a r/w delegation
 194   recalled.store(false);
 195   opened.store(false);
 196   sprintf(filename, "deleg.rorw.%x", getpid());
 197   ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
 198                     O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
 199   ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
 200   std::thread breaker2(open_breaker_func, tcmount, filename, O_RDONLY, &opened);
 201   wait_for_atomic_bool(recalled);
 202   ASSERT_EQ(opened.load(), false);
 203   ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
 204   breaker2.join();
 205   ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
 206   ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
 207
 208   // ensure r/o open does not break a r/o delegation
 209   sprintf(filename, "deleg.rwro.%x", getpid());
 210   ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
 211                     O_RDONLY|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
 212   recalled.store(false);
 213   ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_RD, dummy_deleg_cb, &recalled), 0);
 214   std::thread breaker3(open_breaker_func, tcmount, filename, O_RDONLY, &opened);
 215   breaker3.join();
 216   ASSERT_EQ(recalled.load(), false);
 217
 218   // ensure that r/w open breaks r/o delegation
 219   opened.store(false);
 220   std::thread breaker4(open_breaker_func, tcmount, filename, O_WRONLY, &opened);
 221   wait_for_atomic_bool(recalled);
 222   usleep(1000);
 223   ASSERT_EQ(opened.load(), false);
 224   ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
 225   breaker4.join();
 226   ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
 227   ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
 228
 229   // ensure hardlinking breaks a r/w delegation
 230   recalled.store(false);
 231   char newname[32];
 232   sprintf(filename, "deleg.old.%x", getpid());
 233   sprintf(newname, "deleg.new.%x", getpid());
 234   ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
 235                     O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
 236   ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
 237   std::thread breaker5(namespace_breaker_func, tcmount, DelegTestLink, filename, newname);
 238   wait_for_atomic_bool(recalled);
 239   ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
 240   breaker5.join();
 241   ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
 242   ASSERT_EQ(ceph_ll_unlink(cmount, root, filename, perms), 0);
 243   ASSERT_EQ(ceph_ll_unlink(cmount, root, newname, perms), 0);
 244
 245   // ensure renaming breaks a r/w delegation
 246   recalled.store(false);
 247   ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
 248                     O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
 249   ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
 250   std::thread breaker6(namespace_breaker_func, tcmount, DelegTestRename, filename, newname);
 251   wait_for_atomic_bool(recalled);
 252   ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
 253   breaker6.join();
 254   ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
 255   ASSERT_EQ(ceph_ll_unlink(cmount, root, newname, perms), 0);
 256
 257   // ensure unlinking breaks a r/w delegation
 258   recalled.store(false);
 259   ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
 260                     O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
 261   ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_WR, dummy_deleg_cb, &recalled), 0);
 262   std::thread breaker7(namespace_breaker_func, tcmount, DelegTestUnlink, filename, nullptr);
 263   wait_for_atomic_bool(recalled);
 264   ASSERT_EQ(ceph_ll_delegation(cmount, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, &recalled), 0);
 265   breaker7.join();
 266   ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
 267 }
 268
 269 TEST(LibCephFS, DelegMultiClient) {
 270   struct ceph_mount_info *cmount;
 271
 272   ASSERT_EQ(ceph_create(&cmount, NULL), 0);
 273   ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
 274   ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
 275   ASSERT_EQ(ceph_mount(cmount, "/"), 0);
 276   ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
 277
 278   simple_deleg_test(cmount, nullptr);
 279
 280   ceph_shutdown(cmount);
 281 }
 282
 283 TEST(LibCephFS, DelegSingleClient) {
 284   struct ceph_mount_info *cmount;
 285
 286   ASSERT_EQ(ceph_create(&cmount, NULL), 0);
 287   ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
 288   ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
 289   ASSERT_EQ(ceph_mount(cmount, "/"), 0);
 290   ASSERT_EQ(set_default_deleg_timeout(cmount), 0);
 291
 292   simple_deleg_test(cmount, cmount);
 293
 294   ceph_shutdown(cmount);
 295 }
 296
 297 TEST(LibCephFS, DelegTimeout) {
 298   struct ceph_mount_info *cmount;
 299   ASSERT_EQ(ceph_create(&cmount, NULL), 0);
 300   ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
 301   ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
 302   ASSERT_EQ(ceph_mount(cmount, "/"), 0);
 303   // tweak timeout to run quickly, since we don't plan to return it anyway
 304   ASSERT_EQ(ceph_set_deleg_timeout(cmount, 2), 0);
 305
 306   Inode *root, *file;
 307   ASSERT_EQ(ceph_ll_lookup_root(cmount, &root), 0);
 308
 309   char filename[32];
 310   sprintf(filename, "delegtimeo%x", getpid());
 311
 312   Fh *fh;
 313   struct ceph_statx stx;
 314   UserPerm *perms = ceph_mount_perms(cmount);
 315
 316   ASSERT_EQ(ceph_ll_create(cmount, root, filename, 0666,
 317                     O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
 318
 319   /* Reopen read-only */
 320   ASSERT_EQ(ceph_ll_close(cmount, fh), 0);
 321   ASSERT_EQ(ceph_ll_open(cmount, file, O_RDONLY, &fh, perms), 0);
 322
 323   std::atomic_bool recalled(false);
 324   ASSERT_EQ(ceph_ll_delegation_wait(cmount, fh, CEPH_DELEGATION_RD, dummy_deleg_cb, &recalled), 0);
 325   std::atomic_bool opened(false);
 326   std::thread breaker1(open_breaker_func, nullptr, filename, O_RDWR, &opened);
 327   breaker1.join();
 328   ASSERT_EQ(recalled.load(), true);
 329   ASSERT_EQ(ceph_ll_getattr(cmount, root, &stx, 0, 0, perms), -ENOTCONN);
 330   ceph_release(cmount);
 331 }
 332
 333 TEST(LibCephFS, RecalledGetattr) {
 334   struct ceph_mount_info *cmount1;
 335   ASSERT_EQ(ceph_create(&cmount1, NULL), 0);
 336   ASSERT_EQ(ceph_conf_read_file(cmount1, NULL), 0);
 337   ASSERT_EQ(0, ceph_conf_parse_env(cmount1, NULL));
 338   ASSERT_EQ(ceph_mount(cmount1, "/"), 0);
 339   ASSERT_EQ(set_default_deleg_timeout(cmount1), 0);
 340
 341   Inode *root, *file;
 342   ASSERT_EQ(ceph_ll_lookup_root(cmount1, &root), 0);
 343
 344   char filename[32];
 345   sprintf(filename, "recalledgetattr%x", getpid());
 346
 347   Fh *fh;
 348   struct ceph_statx stx;
 349   UserPerm *perms = ceph_mount_perms(cmount1);
 350
 351   ASSERT_EQ(ceph_ll_create(cmount1, root, filename, 0666,
 352                     O_RDWR|O_CREAT|O_EXCL, &file, &fh, &stx, 0, 0, perms), 0);
 353   ASSERT_EQ(ceph_ll_write(cmount1, fh, 0, sizeof(filename), filename),
 354             static_cast<int>(sizeof(filename)));
 355   ASSERT_EQ(ceph_ll_close(cmount1, fh), 0);
 356
 357   /* New mount for read delegation */
 358   struct ceph_mount_info *cmount2;
 359   ASSERT_EQ(ceph_create(&cmount2, NULL), 0);
 360   ASSERT_EQ(ceph_conf_read_file(cmount2, NULL), 0);
 361   ASSERT_EQ(0, ceph_conf_parse_env(cmount2, NULL));
 362   ASSERT_EQ(ceph_mount(cmount2, "/"), 0);
 363   ASSERT_EQ(set_default_deleg_timeout(cmount2), 0);
 364
 365   ASSERT_EQ(ceph_ll_lookup_root(cmount2, &root), 0);
 366   perms = ceph_mount_perms(cmount2);
 367   ASSERT_EQ(ceph_ll_lookup(cmount2, root, filename, &file, &stx, 0, 0, perms), 0);
 368
 369   ASSERT_EQ(ceph_ll_open(cmount2, file, O_WRONLY, &fh, perms), 0);
 370   ASSERT_EQ(ceph_ll_write(cmount2, fh, 0, sizeof(filename), filename),
 371             static_cast<int>(sizeof(filename)));
 372   ASSERT_EQ(ceph_ll_close(cmount2, fh), 0);
 373
 374   ASSERT_EQ(ceph_ll_open(cmount2, file, O_RDONLY, &fh, perms), 0);
 375
 376   /* Break delegation */
 377   std::atomic_bool recalled(false);
 378   ASSERT_EQ(ceph_ll_delegation_wait(cmount2, fh, CEPH_DELEGATION_RD, dummy_deleg_cb, &recalled), 0);
 379   ASSERT_EQ(ceph_ll_read(cmount2, fh, 0, sizeof(filename), filename),
 380             static_cast<int>(sizeof(filename)));
 381   ASSERT_EQ(ceph_ll_getattr(cmount2, file, &stx, CEPH_STATX_ALL_STATS, 0, perms), 0);
 382   std::atomic_bool opened(false);
 383   std::thread breaker1(open_breaker_func, cmount1, filename, O_WRONLY, &opened);
 384   int i = 0;
 385   do {
 386     ASSERT_EQ(ceph_ll_getattr(cmount2, file, &stx, CEPH_STATX_ALL_STATS, 0, perms), 0);
 387     ASSERT_LT(i++, MAX_WAIT);
 388     usleep(1000);
 389   } while (!recalled.load());
 390   ASSERT_EQ(opened.load(), false);
 391   ASSERT_EQ(ceph_ll_getattr(cmount2, file, &stx, CEPH_STATX_ALL_STATS, 0, perms), 0);
 392   ASSERT_EQ(ceph_ll_delegation(cmount2, fh, CEPH_DELEGATION_NONE, dummy_deleg_cb, nullptr), 0);
 393   breaker1.join();
 394   ASSERT_EQ(ceph_ll_close(cmount2, fh), 0);
 395   ceph_unmount(cmount2);
 396   ceph_release(cmount2);
 397   ceph_unmount(cmount1);
 398   ceph_release(cmount1);
 399 }