ceph/src/test/cls_lock/test_cls_lock.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14
  15 #include <iostream>
  16 #include <errno.h>
  17
  18 #include "include/types.h"
  19 #include "common/Clock.h"
  20 #include "msg/msg_types.h"
  21 #include "include/rados/librados.hpp"
  22
  23 #include "test/librados/test_cxx.h"
  24 #include "gtest/gtest.h"
  25
  26 using namespace librados;
  27
  28 #include "cls/lock/cls_lock_client.h"
  29 #include "cls/lock/cls_lock_ops.h"
  30
  31 using namespace rados::cls::lock;
  32
  33 void lock_info(IoCtx *ioctx, string& oid, string& name, map<locker_id_t, locker_info_t>& lockers,
  34                ClsLockType *assert_type, string *assert_tag)
  35 {
  36   ClsLockType lock_type = LOCK_NONE;
  37   string tag;
  38   lockers.clear();
  39   ASSERT_EQ(0, get_lock_info(ioctx, oid, name, &lockers, &lock_type, &tag));
  40   cout << "lock: " << name << std::endl;
  41   cout << "  lock_type: " << cls_lock_type_str(lock_type) << std::endl;
  42   cout << "  tag: " << tag << std::endl;
  43   cout << "  lockers:" << std::endl;
  44
  45   if (assert_type) {
  46     ASSERT_EQ(*assert_type, lock_type);
  47   }
  48
  49   if (assert_tag) {
  50     ASSERT_EQ(*assert_tag, tag);
  51   }
  52
  53   map<locker_id_t, locker_info_t>::iterator liter;
  54   for (liter = lockers.begin(); liter != lockers.end(); ++liter) {
  55     const locker_id_t& locker = liter->first;
  56     cout << "    " << locker.locker << " expiration=" << liter->second.expiration
  57          << " addr=" << liter->second.addr << " cookie=" << locker.cookie << std::endl;
  58   }
  59 }
  60
  61 void lock_info(IoCtx *ioctx, string& oid, string& name, map<locker_id_t, locker_info_t>& lockers)
  62 {
  63   lock_info(ioctx, oid, name, lockers, NULL, NULL);
  64 }
  65
  66 TEST(ClsLock, TestMultiLocking) {
  67   Rados cluster;
  68   std::string pool_name = get_temp_pool_name();
  69   ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
  70   IoCtx ioctx;
  71   cluster.ioctx_create(pool_name.c_str(), ioctx);
  72   ClsLockType lock_type_shared = LOCK_SHARED;
  73   ClsLockType lock_type_exclusive = LOCK_EXCLUSIVE;
  74
  75
  76   Rados cluster2;
  77   IoCtx ioctx2;
  78   ASSERT_EQ("", connect_cluster_pp(cluster2));
  79   cluster2.ioctx_create(pool_name.c_str(), ioctx2);
  80
  81   string oid = "foo";
  82   bufferlist bl;
  83   string lock_name = "mylock";
  84
  85   ASSERT_EQ(0, ioctx.write(oid, bl, bl.length(), 0));
  86
  87   Lock l(lock_name);
  88   // we set the duration, so the log output contains a locker with a
  89   // non-zero expiration time
  90   l.set_duration(utime_t(120, 0));
  91
  92   /* test lock object */
  93
  94   ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
  95
  96   /* test exclusive lock */
  97   ASSERT_EQ(-EEXIST, l.lock_exclusive(&ioctx, oid));
  98
  99   /* test idempotency */
 100   l.set_may_renew(true);
 101   ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
 102
 103   l.set_may_renew(false);
 104
 105   /* test second client */
 106   Lock l2(lock_name);
 107   ASSERT_EQ(-EBUSY, l2.lock_exclusive(&ioctx2, oid));
 108   ASSERT_EQ(-EBUSY, l2.lock_shared(&ioctx2, oid));
 109
 110   list<string> locks;
 111   ASSERT_EQ(0, list_locks(&ioctx, oid, &locks));
 112
 113   ASSERT_EQ(1, (int)locks.size());
 114   list<string>::iterator iter = locks.begin();
 115   map<locker_id_t, locker_info_t> lockers;
 116   lock_info(&ioctx, oid, *iter, lockers, &lock_type_exclusive, NULL);
 117
 118   ASSERT_EQ(1, (int)lockers.size());
 119
 120   /* test unlock */
 121   ASSERT_EQ(0, l.unlock(&ioctx, oid));
 122   locks.clear();
 123   ASSERT_EQ(0, list_locks(&ioctx, oid, &locks));
 124
 125   /* test shared lock */
 126   ASSERT_EQ(0, l2.lock_shared(&ioctx2, oid));
 127   ASSERT_EQ(0, l.lock_shared(&ioctx, oid));
 128
 129   locks.clear();
 130   ASSERT_EQ(0, list_locks(&ioctx, oid, &locks));
 131   ASSERT_EQ(1, (int)locks.size());
 132   iter = locks.begin();
 133   lock_info(&ioctx, oid, *iter, lockers, &lock_type_shared, NULL);
 134   ASSERT_EQ(2, (int)lockers.size());
 135
 136   /* test break locks */
 137   entity_name_t name = entity_name_t::CLIENT(cluster.get_instance_id());
 138   entity_name_t name2 = entity_name_t::CLIENT(cluster2.get_instance_id());
 139
 140   l2.break_lock(&ioctx2, oid, name);
 141   lock_info(&ioctx, oid, *iter, lockers);
 142   ASSERT_EQ(1, (int)lockers.size());
 143   map<locker_id_t, locker_info_t>::iterator liter = lockers.begin();
 144   const locker_id_t& id = liter->first;
 145   ASSERT_EQ(name2, id.locker);
 146
 147   /* test lock tag */
 148   Lock l_tag(lock_name);
 149   l_tag.set_tag("non-default tag");
 150   ASSERT_EQ(-EBUSY, l_tag.lock_shared(&ioctx, oid));
 151
 152
 153   /* test modify description */
 154   string description = "new description";
 155   l.set_description(description);
 156   ASSERT_EQ(0, l.lock_shared(&ioctx, oid));
 157
 158   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
 159 }
 160
 161 TEST(ClsLock, TestMeta) {
 162   Rados cluster;
 163   std::string pool_name = get_temp_pool_name();
 164   ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
 165   IoCtx ioctx;
 166   cluster.ioctx_create(pool_name.c_str(), ioctx);
 167
 168
 169   Rados cluster2;
 170   IoCtx ioctx2;
 171   ASSERT_EQ("", connect_cluster_pp(cluster2));
 172   cluster2.ioctx_create(pool_name.c_str(), ioctx2);
 173
 174   string oid = "foo";
 175   bufferlist bl;
 176   string lock_name = "mylock";
 177
 178   ASSERT_EQ(0, ioctx.write(oid, bl, bl.length(), 0));
 179
 180   Lock l(lock_name);
 181   ASSERT_EQ(0, l.lock_shared(&ioctx, oid));
 182
 183   /* test lock tag */
 184   Lock l_tag(lock_name);
 185   l_tag.set_tag("non-default tag");
 186   ASSERT_EQ(-EBUSY, l_tag.lock_shared(&ioctx2, oid));
 187
 188
 189   ASSERT_EQ(0, l.unlock(&ioctx, oid));
 190
 191   /* test description */
 192   Lock l2(lock_name);
 193   string description = "new description";
 194   l2.set_description(description);
 195   ASSERT_EQ(0, l2.lock_shared(&ioctx2, oid));
 196
 197   map<locker_id_t, locker_info_t> lockers;
 198   lock_info(&ioctx, oid, lock_name, lockers, NULL, NULL);
 199   ASSERT_EQ(1, (int)lockers.size());
 200
 201   map<locker_id_t, locker_info_t>::iterator iter = lockers.begin();
 202   locker_info_t locker = iter->second;
 203   ASSERT_EQ("new description", locker.description);
 204
 205   ASSERT_EQ(0, l2.unlock(&ioctx2, oid));
 206
 207   /* check new tag */
 208   string new_tag = "new_tag";
 209   l.set_tag(new_tag);
 210   l.set_may_renew(true);
 211   ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
 212   lock_info(&ioctx, oid, lock_name, lockers, NULL, &new_tag);
 213   ASSERT_EQ(1, (int)lockers.size());
 214   l.set_tag("");
 215   ASSERT_EQ(-EBUSY, l.lock_exclusive(&ioctx, oid));
 216   l.set_tag(new_tag);
 217   ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
 218
 219   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
 220 }
 221
 222 TEST(ClsLock, TestCookie) {
 223   Rados cluster;
 224   std::string pool_name = get_temp_pool_name();
 225   ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
 226   IoCtx ioctx;
 227   cluster.ioctx_create(pool_name.c_str(), ioctx);
 228
 229   string oid = "foo";
 230   string lock_name = "mylock";
 231   Lock l(lock_name);
 232
 233   ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
 234
 235   /* new cookie */
 236   string cookie = "new cookie";
 237   l.set_cookie(cookie);
 238   ASSERT_EQ(-EBUSY, l.lock_exclusive(&ioctx, oid));
 239   ASSERT_EQ(-ENOENT, l.unlock(&ioctx, oid));
 240   l.set_cookie("");
 241   ASSERT_EQ(0, l.unlock(&ioctx, oid));
 242
 243   map<locker_id_t, locker_info_t> lockers;
 244   lock_info(&ioctx, oid, lock_name, lockers);
 245   ASSERT_EQ(0, (int)lockers.size());
 246
 247   l.set_cookie(cookie);
 248   ASSERT_EQ(0, l.lock_shared(&ioctx, oid));
 249   l.set_cookie("");
 250   ASSERT_EQ(0, l.lock_shared(&ioctx, oid));
 251
 252   lock_info(&ioctx, oid, lock_name, lockers);
 253   ASSERT_EQ(2, (int)lockers.size());
 254
 255   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
 256 }
 257
 258 TEST(ClsLock, TestMultipleLocks) {
 259   Rados cluster;
 260   std::string pool_name = get_temp_pool_name();
 261   ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
 262   IoCtx ioctx;
 263   cluster.ioctx_create(pool_name.c_str(), ioctx);
 264
 265   string oid = "foo";
 266   Lock l("lock1");
 267   ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
 268
 269   Lock l2("lock2");
 270   ASSERT_EQ(0, l2.lock_exclusive(&ioctx, oid));
 271
 272   list<string> locks;
 273   ASSERT_EQ(0, list_locks(&ioctx, oid, &locks));
 274
 275   ASSERT_EQ(2, (int)locks.size());
 276
 277   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
 278 }
 279
 280 TEST(ClsLock, TestLockDuration) {
 281   Rados cluster;
 282   std::string pool_name = get_temp_pool_name();
 283   ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
 284   IoCtx ioctx;
 285   cluster.ioctx_create(pool_name.c_str(), ioctx);
 286
 287   string oid = "foo";
 288   Lock l("lock");
 289   utime_t dur(5, 0);
 290   l.set_duration(dur);
 291   utime_t start = ceph_clock_now();
 292   ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
 293   int r = l.lock_exclusive(&ioctx, oid);
 294   if (r == 0) {
 295     // it's possible to get success if we were just really slow...
 296     ASSERT_TRUE(ceph_clock_now() > start + dur);
 297   } else {
 298     ASSERT_EQ(-EEXIST, r);
 299   }
 300
 301   sleep(dur.sec());
 302   ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
 303
 304   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
 305 }
 306
 307 TEST(ClsLock, TestAssertLocked) {
 308   Rados cluster;
 309   std::string pool_name = get_temp_pool_name();
 310   ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
 311   IoCtx ioctx;
 312   cluster.ioctx_create(pool_name.c_str(), ioctx);
 313
 314   string oid = "foo";
 315   Lock l("lock1");
 316   ASSERT_EQ(0, l.lock_exclusive(&ioctx, oid));
 317
 318   librados::ObjectWriteOperation op1;
 319   l.assert_locked_exclusive(&op1);
 320   ASSERT_EQ(0, ioctx.operate(oid, &op1));
 321
 322   librados::ObjectWriteOperation op2;
 323   l.assert_locked_shared(&op2);
 324   ASSERT_EQ(-EBUSY, ioctx.operate(oid, &op2));
 325
 326   l.set_tag("tag");
 327   librados::ObjectWriteOperation op3;
 328   l.assert_locked_exclusive(&op3);
 329   ASSERT_EQ(-EBUSY, ioctx.operate(oid, &op3));
 330   l.set_tag("");
 331
 332   l.set_cookie("cookie");
 333   librados::ObjectWriteOperation op4;
 334   l.assert_locked_exclusive(&op4);
 335   ASSERT_EQ(-EBUSY, ioctx.operate(oid, &op4));
 336   l.set_cookie("");
 337
 338   ASSERT_EQ(0, l.unlock(&ioctx, oid));
 339
 340   librados::ObjectWriteOperation op5;
 341   l.assert_locked_exclusive(&op5);
 342   ASSERT_EQ(-EBUSY, ioctx.operate(oid, &op5));
 343
 344   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
 345 }
 346
 347 TEST(ClsLock, TestSetCookie) {
 348   Rados cluster;
 349   std::string pool_name = get_temp_pool_name();
 350   ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
 351   IoCtx ioctx;
 352   cluster.ioctx_create(pool_name.c_str(), ioctx);
 353
 354   string oid = "foo";
 355   string name = "name";
 356   string tag = "tag";
 357   string cookie = "cookie";
 358   string new_cookie = "new cookie";
 359   librados::ObjectWriteOperation op1;
 360   set_cookie(&op1, name, LOCK_SHARED, cookie, tag, new_cookie);
 361   ASSERT_EQ(-ENOENT, ioctx.operate(oid, &op1));
 362
 363   librados::ObjectWriteOperation op2;
 364   lock(&op2, name, LOCK_SHARED, cookie, tag, "", utime_t{}, 0);
 365   ASSERT_EQ(0, ioctx.operate(oid, &op2));
 366
 367   librados::ObjectWriteOperation op3;
 368   lock(&op3, name, LOCK_SHARED, "cookie 2", tag, "", utime_t{}, 0);
 369   ASSERT_EQ(0, ioctx.operate(oid, &op3));
 370
 371   librados::ObjectWriteOperation op4;
 372   set_cookie(&op4, name, LOCK_SHARED, cookie, tag, cookie);
 373   ASSERT_EQ(-EBUSY, ioctx.operate(oid, &op4));
 374
 375   librados::ObjectWriteOperation op5;
 376   set_cookie(&op5, name, LOCK_SHARED, cookie, "wrong tag", new_cookie);
 377   ASSERT_EQ(-EBUSY, ioctx.operate(oid, &op5));
 378
 379   librados::ObjectWriteOperation op6;
 380   set_cookie(&op6, name, LOCK_SHARED, "wrong cookie", tag, new_cookie);
 381   ASSERT_EQ(-EBUSY, ioctx.operate(oid, &op6));
 382
 383   librados::ObjectWriteOperation op7;
 384   set_cookie(&op7, name, LOCK_EXCLUSIVE, cookie, tag, new_cookie);
 385   ASSERT_EQ(-EBUSY, ioctx.operate(oid, &op7));
 386
 387   librados::ObjectWriteOperation op8;
 388   set_cookie(&op8, name, LOCK_SHARED, cookie, tag, "cookie 2");
 389   ASSERT_EQ(-EBUSY, ioctx.operate(oid, &op8));
 390
 391   librados::ObjectWriteOperation op9;
 392   set_cookie(&op9, name, LOCK_SHARED, cookie, tag, new_cookie);
 393   ASSERT_EQ(0, ioctx.operate(oid, &op9));
 394
 395   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
 396 }
 397
 398 TEST(ClsLock, TestRenew) {
 399   Rados cluster;
 400   std::string pool_name = get_temp_pool_name();
 401   ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
 402   IoCtx ioctx;
 403   cluster.ioctx_create(pool_name.c_str(), ioctx);
 404
 405   bufferlist bl;
 406
 407   string oid1 = "foo1";
 408   string lock_name1 = "mylock1";
 409
 410   ASSERT_EQ(0, ioctx.write(oid1, bl, bl.length(), 0));
 411
 412   Lock l1(lock_name1);
 413   utime_t lock_duration1(5, 0);
 414   l1.set_duration(lock_duration1);
 415
 416   ASSERT_EQ(0, l1.lock_exclusive(&ioctx, oid1));
 417   l1.set_may_renew(true);
 418   sleep(2);
 419   ASSERT_EQ(0, l1.lock_exclusive(&ioctx, oid1));
 420   sleep(7);
 421   ASSERT_EQ(0, l1.lock_exclusive(&ioctx, oid1)) <<
 422     "when a cls_lock is set to may_renew, a relock after expiration "
 423     "should still work";
 424   ASSERT_EQ(0, l1.unlock(&ioctx, oid1));
 425
 426   // ***********************************************
 427
 428   string oid2 = "foo2";
 429   string lock_name2 = "mylock2";
 430
 431   ASSERT_EQ(0, ioctx.write(oid2, bl, bl.length(), 0));
 432
 433   Lock l2(lock_name2);
 434   utime_t lock_duration2(5, 0);
 435   l2.set_duration(lock_duration2);
 436
 437   ASSERT_EQ(0, l2.lock_exclusive(&ioctx, oid2));
 438   l2.set_must_renew(true);
 439   sleep(2);
 440   ASSERT_EQ(0, l2.lock_exclusive(&ioctx, oid2));
 441   sleep(7);
 442   ASSERT_EQ(-ENOENT, l2.lock_exclusive(&ioctx, oid2)) <<
 443     "when a cls_lock is set to must_renew, a relock after expiration "
 444     "should fail";
 445   ASSERT_EQ(-ENOENT, l2.unlock(&ioctx, oid2));
 446
 447   // ***********************************************
 448
 449   string oid3 = "foo3";
 450   string lock_name3 = "mylock3";
 451
 452   ASSERT_EQ(0, ioctx.write(oid3, bl, bl.length(), 0));
 453
 454   Lock l3(lock_name3);
 455   l3.set_duration(utime_t(5, 0));
 456   l3.set_must_renew(true);
 457
 458   ASSERT_EQ(-ENOENT, l3.lock_exclusive(&ioctx, oid3)) <<
 459     "unable to create a lock with must_renew";
 460
 461   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
 462 }
 463
 464 TEST(ClsLock, TestExclusiveEphemeralBasic) {
 465   Rados cluster;
 466   std::string pool_name = get_temp_pool_name();
 467   ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
 468   IoCtx ioctx;
 469   cluster.ioctx_create(pool_name.c_str(), ioctx);
 470
 471   bufferlist bl;
 472
 473   string oid1 = "foo1";
 474   string oid2 = "foo2";
 475   string lock_name1 = "mylock1";
 476   string lock_name2 = "mylock2";
 477
 478   Lock l1(lock_name1);
 479   l1.set_duration(utime_t(5, 0));
 480
 481   uint64_t size;
 482   time_t mod_time;
 483
 484   l1.set_may_renew(true);
 485   ASSERT_EQ(0, l1.lock_exclusive_ephemeral(&ioctx, oid1));
 486   ASSERT_EQ(0, ioctx.stat(oid1, &size, &mod_time));
 487   sleep(2);
 488   ASSERT_EQ(0, l1.unlock(&ioctx, oid1));
 489   ASSERT_EQ(-ENOENT, ioctx.stat(oid1, &size, &mod_time));
 490
 491   // ***********************************************
 492
 493   Lock l2(lock_name2);
 494   utime_t lock_duration2(5, 0);
 495   l2.set_duration(utime_t(5, 0));
 496
 497   ASSERT_EQ(0, l2.lock_exclusive(&ioctx, oid2));
 498   ASSERT_EQ(0, ioctx.stat(oid2, &size, &mod_time));
 499   sleep(2);
 500   ASSERT_EQ(0, l2.unlock(&ioctx, oid2));
 501   ASSERT_EQ(0, ioctx.stat(oid2, &size, &mod_time));
 502
 503   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
 504 }
 505
 506
 507 TEST(ClsLock, TestExclusiveEphemeralStealEphemeral) {
 508   Rados cluster;
 509   std::string pool_name = get_temp_pool_name();
 510   ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
 511   IoCtx ioctx;
 512   cluster.ioctx_create(pool_name.c_str(), ioctx);
 513
 514   bufferlist bl;
 515
 516   string oid1 = "foo1";
 517   string lock_name1 = "mylock1";
 518
 519   Lock l1(lock_name1);
 520   l1.set_duration(utime_t(3, 0));
 521
 522   ASSERT_EQ(0, l1.lock_exclusive_ephemeral(&ioctx, oid1));
 523   sleep(4);
 524
 525   // l1 is expired, l2 can take; l2 is also exclusive_ephemeral
 526   Lock l2(lock_name1);
 527   l2.set_duration(utime_t(3, 0));
 528   ASSERT_EQ(0, l2.lock_exclusive_ephemeral(&ioctx, oid1));
 529   sleep(1);
 530   ASSERT_EQ(0, l2.unlock(&ioctx, oid1));
 531
 532   // l2 cannot unlock its expired lock
 533   ASSERT_EQ(-ENOENT, l1.unlock(&ioctx, oid1));
 534
 535   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
 536 }
 537
 538
 539 TEST(ClsLock, TestExclusiveEphemeralStealExclusive) {
 540   Rados cluster;
 541   std::string pool_name = get_temp_pool_name();
 542   ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
 543   IoCtx ioctx;
 544   cluster.ioctx_create(pool_name.c_str(), ioctx);
 545
 546   bufferlist bl;
 547
 548   string oid1 = "foo1";
 549   string lock_name1 = "mylock1";
 550
 551   Lock l1(lock_name1);
 552   l1.set_duration(utime_t(3, 0));
 553
 554   ASSERT_EQ(0, l1.lock_exclusive_ephemeral(&ioctx, oid1));
 555   sleep(4);
 556
 557   // l1 is expired, l2 can take; l2 is exclusive (but not ephemeral)
 558   Lock l2(lock_name1);
 559   l2.set_duration(utime_t(3, 0));
 560   ASSERT_EQ(0, l2.lock_exclusive(&ioctx, oid1));
 561   sleep(1);
 562   ASSERT_EQ(0, l2.unlock(&ioctx, oid1));
 563
 564   // l2 cannot unlock its expired lock
 565   ASSERT_EQ(-ENOENT, l1.unlock(&ioctx, oid1));
 566
 567   ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
 568 }