ceph/src/rgw/driver/rados/rgw_zone.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #include "rgw_zone.h"
   5 #include "rgw_realm_watcher.h"
   6 #include "rgw_sal_config.h"
   7 #include "rgw_sync.h"
   8
   9 #include "services/svc_zone.h"
  10
  11 #define dout_context g_ceph_context
  12 #define dout_subsys ceph_subsys_rgw
  13
  14 using namespace std;
  15 using namespace rgw_zone_defaults;
  16
  17 RGWMetaSyncStatusManager::~RGWMetaSyncStatusManager(){}
  18
  19 #define FIRST_EPOCH 1
  20
  21 struct RGWAccessKey;
  22
  23 /// Generate a random uuid for realm/period/zonegroup/zone ids
  24 static std::string gen_random_uuid()
  25 {
  26   uuid_d uuid;
  27   uuid.generate_random();
  28   return uuid.to_string();
  29 }
  30
  31 void RGWDefaultZoneGroupInfo::dump(Formatter *f) const {
  32   encode_json("default_zonegroup", default_zonegroup, f);
  33 }
  34
  35 void RGWDefaultZoneGroupInfo::decode_json(JSONObj *obj) {
  36
  37   JSONDecoder::decode_json("default_zonegroup", default_zonegroup, obj);
  38   /* backward compatability with region */
  39   if (default_zonegroup.empty()) {
  40     JSONDecoder::decode_json("default_region", default_zonegroup, obj);
  41   }
  42 }
  43
  44 int RGWZoneGroup::create_default(const DoutPrefixProvider *dpp, optional_yield y, bool old_format)
  45 {
  46   name = default_zonegroup_name;
  47   api_name = default_zonegroup_name;
  48   is_master = true;
  49
  50   RGWZoneGroupPlacementTarget placement_target;
  51   placement_target.name = "default-placement";
  52   placement_targets[placement_target.name] = placement_target;
  53   default_placement.name = "default-placement";
  54
  55   RGWZoneParams zone_params(default_zone_name);
  56
  57   int r = zone_params.init(dpp, cct, sysobj_svc, y, false);
  58   if (r < 0) {
  59     ldpp_dout(dpp, 0) << "create_default: error initializing zone params: " << cpp_strerror(-r) << dendl;
  60     return r;
  61   }
  62
  63   r = zone_params.create_default(dpp, y);
  64   if (r < 0 && r != -EEXIST) {
  65     ldpp_dout(dpp, 0) << "create_default: error in create_default  zone params: " << cpp_strerror(-r) << dendl;
  66     return r;
  67   } else if (r == -EEXIST) {
  68     ldpp_dout(dpp, 10) << "zone_params::create_default() returned -EEXIST, we raced with another default zone_params creation" << dendl;
  69     zone_params.clear_id();
  70     r = zone_params.init(dpp, cct, sysobj_svc, y);
  71     if (r < 0) {
  72       ldpp_dout(dpp, 0) << "create_default: error in init existing zone params: " << cpp_strerror(-r) << dendl;
  73       return r;
  74     }
  75     ldpp_dout(dpp, 20) << "zone_params::create_default() " << zone_params.get_name() << " id " << zone_params.get_id()
  76                    << dendl;
  77   }
  78
  79   RGWZone& default_zone = zones[zone_params.get_id()];
  80   default_zone.name = zone_params.get_name();
  81   default_zone.id = zone_params.get_id();
  82   master_zone = default_zone.id;
  83
  84   // enable all supported features
  85   enabled_features.insert(rgw::zone_features::supported.begin(),
  86                           rgw::zone_features::supported.end());
  87   default_zone.supported_features = enabled_features;
  88
  89   r = create(dpp, y);
  90   if (r < 0 && r != -EEXIST) {
  91     ldpp_dout(dpp, 0) << "error storing zone group info: " << cpp_strerror(-r) << dendl;
  92     return r;
  93   }
  94
  95   if (r == -EEXIST) {
  96     ldpp_dout(dpp, 10) << "create_default() returned -EEXIST, we raced with another zonegroup creation" << dendl;
  97     id.clear();
  98     r = init(dpp, cct, sysobj_svc, y);
  99     if (r < 0) {
 100       return r;
 101     }
 102   }
 103
 104   if (old_format) {
 105     name = id;
 106   }
 107
 108   post_process_params(dpp, y);
 109
 110   return 0;
 111 }
 112
 113 int RGWZoneGroup::equals(const string& other_zonegroup) const
 114 {
 115   if (is_master && other_zonegroup.empty())
 116     return true;
 117
 118   return (id  == other_zonegroup);
 119 }
 120
 121 int RGWZoneGroup::add_zone(const DoutPrefixProvider *dpp,
 122                            const RGWZoneParams& zone_params, bool *is_master, bool *read_only,
 123                            const list<string>& endpoints, const string *ptier_type,
 124                            bool *psync_from_all, list<string>& sync_from, list<string>& sync_from_rm,
 125                            string *predirect_zone, std::optional<int> bucket_index_max_shards,
 126                            RGWSyncModulesManager *sync_mgr,
 127                            const rgw::zone_features::set& enable_features,
 128                            const rgw::zone_features::set& disable_features,
 129                            optional_yield y)
 130 {
 131   auto& zone_id = zone_params.get_id();
 132   auto& zone_name = zone_params.get_name();
 133
 134   // check for duplicate zone name on insert
 135   if (!zones.count(zone_id)) {
 136     for (const auto& zone : zones) {
 137       if (zone.second.name == zone_name) {
 138         ldpp_dout(dpp, 0) << "ERROR: found existing zone name " << zone_name
 139             << " (" << zone.first << ") in zonegroup " << get_name() << dendl;
 140         return -EEXIST;
 141       }
 142     }
 143   }
 144
 145   if (is_master) {
 146     if (*is_master) {
 147       if (!master_zone.empty() && master_zone != zone_id) {
 148         ldpp_dout(dpp, 0) << "NOTICE: overriding master zone: " << master_zone << dendl;
 149       }
 150       master_zone = zone_id;
 151     } else if (master_zone == zone_id) {
 152       master_zone.clear();
 153     }
 154   }
 155
 156   RGWZone& zone = zones[zone_id];
 157   zone.name = zone_name;
 158   zone.id = zone_id;
 159   if (!endpoints.empty()) {
 160     zone.endpoints = endpoints;
 161   }
 162   if (read_only) {
 163     zone.read_only = *read_only;
 164   }
 165   if (ptier_type) {
 166     zone.tier_type = *ptier_type;
 167     if (!sync_mgr->get_module(*ptier_type, nullptr)) {
 168       ldpp_dout(dpp, 0) << "ERROR: could not found sync module: " << *ptier_type
 169                     << ",  valid sync modules: "
 170                     << sync_mgr->get_registered_module_names()
 171                     << dendl;
 172       return -ENOENT;
 173     }
 174   }
 175
 176   if (psync_from_all) {
 177     zone.sync_from_all = *psync_from_all;
 178   }
 179
 180   if (predirect_zone) {
 181     zone.redirect_zone = *predirect_zone;
 182   }
 183
 184   if (bucket_index_max_shards) {
 185     zone.bucket_index_max_shards = *bucket_index_max_shards;
 186   }
 187
 188   for (auto add : sync_from) {
 189     zone.sync_from.insert(add);
 190   }
 191
 192   for (auto rm : sync_from_rm) {
 193     zone.sync_from.erase(rm);
 194   }
 195
 196   zone.supported_features.insert(enable_features.begin(),
 197                                  enable_features.end());
 198
 199   for (const auto& feature : disable_features) {
 200     if (enabled_features.contains(feature)) {
 201       lderr(cct) << "ERROR: Cannot disable zone feature \"" << feature
 202           << "\" until it's been disabled in zonegroup " << name << dendl;
 203       return -EINVAL;
 204     }
 205     auto i = zone.supported_features.find(feature);
 206     if (i == zone.supported_features.end()) {
 207       ldout(cct, 1) << "WARNING: zone feature \"" << feature
 208           << "\" was not enabled in zone " << zone.name << dendl;
 209       continue;
 210     }
 211     zone.supported_features.erase(i);
 212   }
 213
 214   post_process_params(dpp, y);
 215
 216   return update(dpp,y);
 217 }
 218
 219
 220 int RGWZoneGroup::rename_zone(const DoutPrefixProvider *dpp,
 221                               const RGWZoneParams& zone_params,
 222                               optional_yield y)
 223 {
 224   RGWZone& zone = zones[zone_params.get_id()];
 225   zone.name = zone_params.get_name();
 226
 227   return update(dpp, y);
 228 }
 229
 230 void RGWZoneGroup::post_process_params(const DoutPrefixProvider *dpp, optional_yield y)
 231 {
 232   bool log_data = zones.size() > 1;
 233
 234   if (master_zone.empty()) {
 235     auto iter = zones.begin();
 236     if (iter != zones.end()) {
 237       master_zone = iter->first;
 238     }
 239   }
 240
 241   for (auto& item : zones) {
 242     RGWZone& zone = item.second;
 243     zone.log_data = log_data;
 244
 245     RGWZoneParams zone_params(zone.id, zone.name);
 246     int ret = zone_params.init(dpp, cct, sysobj_svc, y);
 247     if (ret < 0) {
 248       ldpp_dout(dpp, 0) << "WARNING: could not read zone params for zone id=" << zone.id << " name=" << zone.name << dendl;
 249       continue;
 250     }
 251
 252     for (auto& pitem : zone_params.placement_pools) {
 253       const string& placement_name = pitem.first;
 254       if (placement_targets.find(placement_name) == placement_targets.end()) {
 255         RGWZoneGroupPlacementTarget placement_target;
 256         placement_target.name = placement_name;
 257         placement_targets[placement_name] = placement_target;
 258       }
 259     }
 260   }
 261
 262   if (default_placement.empty() && !placement_targets.empty()) {
 263     default_placement.init(placement_targets.begin()->first, RGW_STORAGE_CLASS_STANDARD);
 264   }
 265 }
 266
 267 int RGWZoneGroup::remove_zone(const DoutPrefixProvider *dpp, const std::string& zone_id, optional_yield y)
 268 {
 269   auto iter = zones.find(zone_id);
 270   if (iter == zones.end()) {
 271     ldpp_dout(dpp, 0) << "zone id " << zone_id << " is not a part of zonegroup "
 272         << name << dendl;
 273     return -ENOENT;
 274   }
 275
 276   zones.erase(iter);
 277
 278   post_process_params(dpp, y);
 279
 280   return update(dpp, y);
 281 }
 282
 283 void RGWDefaultSystemMetaObjInfo::dump(Formatter *f) const {
 284   encode_json("default_id", default_id, f);
 285 }
 286
 287 void RGWDefaultSystemMetaObjInfo::decode_json(JSONObj *obj) {
 288   JSONDecoder::decode_json("default_id", default_id, obj);
 289 }
 290
 291 int RGWSystemMetaObj::rename(const DoutPrefixProvider *dpp, const string& new_name, optional_yield y)
 292 {
 293   string new_id;
 294   int ret = read_id(dpp, new_name, new_id, y);
 295   if (!ret) {
 296     return -EEXIST;
 297   }
 298   if (ret < 0 && ret != -ENOENT) {
 299     ldpp_dout(dpp, 0) << "Error read_id " << new_name << ": " << cpp_strerror(-ret) << dendl;
 300     return ret;
 301   }
 302   string old_name = name;
 303   name = new_name;
 304   ret = update(dpp, y);
 305   if (ret < 0) {
 306     ldpp_dout(dpp, 0) << "Error storing new obj info " << new_name << ": " << cpp_strerror(-ret) << dendl;
 307     return ret;
 308   }
 309   ret = store_name(dpp, true, y);
 310   if (ret < 0) {
 311     ldpp_dout(dpp, 0) << "Error storing new name " << new_name << ": " << cpp_strerror(-ret) << dendl;
 312     return ret;
 313   }
 314   /* delete old name */
 315   rgw_pool pool(get_pool(cct));
 316   string oid = get_names_oid_prefix() + old_name;
 317   rgw_raw_obj old_name_obj(pool, oid);
 318   auto sysobj = sysobj_svc->get_obj(old_name_obj);
 319   ret = sysobj.wop().remove(dpp, y);
 320   if (ret < 0) {
 321     ldpp_dout(dpp, 0) << "Error delete old obj name  " << old_name << ": " << cpp_strerror(-ret) << dendl;
 322     return ret;
 323   }
 324
 325   return ret;
 326 }
 327
 328 int RGWSystemMetaObj::read(const DoutPrefixProvider *dpp, optional_yield y)
 329 {
 330   int ret = read_id(dpp, name, id, y);
 331   if (ret < 0) {
 332     return ret;
 333   }
 334
 335   return read_info(dpp, id, y);
 336 }
 337
 338 int RGWZoneParams::create_default(const DoutPrefixProvider *dpp, optional_yield y, bool old_format)
 339 {
 340   name = default_zone_name;
 341
 342   int r = create(dpp, y);
 343   if (r < 0) {
 344     return r;
 345   }
 346
 347   if (old_format) {
 348     name = id;
 349   }
 350
 351   return r;
 352 }
 353
 354 const string& RGWZoneParams::get_compression_type(const rgw_placement_rule& placement_rule) const
 355 {
 356   static const std::string NONE{"none"};
 357   auto p = placement_pools.find(placement_rule.name);
 358   if (p == placement_pools.end()) {
 359     return NONE;
 360   }
 361   const auto& type = p->second.get_compression_type(placement_rule.get_storage_class());
 362   return !type.empty() ? type : NONE;
 363 }
 364
 365 // run an MD5 hash on the zone_id and return the first 32 bits
 366 static uint32_t gen_short_zone_id(const std::string zone_id)
 367 {
 368   unsigned char md5[CEPH_CRYPTO_MD5_DIGESTSIZE];
 369   MD5 hash;
 370   // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes
 371   hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
 372   hash.Update((const unsigned char *)zone_id.c_str(), zone_id.size());
 373   hash.Final(md5);
 374
 375   uint32_t short_id;
 376   memcpy((char *)&short_id, md5, sizeof(short_id));
 377   return std::max(short_id, 1u);
 378 }
 379
 380 int RGWPeriodMap::update(const RGWZoneGroup& zonegroup, CephContext *cct)
 381 {
 382   if (zonegroup.is_master_zonegroup() && (!master_zonegroup.empty() && zonegroup.get_id() != master_zonegroup)) {
 383     ldout(cct,0) << "Error updating periodmap, multiple master zonegroups configured "<< dendl;
 384     ldout(cct,0) << "master zonegroup: " << master_zonegroup << " and  " << zonegroup.get_id() <<dendl;
 385     return -EINVAL;
 386   }
 387   map<string, RGWZoneGroup>::iterator iter = zonegroups.find(zonegroup.get_id());
 388   if (iter != zonegroups.end()) {
 389     RGWZoneGroup& old_zonegroup = iter->second;
 390     if (!old_zonegroup.api_name.empty()) {
 391       zonegroups_by_api.erase(old_zonegroup.api_name);
 392     }
 393   }
 394   zonegroups[zonegroup.get_id()] = zonegroup;
 395
 396   if (!zonegroup.api_name.empty()) {
 397     zonegroups_by_api[zonegroup.api_name] = zonegroup;
 398   }
 399
 400   if (zonegroup.is_master_zonegroup()) {
 401     master_zonegroup = zonegroup.get_id();
 402   } else if (master_zonegroup == zonegroup.get_id()) {
 403     master_zonegroup = "";
 404   }
 405
 406   for (auto& i : zonegroup.zones) {
 407     auto& zone = i.second;
 408     if (short_zone_ids.find(zone.id) != short_zone_ids.end()) {
 409       continue;
 410     }
 411     // calculate the zone's short id
 412     uint32_t short_id = gen_short_zone_id(zone.id);
 413
 414     // search for an existing zone with the same short id
 415     for (auto& s : short_zone_ids) {
 416       if (s.second == short_id) {
 417         ldout(cct, 0) << "New zone '" << zone.name << "' (" << zone.id
 418             << ") generates the same short_zone_id " << short_id
 419             << " as existing zone id " << s.first << dendl;
 420         return -EEXIST;
 421       }
 422     }
 423
 424     short_zone_ids[zone.id] = short_id;
 425   }
 426
 427   return 0;
 428 }
 429
 430 uint32_t RGWPeriodMap::get_zone_short_id(const string& zone_id) const
 431 {
 432   auto i = short_zone_ids.find(zone_id);
 433   if (i == short_zone_ids.end()) {
 434     return 0;
 435   }
 436   return i->second;
 437 }
 438
 439 bool RGWPeriodMap::find_zone_by_name(const string& zone_name,
 440                                      RGWZoneGroup *zonegroup,
 441                                      RGWZone *zone) const
 442 {
 443   for (auto& iter : zonegroups) {
 444     auto& zg = iter.second;
 445     for (auto& ziter : zg.zones) {
 446       auto& z = ziter.second;
 447
 448       if (z.name == zone_name) {
 449         *zonegroup = zg;
 450         *zone = z;
 451         return true;
 452       }
 453     }
 454   }
 455
 456   return false;
 457 }
 458
 459 namespace rgw {
 460
 461 int read_realm(const DoutPrefixProvider* dpp, optional_yield y,
 462                sal::ConfigStore* cfgstore,
 463                std::string_view realm_id,
 464                std::string_view realm_name,
 465                RGWRealm& info,
 466                std::unique_ptr<sal::RealmWriter>* writer)
 467 {
 468   if (!realm_id.empty()) {
 469     return cfgstore->read_realm_by_id(dpp, y, realm_id, info, writer);
 470   }
 471   if (!realm_name.empty()) {
 472     return cfgstore->read_realm_by_name(dpp, y, realm_name, info, writer);
 473   }
 474   return cfgstore->read_default_realm(dpp, y, info, writer);
 475 }
 476
 477 int create_realm(const DoutPrefixProvider* dpp, optional_yield y,
 478                  sal::ConfigStore* cfgstore, bool exclusive,
 479                  RGWRealm& info,
 480                  std::unique_ptr<sal::RealmWriter>* writer_out)
 481 {
 482   if (info.name.empty()) {
 483     ldpp_dout(dpp, -1) << __func__ << " requires a realm name" << dendl;
 484     return -EINVAL;
 485   }
 486   if (info.id.empty()) {
 487     info.id = gen_random_uuid();
 488   }
 489
 490   // if the realm already has a current_period, just make sure it exists
 491   std::optional<RGWPeriod> period;
 492   if (!info.current_period.empty()) {
 493     period.emplace();
 494     int r = cfgstore->read_period(dpp, y, info.current_period,
 495                                   std::nullopt, *period);
 496     if (r < 0) {
 497       ldpp_dout(dpp, -1) << __func__ << " failed to read realm's current_period="
 498           << info.current_period << " with " << cpp_strerror(r) << dendl;
 499       return r;
 500     }
 501   }
 502
 503   // create the realm
 504   std::unique_ptr<sal::RealmWriter> writer;
 505   int r = cfgstore->create_realm(dpp, y, exclusive, info, &writer);
 506   if (r < 0) {
 507     return r;
 508   }
 509
 510   if (!period) {
 511     // initialize and exclusive-create the initial period
 512     period.emplace();
 513     period->id = gen_random_uuid();
 514     period->period_map.id = period->id;
 515     period->epoch = FIRST_EPOCH;
 516     period->realm_id = info.id;
 517     period->realm_name = info.name;
 518
 519     r = cfgstore->create_period(dpp, y, true, *period);
 520     if (r < 0) {
 521       ldpp_dout(dpp, -1) << __func__ << " failed to create the initial period id="
 522           << period->id << " for realm " << info.name
 523           << " with " << cpp_strerror(r) << dendl;
 524       return r;
 525     }
 526   }
 527
 528   // update the realm's current_period
 529   r = realm_set_current_period(dpp, y, cfgstore, *writer, info, *period);
 530   if (r < 0) {
 531     return r;
 532   }
 533
 534   // try to set as default. may race with another create, so pass exclusive=true
 535   // so we don't override an existing default
 536   r = set_default_realm(dpp, y, cfgstore, info, true);
 537   if (r < 0 && r != -EEXIST) {
 538     ldpp_dout(dpp, 0) << "WARNING: failed to set realm as default: "
 539         << cpp_strerror(r) << dendl;
 540   }
 541
 542   if (writer_out) {
 543     *writer_out = std::move(writer);
 544   }
 545   return 0;
 546 }
 547
 548 int set_default_realm(const DoutPrefixProvider* dpp, optional_yield y,
 549                       sal::ConfigStore* cfgstore, const RGWRealm& info,
 550                       bool exclusive)
 551 {
 552   return cfgstore->write_default_realm_id(dpp, y, exclusive, info.id);
 553 }
 554
 555 int realm_set_current_period(const DoutPrefixProvider* dpp, optional_yield y,
 556                              sal::ConfigStore* cfgstore,
 557                              sal::RealmWriter& writer, RGWRealm& realm,
 558                              const RGWPeriod& period)
 559 {
 560   // update realm epoch to match the period's
 561   if (realm.epoch > period.realm_epoch) {
 562     ldpp_dout(dpp, -1) << __func__ << " with old realm epoch "
 563         << period.realm_epoch << ", current epoch=" << realm.epoch << dendl;
 564     return -EINVAL;
 565   }
 566   if (realm.epoch == period.realm_epoch && realm.current_period != period.id) {
 567     ldpp_dout(dpp, -1) << __func__ << " with same realm epoch "
 568         << period.realm_epoch << ", but different period id "
 569         << period.id << " != " << realm.current_period << dendl;
 570     return -EINVAL;
 571   }
 572
 573   realm.epoch = period.realm_epoch;
 574   realm.current_period = period.id;
 575
 576   // update the realm object
 577   int r = writer.write(dpp, y, realm);
 578   if (r < 0) {
 579     ldpp_dout(dpp, -1) << __func__ << " failed to overwrite realm "
 580         << realm.name << " with " << cpp_strerror(r) << dendl;
 581     return r;
 582   }
 583
 584   // reflect the zonegroup and period config
 585   (void) reflect_period(dpp, y, cfgstore, period);
 586   return 0;
 587 }
 588
 589 int reflect_period(const DoutPrefixProvider* dpp, optional_yield y,
 590                    sal::ConfigStore* cfgstore, const RGWPeriod& info)
 591 {
 592   // overwrite the local period config and zonegroup objects
 593   constexpr bool exclusive = false;
 594
 595   int r = cfgstore->write_period_config(dpp, y, exclusive, info.realm_id,
 596                                         info.period_config);
 597   if (r < 0) {
 598     ldpp_dout(dpp, -1) << __func__ << " failed to store period config for realm id="
 599         << info.realm_id << " with " << cpp_strerror(r) << dendl;
 600     return r;
 601   }
 602
 603   for (auto& [zonegroup_id, zonegroup] : info.period_map.zonegroups) {
 604     r = cfgstore->create_zonegroup(dpp, y, exclusive, zonegroup, nullptr);
 605     if (r < 0) {
 606       ldpp_dout(dpp, -1) << __func__ << " failed to store zonegroup id="
 607           << zonegroup_id << " with " << cpp_strerror(r) << dendl;
 608       return r;
 609     }
 610     if (zonegroup.is_master) {
 611       // set master as default if no default exists
 612       constexpr bool exclusive = true;
 613       r = set_default_zonegroup(dpp, y, cfgstore, zonegroup, exclusive);
 614       if (r == 0) {
 615         ldpp_dout(dpp, 1) << "Set the period's master zonegroup "
 616             << zonegroup.name << " as the default" << dendl;
 617       }
 618     }
 619   }
 620   return 0;
 621 }
 622
 623 std::string get_staging_period_id(std::string_view realm_id)
 624 {
 625   return string_cat_reserve(realm_id, ":staging");
 626 }
 627
 628 void fork_period(const DoutPrefixProvider* dpp, RGWPeriod& info)
 629 {
 630   ldpp_dout(dpp, 20) << __func__ << " realm id=" << info.realm_id
 631       << " period id=" << info.id << dendl;
 632
 633   info.predecessor_uuid = std::move(info.id);
 634   info.id = get_staging_period_id(info.realm_id);
 635   info.period_map.reset();
 636   info.realm_epoch++;
 637 }
 638
 639 int update_period(const DoutPrefixProvider* dpp, optional_yield y,
 640                   sal::ConfigStore* cfgstore, RGWPeriod& info)
 641 {
 642   // clear zone short ids of removed zones. period_map.update() will add the
 643   // remaining zones back
 644   info.period_map.short_zone_ids.clear();
 645
 646   // list all zonegroups in the realm
 647   rgw::sal::ListResult<std::string> listing;
 648   std::array<std::string, 1000> zonegroup_names; // list in pages of 1000
 649   do {
 650     int ret = cfgstore->list_zonegroup_names(dpp, y, listing.next,
 651                                              zonegroup_names, listing);
 652     if (ret < 0) {
 653       std::cerr << "failed to list zonegroups: " << cpp_strerror(-ret) << std::endl;
 654       return -ret;
 655     }
 656     for (const auto& name : listing.entries) {
 657       RGWZoneGroup zg;
 658       ret = cfgstore->read_zonegroup_by_name(dpp, y, name, zg, nullptr);
 659       if (ret < 0) {
 660         ldpp_dout(dpp, 0) << "WARNING: failed to read zonegroup "
 661             << name << ": " << cpp_strerror(-ret) << dendl;
 662         continue;
 663       }
 664
 665       if (zg.realm_id != info.realm_id) {
 666         ldpp_dout(dpp, 20) << "skipping zonegroup " << zg.get_name()
 667             << " with realm id " << zg.realm_id
 668             << ", not on our realm " << info.realm_id << dendl;
 669         continue;
 670       }
 671
 672       if (zg.master_zone.empty()) {
 673         ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() << " should have a master zone " << dendl;
 674         return -EINVAL;
 675       }
 676
 677       if (zg.zones.find(zg.master_zone) == zg.zones.end()) {
 678         ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name()
 679                      << " has a non existent master zone "<< dendl;
 680         return -EINVAL;
 681       }
 682
 683       if (zg.is_master_zonegroup()) {
 684         info.master_zonegroup = zg.get_id();
 685         info.master_zone = zg.master_zone;
 686       }
 687
 688       ret = info.period_map.update(zg, dpp->get_cct());
 689       if (ret < 0) {
 690         return ret;
 691       }
 692     } // foreach name in listing.entries
 693   } while (!listing.next.empty());
 694
 695   // read the realm's current period config
 696   int ret = cfgstore->read_period_config(dpp, y, info.realm_id,
 697                                          info.period_config);
 698   if (ret < 0 && ret != -ENOENT) {
 699     ldpp_dout(dpp, 0) << "ERROR: failed to read period config: "
 700         << cpp_strerror(ret) << dendl;
 701     return ret;
 702   }
 703
 704   return 0;
 705 }
 706
 707 int commit_period(const DoutPrefixProvider* dpp, optional_yield y,
 708                   sal::ConfigStore* cfgstore, sal::Driver* driver,
 709                   RGWRealm& realm, sal::RealmWriter& realm_writer,
 710                   const RGWPeriod& current_period,
 711                   RGWPeriod& info, std::ostream& error_stream,
 712                   bool force_if_stale)
 713 {
 714   auto zone_svc = static_cast<rgw::sal::RadosStore*>(driver)->svc()->zone; // XXX
 715
 716   ldpp_dout(dpp, 20) << __func__ << " realm " << realm.id
 717       << " period " << current_period.id << dendl;
 718   // gateway must be in the master zone to commit
 719   if (info.master_zone != zone_svc->get_zone_params().id) {
 720     error_stream << "Cannot commit period on zone "
 721         << zone_svc->get_zone_params().id << ", it must be sent to "
 722         "the period's master zone " << info.master_zone << '.' << std::endl;
 723     return -EINVAL;
 724   }
 725   // period predecessor must match current period
 726   if (info.predecessor_uuid != current_period.id) {
 727     error_stream << "Period predecessor " << info.predecessor_uuid
 728         << " does not match current period " << current_period.id
 729         << ". Use 'period pull' to get the latest period from the master, "
 730         "reapply your changes, and try again." << std::endl;
 731     return -EINVAL;
 732   }
 733   // realm epoch must be 1 greater than current period
 734   if (info.realm_epoch != current_period.realm_epoch + 1) {
 735     error_stream << "Period's realm epoch " << info.realm_epoch
 736         << " does not come directly after current realm epoch "
 737         << current_period.realm_epoch << ". Use 'realm pull' to get the "
 738         "latest realm and period from the master zone, reapply your changes, "
 739         "and try again." << std::endl;
 740     return -EINVAL;
 741   }
 742   // did the master zone change?
 743   if (info.master_zone != current_period.master_zone) {
 744     // store the current metadata sync status in the period
 745     int r = info.update_sync_status(dpp, driver, current_period,
 746                                     error_stream, force_if_stale);
 747     if (r < 0) {
 748       ldpp_dout(dpp, 0) << "failed to update metadata sync status: "
 749           << cpp_strerror(-r) << dendl;
 750       return r;
 751     }
 752     // create an object with a new period id
 753     info.period_map.id = info.id = gen_random_uuid();
 754     info.epoch = FIRST_EPOCH;
 755
 756     constexpr bool exclusive = true;
 757     r = cfgstore->create_period(dpp, y, exclusive, info);
 758     if (r < 0) {
 759       ldpp_dout(dpp, 0) << "failed to create new period: " << cpp_strerror(-r) << dendl;
 760       return r;
 761     }
 762     // set as current period
 763     r = realm_set_current_period(dpp, y, cfgstore, realm_writer, realm, info);
 764     if (r < 0) {
 765       ldpp_dout(dpp, 0) << "failed to update realm's current period: "
 766           << cpp_strerror(-r) << dendl;
 767       return r;
 768     }
 769     ldpp_dout(dpp, 4) << "Promoted to master zone and committed new period "
 770         << info.id << dendl;
 771     (void) cfgstore->realm_notify_new_period(dpp, y, info);
 772     return 0;
 773   }
 774   // period must be based on current epoch
 775   if (info.epoch != current_period.epoch) {
 776     error_stream << "Period epoch " << info.epoch << " does not match "
 777         "predecessor epoch " << current_period.epoch << ". Use "
 778         "'period pull' to get the latest epoch from the master zone, "
 779         "reapply your changes, and try again." << std::endl;
 780     return -EINVAL;
 781   }
 782   // set period as next epoch
 783   info.id = current_period.id;
 784   info.epoch = current_period.epoch + 1;
 785   info.predecessor_uuid = current_period.predecessor_uuid;
 786   info.realm_epoch = current_period.realm_epoch;
 787   // write the period
 788   constexpr bool exclusive = true;
 789   int r = cfgstore->create_period(dpp, y, exclusive, info);
 790   if (r == -EEXIST) {
 791     // already have this epoch (or a more recent one)
 792     return 0;
 793   }
 794   if (r < 0) {
 795     ldpp_dout(dpp, 0) << "failed to store period: " << cpp_strerror(r) << dendl;
 796     return r;
 797   }
 798   r = reflect_period(dpp, y, cfgstore, info);
 799   if (r < 0) {
 800     ldpp_dout(dpp, 0) << "failed to update local objects: " << cpp_strerror(r) << dendl;
 801     return r;
 802   }
 803   ldpp_dout(dpp, 4) << "Committed new epoch " << info.epoch
 804       << " for period " << info.id << dendl;
 805   (void) cfgstore->realm_notify_new_period(dpp, y, info);
 806   return 0;
 807 }
 808
 809
 810 int read_zonegroup(const DoutPrefixProvider* dpp, optional_yield y,
 811                    sal::ConfigStore* cfgstore,
 812                    std::string_view zonegroup_id,
 813                    std::string_view zonegroup_name,
 814                    RGWZoneGroup& info,
 815                    std::unique_ptr<sal::ZoneGroupWriter>* writer)
 816 {
 817   if (!zonegroup_id.empty()) {
 818     return cfgstore->read_zonegroup_by_id(dpp, y, zonegroup_id, info, writer);
 819   }
 820   if (!zonegroup_name.empty()) {
 821     return cfgstore->read_zonegroup_by_name(dpp, y, zonegroup_name, info, writer);
 822   }
 823
 824   std::string realm_id;
 825   int r = cfgstore->read_default_realm_id(dpp, y, realm_id);
 826   if (r == -ENOENT) {
 827     return cfgstore->read_zonegroup_by_name(dpp, y, default_zonegroup_name,
 828                                             info, writer);
 829   }
 830   if (r < 0) {
 831     return r;
 832   }
 833   return cfgstore->read_default_zonegroup(dpp, y, realm_id, info, writer);
 834 }
 835
 836 int create_zonegroup(const DoutPrefixProvider* dpp, optional_yield y,
 837                      sal::ConfigStore* cfgstore, bool exclusive,
 838                      RGWZoneGroup& info)
 839 {
 840   if (info.name.empty()) {
 841     ldpp_dout(dpp, -1) << __func__ << " requires a zonegroup name" << dendl;
 842     return -EINVAL;
 843   }
 844   if (info.id.empty()) {
 845     info.id = gen_random_uuid();
 846   }
 847
 848   // insert the default placement target if it doesn't exist
 849   constexpr std::string_view default_placement_name = "default-placement";
 850
 851   RGWZoneGroupPlacementTarget placement_target;
 852   placement_target.name = default_placement_name;
 853
 854   info.placement_targets.emplace(default_placement_name, placement_target);
 855   if (info.default_placement.name.empty()) {
 856     info.default_placement.name = default_placement_name;
 857   }
 858
 859   int r = cfgstore->create_zonegroup(dpp, y, exclusive, info, nullptr);
 860   if (r < 0) {
 861     ldpp_dout(dpp, 0) << "failed to create zonegroup with "
 862         << cpp_strerror(r) << dendl;
 863     return r;
 864   }
 865
 866   // try to set as default. may race with another create, so pass exclusive=true
 867   // so we don't override an existing default
 868   r = set_default_zonegroup(dpp, y, cfgstore, info, true);
 869   if (r < 0 && r != -EEXIST) {
 870     ldpp_dout(dpp, 0) << "WARNING: failed to set zonegroup as default: "
 871         << cpp_strerror(r) << dendl;
 872   }
 873
 874   return 0;
 875 }
 876
 877 int set_default_zonegroup(const DoutPrefixProvider* dpp, optional_yield y,
 878                           sal::ConfigStore* cfgstore, const RGWZoneGroup& info,
 879                           bool exclusive)
 880 {
 881   return cfgstore->write_default_zonegroup_id(
 882       dpp, y, exclusive, info.realm_id, info.id);
 883 }
 884
 885 int remove_zone_from_group(const DoutPrefixProvider* dpp,
 886                            RGWZoneGroup& zonegroup,
 887                            const rgw_zone_id& zone_id)
 888 {
 889   auto z = zonegroup.zones.find(zone_id);
 890   if (z == zonegroup.zones.end()) {
 891     return -ENOENT;
 892   }
 893   zonegroup.zones.erase(z);
 894
 895   if (zonegroup.master_zone == zone_id) {
 896     // choose a new master zone
 897     auto m = zonegroup.zones.begin();
 898     if (m != zonegroup.zones.end()) {
 899       zonegroup.master_zone = m->first;
 900       ldpp_dout(dpp, 0) << "NOTICE: promoted " << m->second.name
 901          << " as new master_zone of zonegroup " << zonegroup.name << dendl;
 902     } else {
 903       ldpp_dout(dpp, 0) << "NOTICE: removed master_zone of zonegroup "
 904           << zonegroup.name << dendl;
 905     }
 906   }
 907
 908   const bool log_data = zonegroup.zones.size() > 1;
 909   for (auto& [id, zone] : zonegroup.zones) {
 910     zone.log_data = log_data;
 911   }
 912
 913   return 0;
 914 }
 915
 916 // try to remove the given zone id from every zonegroup in the cluster
 917 static int remove_zone_from_groups(const DoutPrefixProvider* dpp,
 918                                    optional_yield y,
 919                                    sal::ConfigStore* cfgstore,
 920                                    const rgw_zone_id& zone_id)
 921 {
 922   std::array<std::string, 128> zonegroup_names;
 923   sal::ListResult<std::string> listing;
 924   do {
 925     int r = cfgstore->list_zonegroup_names(dpp, y, listing.next,
 926                                            zonegroup_names, listing);
 927     if (r < 0) {
 928       ldpp_dout(dpp, 0) << "failed to list zonegroups with "
 929           << cpp_strerror(r) << dendl;
 930       return r;
 931     }
 932
 933     for (const auto& name : listing.entries) {
 934       RGWZoneGroup zonegroup;
 935       std::unique_ptr<sal::ZoneGroupWriter> writer;
 936       r = cfgstore->read_zonegroup_by_name(dpp, y, name, zonegroup, &writer);
 937       if (r < 0) {
 938         ldpp_dout(dpp, 0) << "WARNING: failed to load zonegroup " << name
 939             << " with " << cpp_strerror(r) << dendl;
 940         continue;
 941       }
 942
 943       r = remove_zone_from_group(dpp, zonegroup, zone_id);
 944       if (r < 0) {
 945         continue;
 946       }
 947
 948       // write the updated zonegroup
 949       r = writer->write(dpp, y, zonegroup);
 950       if (r < 0) {
 951         ldpp_dout(dpp, 0) << "WARNING: failed to write zonegroup " << name
 952             << " with " << cpp_strerror(r) << dendl;
 953         continue;
 954       }
 955       ldpp_dout(dpp, 0) << "Removed zone from zonegroup " << name << dendl;
 956     }
 957   } while (!listing.next.empty());
 958
 959   return 0;
 960 }
 961
 962
 963 int read_zone(const DoutPrefixProvider* dpp, optional_yield y,
 964               sal::ConfigStore* cfgstore,
 965               std::string_view zone_id,
 966               std::string_view zone_name,
 967               RGWZoneParams& info,
 968               std::unique_ptr<sal::ZoneWriter>* writer)
 969 {
 970   if (!zone_id.empty()) {
 971     return cfgstore->read_zone_by_id(dpp, y, zone_id, info, writer);
 972   }
 973   if (!zone_name.empty()) {
 974     return cfgstore->read_zone_by_name(dpp, y, zone_name, info, writer);
 975   }
 976
 977   std::string realm_id;
 978   int r = cfgstore->read_default_realm_id(dpp, y, realm_id);
 979   if (r == -ENOENT) {
 980     return cfgstore->read_zone_by_name(dpp, y, default_zone_name, info, writer);
 981   }
 982   if (r < 0) {
 983     return r;
 984   }
 985   return cfgstore->read_default_zone(dpp, y, realm_id, info, writer);
 986 }
 987
 988 extern int get_zones_pool_set(const DoutPrefixProvider *dpp, optional_yield y,
 989                               rgw::sal::ConfigStore* cfgstore,
 990                               std::string_view my_zone_id,
 991                               std::set<rgw_pool>& pools);
 992
 993 int create_zone(const DoutPrefixProvider* dpp, optional_yield y,
 994                 sal::ConfigStore* cfgstore, bool exclusive,
 995                 RGWZoneParams& info, std::unique_ptr<sal::ZoneWriter>* writer)
 996 {
 997   if (info.name.empty()) {
 998     ldpp_dout(dpp, -1) << __func__ << " requires a zone name" << dendl;
 999     return -EINVAL;
1000   }
1001   if (info.id.empty()) {
1002     info.id = gen_random_uuid();
1003   }
1004
1005   // add default placement with empty pool name
1006   rgw_pool pool;
1007   auto& placement = info.placement_pools["default-placement"];
1008   placement.storage_classes.set_storage_class(
1009       RGW_STORAGE_CLASS_STANDARD, &pool, nullptr);
1010
1011   // build a set of all pool names used by other zones
1012   std::set<rgw_pool> pools;
1013   int r = get_zones_pool_set(dpp, y, cfgstore, info.id, pools);
1014   if (r < 0) {
1015     return r;
1016   }
1017
1018   // initialize pool names with the zone name prefix
1019   r = init_zone_pool_names(dpp, y, pools, info);
1020   if (r < 0) {
1021     return r;
1022   }
1023
1024   r = cfgstore->create_zone(dpp, y, exclusive, info, nullptr);
1025   if (r < 0) {
1026     ldpp_dout(dpp, 0) << "failed to create zone with "
1027         << cpp_strerror(r) << dendl;
1028     return r;
1029   }
1030
1031   // try to set as default. may race with another create, so pass exclusive=true
1032   // so we don't override an existing default
1033   r = set_default_zone(dpp, y, cfgstore, info, true);
1034   if (r < 0 && r != -EEXIST) {
1035     ldpp_dout(dpp, 0) << "WARNING: failed to set zone as default: "
1036         << cpp_strerror(r) << dendl;
1037   }
1038
1039   return 0;
1040
1041 }
1042
1043 int set_default_zone(const DoutPrefixProvider* dpp, optional_yield y,
1044                      sal::ConfigStore* cfgstore, const RGWZoneParams& info,
1045                      bool exclusive)
1046 {
1047   return cfgstore->write_default_zone_id(
1048       dpp, y, exclusive, info.realm_id, info.id);
1049 }
1050
1051 int delete_zone(const DoutPrefixProvider* dpp, optional_yield y,
1052                 sal::ConfigStore* cfgstore, const RGWZoneParams& info,
1053                 sal::ZoneWriter& writer)
1054 {
1055   // remove this zone from any zonegroups that contain it
1056   int r = remove_zone_from_groups(dpp, y, cfgstore, info.id);
1057   if (r < 0) {
1058     return r;
1059   }
1060
1061   return writer.remove(dpp, y);
1062 }
1063
1064 } // namespace rgw
1065
1066 static inline int conf_to_uint64(const JSONFormattable& config, const string& key, uint64_t *pval)
1067 {
1068   string sval;
1069   if (config.find(key, &sval)) {
1070     string err;
1071     uint64_t val = strict_strtoll(sval.c_str(), 10, &err);
1072     if (!err.empty()) {
1073       return -EINVAL;
1074     }
1075     *pval = val;
1076   }
1077   return 0;
1078 }
1079
1080 int RGWZoneGroupPlacementTier::update_params(const JSONFormattable& config)
1081 {
1082   int r = -1;
1083
1084   if (config.exists("retain_head_object")) {
1085     string s = config["retain_head_object"];
1086     if (s == "true") {
1087       retain_head_object = true;
1088     } else {
1089       retain_head_object = false;
1090     }
1091   }
1092
1093   if (tier_type == "cloud-s3") {
1094     r = t.s3.update_params(config);
1095   }
1096
1097   return r;
1098 }
1099
1100 int RGWZoneGroupPlacementTier::clear_params(const JSONFormattable& config)
1101 {
1102   if (config.exists("retain_head_object")) {
1103     retain_head_object = false;
1104   }
1105
1106   if (tier_type == "cloud-s3") {
1107     t.s3.clear_params(config);
1108   }
1109
1110   return 0;
1111 }
1112
1113 int RGWZoneGroupPlacementTierS3::update_params(const JSONFormattable& config)
1114 {
1115   int r = -1;
1116
1117   if (config.exists("endpoint")) {
1118     endpoint = config["endpoint"];
1119   }
1120   if (config.exists("target_path")) {
1121     target_path = config["target_path"];
1122   }
1123   if (config.exists("region")) {
1124     region = config["region"];
1125   }
1126   if (config.exists("host_style")) {
1127     string s;
1128     s = config["host_style"];
1129     if (s != "virtual") {
1130       host_style = PathStyle;
1131     } else {
1132       host_style = VirtualStyle;
1133     }
1134   }
1135   if (config.exists("target_storage_class")) {
1136     target_storage_class = config["target_storage_class"];
1137   }
1138   if (config.exists("access_key")) {
1139     key.id = config["access_key"];
1140   }
1141   if (config.exists("secret")) {
1142     key.key = config["secret"];
1143   }
1144   if (config.exists("multipart_sync_threshold")) {
1145     r = conf_to_uint64(config, "multipart_sync_threshold", &multipart_sync_threshold);
1146     if (r < 0) {
1147       multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE;
1148     }
1149   }
1150
1151   if (config.exists("multipart_min_part_size")) {
1152     r = conf_to_uint64(config, "multipart_min_part_size", &multipart_min_part_size);
1153     if (r < 0) {
1154       multipart_min_part_size = DEFAULT_MULTIPART_SYNC_PART_SIZE;
1155     }
1156   }
1157
1158   if (config.exists("acls")) {
1159     const JSONFormattable& cc = config["acls"];
1160     if (cc.is_array()) {
1161       for (auto& c : cc.array()) {
1162         RGWTierACLMapping m;
1163         m.init(c);
1164         if (!m.source_id.empty()) {
1165           acl_mappings[m.source_id] = m;
1166         }
1167       }
1168     } else {
1169       RGWTierACLMapping m;
1170       m.init(cc);
1171       if (!m.source_id.empty()) {
1172         acl_mappings[m.source_id] = m;
1173       }
1174     }
1175   }
1176   return 0;
1177 }
1178
1179 int RGWZoneGroupPlacementTierS3::clear_params(const JSONFormattable& config)
1180 {
1181   if (config.exists("endpoint")) {
1182     endpoint.clear();
1183   }
1184   if (config.exists("target_path")) {
1185     target_path.clear();
1186   }
1187   if (config.exists("region")) {
1188     region.clear();
1189   }
1190   if (config.exists("host_style")) {
1191     /* default */
1192     host_style = PathStyle;
1193   }
1194   if (config.exists("target_storage_class")) {
1195     target_storage_class.clear();
1196   }
1197   if (config.exists("access_key")) {
1198     key.id.clear();
1199   }
1200   if (config.exists("secret")) {
1201     key.key.clear();
1202   }
1203   if (config.exists("multipart_sync_threshold")) {
1204     multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE;
1205   }
1206   if (config.exists("multipart_min_part_size")) {
1207     multipart_min_part_size = DEFAULT_MULTIPART_SYNC_PART_SIZE;
1208   }
1209   if (config.exists("acls")) {
1210     const JSONFormattable& cc = config["acls"];
1211     if (cc.is_array()) {
1212       for (auto& c : cc.array()) {
1213         RGWTierACLMapping m;
1214         m.init(c);
1215         acl_mappings.erase(m.source_id);
1216       }
1217     } else {
1218       RGWTierACLMapping m;
1219       m.init(cc);
1220       acl_mappings.erase(m.source_id);
1221     }
1222   }
1223   return 0;
1224 }
1225
1226 void rgw_meta_sync_info::generate_test_instances(list<rgw_meta_sync_info*>& o)
1227 {
1228   auto info = new rgw_meta_sync_info;
1229   info->state = rgw_meta_sync_info::StateBuildingFullSyncMaps;
1230   info->period = "periodid";
1231   info->realm_epoch = 5;
1232   o.push_back(info);
1233   o.push_back(new rgw_meta_sync_info);
1234 }
1235
1236 void rgw_meta_sync_marker::generate_test_instances(list<rgw_meta_sync_marker*>& o)
1237 {
1238   auto marker = new rgw_meta_sync_marker;
1239   marker->state = rgw_meta_sync_marker::IncrementalSync;
1240   marker->marker = "01234";
1241   marker->realm_epoch = 5;
1242   o.push_back(marker);
1243   o.push_back(new rgw_meta_sync_marker);
1244 }
1245
1246 void rgw_meta_sync_status::generate_test_instances(list<rgw_meta_sync_status*>& o)
1247 {
1248   o.push_back(new rgw_meta_sync_status);
1249 }
1250
1251 void RGWZoneParams::generate_test_instances(list<RGWZoneParams*> &o)
1252 {
1253   o.push_back(new RGWZoneParams);
1254   o.push_back(new RGWZoneParams);
1255 }
1256
1257 void RGWPeriodLatestEpochInfo::generate_test_instances(list<RGWPeriodLatestEpochInfo*> &o)
1258 {
1259   RGWPeriodLatestEpochInfo *z = new RGWPeriodLatestEpochInfo;
1260   o.push_back(z);
1261   o.push_back(new RGWPeriodLatestEpochInfo);
1262 }
1263
1264 void RGWZoneGroup::generate_test_instances(list<RGWZoneGroup*>& o)
1265 {
1266   RGWZoneGroup *r = new RGWZoneGroup;
1267   o.push_back(r);
1268   o.push_back(new RGWZoneGroup);
1269 }
1270
1271 void RGWPeriodLatestEpochInfo::dump(Formatter *f) const {
1272   encode_json("latest_epoch", epoch, f);
1273 }
1274
1275 void RGWPeriodLatestEpochInfo::decode_json(JSONObj *obj) {
1276   JSONDecoder::decode_json("latest_epoch", epoch, obj);
1277 }
1278
1279 void RGWNameToId::dump(Formatter *f) const {
1280   encode_json("obj_id", obj_id, f);
1281 }
1282
1283 void RGWNameToId::decode_json(JSONObj *obj) {
1284   JSONDecoder::decode_json("obj_id", obj_id, obj);
1285 }
1286