ceph/src/rgw/services/svc_bi_rados.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #include "svc_bi_rados.h"
   5 #include "svc_bilog_rados.h"
   6 #include "svc_zone.h"
   7
   8 #include "rgw/rgw_bucket.h"
   9 #include "rgw/rgw_zone.h"
  10 #include "rgw/rgw_datalog.h"
  11
  12 #include "cls/rgw/cls_rgw_client.h"
  13
  14 #define dout_subsys ceph_subsys_rgw
  15
  16 static string dir_oid_prefix = ".dir.";
  17
  18 RGWSI_BucketIndex_RADOS::RGWSI_BucketIndex_RADOS(CephContext *cct) : RGWSI_BucketIndex(cct)
  19 {
  20 }
  21
  22 void RGWSI_BucketIndex_RADOS::init(RGWSI_Zone *zone_svc,
  23                                    RGWSI_RADOS *rados_svc,
  24                                    RGWSI_BILog_RADOS *bilog_svc,
  25                                    RGWDataChangesLog *datalog_rados_svc)
  26 {
  27   svc.zone = zone_svc;
  28   svc.rados = rados_svc;
  29   svc.bilog = bilog_svc;
  30   svc.datalog_rados = datalog_rados_svc;
  31 }
  32
  33 int RGWSI_BucketIndex_RADOS::open_pool(const rgw_pool& pool,
  34                                        RGWSI_RADOS::Pool *index_pool,
  35                                        bool mostly_omap)
  36 {
  37   *index_pool = svc.rados->pool(pool);
  38   return index_pool->open(RGWSI_RADOS::OpenParams()
  39                           .set_mostly_omap(mostly_omap));
  40 }
  41
  42 int RGWSI_BucketIndex_RADOS::open_bucket_index_pool(const RGWBucketInfo& bucket_info,
  43                                                     RGWSI_RADOS::Pool *index_pool)
  44 {
  45   const rgw_pool& explicit_pool = bucket_info.bucket.explicit_placement.index_pool;
  46
  47   if (!explicit_pool.empty()) {
  48     return open_pool(explicit_pool, index_pool, false);
  49   }
  50
  51   auto& zonegroup = svc.zone->get_zonegroup();
  52   auto& zone_params = svc.zone->get_zone_params();
  53
  54   const rgw_placement_rule *rule = &bucket_info.placement_rule;
  55   if (rule->empty()) {
  56     rule = &zonegroup.default_placement;
  57   }
  58   auto iter = zone_params.placement_pools.find(rule->name);
  59   if (iter == zone_params.placement_pools.end()) {
  60     ldout(cct, 0) << "could not find placement rule " << *rule << " within zonegroup " << dendl;
  61     return -EINVAL;
  62   }
  63
  64   int r = open_pool(iter->second.index_pool, index_pool, true);
  65   if (r < 0)
  66     return r;
  67
  68   return 0;
  69 }
  70
  71 int RGWSI_BucketIndex_RADOS::open_bucket_index_base(const RGWBucketInfo& bucket_info,
  72                                                     RGWSI_RADOS::Pool *index_pool,
  73                                                     string *bucket_oid_base)
  74 {
  75   const rgw_bucket& bucket = bucket_info.bucket;
  76   int r = open_bucket_index_pool(bucket_info, index_pool);
  77   if (r < 0)
  78     return r;
  79
  80   if (bucket.bucket_id.empty()) {
  81     ldout(cct, 0) << "ERROR: empty bucket_id for bucket operation" << dendl;
  82     return -EIO;
  83   }
  84
  85   *bucket_oid_base = dir_oid_prefix;
  86   bucket_oid_base->append(bucket.bucket_id);
  87
  88   return 0;
  89
  90 }
  91
  92 int RGWSI_BucketIndex_RADOS::open_bucket_index(const RGWBucketInfo& bucket_info,
  93                                                RGWSI_RADOS::Pool *index_pool,
  94                                                string *bucket_oid)
  95 {
  96   const rgw_bucket& bucket = bucket_info.bucket;
  97   int r = open_bucket_index_pool(bucket_info, index_pool);
  98   if (r < 0) {
  99     ldout(cct, 20) << __func__ << ": open_bucket_index_pool() returned "
 100                    << r << dendl;
 101     return r;
 102   }
 103
 104   if (bucket.bucket_id.empty()) {
 105     ldout(cct, 0) << "ERROR: empty bucket id for bucket operation" << dendl;
 106     return -EIO;
 107   }
 108
 109   *bucket_oid = dir_oid_prefix;
 110   bucket_oid->append(bucket.bucket_id);
 111
 112   return 0;
 113 }
 114
 115 static void get_bucket_index_objects(const string& bucket_oid_base,
 116                                      uint32_t num_shards,
 117                                      map<int, string> *_bucket_objects,
 118                                      int shard_id = -1)
 119 {
 120   auto& bucket_objects = *_bucket_objects;
 121   if (!num_shards) {
 122     bucket_objects[0] = bucket_oid_base;
 123   } else {
 124     char buf[bucket_oid_base.size() + 32];
 125     if (shard_id < 0) {
 126       for (uint32_t i = 0; i < num_shards; ++i) {
 127         snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), i);
 128         bucket_objects[i] = buf;
 129       }
 130     } else {
 131       if ((uint32_t)shard_id > num_shards) {
 132         return;
 133       }
 134       snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), shard_id);
 135       bucket_objects[shard_id] = buf;
 136     }
 137   }
 138 }
 139
 140 static void get_bucket_instance_ids(const RGWBucketInfo& bucket_info,
 141                                     int shard_id,
 142                                     map<int, string> *result)
 143 {
 144   const rgw_bucket& bucket = bucket_info.bucket;
 145   string plain_id = bucket.name + ":" + bucket.bucket_id;
 146
 147   if (!bucket_info.layout.current_index.layout.normal.num_shards) {
 148     (*result)[0] = plain_id;
 149   } else {
 150     char buf[16];
 151     if (shard_id < 0) {
 152       for (uint32_t i = 0; i < bucket_info.layout.current_index.layout.normal.num_shards; ++i) {
 153         snprintf(buf, sizeof(buf), ":%d", i);
 154         (*result)[i] = plain_id + buf;
 155       }
 156     } else {
 157       if ((uint32_t)shard_id > bucket_info.layout.current_index.layout.normal.num_shards) {
 158         return;
 159       }
 160       snprintf(buf, sizeof(buf), ":%d", shard_id);
 161       (*result)[shard_id] = plain_id + buf;
 162     }
 163   }
 164 }
 165
 166 int RGWSI_BucketIndex_RADOS::open_bucket_index(const RGWBucketInfo& bucket_info,
 167                                                std::optional<int> _shard_id,
 168                                                RGWSI_RADOS::Pool *index_pool,
 169                                                map<int, string> *bucket_objs,
 170                                                map<int, string> *bucket_instance_ids)
 171 {
 172   int shard_id = _shard_id.value_or(-1);
 173   string bucket_oid_base;
 174   int ret = open_bucket_index_base(bucket_info, index_pool, &bucket_oid_base);
 175   if (ret < 0) {
 176     ldout(cct, 20) << __func__ << ": open_bucket_index_pool() returned "
 177                    << ret << dendl;
 178     return ret;
 179   }
 180
 181   get_bucket_index_objects(bucket_oid_base, bucket_info.layout.current_index.layout.normal.num_shards, bucket_objs, shard_id);
 182   if (bucket_instance_ids) {
 183     get_bucket_instance_ids(bucket_info, shard_id, bucket_instance_ids);
 184   }
 185   return 0;
 186 }
 187
 188 void RGWSI_BucketIndex_RADOS::get_bucket_index_object(const string& bucket_oid_base,
 189                                                       uint32_t num_shards,
 190                                                       int shard_id,
 191                                                       uint64_t gen_id,
 192                                                       string *bucket_obj)
 193 {
 194   if (!num_shards) {
 195     // By default with no sharding, we use the bucket oid as itself
 196     (*bucket_obj) = bucket_oid_base;
 197   } else {
 198     char buf[bucket_oid_base.size() + 64];
 199     if (gen_id != 0) {
 200       snprintf(buf, sizeof(buf), "%s.%" PRIu64 ".%d", bucket_oid_base.c_str(), gen_id, shard_id);
 201       (*bucket_obj) = buf;
 202     } else {
 203       // for backward compatibility, gen_id(0) will not be added in the object name
 204       snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), shard_id);
 205       (*bucket_obj) = buf;
 206     }
 207   }
 208 }
 209
 210 int RGWSI_BucketIndex_RADOS::get_bucket_index_object(const string& bucket_oid_base, const string& obj_key,
 211                                                      uint32_t num_shards, rgw::BucketHashType hash_type,
 212                                                      string *bucket_obj, int *shard_id)
 213 {
 214   int r = 0;
 215   switch (hash_type) {
 216     case rgw::BucketHashType::Mod:
 217       if (!num_shards) {
 218         // By default with no sharding, we use the bucket oid as itself
 219         (*bucket_obj) = bucket_oid_base;
 220         if (shard_id) {
 221           *shard_id = -1;
 222         }
 223       } else {
 224         uint32_t sid = bucket_shard_index(obj_key, num_shards);
 225         char buf[bucket_oid_base.size() + 32];
 226         snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), sid);
 227         (*bucket_obj) = buf;
 228         if (shard_id) {
 229           *shard_id = (int)sid;
 230         }
 231       }
 232       break;
 233     default:
 234       r = -ENOTSUP;
 235   }
 236   return r;
 237 }
 238
 239 int RGWSI_BucketIndex_RADOS::open_bucket_index_shard(const RGWBucketInfo& bucket_info,
 240                                                      const string& obj_key,
 241                                                      RGWSI_RADOS::Obj *bucket_obj,
 242                                                      int *shard_id)
 243 {
 244   string bucket_oid_base;
 245
 246   RGWSI_RADOS::Pool pool;
 247
 248   int ret = open_bucket_index_base(bucket_info, &pool, &bucket_oid_base);
 249   if (ret < 0) {
 250     ldout(cct, 20) << __func__ << ": open_bucket_index_pool() returned "
 251                    << ret << dendl;
 252     return ret;
 253   }
 254
 255   string oid;
 256
 257   ret = get_bucket_index_object(bucket_oid_base, obj_key, bucket_info.layout.current_index.layout.normal.num_shards,
 258         bucket_info.layout.current_index.layout.normal.hash_type, &oid, shard_id);
 259   if (ret < 0) {
 260     ldout(cct, 10) << "get_bucket_index_object() returned ret=" << ret << dendl;
 261     return ret;
 262   }
 263
 264   *bucket_obj = svc.rados->obj(pool, oid);
 265
 266   return 0;
 267 }
 268
 269 int RGWSI_BucketIndex_RADOS::open_bucket_index_shard(const RGWBucketInfo& bucket_info,
 270                                                      int shard_id,
 271                                                      const rgw::bucket_index_layout_generation& idx_layout,
 272                                                      RGWSI_RADOS::Obj *bucket_obj)
 273 {
 274   RGWSI_RADOS::Pool index_pool;
 275   string bucket_oid_base;
 276   int ret = open_bucket_index_base(bucket_info, &index_pool, &bucket_oid_base);
 277   if (ret < 0) {
 278     ldout(cct, 20) << __func__ << ": open_bucket_index_pool() returned "
 279                    << ret << dendl;
 280     return ret;
 281   }
 282
 283   string oid;
 284
 285   get_bucket_index_object(bucket_oid_base, idx_layout.layout.normal.num_shards,
 286                           shard_id, idx_layout.gen, &oid);
 287
 288   *bucket_obj = svc.rados->obj(index_pool, oid);
 289
 290   return 0;
 291 }
 292
 293 int RGWSI_BucketIndex_RADOS::cls_bucket_head(const RGWBucketInfo& bucket_info,
 294                                              int shard_id,
 295                                              vector<rgw_bucket_dir_header> *headers,
 296                                              map<int, string> *bucket_instance_ids,
 297                                              optional_yield y)
 298 {
 299   RGWSI_RADOS::Pool index_pool;
 300   map<int, string> oids;
 301   int r = open_bucket_index(bucket_info, shard_id, &index_pool, &oids, bucket_instance_ids);
 302   if (r < 0)
 303     return r;
 304
 305   map<int, struct rgw_cls_list_ret> list_results;
 306   for (auto& iter : oids) {
 307     list_results.emplace(iter.first, rgw_cls_list_ret());
 308   }
 309
 310   r = CLSRGWIssueGetDirHeader(index_pool.ioctx(), oids, list_results, cct->_conf->rgw_bucket_index_max_aio)();
 311   if (r < 0)
 312     return r;
 313
 314   map<int, struct rgw_cls_list_ret>::iterator iter = list_results.begin();
 315   for(; iter != list_results.end(); ++iter) {
 316     headers->push_back(std::move(iter->second.dir.header));
 317   }
 318   return 0;
 319 }
 320
 321
 322 int RGWSI_BucketIndex_RADOS::init_index(RGWBucketInfo& bucket_info)
 323 {
 324   RGWSI_RADOS::Pool index_pool;
 325
 326   string dir_oid = dir_oid_prefix;
 327   int r = open_bucket_index_pool(bucket_info, &index_pool);
 328   if (r < 0) {
 329     return r;
 330   }
 331
 332   dir_oid.append(bucket_info.bucket.bucket_id);
 333
 334   map<int, string> bucket_objs;
 335   get_bucket_index_objects(dir_oid, bucket_info.layout.current_index.layout.normal.num_shards, &bucket_objs);
 336
 337   return CLSRGWIssueBucketIndexInit(index_pool.ioctx(),
 338                                     bucket_objs,
 339                                     cct->_conf->rgw_bucket_index_max_aio)();
 340 }
 341
 342 int RGWSI_BucketIndex_RADOS::clean_index(RGWBucketInfo& bucket_info)
 343 {
 344   RGWSI_RADOS::Pool index_pool;
 345
 346   std::string dir_oid = dir_oid_prefix;
 347   int r = open_bucket_index_pool(bucket_info, &index_pool);
 348   if (r < 0) {
 349     return r;
 350   }
 351
 352   dir_oid.append(bucket_info.bucket.bucket_id);
 353
 354   std::map<int, std::string> bucket_objs;
 355   get_bucket_index_objects(dir_oid, bucket_info.layout.current_index.layout.normal.num_shards, &bucket_objs);
 356
 357   return CLSRGWIssueBucketIndexClean(index_pool.ioctx(),
 358                                      bucket_objs,
 359                                      cct->_conf->rgw_bucket_index_max_aio)();
 360 }
 361
 362 int RGWSI_BucketIndex_RADOS::read_stats(const RGWBucketInfo& bucket_info,
 363                                         RGWBucketEnt *result,
 364                                         optional_yield y)
 365 {
 366   vector<rgw_bucket_dir_header> headers;
 367
 368   result->bucket = bucket_info.bucket;
 369   int r = cls_bucket_head(bucket_info, RGW_NO_SHARD, &headers, nullptr, y);
 370   if (r < 0) {
 371     return r;
 372   }
 373
 374   auto hiter = headers.begin();
 375   for (; hiter != headers.end(); ++hiter) {
 376     RGWObjCategory category = RGWObjCategory::Main;
 377     auto iter = (hiter->stats).find(category);
 378     if (iter != hiter->stats.end()) {
 379       struct rgw_bucket_category_stats& stats = iter->second;
 380       result->count += stats.num_entries;
 381       result->size += stats.total_size;
 382       result->size_rounded += stats.total_size_rounded;
 383     }
 384   }
 385
 386   result->placement_rule = std::move(bucket_info.placement_rule);
 387
 388   return 0;
 389 }
 390
 391 int RGWSI_BucketIndex_RADOS::get_reshard_status(const RGWBucketInfo& bucket_info, list<cls_rgw_bucket_instance_entry> *status)
 392 {
 393   map<int, string> bucket_objs;
 394
 395   RGWSI_RADOS::Pool index_pool;
 396
 397   int r = open_bucket_index(bucket_info,
 398                             std::nullopt,
 399                             &index_pool,
 400                             &bucket_objs,
 401                             nullptr);
 402   if (r < 0) {
 403     return r;
 404   }
 405
 406   for (auto i : bucket_objs) {
 407     cls_rgw_bucket_instance_entry entry;
 408
 409     int ret = cls_rgw_get_bucket_resharding(index_pool.ioctx(), i.second, &entry);
 410     if (ret < 0 && ret != -ENOENT) {
 411       lderr(cct) << "ERROR: " << __func__ << ": cls_rgw_get_bucket_resharding() returned ret=" << ret << dendl;
 412       return ret;
 413     }
 414
 415     status->push_back(entry);
 416   }
 417
 418   return 0;
 419 }
 420
 421 int RGWSI_BucketIndex_RADOS::handle_overwrite(const RGWBucketInfo& info,
 422                                               const RGWBucketInfo& orig_info)
 423 {
 424   bool new_sync_enabled = info.datasync_flag_enabled();
 425   bool old_sync_enabled = orig_info.datasync_flag_enabled();
 426
 427   if (old_sync_enabled != new_sync_enabled) {
 428     int shards_num = info.layout.current_index.layout.normal.num_shards? info.layout.current_index.layout.normal.num_shards : 1;
 429     int shard_id = info.layout.current_index.layout.normal.num_shards? 0 : -1;
 430
 431     int ret;
 432     if (!new_sync_enabled) {
 433       ret = svc.bilog->log_stop(info, -1);
 434     } else {
 435       ret = svc.bilog->log_start(info, -1);
 436     }
 437     if (ret < 0) {
 438       lderr(cct) << "ERROR: failed writing bilog (bucket=" << info.bucket << "); ret=" << ret << dendl;
 439       return ret;
 440     }
 441
 442     for (int i = 0; i < shards_num; ++i, ++shard_id) {
 443       ret = svc.datalog_rados->add_entry(info, shard_id);
 444       if (ret < 0) {
 445         lderr(cct) << "ERROR: failed writing data log (info.bucket=" << info.bucket << ", shard_id=" << shard_id << ")" << dendl;
 446         return ret;
 447       }
 448     }
 449   }
 450
 451   return 0;
 452 }
 453