#include <boost/optional.hpp>
#include "auth/Crypto.h"
+#include "compressor/Compressor.h"
#include "common/armor.h"
#include "common/ceph_json.h"
#include "rgw_replica_log.h"
#include "rgw_orphan.h"
#include "rgw_sync.h"
+#include "rgw_sync_log_trim.h"
#include "rgw_data_sync.h"
#include "rgw_rest_conn.h"
#include "rgw_realm_watcher.h"
#include "rgw_role.h"
#include "rgw_reshard.h"
+#include "rgw_http_client_curl.h"
using namespace std;
static RGWRados *store = NULL;
-void _usage()
+void usage()
{
cout << "usage: radosgw-admin <cmd> [options...]" << std::endl;
cout << "commands:\n";
cout << " bucket rm remove bucket\n";
cout << " bucket check check bucket index\n";
cout << " bucket reshard reshard bucket\n";
+ cout << " bucket sync disable disable bucket sync\n";
+ cout << " bucket sync enable enable bucket sync\n";
cout << " bi get retrieve bucket index object entries\n";
cout << " bi put store bucket index object entries\n";
cout << " bi list list raw bucket index entries\n";
cout << " reshard list list all bucket resharding or scheduled to be reshared\n";
cout << " reshard process process of scheduled reshard jobs\n";
cout << " reshard cancel cancel resharding a bucket\n";
+ cout << " sync error list list sync error\n";
+ cout << " sync error trim trim sync error\n";
cout << "options:\n";
cout << " --tenant=<tenant> tenant name\n";
cout << " --uid=<id> user id\n";
cout << " --access=<access> Set access permissions for sub-user, should be one\n";
cout << " of read, write, readwrite, full\n";
cout << " --display-name=<name>\n";
- cout << " --max_buckets max number of buckets for a user\n";
+ cout << " --max-buckets max number of buckets for a user\n";
cout << " --admin set the admin flag on the user\n";
cout << " --system set the system flag on the user\n";
cout << " --bucket=<bucket>\n";
cout << " --tags-add=<list> list of tags to add for zonegroup placement modify command\n";
cout << " --tags-rm=<list> list of tags to remove for zonegroup placement modify command\n";
cout << " --endpoints=<list> zone endpoints\n";
- cout << " --index
_pool=<pool> placement target index pool\n";
+ cout << " --index
-pool=<pool> placement target index pool\n";
cout << " --data-pool=<pool> placement target data pool\n";
cout << " --data-extra-pool=<pool> placement target data extra (non-ec) pool\n";
cout << " --placement-index-type=<type>\n";
cout << " (NOTE: required to delete a non-empty bucket)\n";
cout << " --sync-stats option to 'user stats', update user stats with current\n";
cout << " stats reported by user's buckets indexes\n";
+ cout << " --reset-stats option to 'user stats', reset stats in accordance with user buckets\n";
cout << " --show-log-entries=<flag> enable/disable dump of log entries on log show\n";
cout << " --show-log-sum=<flag> enable/disable dump of log summation on log show\n";
cout << " --skip-zero-entries log show only dumps entries that don't have zero value\n";
generic_client_usage();
}
-int usage()
-{
- _usage();
- return 1;
-}
-
enum {
OPT_NO_CMD = 0,
OPT_USER_CREATE,
OPT_BUCKET_SYNC_STATUS,
OPT_BUCKET_SYNC_INIT,
OPT_BUCKET_SYNC_RUN,
+ OPT_BUCKET_SYNC_DISABLE,
+ OPT_BUCKET_SYNC_ENABLE,
OPT_BUCKET_RM,
OPT_BUCKET_REWRITE,
OPT_BUCKET_RESHARD,
OPT_MDLOG_FETCH,
OPT_MDLOG_STATUS,
OPT_SYNC_ERROR_LIST,
+ OPT_SYNC_ERROR_TRIM,
OPT_BILOG_LIST,
OPT_BILOG_TRIM,
OPT_BILOG_STATUS,
+ OPT_BILOG_AUTOTRIM,
OPT_DATA_SYNC_STATUS,
OPT_DATA_SYNC_INIT,
OPT_DATA_SYNC_RUN,
} else if (prev_prev_cmd && strcmp(prev_prev_cmd, "bucket") == 0) {
if (strcmp(prev_cmd, "sync") == 0) {
if (strcmp(cmd, "status") == 0)
- return OPT_BUCKET_SYNC_STATUS;
+ return OPT_BUCKET_SYNC_STATUS;
if (strcmp(cmd, "init") == 0)
- return OPT_BUCKET_SYNC_INIT;
+ return OPT_BUCKET_SYNC_INIT;
if (strcmp(cmd, "run") == 0)
- return OPT_BUCKET_SYNC_RUN;
+ return OPT_BUCKET_SYNC_RUN;
+ if (strcmp(cmd, "disable") == 0)
+ return OPT_BUCKET_SYNC_DISABLE;
+ if (strcmp(cmd, "enable") == 0)
+ return OPT_BUCKET_SYNC_ENABLE;
} else if ((strcmp(prev_cmd, "limit") == 0) &&
(strcmp(cmd, "check") == 0)) {
return OPT_BUCKET_LIMIT_CHECK;
(strcmp(prev_cmd, "error") == 0)) {
if (strcmp(cmd, "list") == 0)
return OPT_SYNC_ERROR_LIST;
+ if (strcmp(cmd, "trim") == 0)
+ return OPT_SYNC_ERROR_TRIM;
} else if (strcmp(prev_cmd, "mdlog") == 0) {
if (strcmp(cmd, "list") == 0)
return OPT_MDLOG_LIST;
return OPT_BILOG_TRIM;
if (strcmp(cmd, "status") == 0)
return OPT_BILOG_STATUS;
+ if (strcmp(cmd, "autotrim") == 0)
+ return OPT_BILOG_AUTOTRIM;
} else if (strcmp(prev_cmd, "data") == 0) {
if (strcmp(cmd, "sync") == 0) {
*need_more = true;
return OPT_RESHARD_LIST;
if (strcmp(cmd, "status") == 0)
return OPT_RESHARD_STATUS;
- if (strcmp(cmd, "execute") == 0)
+ if (strcmp(cmd, "process") == 0)
return OPT_RESHARD_PROCESS;
if (strcmp(cmd, "cancel") == 0)
return OPT_RESHARD_CANCEL;
return 0;
}
+int set_bucket_sync_enabled(RGWRados *store, int opt_cmd, const string& tenant_name, const string& bucket_name)
+{
+ RGWBucketInfo bucket_info;
+ map<string, bufferlist> attrs;
+ RGWObjectCtx obj_ctx(store);
+
+ int r = store->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, NULL, &attrs);
+ if (r < 0) {
+ cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
+ return -r;
+ }
+
+ if (opt_cmd == OPT_BUCKET_SYNC_ENABLE) {
+ bucket_info.flags &= ~BUCKET_DATASYNC_DISABLED;
+ } else if (opt_cmd == OPT_BUCKET_SYNC_DISABLE) {
+ bucket_info.flags |= BUCKET_DATASYNC_DISABLED;
+ }
+
+ r = store->put_bucket_instance_info(bucket_info, false, real_time(), &attrs);
+ if (r < 0) {
+ cerr << "ERROR: failed writing bucket instance info: " << cpp_strerror(-r) << std::endl;
+ return -r;
+ }
+
+ int shards_num = bucket_info.num_shards? bucket_info.num_shards : 1;
+ int shard_id = bucket_info.num_shards? 0 : -1;
+
+ if (opt_cmd == OPT_BUCKET_SYNC_DISABLE) {
+ r = store->stop_bi_log_entries(bucket_info, -1);
+ if (r < 0) {
+ lderr(store->ctx()) << "ERROR: failed writing stop bilog" << dendl;
+ return r;
+ }
+ } else {
+ r = store->resync_bi_log_entries(bucket_info, -1);
+ if (r < 0) {
+ lderr(store->ctx()) << "ERROR: failed writing resync bilog" << dendl;
+ return r;
+ }
+ }
+
+ for (int i = 0; i < shards_num; ++i, ++shard_id) {
+ r = store->data_log->add_entry(bucket_info.bucket, shard_id);
+ if (r < 0) {
+ lderr(store->ctx()) << "ERROR: failed writing data log" << dendl;
+ return r;
+ }
+ }
+
+ return 0;
+}
+
+
/// search for a matching zone/zonegroup id and return a connection if found
static boost::optional<RGWRESTConn> get_remote_conn(RGWRados *store,
const RGWZoneGroup& zonegroup,
for (auto iter : store->zone_conn_map) {
const string& source_id = iter.first;
- string zone_name;
string source_str = "source: ";
string s = source_str + source_id;
auto siter = store->zone_by_id.find(source_id);
string start_marker;
string end_marker;
int max_entries = -1;
+ bool max_entries_specified = false;
int admin = false;
bool admin_specified = false;
int system = false;
int include_all = false;
int sync_stats = false;
+ int reset_stats = false;
int bypass_gc = false;
int warnings_only = false;
int inconsistent_index = false;
break;
} else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
usage();
- return 0;
+ assert(false);
} else if (ceph_argparse_witharg(args, i, &val, "-i", "--uid", (char*)NULL)) {
user_id.from_str(val);
} else if (ceph_argparse_witharg(args, i, &val, "--tenant", (char*)NULL)) {
key_type = KEY_TYPE_S3;
} else {
cerr << "bad key type: " << key_type_str << std::endl;
- return usage();
+ usage();
+ assert(false);
}
} else if (ceph_argparse_witharg(args, i, &val, "--job-id", (char*)NULL)) {
job_id = val;
max_buckets_specified = true;
} else if (ceph_argparse_witharg(args, i, &val, "--max-entries", (char*)NULL)) {
max_entries = (int)strict_strtol(val.c_str(), 10, &err);
+ max_entries_specified = true;
if (!err.empty()) {
cerr << "ERROR: failed to parse max entries: " << err << std::endl;
return EINVAL;
bucket_id = val;
if (bucket_id.empty()) {
cerr << "bad bucket-id" << std::endl;
- return usage();
+ usage();
+ assert(false);
}
} else if (ceph_argparse_witharg(args, i, &val, "--format", (char*)NULL)) {
format = val;
// do nothing
} else if (ceph_argparse_binary_flag(args, i, &sync_stats, NULL, "--sync-stats", (char*)NULL)) {
// do nothing
+ } else if (ceph_argparse_binary_flag(args, i, &reset_stats, NULL, "--reset-stats", (char*)NULL)) {
+ // do nothing
} else if (ceph_argparse_binary_flag(args, i, &include_all, NULL, "--include-all", (char*)NULL)) {
// do nothing
} else if (ceph_argparse_binary_flag(args, i, &extra_info, NULL, "--extra-info", (char*)NULL)) {
}
if (args.empty()) {
- return usage();
+ usage();
+ assert(false);
}
else {
const char *prev_cmd = NULL;
opt_cmd = get_cmd(*i, prev_cmd, prev_prev_cmd, &need_more);
if (opt_cmd < 0) {
cerr << "unrecognized arg " << *i << std::endl;
- return usage();
+ usage();
+ assert(false);
}
if (!need_more) {
++i;
prev_cmd = *i;
}
- if (opt_cmd == OPT_NO_CMD)
- return usage();
+ if (opt_cmd == OPT_NO_CMD) {
+ usage();
+ assert(false);
+ }
/* some commands may have an optional extra param */
if (i != args.end()) {
&& opt_cmd != OPT_ROLE_POLICY_PUT
&& opt_cmd != OPT_ROLE_POLICY_LIST
&& opt_cmd != OPT_ROLE_POLICY_GET
- && opt_cmd != OPT_ROLE_POLICY_DELETE) {
+ && opt_cmd != OPT_ROLE_POLICY_DELETE
+ && opt_cmd != OPT_RESHARD_ADD
+ && opt_cmd != OPT_RESHARD_CANCEL
+ && opt_cmd != OPT_RESHARD_STATUS) {
cerr << "ERROR: --tenant is set, but there's no user ID" << std::endl;
return EINVAL;
}
formatter = new JSONFormatter(pretty_format);
else {
cerr << "unrecognized format: " << format << std::endl;
- return usage();
+ usage();
+ assert(false);
}
realm_name = g_conf->rgw_realm;
rgw_user_init(store);
rgw_bucket_init(store->meta_mgr);
+ struct rgw_curl_setup {
+ rgw_curl_setup() {
+ rgw::curl::setup_curl(boost::none);
+ }
+ ~rgw_curl_setup() {
+ rgw::curl::cleanup_curl();
+ }
+ } curl_cleanup;
+
StoreDestructor store_destructor(store);
if (raw_storage_op) {
cerr << "ERROR: --placement-id not specified" << std::endl;
return EINVAL;
}
+ // validate compression type
+ if (compression_type && *compression_type != "random"
+ && !Compressor::get_comp_alg_type(*compression_type)) {
+ std::cerr << "Unrecognized compression type" << std::endl;
+ return EINVAL;
+ }
+
RGWZoneParams zone(zone_id, zone_name);
int ret = zone.init(g_ceph_context, store);
if (ret < 0) {
ret = user.add(user_op, &err_msg);
if (ret < 0) {
cerr << "could not create user: " << err_msg << std::endl;
+ if (ret == -ERR_INVALID_TENANT_NAME)
+ ret = -EINVAL;
+
return -ret;
}
if (!subuser.empty()) {
if (opt_cmd == OPT_LOG_SHOW || opt_cmd == OPT_LOG_RM) {
if (object.empty() && (date.empty() || bucket_name.empty() || bucket_id.empty())) {
cerr << "specify an object or a date, bucket and bucket-id" << std::endl;
- return usage();
+ usage();
+ assert(false);
}
string oid;
if (opt_cmd == OPT_POOL_ADD) {
if (pool_name.empty()) {
cerr << "need to specify pool to add!" << std::endl;
- return usage();
+ usage();
+ assert(false);
}
int ret = store->add_bucket_placement(pool);
if (opt_cmd == OPT_POOL_RM) {
if (pool_name.empty()) {
cerr << "need to specify pool to remove!" << std::endl;
- return usage();
+ usage();
+ assert(false);
}
int ret = store->remove_bucket_placement(pool);
}
if (opt_cmd == OPT_RESHARD_CANCEL) {
- RGWReshard reshard(store);
-
if (bucket_name.empty()) {
cerr << "ERROR: bucket not specified" << std::endl;
return EINVAL;
}
- cls_rgw_reshard_entry entry;
- //entry.tenant = tenant;
- entry.bucket_name = bucket_name;
- //entry.bucket_id = bucket_id;
- int ret = reshard.get(entry);
+
+ rgw_bucket bucket;
+ RGWBucketInfo bucket_info;
+ map<string, bufferlist> attrs;
+ ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket, &attrs);
if (ret < 0) {
- cerr << "Error in getting bucket " << bucket_name << ": " << cpp_strerror(-ret) << std::endl;
+ cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
+ return -ret;
+ }
+
+ RGWBucketReshard br(store, bucket_info, attrs);
+ int ret = br.cancel();
+ if (ret < 0) {
+ if (ret == -EBUSY) {
+ cerr << "There is ongoing resharding, please retry after " << g_conf->rgw_reshard_bucket_lock_duration <<
+ " seconds " << std::endl;
+ } else {
+ cerr << "Error canceling bucket " << bucket_name << " resharding: " << cpp_strerror(-ret) <<
+ std::endl;
+ }
return ret;
}
+ RGWReshard reshard(store);
- /* TBD stop running resharding */
+ cls_rgw_reshard_entry entry;
+ //entry.tenant = tenant;
+ entry.bucket_name = bucket_name;
+ //entry.bucket_id = bucket_id;
- ret =reshard.remove(entry);
- if (ret < 0) {
- cerr << "Error removing bucket " << bucket_name << " for resharding queue: " << cpp_strerror(-ret) <<
- std::endl;
+ ret = reshard.remove(entry);
+ if (ret < 0 && ret != -ENOENT) {
+ cerr << "Error in getting bucket " << bucket_name << ": " << cpp_strerror(-ret) << std::endl;
return ret;
}
}
if (inconsistent_index == false) {
RGWBucketAdminOp::remove_bucket(store, bucket_op, bypass_gc, true);
} else {
+ if (!yes_i_really_mean_it) {
+ cerr << "using --inconsistent_index can corrupt the bucket index " << std::endl
+ << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl;
+ return 1;
+ }
RGWBucketAdminOp::remove_bucket(store, bucket_op, bypass_gc, false);
}
}
}
if (opt_cmd == OPT_USER_STATS) {
+ if (user_id.empty()) {
+ cerr << "ERROR: uid not specified" << std::endl;
+ return EINVAL;
+ }
+
+ string user_str = user_id.to_str();
+ if (reset_stats) {
+ if (!bucket_name.empty()){
+ cerr << "ERROR: recalculate doesn't work on buckets" << std::endl;
+ return EINVAL;
+ }
+ ret = store->cls_user_reset_stats(user_str);
+ if (ret < 0) {
+ cerr << "ERROR: could not clear user stats: " << cpp_strerror(-ret) << std::endl;
+ return -ret;
+ }
+ }
+
if (sync_stats) {
if (!bucket_name.empty()) {
int ret = rgw_bucket_sync_user_stats(store, tenant, bucket_name);
}
}
- if (user_id.empty()) {
- cerr << "ERROR: uid not specified" << std::endl;
- return EINVAL;
- }
cls_user_header header;
- string user_str = user_id.to_str();
int ret = store->cls_user_get_header(user_str, &header);
if (ret < 0) {
if (ret == -ENOENT) { /* in case of ENOENT */
}
void *handle;
int max = 1000;
- int ret = store->meta_mgr->list_keys_init(metadata_key, &handle);
+ int ret = store->meta_mgr->list_keys_init(metadata_key, marker, &handle);
if (ret < 0) {
cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl;
return -ret;
}
bool truncated;
+ uint64_t count = 0;
+ if (max_entries_specified) {
+ formatter->open_object_section("result");
+ }
formatter->open_array_section("keys");
+ uint64_t left;
do {
list<string> keys;
- ret = store->meta_mgr->list_keys_next(handle, max, keys, &truncated);
+ left = (max_entries_specified ? max_entries - count : max);
+ ret = store->meta_mgr->list_keys_next(handle, left, keys, &truncated);
if (ret < 0 && ret != -ENOENT) {
cerr << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << std::endl;
return -ret;
} if (ret != -ENOENT) {
for (list<string>::iterator iter = keys.begin(); iter != keys.end(); ++iter) {
formatter->dump_string("key", *iter);
+ ++count;
}
formatter->flush(cout);
}
- } while (truncated);
+ } while (truncated && left > 0);
formatter->close_section();
+
+ if (max_entries_specified) {
+ encode_json("truncated", truncated, formatter);
+ encode_json("count", count, formatter);
+ if (truncated) {
+ encode_json("marker", store->meta_mgr->get_marker(handle), formatter);
+ }
+ formatter->close_section();
+ }
formatter->flush(cout);
store->meta_mgr->list_keys_complete(handle);
cerr << "ERROR: source zone not specified" << std::endl;
return EINVAL;
}
+
RGWDataSyncStatusManager sync(store, store->get_async_rados(), source_zone);
int ret = sync.init();
cerr << "ERROR: source zone not specified" << std::endl;
return EINVAL;
}
- RGWDataSyncStatusManager sync(store, store->get_async_rados(), source_zone);
- int ret = sync.init();
+ RGWSyncModuleInstanceRef sync_module;
+ int ret = store->get_sync_modules_manager()->create_instance(g_ceph_context, store->get_zone().tier_type,
+ store->get_zone_params().tier_config, &sync_module);
+ if (ret < 0) {
+ lderr(cct) << "ERROR: failed to init sync module instance, ret=" << ret << dendl;
+ return ret;
+ }
+
+ RGWDataSyncStatusManager sync(store, store->get_async_rados(), source_zone, sync_module);
+
+ ret = sync.init();
if (ret < 0) {
cerr << "ERROR: sync.init() returned ret=" << ret << std::endl;
return -ret;
}
}
+ if ((opt_cmd == OPT_BUCKET_SYNC_DISABLE) || (opt_cmd == OPT_BUCKET_SYNC_ENABLE)) {
+ if (bucket_name.empty()) {
+ cerr << "ERROR: bucket not specified" << std::endl;
+ return EINVAL;
+ }
+
+ if (ret < 0) {
+ cerr << "could not init realm " << ": " << cpp_strerror(-ret) << std::endl;
+ return ret;
+ }
+ RGWPeriod period;
+ ret = period.init(g_ceph_context, store, realm_id, realm_name, true);
+ if (ret < 0) {
+ cerr << "failed to init period " << ": " << cpp_strerror(-ret) << std::endl;
+ return ret;
+ }
+
+ if (!store->is_meta_master()) {
+ cerr << "failed to update bucket sync: only allowed on meta master zone " << std::endl;
+ cerr << period.get_master_zone() << " | " << period.get_realm() << std::endl;
+ return EINVAL;
+ }
+
+ rgw_obj obj(bucket, object);
+ ret = set_bucket_sync_enabled(store, opt_cmd, tenant, bucket_name);
+ if (ret < 0)
+ return -ret;
+}
+
if (opt_cmd == OPT_BUCKET_SYNC_STATUS) {
if (source_zone.empty()) {
cerr << "ERROR: source zone not specified" << std::endl;
formatter->flush(cout);
}
+ if (opt_cmd == OPT_SYNC_ERROR_TRIM) {
+ utime_t start_time, end_time;
+ int ret = parse_date_str(start_date, start_time);
+ if (ret < 0)
+ return -ret;
+
+ ret = parse_date_str(end_date, end_time);
+ if (ret < 0)
+ return -ret;
+
+ if (shard_id < 0) {
+ shard_id = 0;
+ }
+
+ for (; shard_id < ERROR_LOGGER_SHARDS; ++shard_id) {
+ string oid = RGWSyncErrorLogger::get_shard_oid(RGW_SYNC_ERROR_LOG_SHARD_PREFIX, shard_id);
+ ret = store->time_log_trim(oid, start_time.to_real_time(), end_time.to_real_time(), start_marker, end_marker);
+ if (ret < 0 && ret != -ENODATA) {
+ cerr << "ERROR: sync error trim: " << cpp_strerror(-ret) << std::endl;
+ return -ret;
+ }
+ if (specified_shard_id) {
+ break;
+ }
+ }
+ }
+
if (opt_cmd == OPT_BILOG_TRIM) {
if (bucket_name.empty()) {
cerr << "ERROR: bucket not specified" << std::endl;
formatter->flush(cout);
}
+ if (opt_cmd == OPT_BILOG_AUTOTRIM) {
+ RGWCoroutinesManager crs(store->ctx(), store->get_cr_registry());
+ RGWHTTPManager http(store->ctx(), crs.get_completion_mgr());
+ int ret = http.set_threaded();
+ if (ret < 0) {
+ cerr << "failed to initialize http client with " << cpp_strerror(ret) << std::endl;
+ return -ret;
+ }
+
+ rgw::BucketTrimConfig config;
+ configure_bucket_trim(store->ctx(), config);
+
+ rgw::BucketTrimManager trim(store, config);
+ ret = trim.init();
+ if (ret < 0) {
+ cerr << "trim manager init failed with " << cpp_strerror(ret) << std::endl;
+ return -ret;
+ }
+ ret = crs.run(trim.create_admin_bucket_trim_cr(&http));
+ if (ret < 0) {
+ cerr << "automated bilog trim failed with " << cpp_strerror(ret) << std::endl;
+ return -ret;
+ }
+ }
+
if (opt_cmd == OPT_DATALOG_LIST) {
formatter->open_array_section("entries");
bool truncated;
projects / ceph.git / blobdiff