1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
9 #include <boost/optional.hpp>
12 #include <liboath/oath.h>
15 #include "auth/Crypto.h"
16 #include "compressor/Compressor.h"
18 #include "common/armor.h"
19 #include "common/ceph_json.h"
20 #include "common/config.h"
21 #include "common/ceph_argparse.h"
22 #include "common/Formatter.h"
23 #include "common/errno.h"
24 #include "common/safe_io.h"
26 #include "include/util.h"
28 #include "cls/rgw/cls_rgw_types.h"
29 #include "cls/rgw/cls_rgw_client.h"
31 #include "global/global_init.h"
33 #include "include/utime.h"
34 #include "include/str_list.h"
37 #include "rgw_bucket.h"
39 #include "rgw_rados.h"
41 #include "rgw_acl_s3.h"
42 #include "rgw_datalog.h"
45 #include "rgw_formats.h"
46 #include "rgw_usage.h"
47 #include "rgw_orphan.h"
49 #include "rgw_trim_bilog.h"
50 #include "rgw_trim_datalog.h"
51 #include "rgw_trim_mdlog.h"
52 #include "rgw_data_sync.h"
53 #include "rgw_rest_conn.h"
54 #include "rgw_realm_watcher.h"
56 #include "rgw_reshard.h"
57 #include "rgw_http_client_curl.h"
59 #include "rgw_pubsub.h"
60 #include "rgw_bucket_sync.h"
61 #include "rgw_sync_checkpoint.h"
64 #include "services/svc_sync_modules.h"
65 #include "services/svc_cls.h"
66 #include "services/svc_bilog_rados.h"
67 #include "services/svc_mdlog.h"
68 #include "services/svc_meta_be_otp.h"
69 #include "services/svc_zone.h"
71 #define dout_context g_ceph_context
72 #define dout_subsys ceph_subsys_rgw
74 #define SECRET_KEY_LEN 40
75 #define PUBLIC_ID_LEN 20
77 static rgw::sal::RGWRadosStore
*store
= NULL
;
79 static const DoutPrefixProvider
* dpp() {
80 struct GlobalPrefix
: public DoutPrefixProvider
{
81 CephContext
*get_cct() const override
{ return store
->ctx(); }
82 unsigned get_subsys() const override
{ return dout_subsys
; }
83 std::ostream
& gen_prefix(std::ostream
& out
) const override
{ return out
; }
85 static GlobalPrefix global_dpp
;
89 #define CHECK_TRUE(x, msg, err) \
92 cerr << msg << std::endl; \
97 #define CHECK_SUCCESS(x, msg) \
101 cerr << msg << ": " << cpp_strerror(-_x_val) << std::endl; \
108 cout
<< "usage: radosgw-admin <cmd> [options...]" << std::endl
;
109 cout
<< "commands:\n";
110 cout
<< " user create create a new user\n" ;
111 cout
<< " user modify modify user\n";
112 cout
<< " user info get user info\n";
113 cout
<< " user rename rename user\n";
114 cout
<< " user rm remove user\n";
115 cout
<< " user suspend suspend a user\n";
116 cout
<< " user enable re-enable user after suspension\n";
117 cout
<< " user check check user info\n";
118 cout
<< " user stats show user stats as accounted by quota subsystem\n";
119 cout
<< " user list list users\n";
120 cout
<< " caps add add user capabilities\n";
121 cout
<< " caps rm remove user capabilities\n";
122 cout
<< " subuser create create a new subuser\n" ;
123 cout
<< " subuser modify modify subuser\n";
124 cout
<< " subuser rm remove subuser\n";
125 cout
<< " key create create access key\n";
126 cout
<< " key rm remove access key\n";
127 cout
<< " bucket list list buckets (specify --allow-unordered for\n";
128 cout
<< " faster, unsorted listing)\n";
129 cout
<< " bucket limit check show bucket sharding stats\n";
130 cout
<< " bucket link link bucket to specified user\n";
131 cout
<< " bucket unlink unlink bucket from specified user\n";
132 cout
<< " bucket stats returns bucket statistics\n";
133 cout
<< " bucket rm remove bucket\n";
134 cout
<< " bucket check check bucket index\n";
135 cout
<< " bucket chown link bucket to specified user and update its object ACLs\n";
136 cout
<< " bucket reshard reshard bucket\n";
137 cout
<< " bucket rewrite rewrite all objects in the specified bucket\n";
138 cout
<< " bucket sync checkpoint poll a bucket's sync status until it catches up to its remote\n";
139 cout
<< " bucket sync disable disable bucket sync\n";
140 cout
<< " bucket sync enable enable bucket sync\n";
141 cout
<< " bucket radoslist list rados objects backing bucket's objects\n";
142 cout
<< " bi get retrieve bucket index object entries\n";
143 cout
<< " bi put store bucket index object entries\n";
144 cout
<< " bi list list raw bucket index entries\n";
145 cout
<< " bi purge purge bucket index entries\n";
146 cout
<< " object rm remove object\n";
147 cout
<< " object put put object\n";
148 cout
<< " object stat stat an object for its metadata\n";
149 cout
<< " object unlink unlink object from bucket index\n";
150 cout
<< " object rewrite rewrite the specified object\n";
151 cout
<< " objects expire run expired objects cleanup\n";
152 cout
<< " objects expire-stale list list stale expired objects (caused by reshard)\n";
153 cout
<< " objects expire-stale rm remove stale expired objects\n";
154 cout
<< " period rm remove a period\n";
155 cout
<< " period get get period info\n";
156 cout
<< " period get-current get current period info\n";
157 cout
<< " period pull pull a period\n";
158 cout
<< " period push push a period\n";
159 cout
<< " period list list all periods\n";
160 cout
<< " period update update the staging period\n";
161 cout
<< " period commit commit the staging period\n";
162 cout
<< " quota set set quota params\n";
163 cout
<< " quota enable enable quota\n";
164 cout
<< " quota disable disable quota\n";
165 cout
<< " global quota get view global quota params\n";
166 cout
<< " global quota set set global quota params\n";
167 cout
<< " global quota enable enable a global quota\n";
168 cout
<< " global quota disable disable a global quota\n";
169 cout
<< " realm create create a new realm\n";
170 cout
<< " realm rm remove a realm\n";
171 cout
<< " realm get show realm info\n";
172 cout
<< " realm get-default get default realm name\n";
173 cout
<< " realm list list realms\n";
174 cout
<< " realm list-periods list all realm periods\n";
175 cout
<< " realm rename rename a realm\n";
176 cout
<< " realm set set realm info (requires infile)\n";
177 cout
<< " realm default set realm as default\n";
178 cout
<< " realm pull pull a realm and its current period\n";
179 cout
<< " zonegroup add add a zone to a zonegroup\n";
180 cout
<< " zonegroup create create a new zone group info\n";
181 cout
<< " zonegroup default set default zone group\n";
182 cout
<< " zonegroup delete delete a zone group info\n";
183 cout
<< " zonegroup get show zone group info\n";
184 cout
<< " zonegroup modify modify an existing zonegroup\n";
185 cout
<< " zonegroup set set zone group info (requires infile)\n";
186 cout
<< " zonegroup rm remove a zone from a zonegroup\n";
187 cout
<< " zonegroup rename rename a zone group\n";
188 cout
<< " zonegroup list list all zone groups set on this cluster\n";
189 cout
<< " zonegroup placement list list zonegroup's placement targets\n";
190 cout
<< " zonegroup placement get get a placement target of a specific zonegroup\n";
191 cout
<< " zonegroup placement add add a placement target id to a zonegroup\n";
192 cout
<< " zonegroup placement modify modify a placement target of a specific zonegroup\n";
193 cout
<< " zonegroup placement rm remove a placement target from a zonegroup\n";
194 cout
<< " zonegroup placement default set a zonegroup's default placement target\n";
195 cout
<< " zone create create a new zone\n";
196 cout
<< " zone rm remove a zone\n";
197 cout
<< " zone get show zone cluster params\n";
198 cout
<< " zone modify modify an existing zone\n";
199 cout
<< " zone set set zone cluster params (requires infile)\n";
200 cout
<< " zone list list all zones set on this cluster\n";
201 cout
<< " zone rename rename a zone\n";
202 cout
<< " zone placement list list zone's placement targets\n";
203 cout
<< " zone placement get get a zone placement target\n";
204 cout
<< " zone placement add add a zone placement target\n";
205 cout
<< " zone placement modify modify a zone placement target\n";
206 cout
<< " zone placement rm remove a zone placement target\n";
207 cout
<< " metadata sync status get metadata sync status\n";
208 cout
<< " metadata sync init init metadata sync\n";
209 cout
<< " metadata sync run run metadata sync\n";
210 cout
<< " data sync status get data sync status of the specified source zone\n";
211 cout
<< " data sync init init data sync for the specified source zone\n";
212 cout
<< " data sync run run data sync for the specified source zone\n";
213 cout
<< " pool add add an existing pool for data placement\n";
214 cout
<< " pool rm remove an existing pool from data placement set\n";
215 cout
<< " pools list list placement active set\n";
216 cout
<< " policy read bucket/object policy\n";
217 cout
<< " log list list log objects\n";
218 cout
<< " log show dump a log from specific object or (bucket + date\n";
219 cout
<< " + bucket-id)\n";
220 cout
<< " (NOTE: required to specify formatting of date\n";
221 cout
<< " to \"YYYY-MM-DD-hh\")\n";
222 cout
<< " log rm remove log object\n";
223 cout
<< " usage show show usage (by user, by bucket, date range)\n";
224 cout
<< " usage trim trim usage (by user, by bucket, date range)\n";
225 cout
<< " usage clear reset all the usage stats for the cluster\n";
226 cout
<< " gc list dump expired garbage collection objects (specify\n";
227 cout
<< " --include-all to list all entries, including unexpired)\n";
228 cout
<< " gc process manually process garbage (specify\n";
229 cout
<< " --include-all to process all entries, including unexpired)\n";
230 cout
<< " lc list list all bucket lifecycle progress\n";
231 cout
<< " lc get get a lifecycle bucket configuration\n";
232 cout
<< " lc process manually process lifecycle\n";
233 cout
<< " lc reshard fix fix LC for a resharded bucket\n";
234 cout
<< " metadata get get metadata info\n";
235 cout
<< " metadata put put metadata info\n";
236 cout
<< " metadata rm remove metadata info\n";
237 cout
<< " metadata list list metadata info\n";
238 cout
<< " mdlog list list metadata log\n";
239 cout
<< " mdlog trim trim metadata log (use marker)\n";
240 cout
<< " mdlog status read metadata log status\n";
241 cout
<< " bilog list list bucket index log\n";
242 cout
<< " bilog trim trim bucket index log (use start-marker, end-marker)\n";
243 cout
<< " bilog status read bucket index log status\n";
244 cout
<< " datalog list list data log\n";
245 cout
<< " datalog trim trim data log\n";
246 cout
<< " datalog status read data log status\n";
247 cout
<< " datalog type change datalog type to --log_type={fifo,omap}\n";
248 cout
<< " orphans find deprecated -- init and run search for leaked rados objects (use job-id, pool)\n";
249 cout
<< " orphans finish deprecated -- clean up search for leaked rados objects\n";
250 cout
<< " orphans list-jobs deprecated -- list the current job-ids for orphans search\n";
251 cout
<< " * the three 'orphans' sub-commands are now deprecated; consider using the `rgw-orphan-list` tool\n";
252 cout
<< " role create create a AWS role for use with STS\n";
253 cout
<< " role rm remove a role\n";
254 cout
<< " role get get a role\n";
255 cout
<< " role list list roles with specified path prefix\n";
256 cout
<< " role modify modify the assume role policy of an existing role\n";
257 cout
<< " role-policy put add/update permission policy to role\n";
258 cout
<< " role-policy list list policies attached to a role\n";
259 cout
<< " role-policy get get the specified inline policy document embedded with the given role\n";
260 cout
<< " role-policy rm remove policy attached to a role\n";
261 cout
<< " reshard add schedule a resharding of a bucket\n";
262 cout
<< " reshard list list all bucket resharding or scheduled to be resharded\n";
263 cout
<< " reshard status read bucket resharding status\n";
264 cout
<< " reshard process process of scheduled reshard jobs\n";
265 cout
<< " reshard cancel cancel resharding a bucket\n";
266 cout
<< " reshard stale-instances list list stale-instances from bucket resharding\n";
267 cout
<< " reshard stale-instances rm cleanup stale-instances from bucket resharding\n";
268 cout
<< " sync error list list sync error\n";
269 cout
<< " sync error trim trim sync error\n";
270 cout
<< " mfa create create a new MFA TOTP token\n";
271 cout
<< " mfa list list MFA TOTP tokens\n";
272 cout
<< " mfa get show MFA TOTP token\n";
273 cout
<< " mfa remove delete MFA TOTP token\n";
274 cout
<< " mfa check check MFA TOTP token\n";
275 cout
<< " mfa resync re-sync MFA TOTP token\n";
276 cout
<< " topic list list bucket notifications/pubsub topics\n";
277 cout
<< " topic get get a bucket notifications/pubsub topic\n";
278 cout
<< " topic rm remove a bucket notifications/pubsub topic\n";
279 cout
<< " subscription get get a pubsub subscription definition\n";
280 cout
<< " subscription rm remove a pubsub subscription\n";
281 cout
<< " subscription pull show events in a pubsub subscription\n";
282 cout
<< " subscription ack ack (remove) an events in a pubsub subscription\n";
283 cout
<< " script put upload a lua script to a context\n";
284 cout
<< " script get get the lua script of a context\n";
285 cout
<< " script rm remove the lua scripts of a context\n";
286 cout
<< " script-package add add a lua package to the scripts allowlist\n";
287 cout
<< " script-package rm remove a lua package from the scripts allowlist\n";
288 cout
<< " script-package list get the lua packages allowlist\n";
289 cout
<< "options:\n";
290 cout
<< " --tenant=<tenant> tenant name\n";
291 cout
<< " --user_ns=<namespace> namespace of user (oidc in case of users authenticated with oidc provider)\n";
292 cout
<< " --uid=<id> user id\n";
293 cout
<< " --new-uid=<id> new user id\n";
294 cout
<< " --subuser=<name> subuser name\n";
295 cout
<< " --access-key=<key> S3 access key\n";
296 cout
<< " --email=<email> user's email address\n";
297 cout
<< " --secret/--secret-key=<key>\n";
298 cout
<< " specify secret key\n";
299 cout
<< " --gen-access-key generate random access key (for S3)\n";
300 cout
<< " --gen-secret generate random secret key\n";
301 cout
<< " --key-type=<type> key type, options are: swift, s3\n";
302 cout
<< " --temp-url-key[-2]=<key> temp url key\n";
303 cout
<< " --access=<access> Set access permissions for sub-user, should be one\n";
304 cout
<< " of read, write, readwrite, full\n";
305 cout
<< " --display-name=<name> user's display name\n";
306 cout
<< " --max-buckets max number of buckets for a user\n";
307 cout
<< " --admin set the admin flag on the user\n";
308 cout
<< " --system set the system flag on the user\n";
309 cout
<< " --op-mask set the op mask on the user\n";
310 cout
<< " --bucket=<bucket> Specify the bucket name. Also used by the quota command.\n";
311 cout
<< " --pool=<pool> Specify the pool name. Also used to scan for leaked rados objects.\n";
312 cout
<< " --object=<object> object name\n";
313 cout
<< " --object-version=<version> object version\n";
314 cout
<< " --date=<date> date in the format yyyy-mm-dd\n";
315 cout
<< " --start-date=<date> start date in the format yyyy-mm-dd\n";
316 cout
<< " --end-date=<date> end date in the format yyyy-mm-dd\n";
317 cout
<< " --bucket-id=<bucket-id> bucket id\n";
318 cout
<< " --bucket-new-name=<bucket>\n";
319 cout
<< " for bucket link: optional new name\n";
320 cout
<< " --shard-id=<shard-id> optional for: \n";
321 cout
<< " mdlog list\n";
322 cout
<< " data sync status\n";
323 cout
<< " required for: \n";
324 cout
<< " mdlog trim\n";
325 cout
<< " --max-entries=<entries> max entries for listing operations\n";
326 cout
<< " --metadata-key=<key> key to retrieve metadata from with metadata get\n";
327 cout
<< " --remote=<remote> zone or zonegroup id of remote gateway\n";
328 cout
<< " --period=<id> period id\n";
329 cout
<< " --url=<url> url for pushing/pulling period/realm\n";
330 cout
<< " --epoch=<number> period epoch\n";
331 cout
<< " --commit commit the period during 'period update'\n";
332 cout
<< " --staging get staging period info\n";
333 cout
<< " --master set as master\n";
334 cout
<< " --master-zone=<id> master zone id\n";
335 cout
<< " --rgw-realm=<name> realm name\n";
336 cout
<< " --realm-id=<id> realm id\n";
337 cout
<< " --realm-new-name=<name> realm new name\n";
338 cout
<< " --rgw-zonegroup=<name> zonegroup name\n";
339 cout
<< " --zonegroup-id=<id> zonegroup id\n";
340 cout
<< " --zonegroup-new-name=<name>\n";
341 cout
<< " zonegroup new name\n";
342 cout
<< " --rgw-zone=<name> name of zone in which radosgw is running\n";
343 cout
<< " --zone-id=<id> zone id\n";
344 cout
<< " --zone-new-name=<name> zone new name\n";
345 cout
<< " --source-zone specify the source zone (for data sync)\n";
346 cout
<< " --default set entity (realm, zonegroup, zone) as default\n";
347 cout
<< " --read-only set zone as read-only (when adding to zonegroup)\n";
348 cout
<< " --redirect-zone specify zone id to redirect when response is 404 (not found)\n";
349 cout
<< " --placement-id placement id for zonegroup placement commands\n";
350 cout
<< " --storage-class storage class for zonegroup placement commands\n";
351 cout
<< " --tags=<list> list of tags for zonegroup placement add and modify commands\n";
352 cout
<< " --tags-add=<list> list of tags to add for zonegroup placement modify command\n";
353 cout
<< " --tags-rm=<list> list of tags to remove for zonegroup placement modify command\n";
354 cout
<< " --endpoints=<list> zone endpoints\n";
355 cout
<< " --index-pool=<pool> placement target index pool\n";
356 cout
<< " --data-pool=<pool> placement target data pool\n";
357 cout
<< " --data-extra-pool=<pool> placement target data extra (non-ec) pool\n";
358 cout
<< " --placement-index-type=<type>\n";
359 cout
<< " placement target index type (normal, indexless, or #id)\n";
360 cout
<< " --compression=<type> placement target compression type (plugin name or empty/none)\n";
361 cout
<< " --tier-type=<type> zone tier type\n";
362 cout
<< " --tier-config=<k>=<v>[,...]\n";
363 cout
<< " set zone tier config keys, values\n";
364 cout
<< " --tier-config-rm=<k>[,...]\n";
365 cout
<< " unset zone tier config keys\n";
366 cout
<< " --sync-from-all[=false] set/reset whether zone syncs from all zonegroup peers\n";
367 cout
<< " --sync-from=[zone-name][,...]\n";
368 cout
<< " set list of zones to sync from\n";
369 cout
<< " --sync-from-rm=[zone-name][,...]\n";
370 cout
<< " remove zones from list of zones to sync from\n";
371 cout
<< " --bucket-index-max-shards override a zone/zonegroup's default bucket index shard count\n";
372 cout
<< " --fix besides checking bucket index, will also fix it\n";
373 cout
<< " --check-objects bucket check: rebuilds bucket index according to\n";
374 cout
<< " actual objects state\n";
375 cout
<< " --format=<format> specify output format for certain operations: xml,\n";
377 cout
<< " --purge-data when specified, user removal will also purge all the\n";
378 cout
<< " user data\n";
379 cout
<< " --purge-keys when specified, subuser removal will also purge all the\n";
380 cout
<< " subuser keys\n";
381 cout
<< " --purge-objects remove a bucket's objects before deleting it\n";
382 cout
<< " (NOTE: required to delete a non-empty bucket)\n";
383 cout
<< " --sync-stats option to 'user stats', update user stats with current\n";
384 cout
<< " stats reported by user's buckets indexes\n";
385 cout
<< " --reset-stats option to 'user stats', reset stats in accordance with user buckets\n";
386 cout
<< " --show-log-entries=<flag> enable/disable dump of log entries on log show\n";
387 cout
<< " --show-log-sum=<flag> enable/disable dump of log summation on log show\n";
388 cout
<< " --skip-zero-entries log show only dumps entries that don't have zero value\n";
389 cout
<< " in one of the numeric field\n";
390 cout
<< " --infile=<file> specify a file to read in when setting data\n";
391 cout
<< " --categories=<list> comma separated list of categories, used in usage show\n";
392 cout
<< " --caps=<caps> list of caps (e.g., \"usage=read, write; user=read\")\n";
393 cout
<< " --op-mask=<op-mask> permission of user's operations (e.g., \"read, write, delete, *\")\n";
394 cout
<< " --yes-i-really-mean-it required for certain operations\n";
395 cout
<< " --warnings-only when specified with bucket limit check, list\n";
396 cout
<< " only buckets nearing or over the current max\n";
397 cout
<< " objects per shard value\n";
398 cout
<< " --bypass-gc when specified with bucket deletion, triggers\n";
399 cout
<< " object deletions by not involving GC\n";
400 cout
<< " --inconsistent-index when specified with bucket deletion and bypass-gc set to true,\n";
401 cout
<< " ignores bucket index consistency\n";
402 cout
<< " --min-rewrite-size min object size for bucket rewrite (default 4M)\n";
403 cout
<< " --max-rewrite-size max object size for bucket rewrite (default ULLONG_MAX)\n";
404 cout
<< " --min-rewrite-stripe-size min stripe size for object rewrite (default 0)\n";
405 cout
<< " --trim-delay-ms time interval in msec to limit the frequency of sync error log entries trimming operations,\n";
406 cout
<< " the trimming process will sleep the specified msec for every 1000 entries trimmed\n";
407 cout
<< " --max-concurrent-ios maximum concurrent ios for bucket operations (default: 32)\n";
409 cout
<< "<date> := \"YYYY-MM-DD[ hh:mm:ss]\"\n";
410 cout
<< "\nQuota options:\n";
411 cout
<< " --max-objects specify max objects (negative value to disable)\n";
412 cout
<< " --max-size specify max size (in B/K/M/G/T, negative value to disable)\n";
413 cout
<< " --quota-scope scope of quota (bucket, user)\n";
414 cout
<< "\nOrphans search options:\n";
415 cout
<< " --num-shards num of shards to use for keeping the temporary scan info\n";
416 cout
<< " --orphan-stale-secs num of seconds to wait before declaring an object to be an orphan (default: 86400)\n";
417 cout
<< " --job-id set the job id (for orphans find)\n";
418 cout
<< " --detail detailed mode, log and stat head objects as well\n";
419 cout
<< "\nOrphans list-jobs options:\n";
420 cout
<< " --extra-info provide extra info in job list\n";
421 cout
<< "\nRole options:\n";
422 cout
<< " --role-name name of the role to create\n";
423 cout
<< " --path path to the role\n";
424 cout
<< " --assume-role-policy-doc the trust relationship policy document that grants an entity permission to assume the role\n";
425 cout
<< " --policy-name name of the policy document\n";
426 cout
<< " --policy-doc permission policy document\n";
427 cout
<< " --path-prefix path prefix for filtering roles\n";
428 cout
<< "\nMFA options:\n";
429 cout
<< " --totp-serial a string that represents the ID of a TOTP token\n";
430 cout
<< " --totp-seed the secret seed that is used to calculate the TOTP\n";
431 cout
<< " --totp-seconds the time resolution that is being used for TOTP generation\n";
432 cout
<< " --totp-window the number of TOTP tokens that are checked before and after the current token when validating token\n";
433 cout
<< " --totp-pin the valid value of a TOTP token at a certain time\n";
434 cout
<< "\nBucket notifications/pubsub options:\n";
435 cout
<< " --topic bucket notifications/pubsub topic name\n";
436 cout
<< " --subscription pubsub subscription name\n";
437 cout
<< " --event-id event id in a pubsub subscription\n";
438 cout
<< "\nScript options:\n";
439 cout
<< " --context context in which the script runs. one of: preRequest, postRequest\n";
440 cout
<< " --package name of the lua package that should be added/removed to/from the allowlist\n";
441 cout
<< " --allow-compilation package is allowed to compile C code as part of its installation\n";
442 cout
<< "\nradoslist options:\n";
443 cout
<< " --rgw-obj-fs the field separator that will separate the rados\n";
444 cout
<< " object name from the rgw object name;\n";
445 cout
<< " additionally rados objects for incomplete\n";
446 cout
<< " multipart uploads will not be output\n";
448 generic_client_usage();
459 using Aliases
= std::vector
<std::set
<string
> >;
460 using Commands
= std::vector
<Def
>;
464 map
<string
, Node
> next
;
465 set
<string
> expected
; /* separate un-normalized list */
470 map
<string
, string
> alias_map
;
472 string
normalize_alias(const string
& s
) const {
473 auto iter
= alias_map
.find(s
);
474 if (iter
== alias_map
.end()) {
480 void init_alias_map(Aliases
& aliases
) {
481 for (auto& alias_set
: aliases
) {
482 std::optional
<string
> first
;
484 for (auto& alias
: alias_set
) {
488 alias_map
[alias
] = *first
;
494 bool gen_next_expected(Node
*node
, vector
<string
> *expected
, bool ret
) {
495 for (auto& next_cmd
: node
->expected
) {
496 expected
->push_back(next_cmd
);
506 SimpleCmd(std::optional
<Commands
> cmds
,
507 std::optional
<Aliases
> aliases
) {
509 add_aliases(*aliases
);
517 void add_aliases(Aliases
& aliases
) {
518 init_alias_map(aliases
);
521 void add_commands(std::vector
<Def
>& cmds
) {
522 for (auto& cmd
: cmds
) {
523 vector
<string
> words
;
524 get_str_vec(cmd
.cmd
, " ", words
);
526 auto node
= &cmd_root
;
527 for (auto& word
: words
) {
528 auto norm
= normalize_alias(word
);
531 node
->expected
.insert(word
);
533 node
= &node
->next
[norm
];
535 if (norm
== "[*]") { /* optional param at the end */
536 parent
->next
["*"] = *node
; /* can be also looked up by '*' */
537 parent
->opt
= cmd
.opt
;
545 template <class Container
>
546 bool find_command(Container
& args
,
548 vector
<string
> *extra_args
,
550 vector
<string
> *expected
) {
551 auto node
= &cmd_root
;
553 std::optional
<std::any
> found_opt
;
555 for (auto& arg
: args
) {
556 string norm
= normalize_alias(arg
);
557 auto iter
= node
->next
.find(norm
);
558 if (iter
== node
->next
.end()) {
559 iter
= node
->next
.find("*");
560 if (iter
== node
->next
.end()) {
561 *error
= string("ERROR: Unrecognized argument: '") + arg
+ "'";
562 return gen_next_expected(node
, expected
, false);
564 extra_args
->push_back(arg
);
566 found_opt
= node
->opt
;
569 node
= &(iter
->second
);
572 *opt_cmd
= found_opt
.value_or(node
->opt
);
574 if (!opt_cmd
->has_value()) {
575 *error
="ERROR: Unknown command";
576 return gen_next_expected(node
, expected
, false);
584 namespace rgw_admin
{
609 BUCKET_SYNC_CHECKPOINT
,
638 OBJECTS_EXPIRE_STALE_LIST
,
639 OBJECTS_EXPIRE_STALE_RM
,
668 ZONEGROUP_PLACEMENT_ADD
,
669 ZONEGROUP_PLACEMENT_MODIFY
,
670 ZONEGROUP_PLACEMENT_RM
,
671 ZONEGROUP_PLACEMENT_LIST
,
672 ZONEGROUP_PLACEMENT_GET
,
673 ZONEGROUP_PLACEMENT_DEFAULT
,
683 ZONE_PLACEMENT_MODIFY
,
693 METADATA_SYNC_STATUS
,
707 SYNC_GROUP_FLOW_CREATE
,
708 SYNC_GROUP_FLOW_REMOVE
,
709 SYNC_GROUP_PIPE_CREATE
,
710 SYNC_GROUP_PIPE_MODIFY
,
711 SYNC_GROUP_PIPE_REMOVE
,
747 GLOBAL_QUOTA_DISABLE
,
770 RESHARD_STALE_INSTANCES_LIST
,
771 RESHARD_STALE_INSTANCES_DELETE
,
773 // TODO add "subscription list" command
790 using namespace rgw_admin
;
792 static SimpleCmd::Commands all_cmds
= {
793 { "user create", OPT::USER_CREATE
},
794 { "user info", OPT::USER_INFO
},
795 { "user modify", OPT::USER_MODIFY
},
796 { "user rename", OPT::USER_RENAME
},
797 { "user rm", OPT::USER_RM
},
798 { "user suspend", OPT::USER_SUSPEND
},
799 { "user enable", OPT::USER_ENABLE
},
800 { "user check", OPT::USER_CHECK
},
801 { "user stats", OPT::USER_STATS
},
802 { "user list", OPT::USER_LIST
},
803 { "subuser create", OPT::SUBUSER_CREATE
},
804 { "subuser modify", OPT::SUBUSER_MODIFY
},
805 { "subuser rm", OPT::SUBUSER_RM
},
806 { "key create", OPT::KEY_CREATE
},
807 { "key rm", OPT::KEY_RM
},
808 { "buckets list", OPT::BUCKETS_LIST
},
809 { "bucket list", OPT::BUCKETS_LIST
},
810 { "bucket limit check", OPT::BUCKET_LIMIT_CHECK
},
811 { "bucket link", OPT::BUCKET_LINK
},
812 { "bucket unlink", OPT::BUCKET_UNLINK
},
813 { "bucket stats", OPT::BUCKET_STATS
},
814 { "bucket check", OPT::BUCKET_CHECK
},
815 { "bucket sync checkpoint", OPT::BUCKET_SYNC_CHECKPOINT
},
816 { "bucket sync info", OPT::BUCKET_SYNC_INFO
},
817 { "bucket sync status", OPT::BUCKET_SYNC_STATUS
},
818 { "bucket sync markers", OPT::BUCKET_SYNC_MARKERS
},
819 { "bucket sync init", OPT::BUCKET_SYNC_INIT
},
820 { "bucket sync run", OPT::BUCKET_SYNC_RUN
},
821 { "bucket sync disable", OPT::BUCKET_SYNC_DISABLE
},
822 { "bucket sync enable", OPT::BUCKET_SYNC_ENABLE
},
823 { "bucket rm", OPT::BUCKET_RM
},
824 { "bucket rewrite", OPT::BUCKET_REWRITE
},
825 { "bucket reshard", OPT::BUCKET_RESHARD
},
826 { "bucket chown", OPT::BUCKET_CHOWN
},
827 { "bucket radoslist", OPT::BUCKET_RADOS_LIST
},
828 { "bucket rados list", OPT::BUCKET_RADOS_LIST
},
829 { "policy", OPT::POLICY
},
830 { "pool add", OPT::POOL_ADD
},
831 { "pool rm", OPT::POOL_RM
},
832 { "pool list", OPT::POOLS_LIST
},
833 { "pools list", OPT::POOLS_LIST
},
834 { "log list", OPT::LOG_LIST
},
835 { "log show", OPT::LOG_SHOW
},
836 { "log rm", OPT::LOG_RM
},
837 { "usage show", OPT::USAGE_SHOW
},
838 { "usage trim", OPT::USAGE_TRIM
},
839 { "usage clear", OPT::USAGE_CLEAR
},
840 { "object put", OPT::OBJECT_PUT
},
841 { "object rm", OPT::OBJECT_RM
},
842 { "object unlink", OPT::OBJECT_UNLINK
},
843 { "object stat", OPT::OBJECT_STAT
},
844 { "object rewrite", OPT::OBJECT_REWRITE
},
845 { "objects expire", OPT::OBJECTS_EXPIRE
},
846 { "objects expire-stale list", OPT::OBJECTS_EXPIRE_STALE_LIST
},
847 { "objects expire-stale rm", OPT::OBJECTS_EXPIRE_STALE_RM
},
848 { "bi get", OPT::BI_GET
},
849 { "bi put", OPT::BI_PUT
},
850 { "bi list", OPT::BI_LIST
},
851 { "bi purge", OPT::BI_PURGE
},
852 { "olh get", OPT::OLH_GET
},
853 { "olh readlog", OPT::OLH_READLOG
},
854 { "quota set", OPT::QUOTA_SET
},
855 { "quota enable", OPT::QUOTA_ENABLE
},
856 { "quota disable", OPT::QUOTA_DISABLE
},
857 { "gc list", OPT::GC_LIST
},
858 { "gc process", OPT::GC_PROCESS
},
859 { "lc list", OPT::LC_LIST
},
860 { "lc get", OPT::LC_GET
},
861 { "lc process", OPT::LC_PROCESS
},
862 { "lc reshard fix", OPT::LC_RESHARD_FIX
},
863 { "orphans find", OPT::ORPHANS_FIND
},
864 { "orphans finish", OPT::ORPHANS_FINISH
},
865 { "orphans list jobs", OPT::ORPHANS_LIST_JOBS
},
866 { "orphans list-jobs", OPT::ORPHANS_LIST_JOBS
},
867 { "zonegroup add", OPT::ZONEGROUP_ADD
},
868 { "zonegroup create", OPT::ZONEGROUP_CREATE
},
869 { "zonegroup default", OPT::ZONEGROUP_DEFAULT
},
870 { "zonegroup delete", OPT::ZONEGROUP_DELETE
},
871 { "zonegroup get", OPT::ZONEGROUP_GET
},
872 { "zonegroup modify", OPT::ZONEGROUP_MODIFY
},
873 { "zonegroup set", OPT::ZONEGROUP_SET
},
874 { "zonegroup list", OPT::ZONEGROUP_LIST
},
875 { "zonegroups list", OPT::ZONEGROUP_LIST
},
876 { "zonegroup remove", OPT::ZONEGROUP_REMOVE
},
877 { "zonegroup remove zone", OPT::ZONEGROUP_REMOVE
},
878 { "zonegroup rename", OPT::ZONEGROUP_RENAME
},
879 { "zonegroup placement add", OPT::ZONEGROUP_PLACEMENT_ADD
},
880 { "zonegroup placement modify", OPT::ZONEGROUP_PLACEMENT_MODIFY
},
881 { "zonegroup placement rm", OPT::ZONEGROUP_PLACEMENT_RM
},
882 { "zonegroup placement list", OPT::ZONEGROUP_PLACEMENT_LIST
},
883 { "zonegroup placement get", OPT::ZONEGROUP_PLACEMENT_GET
},
884 { "zonegroup placement default", OPT::ZONEGROUP_PLACEMENT_DEFAULT
},
885 { "zone create", OPT::ZONE_CREATE
},
886 { "zone delete", OPT::ZONE_DELETE
},
887 { "zone get", OPT::ZONE_GET
},
888 { "zone modify", OPT::ZONE_MODIFY
},
889 { "zone set", OPT::ZONE_SET
},
890 { "zone list", OPT::ZONE_LIST
},
891 { "zones list", OPT::ZONE_LIST
},
892 { "zone rename", OPT::ZONE_RENAME
},
893 { "zone default", OPT::ZONE_DEFAULT
},
894 { "zone placement add", OPT::ZONE_PLACEMENT_ADD
},
895 { "zone placement modify", OPT::ZONE_PLACEMENT_MODIFY
},
896 { "zone placement rm", OPT::ZONE_PLACEMENT_RM
},
897 { "zone placement list", OPT::ZONE_PLACEMENT_LIST
},
898 { "zone placement get", OPT::ZONE_PLACEMENT_GET
},
899 { "caps add", OPT::CAPS_ADD
},
900 { "caps rm", OPT::CAPS_RM
},
901 { "metadata get [*]", OPT::METADATA_GET
},
902 { "metadata put [*]", OPT::METADATA_PUT
},
903 { "metadata rm [*]", OPT::METADATA_RM
},
904 { "metadata list [*]", OPT::METADATA_LIST
},
905 { "metadata sync status", OPT::METADATA_SYNC_STATUS
},
906 { "metadata sync init", OPT::METADATA_SYNC_INIT
},
907 { "metadata sync run", OPT::METADATA_SYNC_RUN
},
908 { "mdlog list", OPT::MDLOG_LIST
},
909 { "mdlog autotrim", OPT::MDLOG_AUTOTRIM
},
910 { "mdlog trim", OPT::MDLOG_TRIM
},
911 { "mdlog fetch", OPT::MDLOG_FETCH
},
912 { "mdlog status", OPT::MDLOG_STATUS
},
913 { "sync error list", OPT::SYNC_ERROR_LIST
},
914 { "sync error trim", OPT::SYNC_ERROR_TRIM
},
915 { "sync policy get", OPT::SYNC_POLICY_GET
},
916 { "sync group create", OPT::SYNC_GROUP_CREATE
},
917 { "sync group modify", OPT::SYNC_GROUP_MODIFY
},
918 { "sync group get", OPT::SYNC_GROUP_GET
},
919 { "sync group remove", OPT::SYNC_GROUP_REMOVE
},
920 { "sync group flow create", OPT::SYNC_GROUP_FLOW_CREATE
},
921 { "sync group flow remove", OPT::SYNC_GROUP_FLOW_REMOVE
},
922 { "sync group pipe create", OPT::SYNC_GROUP_PIPE_CREATE
},
923 { "sync group pipe modify", OPT::SYNC_GROUP_PIPE_MODIFY
},
924 { "sync group pipe remove", OPT::SYNC_GROUP_PIPE_REMOVE
},
925 { "bilog list", OPT::BILOG_LIST
},
926 { "bilog trim", OPT::BILOG_TRIM
},
927 { "bilog status", OPT::BILOG_STATUS
},
928 { "bilog autotrim", OPT::BILOG_AUTOTRIM
},
929 { "data sync status", OPT::DATA_SYNC_STATUS
},
930 { "data sync init", OPT::DATA_SYNC_INIT
},
931 { "data sync run", OPT::DATA_SYNC_RUN
},
932 { "datalog list", OPT::DATALOG_LIST
},
933 { "datalog status", OPT::DATALOG_STATUS
},
934 { "datalog autotrim", OPT::DATALOG_AUTOTRIM
},
935 { "datalog trim", OPT::DATALOG_TRIM
},
936 { "datalog type", OPT::DATALOG_TYPE
},
937 { "datalog prune", OPT::DATALOG_PRUNE
},
938 { "realm create", OPT::REALM_CREATE
},
939 { "realm delete", OPT::REALM_DELETE
},
940 { "realm get", OPT::REALM_GET
},
941 { "realm get default", OPT::REALM_GET_DEFAULT
},
942 { "realm get-default", OPT::REALM_GET_DEFAULT
},
943 { "realm list", OPT::REALM_LIST
},
944 { "realm list periods", OPT::REALM_LIST_PERIODS
},
945 { "realm list-periods", OPT::REALM_LIST_PERIODS
},
946 { "realm rename", OPT::REALM_RENAME
},
947 { "realm set", OPT::REALM_SET
},
948 { "realm default", OPT::REALM_DEFAULT
},
949 { "realm pull", OPT::REALM_PULL
},
950 { "period delete", OPT::PERIOD_DELETE
},
951 { "period get", OPT::PERIOD_GET
},
952 { "period get-current", OPT::PERIOD_GET_CURRENT
},
953 { "period get current", OPT::PERIOD_GET_CURRENT
},
954 { "period pull", OPT::PERIOD_PULL
},
955 { "period push", OPT::PERIOD_PUSH
},
956 { "period list", OPT::PERIOD_LIST
},
957 { "period update", OPT::PERIOD_UPDATE
},
958 { "period commit", OPT::PERIOD_COMMIT
},
959 { "global quota get", OPT::GLOBAL_QUOTA_GET
},
960 { "global quota set", OPT::GLOBAL_QUOTA_SET
},
961 { "global quota enable", OPT::GLOBAL_QUOTA_ENABLE
},
962 { "global quota disable", OPT::GLOBAL_QUOTA_DISABLE
},
963 { "sync info", OPT::SYNC_INFO
},
964 { "sync status", OPT::SYNC_STATUS
},
965 { "role create", OPT::ROLE_CREATE
},
966 { "role delete", OPT::ROLE_DELETE
},
967 { "role get", OPT::ROLE_GET
},
968 { "role modify", OPT::ROLE_MODIFY
},
969 { "role list", OPT::ROLE_LIST
},
970 { "role policy put", OPT::ROLE_POLICY_PUT
},
971 { "role-policy put", OPT::ROLE_POLICY_PUT
},
972 { "role policy list", OPT::ROLE_POLICY_LIST
},
973 { "role-policy list", OPT::ROLE_POLICY_LIST
},
974 { "role policy get", OPT::ROLE_POLICY_GET
},
975 { "role-policy get", OPT::ROLE_POLICY_GET
},
976 { "role policy delete", OPT::ROLE_POLICY_DELETE
},
977 { "role-policy delete", OPT::ROLE_POLICY_DELETE
},
978 { "reshard bucket", OPT::BUCKET_RESHARD
},
979 { "reshard add", OPT::RESHARD_ADD
},
980 { "reshard list", OPT::RESHARD_LIST
},
981 { "reshard status", OPT::RESHARD_STATUS
},
982 { "reshard process", OPT::RESHARD_PROCESS
},
983 { "reshard cancel", OPT::RESHARD_CANCEL
},
984 { "mfa create", OPT::MFA_CREATE
},
985 { "mfa remove", OPT::MFA_REMOVE
},
986 { "mfa get", OPT::MFA_GET
},
987 { "mfa list", OPT::MFA_LIST
},
988 { "mfa check", OPT::MFA_CHECK
},
989 { "mfa resync", OPT::MFA_RESYNC
},
990 { "reshard stale-instances list", OPT::RESHARD_STALE_INSTANCES_LIST
},
991 { "reshard stale list", OPT::RESHARD_STALE_INSTANCES_LIST
},
992 { "reshard stale-instances delete", OPT::RESHARD_STALE_INSTANCES_DELETE
},
993 { "reshard stale delete", OPT::RESHARD_STALE_INSTANCES_DELETE
},
994 { "topic list", OPT::PUBSUB_TOPICS_LIST
},
995 { "topic get", OPT::PUBSUB_TOPIC_GET
},
996 { "topic rm", OPT::PUBSUB_TOPIC_RM
},
997 { "subscription get", OPT::PUBSUB_SUB_GET
},
998 { "subscription rm", OPT::PUBSUB_SUB_RM
},
999 { "subscription pull", OPT::PUBSUB_SUB_PULL
},
1000 { "subscription ack", OPT::PUBSUB_EVENT_RM
},
1001 { "script put", OPT::SCRIPT_PUT
},
1002 { "script get", OPT::SCRIPT_GET
},
1003 { "script rm", OPT::SCRIPT_RM
},
1004 { "script-package add", OPT::SCRIPT_PACKAGE_ADD
},
1005 { "script-package rm", OPT::SCRIPT_PACKAGE_RM
},
1006 { "script-package list", OPT::SCRIPT_PACKAGE_LIST
},
1009 static SimpleCmd::Aliases cmd_aliases
= {
1010 { "delete", "del" },
1017 BIIndexType
get_bi_index_type(const string
& type_str
) {
1018 if (type_str
== "plain")
1019 return BIIndexType::Plain
;
1020 if (type_str
== "instance")
1021 return BIIndexType::Instance
;
1022 if (type_str
== "olh")
1023 return BIIndexType::OLH
;
1025 return BIIndexType::Invalid
;
1028 log_type
get_log_type(const string
& type_str
) {
1029 if (strcasecmp(type_str
.c_str(), "fifo") == 0)
1030 return log_type::fifo
;
1031 if (strcasecmp(type_str
.c_str(), "omap") == 0)
1032 return log_type::omap
;
1034 return static_cast<log_type
>(0xff);
1037 void dump_bi_entry(bufferlist
& bl
, BIIndexType index_type
, Formatter
*formatter
)
1039 auto iter
= bl
.cbegin();
1040 switch (index_type
) {
1041 case BIIndexType::Plain
:
1042 case BIIndexType::Instance
:
1044 rgw_bucket_dir_entry entry
;
1045 decode(entry
, iter
);
1046 encode_json("entry", entry
, formatter
);
1049 case BIIndexType::OLH
:
1051 rgw_bucket_olh_entry entry
;
1052 decode(entry
, iter
);
1053 encode_json("entry", entry
, formatter
);
1062 static void show_user_info(RGWUserInfo
& info
, Formatter
*formatter
)
1064 encode_json("user_info", info
, formatter
);
1065 formatter
->flush(cout
);
1069 static void show_perm_policy(string perm_policy
, Formatter
* formatter
)
1071 formatter
->open_object_section("role");
1072 formatter
->dump_string("Permission policy", perm_policy
);
1073 formatter
->close_section();
1074 formatter
->flush(cout
);
1077 static void show_policy_names(std::vector
<string
> policy_names
, Formatter
* formatter
)
1079 formatter
->open_array_section("PolicyNames");
1080 for (const auto& it
: policy_names
) {
1081 formatter
->dump_string("policyname", it
);
1083 formatter
->close_section();
1084 formatter
->flush(cout
);
1087 static void show_role_info(RGWRole
& role
, Formatter
* formatter
)
1089 formatter
->open_object_section("role");
1090 role
.dump(formatter
);
1091 formatter
->close_section();
1092 formatter
->flush(cout
);
1095 static void show_roles_info(vector
<RGWRole
>& roles
, Formatter
* formatter
)
1097 formatter
->open_array_section("Roles");
1098 for (const auto& it
: roles
) {
1099 formatter
->open_object_section("role");
1101 formatter
->close_section();
1103 formatter
->close_section();
1104 formatter
->flush(cout
);
1107 static void show_reshard_status(
1108 const list
<cls_rgw_bucket_instance_entry
>& status
, Formatter
*formatter
)
1110 formatter
->open_array_section("status");
1111 for (const auto& entry
: status
) {
1112 formatter
->open_object_section("entry");
1113 formatter
->dump_string("reshard_status", to_string(entry
.reshard_status
));
1114 formatter
->dump_string("new_bucket_instance_id",
1115 entry
.new_bucket_instance_id
);
1116 formatter
->dump_int("num_shards", entry
.num_shards
);
1117 formatter
->close_section();
1119 formatter
->close_section();
1120 formatter
->flush(cout
);
1123 class StoreDestructor
{
1124 rgw::sal::RGWRadosStore
*store
;
1126 explicit StoreDestructor(rgw::sal::RGWRadosStore
*_s
) : store(_s
) {}
1127 ~StoreDestructor() {
1128 RGWStoreManager::close_storage(store
);
1129 rgw_http_client_cleanup();
1133 static int init_bucket(const string
& tenant_name
,
1134 const string
& bucket_name
,
1135 const string
& bucket_id
,
1136 RGWBucketInfo
& bucket_info
,
1138 map
<string
, bufferlist
> *pattrs
= nullptr)
1140 if (!bucket_name
.empty()) {
1141 auto obj_ctx
= store
->svc()->sysobj
->init_obj_ctx();
1143 if (bucket_id
.empty()) {
1144 r
= store
->getRados()->get_bucket_info(store
->svc(), tenant_name
, bucket_name
, bucket_info
, nullptr, null_yield
, pattrs
);
1146 string bucket_instance_id
= bucket_name
+ ":" + bucket_id
;
1147 r
= store
->getRados()->get_bucket_instance_info(obj_ctx
, bucket_instance_id
, bucket_info
, NULL
, pattrs
, null_yield
);
1150 cerr
<< "could not get bucket info for bucket=" << bucket_name
<< std::endl
;
1153 bucket
= bucket_info
.bucket
;
1158 static int init_bucket(const rgw_bucket
& b
,
1159 RGWBucketInfo
& bucket_info
,
1161 map
<string
, bufferlist
> *pattrs
= nullptr)
1163 return init_bucket(b
.tenant
, b
.name
, b
.bucket_id
,
1164 bucket_info
, bucket
, pattrs
);
1167 static int read_input(const string
& infile
, bufferlist
& bl
)
1170 if (infile
.size()) {
1171 fd
= open(infile
.c_str(), O_RDONLY
);
1174 cerr
<< "error reading input file " << infile
<< std::endl
;
1179 #define READ_CHUNK 8196
1184 char buf
[READ_CHUNK
];
1186 r
= safe_read(fd
, buf
, READ_CHUNK
);
1189 cerr
<< "error while reading input" << std::endl
;
1197 if (infile
.size()) {
1204 static int read_decode_json(const string
& infile
, T
& t
)
1207 int ret
= read_input(infile
, bl
);
1209 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
1213 if (!p
.parse(bl
.c_str(), bl
.length())) {
1214 cout
<< "failed to parse JSON" << std::endl
;
1219 decode_json_obj(t
, &p
);
1220 } catch (const JSONDecoder::err
& e
) {
1221 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
1227 template <class T
, class K
>
1228 static int read_decode_json(const string
& infile
, T
& t
, K
*k
)
1231 int ret
= read_input(infile
, bl
);
1233 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
1237 if (!p
.parse(bl
.c_str(), bl
.length())) {
1238 cout
<< "failed to parse JSON" << std::endl
;
1243 t
.decode_json(&p
, k
);
1244 } catch (const JSONDecoder::err
& e
) {
1245 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
1252 static bool decode_dump(const char *field_name
, bufferlist
& bl
, Formatter
*f
)
1256 auto iter
= bl
.cbegin();
1260 } catch (buffer::error
& err
) {
1264 encode_json(field_name
, t
, f
);
1269 static bool dump_string(const char *field_name
, bufferlist
& bl
, Formatter
*f
)
1271 string val
= bl
.to_str();
1272 f
->dump_string(field_name
, val
.c_str() /* hide encoded null termination chars */);
1277 void set_quota_info(RGWQuotaInfo
& quota
, OPT opt_cmd
, int64_t max_size
, int64_t max_objects
,
1278 bool have_max_size
, bool have_max_objects
)
1281 case OPT::QUOTA_ENABLE
:
1282 case OPT::GLOBAL_QUOTA_ENABLE
:
1283 quota
.enabled
= true;
1285 // falling through on purpose
1287 case OPT::QUOTA_SET
:
1288 case OPT::GLOBAL_QUOTA_SET
:
1289 if (have_max_objects
) {
1290 if (max_objects
< 0) {
1291 quota
.max_objects
= -1;
1293 quota
.max_objects
= max_objects
;
1296 if (have_max_size
) {
1298 quota
.max_size
= -1;
1300 quota
.max_size
= rgw_rounded_kb(max_size
) * 1024;
1304 case OPT::QUOTA_DISABLE
:
1305 case OPT::GLOBAL_QUOTA_DISABLE
:
1306 quota
.enabled
= false;
1313 int set_bucket_quota(rgw::sal::RGWRadosStore
*store
, OPT opt_cmd
,
1314 const string
& tenant_name
, const string
& bucket_name
,
1315 int64_t max_size
, int64_t max_objects
,
1316 bool have_max_size
, bool have_max_objects
)
1318 RGWBucketInfo bucket_info
;
1319 map
<string
, bufferlist
> attrs
;
1320 int r
= store
->getRados()->get_bucket_info(store
->svc(), tenant_name
, bucket_name
, bucket_info
, NULL
, null_yield
, &attrs
);
1322 cerr
<< "could not get bucket info for bucket=" << bucket_name
<< ": " << cpp_strerror(-r
) << std::endl
;
1326 set_quota_info(bucket_info
.quota
, opt_cmd
, max_size
, max_objects
, have_max_size
, have_max_objects
);
1328 r
= store
->getRados()->put_bucket_instance_info(bucket_info
, false, real_time(), &attrs
);
1330 cerr
<< "ERROR: failed writing bucket instance info: " << cpp_strerror(-r
) << std::endl
;
1336 int set_user_bucket_quota(OPT opt_cmd
, RGWUser
& user
, RGWUserAdminOpState
& op_state
, int64_t max_size
, int64_t max_objects
,
1337 bool have_max_size
, bool have_max_objects
)
1339 RGWUserInfo
& user_info
= op_state
.get_user_info();
1341 set_quota_info(user_info
.bucket_quota
, opt_cmd
, max_size
, max_objects
, have_max_size
, have_max_objects
);
1343 op_state
.set_bucket_quota(user_info
.bucket_quota
);
1346 int r
= user
.modify(op_state
, null_yield
, &err
);
1348 cerr
<< "ERROR: failed updating user info: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
1354 int set_user_quota(OPT opt_cmd
, RGWUser
& user
, RGWUserAdminOpState
& op_state
, int64_t max_size
, int64_t max_objects
,
1355 bool have_max_size
, bool have_max_objects
)
1357 RGWUserInfo
& user_info
= op_state
.get_user_info();
1359 set_quota_info(user_info
.user_quota
, opt_cmd
, max_size
, max_objects
, have_max_size
, have_max_objects
);
1361 op_state
.set_user_quota(user_info
.user_quota
);
1364 int r
= user
.modify(op_state
, null_yield
, &err
);
1366 cerr
<< "ERROR: failed updating user info: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
1372 int check_min_obj_stripe_size(rgw::sal::RGWRadosStore
*store
, RGWBucketInfo
& bucket_info
, rgw::sal::RGWObject
* obj
, uint64_t min_stripe_size
, bool *need_rewrite
)
1374 RGWObjectCtx
obj_ctx(store
);
1375 int ret
= obj
->get_obj_attrs(&obj_ctx
, null_yield
);
1377 lderr(store
->ctx()) << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret
) << dendl
;
1381 map
<string
, bufferlist
>::iterator iter
;
1382 iter
= obj
->get_attrs().find(RGW_ATTR_MANIFEST
);
1383 if (iter
== obj
->get_attrs().end()) {
1384 *need_rewrite
= (obj
->get_obj_size() >= min_stripe_size
);
1388 RGWObjManifest manifest
;
1391 bufferlist
& bl
= iter
->second
;
1392 auto biter
= bl
.cbegin();
1393 decode(manifest
, biter
);
1394 } catch (buffer::error
& err
) {
1395 ldout(store
->ctx(), 0) << "ERROR: failed to decode manifest" << dendl
;
1399 map
<uint64_t, RGWObjManifestPart
>& objs
= manifest
.get_explicit_objs();
1400 map
<uint64_t, RGWObjManifestPart
>::iterator oiter
;
1401 for (oiter
= objs
.begin(); oiter
!= objs
.end(); ++oiter
) {
1402 RGWObjManifestPart
& part
= oiter
->second
;
1404 if (part
.size
>= min_stripe_size
) {
1405 *need_rewrite
= true;
1409 *need_rewrite
= false;
1415 int check_obj_locator_underscore(RGWBucketInfo
& bucket_info
, rgw_obj
& obj
, rgw_obj_key
& key
, bool fix
, bool remove_bad
, Formatter
*f
) {
1416 f
->open_object_section("object");
1417 f
->open_object_section("key");
1418 f
->dump_string("type", "head");
1419 f
->dump_string("name", key
.name
);
1420 f
->dump_string("instance", key
.instance
);
1426 get_obj_bucket_and_oid_loc(obj
, oid
, locator
);
1428 f
->dump_string("oid", oid
);
1429 f
->dump_string("locator", locator
);
1432 RGWObjectCtx
obj_ctx(store
);
1434 RGWRados::Object
op_target(store
->getRados(), bucket_info
, obj_ctx
, obj
);
1435 RGWRados::Object::Read
read_op(&op_target
);
1437 int ret
= read_op
.prepare(null_yield
);
1438 bool needs_fixing
= (ret
== -ENOENT
);
1440 f
->dump_bool("needs_fixing", needs_fixing
);
1442 string status
= (needs_fixing
? "needs_fixing" : "ok");
1444 if ((needs_fixing
|| remove_bad
) && fix
) {
1445 ret
= store
->getRados()->fix_head_obj_locator(bucket_info
, needs_fixing
, remove_bad
, key
);
1447 cerr
<< "ERROR: fix_head_object_locator() returned ret=" << ret
<< std::endl
;
1454 f
->dump_string("status", status
);
1461 int check_obj_tail_locator_underscore(RGWBucketInfo
& bucket_info
, rgw_obj
& obj
, rgw_obj_key
& key
, bool fix
, Formatter
*f
) {
1462 f
->open_object_section("object");
1463 f
->open_object_section("key");
1464 f
->dump_string("type", "tail");
1465 f
->dump_string("name", key
.name
);
1466 f
->dump_string("instance", key
.instance
);
1472 int ret
= store
->getRados()->fix_tail_obj_locator(bucket_info
, key
, fix
, &needs_fixing
, null_yield
);
1474 cerr
<< "ERROR: fix_tail_object_locator_underscore() returned ret=" << ret
<< std::endl
;
1477 status
= (needs_fixing
&& !fix
? "needs_fixing" : "ok");
1480 f
->dump_bool("needs_fixing", needs_fixing
);
1481 f
->dump_string("status", status
);
1488 int do_check_object_locator(const string
& tenant_name
, const string
& bucket_name
,
1489 bool fix
, bool remove_bad
, Formatter
*f
)
1491 if (remove_bad
&& !fix
) {
1492 cerr
<< "ERROR: can't have remove_bad specified without fix" << std::endl
;
1496 RGWBucketInfo bucket_info
;
1500 f
->open_object_section("bucket");
1501 f
->dump_string("bucket", bucket_name
);
1502 int ret
= init_bucket(tenant_name
, bucket_name
, bucket_id
, bucket_info
, bucket
);
1504 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
1510 int max_entries
= 1000;
1514 vector
<rgw_bucket_dir_entry
> result
;
1515 map
<string
, bool> common_prefixes
;
1518 RGWRados::Bucket
target(store
->getRados(), bucket_info
);
1519 RGWRados::Bucket::List
list_op(&target
);
1523 list_op
.params
.prefix
= prefix
;
1524 list_op
.params
.delim
= delim
;
1525 list_op
.params
.marker
= rgw_obj_key(marker
);
1526 list_op
.params
.ns
= ns
;
1527 list_op
.params
.enforce_ns
= true;
1528 list_op
.params
.list_versions
= true;
1530 f
->open_array_section("check_objects");
1532 ret
= list_op
.list_objects(max_entries
- count
, &result
, &common_prefixes
, &truncated
, null_yield
);
1534 cerr
<< "ERROR: store->list_objects(): " << cpp_strerror(-ret
) << std::endl
;
1538 count
+= result
.size();
1540 for (vector
<rgw_bucket_dir_entry
>::iterator iter
= result
.begin(); iter
!= result
.end(); ++iter
) {
1541 rgw_obj_key key
= iter
->key
;
1542 rgw_obj
obj(bucket
, key
);
1544 if (key
.name
[0] == '_') {
1545 ret
= check_obj_locator_underscore(bucket_info
, obj
, key
, fix
, remove_bad
, f
);
1548 ret
= check_obj_tail_locator_underscore(bucket_info
, obj
, key
, fix
, f
);
1550 cerr
<< "ERROR: check_obj_tail_locator_underscore(): " << cpp_strerror(-ret
) << std::endl
;
1557 } while (truncated
&& count
< max_entries
);
1566 /// search for a matching zone/zonegroup id and return a connection if found
1567 static boost::optional
<RGWRESTConn
> get_remote_conn(rgw::sal::RGWRadosStore
*store
,
1568 const RGWZoneGroup
& zonegroup
,
1569 const std::string
& remote
)
1571 boost::optional
<RGWRESTConn
> conn
;
1572 if (remote
== zonegroup
.get_id()) {
1573 conn
.emplace(store
->ctx(), store
->svc()->zone
, remote
, zonegroup
.endpoints
);
1575 for (const auto& z
: zonegroup
.zones
) {
1576 const auto& zone
= z
.second
;
1577 if (remote
== zone
.id
) {
1578 conn
.emplace(store
->ctx(), store
->svc()->zone
, remote
, zone
.endpoints
);
1586 /// search each zonegroup for a connection
1587 static boost::optional
<RGWRESTConn
> get_remote_conn(rgw::sal::RGWRadosStore
*store
,
1588 const RGWPeriodMap
& period_map
,
1589 const std::string
& remote
)
1591 boost::optional
<RGWRESTConn
> conn
;
1592 for (const auto& zg
: period_map
.zonegroups
) {
1593 conn
= get_remote_conn(store
, zg
.second
, remote
);
1601 // we expect a very small response
1602 static constexpr size_t MAX_REST_RESPONSE
= 128 * 1024;
1604 static int send_to_remote_gateway(RGWRESTConn
* conn
, req_info
& info
,
1605 bufferlist
& in_data
, JSONParser
& parser
)
1611 ceph::bufferlist response
;
1613 int ret
= conn
->forward(user
, info
, nullptr, MAX_REST_RESPONSE
, &in_data
, &response
, null_yield
);
1615 int parse_ret
= parser
.parse(response
.c_str(), response
.length());
1616 if (parse_ret
< 0) {
1617 cerr
<< "failed to parse response" << std::endl
;
1623 static int send_to_url(const string
& url
, const string
& access
,
1624 const string
& secret
, req_info
& info
,
1625 bufferlist
& in_data
, JSONParser
& parser
)
1627 if (access
.empty() || secret
.empty()) {
1628 cerr
<< "An --access-key and --secret must be provided with --url." << std::endl
;
1636 RGWRESTSimpleRequest
req(g_ceph_context
, info
.method
, url
, NULL
, ¶ms
);
1638 bufferlist response
;
1639 int ret
= req
.forward_request(key
, info
, MAX_REST_RESPONSE
, &in_data
, &response
, null_yield
);
1641 int parse_ret
= parser
.parse(response
.c_str(), response
.length());
1642 if (parse_ret
< 0) {
1643 cout
<< "failed to parse response" << std::endl
;
1649 static int send_to_remote_or_url(RGWRESTConn
*conn
, const string
& url
,
1650 const string
& access
, const string
& secret
,
1651 req_info
& info
, bufferlist
& in_data
,
1655 return send_to_remote_gateway(conn
, info
, in_data
, parser
);
1657 return send_to_url(url
, access
, secret
, info
, in_data
, parser
);
1660 static int commit_period(RGWRealm
& realm
, RGWPeriod
& period
,
1661 string remote
, const string
& url
,
1662 const string
& access
, const string
& secret
,
1665 auto& master_zone
= period
.get_master_zone();
1666 if (master_zone
.empty()) {
1667 cerr
<< "cannot commit period: period does not have a master zone of a master zonegroup" << std::endl
;
1670 // are we the period's master zone?
1671 if (store
->svc()->zone
->zone_id() == master_zone
) {
1672 // read the current period
1673 RGWPeriod current_period
;
1674 int ret
= current_period
.init(g_ceph_context
,
1675 store
->svc()->sysobj
, realm
.get_id(),
1678 cerr
<< "Error initializing current period: "
1679 << cpp_strerror(-ret
) << std::endl
;
1682 // the master zone can commit locally
1683 ret
= period
.commit(store
, realm
, current_period
, cerr
, null_yield
, force
);
1685 cerr
<< "failed to commit period: " << cpp_strerror(-ret
) << std::endl
;
1690 if (remote
.empty() && url
.empty()) {
1691 // use the new master zone's connection
1692 remote
= master_zone
.id
;
1693 cerr
<< "Sending period to new master zone " << remote
<< std::endl
;
1695 boost::optional
<RGWRESTConn
> conn
;
1696 RGWRESTConn
*remote_conn
= nullptr;
1697 if (!remote
.empty()) {
1698 conn
= get_remote_conn(store
, period
.get_map(), remote
);
1700 cerr
<< "failed to find a zone or zonegroup for remote "
1701 << remote
<< std::endl
;
1704 remote_conn
= &*conn
;
1707 // push period to the master with an empty period id
1708 period
.set_id(string());
1711 req_info
info(g_ceph_context
, &env
);
1712 info
.method
= "POST";
1713 info
.request_uri
= "/admin/realm/period";
1715 // json format into a bufferlist
1716 JSONFormatter
jf(false);
1717 encode_json("period", period
, &jf
);
1722 int ret
= send_to_remote_or_url(remote_conn
, url
, access
, secret
, info
, bl
, p
);
1724 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
1726 // did we parse an error message?
1727 auto message
= p
.find_obj("Message");
1729 cerr
<< "Reason: " << message
->get_data() << std::endl
;
1734 // decode the response and store it back
1736 decode_json_obj(period
, &p
);
1737 } catch (const JSONDecoder::err
& e
) {
1738 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
1741 if (period
.get_id().empty()) {
1742 cerr
<< "Period commit got back an empty period id" << std::endl
;
1745 // the master zone gave us back the period that it committed, so it's
1746 // safe to save it as our latest epoch
1747 ret
= period
.store_info(false, null_yield
);
1749 cerr
<< "Error storing committed period " << period
.get_id() << ": "
1750 << cpp_strerror(ret
) << std::endl
;
1753 ret
= period
.set_latest_epoch(null_yield
, period
.get_epoch());
1755 cerr
<< "Error updating period epoch: " << cpp_strerror(ret
) << std::endl
;
1758 ret
= period
.reflect(null_yield
);
1760 cerr
<< "Error updating local objects: " << cpp_strerror(ret
) << std::endl
;
1763 realm
.notify_new_period(period
, null_yield
);
1767 static int update_period(const string
& realm_id
, const string
& realm_name
,
1768 const string
& period_id
, const string
& period_epoch
,
1769 bool commit
, const string
& remote
, const string
& url
,
1770 const string
& access
, const string
& secret
,
1771 Formatter
*formatter
, bool force
)
1773 RGWRealm
realm(realm_id
, realm_name
);
1774 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
1776 cerr
<< "Error initializing realm " << cpp_strerror(-ret
) << std::endl
;
1780 if (!period_epoch
.empty()) {
1781 epoch
= atoi(period_epoch
.c_str());
1783 RGWPeriod
period(period_id
, epoch
);
1784 ret
= period
.init(g_ceph_context
, store
->svc()->sysobj
, realm
.get_id(), null_yield
);
1786 cerr
<< "period init failed: " << cpp_strerror(-ret
) << std::endl
;
1790 ret
= period
.update(null_yield
);
1792 // Dropping the error message here, as both the ret codes were handled in
1796 ret
= period
.store_info(false, null_yield
);
1798 cerr
<< "failed to store period: " << cpp_strerror(-ret
) << std::endl
;
1802 ret
= commit_period(realm
, period
, remote
, url
, access
, secret
, force
);
1804 cerr
<< "failed to commit period: " << cpp_strerror(-ret
) << std::endl
;
1808 encode_json("period", period
, formatter
);
1809 formatter
->flush(cout
);
1813 static int init_bucket_for_sync(const string
& tenant
, const string
& bucket_name
,
1814 const string
& bucket_id
, rgw_bucket
& bucket
)
1816 RGWBucketInfo bucket_info
;
1818 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
1820 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
1827 static int do_period_pull(RGWRESTConn
*remote_conn
, const string
& url
,
1828 const string
& access_key
, const string
& secret_key
,
1829 const string
& realm_id
, const string
& realm_name
,
1830 const string
& period_id
, const string
& period_epoch
,
1834 req_info
info(g_ceph_context
, &env
);
1835 info
.method
= "GET";
1836 info
.request_uri
= "/admin/realm/period";
1838 map
<string
, string
> ¶ms
= info
.args
.get_params();
1839 if (!realm_id
.empty())
1840 params
["realm_id"] = realm_id
;
1841 if (!realm_name
.empty())
1842 params
["realm_name"] = realm_name
;
1843 if (!period_id
.empty())
1844 params
["period_id"] = period_id
;
1845 if (!period_epoch
.empty())
1846 params
["epoch"] = period_epoch
;
1850 int ret
= send_to_remote_or_url(remote_conn
, url
, access_key
, secret_key
,
1853 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
1856 ret
= period
->init(g_ceph_context
, store
->svc()->sysobj
, null_yield
, false);
1858 cerr
<< "faile to init period " << cpp_strerror(-ret
) << std::endl
;
1862 decode_json_obj(*period
, &p
);
1863 } catch (const JSONDecoder::err
& e
) {
1864 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
1867 ret
= period
->store_info(false, null_yield
);
1869 cerr
<< "Error storing period " << period
->get_id() << ": " << cpp_strerror(ret
) << std::endl
;
1871 // store latest epoch (ignore errors)
1872 period
->update_latest_epoch(period
->get_epoch(), null_yield
);
1876 static int read_current_period_id(rgw::sal::RGWRadosStore
* store
, const std::string
& realm_id
,
1877 const std::string
& realm_name
,
1878 std::string
* period_id
)
1880 RGWRealm
realm(realm_id
, realm_name
);
1881 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
1883 std::cerr
<< "failed to read realm: " << cpp_strerror(-ret
) << std::endl
;
1886 *period_id
= realm
.get_current_period();
1890 void flush_ss(stringstream
& ss
, list
<string
>& l
)
1892 if (!ss
.str().empty()) {
1893 l
.push_back(ss
.str());
1898 stringstream
& push_ss(stringstream
& ss
, list
<string
>& l
, int tab
= 0)
1902 ss
<< setw(tab
) << "" << setw(1);
1907 static void get_md_sync_status(list
<string
>& status
)
1909 RGWMetaSyncStatusManager
sync(store
, store
->svc()->rados
->get_async_processor());
1911 int ret
= sync
.init();
1913 status
.push_back(string("failed to retrieve sync info: sync.init() failed: ") + cpp_strerror(-ret
));
1917 rgw_meta_sync_status sync_status
;
1918 ret
= sync
.read_sync_status(&sync_status
);
1920 status
.push_back(string("failed to read sync status: ") + cpp_strerror(-ret
));
1925 switch (sync_status
.sync_info
.state
) {
1926 case rgw_meta_sync_info::StateInit
:
1927 status_str
= "init";
1929 case rgw_meta_sync_info::StateBuildingFullSyncMaps
:
1930 status_str
= "preparing for full sync";
1932 case rgw_meta_sync_info::StateSync
:
1933 status_str
= "syncing";
1936 status_str
= "unknown";
1939 status
.push_back(status_str
);
1941 uint64_t full_total
= 0;
1942 uint64_t full_complete
= 0;
1946 int total_shards
= 0;
1947 set
<int> shards_behind_set
;
1949 for (auto marker_iter
: sync_status
.sync_markers
) {
1950 full_total
+= marker_iter
.second
.total_entries
;
1952 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::FullSync
) {
1954 full_complete
+= marker_iter
.second
.pos
;
1955 int shard_id
= marker_iter
.first
;
1956 shards_behind_set
.insert(shard_id
);
1958 full_complete
+= marker_iter
.second
.total_entries
;
1960 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::IncrementalSync
) {
1966 push_ss(ss
, status
) << "full sync: " << num_full
<< "/" << total_shards
<< " shards";
1969 push_ss(ss
, status
) << "full sync: " << full_total
- full_complete
<< " entries to sync";
1972 push_ss(ss
, status
) << "incremental sync: " << num_inc
<< "/" << total_shards
<< " shards";
1974 map
<int, RGWMetadataLogInfo
> master_shards_info
;
1975 string master_period
= store
->svc()->zone
->get_current_period_id();
1977 ret
= sync
.read_master_log_shards_info(master_period
, &master_shards_info
);
1979 status
.push_back(string("failed to fetch master sync status: ") + cpp_strerror(-ret
));
1983 map
<int, string
> shards_behind
;
1984 if (sync_status
.sync_info
.period
!= master_period
) {
1985 status
.push_back(string("master is on a different period: master_period=" +
1986 master_period
+ " local_period=" + sync_status
.sync_info
.period
));
1988 for (auto local_iter
: sync_status
.sync_markers
) {
1989 int shard_id
= local_iter
.first
;
1990 auto iter
= master_shards_info
.find(shard_id
);
1992 if (iter
== master_shards_info
.end()) {
1994 derr
<< "ERROR: could not find remote sync shard status for shard_id=" << shard_id
<< dendl
;
1997 auto master_marker
= iter
->second
.marker
;
1998 if (local_iter
.second
.state
== rgw_meta_sync_marker::SyncState::IncrementalSync
&&
1999 master_marker
> local_iter
.second
.marker
) {
2000 shards_behind
[shard_id
] = local_iter
.second
.marker
;
2001 shards_behind_set
.insert(shard_id
);
2006 int total_behind
= shards_behind
.size() + (sync_status
.sync_info
.num_shards
- num_inc
);
2007 if (total_behind
== 0) {
2008 push_ss(ss
, status
) << "metadata is caught up with master";
2010 push_ss(ss
, status
) << "metadata is behind on " << total_behind
<< " shards";
2012 push_ss(ss
, status
) << "behind shards: " << "[" << shards_behind_set
<< "]";
2014 map
<int, rgw_mdlog_shard_data
> master_pos
;
2015 ret
= sync
.read_master_log_shards_next(sync_status
.sync_info
.period
, shards_behind
, &master_pos
);
2017 derr
<< "ERROR: failed to fetch master next positions (" << cpp_strerror(-ret
) << ")" << dendl
;
2019 std::optional
<std::pair
<int, ceph::real_time
>> oldest
;
2021 for (auto iter
: master_pos
) {
2022 rgw_mdlog_shard_data
& shard_data
= iter
.second
;
2024 if (!shard_data
.entries
.empty()) {
2025 rgw_mdlog_entry
& entry
= shard_data
.entries
.front();
2027 oldest
.emplace(iter
.first
, entry
.timestamp
);
2028 } else if (!ceph::real_clock::is_zero(entry
.timestamp
) && entry
.timestamp
< oldest
->second
) {
2029 oldest
.emplace(iter
.first
, entry
.timestamp
);
2035 push_ss(ss
, status
) << "oldest incremental change not applied: "
2036 << oldest
->second
<< " [" << oldest
->first
<< ']';
2041 flush_ss(ss
, status
);
2044 static void get_data_sync_status(const rgw_zone_id
& source_zone
, list
<string
>& status
, int tab
)
2050 if (!store
->svc()->zone
->find_zone(source_zone
, &sz
)) {
2051 push_ss(ss
, status
, tab
) << string("zone not found");
2052 flush_ss(ss
, status
);
2056 if (!store
->svc()->zone
->zone_syncs_from(store
->svc()->zone
->get_zone(), *sz
)) {
2057 push_ss(ss
, status
, tab
) << string("not syncing from zone");
2058 flush_ss(ss
, status
);
2061 RGWDataSyncStatusManager
sync(store
, store
->svc()->rados
->get_async_processor(), source_zone
, nullptr);
2063 int ret
= sync
.init();
2065 push_ss(ss
, status
, tab
) << string("failed to retrieve sync info: ") + cpp_strerror(-ret
);
2066 flush_ss(ss
, status
);
2070 rgw_data_sync_status sync_status
;
2071 ret
= sync
.read_sync_status(&sync_status
);
2072 if (ret
< 0 && ret
!= -ENOENT
) {
2073 push_ss(ss
, status
, tab
) << string("failed read sync status: ") + cpp_strerror(-ret
);
2077 set
<int> recovering_shards
;
2078 ret
= sync
.read_recovering_shards(sync_status
.sync_info
.num_shards
, recovering_shards
);
2079 if (ret
< 0 && ret
!= ENOENT
) {
2080 push_ss(ss
, status
, tab
) << string("failed read recovering shards: ") + cpp_strerror(-ret
);
2085 switch (sync_status
.sync_info
.state
) {
2086 case rgw_data_sync_info::StateInit
:
2087 status_str
= "init";
2089 case rgw_data_sync_info::StateBuildingFullSyncMaps
:
2090 status_str
= "preparing for full sync";
2092 case rgw_data_sync_info::StateSync
:
2093 status_str
= "syncing";
2096 status_str
= "unknown";
2099 push_ss(ss
, status
, tab
) << status_str
;
2101 uint64_t full_total
= 0;
2102 uint64_t full_complete
= 0;
2106 int total_shards
= 0;
2107 set
<int> shards_behind_set
;
2109 for (auto marker_iter
: sync_status
.sync_markers
) {
2110 full_total
+= marker_iter
.second
.total_entries
;
2112 if (marker_iter
.second
.state
== rgw_data_sync_marker::SyncState::FullSync
) {
2114 full_complete
+= marker_iter
.second
.pos
;
2115 int shard_id
= marker_iter
.first
;
2116 shards_behind_set
.insert(shard_id
);
2118 full_complete
+= marker_iter
.second
.total_entries
;
2120 if (marker_iter
.second
.state
== rgw_data_sync_marker::SyncState::IncrementalSync
) {
2125 push_ss(ss
, status
, tab
) << "full sync: " << num_full
<< "/" << total_shards
<< " shards";
2128 push_ss(ss
, status
, tab
) << "full sync: " << full_total
- full_complete
<< " buckets to sync";
2131 push_ss(ss
, status
, tab
) << "incremental sync: " << num_inc
<< "/" << total_shards
<< " shards";
2133 map
<int, RGWDataChangesLogInfo
> source_shards_info
;
2135 ret
= sync
.read_source_log_shards_info(&source_shards_info
);
2137 push_ss(ss
, status
, tab
) << string("failed to fetch source sync status: ") + cpp_strerror(-ret
);
2141 map
<int, string
> shards_behind
;
2143 for (auto local_iter
: sync_status
.sync_markers
) {
2144 int shard_id
= local_iter
.first
;
2145 auto iter
= source_shards_info
.find(shard_id
);
2147 if (iter
== source_shards_info
.end()) {
2149 derr
<< "ERROR: could not find remote sync shard status for shard_id=" << shard_id
<< dendl
;
2152 auto master_marker
= iter
->second
.marker
;
2153 if (local_iter
.second
.state
== rgw_data_sync_marker::SyncState::IncrementalSync
&&
2154 master_marker
> local_iter
.second
.marker
) {
2155 shards_behind
[shard_id
] = local_iter
.second
.marker
;
2156 shards_behind_set
.insert(shard_id
);
2160 int total_behind
= shards_behind
.size() + (sync_status
.sync_info
.num_shards
- num_inc
);
2161 int total_recovering
= recovering_shards
.size();
2162 if (total_behind
== 0 && total_recovering
== 0) {
2163 push_ss(ss
, status
, tab
) << "data is caught up with source";
2164 } else if (total_behind
> 0) {
2165 push_ss(ss
, status
, tab
) << "data is behind on " << total_behind
<< " shards";
2167 push_ss(ss
, status
, tab
) << "behind shards: " << "[" << shards_behind_set
<< "]" ;
2169 map
<int, rgw_datalog_shard_data
> master_pos
;
2170 ret
= sync
.read_source_log_shards_next(shards_behind
, &master_pos
);
2172 derr
<< "ERROR: failed to fetch next positions (" << cpp_strerror(-ret
) << ")" << dendl
;
2174 std::optional
<std::pair
<int, ceph::real_time
>> oldest
;
2176 for (auto iter
: master_pos
) {
2177 rgw_datalog_shard_data
& shard_data
= iter
.second
;
2179 if (!shard_data
.entries
.empty()) {
2180 rgw_datalog_entry
& entry
= shard_data
.entries
.front();
2182 oldest
.emplace(iter
.first
, entry
.timestamp
);
2183 } else if (!ceph::real_clock::is_zero(entry
.timestamp
) && entry
.timestamp
< oldest
->second
) {
2184 oldest
.emplace(iter
.first
, entry
.timestamp
);
2190 push_ss(ss
, status
, tab
) << "oldest incremental change not applied: "
2191 << oldest
->second
<< " [" << oldest
->first
<< ']';
2196 if (total_recovering
> 0) {
2197 push_ss(ss
, status
, tab
) << total_recovering
<< " shards are recovering";
2198 push_ss(ss
, status
, tab
) << "recovering shards: " << "[" << recovering_shards
<< "]";
2201 flush_ss(ss
, status
);
2204 static void tab_dump(const string
& header
, int width
, const list
<string
>& entries
)
2208 for (auto e
: entries
) {
2209 cout
<< std::setw(width
) << s
<< std::setw(1) << " " << e
<< std::endl
;
2215 static void sync_status(Formatter
*formatter
)
2217 const RGWRealm
& realm
= store
->svc()->zone
->get_realm();
2218 const RGWZoneGroup
& zonegroup
= store
->svc()->zone
->get_zonegroup();
2219 const RGWZone
& zone
= store
->svc()->zone
->get_zone();
2223 cout
<< std::setw(width
) << "realm" << std::setw(1) << " " << realm
.get_id() << " (" << realm
.get_name() << ")" << std::endl
;
2224 cout
<< std::setw(width
) << "zonegroup" << std::setw(1) << " " << zonegroup
.get_id() << " (" << zonegroup
.get_name() << ")" << std::endl
;
2225 cout
<< std::setw(width
) << "zone" << std::setw(1) << " " << zone
.id
<< " (" << zone
.name
<< ")" << std::endl
;
2227 list
<string
> md_status
;
2229 if (store
->svc()->zone
->is_meta_master()) {
2230 md_status
.push_back("no sync (zone is master)");
2232 get_md_sync_status(md_status
);
2235 tab_dump("metadata sync", width
, md_status
);
2237 list
<string
> data_status
;
2239 auto& zone_conn_map
= store
->svc()->zone
->get_zone_conn_map();
2241 for (auto iter
: zone_conn_map
) {
2242 const rgw_zone_id
& source_id
= iter
.first
;
2243 string source_str
= "source: ";
2244 string s
= source_str
+ source_id
.id
;
2246 if (store
->svc()->zone
->find_zone(source_id
, &sz
)) {
2247 s
+= string(" (") + sz
->name
+ ")";
2249 data_status
.push_back(s
);
2250 get_data_sync_status(source_id
, data_status
, source_str
.size());
2253 tab_dump("data sync", width
, data_status
);
2257 int w
; // indent width
2258 std::string_view header
;
2259 indented(int w
, std::string_view header
= "") : w(w
), header(header
) {}
2261 std::ostream
& operator<<(std::ostream
& out
, const indented
& h
) {
2262 return out
<< std::setw(h
.w
) << h
.header
<< std::setw(1) << ' ';
2265 static int bucket_source_sync_status(rgw::sal::RGWRadosStore
*store
, const RGWZone
& zone
,
2266 const RGWZone
& source
, RGWRESTConn
*conn
,
2267 const RGWBucketInfo
& bucket_info
,
2268 rgw_sync_bucket_pipe pipe
,
2269 int width
, std::ostream
& out
)
2271 out
<< indented
{width
, "source zone"} << source
.id
<< " (" << source
.name
<< ")" << std::endl
;
2273 // syncing from this zone?
2274 if (!zone
.syncs_from(source
.name
)) {
2275 out
<< indented
{width
} << "does not sync from zone\n";
2279 if (!pipe
.source
.bucket
) {
2280 lderr(store
->ctx()) << __func__
<< "(): missing source bucket" << dendl
;
2284 RGWBucketInfo source_bucket_info
;
2285 rgw_bucket source_bucket
;
2286 int r
= init_bucket(*pipe
.source
.bucket
, source_bucket_info
, source_bucket
);
2288 lderr(store
->ctx()) << "failed to read source bucket info: " << cpp_strerror(r
) << dendl
;
2292 pipe
.source
.bucket
= source_bucket
;
2293 pipe
.dest
.bucket
= bucket_info
.bucket
;
2295 std::vector
<rgw_bucket_shard_sync_info
> status
;
2296 r
= rgw_bucket_sync_status(dpp(), store
, pipe
, bucket_info
, &source_bucket_info
, &status
);
2298 lderr(store
->ctx()) << "failed to read bucket sync status: " << cpp_strerror(r
) << dendl
;
2302 out
<< indented
{width
, "source bucket"} << source_bucket_info
.bucket
.get_key() << std::endl
;
2306 uint64_t full_complete
= 0;
2307 const size_t total_shards
= status
.size();
2309 using BucketSyncState
= rgw_bucket_shard_sync_info::SyncState
;
2310 for (size_t shard_id
= 0; shard_id
< total_shards
; shard_id
++) {
2311 auto& m
= status
[shard_id
];
2312 if (m
.state
== BucketSyncState::StateFullSync
) {
2314 full_complete
+= m
.full_marker
.count
;
2315 } else if (m
.state
== BucketSyncState::StateIncrementalSync
) {
2320 out
<< indented
{width
} << "full sync: " << num_full
<< "/" << total_shards
<< " shards\n";
2322 out
<< indented
{width
} << "full sync: " << full_complete
<< " objects completed\n";
2324 out
<< indented
{width
} << "incremental sync: " << num_inc
<< "/" << total_shards
<< " shards\n";
2326 BucketIndexShardsManager remote_markers
;
2327 r
= rgw_read_remote_bilog_info(conn
, source_bucket
, remote_markers
, null_yield
);
2329 lderr(store
->ctx()) << "failed to read remote log: " << cpp_strerror(r
) << dendl
;
2333 std::set
<int> shards_behind
;
2334 for (auto& r
: remote_markers
.get()) {
2335 auto shard_id
= r
.first
;
2336 auto& m
= status
[shard_id
];
2337 if (r
.second
.empty()) {
2338 continue; // empty bucket index shard
2340 auto pos
= BucketIndexShardsManager::get_shard_marker(m
.inc_marker
.position
);
2341 if (m
.state
!= BucketSyncState::StateIncrementalSync
|| pos
!= r
.second
) {
2342 shards_behind
.insert(shard_id
);
2345 if (!shards_behind
.empty()) {
2346 out
<< indented
{width
} << "bucket is behind on " << shards_behind
.size() << " shards\n";
2347 out
<< indented
{width
} << "behind shards: [" << shards_behind
<< "]\n" ;
2348 } else if (!num_full
) {
2349 out
<< indented
{width
} << "bucket is caught up with source\n";
2354 void encode_json(const char *name
, const RGWBucketSyncFlowManager::pipe_set
& pset
, Formatter
*f
)
2356 Formatter::ObjectSection
top_section(*f
, name
);
2357 Formatter::ArraySection
as(*f
, "entries");
2359 for (auto& pipe_handler
: pset
) {
2360 Formatter::ObjectSection
hs(*f
, "handler");
2361 encode_json("source", pipe_handler
.source
, f
);
2362 encode_json("dest", pipe_handler
.dest
, f
);
2366 static std::vector
<string
> convert_bucket_set_to_str_vec(const std::set
<rgw_bucket
>& bs
)
2368 std::vector
<string
> result
;
2369 result
.reserve(bs
.size());
2370 for (auto& b
: bs
) {
2371 result
.push_back(b
.get_key());
2376 static void get_hint_entities(const std::set
<rgw_zone_id
>& zones
, const std::set
<rgw_bucket
>& buckets
,
2377 std::set
<rgw_sync_bucket_entity
> *hint_entities
)
2379 for (auto& zone_id
: zones
) {
2380 for (auto& b
: buckets
) {
2381 RGWBucketInfo hint_bucket_info
;
2382 rgw_bucket hint_bucket
;
2383 int ret
= init_bucket(b
, hint_bucket_info
, hint_bucket
);
2385 ldout(store
->ctx(), 20) << "could not init bucket info for hint bucket=" << b
<< " ... skipping" << dendl
;
2389 hint_entities
->insert(rgw_sync_bucket_entity(zone_id
, hint_bucket
));
2394 static rgw_zone_id
resolve_zone_id(const string
& s
)
2399 if (store
->svc()->zone
->find_zone(s
, &zone
)) {
2400 return rgw_zone_id(s
);
2402 if (store
->svc()->zone
->find_zone_id_by_name(s
, &result
)) {
2405 return rgw_zone_id(s
);
2408 rgw_zone_id
validate_zone_id(const rgw_zone_id
& zone_id
)
2410 return resolve_zone_id(zone_id
.id
);
2413 static int sync_info(std::optional
<rgw_zone_id
> opt_target_zone
, std::optional
<rgw_bucket
> opt_bucket
, Formatter
*formatter
)
2415 rgw_zone_id zone_id
= opt_target_zone
.value_or(store
->svc()->zone
->zone_id());
2417 auto zone_policy_handler
= store
->svc()->zone
->get_sync_policy_handler(zone_id
);
2419 RGWBucketSyncPolicyHandlerRef bucket_handler
;
2421 std::optional
<rgw_bucket
> eff_bucket
= opt_bucket
;
2423 auto handler
= zone_policy_handler
;
2427 RGWBucketInfo bucket_info
;
2428 map
<string
, bufferlist
> bucket_attrs
;
2430 int ret
= init_bucket(*eff_bucket
, bucket_info
, bucket
, &bucket_attrs
);
2431 if (ret
< 0 && ret
!= -ENOENT
) {
2432 cerr
<< "ERROR: init_bucket failed: " << cpp_strerror(-ret
) << std::endl
;
2437 bucket_handler
.reset(handler
->alloc_child(bucket_info
, std::move(bucket_attrs
)));
2439 cerr
<< "WARNING: bucket not found, simulating result" << std::endl
;
2440 bucket_handler
.reset(handler
->alloc_child(*eff_bucket
, nullopt
));
2443 ret
= bucket_handler
->init(null_yield
);
2445 cerr
<< "ERROR: failed to init bucket sync policy handler: " << cpp_strerror(-ret
) << " (ret=" << ret
<< ")" << std::endl
;
2449 handler
= bucket_handler
;
2452 std::set
<rgw_sync_bucket_pipe
> sources
;
2453 std::set
<rgw_sync_bucket_pipe
> dests
;
2455 handler
->get_pipes(&sources
, &dests
, std::nullopt
);
2457 auto source_hints_vec
= convert_bucket_set_to_str_vec(handler
->get_source_hints());
2458 auto target_hints_vec
= convert_bucket_set_to_str_vec(handler
->get_target_hints());
2460 std::set
<rgw_sync_bucket_pipe
> resolved_sources
;
2461 std::set
<rgw_sync_bucket_pipe
> resolved_dests
;
2463 rgw_sync_bucket_entity
self_entity(zone_id
, opt_bucket
);
2465 set
<rgw_zone_id
> source_zones
;
2466 set
<rgw_zone_id
> target_zones
;
2468 zone_policy_handler
->reflect(nullptr, nullptr,
2472 false); /* relaxed: also get all zones that we allow to sync to/from */
2474 std::set
<rgw_sync_bucket_entity
> hint_entities
;
2476 get_hint_entities(source_zones
, handler
->get_source_hints(), &hint_entities
);
2477 get_hint_entities(target_zones
, handler
->get_target_hints(), &hint_entities
);
2479 for (auto& hint_entity
: hint_entities
) {
2480 if (!hint_entity
.zone
||
2481 !hint_entity
.bucket
) {
2482 continue; /* shouldn't really happen */
2485 auto zid
= validate_zone_id(*hint_entity
.zone
);
2486 auto& hint_bucket
= *hint_entity
.bucket
;
2488 RGWBucketSyncPolicyHandlerRef hint_bucket_handler
;
2489 int r
= store
->ctl()->bucket
->get_sync_policy_handler(zid
, hint_bucket
, &hint_bucket_handler
, null_yield
);
2491 ldout(store
->ctx(), 20) << "could not get bucket sync policy handler for hint bucket=" << hint_bucket
<< " ... skipping" << dendl
;
2495 hint_bucket_handler
->get_pipes(&resolved_dests
,
2497 self_entity
); /* flipping resolved dests and sources as these are
2498 relative to the remote entity */
2502 Formatter::ObjectSection
os(*formatter
, "result");
2503 encode_json("sources", sources
, formatter
);
2504 encode_json("dests", dests
, formatter
);
2506 Formatter::ObjectSection
hints_section(*formatter
, "hints");
2507 encode_json("sources", source_hints_vec
, formatter
);
2508 encode_json("dests", target_hints_vec
, formatter
);
2511 Formatter::ObjectSection
resolved_hints_section(*formatter
, "resolved-hints-1");
2512 encode_json("sources", resolved_sources
, formatter
);
2513 encode_json("dests", resolved_dests
, formatter
);
2516 Formatter::ObjectSection
resolved_hints_section(*formatter
, "resolved-hints");
2517 encode_json("sources", handler
->get_resolved_source_hints(), formatter
);
2518 encode_json("dests", handler
->get_resolved_dest_hints(), formatter
);
2522 formatter
->flush(cout
);
2527 static int bucket_sync_info(rgw::sal::RGWRadosStore
*store
, const RGWBucketInfo
& info
,
2530 const RGWRealm
& realm
= store
->svc()->zone
->get_realm();
2531 const RGWZoneGroup
& zonegroup
= store
->svc()->zone
->get_zonegroup();
2532 const RGWZone
& zone
= store
->svc()->zone
->get_zone();
2533 constexpr int width
= 15;
2535 out
<< indented
{width
, "realm"} << realm
.get_id() << " (" << realm
.get_name() << ")\n";
2536 out
<< indented
{width
, "zonegroup"} << zonegroup
.get_id() << " (" << zonegroup
.get_name() << ")\n";
2537 out
<< indented
{width
, "zone"} << zone
.id
<< " (" << zone
.name
<< ")\n";
2538 out
<< indented
{width
, "bucket"} << info
.bucket
<< "\n\n";
2540 if (!store
->ctl()->bucket
->bucket_imports_data(info
.bucket
, null_yield
)) {
2541 out
<< "Sync is disabled for bucket " << info
.bucket
.name
<< '\n';
2545 RGWBucketSyncPolicyHandlerRef handler
;
2547 int r
= store
->ctl()->bucket
->get_sync_policy_handler(std::nullopt
, info
.bucket
, &handler
, null_yield
);
2549 lderr(store
->ctx()) << "ERROR: failed to get policy handler for bucket (" << info
.bucket
<< "): r=" << r
<< ": " << cpp_strerror(-r
) << dendl
;
2553 auto& sources
= handler
->get_sources();
2555 for (auto& m
: sources
) {
2556 auto& zone
= m
.first
;
2557 out
<< indented
{width
, "source zone"} << zone
<< std::endl
;
2558 for (auto& pipe_handler
: m
.second
) {
2559 out
<< indented
{width
, "bucket"} << *pipe_handler
.source
.bucket
<< std::endl
;
2566 static int bucket_sync_status(rgw::sal::RGWRadosStore
*store
, const RGWBucketInfo
& info
,
2567 const rgw_zone_id
& source_zone_id
,
2568 std::optional
<rgw_bucket
>& opt_source_bucket
,
2571 const RGWRealm
& realm
= store
->svc()->zone
->get_realm();
2572 const RGWZoneGroup
& zonegroup
= store
->svc()->zone
->get_zonegroup();
2573 const RGWZone
& zone
= store
->svc()->zone
->get_zone();
2574 constexpr int width
= 15;
2576 out
<< indented
{width
, "realm"} << realm
.get_id() << " (" << realm
.get_name() << ")\n";
2577 out
<< indented
{width
, "zonegroup"} << zonegroup
.get_id() << " (" << zonegroup
.get_name() << ")\n";
2578 out
<< indented
{width
, "zone"} << zone
.id
<< " (" << zone
.name
<< ")\n";
2579 out
<< indented
{width
, "bucket"} << info
.bucket
<< "\n\n";
2581 if (!store
->ctl()->bucket
->bucket_imports_data(info
.bucket
, null_yield
)) {
2582 out
<< "Sync is disabled for bucket " << info
.bucket
.name
<< " or bucket has no sync sources" << std::endl
;
2586 RGWBucketSyncPolicyHandlerRef handler
;
2588 int r
= store
->ctl()->bucket
->get_sync_policy_handler(std::nullopt
, info
.bucket
, &handler
, null_yield
);
2590 lderr(store
->ctx()) << "ERROR: failed to get policy handler for bucket (" << info
.bucket
<< "): r=" << r
<< ": " << cpp_strerror(-r
) << dendl
;
2594 auto sources
= handler
->get_all_sources();
2596 auto& zone_conn_map
= store
->svc()->zone
->get_zone_conn_map();
2597 set
<rgw_zone_id
> zone_ids
;
2599 if (!source_zone_id
.empty()) {
2600 auto z
= zonegroup
.zones
.find(source_zone_id
);
2601 if (z
== zonegroup
.zones
.end()) {
2602 lderr(store
->ctx()) << "Source zone not found in zonegroup "
2603 << zonegroup
.get_name() << dendl
;
2606 auto c
= zone_conn_map
.find(source_zone_id
);
2607 if (c
== zone_conn_map
.end()) {
2608 lderr(store
->ctx()) << "No connection to zone " << z
->second
.name
<< dendl
;
2611 zone_ids
.insert(source_zone_id
);
2613 for (const auto& entry
: zonegroup
.zones
) {
2614 zone_ids
.insert(entry
.second
.id
);
2618 for (auto& zone_id
: zone_ids
) {
2619 auto z
= zonegroup
.zones
.find(zone_id
.id
);
2620 if (z
== zonegroup
.zones
.end()) { /* should't happen */
2623 auto c
= zone_conn_map
.find(zone_id
.id
);
2624 if (c
== zone_conn_map
.end()) { /* should't happen */
2628 for (auto& entry
: sources
) {
2629 auto& pipe
= entry
.second
;
2630 if (opt_source_bucket
&&
2631 pipe
.source
.bucket
!= opt_source_bucket
) {
2634 if (pipe
.source
.zone
.value_or(rgw_zone_id()) == z
->second
.id
) {
2635 bucket_source_sync_status(store
, zone
, z
->second
,
2646 static void parse_tier_config_param(const string
& s
, map
<string
, string
, ltstr_nocase
>& out
)
2654 confs
.push_back(cur_conf
);
2661 } else if (c
== '}') {
2666 if (!cur_conf
.empty()) {
2667 confs
.push_back(cur_conf
);
2670 for (auto c
: confs
) {
2671 ssize_t pos
= c
.find("=");
2675 out
[c
.substr(0, pos
)] = c
.substr(pos
+ 1);
2680 static int check_pool_support_omap(const rgw_pool
& pool
)
2682 librados::IoCtx io_ctx
;
2683 int ret
= store
->getRados()->get_rados_handle()->ioctx_create(pool
.to_str().c_str(), io_ctx
);
2685 // the pool may not exist at this moment, we have no way to check if it supports omap.
2689 ret
= io_ctx
.omap_clear("__omap_test_not_exist_oid__");
2690 if (ret
== -EOPNOTSUPP
) {
2698 int check_reshard_bucket_params(rgw::sal::RGWRadosStore
*store
,
2699 const string
& bucket_name
,
2700 const string
& tenant
,
2701 const string
& bucket_id
,
2702 bool num_shards_specified
,
2704 int yes_i_really_mean_it
,
2706 RGWBucketInfo
& bucket_info
,
2707 map
<string
, bufferlist
>& attrs
)
2709 if (bucket_name
.empty()) {
2710 cerr
<< "ERROR: bucket not specified" << std::endl
;
2714 if (!num_shards_specified
) {
2715 cerr
<< "ERROR: --num-shards not specified" << std::endl
;
2719 if (num_shards
> (int)store
->getRados()->get_max_bucket_shards()) {
2720 cerr
<< "ERROR: num_shards too high, max value: " << store
->getRados()->get_max_bucket_shards() << std::endl
;
2724 if (num_shards
< 0) {
2725 cerr
<< "ERROR: num_shards must be non-negative integer" << std::endl
;
2729 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
, &attrs
);
2731 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
2735 if (bucket_info
.reshard_status
!= cls_rgw_reshard_status::NOT_RESHARDING
) {
2736 // if in_progress or done then we have an old BucketInfo
2737 cerr
<< "ERROR: the bucket is currently undergoing resharding and "
2738 "cannot be added to the reshard list at this time" << std::endl
;
2742 int num_source_shards
= (bucket_info
.layout
.current_index
.layout
.normal
.num_shards
> 0 ? bucket_info
.layout
.current_index
.layout
.normal
.num_shards
: 1);
2744 if (num_shards
<= num_source_shards
&& !yes_i_really_mean_it
) {
2745 cerr
<< "num shards is less or equal to current shards count" << std::endl
2746 << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
2752 static int scan_totp(CephContext
*cct
, ceph::real_time
& now
, rados::cls::otp::otp_info_t
& totp
, vector
<string
>& pins
,
2755 #define MAX_TOTP_SKEW_HOURS (24 * 7)
2756 ceph_assert(pins
.size() == 2);
2758 time_t start_time
= ceph::real_clock::to_time_t(now
);
2759 time_t time_ofs
= 0, time_ofs_abs
= 0;
2760 time_t step_size
= totp
.step_size
;
2761 if (step_size
== 0) {
2762 step_size
= OATH_TOTP_DEFAULT_TIME_STEP_SIZE
;
2767 uint32_t max_skew
= MAX_TOTP_SKEW_HOURS
* 3600;
2769 while (time_ofs_abs
< max_skew
) {
2770 int rc
= oath_totp_validate2(totp
.seed_bin
.c_str(), totp
.seed_bin
.length(),
2777 if (rc
!= OATH_INVALID_OTP
) {
2778 rc
= oath_totp_validate2(totp
.seed_bin
.c_str(), totp
.seed_bin
.length(),
2781 time_ofs
- step_size
, /* smaller time_ofs moves time forward */
2785 if (rc
!= OATH_INVALID_OTP
) {
2786 *pofs
= time_ofs
- step_size
+ step_size
* totp
.window
/ 2;
2787 ldout(cct
, 20) << "found at time=" << start_time
- time_ofs
<< " time_ofs=" << time_ofs
<< dendl
;
2792 time_ofs_abs
= (++count
) * step_size
;
2793 time_ofs
= sign
* time_ofs_abs
;
2799 static int trim_sync_error_log(int shard_id
, const string
& marker
, int delay_ms
)
2801 auto oid
= RGWSyncErrorLogger::get_shard_oid(RGW_SYNC_ERROR_LOG_SHARD_PREFIX
,
2803 // call cls_log_trim() until it returns -ENODATA
2805 int ret
= store
->svc()->cls
->timelog
.trim(oid
, {}, {}, {}, marker
, nullptr,
2807 if (ret
== -ENODATA
) {
2814 std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms
));
2820 const string
& get_tier_type(rgw::sal::RGWRadosStore
*store
) {
2821 return store
->svc()->zone
->get_zone().tier_type
;
2824 static bool symmetrical_flow_opt(const string
& opt
)
2826 return (opt
== "symmetrical" || opt
== "symmetric");
2829 static bool directional_flow_opt(const string
& opt
)
2831 return (opt
== "directional" || opt
== "direction");
2835 static bool require_opt(std::optional
<T
> opt
, bool extra_check
= true)
2837 if (!opt
|| !extra_check
) {
2844 static bool require_non_empty_opt(std::optional
<T
> opt
, bool extra_check
= true)
2846 if (!opt
|| opt
->empty() || !extra_check
) {
2853 static void show_result(T
& obj
,
2854 Formatter
*formatter
,
2857 encode_json("obj", obj
, formatter
);
2859 formatter
->flush(cout
);
2862 void init_optional_bucket(std::optional
<rgw_bucket
>& opt_bucket
,
2863 std::optional
<string
>& opt_tenant
,
2864 std::optional
<string
>& opt_bucket_name
,
2865 std::optional
<string
>& opt_bucket_id
)
2867 if (opt_tenant
|| opt_bucket_name
|| opt_bucket_id
) {
2868 opt_bucket
.emplace();
2870 opt_bucket
->tenant
= *opt_tenant
;
2872 if (opt_bucket_name
) {
2873 opt_bucket
->name
= *opt_bucket_name
;
2875 if (opt_bucket_id
) {
2876 opt_bucket
->bucket_id
= *opt_bucket_id
;
2881 class SyncPolicyContext
2883 RGWZoneGroup zonegroup
;
2885 std::optional
<rgw_bucket
> bucket
;
2886 RGWBucketInfo bucket_info
;
2887 map
<string
, bufferlist
> bucket_attrs
;
2889 rgw_sync_policy_info
*policy
{nullptr};
2891 std::optional
<rgw_user
> owner
;
2894 SyncPolicyContext(const string
& zonegroup_id
,
2895 const string
& zonegroup_name
,
2896 std::optional
<rgw_bucket
> _bucket
) : zonegroup(zonegroup_id
, zonegroup_name
),
2900 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
2902 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
2907 policy
= &zonegroup
.sync_policy
;
2911 ret
= init_bucket(*bucket
, bucket_info
, *bucket
, &bucket_attrs
);
2913 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
2917 owner
= bucket_info
.owner
;
2919 if (!bucket_info
.sync_policy
) {
2920 rgw_sync_policy_info new_policy
;
2921 bucket_info
.set_sync_policy(std::move(new_policy
));
2924 policy
= &(*bucket_info
.sync_policy
);
2929 int write_policy() {
2931 int ret
= zonegroup
.update(null_yield
);
2933 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
2939 int ret
= store
->getRados()->put_bucket_instance_info(bucket_info
, false, real_time(), &bucket_attrs
);
2941 cerr
<< "failed to store bucket info: " << cpp_strerror(-ret
) << std::endl
;
2948 rgw_sync_policy_info
& get_policy() {
2952 std::optional
<rgw_user
>& get_owner() {
2957 void resolve_zone_id_opt(std::optional
<string
>& zone_name
, std::optional
<rgw_zone_id
>& zone_id
)
2959 if (!zone_name
|| zone_id
) {
2963 if (!store
->svc()->zone
->find_zone_id_by_name(*zone_name
, &(*zone_id
))) {
2964 cerr
<< "WARNING: cannot find source zone id for name=" << *zone_name
<< std::endl
;
2965 zone_id
= rgw_zone_id(*zone_name
);
2968 void resolve_zone_ids_opt(std::optional
<vector
<string
> >& names
, std::optional
<vector
<rgw_zone_id
> >& ids
)
2970 if (!names
|| ids
) {
2974 for (auto& name
: *names
) {
2976 if (!store
->svc()->zone
->find_zone_id_by_name(name
, &zid
)) {
2977 cerr
<< "WARNING: cannot find source zone id for name=" << name
<< std::endl
;
2978 zid
= rgw_zone_id(name
);
2980 ids
->push_back(zid
);
2984 static vector
<rgw_zone_id
> zone_ids_from_str(const string
& val
)
2986 vector
<rgw_zone_id
> result
;
2988 get_str_vec(val
, v
);
2990 result
.push_back(rgw_zone_id(z
));
2995 class JSONFormatter_PrettyZone
: public JSONFormatter
{
2996 class Handler
: public JSONEncodeFilter::Handler
<rgw_zone_id
> {
2997 void encode_json(const char *name
, const void *pval
, ceph::Formatter
*f
) const override
{
2998 auto zone_id
= *(static_cast<const rgw_zone_id
*>(pval
));
3001 if (store
->svc()->zone
->find_zone(zone_id
, &zone
)) {
3002 zone_name
= zone
->name
;
3004 cerr
<< "WARNING: cannot find zone name for id=" << zone_id
<< std::endl
;
3005 zone_name
= zone_id
.id
;
3008 ::encode_json(name
, zone_name
, f
);
3010 } zone_id_type_handler
;
3012 JSONEncodeFilter encode_filter
;
3014 JSONFormatter_PrettyZone(bool pretty_format
) : JSONFormatter(pretty_format
) {
3015 encode_filter
.register_type(&zone_id_type_handler
);
3018 void *get_external_feature_handler(const std::string
& feature
) override
{
3019 if (feature
!= "JSONEncodeFilter") {
3022 return &encode_filter
;
3026 int main(int argc
, const char **argv
)
3028 vector
<const char*> args
;
3029 argv_to_vec(argc
, (const char **)argv
, args
);
3031 cerr
<< argv
[0] << ": -h or --help for usage" << std::endl
;
3034 if (ceph_argparse_need_usage(args
)) {
3039 auto cct
= global_init(NULL
, args
, CEPH_ENTITY_TYPE_CLIENT
,
3040 CODE_ENVIRONMENT_UTILITY
, 0);
3042 // for region -> zonegroup conversion (must happen before common_init_finish())
3043 if (!g_conf()->rgw_region
.empty() && g_conf()->rgw_zonegroup
.empty()) {
3044 g_conf().set_val_or_die("rgw_zonegroup", g_conf()->rgw_region
.c_str());
3047 common_init_finish(g_ceph_context
);
3052 rgw_user new_user_id
;
3053 std::string access_key
, secret_key
, user_email
, display_name
;
3054 std::string bucket_name
, pool_name
, object
;
3056 std::string date
, subuser
, access
, format
;
3057 std::string start_date
, end_date
;
3058 std::string key_type_str
;
3059 std::string period_id
, period_epoch
, remote
, url
;
3060 std::string master_zone
;
3061 std::string realm_name
, realm_id
, realm_new_name
;
3062 std::string zone_name
, zone_id
, zone_new_name
;
3063 std::string zonegroup_name
, zonegroup_id
, zonegroup_new_name
;
3064 std::string api_name
;
3065 std::string role_name
, path
, assume_role_doc
, policy_name
, perm_policy_doc
, path_prefix
;
3066 std::string redirect_zone
;
3067 bool redirect_zone_set
= false;
3068 list
<string
> endpoints
;
3070 int sync_from_all_specified
= false;
3071 bool sync_from_all
= false;
3072 list
<string
> sync_from
;
3073 list
<string
> sync_from_rm
;
3075 int set_default
= 0;
3076 bool is_master
= false;
3077 bool is_master_set
= false;
3079 bool read_only
= false;
3080 int is_read_only_set
= false;
3082 int staging
= false;
3083 int key_type
= KEY_TYPE_UNDEFINED
;
3085 uint32_t perm_mask
= 0;
3087 OPT opt_cmd
= OPT::NO_CMD
;
3088 int gen_access_key
= 0;
3089 int gen_secret_key
= 0;
3090 bool set_perm
= false;
3091 bool set_temp_url_key
= false;
3092 map
<int, string
> temp_url_keys
;
3094 string new_bucket_name
;
3095 std::unique_ptr
<Formatter
> formatter
;
3096 std::unique_ptr
<Formatter
> zone_formatter
;
3097 int purge_data
= false;
3098 int pretty_format
= false;
3099 int show_log_entries
= true;
3100 int show_log_sum
= true;
3101 int skip_zero_entries
= false; // log show
3102 int purge_keys
= false;
3103 int yes_i_really_mean_it
= false;
3104 int delete_child_objects
= false;
3106 int remove_bad
= false;
3107 int check_head_obj_locator
= false;
3108 int max_buckets
= -1;
3109 bool max_buckets_specified
= false;
3110 map
<string
, bool> categories
;
3112 int check_objects
= false;
3113 RGWUserAdminOpState user_op
;
3114 RGWBucketAdminOpState bucket_op
;
3116 string metadata_key
;
3117 RGWObjVersionTracker objv_tracker
;
3119 string start_marker
;
3121 int max_entries
= -1;
3122 bool max_entries_specified
= false;
3124 bool admin_specified
= false;
3126 bool system_specified
= false;
3128 bool specified_shard_id
= false;
3133 string object_version
;
3134 string placement_id
;
3135 std::optional
<string
> opt_storage_class
;
3137 list
<string
> tags_add
;
3138 list
<string
> tags_rm
;
3140 int64_t max_objects
= -1;
3141 int64_t max_size
= -1;
3142 bool have_max_objects
= false;
3143 bool have_max_size
= false;
3144 int include_all
= false;
3145 int allow_unordered
= false;
3147 int sync_stats
= false;
3148 int reset_stats
= false;
3149 int bypass_gc
= false;
3150 int warnings_only
= false;
3151 int inconsistent_index
= false;
3153 int verbose
= false;
3155 int extra_info
= false;
3157 uint64_t min_rewrite_size
= 4 * 1024 * 1024;
3158 uint64_t max_rewrite_size
= ULLONG_MAX
;
3159 uint64_t min_rewrite_stripe_size
= 0;
3161 BIIndexType bi_index_type
= BIIndexType::Plain
;
3162 std::optional
<log_type
> opt_log_type
;
3166 bool num_shards_specified
= false;
3167 std::optional
<int> bucket_index_max_shards
;
3168 int max_concurrent_ios
= 32;
3169 uint64_t orphan_stale_secs
= (24 * 3600);
3173 std::ostringstream errs
;
3176 string source_zone_name
;
3177 rgw_zone_id source_zone
; /* zone id */
3180 bool tier_type_specified
= false;
3182 map
<string
, string
, ltstr_nocase
> tier_config_add
;
3183 map
<string
, string
, ltstr_nocase
> tier_config_rm
;
3185 boost::optional
<string
> index_pool
;
3186 boost::optional
<string
> data_pool
;
3187 boost::optional
<string
> data_extra_pool
;
3188 rgw::BucketIndexType placement_index_type
= rgw::BucketIndexType::Normal
;
3189 bool index_type_specified
= false;
3191 boost::optional
<std::string
> compression_type
;
3195 string totp_seed_type
= "hex";
3196 vector
<string
> totp_pin
;
3197 int totp_seconds
= 0;
3198 int totp_window
= 0;
3199 int trim_delay_ms
= 0;
3205 std::optional
<std::string
> str_script_ctx
;
3206 std::optional
<std::string
> script_package
;
3207 int allow_compilation
= false;
3209 std::optional
<string
> opt_group_id
;
3210 std::optional
<string
> opt_status
;
3211 std::optional
<string
> opt_flow_type
;
3212 std::optional
<vector
<string
> > opt_zone_names
;
3213 std::optional
<vector
<rgw_zone_id
> > opt_zone_ids
;
3214 std::optional
<string
> opt_flow_id
;
3215 std::optional
<string
> opt_source_zone_name
;
3216 std::optional
<rgw_zone_id
> opt_source_zone_id
;
3217 std::optional
<string
> opt_dest_zone_name
;
3218 std::optional
<rgw_zone_id
> opt_dest_zone_id
;
3219 std::optional
<vector
<string
> > opt_source_zone_names
;
3220 std::optional
<vector
<rgw_zone_id
> > opt_source_zone_ids
;
3221 std::optional
<vector
<string
> > opt_dest_zone_names
;
3222 std::optional
<vector
<rgw_zone_id
> > opt_dest_zone_ids
;
3223 std::optional
<string
> opt_pipe_id
;
3224 std::optional
<rgw_bucket
> opt_bucket
;
3225 std::optional
<string
> opt_tenant
;
3226 std::optional
<string
> opt_bucket_name
;
3227 std::optional
<string
> opt_bucket_id
;
3228 std::optional
<rgw_bucket
> opt_source_bucket
;
3229 std::optional
<string
> opt_source_tenant
;
3230 std::optional
<string
> opt_source_bucket_name
;
3231 std::optional
<string
> opt_source_bucket_id
;
3232 std::optional
<rgw_bucket
> opt_dest_bucket
;
3233 std::optional
<string
> opt_dest_tenant
;
3234 std::optional
<string
> opt_dest_bucket_name
;
3235 std::optional
<string
> opt_dest_bucket_id
;
3236 std::optional
<string
> opt_effective_zone_name
;
3237 std::optional
<rgw_zone_id
> opt_effective_zone_id
;
3239 std::optional
<string
> opt_prefix
;
3240 std::optional
<string
> opt_prefix_rm
;
3242 std::optional
<int> opt_priority
;
3243 std::optional
<string
> opt_mode
;
3244 std::optional
<rgw_user
> opt_dest_owner
;
3245 ceph::timespan opt_retry_delay_ms
= std::chrono::milliseconds(2000);
3246 ceph::timespan opt_timeout_sec
= std::chrono::seconds(60);
3248 SimpleCmd
cmd(all_cmds
, cmd_aliases
);
3250 std::optional
<std::string
> rgw_obj_fs
; // radoslist field separator
3252 for (std::vector
<const char*>::iterator i
= args
.begin(); i
!= args
.end(); ) {
3253 if (ceph_argparse_double_dash(args
, i
)) {
3255 } else if (ceph_argparse_witharg(args
, i
, &val
, "-i", "--uid", (char*)NULL
)) {
3256 user_id
.from_str(val
);
3257 if (user_id
.empty()) {
3258 cerr
<< "no value for uid" << std::endl
;
3261 } else if (ceph_argparse_witharg(args
, i
, &val
, "-i", "--new-uid", (char*)NULL
)) {
3262 new_user_id
.from_str(val
);
3263 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tenant", (char*)NULL
)) {
3266 } else if (ceph_argparse_witharg(args
, i
, &val
, "--user_ns", (char*)NULL
)) {
3268 } else if (ceph_argparse_witharg(args
, i
, &val
, "--access-key", (char*)NULL
)) {
3270 } else if (ceph_argparse_witharg(args
, i
, &val
, "--subuser", (char*)NULL
)) {
3272 } else if (ceph_argparse_witharg(args
, i
, &val
, "--secret", "--secret-key", (char*)NULL
)) {
3274 } else if (ceph_argparse_witharg(args
, i
, &val
, "-e", "--email", (char*)NULL
)) {
3276 user_op
.user_email_specified
=true;
3277 } else if (ceph_argparse_witharg(args
, i
, &val
, "-n", "--display-name", (char*)NULL
)) {
3279 } else if (ceph_argparse_witharg(args
, i
, &val
, "-b", "--bucket", (char*)NULL
)) {
3281 opt_bucket_name
= val
;
3282 } else if (ceph_argparse_witharg(args
, i
, &val
, "-p", "--pool", (char*)NULL
)) {
3284 pool
= rgw_pool(pool_name
);
3285 } else if (ceph_argparse_witharg(args
, i
, &val
, "-o", "--object", (char*)NULL
)) {
3287 } else if (ceph_argparse_witharg(args
, i
, &val
, "--object-version", (char*)NULL
)) {
3288 object_version
= val
;
3289 } else if (ceph_argparse_witharg(args
, i
, &val
, "--client-id", (char*)NULL
)) {
3291 } else if (ceph_argparse_witharg(args
, i
, &val
, "--op-id", (char*)NULL
)) {
3293 } else if (ceph_argparse_witharg(args
, i
, &val
, "--op-mask", (char*)NULL
)) {
3295 } else if (ceph_argparse_witharg(args
, i
, &val
, "--key-type", (char*)NULL
)) {
3297 if (key_type_str
.compare("swift") == 0) {
3298 key_type
= KEY_TYPE_SWIFT
;
3299 } else if (key_type_str
.compare("s3") == 0) {
3300 key_type
= KEY_TYPE_S3
;
3302 cerr
<< "bad key type: " << key_type_str
<< std::endl
;
3305 } else if (ceph_argparse_witharg(args
, i
, &val
, "--job-id", (char*)NULL
)) {
3307 } else if (ceph_argparse_binary_flag(args
, i
, &gen_access_key
, NULL
, "--gen-access-key", (char*)NULL
)) {
3309 } else if (ceph_argparse_binary_flag(args
, i
, &gen_secret_key
, NULL
, "--gen-secret", (char*)NULL
)) {
3311 } else if (ceph_argparse_binary_flag(args
, i
, &show_log_entries
, NULL
, "--show-log-entries", (char*)NULL
)) {
3313 } else if (ceph_argparse_binary_flag(args
, i
, &show_log_sum
, NULL
, "--show-log-sum", (char*)NULL
)) {
3315 } else if (ceph_argparse_binary_flag(args
, i
, &skip_zero_entries
, NULL
, "--skip-zero-entries", (char*)NULL
)) {
3317 } else if (ceph_argparse_binary_flag(args
, i
, &admin
, NULL
, "--admin", (char*)NULL
)) {
3318 admin_specified
= true;
3319 } else if (ceph_argparse_binary_flag(args
, i
, &system
, NULL
, "--system", (char*)NULL
)) {
3320 system_specified
= true;
3321 } else if (ceph_argparse_binary_flag(args
, i
, &verbose
, NULL
, "--verbose", (char*)NULL
)) {
3323 } else if (ceph_argparse_binary_flag(args
, i
, &staging
, NULL
, "--staging", (char*)NULL
)) {
3325 } else if (ceph_argparse_binary_flag(args
, i
, &commit
, NULL
, "--commit", (char*)NULL
)) {
3327 } else if (ceph_argparse_witharg(args
, i
, &val
, "--min-rewrite-size", (char*)NULL
)) {
3328 min_rewrite_size
= (uint64_t)atoll(val
.c_str());
3329 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-rewrite-size", (char*)NULL
)) {
3330 max_rewrite_size
= (uint64_t)atoll(val
.c_str());
3331 } else if (ceph_argparse_witharg(args
, i
, &val
, "--min-rewrite-stripe-size", (char*)NULL
)) {
3332 min_rewrite_stripe_size
= (uint64_t)atoll(val
.c_str());
3333 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-buckets", (char*)NULL
)) {
3334 max_buckets
= (int)strict_strtol(val
.c_str(), 10, &err
);
3336 cerr
<< "ERROR: failed to parse max buckets: " << err
<< std::endl
;
3339 max_buckets_specified
= true;
3340 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-entries", (char*)NULL
)) {
3341 max_entries
= (int)strict_strtol(val
.c_str(), 10, &err
);
3342 max_entries_specified
= true;
3344 cerr
<< "ERROR: failed to parse max entries: " << err
<< std::endl
;
3347 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-size", (char*)NULL
)) {
3348 max_size
= strict_iec_cast
<long long>(val
.c_str(), &err
);
3350 cerr
<< "ERROR: failed to parse max size: " << err
<< std::endl
;
3353 have_max_size
= true;
3354 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-objects", (char*)NULL
)) {
3355 max_objects
= (int64_t)strict_strtoll(val
.c_str(), 10, &err
);
3357 cerr
<< "ERROR: failed to parse max objects: " << err
<< std::endl
;
3360 have_max_objects
= true;
3361 } else if (ceph_argparse_witharg(args
, i
, &val
, "--date", "--time", (char*)NULL
)) {
3363 if (end_date
.empty())
3365 } else if (ceph_argparse_witharg(args
, i
, &val
, "--start-date", "--start-time", (char*)NULL
)) {
3367 } else if (ceph_argparse_witharg(args
, i
, &val
, "--end-date", "--end-time", (char*)NULL
)) {
3369 } else if (ceph_argparse_witharg(args
, i
, &val
, "--num-shards", (char*)NULL
)) {
3370 num_shards
= (int)strict_strtol(val
.c_str(), 10, &err
);
3372 cerr
<< "ERROR: failed to parse num shards: " << err
<< std::endl
;
3375 num_shards_specified
= true;
3376 } else if (ceph_argparse_witharg(args
, i
, &val
, "--bucket-index-max-shards", (char*)NULL
)) {
3377 bucket_index_max_shards
= (int)strict_strtol(val
.c_str(), 10, &err
);
3379 cerr
<< "ERROR: failed to parse bucket-index-max-shards: " << err
<< std::endl
;
3382 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-concurrent-ios", (char*)NULL
)) {
3383 max_concurrent_ios
= (int)strict_strtol(val
.c_str(), 10, &err
);
3385 cerr
<< "ERROR: failed to parse max concurrent ios: " << err
<< std::endl
;
3388 } else if (ceph_argparse_witharg(args
, i
, &val
, "--orphan-stale-secs", (char*)NULL
)) {
3389 orphan_stale_secs
= (uint64_t)strict_strtoll(val
.c_str(), 10, &err
);
3391 cerr
<< "ERROR: failed to parse orphan stale secs: " << err
<< std::endl
;
3394 } else if (ceph_argparse_witharg(args
, i
, &val
, "--shard-id", (char*)NULL
)) {
3395 shard_id
= (int)strict_strtol(val
.c_str(), 10, &err
);
3397 cerr
<< "ERROR: failed to parse shard id: " << err
<< std::endl
;
3400 specified_shard_id
= true;
3401 } else if (ceph_argparse_witharg(args
, i
, &val
, "--access", (char*)NULL
)) {
3403 perm_mask
= rgw_str_to_perm(access
.c_str());
3405 } else if (ceph_argparse_witharg(args
, i
, &val
, "--temp-url-key", (char*)NULL
)) {
3406 temp_url_keys
[0] = val
;
3407 set_temp_url_key
= true;
3408 } else if (ceph_argparse_witharg(args
, i
, &val
, "--temp-url-key2", "--temp-url-key-2", (char*)NULL
)) {
3409 temp_url_keys
[1] = val
;
3410 set_temp_url_key
= true;
3411 } else if (ceph_argparse_witharg(args
, i
, &val
, "--bucket-id", (char*)NULL
)) {
3413 opt_bucket_id
= val
;
3414 if (bucket_id
.empty()) {
3415 cerr
<< "no value for bucket-id" << std::endl
;
3418 } else if (ceph_argparse_witharg(args
, i
, &val
, "--bucket-new-name", (char*)NULL
)) {
3419 new_bucket_name
= val
;
3420 } else if (ceph_argparse_witharg(args
, i
, &val
, "--format", (char*)NULL
)) {
3422 } else if (ceph_argparse_witharg(args
, i
, &val
, "--categories", (char*)NULL
)) {
3423 string cat_str
= val
;
3424 list
<string
> cat_list
;
3425 list
<string
>::iterator iter
;
3426 get_str_list(cat_str
, cat_list
);
3427 for (iter
= cat_list
.begin(); iter
!= cat_list
.end(); ++iter
) {
3428 categories
[*iter
] = true;
3430 } else if (ceph_argparse_binary_flag(args
, i
, &delete_child_objects
, NULL
, "--purge-objects", (char*)NULL
)) {
3432 } else if (ceph_argparse_binary_flag(args
, i
, &pretty_format
, NULL
, "--pretty-format", (char*)NULL
)) {
3434 } else if (ceph_argparse_binary_flag(args
, i
, &purge_data
, NULL
, "--purge-data", (char*)NULL
)) {
3435 delete_child_objects
= purge_data
;
3436 } else if (ceph_argparse_binary_flag(args
, i
, &purge_keys
, NULL
, "--purge-keys", (char*)NULL
)) {
3438 } else if (ceph_argparse_binary_flag(args
, i
, &yes_i_really_mean_it
, NULL
, "--yes-i-really-mean-it", (char*)NULL
)) {
3440 } else if (ceph_argparse_binary_flag(args
, i
, &fix
, NULL
, "--fix", (char*)NULL
)) {
3442 } else if (ceph_argparse_binary_flag(args
, i
, &remove_bad
, NULL
, "--remove-bad", (char*)NULL
)) {
3444 } else if (ceph_argparse_binary_flag(args
, i
, &check_head_obj_locator
, NULL
, "--check-head-obj-locator", (char*)NULL
)) {
3446 } else if (ceph_argparse_binary_flag(args
, i
, &check_objects
, NULL
, "--check-objects", (char*)NULL
)) {
3448 } else if (ceph_argparse_binary_flag(args
, i
, &sync_stats
, NULL
, "--sync-stats", (char*)NULL
)) {
3450 } else if (ceph_argparse_binary_flag(args
, i
, &reset_stats
, NULL
, "--reset-stats", (char*)NULL
)) {
3452 } else if (ceph_argparse_binary_flag(args
, i
, &include_all
, NULL
, "--include-all", (char*)NULL
)) {
3454 } else if (ceph_argparse_binary_flag(args
, i
, &allow_unordered
, NULL
, "--allow-unordered", (char*)NULL
)) {
3456 } else if (ceph_argparse_binary_flag(args
, i
, &extra_info
, NULL
, "--extra-info", (char*)NULL
)) {
3458 } else if (ceph_argparse_binary_flag(args
, i
, &bypass_gc
, NULL
, "--bypass-gc", (char*)NULL
)) {
3460 } else if (ceph_argparse_binary_flag(args
, i
, &warnings_only
, NULL
, "--warnings-only", (char*)NULL
)) {
3462 } else if (ceph_argparse_binary_flag(args
, i
, &inconsistent_index
, NULL
, "--inconsistent-index", (char*)NULL
)) {
3464 } else if (ceph_argparse_witharg(args
, i
, &val
, "--caps", (char*)NULL
)) {
3466 } else if (ceph_argparse_witharg(args
, i
, &val
, "-i", "--infile", (char*)NULL
)) {
3468 } else if (ceph_argparse_witharg(args
, i
, &val
, "--metadata-key", (char*)NULL
)) {
3470 } else if (ceph_argparse_witharg(args
, i
, &val
, "--marker", (char*)NULL
)) {
3472 } else if (ceph_argparse_witharg(args
, i
, &val
, "--start-marker", (char*)NULL
)) {
3474 } else if (ceph_argparse_witharg(args
, i
, &val
, "--end-marker", (char*)NULL
)) {
3476 } else if (ceph_argparse_witharg(args
, i
, &val
, "--quota-scope", (char*)NULL
)) {
3478 } else if (ceph_argparse_witharg(args
, i
, &val
, "--index-type", (char*)NULL
)) {
3479 string index_type_str
= val
;
3480 bi_index_type
= get_bi_index_type(index_type_str
);
3481 if (bi_index_type
== BIIndexType::Invalid
) {
3482 cerr
<< "ERROR: invalid bucket index entry type" << std::endl
;
3485 } else if (ceph_argparse_witharg(args
, i
, &val
, "--log-type", (char*)NULL
)) {
3486 string log_type_str
= val
;
3487 auto l
= get_log_type(log_type_str
);
3488 if (l
== static_cast<log_type
>(0xff)) {
3489 cerr
<< "ERROR: invalid log type" << std::endl
;
3493 } else if (ceph_argparse_binary_flag(args
, i
, &is_master_int
, NULL
, "--master", (char*)NULL
)) {
3494 is_master
= (bool)is_master_int
;
3495 is_master_set
= true;
3496 } else if (ceph_argparse_binary_flag(args
, i
, &set_default
, NULL
, "--default", (char*)NULL
)) {
3498 } else if (ceph_argparse_witharg(args
, i
, &val
, "--redirect-zone", (char*)NULL
)) {
3499 redirect_zone
= val
;
3500 redirect_zone_set
= true;
3501 } else if (ceph_argparse_binary_flag(args
, i
, &read_only_int
, NULL
, "--read-only", (char*)NULL
)) {
3502 read_only
= (bool)read_only_int
;
3503 is_read_only_set
= true;
3504 } else if (ceph_argparse_witharg(args
, i
, &val
, "--master-zone", (char*)NULL
)) {
3506 } else if (ceph_argparse_witharg(args
, i
, &val
, "--period", (char*)NULL
)) {
3508 } else if (ceph_argparse_witharg(args
, i
, &val
, "--epoch", (char*)NULL
)) {
3510 } else if (ceph_argparse_witharg(args
, i
, &val
, "--remote", (char*)NULL
)) {
3512 } else if (ceph_argparse_witharg(args
, i
, &val
, "--url", (char*)NULL
)) {
3514 } else if (ceph_argparse_witharg(args
, i
, &val
, "--realm-id", (char*)NULL
)) {
3516 } else if (ceph_argparse_witharg(args
, i
, &val
, "--realm-new-name", (char*)NULL
)) {
3517 realm_new_name
= val
;
3518 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zonegroup-id", (char*)NULL
)) {
3520 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zonegroup-new-name", (char*)NULL
)) {
3521 zonegroup_new_name
= val
;
3522 } else if (ceph_argparse_witharg(args
, i
, &val
, "--placement-id", (char*)NULL
)) {
3524 } else if (ceph_argparse_witharg(args
, i
, &val
, "--storage-class", (char*)NULL
)) {
3525 opt_storage_class
= val
;
3526 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tags", (char*)NULL
)) {
3527 get_str_list(val
, ",", tags
);
3528 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tags-add", (char*)NULL
)) {
3529 get_str_list(val
, ",", tags_add
);
3530 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tags-rm", (char*)NULL
)) {
3531 get_str_list(val
, ",", tags_rm
);
3532 } else if (ceph_argparse_witharg(args
, i
, &val
, "--api-name", (char*)NULL
)) {
3534 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zone-id", (char*)NULL
)) {
3536 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zone-new-name", (char*)NULL
)) {
3537 zone_new_name
= val
;
3538 } else if (ceph_argparse_witharg(args
, i
, &val
, "--endpoints", (char*)NULL
)) {
3539 get_str_list(val
, endpoints
);
3540 } else if (ceph_argparse_witharg(args
, i
, &val
, "--sync-from", (char*)NULL
)) {
3541 get_str_list(val
, sync_from
);
3542 } else if (ceph_argparse_witharg(args
, i
, &val
, "--sync-from-rm", (char*)NULL
)) {
3543 get_str_list(val
, sync_from_rm
);
3544 } else if (ceph_argparse_binary_flag(args
, i
, &tmp_int
, NULL
, "--sync-from-all", (char*)NULL
)) {
3545 sync_from_all
= (bool)tmp_int
;
3546 sync_from_all_specified
= true;
3547 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zone", (char*)NULL
)) {
3548 source_zone_name
= val
;
3549 opt_source_zone_name
= val
;
3550 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zone-id", (char*)NULL
)) {
3551 opt_source_zone_id
= val
;
3552 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zone", (char*)NULL
)) {
3553 opt_dest_zone_name
= val
;
3554 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zone-id", (char*)NULL
)) {
3555 opt_dest_zone_id
= val
;
3556 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tier-type", (char*)NULL
)) {
3558 tier_type_specified
= true;
3559 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tier-config", (char*)NULL
)) {
3560 parse_tier_config_param(val
, tier_config_add
);
3561 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tier-config-rm", (char*)NULL
)) {
3562 parse_tier_config_param(val
, tier_config_rm
);
3563 } else if (ceph_argparse_witharg(args
, i
, &val
, "--index-pool", (char*)NULL
)) {
3565 } else if (ceph_argparse_witharg(args
, i
, &val
, "--data-pool", (char*)NULL
)) {
3567 } else if (ceph_argparse_witharg(args
, i
, &val
, "--data-extra-pool", (char*)NULL
)) {
3568 data_extra_pool
= val
;
3569 } else if (ceph_argparse_witharg(args
, i
, &val
, "--placement-index-type", (char*)NULL
)) {
3570 if (val
== "normal") {
3571 placement_index_type
= rgw::BucketIndexType::Normal
;
3572 } else if (val
== "indexless") {
3573 placement_index_type
= rgw::BucketIndexType::Indexless
;
3575 placement_index_type
= (rgw::BucketIndexType
)strict_strtol(val
.c_str(), 10, &err
);
3577 cerr
<< "ERROR: failed to parse index type index: " << err
<< std::endl
;
3581 index_type_specified
= true;
3582 } else if (ceph_argparse_witharg(args
, i
, &val
, "--compression", (char*)NULL
)) {
3583 compression_type
= val
;
3584 } else if (ceph_argparse_witharg(args
, i
, &val
, "--role-name", (char*)NULL
)) {
3586 } else if (ceph_argparse_witharg(args
, i
, &val
, "--path", (char*)NULL
)) {
3588 } else if (ceph_argparse_witharg(args
, i
, &val
, "--assume-role-policy-doc", (char*)NULL
)) {
3589 assume_role_doc
= val
;
3590 } else if (ceph_argparse_witharg(args
, i
, &val
, "--policy-name", (char*)NULL
)) {
3592 } else if (ceph_argparse_witharg(args
, i
, &val
, "--policy-doc", (char*)NULL
)) {
3593 perm_policy_doc
= val
;
3594 } else if (ceph_argparse_witharg(args
, i
, &val
, "--path-prefix", (char*)NULL
)) {
3596 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-serial", (char*)NULL
)) {
3598 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-pin", (char*)NULL
)) {
3599 totp_pin
.push_back(val
);
3600 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-seed", (char*)NULL
)) {
3602 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-seed-type", (char*)NULL
)) {
3603 totp_seed_type
= val
;
3604 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-seconds", (char*)NULL
)) {
3605 totp_seconds
= atoi(val
.c_str());
3606 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-window", (char*)NULL
)) {
3607 totp_window
= atoi(val
.c_str());
3608 } else if (ceph_argparse_witharg(args
, i
, &val
, "--trim-delay-ms", (char*)NULL
)) {
3609 trim_delay_ms
= atoi(val
.c_str());
3610 } else if (ceph_argparse_witharg(args
, i
, &val
, "--topic", (char*)NULL
)) {
3612 } else if (ceph_argparse_witharg(args
, i
, &val
, "--subscription", (char*)NULL
)) {
3614 } else if (ceph_argparse_witharg(args
, i
, &val
, "--event-id", (char*)NULL
)) {
3616 } else if (ceph_argparse_witharg(args
, i
, &val
, "--group-id", (char*)NULL
)) {
3618 } else if (ceph_argparse_witharg(args
, i
, &val
, "--status", (char*)NULL
)) {
3620 } else if (ceph_argparse_witharg(args
, i
, &val
, "--flow-type", (char*)NULL
)) {
3621 opt_flow_type
= val
;
3622 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zones", "--zone-names", (char*)NULL
)) {
3624 get_str_vec(val
, v
);
3625 opt_zone_names
= std::move(v
);
3626 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zone-ids", (char*)NULL
)) {
3627 opt_zone_ids
= zone_ids_from_str(val
);
3628 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zones", "--source-zone-names", (char*)NULL
)) {
3630 get_str_vec(val
, v
);
3631 opt_source_zone_names
= std::move(v
);
3632 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zone-ids", (char*)NULL
)) {
3633 opt_source_zone_ids
= zone_ids_from_str(val
);
3634 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zones", "--dest-zone-names", (char*)NULL
)) {
3636 get_str_vec(val
, v
);
3637 opt_dest_zone_names
= std::move(v
);
3638 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zone-ids", (char*)NULL
)) {
3639 opt_dest_zone_ids
= zone_ids_from_str(val
);
3640 } else if (ceph_argparse_witharg(args
, i
, &val
, "--flow-id", (char*)NULL
)) {
3642 } else if (ceph_argparse_witharg(args
, i
, &val
, "--pipe-id", (char*)NULL
)) {
3644 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-tenant", (char*)NULL
)) {
3645 opt_source_tenant
= val
;
3646 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-bucket", (char*)NULL
)) {
3647 opt_source_bucket_name
= val
;
3648 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-bucket-id", (char*)NULL
)) {
3649 opt_source_bucket_id
= val
;
3650 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-tenant", (char*)NULL
)) {
3651 opt_dest_tenant
= val
;
3652 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-bucket", (char*)NULL
)) {
3653 opt_dest_bucket_name
= val
;
3654 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-bucket-id", (char*)NULL
)) {
3655 opt_dest_bucket_id
= val
;
3656 } else if (ceph_argparse_witharg(args
, i
, &val
, "--effective-zone-name", "--effective-zone", (char*)NULL
)) {
3657 opt_effective_zone_name
= val
;
3658 } else if (ceph_argparse_witharg(args
, i
, &val
, "--effective-zone-id", (char*)NULL
)) {
3659 opt_effective_zone_id
= rgw_zone_id(val
);
3660 } else if (ceph_argparse_witharg(args
, i
, &val
, "--prefix", (char*)NULL
)) {
3662 } else if (ceph_argparse_witharg(args
, i
, &val
, "--prefix-rm", (char*)NULL
)) {
3663 opt_prefix_rm
= val
;
3664 } else if (ceph_argparse_witharg(args
, i
, &val
, "--priority", (char*)NULL
)) {
3665 opt_priority
= atoi(val
.c_str());
3666 } else if (ceph_argparse_witharg(args
, i
, &val
, "--mode", (char*)NULL
)) {
3668 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-owner", (char*)NULL
)) {
3669 opt_dest_owner
.emplace(val
);
3670 opt_dest_owner
= val
;
3671 } else if (ceph_argparse_witharg(args
, i
, &val
, "--retry-delay-ms", (char*)NULL
)) {
3672 opt_retry_delay_ms
= std::chrono::milliseconds(atoi(val
.c_str()));
3673 } else if (ceph_argparse_witharg(args
, i
, &val
, "--timeout-sec", (char*)NULL
)) {
3674 opt_timeout_sec
= std::chrono::seconds(atoi(val
.c_str()));
3675 } else if (ceph_argparse_binary_flag(args
, i
, &detail
, NULL
, "--detail", (char*)NULL
)) {
3677 } else if (ceph_argparse_witharg(args
, i
, &val
, "--context", (char*)NULL
)) {
3678 str_script_ctx
= val
;
3679 } else if (ceph_argparse_witharg(args
, i
, &val
, "--package", (char*)NULL
)) {
3680 script_package
= val
;
3681 } else if (ceph_argparse_binary_flag(args
, i
, &allow_compilation
, NULL
, "--allow-compilation", (char*)NULL
)) {
3683 } else if (ceph_argparse_witharg(args
, i
, &val
, "--rgw-obj-fs", (char*)NULL
)) {
3685 } else if (strncmp(*i
, "-", 1) == 0) {
3686 cerr
<< "ERROR: invalid flag " << *i
<< std::endl
;
3698 std::vector
<string
> extra_args
;
3699 std::vector
<string
> expected
;
3703 if (!cmd
.find_command(args
, &_opt_cmd
, &extra_args
, &err
, &expected
)) {
3704 if (!expected
.empty()) {
3705 cerr
<< err
<< std::endl
;
3706 cerr
<< "Expected one of the following:" << std::endl
;
3707 for (auto& exp
: expected
) {
3708 if (exp
== "*" || exp
== "[*]") {
3711 cerr
<< " " << exp
<< std::endl
;
3714 cerr
<< "Command not found:";
3715 for (auto& arg
: args
) {
3723 opt_cmd
= std::any_cast
<OPT
>(_opt_cmd
);
3725 /* some commands may have an optional extra param */
3726 if (!extra_args
.empty()) {
3728 case OPT::METADATA_GET
:
3729 case OPT::METADATA_PUT
:
3730 case OPT::METADATA_RM
:
3731 case OPT::METADATA_LIST
:
3732 metadata_key
= extra_args
[0];
3739 init_optional_bucket(opt_bucket
, opt_tenant
,
3740 opt_bucket_name
, opt_bucket_id
);
3741 init_optional_bucket(opt_source_bucket
, opt_source_tenant
,
3742 opt_source_bucket_name
, opt_source_bucket_id
);
3743 init_optional_bucket(opt_dest_bucket
, opt_dest_tenant
,
3744 opt_dest_bucket_name
, opt_dest_bucket_id
);
3746 if (tenant
.empty()) {
3747 tenant
= user_id
.tenant
;
3749 if (user_id
.empty() && opt_cmd
!= OPT::ROLE_CREATE
3750 && opt_cmd
!= OPT::ROLE_DELETE
3751 && opt_cmd
!= OPT::ROLE_GET
3752 && opt_cmd
!= OPT::ROLE_MODIFY
3753 && opt_cmd
!= OPT::ROLE_LIST
3754 && opt_cmd
!= OPT::ROLE_POLICY_PUT
3755 && opt_cmd
!= OPT::ROLE_POLICY_LIST
3756 && opt_cmd
!= OPT::ROLE_POLICY_GET
3757 && opt_cmd
!= OPT::ROLE_POLICY_DELETE
3758 && opt_cmd
!= OPT::RESHARD_ADD
3759 && opt_cmd
!= OPT::RESHARD_CANCEL
3760 && opt_cmd
!= OPT::RESHARD_STATUS
) {
3761 cerr
<< "ERROR: --tenant is set, but there's no user ID" << std::endl
;
3764 user_id
.tenant
= tenant
;
3766 if (user_ns
.empty()) {
3767 user_ns
= user_id
.ns
;
3769 user_id
.ns
= user_ns
;
3772 if (!new_user_id
.empty() && !tenant
.empty()) {
3773 new_user_id
.tenant
= tenant
;
3776 /* check key parameter conflict */
3777 if ((!access_key
.empty()) && gen_access_key
) {
3778 cerr
<< "ERROR: key parameter conflict, --access-key & --gen-access-key" << std::endl
;
3781 if ((!secret_key
.empty()) && gen_secret_key
) {
3782 cerr
<< "ERROR: key parameter conflict, --secret & --gen-secret" << std::endl
;
3787 // default to pretty json
3788 if (format
.empty()) {
3790 pretty_format
= true;
3793 if (format
== "xml")
3794 formatter
= make_unique
<XMLFormatter
>(new XMLFormatter(pretty_format
));
3795 else if (format
== "json")
3796 formatter
= make_unique
<JSONFormatter
>(new JSONFormatter(pretty_format
));
3798 cerr
<< "unrecognized format: " << format
<< std::endl
;
3802 zone_formatter
= std::make_unique
<JSONFormatter_PrettyZone
>(new JSONFormatter_PrettyZone(pretty_format
));
3804 realm_name
= g_conf()->rgw_realm
;
3805 zone_name
= g_conf()->rgw_zone
;
3806 zonegroup_name
= g_conf()->rgw_zonegroup
;
3808 RGWStreamFlusher
f(formatter
.get(), cout
);
3810 // not a raw op if 'period update' needs to commit to master
3811 bool raw_period_update
= opt_cmd
== OPT::PERIOD_UPDATE
&& !commit
;
3812 // not a raw op if 'period pull' needs to read zone/period configuration
3813 bool raw_period_pull
= opt_cmd
== OPT::PERIOD_PULL
&& !url
.empty();
3815 std::set
<OPT
> raw_storage_ops_list
= {OPT::ZONEGROUP_ADD
, OPT::ZONEGROUP_CREATE
, OPT::ZONEGROUP_DELETE
,
3816 OPT::ZONEGROUP_GET
, OPT::ZONEGROUP_LIST
,
3817 OPT::ZONEGROUP_SET
, OPT::ZONEGROUP_DEFAULT
,
3818 OPT::ZONEGROUP_RENAME
, OPT::ZONEGROUP_MODIFY
,
3819 OPT::ZONEGROUP_REMOVE
,
3820 OPT::ZONEGROUP_PLACEMENT_ADD
, OPT::ZONEGROUP_PLACEMENT_RM
,
3821 OPT::ZONEGROUP_PLACEMENT_MODIFY
, OPT::ZONEGROUP_PLACEMENT_LIST
,
3822 OPT::ZONEGROUP_PLACEMENT_GET
,
3823 OPT::ZONEGROUP_PLACEMENT_DEFAULT
,
3824 OPT::ZONE_CREATE
, OPT::ZONE_DELETE
,
3825 OPT::ZONE_GET
, OPT::ZONE_SET
, OPT::ZONE_RENAME
,
3826 OPT::ZONE_LIST
, OPT::ZONE_MODIFY
, OPT::ZONE_DEFAULT
,
3827 OPT::ZONE_PLACEMENT_ADD
, OPT::ZONE_PLACEMENT_RM
,
3828 OPT::ZONE_PLACEMENT_MODIFY
, OPT::ZONE_PLACEMENT_LIST
,
3829 OPT::ZONE_PLACEMENT_GET
,
3831 OPT::PERIOD_DELETE
, OPT::PERIOD_GET
,
3832 OPT::PERIOD_GET_CURRENT
, OPT::PERIOD_LIST
,
3833 OPT::GLOBAL_QUOTA_GET
, OPT::GLOBAL_QUOTA_SET
,
3834 OPT::GLOBAL_QUOTA_ENABLE
, OPT::GLOBAL_QUOTA_DISABLE
,
3835 OPT::REALM_DELETE
, OPT::REALM_GET
, OPT::REALM_LIST
,
3836 OPT::REALM_LIST_PERIODS
,
3837 OPT::REALM_GET_DEFAULT
,
3838 OPT::REALM_RENAME
, OPT::REALM_SET
,
3839 OPT::REALM_DEFAULT
, OPT::REALM_PULL
};
3841 std::set
<OPT
> readonly_ops_list
= {
3845 OPT::BUCKET_LIMIT_CHECK
,
3847 OPT::BUCKET_SYNC_CHECKPOINT
,
3848 OPT::BUCKET_SYNC_INFO
,
3849 OPT::BUCKET_SYNC_STATUS
,
3850 OPT::BUCKET_SYNC_MARKERS
,
3861 OPT::ORPHANS_LIST_JOBS
,
3863 OPT::ZONEGROUP_LIST
,
3864 OPT::ZONEGROUP_PLACEMENT_LIST
,
3865 OPT::ZONEGROUP_PLACEMENT_GET
,
3868 OPT::ZONE_PLACEMENT_LIST
,
3869 OPT::ZONE_PLACEMENT_GET
,
3872 OPT::METADATA_SYNC_STATUS
,
3875 OPT::SYNC_ERROR_LIST
,
3876 OPT::SYNC_GROUP_GET
,
3877 OPT::SYNC_POLICY_GET
,
3880 OPT::DATA_SYNC_STATUS
,
3882 OPT::DATALOG_STATUS
,
3884 OPT::REALM_GET_DEFAULT
,
3886 OPT::REALM_LIST_PERIODS
,
3888 OPT::PERIOD_GET_CURRENT
,
3890 OPT::GLOBAL_QUOTA_GET
,
3895 OPT::ROLE_POLICY_LIST
,
3896 OPT::ROLE_POLICY_GET
,
3898 OPT::RESHARD_STATUS
,
3899 OPT::PUBSUB_TOPICS_LIST
,
3900 OPT::PUBSUB_TOPIC_GET
,
3901 OPT::PUBSUB_SUB_GET
,
3902 OPT::PUBSUB_SUB_PULL
,
3907 bool raw_storage_op
= (raw_storage_ops_list
.find(opt_cmd
) != raw_storage_ops_list
.end() ||
3908 raw_period_update
|| raw_period_pull
);
3909 bool need_cache
= readonly_ops_list
.find(opt_cmd
) == readonly_ops_list
.end();
3911 if (raw_storage_op
) {
3912 store
= RGWStoreManager::get_raw_storage(g_ceph_context
);
3914 store
= RGWStoreManager::get_storage(g_ceph_context
, false, false, false, false, false,
3915 need_cache
&& g_conf()->rgw_cache_enabled
);
3918 cerr
<< "couldn't init storage provider" << std::endl
;
3922 if (!source_zone_name
.empty()) {
3923 if (!store
->svc()->zone
->find_zone_id_by_name(source_zone_name
, &source_zone
)) {
3924 cerr
<< "WARNING: cannot find source zone id for name=" << source_zone_name
<< std::endl
;
3925 source_zone
= source_zone_name
;
3929 rgw_http_client_init(g_ceph_context
);
3931 struct rgw_curl_setup
{
3933 rgw::curl::setup_curl(boost::none
);
3936 rgw::curl::cleanup_curl();
3942 StoreDestructor
store_destructor(store
);
3944 if (raw_storage_op
) {
3946 case OPT::PERIOD_DELETE
:
3948 if (period_id
.empty()) {
3949 cerr
<< "missing period id" << std::endl
;
3952 RGWPeriod
period(period_id
);
3953 int ret
= period
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
3955 cerr
<< "period.init failed: " << cpp_strerror(-ret
) << std::endl
;
3958 ret
= period
.delete_obj(null_yield
);
3960 cerr
<< "ERROR: couldn't delete period: " << cpp_strerror(-ret
) << std::endl
;
3966 case OPT::PERIOD_GET
:
3969 if (!period_epoch
.empty()) {
3970 epoch
= atoi(period_epoch
.c_str());
3973 RGWRealm
realm(realm_id
, realm_name
);
3974 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
3976 cerr
<< "Error initializing realm " << cpp_strerror(-ret
) << std::endl
;
3979 realm_id
= realm
.get_id();
3980 realm_name
= realm
.get_name();
3981 period_id
= RGWPeriod::get_staging_id(realm_id
);
3984 RGWPeriod
period(period_id
, epoch
);
3985 int ret
= period
.init(g_ceph_context
, store
->svc()->sysobj
, realm_id
,
3986 null_yield
, realm_name
);
3988 cerr
<< "period init failed: " << cpp_strerror(-ret
) << std::endl
;
3991 encode_json("period", period
, formatter
.get());
3992 formatter
->flush(cout
);
3995 case OPT::PERIOD_GET_CURRENT
:
3997 int ret
= read_current_period_id(store
, realm_id
, realm_name
, &period_id
);
4001 formatter
->open_object_section("period_get_current");
4002 encode_json("current_period", period_id
, formatter
.get());
4003 formatter
->close_section();
4004 formatter
->flush(cout
);
4007 case OPT::PERIOD_LIST
:
4009 list
<string
> periods
;
4010 int ret
= store
->svc()->zone
->list_periods(periods
);
4012 cerr
<< "failed to list periods: " << cpp_strerror(-ret
) << std::endl
;
4015 formatter
->open_object_section("periods_list");
4016 encode_json("periods", periods
, formatter
.get());
4017 formatter
->close_section();
4018 formatter
->flush(cout
);
4021 case OPT::PERIOD_UPDATE
:
4023 int ret
= update_period(realm_id
, realm_name
, period_id
, period_epoch
,
4024 commit
, remote
, url
, access_key
, secret_key
,
4025 formatter
.get(), yes_i_really_mean_it
);
4031 case OPT::PERIOD_PULL
:
4033 boost::optional
<RGWRESTConn
> conn
;
4034 RGWRESTConn
*remote_conn
= nullptr;
4036 // load current period for endpoints
4037 RGWRealm
realm(realm_id
, realm_name
);
4038 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4040 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
4043 RGWPeriod
current_period(realm
.get_current_period());
4044 ret
= current_period
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4046 cerr
<< "failed to init current period: " << cpp_strerror(-ret
) << std::endl
;
4049 if (remote
.empty()) {
4050 // use realm master zone as remote
4051 remote
= current_period
.get_master_zone().id
;
4053 conn
= get_remote_conn(store
, current_period
.get_map(), remote
);
4055 cerr
<< "failed to find a zone or zonegroup for remote "
4056 << remote
<< std::endl
;
4059 remote_conn
= &*conn
;
4063 int ret
= do_period_pull(remote_conn
, url
, access_key
, secret_key
,
4064 realm_id
, realm_name
, period_id
, period_epoch
,
4067 cerr
<< "period pull failed: " << cpp_strerror(-ret
) << std::endl
;
4071 encode_json("period", period
, formatter
.get());
4072 formatter
->flush(cout
);
4075 case OPT::GLOBAL_QUOTA_GET
:
4076 case OPT::GLOBAL_QUOTA_SET
:
4077 case OPT::GLOBAL_QUOTA_ENABLE
:
4078 case OPT::GLOBAL_QUOTA_DISABLE
:
4080 if (realm_id
.empty()) {
4081 RGWRealm
realm(g_ceph_context
, store
->svc()->sysobj
);
4082 if (!realm_name
.empty()) {
4083 // look up realm_id for the given realm_name
4084 int ret
= realm
.read_id(realm_name
, realm_id
, null_yield
);
4086 cerr
<< "ERROR: failed to read realm for " << realm_name
4087 << ": " << cpp_strerror(-ret
) << std::endl
;
4091 // use default realm_id when none is given
4092 int ret
= realm
.read_default_id(realm_id
, null_yield
);
4093 if (ret
< 0 && ret
!= -ENOENT
) { // on ENOENT, use empty realm_id
4094 cerr
<< "ERROR: failed to read default realm: "
4095 << cpp_strerror(-ret
) << std::endl
;
4101 RGWPeriodConfig period_config
;
4102 int ret
= period_config
.read(store
->svc()->sysobj
, realm_id
, null_yield
);
4103 if (ret
< 0 && ret
!= -ENOENT
) {
4104 cerr
<< "ERROR: failed to read period config: "
4105 << cpp_strerror(-ret
) << std::endl
;
4109 formatter
->open_object_section("period_config");
4110 if (quota_scope
== "bucket") {
4111 set_quota_info(period_config
.bucket_quota
, opt_cmd
,
4112 max_size
, max_objects
,
4113 have_max_size
, have_max_objects
);
4114 encode_json("bucket quota", period_config
.bucket_quota
, formatter
.get());
4115 } else if (quota_scope
== "user") {
4116 set_quota_info(period_config
.user_quota
, opt_cmd
,
4117 max_size
, max_objects
,
4118 have_max_size
, have_max_objects
);
4119 encode_json("user quota", period_config
.user_quota
, formatter
.get());
4120 } else if (quota_scope
.empty() && opt_cmd
== OPT::GLOBAL_QUOTA_GET
) {
4121 // if no scope is given for GET, print both
4122 encode_json("bucket quota", period_config
.bucket_quota
, formatter
.get());
4123 encode_json("user quota", period_config
.user_quota
, formatter
.get());
4125 cerr
<< "ERROR: invalid quota scope specification. Please specify "
4126 "either --quota-scope=bucket, or --quota-scope=user" << std::endl
;
4129 formatter
->close_section();
4131 if (opt_cmd
!= OPT::GLOBAL_QUOTA_GET
) {
4132 // write the modified period config
4133 ret
= period_config
.write(store
->svc()->sysobj
, realm_id
, null_yield
);
4135 cerr
<< "ERROR: failed to write period config: "
4136 << cpp_strerror(-ret
) << std::endl
;
4139 if (!realm_id
.empty()) {
4140 cout
<< "Global quota changes saved. Use 'period update' to apply "
4141 "them to the staging period, and 'period commit' to commit the "
4142 "new period." << std::endl
;
4144 cout
<< "Global quota changes saved. They will take effect as "
4145 "the gateways are restarted." << std::endl
;
4149 formatter
->flush(cout
);
4152 case OPT::REALM_CREATE
:
4154 if (realm_name
.empty()) {
4155 cerr
<< "missing realm name" << std::endl
;
4159 RGWRealm
realm(realm_name
, g_ceph_context
, store
->svc()->sysobj
);
4160 int ret
= realm
.create(null_yield
);
4162 cerr
<< "ERROR: couldn't create realm " << realm_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
4167 ret
= realm
.set_as_default(null_yield
);
4169 cerr
<< "failed to set realm " << realm_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
4173 encode_json("realm", realm
, formatter
.get());
4174 formatter
->flush(cout
);
4177 case OPT::REALM_DELETE
:
4179 RGWRealm
realm(realm_id
, realm_name
);
4180 if (realm_name
.empty() && realm_id
.empty()) {
4181 cerr
<< "missing realm name or id" << std::endl
;
4184 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4186 cerr
<< "realm.init failed: " << cpp_strerror(-ret
) << std::endl
;
4189 ret
= realm
.delete_obj(null_yield
);
4191 cerr
<< "ERROR: couldn't : " << cpp_strerror(-ret
) << std::endl
;
4197 case OPT::REALM_GET
:
4199 RGWRealm
realm(realm_id
, realm_name
);
4200 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4202 if (ret
== -ENOENT
&& realm_name
.empty() && realm_id
.empty()) {
4203 cerr
<< "missing realm name or id, or default realm not found" << std::endl
;
4205 cerr
<< "realm.init failed: " << cpp_strerror(-ret
) << std::endl
;
4209 encode_json("realm", realm
, formatter
.get());
4210 formatter
->flush(cout
);
4213 case OPT::REALM_GET_DEFAULT
:
4215 RGWRealm
realm(g_ceph_context
, store
->svc()->sysobj
);
4217 int ret
= realm
.read_default_id(default_id
, null_yield
);
4218 if (ret
== -ENOENT
) {
4219 cout
<< "No default realm is set" << std::endl
;
4221 } else if (ret
< 0) {
4222 cerr
<< "Error reading default realm:" << cpp_strerror(-ret
) << std::endl
;
4225 cout
<< "default realm: " << default_id
<< std::endl
;
4228 case OPT::REALM_LIST
:
4230 RGWRealm
realm(g_ceph_context
, store
->svc()->sysobj
);
4232 int ret
= realm
.read_default_id(default_id
, null_yield
);
4233 if (ret
< 0 && ret
!= -ENOENT
) {
4234 cerr
<< "could not determine default realm: " << cpp_strerror(-ret
) << std::endl
;
4236 list
<string
> realms
;
4237 ret
= store
->svc()->zone
->list_realms(realms
);
4239 cerr
<< "failed to list realms: " << cpp_strerror(-ret
) << std::endl
;
4242 formatter
->open_object_section("realms_list");
4243 encode_json("default_info", default_id
, formatter
.get());
4244 encode_json("realms", realms
, formatter
.get());
4245 formatter
->close_section();
4246 formatter
->flush(cout
);
4249 case OPT::REALM_LIST_PERIODS
:
4251 int ret
= read_current_period_id(store
, realm_id
, realm_name
, &period_id
);
4255 list
<string
> periods
;
4256 ret
= store
->svc()->zone
->list_periods(period_id
, periods
, null_yield
);
4258 cerr
<< "list periods failed: " << cpp_strerror(-ret
) << std::endl
;
4261 formatter
->open_object_section("realm_periods_list");
4262 encode_json("current_period", period_id
, formatter
.get());
4263 encode_json("periods", periods
, formatter
.get());
4264 formatter
->close_section();
4265 formatter
->flush(cout
);
4269 case OPT::REALM_RENAME
:
4271 RGWRealm
realm(realm_id
, realm_name
);
4272 if (realm_new_name
.empty()) {
4273 cerr
<< "missing realm new name" << std::endl
;
4276 if (realm_name
.empty() && realm_id
.empty()) {
4277 cerr
<< "missing realm name or id" << std::endl
;
4280 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4282 cerr
<< "realm.init failed: " << cpp_strerror(-ret
) << std::endl
;
4285 ret
= realm
.rename(realm_new_name
, null_yield
);
4287 cerr
<< "realm.rename failed: " << cpp_strerror(-ret
) << std::endl
;
4290 cout
<< "Realm name updated. Note that this change only applies to "
4291 "the current cluster, so this command must be run separately "
4292 "on each of the realm's other clusters." << std::endl
;
4295 case OPT::REALM_SET
:
4297 if (realm_id
.empty() && realm_name
.empty()) {
4298 cerr
<< "no realm name or id provided" << std::endl
;
4301 RGWRealm
realm(realm_id
, realm_name
);
4302 bool new_realm
= false;
4303 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4304 if (ret
< 0 && ret
!= -ENOENT
) {
4305 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
4307 } else if (ret
== -ENOENT
) {
4310 ret
= read_decode_json(infile
, realm
);
4314 if (!realm_name
.empty() && realm
.get_name() != realm_name
) {
4315 cerr
<< "mismatch between --rgw-realm " << realm_name
<< " and json input file name " <<
4316 realm
.get_name() << std::endl
;
4321 cout
<< "clearing period and epoch for new realm" << std::endl
;
4322 realm
.clear_current_period_and_epoch();
4323 ret
= realm
.create(null_yield
);
4325 cerr
<< "ERROR: couldn't create new realm: " << cpp_strerror(-ret
) << std::endl
;
4329 ret
= realm
.update(null_yield
);
4331 cerr
<< "ERROR: couldn't store realm info: " << cpp_strerror(-ret
) << std::endl
;
4337 ret
= realm
.set_as_default(null_yield
);
4339 cerr
<< "failed to set realm " << realm_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
4342 encode_json("realm", realm
, formatter
.get());
4343 formatter
->flush(cout
);
4347 case OPT::REALM_DEFAULT
:
4349 RGWRealm
realm(realm_id
, realm_name
);
4350 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4352 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
4355 ret
= realm
.set_as_default(null_yield
);
4357 cerr
<< "failed to set realm as default: " << cpp_strerror(-ret
) << std::endl
;
4362 case OPT::REALM_PULL
:
4365 cerr
<< "A --url must be provided." << std::endl
;
4369 req_info
info(g_ceph_context
, &env
);
4370 info
.method
= "GET";
4371 info
.request_uri
= "/admin/realm";
4373 map
<string
, string
> ¶ms
= info
.args
.get_params();
4374 if (!realm_id
.empty())
4375 params
["id"] = realm_id
;
4376 if (!realm_name
.empty())
4377 params
["name"] = realm_name
;
4381 int ret
= send_to_url(url
, access_key
, secret_key
, info
, bl
, p
);
4383 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
4384 if (ret
== -EACCES
) {
4385 cerr
<< "If the realm has been changed on the master zone, the "
4386 "master zone's gateway may need to be restarted to recognize "
4387 "this user." << std::endl
;
4392 realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
, false);
4394 decode_json_obj(realm
, &p
);
4395 } catch (const JSONDecoder::err
& e
) {
4396 cerr
<< "failed to decode JSON response: " << e
.what() << std::endl
;
4400 auto& current_period
= realm
.get_current_period();
4401 if (!current_period
.empty()) {
4402 // pull the latest epoch of the realm's current period
4403 ret
= do_period_pull(nullptr, url
, access_key
, secret_key
,
4404 realm_id
, realm_name
, current_period
, "",
4407 cerr
<< "could not fetch period " << current_period
<< std::endl
;
4411 ret
= realm
.create(null_yield
, false);
4412 if (ret
< 0 && ret
!= -EEXIST
) {
4413 cerr
<< "Error storing realm " << realm
.get_id() << ": "
4414 << cpp_strerror(ret
) << std::endl
;
4416 } else if (ret
==-EEXIST
) {
4417 ret
= realm
.update(null_yield
);
4419 cerr
<< "Error storing realm " << realm
.get_id() << ": "
4420 << cpp_strerror(ret
) << std::endl
;
4425 ret
= realm
.set_as_default(null_yield
);
4427 cerr
<< "failed to set realm " << realm_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
4431 encode_json("realm", realm
, formatter
.get());
4432 formatter
->flush(cout
);
4436 case OPT::ZONEGROUP_ADD
:
4438 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
4439 cerr
<< "no zonegroup name or id provided" << std::endl
;
4443 RGWZoneGroup
zonegroup(zonegroup_id
,zonegroup_name
);
4444 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4446 cerr
<< "failed to initialize zonegroup " << zonegroup_name
<< " id " << zonegroup_id
<< " :"
4447 << cpp_strerror(-ret
) << std::endl
;
4450 RGWZoneParams
zone(zone_id
, zone_name
);
4451 ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4453 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
4456 if (zone
.realm_id
!= zonegroup
.realm_id
) {
4457 zone
.realm_id
= zonegroup
.realm_id
;
4458 ret
= zone
.update(null_yield
);
4460 cerr
<< "failed to save zone info: " << cpp_strerror(-ret
) << std::endl
;
4465 string
*ptier_type
= (tier_type_specified
? &tier_type
: nullptr);
4467 for (auto a
: tier_config_add
) {
4468 int r
= zone
.tier_config
.set(a
.first
, a
.second
);
4470 cerr
<< "ERROR: failed to set configurable: " << a
<< std::endl
;
4475 bool *psync_from_all
= (sync_from_all_specified
? &sync_from_all
: nullptr);
4476 string
*predirect_zone
= (redirect_zone_set
? &redirect_zone
: nullptr);
4478 ret
= zonegroup
.add_zone(zone
,
4479 (is_master_set
? &is_master
: NULL
),
4480 (is_read_only_set
? &read_only
: NULL
),
4481 endpoints
, ptier_type
,
4482 psync_from_all
, sync_from
, sync_from_rm
,
4483 predirect_zone
, bucket_index_max_shards
,
4484 store
->svc()->sync_modules
->get_manager(),
4487 cerr
<< "failed to add zone " << zone_name
<< " to zonegroup " << zonegroup
.get_name() << ": "
4488 << cpp_strerror(-ret
) << std::endl
;
4492 encode_json("zonegroup", zonegroup
, formatter
.get());
4493 formatter
->flush(cout
);
4496 case OPT::ZONEGROUP_CREATE
:
4498 if (zonegroup_name
.empty()) {
4499 cerr
<< "Missing zonegroup name" << std::endl
;
4502 RGWRealm
realm(realm_id
, realm_name
);
4503 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4505 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
4509 RGWZoneGroup
zonegroup(zonegroup_name
, is_master
, g_ceph_context
, store
->svc()->sysobj
, realm
.get_id(), endpoints
);
4510 zonegroup
.api_name
= (api_name
.empty() ? zonegroup_name
: api_name
);
4511 ret
= zonegroup
.create(null_yield
);
4513 cerr
<< "failed to create zonegroup " << zonegroup_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
4518 ret
= zonegroup
.set_as_default(null_yield
);
4520 cerr
<< "failed to set zonegroup " << zonegroup_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
4524 encode_json("zonegroup", zonegroup
, formatter
.get());
4525 formatter
->flush(cout
);
4528 case OPT::ZONEGROUP_DEFAULT
:
4530 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
4531 cerr
<< "no zonegroup name or id provided" << std::endl
;
4535 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
4536 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4538 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4542 ret
= zonegroup
.set_as_default(null_yield
);
4544 cerr
<< "failed to set zonegroup as default: " << cpp_strerror(-ret
) << std::endl
;
4549 case OPT::ZONEGROUP_DELETE
:
4551 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
4552 cerr
<< "no zonegroup name or id provided" << std::endl
;
4555 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
4556 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
,
4559 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4562 ret
= zonegroup
.delete_obj(null_yield
);
4564 cerr
<< "ERROR: couldn't delete zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4569 case OPT::ZONEGROUP_GET
:
4571 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
4572 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4574 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4578 encode_json("zonegroup", zonegroup
, formatter
.get());
4579 formatter
->flush(cout
);
4582 case OPT::ZONEGROUP_LIST
:
4584 RGWZoneGroup zonegroup
;
4585 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
,
4588 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4592 list
<string
> zonegroups
;
4593 ret
= store
->svc()->zone
->list_zonegroups(zonegroups
);
4595 cerr
<< "failed to list zonegroups: " << cpp_strerror(-ret
) << std::endl
;
4598 string default_zonegroup
;
4599 ret
= zonegroup
.read_default_id(default_zonegroup
, null_yield
);
4600 if (ret
< 0 && ret
!= -ENOENT
) {
4601 cerr
<< "could not determine default zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4603 formatter
->open_object_section("zonegroups_list");
4604 encode_json("default_info", default_zonegroup
, formatter
.get());
4605 encode_json("zonegroups", zonegroups
, formatter
.get());
4606 formatter
->close_section();
4607 formatter
->flush(cout
);
4610 case OPT::ZONEGROUP_MODIFY
:
4612 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
4613 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4615 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4619 bool need_update
= false;
4621 if (!master_zone
.empty()) {
4622 zonegroup
.master_zone
= master_zone
;
4626 if (is_master_set
) {
4627 zonegroup
.update_master(is_master
, null_yield
);
4631 if (!endpoints
.empty()) {
4632 zonegroup
.endpoints
= endpoints
;
4636 if (!api_name
.empty()) {
4637 zonegroup
.api_name
= api_name
;
4641 if (!realm_id
.empty()) {
4642 zonegroup
.realm_id
= realm_id
;
4644 } else if (!realm_name
.empty()) {
4645 // get realm id from name
4646 RGWRealm realm
{g_ceph_context
, store
->svc()->sysobj
};
4647 ret
= realm
.read_id(realm_name
, zonegroup
.realm_id
, null_yield
);
4649 cerr
<< "failed to find realm by name " << realm_name
<< std::endl
;
4655 if (bucket_index_max_shards
) {
4656 for (auto& [name
, zone
] : zonegroup
.zones
) {
4657 zone
.bucket_index_max_shards
= *bucket_index_max_shards
;
4663 ret
= zonegroup
.update(null_yield
);
4665 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4671 ret
= zonegroup
.set_as_default(null_yield
);
4673 cerr
<< "failed to set zonegroup " << zonegroup_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
4677 encode_json("zonegroup", zonegroup
, formatter
.get());
4678 formatter
->flush(cout
);
4681 case OPT::ZONEGROUP_SET
:
4683 RGWRealm
realm(realm_id
, realm_name
);
4684 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4685 bool default_realm_not_exist
= (ret
== -ENOENT
&& realm_id
.empty() && realm_name
.empty());
4687 if (ret
< 0 && !default_realm_not_exist
) {
4688 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
4692 RGWZoneGroup zonegroup
;
4693 ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
,
4696 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4699 ret
= read_decode_json(infile
, zonegroup
);
4703 if (zonegroup
.realm_id
.empty() && !default_realm_not_exist
) {
4704 zonegroup
.realm_id
= realm
.get_id();
4706 ret
= zonegroup
.create(null_yield
);
4707 if (ret
< 0 && ret
!= -EEXIST
) {
4708 cerr
<< "ERROR: couldn't create zonegroup info: " << cpp_strerror(-ret
) << std::endl
;
4710 } else if (ret
== -EEXIST
) {
4711 ret
= zonegroup
.update(null_yield
);
4713 cerr
<< "ERROR: couldn't store zonegroup info: " << cpp_strerror(-ret
) << std::endl
;
4719 ret
= zonegroup
.set_as_default(null_yield
);
4721 cerr
<< "failed to set zonegroup " << zonegroup_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
4725 encode_json("zonegroup", zonegroup
, formatter
.get());
4726 formatter
->flush(cout
);
4729 case OPT::ZONEGROUP_REMOVE
:
4731 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
4732 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4734 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4738 if (zone_id
.empty()) {
4739 if (zone_name
.empty()) {
4740 cerr
<< "no --zone-id or --rgw-zone name provided" << std::endl
;
4743 // look up zone id by name
4744 for (auto& z
: zonegroup
.zones
) {
4745 if (zone_name
== z
.second
.name
) {
4746 zone_id
= z
.second
.id
;
4750 if (zone_id
.empty()) {
4751 cerr
<< "zone name " << zone_name
<< " not found in zonegroup "
4752 << zonegroup
.get_name() << std::endl
;
4757 ret
= zonegroup
.remove_zone(zone_id
, null_yield
);
4759 cerr
<< "failed to remove zone: " << cpp_strerror(-ret
) << std::endl
;
4763 encode_json("zonegroup", zonegroup
, formatter
.get());
4764 formatter
->flush(cout
);
4767 case OPT::ZONEGROUP_RENAME
:
4769 if (zonegroup_new_name
.empty()) {
4770 cerr
<< " missing zonegroup new name" << std::endl
;
4773 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
4774 cerr
<< "no zonegroup name or id provided" << std::endl
;
4777 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
4778 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4780 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4783 ret
= zonegroup
.rename(zonegroup_new_name
, null_yield
);
4785 cerr
<< "failed to rename zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4790 case OPT::ZONEGROUP_PLACEMENT_LIST
:
4792 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
4793 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
,
4796 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4800 encode_json("placement_targets", zonegroup
.placement_targets
, formatter
.get());
4801 formatter
->flush(cout
);
4804 case OPT::ZONEGROUP_PLACEMENT_GET
:
4806 if (placement_id
.empty()) {
4807 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
4811 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
4812 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4814 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4818 auto p
= zonegroup
.placement_targets
.find(placement_id
);
4819 if (p
== zonegroup
.placement_targets
.end()) {
4820 cerr
<< "failed to find a zonegroup placement target named '" << placement_id
<< "'" << std::endl
;
4823 encode_json("placement_targets", p
->second
, formatter
.get());
4824 formatter
->flush(cout
);
4827 case OPT::ZONEGROUP_PLACEMENT_ADD
:
4828 case OPT::ZONEGROUP_PLACEMENT_MODIFY
:
4829 case OPT::ZONEGROUP_PLACEMENT_RM
:
4830 case OPT::ZONEGROUP_PLACEMENT_DEFAULT
:
4832 if (placement_id
.empty()) {
4833 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
4837 rgw_placement_rule rule
;
4838 rule
.from_str(placement_id
);
4840 if (!rule
.storage_class
.empty() && opt_storage_class
&&
4841 rule
.storage_class
!= *opt_storage_class
) {
4842 cerr
<< "ERROR: provided contradicting storage class configuration" << std::endl
;
4844 } else if (rule
.storage_class
.empty()) {
4845 rule
.storage_class
= opt_storage_class
.value_or(string());
4848 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
4849 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4851 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4855 if (opt_cmd
== OPT::ZONEGROUP_PLACEMENT_ADD
||
4856 opt_cmd
== OPT::ZONEGROUP_PLACEMENT_MODIFY
) {
4857 RGWZoneGroupPlacementTarget
& target
= zonegroup
.placement_targets
[placement_id
];
4858 if (!tags
.empty()) {
4859 target
.tags
.clear();
4860 for (auto& t
: tags
) {
4861 target
.tags
.insert(t
);
4864 target
.name
= placement_id
;
4865 for (auto& t
: tags_rm
) {
4866 target
.tags
.erase(t
);
4868 for (auto& t
: tags_add
) {
4869 target
.tags
.insert(t
);
4871 target
.storage_classes
.insert(rule
.get_storage_class());
4872 } else if (opt_cmd
== OPT::ZONEGROUP_PLACEMENT_RM
) {
4873 if (!opt_storage_class
||
4874 opt_storage_class
->empty()) {
4875 zonegroup
.placement_targets
.erase(placement_id
);
4877 auto iter
= zonegroup
.placement_targets
.find(placement_id
);
4878 if (iter
!= zonegroup
.placement_targets
.end()) {
4879 RGWZoneGroupPlacementTarget
& info
= zonegroup
.placement_targets
[placement_id
];
4880 info
.storage_classes
.erase(*opt_storage_class
);
4883 } else if (opt_cmd
== OPT::ZONEGROUP_PLACEMENT_DEFAULT
) {
4884 if (!zonegroup
.placement_targets
.count(placement_id
)) {
4885 cerr
<< "failed to find a zonegroup placement target named '"
4886 << placement_id
<< "'" << std::endl
;
4889 zonegroup
.default_placement
= rule
;
4892 zonegroup
.post_process_params(null_yield
);
4893 ret
= zonegroup
.update(null_yield
);
4895 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
4899 encode_json("placement_targets", zonegroup
.placement_targets
, formatter
.get());
4900 formatter
->flush(cout
);
4903 case OPT::ZONE_CREATE
:
4905 if (zone_name
.empty()) {
4906 cerr
<< "zone name not provided" << std::endl
;
4910 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
4911 /* if the user didn't provide zonegroup info , create stand alone zone */
4912 if (!zonegroup_id
.empty() || !zonegroup_name
.empty()) {
4913 ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4915 cerr
<< "unable to initialize zonegroup " << zonegroup_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
4918 if (realm_id
.empty() && realm_name
.empty()) {
4919 realm_id
= zonegroup
.realm_id
;
4923 RGWZoneParams
zone(zone_id
, zone_name
);
4924 ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
, false);
4926 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
4930 zone
.system_key
.id
= access_key
;
4931 zone
.system_key
.key
= secret_key
;
4932 zone
.realm_id
= realm_id
;
4933 for (auto a
: tier_config_add
) {
4934 int r
= zone
.tier_config
.set(a
.first
, a
.second
);
4936 cerr
<< "ERROR: failed to set configurable: " << a
<< std::endl
;
4941 ret
= zone
.create(null_yield
);
4943 cerr
<< "failed to create zone " << zone_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
4947 if (!zonegroup_id
.empty() || !zonegroup_name
.empty()) {
4948 string
*ptier_type
= (tier_type_specified
? &tier_type
: nullptr);
4949 bool *psync_from_all
= (sync_from_all_specified
? &sync_from_all
: nullptr);
4950 string
*predirect_zone
= (redirect_zone_set
? &redirect_zone
: nullptr);
4951 ret
= zonegroup
.add_zone(zone
,
4952 (is_master_set
? &is_master
: NULL
),
4953 (is_read_only_set
? &read_only
: NULL
),
4957 sync_from
, sync_from_rm
,
4958 predirect_zone
, bucket_index_max_shards
,
4959 store
->svc()->sync_modules
->get_manager(),
4962 cerr
<< "failed to add zone " << zone_name
<< " to zonegroup " << zonegroup
.get_name()
4963 << ": " << cpp_strerror(-ret
) << std::endl
;
4969 ret
= zone
.set_as_default(null_yield
);
4971 cerr
<< "failed to set zone " << zone_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
4975 encode_json("zone", zone
, formatter
.get());
4976 formatter
->flush(cout
);
4979 case OPT::ZONE_DEFAULT
:
4981 RGWZoneGroup
zonegroup(zonegroup_id
,zonegroup_name
);
4982 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4984 cerr
<< "WARNING: failed to initialize zonegroup " << zonegroup_name
<< std::endl
;
4986 if (zone_id
.empty() && zone_name
.empty()) {
4987 cerr
<< "no zone name or id provided" << std::endl
;
4990 RGWZoneParams
zone(zone_id
, zone_name
);
4991 ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
4993 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
4996 ret
= zone
.set_as_default(null_yield
);
4998 cerr
<< "failed to set zone as default: " << cpp_strerror(-ret
) << std::endl
;
5003 case OPT::ZONE_DELETE
:
5005 if (zone_id
.empty() && zone_name
.empty()) {
5006 cerr
<< "no zone name or id provided" << std::endl
;
5009 RGWZoneParams
zone(zone_id
, zone_name
);
5010 int ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5012 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
5016 list
<string
> zonegroups
;
5017 ret
= store
->svc()->zone
->list_zonegroups(zonegroups
);
5019 cerr
<< "failed to list zonegroups: " << cpp_strerror(-ret
) << std::endl
;
5023 for (list
<string
>::iterator iter
= zonegroups
.begin(); iter
!= zonegroups
.end(); ++iter
) {
5024 RGWZoneGroup
zonegroup(string(), *iter
);
5025 int ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5027 cerr
<< "WARNING: failed to initialize zonegroup " << zonegroup_name
<< std::endl
;
5030 ret
= zonegroup
.remove_zone(zone
.get_id(), null_yield
);
5031 if (ret
< 0 && ret
!= -ENOENT
) {
5032 cerr
<< "failed to remove zone " << zone_name
<< " from zonegroup " << zonegroup
.get_name() << ": "
5033 << cpp_strerror(-ret
) << std::endl
;
5037 ret
= zone
.delete_obj(null_yield
);
5039 cerr
<< "failed to delete zone " << zone_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
5046 RGWZoneParams
zone(zone_id
, zone_name
);
5047 int ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5049 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
5052 encode_json("zone", zone
, formatter
.get());
5053 formatter
->flush(cout
);
5058 RGWZoneParams
zone(zone_name
);
5059 int ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
,
5065 ret
= zone
.read(null_yield
);
5066 if (ret
< 0 && ret
!= -ENOENT
) {
5067 cerr
<< "zone.read() returned ret=" << ret
<< std::endl
;
5071 string orig_id
= zone
.get_id();
5073 ret
= read_decode_json(infile
, zone
);
5078 if(zone
.realm_id
.empty()) {
5079 RGWRealm
realm(realm_id
, realm_name
);
5080 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5081 if (ret
< 0 && ret
!= -ENOENT
) {
5082 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
5085 zone
.realm_id
= realm
.get_id();
5088 if( !zone_name
.empty() && !zone
.get_name().empty() && zone
.get_name() != zone_name
) {
5089 cerr
<< "Error: zone name" << zone_name
<< " is different than the zone name " << zone
.get_name() << " in the provided json " << std::endl
;
5093 if (zone
.get_name().empty()) {
5094 zone
.set_name(zone_name
);
5095 if (zone
.get_name().empty()) {
5096 cerr
<< "no zone name specified" << std::endl
;
5101 zone_name
= zone
.get_name();
5103 if (zone
.get_id().empty()) {
5104 zone
.set_id(orig_id
);
5107 if (zone
.get_id().empty()) {
5108 cerr
<< "no zone name id the json provided, assuming old format" << std::endl
;
5109 if (zone_name
.empty()) {
5110 cerr
<< "missing zone name" << std::endl
;
5113 zone
.set_name(zone_name
);
5114 zone
.set_id(zone_name
);
5117 cerr
<< "zone id " << zone
.get_id();
5118 ret
= zone
.fix_pool_names(null_yield
);
5120 cerr
<< "ERROR: couldn't fix zone: " << cpp_strerror(-ret
) << std::endl
;
5123 ret
= zone
.write(false, null_yield
);
5125 cerr
<< "ERROR: couldn't create zone: " << cpp_strerror(-ret
) << std::endl
;
5130 ret
= zone
.set_as_default(null_yield
);
5132 cerr
<< "failed to set zone " << zone_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5136 encode_json("zone", zone
, formatter
.get());
5137 formatter
->flush(cout
);
5140 case OPT::ZONE_LIST
:
5143 int ret
= store
->svc()->zone
->list_zones(zones
);
5145 cerr
<< "failed to list zones: " << cpp_strerror(-ret
) << std::endl
;
5150 ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
, false);
5152 cerr
<< "failed to init zone: " << cpp_strerror(-ret
) << std::endl
;
5155 string default_zone
;
5156 ret
= zone
.read_default_id(default_zone
, null_yield
);
5157 if (ret
< 0 && ret
!= -ENOENT
) {
5158 cerr
<< "could not determine default zone: " << cpp_strerror(-ret
) << std::endl
;
5160 formatter
->open_object_section("zones_list");
5161 encode_json("default_info", default_zone
, formatter
.get());
5162 encode_json("zones", zones
, formatter
.get());
5163 formatter
->close_section();
5164 formatter
->flush(cout
);
5167 case OPT::ZONE_MODIFY
:
5169 RGWZoneParams
zone(zone_id
, zone_name
);
5170 int ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5172 cerr
<< "failed to init zone: " << cpp_strerror(-ret
) << std::endl
;
5176 bool need_zone_update
= false;
5177 if (!access_key
.empty()) {
5178 zone
.system_key
.id
= access_key
;
5179 need_zone_update
= true;
5182 if (!secret_key
.empty()) {
5183 zone
.system_key
.key
= secret_key
;
5184 need_zone_update
= true;
5187 if (!realm_id
.empty()) {
5188 zone
.realm_id
= realm_id
;
5189 need_zone_update
= true;
5190 } else if (!realm_name
.empty()) {
5191 // get realm id from name
5192 RGWRealm realm
{g_ceph_context
, store
->svc()->sysobj
};
5193 ret
= realm
.read_id(realm_name
, zone
.realm_id
, null_yield
);
5195 cerr
<< "failed to find realm by name " << realm_name
<< std::endl
;
5198 need_zone_update
= true;
5201 if (tier_config_add
.size() > 0) {
5202 for (auto add
: tier_config_add
) {
5203 int r
= zone
.tier_config
.set(add
.first
, add
.second
);
5205 cerr
<< "ERROR: failed to set configurable: " << add
<< std::endl
;
5209 need_zone_update
= true;
5212 for (auto rm
: tier_config_rm
) {
5213 if (!rm
.first
.empty()) { /* otherwise will remove the entire config */
5214 zone
.tier_config
.erase(rm
.first
);
5215 need_zone_update
= true;
5219 if (need_zone_update
) {
5220 ret
= zone
.update(null_yield
);
5222 cerr
<< "failed to save zone info: " << cpp_strerror(-ret
) << std::endl
;
5227 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
5228 ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5230 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5233 string
*ptier_type
= (tier_type_specified
? &tier_type
: nullptr);
5235 bool *psync_from_all
= (sync_from_all_specified
? &sync_from_all
: nullptr);
5236 string
*predirect_zone
= (redirect_zone_set
? &redirect_zone
: nullptr);
5238 ret
= zonegroup
.add_zone(zone
,
5239 (is_master_set
? &is_master
: NULL
),
5240 (is_read_only_set
? &read_only
: NULL
),
5241 endpoints
, ptier_type
,
5242 psync_from_all
, sync_from
, sync_from_rm
,
5243 predirect_zone
, bucket_index_max_shards
,
5244 store
->svc()->sync_modules
->get_manager(),
5247 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5251 ret
= zonegroup
.update(null_yield
);
5253 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5258 ret
= zone
.set_as_default(null_yield
);
5260 cerr
<< "failed to set zone " << zone_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5264 encode_json("zone", zone
, formatter
.get());
5265 formatter
->flush(cout
);
5268 case OPT::ZONE_RENAME
:
5270 if (zone_new_name
.empty()) {
5271 cerr
<< " missing zone new name" << std::endl
;
5274 if (zone_id
.empty() && zone_name
.empty()) {
5275 cerr
<< "no zone name or id provided" << std::endl
;
5278 RGWZoneParams
zone(zone_id
,zone_name
);
5279 int ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5281 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
5284 ret
= zone
.rename(zone_new_name
, null_yield
);
5286 cerr
<< "failed to rename zone " << zone_name
<< " to " << zone_new_name
<< ": " << cpp_strerror(-ret
)
5290 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
5291 ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5293 cerr
<< "WARNING: failed to initialize zonegroup " << zonegroup_name
<< std::endl
;
5295 ret
= zonegroup
.rename_zone(zone
, null_yield
);
5297 cerr
<< "Error in zonegroup rename for " << zone_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
5303 case OPT::ZONE_PLACEMENT_ADD
:
5304 case OPT::ZONE_PLACEMENT_MODIFY
:
5305 case OPT::ZONE_PLACEMENT_RM
:
5307 if (placement_id
.empty()) {
5308 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
5311 // validate compression type
5312 if (compression_type
&& *compression_type
!= "random"
5313 && !Compressor::get_comp_alg_type(*compression_type
)) {
5314 std::cerr
<< "Unrecognized compression type" << std::endl
;
5318 RGWZoneParams
zone(zone_id
, zone_name
);
5319 int ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5321 cerr
<< "failed to init zone: " << cpp_strerror(-ret
) << std::endl
;
5325 if (opt_cmd
== OPT::ZONE_PLACEMENT_ADD
||
5326 opt_cmd
== OPT::ZONE_PLACEMENT_MODIFY
) {
5327 RGWZoneGroup
zonegroup(zonegroup_id
, zonegroup_name
);
5328 ret
= zonegroup
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5330 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5334 auto ptiter
= zonegroup
.placement_targets
.find(placement_id
);
5335 if (ptiter
== zonegroup
.placement_targets
.end()) {
5336 cerr
<< "ERROR: placement id '" << placement_id
<< "' is not configured in zonegroup placement targets" << std::endl
;
5340 string storage_class
= rgw_placement_rule::get_canonical_storage_class(opt_storage_class
.value_or(string()));
5341 if (ptiter
->second
.storage_classes
.find(storage_class
) == ptiter
->second
.storage_classes
.end()) {
5342 cerr
<< "ERROR: storage class '" << storage_class
<< "' is not defined in zonegroup '" << placement_id
<< "' placement target" << std::endl
;
5346 RGWZonePlacementInfo
& info
= zone
.placement_pools
[placement_id
];
5348 string opt_index_pool
= index_pool
.value_or(string());
5349 string opt_data_pool
= data_pool
.value_or(string());
5351 if (!opt_index_pool
.empty()) {
5352 info
.index_pool
= opt_index_pool
;
5355 if (info
.index_pool
.empty()) {
5356 cerr
<< "ERROR: index pool not configured, need to specify --index-pool" << std::endl
;
5360 if (opt_data_pool
.empty()) {
5361 const RGWZoneStorageClass
*porig_sc
{nullptr};
5362 if (info
.storage_classes
.find(storage_class
, &porig_sc
)) {
5363 if (porig_sc
->data_pool
) {
5364 opt_data_pool
= porig_sc
->data_pool
->to_str();
5367 if (opt_data_pool
.empty()) {
5368 cerr
<< "ERROR: data pool not configured, need to specify --data-pool" << std::endl
;
5373 rgw_pool dp
= opt_data_pool
;
5374 info
.storage_classes
.set_storage_class(storage_class
, &dp
, compression_type
.get_ptr());
5376 if (data_extra_pool
) {
5377 info
.data_extra_pool
= *data_extra_pool
;
5379 if (index_type_specified
) {
5380 info
.index_type
= placement_index_type
;
5383 ret
= check_pool_support_omap(info
.get_data_extra_pool());
5385 cerr
<< "ERROR: the data extra (non-ec) pool '" << info
.get_data_extra_pool()
5386 << "' does not support omap" << std::endl
;
5389 } else if (opt_cmd
== OPT::ZONE_PLACEMENT_RM
) {
5390 if (!opt_storage_class
||
5391 opt_storage_class
->empty()) {
5392 zone
.placement_pools
.erase(placement_id
);
5394 auto iter
= zone
.placement_pools
.find(placement_id
);
5395 if (iter
!= zone
.placement_pools
.end()) {
5396 RGWZonePlacementInfo
& info
= zone
.placement_pools
[placement_id
];
5397 info
.storage_classes
.remove_storage_class(*opt_storage_class
);
5402 ret
= zone
.update(null_yield
);
5404 cerr
<< "failed to save zone info: " << cpp_strerror(-ret
) << std::endl
;
5408 encode_json("zone", zone
, formatter
.get());
5409 formatter
->flush(cout
);
5412 case OPT::ZONE_PLACEMENT_LIST
:
5414 RGWZoneParams
zone(zone_id
, zone_name
);
5415 int ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5417 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
5420 encode_json("placement_pools", zone
.placement_pools
, formatter
.get());
5421 formatter
->flush(cout
);
5424 case OPT::ZONE_PLACEMENT_GET
:
5426 if (placement_id
.empty()) {
5427 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
5431 RGWZoneParams
zone(zone_id
, zone_name
);
5432 int ret
= zone
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5434 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
5437 auto p
= zone
.placement_pools
.find(placement_id
);
5438 if (p
== zone
.placement_pools
.end()) {
5439 cerr
<< "ERROR: zone placement target '" << placement_id
<< "' not found" << std::endl
;
5442 encode_json("placement_pools", p
->second
, formatter
.get());
5443 formatter
->flush(cout
);
5451 resolve_zone_id_opt(opt_effective_zone_name
, opt_effective_zone_id
);
5452 resolve_zone_id_opt(opt_source_zone_name
, opt_source_zone_id
);
5453 resolve_zone_id_opt(opt_dest_zone_name
, opt_dest_zone_id
);
5454 resolve_zone_ids_opt(opt_zone_names
, opt_zone_ids
);
5455 resolve_zone_ids_opt(opt_source_zone_names
, opt_source_zone_ids
);
5456 resolve_zone_ids_opt(opt_dest_zone_names
, opt_dest_zone_ids
);
5458 bool non_master_cmd
= (!store
->svc()->zone
->is_meta_master() && !yes_i_really_mean_it
);
5459 std::set
<OPT
> non_master_ops_list
= {OPT::USER_CREATE
, OPT::USER_RM
,
5460 OPT::USER_MODIFY
, OPT::USER_ENABLE
,
5461 OPT::USER_SUSPEND
, OPT::SUBUSER_CREATE
,
5462 OPT::SUBUSER_MODIFY
, OPT::SUBUSER_RM
,
5463 OPT::BUCKET_LINK
, OPT::BUCKET_UNLINK
,
5464 OPT::BUCKET_RESHARD
, OPT::BUCKET_RM
,
5465 OPT::BUCKET_CHOWN
, OPT::METADATA_PUT
,
5466 OPT::METADATA_RM
, OPT::RESHARD_CANCEL
,
5467 OPT::RESHARD_ADD
, OPT::MFA_CREATE
,
5468 OPT::MFA_REMOVE
, OPT::MFA_RESYNC
,
5469 OPT::CAPS_ADD
, OPT::CAPS_RM
};
5471 bool print_warning_message
= (non_master_ops_list
.find(opt_cmd
) != non_master_ops_list
.end() &&
5474 if (print_warning_message
) {
5475 cerr
<< "Please run the command on master zone. Performing this operation on non-master zone leads to inconsistent metadata between zones" << std::endl
;
5476 cerr
<< "Are you sure you want to go ahead? (requires --yes-i-really-mean-it)" << std::endl
;
5480 if (!user_id
.empty()) {
5481 user_op
.set_user_id(user_id
);
5482 bucket_op
.set_user_id(user_id
);
5485 if (!display_name
.empty())
5486 user_op
.set_display_name(display_name
);
5488 if (!user_email
.empty())
5489 user_op
.set_user_email(user_email
);
5491 if (!user_id
.empty()) {
5492 user_op
.set_new_user_id(new_user_id
);
5495 if (!access_key
.empty())
5496 user_op
.set_access_key(access_key
);
5498 if (!secret_key
.empty())
5499 user_op
.set_secret_key(secret_key
);
5501 if (!subuser
.empty())
5502 user_op
.set_subuser(subuser
);
5505 user_op
.set_caps(caps
);
5507 user_op
.set_purge_data(purge_data
);
5510 user_op
.set_purge_keys();
5513 user_op
.set_generate_key();
5516 user_op
.set_gen_secret(); // assume that a key pair should be created
5518 if (max_buckets_specified
)
5519 user_op
.set_max_buckets(max_buckets
);
5521 if (admin_specified
)
5522 user_op
.set_admin(admin
);
5524 if (system_specified
)
5525 user_op
.set_system(system
);
5528 user_op
.set_perm(perm_mask
);
5530 if (set_temp_url_key
) {
5531 map
<int, string
>::iterator iter
= temp_url_keys
.begin();
5532 for (; iter
!= temp_url_keys
.end(); ++iter
) {
5533 user_op
.set_temp_url_key(iter
->second
, iter
->first
);
5537 if (!op_mask_str
.empty()) {
5539 int ret
= rgw_parse_op_type_list(op_mask_str
, &op_mask
);
5541 cerr
<< "failed to parse op_mask: " << cpp_strerror(-ret
) << std::endl
;
5545 user_op
.set_op_mask(op_mask
);
5548 if (key_type
!= KEY_TYPE_UNDEFINED
)
5549 user_op
.set_key_type(key_type
);
5551 // set suspension operation parameters
5552 if (opt_cmd
== OPT::USER_ENABLE
)
5553 user_op
.set_suspension(false);
5554 else if (opt_cmd
== OPT::USER_SUSPEND
)
5555 user_op
.set_suspension(true);
5557 if (!placement_id
.empty() ||
5558 (opt_storage_class
&& !opt_storage_class
->empty())) {
5559 rgw_placement_rule target_rule
;
5560 target_rule
.name
= placement_id
;
5561 target_rule
.storage_class
= *opt_storage_class
;
5562 if (!store
->svc()->zone
->get_zone_params().valid_placement(target_rule
)) {
5563 cerr
<< "NOTICE: invalid dest placement: " << target_rule
.to_str() << std::endl
;
5566 user_op
.set_default_placement(target_rule
);
5569 if (!tags
.empty()) {
5570 user_op
.set_placement_tags(tags
);
5573 // RGWUser to use for user operations
5576 if (!(user_id
.empty() && access_key
.empty()) || !subuser
.empty()) {
5577 ret
= user
.init(store
, user_op
, null_yield
);
5579 cerr
<< "user.init failed: " << cpp_strerror(-ret
) << std::endl
;
5584 /* populate bucket operation */
5585 bucket_op
.set_bucket_name(bucket_name
);
5586 bucket_op
.set_object(object
);
5587 bucket_op
.set_check_objects(check_objects
);
5588 bucket_op
.set_delete_children(delete_child_objects
);
5589 bucket_op
.set_fix_index(fix
);
5590 bucket_op
.set_max_aio(max_concurrent_ios
);
5592 // required to gather errors from operations
5593 std::string err_msg
;
5595 bool output_user_info
= true;
5598 case OPT::USER_INFO
:
5599 if (user_id
.empty() && access_key
.empty()) {
5600 cerr
<< "ERROR: --uid or --access-key required" << std::endl
;
5604 case OPT::USER_CREATE
:
5605 if (!user_op
.has_existing_user()) {
5606 user_op
.set_generate_key(); // generate a new key by default
5608 ret
= user
.add(user_op
, null_yield
, &err_msg
);
5610 cerr
<< "could not create user: " << err_msg
<< std::endl
;
5611 if (ret
== -ERR_INVALID_TENANT_NAME
)
5616 if (!subuser
.empty()) {
5617 ret
= user
.subusers
.add(user_op
, null_yield
, &err_msg
);
5619 cerr
<< "could not create subuser: " << err_msg
<< std::endl
;
5625 ret
= user
.remove(user_op
, null_yield
, &err_msg
);
5627 cerr
<< "could not remove user: " << err_msg
<< std::endl
;
5631 output_user_info
= false;
5633 case OPT::USER_RENAME
:
5634 if (yes_i_really_mean_it
) {
5635 user_op
.set_overwrite_new_user(true);
5637 ret
= user
.rename(user_op
, null_yield
, &err_msg
);
5639 if (ret
== -EEXIST
) {
5640 err_msg
+= ". to overwrite this user, add --yes-i-really-mean-it";
5642 cerr
<< "could not rename user: " << err_msg
<< std::endl
;
5647 case OPT::USER_ENABLE
:
5648 case OPT::USER_SUSPEND
:
5649 case OPT::USER_MODIFY
:
5650 ret
= user
.modify(user_op
, null_yield
, &err_msg
);
5652 cerr
<< "could not modify user: " << err_msg
<< std::endl
;
5657 case OPT::SUBUSER_CREATE
:
5658 ret
= user
.subusers
.add(user_op
, null_yield
, &err_msg
);
5660 cerr
<< "could not create subuser: " << err_msg
<< std::endl
;
5665 case OPT::SUBUSER_MODIFY
:
5666 ret
= user
.subusers
.modify(user_op
, null_yield
, &err_msg
);
5668 cerr
<< "could not modify subuser: " << err_msg
<< std::endl
;
5673 case OPT::SUBUSER_RM
:
5674 ret
= user
.subusers
.remove(user_op
, null_yield
, &err_msg
);
5676 cerr
<< "could not remove subuser: " << err_msg
<< std::endl
;
5682 ret
= user
.caps
.add(user_op
, null_yield
, &err_msg
);
5684 cerr
<< "could not add caps: " << err_msg
<< std::endl
;
5690 ret
= user
.caps
.remove(user_op
, null_yield
, &err_msg
);
5692 cerr
<< "could not remove caps: " << err_msg
<< std::endl
;
5697 case OPT::KEY_CREATE
:
5698 ret
= user
.keys
.add(user_op
, null_yield
, &err_msg
);
5700 cerr
<< "could not create key: " << err_msg
<< std::endl
;
5706 ret
= user
.keys
.remove(user_op
, null_yield
, &err_msg
);
5708 cerr
<< "could not remove key: " << err_msg
<< std::endl
;
5712 case OPT::PERIOD_PUSH
:
5715 req_info
info(g_ceph_context
, &env
);
5716 info
.method
= "POST";
5717 info
.request_uri
= "/admin/realm/period";
5719 map
<string
, string
> ¶ms
= info
.args
.get_params();
5720 if (!realm_id
.empty())
5721 params
["realm_id"] = realm_id
;
5722 if (!realm_name
.empty())
5723 params
["realm_name"] = realm_name
;
5724 if (!period_id
.empty())
5725 params
["period_id"] = period_id
;
5726 if (!period_epoch
.empty())
5727 params
["epoch"] = period_epoch
;
5730 RGWPeriod
period(period_id
);
5731 int ret
= period
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5733 cerr
<< "period init failed: " << cpp_strerror(-ret
) << std::endl
;
5736 // json format into a bufferlist
5737 JSONFormatter
jf(false);
5738 encode_json("period", period
, &jf
);
5743 ret
= send_to_remote_or_url(nullptr, url
, access_key
, secret_key
,
5746 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
5751 case OPT::PERIOD_UPDATE
:
5753 int ret
= update_period(realm_id
, realm_name
, period_id
, period_epoch
,
5754 commit
, remote
, url
, access_key
, secret_key
,
5755 formatter
.get(), yes_i_really_mean_it
);
5761 case OPT::PERIOD_COMMIT
:
5763 // read realm and staging period
5764 RGWRealm
realm(realm_id
, realm_name
);
5765 int ret
= realm
.init(g_ceph_context
, store
->svc()->sysobj
, null_yield
);
5767 cerr
<< "Error initializing realm: " << cpp_strerror(-ret
) << std::endl
;
5770 RGWPeriod
period(RGWPeriod::get_staging_id(realm
.get_id()), 1);
5771 ret
= period
.init(g_ceph_context
, store
->svc()->sysobj
, realm
.get_id(), null_yield
);
5773 cerr
<< "period init failed: " << cpp_strerror(-ret
) << std::endl
;
5776 ret
= commit_period(realm
, period
, remote
, url
, access_key
, secret_key
,
5777 yes_i_really_mean_it
);
5779 cerr
<< "failed to commit period: " << cpp_strerror(-ret
) << std::endl
;
5783 encode_json("period", period
, formatter
.get());
5784 formatter
->flush(cout
);
5787 case OPT::ROLE_CREATE
:
5789 if (role_name
.empty()) {
5790 cerr
<< "ERROR: role name is empty" << std::endl
;
5794 if (assume_role_doc
.empty()) {
5795 cerr
<< "ERROR: assume role policy document is empty" << std::endl
;
5798 bufferlist bl
= bufferlist::static_from_string(assume_role_doc
);
5800 const rgw::IAM::Policy
p(g_ceph_context
, tenant
, bl
);
5801 } catch (rgw::IAM::PolicyParseException
& e
) {
5802 cerr
<< "failed to parse policy: " << e
.what() << std::endl
;
5805 RGWRole
role(g_ceph_context
, store
->getRados()->pctl
, role_name
, path
, assume_role_doc
, tenant
);
5806 ret
= role
.create(true, null_yield
);
5810 show_role_info(role
, formatter
.get());
5813 case OPT::ROLE_DELETE
:
5815 if (role_name
.empty()) {
5816 cerr
<< "ERROR: empty role name" << std::endl
;
5819 RGWRole
role(g_ceph_context
, store
->getRados()->pctl
, role_name
, tenant
);
5820 ret
= role
.delete_obj(null_yield
);
5824 cout
<< "role: " << role_name
<< " successfully deleted" << std::endl
;
5829 if (role_name
.empty()) {
5830 cerr
<< "ERROR: empty role name" << std::endl
;
5833 RGWRole
role(g_ceph_context
, store
->getRados()->pctl
, role_name
, tenant
);
5834 ret
= role
.get(null_yield
);
5838 show_role_info(role
, formatter
.get());
5841 case OPT::ROLE_MODIFY
:
5843 if (role_name
.empty()) {
5844 cerr
<< "ERROR: role name is empty" << std::endl
;
5848 if (assume_role_doc
.empty()) {
5849 cerr
<< "ERROR: assume role policy document is empty" << std::endl
;
5853 bufferlist bl
= bufferlist::static_from_string(assume_role_doc
);
5855 const rgw::IAM::Policy
p(g_ceph_context
, tenant
, bl
);
5856 } catch (rgw::IAM::PolicyParseException
& e
) {
5857 cerr
<< "failed to parse policy: " << e
.what() << std::endl
;
5861 RGWRole
role(g_ceph_context
, store
->getRados()->pctl
, role_name
, tenant
);
5862 ret
= role
.get(null_yield
);
5866 role
.update_trust_policy(assume_role_doc
);
5867 ret
= role
.update(null_yield
);
5871 cout
<< "Assume role policy document updated successfully for role: " << role_name
<< std::endl
;
5874 case OPT::ROLE_LIST
:
5876 vector
<RGWRole
> result
;
5877 ret
= RGWRole::get_roles_by_path_prefix(store
->getRados(), g_ceph_context
, path_prefix
, tenant
, result
, null_yield
);
5881 show_roles_info(result
, formatter
.get());
5884 case OPT::ROLE_POLICY_PUT
:
5886 if (role_name
.empty()) {
5887 cerr
<< "role name is empty" << std::endl
;
5891 if (policy_name
.empty()) {
5892 cerr
<< "policy name is empty" << std::endl
;
5896 if (perm_policy_doc
.empty()) {
5897 cerr
<< "permission policy document is empty" << std::endl
;
5901 bufferlist bl
= bufferlist::static_from_string(perm_policy_doc
);
5903 const rgw::IAM::Policy
p(g_ceph_context
, tenant
, bl
);
5904 } catch (rgw::IAM::PolicyParseException
& e
) {
5905 cerr
<< "failed to parse perm policy: " << e
.what() << std::endl
;
5909 RGWRole
role(g_ceph_context
, store
->getRados()->pctl
, role_name
, tenant
);
5910 ret
= role
.get(null_yield
);
5914 role
.set_perm_policy(policy_name
, perm_policy_doc
);
5915 ret
= role
.update(null_yield
);
5919 cout
<< "Permission policy attached successfully" << std::endl
;
5922 case OPT::ROLE_POLICY_LIST
:
5924 if (role_name
.empty()) {
5925 cerr
<< "ERROR: Role name is empty" << std::endl
;
5928 RGWRole
role(g_ceph_context
, store
->getRados()->pctl
, role_name
, tenant
);
5929 ret
= role
.get(null_yield
);
5933 std::vector
<string
> policy_names
= role
.get_role_policy_names();
5934 show_policy_names(policy_names
, formatter
.get());
5937 case OPT::ROLE_POLICY_GET
:
5939 if (role_name
.empty()) {
5940 cerr
<< "ERROR: role name is empty" << std::endl
;
5944 if (policy_name
.empty()) {
5945 cerr
<< "ERROR: policy name is empty" << std::endl
;
5948 RGWRole
role(g_ceph_context
, store
->getRados()->pctl
, role_name
, tenant
);
5949 int ret
= role
.get(null_yield
);
5954 ret
= role
.get_role_policy(policy_name
, perm_policy
);
5958 show_perm_policy(perm_policy
, formatter
.get());
5961 case OPT::ROLE_POLICY_DELETE
:
5963 if (role_name
.empty()) {
5964 cerr
<< "ERROR: role name is empty" << std::endl
;
5968 if (policy_name
.empty()) {
5969 cerr
<< "ERROR: policy name is empty" << std::endl
;
5972 RGWRole
role(g_ceph_context
, store
->getRados()->pctl
, role_name
, tenant
);
5973 ret
= role
.get(null_yield
);
5977 ret
= role
.delete_policy(policy_name
);
5981 ret
= role
.update(null_yield
);
5985 cout
<< "Policy: " << policy_name
<< " successfully deleted for role: "
5986 << role_name
<< std::endl
;
5990 output_user_info
= false;
5993 // output the result of a user operation
5994 if (output_user_info
) {
5995 ret
= user
.info(info
, &err_msg
);
5997 cerr
<< "could not fetch user info: " << err_msg
<< std::endl
;
6000 show_user_info(info
, formatter
.get());
6003 if (opt_cmd
== OPT::POLICY
) {
6004 if (format
== "xml") {
6005 int ret
= RGWBucketAdminOp::dump_s3_policy(store
, bucket_op
, cout
);
6007 cerr
<< "ERROR: failed to get policy: " << cpp_strerror(-ret
) << std::endl
;
6011 int ret
= RGWBucketAdminOp::get_policy(store
, bucket_op
, f
);
6013 cerr
<< "ERROR: failed to get policy: " << cpp_strerror(-ret
) << std::endl
;
6019 if (opt_cmd
== OPT::BUCKET_LIMIT_CHECK
) {
6021 std::list
<std::string
> user_ids
;
6022 metadata_key
= "user";
6027 if (! user_id
.empty()) {
6028 user_ids
.push_back(user_id
.id
);
6030 RGWBucketAdminOp::limit_check(store
, bucket_op
, user_ids
, f
,
6031 null_yield
, warnings_only
);
6033 /* list users in groups of max-keys, then perform user-bucket
6034 * limit-check on each group */
6035 ret
= store
->ctl()->meta
.mgr
->list_keys_init(metadata_key
, &handle
);
6037 cerr
<< "ERROR: buckets limit check can't get user metadata_key: "
6038 << cpp_strerror(-ret
) << std::endl
;
6043 ret
= store
->ctl()->meta
.mgr
->list_keys_next(handle
, max
, user_ids
,
6045 if (ret
< 0 && ret
!= -ENOENT
) {
6046 cerr
<< "ERROR: buckets limit check lists_keys_next(): "
6047 << cpp_strerror(-ret
) << std::endl
;
6050 /* ok, do the limit checks for this group */
6052 RGWBucketAdminOp::limit_check(store
, bucket_op
, user_ids
, f
,
6053 null_yield
, warnings_only
);
6058 } while (truncated
);
6059 store
->ctl()->meta
.mgr
->list_keys_complete(handle
);
6062 } /* OPT::BUCKET_LIMIT_CHECK */
6064 if (opt_cmd
== OPT::BUCKETS_LIST
) {
6065 if (bucket_name
.empty()) {
6066 if (!user_id
.empty()) {
6067 if (!user_op
.has_existing_user()) {
6068 cerr
<< "ERROR: could not find user: " << user_id
<< std::endl
;
6072 RGWBucketAdminOp::info(store
, bucket_op
, f
, null_yield
);
6074 RGWBucketInfo bucket_info
;
6075 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
6077 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
6080 formatter
->open_array_section("entries");
6082 bool truncated
= false;
6085 static constexpr int MAX_PAGINATE_SIZE
= 10000;
6086 static constexpr int DEFAULT_MAX_ENTRIES
= 1000;
6088 if (max_entries
< 0) {
6089 max_entries
= DEFAULT_MAX_ENTRIES
;
6091 const int paginate_size
= std::min(max_entries
, MAX_PAGINATE_SIZE
);
6095 vector
<rgw_bucket_dir_entry
> result
;
6096 map
<string
, bool> common_prefixes
;
6099 RGWRados::Bucket
target(store
->getRados(), bucket_info
);
6100 RGWRados::Bucket::List
list_op(&target
);
6102 list_op
.params
.prefix
= prefix
;
6103 list_op
.params
.delim
= delim
;
6104 list_op
.params
.marker
= rgw_obj_key(marker
);
6105 list_op
.params
.ns
= ns
;
6106 list_op
.params
.enforce_ns
= false;
6107 list_op
.params
.list_versions
= true;
6108 list_op
.params
.allow_unordered
= bool(allow_unordered
);
6111 const int remaining
= max_entries
- count
;
6112 ret
= list_op
.list_objects(std::min(remaining
, paginate_size
),
6113 &result
, &common_prefixes
, &truncated
,
6116 cerr
<< "ERROR: store->list_objects(): " << cpp_strerror(-ret
) << std::endl
;
6120 count
+= result
.size();
6122 for (const auto& entry
: result
) {
6123 encode_json("entry", entry
, formatter
.get());
6125 formatter
->flush(cout
);
6126 } while (truncated
&& count
< max_entries
);
6128 formatter
->close_section();
6129 formatter
->flush(cout
);
6130 } /* have bucket_name */
6131 } /* OPT::BUCKETS_LIST */
6133 if (opt_cmd
== OPT::BUCKET_RADOS_LIST
) {
6134 RGWRadosList
lister(store
,
6135 max_concurrent_ios
, orphan_stale_secs
, tenant
);
6137 lister
.set_field_separator(*rgw_obj_fs
);
6140 if (bucket_name
.empty()) {
6143 ret
= lister
.run(bucket_name
);
6148 "ERROR: bucket radoslist failed to finish before " <<
6149 "encountering error: " << cpp_strerror(-ret
) << std::endl
;
6150 std::cerr
<< "************************************"
6151 "************************************" << std::endl
;
6152 std::cerr
<< "WARNING: THE RESULTS ARE NOT RELIABLE AND SHOULD NOT " <<
6153 "BE USED IN DELETING ORPHANS" << std::endl
;
6154 std::cerr
<< "************************************"
6155 "************************************" << std::endl
;
6160 if (opt_cmd
== OPT::BUCKET_STATS
) {
6161 if (bucket_name
.empty() && !bucket_id
.empty()) {
6163 if (!rgw_find_bucket_by_id(store
->ctx(), store
->ctl()->meta
.mgr
, marker
, bucket_id
, &bucket
)) {
6164 cerr
<< "failure: no such bucket id" << std::endl
;
6167 bucket_op
.set_tenant(bucket
.tenant
);
6168 bucket_op
.set_bucket_name(bucket
.name
);
6170 bucket_op
.set_fetch_stats(true);
6172 int r
= RGWBucketAdminOp::info(store
, bucket_op
, f
, null_yield
);
6174 cerr
<< "failure: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
6179 if (opt_cmd
== OPT::BUCKET_LINK
) {
6180 bucket_op
.set_bucket_id(bucket_id
);
6181 bucket_op
.set_new_bucket_name(new_bucket_name
);
6183 int r
= RGWBucketAdminOp::link(store
, bucket_op
, &err
);
6185 cerr
<< "failure: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
6190 if (opt_cmd
== OPT::BUCKET_UNLINK
) {
6191 int r
= RGWBucketAdminOp::unlink(store
, bucket_op
);
6193 cerr
<< "failure: " << cpp_strerror(-r
) << std::endl
;
6198 if (opt_cmd
== OPT::BUCKET_CHOWN
) {
6200 bucket_op
.set_bucket_name(bucket_name
);
6201 bucket_op
.set_new_bucket_name(new_bucket_name
);
6205 int r
= RGWBucketAdminOp::chown(store
, bucket_op
, marker
, &err
);
6207 cerr
<< "failure: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
6212 if (opt_cmd
== OPT::LOG_LIST
) {
6214 if (date
.size() && date
.size() != 10) {
6215 cerr
<< "bad date format for '" << date
<< "', expect YYYY-MM-DD" << std::endl
;
6220 formatter
->open_array_section("logs");
6222 int r
= store
->getRados()->log_list_init(date
, &h
);
6227 cerr
<< "log list: error " << r
<< std::endl
;
6232 int r
= store
->getRados()->log_list_next(h
, &name
);
6236 cerr
<< "log list: error " << r
<< std::endl
;
6239 formatter
->dump_string("object", name
);
6242 formatter
->close_section();
6243 formatter
->flush(cout
);
6247 if (opt_cmd
== OPT::LOG_SHOW
|| opt_cmd
== OPT::LOG_RM
) {
6248 if (object
.empty() && (date
.empty() || bucket_name
.empty() || bucket_id
.empty())) {
6249 cerr
<< "specify an object or a date, bucket and bucket-id" << std::endl
;
6254 if (!object
.empty()) {
6264 if (opt_cmd
== OPT::LOG_SHOW
) {
6267 int r
= store
->getRados()->log_show_init(oid
, &h
);
6269 cerr
<< "error opening log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
6274 formatter
->open_object_section("log");
6276 struct rgw_log_entry entry
;
6278 // peek at first entry to get bucket metadata
6279 r
= store
->getRados()->log_show_next(h
, &entry
);
6281 cerr
<< "error reading log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
6284 formatter
->dump_string("bucket_id", entry
.bucket_id
);
6285 formatter
->dump_string("bucket_owner", entry
.bucket_owner
.to_str());
6286 formatter
->dump_string("bucket", entry
.bucket
);
6288 uint64_t agg_time
= 0;
6289 uint64_t agg_bytes_sent
= 0;
6290 uint64_t agg_bytes_received
= 0;
6291 uint64_t total_entries
= 0;
6293 if (show_log_entries
)
6294 formatter
->open_array_section("log_entries");
6297 using namespace std::chrono
;
6298 uint64_t total_time
= duration_cast
<milliseconds
>(entry
.total_time
).count();
6300 agg_time
+= total_time
;
6301 agg_bytes_sent
+= entry
.bytes_sent
;
6302 agg_bytes_received
+= entry
.bytes_received
;
6305 if (skip_zero_entries
&& entry
.bytes_sent
== 0 &&
6306 entry
.bytes_received
== 0)
6309 if (show_log_entries
) {
6311 rgw_format_ops_log_entry(entry
, formatter
.get());
6312 formatter
->flush(cout
);
6315 r
= store
->getRados()->log_show_next(h
, &entry
);
6319 cerr
<< "error reading log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
6322 if (show_log_entries
)
6323 formatter
->close_section();
6326 formatter
->open_object_section("log_sum");
6327 formatter
->dump_int("bytes_sent", agg_bytes_sent
);
6328 formatter
->dump_int("bytes_received", agg_bytes_received
);
6329 formatter
->dump_int("total_time", agg_time
);
6330 formatter
->dump_int("total_entries", total_entries
);
6331 formatter
->close_section();
6333 formatter
->close_section();
6334 formatter
->flush(cout
);
6337 if (opt_cmd
== OPT::LOG_RM
) {
6338 int r
= store
->getRados()->log_remove(oid
);
6340 cerr
<< "error removing log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
6346 if (opt_cmd
== OPT::POOL_ADD
) {
6347 if (pool_name
.empty()) {
6348 cerr
<< "need to specify pool to add!" << std::endl
;
6352 int ret
= store
->svc()->zone
->add_bucket_placement(pool
, null_yield
);
6354 cerr
<< "failed to add bucket placement: " << cpp_strerror(-ret
) << std::endl
;
6357 if (opt_cmd
== OPT::POOL_RM
) {
6358 if (pool_name
.empty()) {
6359 cerr
<< "need to specify pool to remove!" << std::endl
;
6363 int ret
= store
->svc()->zone
->remove_bucket_placement(pool
, null_yield
);
6365 cerr
<< "failed to remove bucket placement: " << cpp_strerror(-ret
) << std::endl
;
6368 if (opt_cmd
== OPT::POOLS_LIST
) {
6369 set
<rgw_pool
> pools
;
6370 int ret
= store
->svc()->zone
->list_placement_set(pools
, null_yield
);
6372 cerr
<< "could not list placement set: " << cpp_strerror(-ret
) << std::endl
;
6376 formatter
->open_array_section("pools");
6377 for (auto siter
= pools
.begin(); siter
!= pools
.end(); ++siter
) {
6378 formatter
->open_object_section("pool");
6379 formatter
->dump_string("name", siter
->to_str());
6380 formatter
->close_section();
6382 formatter
->close_section();
6383 formatter
->flush(cout
);
6387 if (opt_cmd
== OPT::USAGE_SHOW
) {
6388 uint64_t start_epoch
= 0;
6389 uint64_t end_epoch
= (uint64_t)-1;
6393 if (!start_date
.empty()) {
6394 ret
= utime_t::parse_date(start_date
, &start_epoch
, NULL
);
6396 cerr
<< "ERROR: failed to parse start date" << std::endl
;
6400 if (!end_date
.empty()) {
6401 ret
= utime_t::parse_date(end_date
, &end_epoch
, NULL
);
6403 cerr
<< "ERROR: failed to parse end date" << std::endl
;
6409 ret
= RGWUsage::show(store
->getRados(), user_id
, bucket_name
, start_epoch
, end_epoch
,
6410 show_log_entries
, show_log_sum
, &categories
,
6413 cerr
<< "ERROR: failed to show usage" << std::endl
;
6418 if (opt_cmd
== OPT::USAGE_TRIM
) {
6419 if (user_id
.empty() && bucket_name
.empty() &&
6420 start_date
.empty() && end_date
.empty() && !yes_i_really_mean_it
) {
6421 cerr
<< "usage trim without user/date/bucket specified will remove *all* users data" << std::endl
;
6422 cerr
<< "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
6426 uint64_t start_epoch
= 0;
6427 uint64_t end_epoch
= (uint64_t)-1;
6430 if (!start_date
.empty()) {
6431 ret
= utime_t::parse_date(start_date
, &start_epoch
, NULL
);
6433 cerr
<< "ERROR: failed to parse start date" << std::endl
;
6438 if (!end_date
.empty()) {
6439 ret
= utime_t::parse_date(end_date
, &end_epoch
, NULL
);
6441 cerr
<< "ERROR: failed to parse end date" << std::endl
;
6446 ret
= RGWUsage::trim(store
->getRados(), user_id
, bucket_name
, start_epoch
, end_epoch
);
6448 cerr
<< "ERROR: read_usage() returned ret=" << ret
<< std::endl
;
6453 if (opt_cmd
== OPT::USAGE_CLEAR
) {
6454 if (!yes_i_really_mean_it
) {
6455 cerr
<< "usage clear would remove *all* users usage data for all time" << std::endl
;
6456 cerr
<< "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
6460 ret
= RGWUsage::clear(store
->getRados());
6467 if (opt_cmd
== OPT::OLH_GET
|| opt_cmd
== OPT::OLH_READLOG
) {
6468 if (bucket_name
.empty()) {
6469 cerr
<< "ERROR: bucket not specified" << std::endl
;
6472 if (object
.empty()) {
6473 cerr
<< "ERROR: object not specified" << std::endl
;
6478 if (opt_cmd
== OPT::OLH_GET
) {
6479 RGWBucketInfo bucket_info
;
6480 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
6482 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
6486 rgw_obj
obj(bucket
, object
);
6487 ret
= store
->getRados()->get_olh(bucket_info
, obj
, &olh
);
6489 cerr
<< "ERROR: failed reading olh: " << cpp_strerror(-ret
) << std::endl
;
6492 encode_json("olh", olh
, formatter
.get());
6493 formatter
->flush(cout
);
6496 if (opt_cmd
== OPT::OLH_READLOG
) {
6497 RGWBucketInfo bucket_info
;
6498 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
6500 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
6503 map
<uint64_t, vector
<rgw_bucket_olh_log_entry
> > log
;
6506 RGWObjectCtx
rctx(store
);
6507 rgw_obj
obj(bucket
, object
);
6511 ret
= store
->getRados()->get_obj_state(&rctx
, bucket_info
, obj
, &state
, false, null_yield
); /* don't follow olh */
6516 ret
= store
->getRados()->bucket_index_read_olh_log(bucket_info
, *state
, obj
, 0, &log
, &is_truncated
);
6518 cerr
<< "ERROR: failed reading olh: " << cpp_strerror(-ret
) << std::endl
;
6521 formatter
->open_object_section("result");
6522 encode_json("is_truncated", is_truncated
, formatter
.get());
6523 encode_json("log", log
, formatter
.get());
6524 formatter
->close_section();
6525 formatter
->flush(cout
);
6528 if (opt_cmd
== OPT::BI_GET
) {
6529 if (bucket_name
.empty()) {
6530 cerr
<< "ERROR: bucket name not specified" << std::endl
;
6533 if (object
.empty()) {
6534 cerr
<< "ERROR: object not specified" << std::endl
;
6537 RGWBucketInfo bucket_info
;
6538 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
6540 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
6543 rgw_obj
obj(bucket
, object
);
6544 if (!object_version
.empty()) {
6545 obj
.key
.set_instance(object_version
);
6548 rgw_cls_bi_entry entry
;
6550 ret
= store
->getRados()->bi_get(bucket_info
, obj
, bi_index_type
, &entry
);
6552 cerr
<< "ERROR: bi_get(): " << cpp_strerror(-ret
) << std::endl
;
6556 encode_json("entry", entry
, formatter
.get());
6557 formatter
->flush(cout
);
6560 if (opt_cmd
== OPT::BI_PUT
) {
6561 if (bucket_name
.empty()) {
6562 cerr
<< "ERROR: bucket name not specified" << std::endl
;
6565 RGWBucketInfo bucket_info
;
6566 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
6568 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
6572 rgw_cls_bi_entry entry
;
6573 cls_rgw_obj_key key
;
6574 ret
= read_decode_json(infile
, entry
, &key
);
6579 rgw_obj
obj(bucket
, key
);
6581 ret
= store
->getRados()->bi_put(bucket
, obj
, entry
);
6583 cerr
<< "ERROR: bi_put(): " << cpp_strerror(-ret
) << std::endl
;
6588 if (opt_cmd
== OPT::BI_LIST
) {
6589 if (bucket_name
.empty()) {
6590 cerr
<< "ERROR: bucket name not specified" << std::endl
;
6593 RGWBucketInfo bucket_info
;
6594 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
6596 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
6600 list
<rgw_cls_bi_entry
> entries
;
6602 if (max_entries
< 0) {
6606 int max_shards
= (bucket_info
.layout
.current_index
.layout
.normal
.num_shards
> 0 ? bucket_info
.layout
.current_index
.layout
.normal
.num_shards
: 1);
6608 formatter
->open_array_section("entries");
6610 int i
= (specified_shard_id
? shard_id
: 0);
6611 for (; i
< max_shards
; i
++) {
6612 RGWRados::BucketShard
bs(store
->getRados());
6613 int shard_id
= (bucket_info
.layout
.current_index
.layout
.normal
.num_shards
> 0 ? i
: -1);
6615 int ret
= bs
.init(bucket
, shard_id
, bucket_info
.layout
.current_index
, nullptr /* no RGWBucketInfo */);
6619 cerr
<< "ERROR: bs.init(bucket=" << bucket
<< ", shard=" << shard_id
<< "): " << cpp_strerror(-ret
) << std::endl
;
6625 ret
= store
->getRados()->bi_list(bs
, object
, marker
, max_entries
, &entries
, &is_truncated
);
6627 cerr
<< "ERROR: bi_list(): " << cpp_strerror(-ret
) << std::endl
;
6631 list
<rgw_cls_bi_entry
>::iterator iter
;
6632 for (iter
= entries
.begin(); iter
!= entries
.end(); ++iter
) {
6633 rgw_cls_bi_entry
& entry
= *iter
;
6634 encode_json("entry", entry
, formatter
.get());
6637 formatter
->flush(cout
);
6638 } while (is_truncated
);
6639 formatter
->flush(cout
);
6641 if (specified_shard_id
)
6644 formatter
->close_section();
6645 formatter
->flush(cout
);
6648 if (opt_cmd
== OPT::BI_PURGE
) {
6649 if (bucket_name
.empty()) {
6650 cerr
<< "ERROR: bucket name not specified" << std::endl
;
6653 RGWBucketInfo bucket_info
;
6654 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
6656 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
6660 RGWBucketInfo cur_bucket_info
;
6661 rgw_bucket cur_bucket
;
6662 ret
= init_bucket(tenant
, bucket_name
, string(), cur_bucket_info
, cur_bucket
);
6664 cerr
<< "ERROR: could not init current bucket info for bucket_name=" << bucket_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
6668 if (cur_bucket_info
.bucket
.bucket_id
== bucket_info
.bucket
.bucket_id
&& !yes_i_really_mean_it
) {
6669 cerr
<< "specified bucket instance points to a current bucket instance" << std::endl
;
6670 cerr
<< "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
6674 int max_shards
= (bucket_info
.layout
.current_index
.layout
.normal
.num_shards
> 0 ? bucket_info
.layout
.current_index
.layout
.normal
.num_shards
: 1);
6676 for (int i
= 0; i
< max_shards
; i
++) {
6677 RGWRados::BucketShard
bs(store
->getRados());
6678 int shard_id
= (bucket_info
.layout
.current_index
.layout
.normal
.num_shards
> 0 ? i
: -1);
6679 int ret
= bs
.init(bucket
, shard_id
, bucket_info
.layout
.current_index
, nullptr /* no RGWBucketInfo */);
6681 cerr
<< "ERROR: bs.init(bucket=" << bucket
<< ", shard=" << shard_id
<< "): " << cpp_strerror(-ret
) << std::endl
;
6685 ret
= store
->getRados()->bi_remove(bs
);
6687 cerr
<< "ERROR: failed to remove bucket index object: " << cpp_strerror(-ret
) << std::endl
;
6693 if (opt_cmd
== OPT::OBJECT_PUT
) {
6694 if (bucket_name
.empty()) {
6695 cerr
<< "ERROR: bucket not specified" << std::endl
;
6698 if (object
.empty()) {
6699 cerr
<< "ERROR: object not specified" << std::endl
;
6703 RGWDataAccess
data_access(store
);
6704 rgw_obj_key
key(object
, object_version
);
6706 RGWDataAccess::BucketRef b
;
6707 RGWDataAccess::ObjectRef obj
;
6709 int ret
= data_access
.get_bucket(tenant
, bucket_name
, bucket_id
, &b
, null_yield
);
6711 cerr
<< "ERROR: failed to init bucket: " << cpp_strerror(-ret
) << std::endl
;
6715 ret
= b
->get_object(key
, &obj
);
6717 cerr
<< "ERROR: failed to get object: " << cpp_strerror(-ret
) << std::endl
;
6722 ret
= read_input(infile
, bl
);
6724 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
6727 map
<string
, bufferlist
> attrs
;
6728 ret
= obj
->put(bl
, attrs
, dpp(), null_yield
);
6730 cerr
<< "ERROR: put object returned error: " << cpp_strerror(-ret
) << std::endl
;
6734 if (opt_cmd
== OPT::OBJECT_RM
) {
6735 RGWBucketInfo bucket_info
;
6736 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
6738 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
6741 rgw_obj_key
key(object
, object_version
);
6742 ret
= rgw_remove_object(store
, bucket_info
, bucket
, key
);
6745 cerr
<< "ERROR: object remove returned: " << cpp_strerror(-ret
) << std::endl
;
6750 if (opt_cmd
== OPT::OBJECT_REWRITE
) {
6751 if (bucket_name
.empty()) {
6752 cerr
<< "ERROR: bucket not specified" << std::endl
;
6755 if (object
.empty()) {
6756 cerr
<< "ERROR: object not specified" << std::endl
;
6760 RGWBucketInfo bucket_info
;
6761 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
6763 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
6767 rgw::sal::RGWRadosBucket
rbucket(store
, bucket
);
6768 rgw::sal::RGWRadosObject
obj(store
, object
, &rbucket
);
6769 obj
.set_instance(object_version
);
6770 bool need_rewrite
= true;
6771 if (min_rewrite_stripe_size
> 0) {
6772 ret
= check_min_obj_stripe_size(store
, bucket_info
, &obj
, min_rewrite_stripe_size
, &need_rewrite
);
6774 ldout(store
->ctx(), 0) << "WARNING: check_min_obj_stripe_size failed, r=" << ret
<< dendl
;
6778 ret
= store
->getRados()->rewrite_obj(bucket_info
, &obj
, dpp(), null_yield
);
6780 cerr
<< "ERROR: object rewrite returned: " << cpp_strerror(-ret
) << std::endl
;
6784 ldout(store
->ctx(), 20) << "skipped object" << dendl
;
6788 if (opt_cmd
== OPT::OBJECTS_EXPIRE
) {
6789 if (!store
->getRados()->process_expire_objects()) {
6790 cerr
<< "ERROR: process_expire_objects() processing returned error." << std::endl
;
6795 if (opt_cmd
== OPT::OBJECTS_EXPIRE_STALE_LIST
) {
6796 ret
= RGWBucketAdminOp::fix_obj_expiry(store
, bucket_op
, f
, true);
6798 cerr
<< "ERROR: listing returned " << cpp_strerror(-ret
) << std::endl
;
6803 if (opt_cmd
== OPT::OBJECTS_EXPIRE_STALE_RM
) {
6804 ret
= RGWBucketAdminOp::fix_obj_expiry(store
, bucket_op
, f
, false);
6806 cerr
<< "ERROR: removing returned " << cpp_strerror(-ret
) << std::endl
;
6811 if (opt_cmd
== OPT::BUCKET_REWRITE
) {
6812 if (bucket_name
.empty()) {
6813 cerr
<< "ERROR: bucket not specified" << std::endl
;
6817 RGWBucketInfo bucket_info
;
6818 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
6820 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
6824 uint64_t start_epoch
= 0;
6825 uint64_t end_epoch
= 0;
6827 if (!end_date
.empty()) {
6828 int ret
= utime_t::parse_date(end_date
, &end_epoch
, NULL
);
6830 cerr
<< "ERROR: failed to parse end date" << std::endl
;
6834 if (!start_date
.empty()) {
6835 int ret
= utime_t::parse_date(start_date
, &start_epoch
, NULL
);
6837 cerr
<< "ERROR: failed to parse start date" << std::endl
;
6842 bool is_truncated
= true;
6843 bool cls_filtered
= true;
6845 rgw_obj_index_key marker
;
6846 string empty_prefix
;
6847 string empty_delimiter
;
6849 formatter
->open_object_section("result");
6850 formatter
->dump_string("bucket", bucket_name
);
6851 formatter
->open_array_section("objects");
6853 constexpr uint32_t NUM_ENTRIES
= 1000;
6854 uint16_t expansion_factor
= 1;
6855 while (is_truncated
) {
6856 RGWRados::ent_map_t result
;
6857 result
.reserve(NUM_ENTRIES
);
6859 int r
= store
->getRados()->cls_bucket_list_ordered(
6860 bucket_info
, RGW_NO_SHARD
,
6861 marker
, empty_prefix
, empty_delimiter
,
6862 NUM_ENTRIES
, true, expansion_factor
,
6863 result
, &is_truncated
, &cls_filtered
, &marker
,
6865 rgw_bucket_object_check_filter
);
6866 if (r
< 0 && r
!= -ENOENT
) {
6867 cerr
<< "ERROR: failed operation r=" << r
<< std::endl
;
6868 } else if (r
== -ENOENT
) {
6872 if (result
.size() < NUM_ENTRIES
/ 8) {
6874 } else if (result
.size() > NUM_ENTRIES
* 7 / 8 &&
6875 expansion_factor
> 1) {
6879 for (auto iter
= result
.begin(); iter
!= result
.end(); ++iter
) {
6880 rgw_obj_key key
= iter
->second
.key
;
6881 rgw_bucket_dir_entry
& entry
= iter
->second
;
6883 formatter
->open_object_section("object");
6884 formatter
->dump_string("name", key
.name
);
6885 formatter
->dump_string("instance", key
.instance
);
6886 formatter
->dump_int("size", entry
.meta
.size
);
6887 utime_t
ut(entry
.meta
.mtime
);
6888 ut
.gmtime(formatter
->dump_stream("mtime"));
6890 if ((entry
.meta
.size
< min_rewrite_size
) ||
6891 (entry
.meta
.size
> max_rewrite_size
) ||
6892 (start_epoch
> 0 && start_epoch
> (uint64_t)ut
.sec()) ||
6893 (end_epoch
> 0 && end_epoch
< (uint64_t)ut
.sec())) {
6894 formatter
->dump_string("status", "Skipped");
6896 rgw::sal::RGWRadosBucket
rbucket(store
, bucket
);
6897 rgw::sal::RGWRadosObject
obj(store
, key
, &rbucket
);
6899 bool need_rewrite
= true;
6900 if (min_rewrite_stripe_size
> 0) {
6901 r
= check_min_obj_stripe_size(store
, bucket_info
, &obj
, min_rewrite_stripe_size
, &need_rewrite
);
6903 ldout(store
->ctx(), 0) << "WARNING: check_min_obj_stripe_size failed, r=" << r
<< dendl
;
6906 if (!need_rewrite
) {
6907 formatter
->dump_string("status", "Skipped");
6909 r
= store
->getRados()->rewrite_obj(bucket_info
, &obj
, dpp(), null_yield
);
6911 formatter
->dump_string("status", "Success");
6913 formatter
->dump_string("status", cpp_strerror(-r
));
6917 formatter
->dump_int("flags", entry
.flags
);
6919 formatter
->close_section();
6920 formatter
->flush(cout
);
6923 formatter
->close_section();
6924 formatter
->close_section();
6925 formatter
->flush(cout
);
6928 if (opt_cmd
== OPT::BUCKET_RESHARD
) {
6930 RGWBucketInfo bucket_info
;
6931 map
<string
, bufferlist
> attrs
;
6933 int ret
= check_reshard_bucket_params(store
,
6937 num_shards_specified
,
6939 yes_i_really_mean_it
,
6947 RGWBucketReshard
br(store
, bucket_info
, attrs
, nullptr /* no callback */);
6949 #define DEFAULT_RESHARD_MAX_ENTRIES 1000
6950 if (max_entries
< 1) {
6951 max_entries
= DEFAULT_RESHARD_MAX_ENTRIES
;
6954 return br
.execute(num_shards
, max_entries
,
6955 verbose
, &cout
, formatter
.get());
6958 if (opt_cmd
== OPT::RESHARD_ADD
) {
6960 RGWBucketInfo bucket_info
;
6961 map
<string
, bufferlist
> attrs
;
6963 int ret
= check_reshard_bucket_params(store
,
6967 num_shards_specified
,
6969 yes_i_really_mean_it
,
6977 int num_source_shards
= (bucket_info
.layout
.current_index
.layout
.normal
.num_shards
> 0 ? bucket_info
.layout
.current_index
.layout
.normal
.num_shards
: 1);
6979 RGWReshard
reshard(store
);
6980 cls_rgw_reshard_entry entry
;
6981 entry
.time
= real_clock::now();
6982 entry
.tenant
= tenant
;
6983 entry
.bucket_name
= bucket_name
;
6984 entry
.bucket_id
= bucket_info
.bucket
.bucket_id
;
6985 entry
.old_num_shards
= num_source_shards
;
6986 entry
.new_num_shards
= num_shards
;
6988 return reshard
.add(entry
);
6991 if (opt_cmd
== OPT::RESHARD_LIST
) {
6992 list
<cls_rgw_reshard_entry
> entries
;
6995 if (max_entries
< 0) {
7000 store
->ctx()->_conf
.get_val
<uint64_t>("rgw_reshard_num_logs");
7002 RGWReshard
reshard(store
);
7004 formatter
->open_array_section("reshard");
7005 for (int i
= 0; i
< num_logshards
; i
++) {
7006 bool is_truncated
= true;
7010 ret
= reshard
.list(i
, marker
, max_entries
- count
, entries
, &is_truncated
);
7012 cerr
<< "Error listing resharding buckets: " << cpp_strerror(-ret
) << std::endl
;
7015 for (auto iter
=entries
.begin(); iter
!= entries
.end(); ++iter
) {
7016 cls_rgw_reshard_entry
& entry
= *iter
;
7017 encode_json("entry", entry
, formatter
.get());
7018 entry
.get_key(&marker
);
7020 count
+= entries
.size();
7021 formatter
->flush(cout
);
7022 } while (is_truncated
&& count
< max_entries
);
7024 if (count
>= max_entries
) {
7029 formatter
->close_section();
7030 formatter
->flush(cout
);
7034 if (opt_cmd
== OPT::RESHARD_STATUS
) {
7035 if (bucket_name
.empty()) {
7036 cerr
<< "ERROR: bucket not specified" << std::endl
;
7041 RGWBucketInfo bucket_info
;
7042 map
<string
, bufferlist
> attrs
;
7043 ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
, &attrs
);
7045 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7049 RGWBucketReshard
br(store
, bucket_info
, attrs
, nullptr /* no callback */);
7050 list
<cls_rgw_bucket_instance_entry
> status
;
7051 int r
= br
.get_status(&status
);
7053 cerr
<< "ERROR: could not get resharding status for bucket " <<
7054 bucket_name
<< std::endl
;
7058 show_reshard_status(status
, formatter
.get());
7061 if (opt_cmd
== OPT::RESHARD_PROCESS
) {
7062 RGWReshard
reshard(store
, true, &cout
);
7064 int ret
= reshard
.process_all_logshards();
7066 cerr
<< "ERROR: failed to process reshard logs, error=" << cpp_strerror(-ret
) << std::endl
;
7071 if (opt_cmd
== OPT::RESHARD_CANCEL
) {
7072 if (bucket_name
.empty()) {
7073 cerr
<< "ERROR: bucket not specified" << std::endl
;
7078 RGWBucketInfo bucket_info
;
7079 map
<string
, bufferlist
> attrs
;
7080 bool bucket_initable
= true;
7081 ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
,
7084 if (yes_i_really_mean_it
) {
7085 bucket_initable
= false;
7087 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) <<
7088 "; if you want to cancel the reshard request nonetheless, please "
7089 "use the --yes-i-really-mean-it option" << std::endl
;
7094 if (bucket_initable
) {
7095 // we did not encounter an error, so let's work with the bucket
7096 RGWBucketReshard
br(store
, bucket_info
, attrs
,
7097 nullptr /* no callback */);
7098 int ret
= br
.cancel();
7100 if (ret
== -EBUSY
) {
7101 cerr
<< "There is ongoing resharding, please retry after " <<
7102 store
->ctx()->_conf
.get_val
<uint64_t>(
7103 "rgw_reshard_bucket_lock_duration") <<
7104 " seconds " << std::endl
;
7106 cerr
<< "Error canceling bucket " << bucket_name
<<
7107 " resharding: " << cpp_strerror(-ret
) << std::endl
;
7113 RGWReshard
reshard(store
);
7115 cls_rgw_reshard_entry entry
;
7116 entry
.tenant
= tenant
;
7117 entry
.bucket_name
= bucket_name
;
7118 //entry.bucket_id = bucket_id;
7120 ret
= reshard
.remove(entry
);
7121 if (ret
< 0 && ret
!= -ENOENT
) {
7122 cerr
<< "Error in updating reshard log with bucket " <<
7123 bucket_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
7126 } // OPT_RESHARD_CANCEL
7128 if (opt_cmd
== OPT::OBJECT_UNLINK
) {
7129 RGWBucketInfo bucket_info
;
7130 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
7132 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7135 list
<rgw_obj_index_key
> oid_list
;
7136 rgw_obj_key
key(object
, object_version
);
7137 rgw_obj_index_key index_key
;
7138 key
.get_index_key(&index_key
);
7139 oid_list
.push_back(index_key
);
7140 ret
= store
->getRados()->remove_objs_from_index(bucket_info
, oid_list
);
7142 cerr
<< "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret
) << std::endl
;
7147 if (opt_cmd
== OPT::OBJECT_STAT
) {
7148 RGWBucketInfo bucket_info
;
7149 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
7151 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7154 rgw_obj
obj(bucket
, object
);
7155 obj
.key
.set_instance(object_version
);
7158 map
<string
, bufferlist
> attrs
;
7159 RGWObjectCtx
obj_ctx(store
);
7160 RGWRados::Object
op_target(store
->getRados(), bucket_info
, obj_ctx
, obj
);
7161 RGWRados::Object::Read
read_op(&op_target
);
7163 read_op
.params
.attrs
= &attrs
;
7164 read_op
.params
.obj_size
= &obj_size
;
7166 ret
= read_op
.prepare(null_yield
);
7168 cerr
<< "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret
) << std::endl
;
7171 formatter
->open_object_section("object_metadata");
7172 formatter
->dump_string("name", object
);
7173 formatter
->dump_unsigned("size", obj_size
);
7175 map
<string
, bufferlist
>::iterator iter
;
7176 map
<string
, bufferlist
> other_attrs
;
7177 for (iter
= attrs
.begin(); iter
!= attrs
.end(); ++iter
) {
7178 bufferlist
& bl
= iter
->second
;
7179 bool handled
= false;
7180 if (iter
->first
== RGW_ATTR_MANIFEST
) {
7181 handled
= decode_dump
<RGWObjManifest
>("manifest", bl
, formatter
.get());
7182 } else if (iter
->first
== RGW_ATTR_ACL
) {
7183 handled
= decode_dump
<RGWAccessControlPolicy
>("policy", bl
, formatter
.get());
7184 } else if (iter
->first
== RGW_ATTR_ID_TAG
) {
7185 handled
= dump_string("tag", bl
, formatter
.get());
7186 } else if (iter
->first
== RGW_ATTR_ETAG
) {
7187 handled
= dump_string("etag", bl
, formatter
.get());
7188 } else if (iter
->first
== RGW_ATTR_COMPRESSION
) {
7189 handled
= decode_dump
<RGWCompressionInfo
>("compression", bl
, formatter
.get());
7190 } else if (iter
->first
== RGW_ATTR_DELETE_AT
) {
7191 handled
= decode_dump
<utime_t
>("delete_at", bl
, formatter
.get());
7195 other_attrs
[iter
->first
] = bl
;
7198 formatter
->open_object_section("attrs");
7199 for (iter
= other_attrs
.begin(); iter
!= other_attrs
.end(); ++iter
) {
7200 dump_string(iter
->first
.c_str(), iter
->second
, formatter
.get());
7202 formatter
->close_section();
7203 formatter
->close_section();
7204 formatter
->flush(cout
);
7207 if (opt_cmd
== OPT::BUCKET_CHECK
) {
7208 if (check_head_obj_locator
) {
7209 if (bucket_name
.empty()) {
7210 cerr
<< "ERROR: need to specify bucket name" << std::endl
;
7213 do_check_object_locator(tenant
, bucket_name
, fix
, remove_bad
, formatter
.get());
7215 RGWBucketAdminOp::check_index(store
, bucket_op
, f
, null_yield
);
7219 if (opt_cmd
== OPT::BUCKET_RM
) {
7220 if (!inconsistent_index
) {
7221 RGWBucketAdminOp::remove_bucket(store
, bucket_op
, null_yield
, bypass_gc
, true);
7223 if (!yes_i_really_mean_it
) {
7224 cerr
<< "using --inconsistent_index can corrupt the bucket index " << std::endl
7225 << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
7228 RGWBucketAdminOp::remove_bucket(store
, bucket_op
, null_yield
, bypass_gc
, false);
7232 if (opt_cmd
== OPT::GC_LIST
) {
7235 bool processing_queue
= false;
7236 formatter
->open_array_section("entries");
7239 list
<cls_rgw_gc_obj_info
> result
;
7240 int ret
= store
->getRados()->list_gc_objs(&index
, marker
, 1000, !include_all
, result
, &truncated
, processing_queue
);
7242 cerr
<< "ERROR: failed to list objs: " << cpp_strerror(-ret
) << std::endl
;
7247 list
<cls_rgw_gc_obj_info
>::iterator iter
;
7248 for (iter
= result
.begin(); iter
!= result
.end(); ++iter
) {
7249 cls_rgw_gc_obj_info
& info
= *iter
;
7250 formatter
->open_object_section("chain_info");
7251 formatter
->dump_string("tag", info
.tag
);
7252 formatter
->dump_stream("time") << info
.time
;
7253 formatter
->open_array_section("objs");
7254 list
<cls_rgw_obj
>::iterator liter
;
7255 cls_rgw_obj_chain
& chain
= info
.chain
;
7256 for (liter
= chain
.objs
.begin(); liter
!= chain
.objs
.end(); ++liter
) {
7257 cls_rgw_obj
& obj
= *liter
;
7258 encode_json("obj", obj
, formatter
.get());
7260 formatter
->close_section(); // objs
7261 formatter
->close_section(); // obj_chain
7262 formatter
->flush(cout
);
7264 } while (truncated
);
7265 formatter
->close_section();
7266 formatter
->flush(cout
);
7269 if (opt_cmd
== OPT::GC_PROCESS
) {
7270 int ret
= store
->getRados()->process_gc(!include_all
);
7272 cerr
<< "ERROR: gc processing returned error: " << cpp_strerror(-ret
) << std::endl
;
7277 if (opt_cmd
== OPT::LC_LIST
) {
7278 formatter
->open_array_section("lifecycle_list");
7279 vector
<rgw::sal::Lifecycle::LCEntry
> bucket_lc_map
;
7282 #define MAX_LC_LIST_ENTRIES 100
7283 if (max_entries
< 0) {
7284 max_entries
= MAX_LC_LIST_ENTRIES
;
7287 int ret
= store
->getRados()->list_lc_progress(marker
, max_entries
,
7288 bucket_lc_map
, index
);
7290 cerr
<< "ERROR: failed to list objs: " << cpp_strerror(-ret
)
7294 for (const auto& entry
: bucket_lc_map
) {
7295 formatter
->open_object_section("bucket_lc_info");
7296 formatter
->dump_string("bucket", entry
.bucket
);
7298 time_t t
{time_t(entry
.start_time
)};
7300 exp_buf
, sizeof(exp_buf
),
7301 "%a, %d %b %Y %T %Z", std::gmtime(&t
))) {
7302 formatter
->dump_string("started", exp_buf
);
7304 string lc_status
= LC_STATUS
[entry
.status
];
7305 formatter
->dump_string("status", lc_status
);
7306 formatter
->close_section(); // objs
7307 formatter
->flush(cout
);
7309 } while (!bucket_lc_map
.empty());
7311 formatter
->close_section(); //lifecycle list
7312 formatter
->flush(cout
);
7316 if (opt_cmd
== OPT::LC_GET
) {
7317 if (bucket_name
.empty()) {
7318 cerr
<< "ERROR: bucket not specified" << std::endl
;
7323 RGWBucketInfo bucket_info
;
7324 map
<string
, bufferlist
> attrs
;
7325 RGWLifecycleConfiguration config
;
7326 ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
, &attrs
);
7328 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7332 auto aiter
= attrs
.find(RGW_ATTR_LC
);
7333 if (aiter
== attrs
.end()) {
7337 bufferlist::const_iterator iter
{&aiter
->second
};
7339 config
.decode(iter
);
7340 } catch (const buffer::error
& e
) {
7341 cerr
<< "ERROR: decode life cycle config failed" << std::endl
;
7345 encode_json("result", config
, formatter
.get());
7346 formatter
->flush(cout
);
7349 if (opt_cmd
== OPT::LC_PROCESS
) {
7350 int ret
= store
->getRados()->process_lc();
7352 cerr
<< "ERROR: lc processing returned error: " << cpp_strerror(-ret
) << std::endl
;
7358 if (opt_cmd
== OPT::LC_RESHARD_FIX
) {
7359 ret
= RGWBucketAdminOp::fix_lc_shards(store
, bucket_op
,f
);
7361 cerr
<< "ERROR: listing stale instances" << cpp_strerror(-ret
) << std::endl
;
7366 if (opt_cmd
== OPT::ORPHANS_FIND
) {
7367 if (!yes_i_really_mean_it
) {
7368 cerr
<< "this command is now deprecated; please consider using the rgw-orphan-list tool; "
7369 << "accidental removal of active objects cannot be reversed; "
7370 << "do you really mean it? (requires --yes-i-really-mean-it)"
7374 cerr
<< "IMPORTANT: this command is now deprecated; please consider using the rgw-orphan-list tool"
7378 RGWOrphanSearch
search(store
, max_concurrent_ios
, orphan_stale_secs
);
7380 if (job_id
.empty()) {
7381 cerr
<< "ERROR: --job-id not specified" << std::endl
;
7384 if (pool_name
.empty()) {
7385 cerr
<< "ERROR: --pool not specified" << std::endl
;
7389 RGWOrphanSearchInfo info
;
7392 info
.job_name
= job_id
;
7393 info
.num_shards
= num_shards
;
7395 int ret
= search
.init(job_id
, &info
, detail
);
7397 cerr
<< "could not init search, ret=" << ret
<< std::endl
;
7406 if (opt_cmd
== OPT::ORPHANS_FINISH
) {
7407 if (!yes_i_really_mean_it
) {
7408 cerr
<< "this command is now deprecated; please consider using the rgw-orphan-list tool; "
7409 << "accidental removal of active objects cannot be reversed; "
7410 << "do you really mean it? (requires --yes-i-really-mean-it)"
7414 cerr
<< "IMPORTANT: this command is now deprecated; please consider using the rgw-orphan-list tool"
7418 RGWOrphanSearch
search(store
, max_concurrent_ios
, orphan_stale_secs
);
7420 if (job_id
.empty()) {
7421 cerr
<< "ERROR: --job-id not specified" << std::endl
;
7424 int ret
= search
.init(job_id
, NULL
);
7426 if (ret
== -ENOENT
) {
7427 cerr
<< "job not found" << std::endl
;
7431 ret
= search
.finish();
7437 if (opt_cmd
== OPT::ORPHANS_LIST_JOBS
){
7438 if (!yes_i_really_mean_it
) {
7439 cerr
<< "this command is now deprecated; please consider using the rgw-orphan-list tool; "
7440 << "do you really mean it? (requires --yes-i-really-mean-it)"
7444 cerr
<< "IMPORTANT: this command is now deprecated; please consider using the rgw-orphan-list tool"
7448 RGWOrphanStore
orphan_store(store
);
7449 int ret
= orphan_store
.init();
7451 cerr
<< "connection to cluster failed!" << std::endl
;
7455 map
<string
,RGWOrphanSearchState
> m
;
7456 ret
= orphan_store
.list_jobs(m
);
7458 cerr
<< "job list failed" << std::endl
;
7461 formatter
->open_array_section("entries");
7462 for (const auto &it
: m
){
7464 formatter
->dump_string("job-id",it
.first
);
7466 encode_json("orphan_search_state", it
.second
, formatter
.get());
7469 formatter
->close_section();
7470 formatter
->flush(cout
);
7473 if (opt_cmd
== OPT::USER_CHECK
) {
7474 check_bad_user_bucket_mapping(store
, user_id
, fix
, null_yield
);
7477 if (opt_cmd
== OPT::USER_STATS
) {
7478 if (user_id
.empty()) {
7479 cerr
<< "ERROR: uid not specified" << std::endl
;
7484 if (!bucket_name
.empty()) {
7485 cerr
<< "ERROR: --reset-stats does not work on buckets and "
7486 "bucket specified" << std::endl
;
7490 cerr
<< "ERROR: sync-stats includes the reset-stats functionality, "
7491 "so at most one of the two should be specified" << std::endl
;
7494 ret
= store
->ctl()->user
->reset_stats(user_id
, null_yield
);
7496 cerr
<< "ERROR: could not reset user stats: " << cpp_strerror(-ret
) <<
7503 if (!bucket_name
.empty()) {
7504 RGWBucketInfo bucket_info
;
7505 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
7507 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7510 ret
= store
->ctl()->bucket
->sync_user_stats(user_id
, bucket_info
, null_yield
);
7512 cerr
<< "ERROR: could not sync bucket stats: " <<
7513 cpp_strerror(-ret
) << std::endl
;
7517 int ret
= rgw_user_sync_all_stats(store
, user_id
, null_yield
);
7519 cerr
<< "ERROR: could not sync user stats: " <<
7520 cpp_strerror(-ret
) << std::endl
;
7526 RGWStorageStats stats
;
7527 ceph::real_time last_stats_sync
;
7528 ceph::real_time last_stats_update
;
7529 int ret
= store
->ctl()->user
->read_stats(user_id
, &stats
, null_yield
,
7531 &last_stats_update
);
7533 if (ret
== -ENOENT
) { /* in case of ENOENT */
7534 cerr
<< "User has not been initialized or user does not exist" << std::endl
;
7536 cerr
<< "ERROR: can't read user: " << cpp_strerror(ret
) << std::endl
;
7543 Formatter::ObjectSection
os(*formatter
, "result");
7544 encode_json("stats", stats
, formatter
.get());
7545 utime_t
last_sync_ut(last_stats_sync
);
7546 encode_json("last_stats_sync", last_sync_ut
, formatter
.get());
7547 utime_t
last_update_ut(last_stats_update
);
7548 encode_json("last_stats_update", last_update_ut
, formatter
.get());
7550 formatter
->flush(cout
);
7553 if (opt_cmd
== OPT::METADATA_GET
) {
7554 int ret
= store
->ctl()->meta
.mgr
->get(metadata_key
, formatter
.get(), null_yield
);
7556 cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
7560 formatter
->flush(cout
);
7563 if (opt_cmd
== OPT::METADATA_PUT
) {
7565 int ret
= read_input(infile
, bl
);
7567 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
7570 ret
= store
->ctl()->meta
.mgr
->put(metadata_key
, bl
, null_yield
, RGWMDLogSyncType::APPLY_ALWAYS
, false);
7572 cerr
<< "ERROR: can't put key: " << cpp_strerror(-ret
) << std::endl
;
7577 if (opt_cmd
== OPT::METADATA_RM
) {
7578 int ret
= store
->ctl()->meta
.mgr
->remove(metadata_key
, null_yield
);
7580 cerr
<< "ERROR: can't remove key: " << cpp_strerror(-ret
) << std::endl
;
7585 if (opt_cmd
== OPT::METADATA_LIST
|| opt_cmd
== OPT::USER_LIST
) {
7586 if (opt_cmd
== OPT::USER_LIST
) {
7587 metadata_key
= "user";
7591 int ret
= store
->ctl()->meta
.mgr
->list_keys_init(metadata_key
, marker
, &handle
);
7593 cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
7600 if (max_entries_specified
) {
7601 formatter
->open_object_section("result");
7603 formatter
->open_array_section("keys");
7608 left
= (max_entries_specified
? max_entries
- count
: max
);
7609 ret
= store
->ctl()->meta
.mgr
->list_keys_next(handle
, left
, keys
, &truncated
);
7610 if (ret
< 0 && ret
!= -ENOENT
) {
7611 cerr
<< "ERROR: lists_keys_next(): " << cpp_strerror(-ret
) << std::endl
;
7613 } if (ret
!= -ENOENT
) {
7614 for (list
<string
>::iterator iter
= keys
.begin(); iter
!= keys
.end(); ++iter
) {
7615 formatter
->dump_string("key", *iter
);
7618 formatter
->flush(cout
);
7620 } while (truncated
&& left
> 0);
7622 formatter
->close_section();
7624 if (max_entries_specified
) {
7625 encode_json("truncated", truncated
, formatter
.get());
7626 encode_json("count", count
, formatter
.get());
7628 encode_json("marker", store
->ctl()->meta
.mgr
->get_marker(handle
), formatter
.get());
7630 formatter
->close_section();
7632 formatter
->flush(cout
);
7634 store
->ctl()->meta
.mgr
->list_keys_complete(handle
);
7637 if (opt_cmd
== OPT::MDLOG_LIST
) {
7638 if (!start_date
.empty()) {
7639 std::cerr
<< "start-date not allowed." << std::endl
;
7642 if (!end_date
.empty()) {
7643 std::cerr
<< "end-date not allowed." << std::endl
;
7646 if (!end_marker
.empty()) {
7647 std::cerr
<< "end-marker not allowed." << std::endl
;
7650 if (!start_marker
.empty()) {
7651 if (marker
.empty()) {
7652 marker
= start_marker
;
7654 std::cerr
<< "start-marker and marker not both allowed." << std::endl
;
7659 int i
= (specified_shard_id
? shard_id
: 0);
7661 if (period_id
.empty()) {
7662 int ret
= read_current_period_id(store
, realm_id
, realm_name
, &period_id
);
7666 std::cerr
<< "No --period given, using current period="
7667 << period_id
<< std::endl
;
7669 RGWMetadataLog
*meta_log
= store
->svc()->mdlog
->get_log(period_id
);
7671 formatter
->open_array_section("entries");
7672 for (; i
< g_ceph_context
->_conf
->rgw_md_log_max_shards
; i
++) {
7674 list
<cls_log_entry
> entries
;
7676 meta_log
->init_list_entries(i
, {}, {}, marker
, &handle
);
7679 int ret
= meta_log
->list_entries(handle
, 1000, entries
, NULL
, &truncated
);
7681 cerr
<< "ERROR: meta_log->list_entries(): " << cpp_strerror(-ret
) << std::endl
;
7685 for (list
<cls_log_entry
>::iterator iter
= entries
.begin(); iter
!= entries
.end(); ++iter
) {
7686 cls_log_entry
& entry
= *iter
;
7687 store
->ctl()->meta
.mgr
->dump_log_entry(entry
, formatter
.get());
7689 formatter
->flush(cout
);
7690 } while (truncated
);
7692 meta_log
->complete_list_entries(handle
);
7694 if (specified_shard_id
)
7699 formatter
->close_section();
7700 formatter
->flush(cout
);
7703 if (opt_cmd
== OPT::MDLOG_STATUS
) {
7704 int i
= (specified_shard_id
? shard_id
: 0);
7706 if (period_id
.empty()) {
7707 int ret
= read_current_period_id(store
, realm_id
, realm_name
, &period_id
);
7711 std::cerr
<< "No --period given, using current period="
7712 << period_id
<< std::endl
;
7714 RGWMetadataLog
*meta_log
= store
->svc()->mdlog
->get_log(period_id
);
7716 formatter
->open_array_section("entries");
7718 for (; i
< g_ceph_context
->_conf
->rgw_md_log_max_shards
; i
++) {
7719 RGWMetadataLogInfo info
;
7720 meta_log
->get_info(i
, &info
);
7722 ::encode_json("info", info
, formatter
.get());
7724 if (specified_shard_id
)
7729 formatter
->close_section();
7730 formatter
->flush(cout
);
7733 if (opt_cmd
== OPT::MDLOG_AUTOTRIM
) {
7734 // need a full history for purging old mdlog periods
7735 store
->svc()->mdlog
->init_oldest_log_period(null_yield
);
7737 RGWCoroutinesManager
crs(store
->ctx(), store
->getRados()->get_cr_registry());
7738 RGWHTTPManager
http(store
->ctx(), crs
.get_completion_mgr());
7739 int ret
= http
.start();
7741 cerr
<< "failed to initialize http client with " << cpp_strerror(ret
) << std::endl
;
7745 auto num_shards
= g_conf()->rgw_md_log_max_shards
;
7746 ret
= crs
.run(create_admin_meta_log_trim_cr(dpp(), store
, &http
, num_shards
));
7748 cerr
<< "automated mdlog trim failed with " << cpp_strerror(ret
) << std::endl
;
7753 if (opt_cmd
== OPT::MDLOG_TRIM
) {
7754 if (!start_date
.empty()) {
7755 std::cerr
<< "start-date not allowed." << std::endl
;
7758 if (!end_date
.empty()) {
7759 std::cerr
<< "end-date not allowed." << std::endl
;
7762 if (!start_marker
.empty()) {
7763 std::cerr
<< "start-marker not allowed." << std::endl
;
7766 if (!end_marker
.empty()) {
7767 if (marker
.empty()) {
7768 marker
= end_marker
;
7770 std::cerr
<< "end-marker and marker not both allowed." << std::endl
;
7775 if (!specified_shard_id
) {
7776 cerr
<< "ERROR: shard-id must be specified for trim operation" << std::endl
;
7780 if (period_id
.empty()) {
7781 std::cerr
<< "missing --period argument" << std::endl
;
7784 RGWMetadataLog
*meta_log
= store
->svc()->mdlog
->get_log(period_id
);
7786 // trim until -ENODATA
7788 ret
= meta_log
->trim(shard_id
, {}, {}, {}, marker
);
7790 if (ret
< 0 && ret
!= -ENODATA
) {
7791 cerr
<< "ERROR: meta_log->trim(): " << cpp_strerror(-ret
) << std::endl
;
7796 if (opt_cmd
== OPT::SYNC_INFO
) {
7797 sync_info(opt_effective_zone_id
, opt_bucket
, zone_formatter
.get());
7800 if (opt_cmd
== OPT::SYNC_STATUS
) {
7801 sync_status(formatter
.get());
7804 if (opt_cmd
== OPT::METADATA_SYNC_STATUS
) {
7805 RGWMetaSyncStatusManager
sync(store
, store
->svc()->rados
->get_async_processor());
7807 int ret
= sync
.init();
7809 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
7813 rgw_meta_sync_status sync_status
;
7814 ret
= sync
.read_sync_status(&sync_status
);
7816 cerr
<< "ERROR: sync.read_sync_status() returned ret=" << ret
<< std::endl
;
7820 formatter
->open_object_section("summary");
7821 encode_json("sync_status", sync_status
, formatter
.get());
7823 uint64_t full_total
= 0;
7824 uint64_t full_complete
= 0;
7826 for (auto marker_iter
: sync_status
.sync_markers
) {
7827 full_total
+= marker_iter
.second
.total_entries
;
7828 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::FullSync
) {
7829 full_complete
+= marker_iter
.second
.pos
;
7831 full_complete
+= marker_iter
.second
.total_entries
;
7835 formatter
->open_object_section("full_sync");
7836 encode_json("total", full_total
, formatter
.get());
7837 encode_json("complete", full_complete
, formatter
.get());
7838 formatter
->close_section();
7839 formatter
->close_section();
7841 formatter
->flush(cout
);
7845 if (opt_cmd
== OPT::METADATA_SYNC_INIT
) {
7846 RGWMetaSyncStatusManager
sync(store
, store
->svc()->rados
->get_async_processor());
7848 int ret
= sync
.init();
7850 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
7853 ret
= sync
.init_sync_status();
7855 cerr
<< "ERROR: sync.init_sync_status() returned ret=" << ret
<< std::endl
;
7861 if (opt_cmd
== OPT::METADATA_SYNC_RUN
) {
7862 RGWMetaSyncStatusManager
sync(store
, store
->svc()->rados
->get_async_processor());
7864 int ret
= sync
.init();
7866 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
7870 ret
= sync
.run(null_yield
);
7872 cerr
<< "ERROR: sync.run() returned ret=" << ret
<< std::endl
;
7877 if (opt_cmd
== OPT::DATA_SYNC_STATUS
) {
7878 if (source_zone
.empty()) {
7879 cerr
<< "ERROR: source zone not specified" << std::endl
;
7882 RGWDataSyncStatusManager
sync(store
, store
->svc()->rados
->get_async_processor(), source_zone
, nullptr);
7884 int ret
= sync
.init();
7886 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
7890 rgw_data_sync_status sync_status
;
7891 if (specified_shard_id
) {
7892 set
<string
> pending_buckets
;
7893 set
<string
> recovering_buckets
;
7894 rgw_data_sync_marker sync_marker
;
7895 ret
= sync
.read_shard_status(shard_id
, pending_buckets
, recovering_buckets
, &sync_marker
,
7896 max_entries_specified
? max_entries
: 20);
7897 if (ret
< 0 && ret
!= -ENOENT
) {
7898 cerr
<< "ERROR: sync.read_shard_status() returned ret=" << ret
<< std::endl
;
7901 formatter
->open_object_section("summary");
7902 encode_json("shard_id", shard_id
, formatter
.get());
7903 encode_json("marker", sync_marker
, formatter
.get());
7904 encode_json("pending_buckets", pending_buckets
, formatter
.get());
7905 encode_json("recovering_buckets", recovering_buckets
, formatter
.get());
7906 formatter
->close_section();
7907 formatter
->flush(cout
);
7909 ret
= sync
.read_sync_status(&sync_status
);
7910 if (ret
< 0 && ret
!= -ENOENT
) {
7911 cerr
<< "ERROR: sync.read_sync_status() returned ret=" << ret
<< std::endl
;
7915 formatter
->open_object_section("summary");
7916 encode_json("sync_status", sync_status
, formatter
.get());
7918 uint64_t full_total
= 0;
7919 uint64_t full_complete
= 0;
7921 for (auto marker_iter
: sync_status
.sync_markers
) {
7922 full_total
+= marker_iter
.second
.total_entries
;
7923 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::FullSync
) {
7924 full_complete
+= marker_iter
.second
.pos
;
7926 full_complete
+= marker_iter
.second
.total_entries
;
7930 formatter
->open_object_section("full_sync");
7931 encode_json("total", full_total
, formatter
.get());
7932 encode_json("complete", full_complete
, formatter
.get());
7933 formatter
->close_section();
7934 formatter
->close_section();
7936 formatter
->flush(cout
);
7940 if (opt_cmd
== OPT::DATA_SYNC_INIT
) {
7941 if (source_zone
.empty()) {
7942 cerr
<< "ERROR: source zone not specified" << std::endl
;
7946 RGWDataSyncStatusManager
sync(store
, store
->svc()->rados
->get_async_processor(), source_zone
, nullptr);
7948 int ret
= sync
.init();
7950 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
7954 ret
= sync
.init_sync_status();
7956 cerr
<< "ERROR: sync.init_sync_status() returned ret=" << ret
<< std::endl
;
7961 if (opt_cmd
== OPT::DATA_SYNC_RUN
) {
7962 if (source_zone
.empty()) {
7963 cerr
<< "ERROR: source zone not specified" << std::endl
;
7967 RGWSyncModuleInstanceRef sync_module
;
7968 int ret
= store
->svc()->sync_modules
->get_manager()->create_instance(g_ceph_context
, store
->svc()->zone
->get_zone().tier_type
,
7969 store
->svc()->zone
->get_zone_params().tier_config
, &sync_module
);
7971 lderr(cct
) << "ERROR: failed to init sync module instance, ret=" << ret
<< dendl
;
7975 RGWDataSyncStatusManager
sync(store
, store
->svc()->rados
->get_async_processor(), source_zone
, nullptr, sync_module
);
7979 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
7985 cerr
<< "ERROR: sync.run() returned ret=" << ret
<< std::endl
;
7990 if (opt_cmd
== OPT::BUCKET_SYNC_INIT
) {
7991 if (source_zone
.empty()) {
7992 cerr
<< "ERROR: source zone not specified" << std::endl
;
7995 if (bucket_name
.empty()) {
7996 cerr
<< "ERROR: bucket not specified" << std::endl
;
8000 int ret
= init_bucket_for_sync(tenant
, bucket_name
, bucket_id
, bucket
);
8004 auto opt_sb
= opt_source_bucket
;
8005 if (opt_sb
&& opt_sb
->bucket_id
.empty()) {
8008 int ret
= init_bucket_for_sync(opt_sb
->tenant
, opt_sb
->name
, sbid
, sbuck
);
8015 RGWBucketPipeSyncStatusManager
sync(store
, source_zone
, opt_sb
, bucket
);
8019 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
8022 ret
= sync
.init_sync_status();
8024 cerr
<< "ERROR: sync.init_sync_status() returned ret=" << ret
<< std::endl
;
8029 if (opt_cmd
== OPT::BUCKET_SYNC_CHECKPOINT
) {
8030 std::optional
<rgw_zone_id
> opt_source_zone
;
8031 if (!source_zone
.empty()) {
8032 opt_source_zone
= source_zone
;
8034 if (bucket_name
.empty()) {
8035 cerr
<< "ERROR: bucket not specified" << std::endl
;
8038 RGWBucketInfo bucket_info
;
8040 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
8045 if (!store
->ctl()->bucket
->bucket_imports_data(bucket_info
.bucket
, null_yield
)) {
8046 std::cout
<< "Sync is disabled for bucket " << bucket_name
<< std::endl
;
8050 RGWBucketSyncPolicyHandlerRef handler
;
8051 ret
= store
->ctl()->bucket
->get_sync_policy_handler(std::nullopt
, bucket
, &handler
, null_yield
);
8053 std::cerr
<< "ERROR: failed to get policy handler for bucket ("
8054 << bucket_info
.bucket
<< "): r=" << ret
<< ": " << cpp_strerror(-ret
) << std::endl
;
8058 auto timeout_at
= ceph::coarse_mono_clock::now() + opt_timeout_sec
;
8059 ret
= rgw_bucket_sync_checkpoint(dpp(), store
, *handler
, bucket_info
,
8060 opt_source_zone
, opt_source_bucket
,
8061 opt_retry_delay_ms
, timeout_at
);
8063 lderr(store
->ctx()) << "bucket sync checkpoint failed: " << cpp_strerror(ret
) << dendl
;
8068 if ((opt_cmd
== OPT::BUCKET_SYNC_DISABLE
) || (opt_cmd
== OPT::BUCKET_SYNC_ENABLE
)) {
8069 if (bucket_name
.empty()) {
8070 cerr
<< "ERROR: bucket not specified" << std::endl
;
8073 if (opt_cmd
== OPT::BUCKET_SYNC_DISABLE
) {
8074 bucket_op
.set_sync_bucket(false);
8076 bucket_op
.set_sync_bucket(true);
8078 bucket_op
.set_tenant(tenant
);
8080 ret
= RGWBucketAdminOp::sync_bucket(store
, bucket_op
, &err_msg
);
8082 cerr
<< err_msg
<< std::endl
;
8087 if (opt_cmd
== OPT::BUCKET_SYNC_INFO
) {
8088 if (bucket_name
.empty()) {
8089 cerr
<< "ERROR: bucket not specified" << std::endl
;
8092 RGWBucketInfo bucket_info
;
8094 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
8098 bucket_sync_info(store
, bucket_info
, std::cout
);
8101 if (opt_cmd
== OPT::BUCKET_SYNC_STATUS
) {
8102 if (bucket_name
.empty()) {
8103 cerr
<< "ERROR: bucket not specified" << std::endl
;
8106 RGWBucketInfo bucket_info
;
8108 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
8112 bucket_sync_status(store
, bucket_info
, source_zone
, opt_source_bucket
, std::cout
);
8115 if (opt_cmd
== OPT::BUCKET_SYNC_MARKERS
) {
8116 if (source_zone
.empty()) {
8117 cerr
<< "ERROR: source zone not specified" << std::endl
;
8120 if (bucket_name
.empty()) {
8121 cerr
<< "ERROR: bucket not specified" << std::endl
;
8125 int ret
= init_bucket_for_sync(tenant
, bucket_name
, bucket_id
, bucket
);
8129 RGWBucketPipeSyncStatusManager
sync(store
, source_zone
, opt_source_bucket
, bucket
);
8133 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
8136 ret
= sync
.read_sync_status();
8138 cerr
<< "ERROR: sync.read_sync_status() returned ret=" << ret
<< std::endl
;
8142 map
<int, rgw_bucket_shard_sync_info
>& sync_status
= sync
.get_sync_status();
8144 encode_json("sync_status", sync_status
, formatter
.get());
8145 formatter
->flush(cout
);
8148 if (opt_cmd
== OPT::BUCKET_SYNC_RUN
) {
8149 if (source_zone
.empty()) {
8150 cerr
<< "ERROR: source zone not specified" << std::endl
;
8153 if (bucket_name
.empty()) {
8154 cerr
<< "ERROR: bucket not specified" << std::endl
;
8158 int ret
= init_bucket_for_sync(tenant
, bucket_name
, bucket_id
, bucket
);
8162 RGWBucketPipeSyncStatusManager
sync(store
, source_zone
, opt_source_bucket
, bucket
);
8166 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
8172 cerr
<< "ERROR: sync.run() returned ret=" << ret
<< std::endl
;
8177 if (opt_cmd
== OPT::BILOG_LIST
) {
8178 if (bucket_name
.empty()) {
8179 cerr
<< "ERROR: bucket not specified" << std::endl
;
8182 RGWBucketInfo bucket_info
;
8183 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
8185 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8188 formatter
->open_array_section("entries");
8191 if (max_entries
< 0)
8195 list
<rgw_bi_log_entry
> entries
;
8196 ret
= store
->svc()->bilog_rados
->log_list(bucket_info
, shard_id
, marker
, max_entries
- count
, entries
, &truncated
);
8198 cerr
<< "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret
) << std::endl
;
8202 count
+= entries
.size();
8204 for (list
<rgw_bi_log_entry
>::iterator iter
= entries
.begin(); iter
!= entries
.end(); ++iter
) {
8205 rgw_bi_log_entry
& entry
= *iter
;
8206 encode_json("entry", entry
, formatter
.get());
8210 formatter
->flush(cout
);
8211 } while (truncated
&& count
< max_entries
);
8213 formatter
->close_section();
8214 formatter
->flush(cout
);
8217 if (opt_cmd
== OPT::SYNC_ERROR_LIST
) {
8218 if (max_entries
< 0) {
8221 if (!start_date
.empty()) {
8222 std::cerr
<< "start-date not allowed." << std::endl
;
8225 if (!end_date
.empty()) {
8226 std::cerr
<< "end-date not allowed." << std::endl
;
8229 if (!end_marker
.empty()) {
8230 std::cerr
<< "end-marker not allowed." << std::endl
;
8233 if (!start_marker
.empty()) {
8234 if (marker
.empty()) {
8235 marker
= start_marker
;
8237 std::cerr
<< "start-marker and marker not both allowed." << std::endl
;
8248 formatter
->open_array_section("entries");
8250 for (; shard_id
< ERROR_LOGGER_SHARDS
; ++shard_id
) {
8251 formatter
->open_object_section("shard");
8252 encode_json("shard_id", shard_id
, formatter
.get());
8253 formatter
->open_array_section("entries");
8256 string oid
= RGWSyncErrorLogger::get_shard_oid(RGW_SYNC_ERROR_LOG_SHARD_PREFIX
, shard_id
);
8259 list
<cls_log_entry
> entries
;
8260 ret
= store
->svc()->cls
->timelog
.list(oid
, {}, {}, max_entries
- count
, entries
, marker
, &marker
, &truncated
,
8262 if (ret
== -ENOENT
) {
8266 cerr
<< "ERROR: svc.cls->timelog.list(): " << cpp_strerror(-ret
) << std::endl
;
8270 count
+= entries
.size();
8272 for (auto& cls_entry
: entries
) {
8273 rgw_sync_error_info log_entry
;
8275 auto iter
= cls_entry
.data
.cbegin();
8277 decode(log_entry
, iter
);
8278 } catch (buffer::error
& err
) {
8279 cerr
<< "ERROR: failed to decode log entry" << std::endl
;
8282 formatter
->open_object_section("entry");
8283 encode_json("id", cls_entry
.id
, formatter
.get());
8284 encode_json("section", cls_entry
.section
, formatter
.get());
8285 encode_json("name", cls_entry
.name
, formatter
.get());
8286 encode_json("timestamp", cls_entry
.timestamp
, formatter
.get());
8287 encode_json("info", log_entry
, formatter
.get());
8288 formatter
->close_section();
8289 formatter
->flush(cout
);
8291 } while (truncated
&& count
< max_entries
);
8293 formatter
->close_section();
8294 formatter
->close_section();
8296 if (specified_shard_id
) {
8301 formatter
->close_section();
8302 formatter
->flush(cout
);
8305 if (opt_cmd
== OPT::SYNC_ERROR_TRIM
) {
8306 if (!start_date
.empty()) {
8307 std::cerr
<< "start-date not allowed." << std::endl
;
8310 if (!end_date
.empty()) {
8311 std::cerr
<< "end-date not allowed." << std::endl
;
8314 if (!start_marker
.empty()) {
8315 std::cerr
<< "end-date not allowed." << std::endl
;
8318 if (!end_marker
.empty()) {
8319 std::cerr
<< "end-date not allowed." << std::endl
;
8327 for (; shard_id
< ERROR_LOGGER_SHARDS
; ++shard_id
) {
8328 ret
= trim_sync_error_log(shard_id
, marker
, trim_delay_ms
);
8330 cerr
<< "ERROR: sync error trim: " << cpp_strerror(-ret
) << std::endl
;
8333 if (specified_shard_id
) {
8339 if (opt_cmd
== OPT::SYNC_GROUP_CREATE
||
8340 opt_cmd
== OPT::SYNC_GROUP_MODIFY
) {
8341 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
8342 CHECK_TRUE(require_opt(opt_status
), "ERROR: --status is not specified (options: forbidden, allowed, enabled)", EINVAL
);
8344 SyncPolicyContext
sync_policy_ctx(zonegroup_id
, zonegroup_name
, opt_bucket
);
8345 ret
= sync_policy_ctx
.init();
8349 auto& sync_policy
= sync_policy_ctx
.get_policy();
8351 if (opt_cmd
== OPT::SYNC_GROUP_MODIFY
) {
8352 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
8353 if (iter
== sync_policy
.groups
.end()) {
8354 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
8359 auto& group
= sync_policy
.groups
[*opt_group_id
];
8360 group
.id
= *opt_group_id
;
8363 if (!group
.set_status(*opt_status
)) {
8364 cerr
<< "ERROR: unrecognized status (options: forbidden, allowed, enabled)" << std::endl
;
8369 ret
= sync_policy_ctx
.write_policy();
8374 show_result(sync_policy
, zone_formatter
.get(), cout
);
8377 if (opt_cmd
== OPT::SYNC_GROUP_GET
) {
8378 SyncPolicyContext
sync_policy_ctx(zonegroup_id
, zonegroup_name
, opt_bucket
);
8379 ret
= sync_policy_ctx
.init();
8383 auto& sync_policy
= sync_policy_ctx
.get_policy();
8385 auto& groups
= sync_policy
.groups
;
8387 if (!opt_group_id
) {
8388 show_result(groups
, zone_formatter
.get(), cout
);
8390 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
8391 if (iter
== sync_policy
.groups
.end()) {
8392 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
8396 show_result(iter
->second
, zone_formatter
.get(), cout
);
8400 if (opt_cmd
== OPT::SYNC_GROUP_REMOVE
) {
8401 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
8403 SyncPolicyContext
sync_policy_ctx(zonegroup_id
, zonegroup_name
, opt_bucket
);
8404 ret
= sync_policy_ctx
.init();
8408 auto& sync_policy
= sync_policy_ctx
.get_policy();
8410 sync_policy
.groups
.erase(*opt_group_id
);
8412 ret
= sync_policy_ctx
.write_policy();
8418 Formatter::ObjectSection
os(*zone_formatter
.get(), "result");
8419 encode_json("sync_policy", sync_policy
, zone_formatter
.get());
8422 zone_formatter
->flush(cout
);
8425 if (opt_cmd
== OPT::SYNC_GROUP_FLOW_CREATE
) {
8426 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
8427 CHECK_TRUE(require_opt(opt_flow_id
), "ERROR: --flow-id not specified", EINVAL
);
8428 CHECK_TRUE(require_opt(opt_flow_type
,
8429 (symmetrical_flow_opt(*opt_flow_type
) ||
8430 directional_flow_opt(*opt_flow_type
))),
8431 "ERROR: --flow-type not specified or invalid (options: symmetrical, directional)", EINVAL
);
8433 SyncPolicyContext
sync_policy_ctx(zonegroup_id
, zonegroup_name
, opt_bucket
);
8434 ret
= sync_policy_ctx
.init();
8438 auto& sync_policy
= sync_policy_ctx
.get_policy();
8440 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
8441 if (iter
== sync_policy
.groups
.end()) {
8442 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
8446 auto& group
= iter
->second
;
8448 if (symmetrical_flow_opt(*opt_flow_type
)) {
8449 CHECK_TRUE(require_non_empty_opt(opt_zone_ids
), "ERROR: --zones not provided for symmetrical flow, or is empty", EINVAL
);
8451 rgw_sync_symmetric_group
*flow_group
;
8453 group
.data_flow
.find_or_create_symmetrical(*opt_flow_id
, &flow_group
);
8455 for (auto& z
: *opt_zone_ids
) {
8456 flow_group
->zones
.insert(z
);
8458 } else { /* directional */
8459 CHECK_TRUE(require_non_empty_opt(opt_source_zone_id
), "ERROR: --source-zone not provided for directional flow rule, or is empty", EINVAL
);
8460 CHECK_TRUE(require_non_empty_opt(opt_dest_zone_id
), "ERROR: --dest-zone not provided for directional flow rule, or is empty", EINVAL
);
8462 rgw_sync_directional_rule
*flow_rule
;
8464 group
.data_flow
.find_or_create_directional(*opt_source_zone_id
, *opt_dest_zone_id
, &flow_rule
);
8467 ret
= sync_policy_ctx
.write_policy();
8472 show_result(sync_policy
, zone_formatter
.get(), cout
);
8475 if (opt_cmd
== OPT::SYNC_GROUP_FLOW_REMOVE
) {
8476 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
8477 CHECK_TRUE(require_opt(opt_flow_id
), "ERROR: --flow-id not specified", EINVAL
);
8478 CHECK_TRUE(require_opt(opt_flow_type
,
8479 (symmetrical_flow_opt(*opt_flow_type
) ||
8480 directional_flow_opt(*opt_flow_type
))),
8481 "ERROR: --flow-type not specified or invalid (options: symmetrical, directional)", EINVAL
);
8483 SyncPolicyContext
sync_policy_ctx(zonegroup_id
, zonegroup_name
, opt_bucket
);
8484 ret
= sync_policy_ctx
.init();
8488 auto& sync_policy
= sync_policy_ctx
.get_policy();
8490 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
8491 if (iter
== sync_policy
.groups
.end()) {
8492 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
8496 auto& group
= iter
->second
;
8498 if (symmetrical_flow_opt(*opt_flow_type
)) {
8499 group
.data_flow
.remove_symmetrical(*opt_flow_id
, opt_zone_ids
);
8500 } else { /* directional */
8501 CHECK_TRUE(require_non_empty_opt(opt_source_zone_id
), "ERROR: --source-zone not provided for directional flow rule, or is empty", EINVAL
);
8502 CHECK_TRUE(require_non_empty_opt(opt_dest_zone_id
), "ERROR: --dest-zone not provided for directional flow rule, or is empty", EINVAL
);
8504 group
.data_flow
.remove_directional(*opt_source_zone_id
, *opt_dest_zone_id
);
8507 ret
= sync_policy_ctx
.write_policy();
8512 show_result(sync_policy
, zone_formatter
.get(), cout
);
8515 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_CREATE
||
8516 opt_cmd
== OPT::SYNC_GROUP_PIPE_MODIFY
) {
8517 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
8518 CHECK_TRUE(require_opt(opt_pipe_id
), "ERROR: --pipe-id not specified", EINVAL
);
8519 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_CREATE
) {
8520 CHECK_TRUE(require_non_empty_opt(opt_source_zone_ids
), "ERROR: --source-zones not provided or is empty; should be list of zones or '*'", EINVAL
);
8521 CHECK_TRUE(require_non_empty_opt(opt_dest_zone_ids
), "ERROR: --dest-zones not provided or is empty; should be list of zones or '*'", EINVAL
);
8524 SyncPolicyContext
sync_policy_ctx(zonegroup_id
, zonegroup_name
, opt_bucket
);
8525 ret
= sync_policy_ctx
.init();
8529 auto& sync_policy
= sync_policy_ctx
.get_policy();
8531 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
8532 if (iter
== sync_policy
.groups
.end()) {
8533 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
8537 auto& group
= iter
->second
;
8539 rgw_sync_bucket_pipes
*pipe
;
8541 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_CREATE
) {
8542 group
.find_pipe(*opt_pipe_id
, true, &pipe
);
8544 if (!group
.find_pipe(*opt_pipe_id
, false, &pipe
)) {
8545 cerr
<< "ERROR: could not find pipe '" << *opt_pipe_id
<< "'" << std::endl
;
8550 pipe
->source
.add_zones(*opt_source_zone_ids
);
8551 pipe
->source
.set_bucket(opt_source_tenant
,
8552 opt_source_bucket_name
,
8553 opt_source_bucket_id
);
8554 pipe
->dest
.add_zones(*opt_dest_zone_ids
);
8555 pipe
->dest
.set_bucket(opt_dest_tenant
,
8556 opt_dest_bucket_name
,
8557 opt_dest_bucket_id
);
8559 pipe
->params
.source
.filter
.set_prefix(opt_prefix
, !!opt_prefix_rm
);
8560 pipe
->params
.source
.filter
.set_tags(tags_add
, tags_rm
);
8561 if (opt_dest_owner
) {
8562 pipe
->params
.dest
.set_owner(*opt_dest_owner
);
8564 if (opt_storage_class
) {
8565 pipe
->params
.dest
.set_storage_class(*opt_storage_class
);
8568 pipe
->params
.priority
= *opt_priority
;
8571 if (*opt_mode
== "system") {
8572 pipe
->params
.mode
= rgw_sync_pipe_params::MODE_SYSTEM
;
8573 } else if (*opt_mode
== "user") {
8574 pipe
->params
.mode
= rgw_sync_pipe_params::MODE_USER
;
8576 cerr
<< "ERROR: bad mode value: should be one of the following: system, user" << std::endl
;
8581 if (!user_id
.empty()) {
8582 pipe
->params
.user
= user_id
;
8583 } else if (pipe
->params
.user
.empty()) {
8584 auto owner
= sync_policy_ctx
.get_owner();
8586 pipe
->params
.user
= *owner
;
8590 ret
= sync_policy_ctx
.write_policy();
8595 show_result(sync_policy
, zone_formatter
.get(), cout
);
8598 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_REMOVE
) {
8599 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
8600 CHECK_TRUE(require_opt(opt_pipe_id
), "ERROR: --pipe-id not specified", EINVAL
);
8602 SyncPolicyContext
sync_policy_ctx(zonegroup_id
, zonegroup_name
, opt_bucket
);
8603 ret
= sync_policy_ctx
.init();
8607 auto& sync_policy
= sync_policy_ctx
.get_policy();
8609 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
8610 if (iter
== sync_policy
.groups
.end()) {
8611 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
8615 auto& group
= iter
->second
;
8617 rgw_sync_bucket_pipes
*pipe
;
8619 if (!group
.find_pipe(*opt_pipe_id
, false, &pipe
)) {
8620 cerr
<< "ERROR: could not find pipe '" << *opt_pipe_id
<< "'" << std::endl
;
8624 if (opt_source_zone_ids
) {
8625 pipe
->source
.remove_zones(*opt_source_zone_ids
);
8628 pipe
->source
.remove_bucket(opt_source_tenant
,
8629 opt_source_bucket_name
,
8630 opt_source_bucket_id
);
8631 if (opt_dest_zone_ids
) {
8632 pipe
->dest
.remove_zones(*opt_dest_zone_ids
);
8634 pipe
->dest
.remove_bucket(opt_dest_tenant
,
8635 opt_dest_bucket_name
,
8636 opt_dest_bucket_id
);
8638 if (!(opt_source_zone_ids
||
8639 opt_source_tenant
||
8640 opt_source_bucket
||
8641 opt_source_bucket_id
||
8642 opt_dest_zone_ids
||
8645 opt_dest_bucket_id
)) {
8646 group
.remove_pipe(*opt_pipe_id
);
8649 ret
= sync_policy_ctx
.write_policy();
8654 show_result(sync_policy
, zone_formatter
.get(), cout
);
8657 if (opt_cmd
== OPT::SYNC_POLICY_GET
) {
8658 SyncPolicyContext
sync_policy_ctx(zonegroup_id
, zonegroup_name
, opt_bucket
);
8659 ret
= sync_policy_ctx
.init();
8663 auto& sync_policy
= sync_policy_ctx
.get_policy();
8665 show_result(sync_policy
, zone_formatter
.get(), cout
);
8668 if (opt_cmd
== OPT::BILOG_TRIM
) {
8669 if (bucket_name
.empty()) {
8670 cerr
<< "ERROR: bucket not specified" << std::endl
;
8673 RGWBucketInfo bucket_info
;
8674 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
8676 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8679 ret
= store
->svc()->bilog_rados
->log_trim(bucket_info
, shard_id
, start_marker
, end_marker
);
8681 cerr
<< "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret
) << std::endl
;
8686 if (opt_cmd
== OPT::BILOG_STATUS
) {
8687 if (bucket_name
.empty()) {
8688 cerr
<< "ERROR: bucket not specified" << std::endl
;
8691 RGWBucketInfo bucket_info
;
8692 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
8694 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8697 map
<int, string
> markers
;
8698 ret
= store
->svc()->bilog_rados
->get_log_status(bucket_info
, shard_id
,
8699 &markers
, null_yield
);
8701 cerr
<< "ERROR: get_bi_log_status(): " << cpp_strerror(-ret
) << std::endl
;
8704 formatter
->open_object_section("entries");
8705 encode_json("markers", markers
, formatter
.get());
8706 formatter
->close_section();
8707 formatter
->flush(cout
);
8710 if (opt_cmd
== OPT::BILOG_AUTOTRIM
) {
8711 RGWCoroutinesManager
crs(store
->ctx(), store
->getRados()->get_cr_registry());
8712 RGWHTTPManager
http(store
->ctx(), crs
.get_completion_mgr());
8713 int ret
= http
.start();
8715 cerr
<< "failed to initialize http client with " << cpp_strerror(ret
) << std::endl
;
8719 rgw::BucketTrimConfig config
;
8720 configure_bucket_trim(store
->ctx(), config
);
8722 rgw::BucketTrimManager
trim(store
, config
);
8725 cerr
<< "trim manager init failed with " << cpp_strerror(ret
) << std::endl
;
8728 ret
= crs
.run(trim
.create_admin_bucket_trim_cr(&http
));
8730 cerr
<< "automated bilog trim failed with " << cpp_strerror(ret
) << std::endl
;
8735 if (opt_cmd
== OPT::DATALOG_LIST
) {
8736 formatter
->open_array_section("entries");
8739 if (max_entries
< 0)
8741 if (!start_date
.empty()) {
8742 std::cerr
<< "start-date not allowed." << std::endl
;
8745 if (!end_date
.empty()) {
8746 std::cerr
<< "end-date not allowed." << std::endl
;
8749 if (!end_marker
.empty()) {
8750 std::cerr
<< "end-marker not allowed." << std::endl
;
8753 if (!start_marker
.empty()) {
8754 if (marker
.empty()) {
8755 marker
= start_marker
;
8757 std::cerr
<< "start-marker and marker not both allowed." << std::endl
;
8762 auto datalog_svc
= store
->svc()->datalog_rados
;
8763 RGWDataChangesLog::LogMarker log_marker
;
8766 std::vector
<rgw_data_change_log_entry
> entries
;
8767 if (specified_shard_id
) {
8768 ret
= datalog_svc
->list_entries(shard_id
, max_entries
- count
,
8772 std::make_optional(marker
),
8773 &marker
, &truncated
);
8775 ret
= datalog_svc
->list_entries(max_entries
- count
, entries
,
8776 log_marker
, &truncated
);
8779 cerr
<< "ERROR: datalog_svc->list_entries(): " << cpp_strerror(-ret
) << std::endl
;
8783 count
+= entries
.size();
8785 for (const auto& entry
: entries
) {
8787 encode_json("entry", entry
.entry
, formatter
.get());
8789 encode_json("entry", entry
, formatter
.get());
8792 formatter
.get()->flush(cout
);
8793 } while (truncated
&& count
< max_entries
);
8795 formatter
->close_section();
8796 formatter
->flush(cout
);
8799 if (opt_cmd
== OPT::DATALOG_STATUS
) {
8800 int i
= (specified_shard_id
? shard_id
: 0);
8802 formatter
->open_array_section("entries");
8803 for (; i
< g_ceph_context
->_conf
->rgw_data_log_num_shards
; i
++) {
8804 list
<cls_log_entry
> entries
;
8806 RGWDataChangesLogInfo info
;
8807 store
->svc()->datalog_rados
->get_info(i
, &info
);
8809 ::encode_json("info", info
, formatter
.get());
8811 if (specified_shard_id
)
8815 formatter
->close_section();
8816 formatter
->flush(cout
);
8819 if (opt_cmd
== OPT::DATALOG_AUTOTRIM
) {
8820 RGWCoroutinesManager
crs(store
->ctx(), store
->getRados()->get_cr_registry());
8821 RGWHTTPManager
http(store
->ctx(), crs
.get_completion_mgr());
8822 int ret
= http
.start();
8824 cerr
<< "failed to initialize http client with " << cpp_strerror(ret
) << std::endl
;
8828 auto num_shards
= g_conf()->rgw_data_log_num_shards
;
8829 std::vector
<std::string
> markers(num_shards
);
8830 ret
= crs
.run(create_admin_data_log_trim_cr(store
, &http
, num_shards
, markers
));
8832 cerr
<< "automated datalog trim failed with " << cpp_strerror(ret
) << std::endl
;
8837 if (opt_cmd
== OPT::DATALOG_TRIM
) {
8838 if (!start_date
.empty()) {
8839 std::cerr
<< "start-date not allowed." << std::endl
;
8842 if (!end_date
.empty()) {
8843 std::cerr
<< "end-date not allowed." << std::endl
;
8846 if (!start_marker
.empty()) {
8847 std::cerr
<< "start-marker not allowed." << std::endl
;
8850 if (!end_marker
.empty()) {
8851 if (marker
.empty()) {
8852 marker
= end_marker
;
8854 std::cerr
<< "end-marker and marker not both allowed." << std::endl
;
8859 if (!specified_shard_id
) {
8860 cerr
<< "ERROR: requires a --shard-id" << std::endl
;
8864 // loop until -ENODATA
8866 auto datalog
= store
->svc()->datalog_rados
;
8867 ret
= datalog
->trim_entries(shard_id
, marker
);
8870 if (ret
< 0 && ret
!= -ENODATA
) {
8871 cerr
<< "ERROR: trim_entries(): " << cpp_strerror(-ret
) << std::endl
;
8876 if (opt_cmd
== OPT::DATALOG_TYPE
) {
8877 if (!opt_log_type
) {
8878 std::cerr
<< "log-type not specified." << std::endl
;
8881 auto datalog
= static_cast<rgw::sal::RGWRadosStore
*>(store
)->svc()->datalog_rados
;
8882 ret
= datalog
->change_format(*opt_log_type
, null_yield
);
8884 cerr
<< "ERROR: change_format(): " << cpp_strerror(-ret
) << std::endl
;
8889 if (opt_cmd
== OPT::DATALOG_PRUNE
) {
8890 auto datalog
= static_cast<rgw::sal::RGWRadosStore
*>(store
)->svc()->datalog_rados
;
8891 std::optional
<uint64_t> through
;
8892 ret
= datalog
->trim_generations(through
);
8895 cerr
<< "ERROR: trim_generations(): " << cpp_strerror(-ret
) << std::endl
;
8900 std::cout
<< "Pruned " << *through
<< " empty generations." << std::endl
;
8902 std::cout
<< "No empty generations." << std::endl
;
8906 bool quota_op
= (opt_cmd
== OPT::QUOTA_SET
|| opt_cmd
== OPT::QUOTA_ENABLE
|| opt_cmd
== OPT::QUOTA_DISABLE
);
8909 if (bucket_name
.empty() && user_id
.empty()) {
8910 cerr
<< "ERROR: bucket name or uid is required for quota operation" << std::endl
;
8914 if (!bucket_name
.empty()) {
8915 if (!quota_scope
.empty() && quota_scope
!= "bucket") {
8916 cerr
<< "ERROR: invalid quota scope specification." << std::endl
;
8919 set_bucket_quota(store
, opt_cmd
, tenant
, bucket_name
,
8920 max_size
, max_objects
, have_max_size
, have_max_objects
);
8921 } else if (!user_id
.empty()) {
8922 if (quota_scope
== "bucket") {
8923 return set_user_bucket_quota(opt_cmd
, user
, user_op
, max_size
, max_objects
, have_max_size
, have_max_objects
);
8924 } else if (quota_scope
== "user") {
8925 return set_user_quota(opt_cmd
, user
, user_op
, max_size
, max_objects
, have_max_size
, have_max_objects
);
8927 cerr
<< "ERROR: invalid quota scope specification. Please specify either --quota-scope=bucket, or --quota-scope=user" << std::endl
;
8933 if (opt_cmd
== OPT::MFA_CREATE
) {
8934 rados::cls::otp::otp_info_t config
;
8936 if (user_id
.empty()) {
8937 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
8941 if (totp_serial
.empty()) {
8942 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
8946 if (totp_seed
.empty()) {
8947 cerr
<< "ERROR: TOTP device seed was not provided (via --totp-seed)" << std::endl
;
8952 rados::cls::otp::SeedType seed_type
;
8953 if (totp_seed_type
== "hex") {
8954 seed_type
= rados::cls::otp::OTP_SEED_HEX
;
8955 } else if (totp_seed_type
== "base32") {
8956 seed_type
= rados::cls::otp::OTP_SEED_BASE32
;
8958 cerr
<< "ERROR: invalid seed type: " << totp_seed_type
<< std::endl
;
8962 config
.id
= totp_serial
;
8963 config
.seed
= totp_seed
;
8964 config
.seed_type
= seed_type
;
8966 if (totp_seconds
> 0) {
8967 config
.step_size
= totp_seconds
;
8970 if (totp_window
> 0) {
8971 config
.window
= totp_window
;
8974 real_time mtime
= real_clock::now();
8975 string oid
= store
->svc()->cls
->mfa
.get_mfa_oid(user_id
);
8977 int ret
= store
->ctl()->meta
.mgr
->mutate(RGWSI_MetaBackend_OTP::get_meta_key(user_id
),
8978 mtime
, &objv_tracker
,
8982 return store
->svc()->cls
->mfa
.create_mfa(user_id
, config
, &objv_tracker
, mtime
, null_yield
);
8985 cerr
<< "MFA creation failed, error: " << cpp_strerror(-ret
) << std::endl
;
8989 RGWUserInfo
& user_info
= user_op
.get_user_info();
8990 user_info
.mfa_ids
.insert(totp_serial
);
8991 user_op
.set_mfa_ids(user_info
.mfa_ids
);
8993 ret
= user
.modify(user_op
, null_yield
, &err
);
8995 cerr
<< "ERROR: failed storing user info, error: " << err
<< std::endl
;
9000 if (opt_cmd
== OPT::MFA_REMOVE
) {
9001 if (user_id
.empty()) {
9002 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
9006 if (totp_serial
.empty()) {
9007 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
9011 real_time mtime
= real_clock::now();
9013 int ret
= store
->ctl()->meta
.mgr
->mutate(RGWSI_MetaBackend_OTP::get_meta_key(user_id
),
9014 mtime
, &objv_tracker
,
9018 return store
->svc()->cls
->mfa
.remove_mfa(user_id
, totp_serial
, &objv_tracker
, mtime
, null_yield
);
9021 cerr
<< "MFA removal failed, error: " << cpp_strerror(-ret
) << std::endl
;
9025 RGWUserInfo
& user_info
= user_op
.get_user_info();
9026 user_info
.mfa_ids
.erase(totp_serial
);
9027 user_op
.set_mfa_ids(user_info
.mfa_ids
);
9029 ret
= user
.modify(user_op
, null_yield
, &err
);
9031 cerr
<< "ERROR: failed storing user info, error: " << err
<< std::endl
;
9036 if (opt_cmd
== OPT::MFA_GET
) {
9037 if (user_id
.empty()) {
9038 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
9042 if (totp_serial
.empty()) {
9043 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
9047 rados::cls::otp::otp_info_t result
;
9048 int ret
= store
->svc()->cls
->mfa
.get_mfa(user_id
, totp_serial
, &result
, null_yield
);
9050 if (ret
== -ENOENT
|| ret
== -ENODATA
) {
9051 cerr
<< "MFA serial id not found" << std::endl
;
9053 cerr
<< "MFA retrieval failed, error: " << cpp_strerror(-ret
) << std::endl
;
9057 formatter
->open_object_section("result");
9058 encode_json("entry", result
, formatter
.get());
9059 formatter
->close_section();
9060 formatter
->flush(cout
);
9063 if (opt_cmd
== OPT::MFA_LIST
) {
9064 if (user_id
.empty()) {
9065 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
9069 list
<rados::cls::otp::otp_info_t
> result
;
9070 int ret
= store
->svc()->cls
->mfa
.list_mfa(user_id
, &result
, null_yield
);
9072 cerr
<< "MFA listing failed, error: " << cpp_strerror(-ret
) << std::endl
;
9075 formatter
->open_object_section("result");
9076 encode_json("entries", result
, formatter
.get());
9077 formatter
->close_section();
9078 formatter
->flush(cout
);
9081 if (opt_cmd
== OPT::MFA_CHECK
) {
9082 if (user_id
.empty()) {
9083 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
9087 if (totp_serial
.empty()) {
9088 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
9092 if (totp_pin
.empty()) {
9093 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-pin)" << std::endl
;
9097 list
<rados::cls::otp::otp_info_t
> result
;
9098 int ret
= store
->svc()->cls
->mfa
.check_mfa(user_id
, totp_serial
, totp_pin
.front(), null_yield
);
9100 cerr
<< "MFA check failed, error: " << cpp_strerror(-ret
) << std::endl
;
9104 cout
<< "ok" << std::endl
;
9107 if (opt_cmd
== OPT::MFA_RESYNC
) {
9108 if (user_id
.empty()) {
9109 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
9113 if (totp_serial
.empty()) {
9114 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
9118 if (totp_pin
.size() != 2) {
9119 cerr
<< "ERROR: missing two --totp-pin params (--totp-pin=<first> --totp-pin=<second>)" << std::endl
;
9122 rados::cls::otp::otp_info_t config
;
9123 int ret
= store
->svc()->cls
->mfa
.get_mfa(user_id
, totp_serial
, &config
, null_yield
);
9125 if (ret
== -ENOENT
|| ret
== -ENODATA
) {
9126 cerr
<< "MFA serial id not found" << std::endl
;
9128 cerr
<< "MFA retrieval failed, error: " << cpp_strerror(-ret
) << std::endl
;
9133 ceph::real_time now
;
9135 ret
= store
->svc()->cls
->mfa
.otp_get_current_time(user_id
, &now
, null_yield
);
9137 cerr
<< "ERROR: failed to fetch current time from osd: " << cpp_strerror(-ret
) << std::endl
;
9142 ret
= scan_totp(store
->ctx(), now
, config
, totp_pin
, &time_ofs
);
9144 if (ret
== -ENOENT
) {
9145 cerr
<< "failed to resync, TOTP values not found in range" << std::endl
;
9147 cerr
<< "ERROR: failed to scan for TOTP values: " << cpp_strerror(-ret
) << std::endl
;
9152 config
.time_ofs
= time_ofs
;
9154 /* now update the backend */
9155 real_time mtime
= real_clock::now();
9157 ret
= store
->ctl()->meta
.mgr
->mutate(RGWSI_MetaBackend_OTP::get_meta_key(user_id
),
9158 mtime
, &objv_tracker
,
9162 return store
->svc()->cls
->mfa
.create_mfa(user_id
, config
, &objv_tracker
, mtime
, null_yield
);
9165 cerr
<< "MFA update failed, error: " << cpp_strerror(-ret
) << std::endl
;
9171 if (opt_cmd
== OPT::RESHARD_STALE_INSTANCES_LIST
) {
9172 if (!store
->svc()->zone
->can_reshard() && !yes_i_really_mean_it
) {
9173 cerr
<< "Resharding disabled in a multisite env, stale instances unlikely from resharding" << std::endl
;
9174 cerr
<< "These instances may not be safe to delete." << std::endl
;
9175 cerr
<< "Use --yes-i-really-mean-it to force displaying these instances." << std::endl
;
9179 ret
= RGWBucketAdminOp::list_stale_instances(store
, bucket_op
,f
);
9181 cerr
<< "ERROR: listing stale instances" << cpp_strerror(-ret
) << std::endl
;
9185 if (opt_cmd
== OPT::RESHARD_STALE_INSTANCES_DELETE
) {
9186 if (!store
->svc()->zone
->can_reshard()) {
9187 cerr
<< "Resharding disabled in a multisite env. Stale instances are not safe to be deleted." << std::endl
;
9191 ret
= RGWBucketAdminOp::clear_stale_instances(store
, bucket_op
,f
);
9193 cerr
<< "ERROR: deleting stale instances" << cpp_strerror(-ret
) << std::endl
;
9197 if (opt_cmd
== OPT::PUBSUB_TOPICS_LIST
) {
9199 RGWPubSub
ps(store
, tenant
);
9203 if (!bucket_name
.empty()) {
9204 rgw_pubsub_bucket_topics result
;
9205 RGWBucketInfo bucket_info
;
9206 int ret
= init_bucket(tenant
, bucket_name
, bucket_id
, bucket_info
, bucket
);
9208 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
9212 auto b
= ps
.get_bucket(bucket_info
.bucket
);
9213 ret
= b
->get_topics(&result
);
9215 cerr
<< "ERROR: could not get topics: " << cpp_strerror(-ret
) << std::endl
;
9218 encode_json("result", result
, formatter
.get());
9220 rgw_pubsub_topics result
;
9221 int ret
= ps
.get_topics(&result
);
9223 cerr
<< "ERROR: could not get topics: " << cpp_strerror(-ret
) << std::endl
;
9226 encode_json("result", result
, formatter
.get());
9228 formatter
->flush(cout
);
9231 if (opt_cmd
== OPT::PUBSUB_TOPIC_GET
) {
9232 if (topic_name
.empty()) {
9233 cerr
<< "ERROR: topic name was not provided (via --topic)" << std::endl
;
9237 RGWPubSub
ps(store
, tenant
);
9239 rgw_pubsub_topic_subs topic
;
9240 ret
= ps
.get_topic(topic_name
, &topic
);
9242 cerr
<< "ERROR: could not get topic: " << cpp_strerror(-ret
) << std::endl
;
9245 encode_json("topic", topic
, formatter
.get());
9246 formatter
->flush(cout
);
9249 if (opt_cmd
== OPT::PUBSUB_TOPIC_RM
) {
9250 if (topic_name
.empty()) {
9251 cerr
<< "ERROR: topic name was not provided (via --topic)" << std::endl
;
9255 RGWPubSub
ps(store
, tenant
);
9257 ret
= ps
.remove_topic(topic_name
, null_yield
);
9259 cerr
<< "ERROR: could not remove topic: " << cpp_strerror(-ret
) << std::endl
;
9264 if (opt_cmd
== OPT::PUBSUB_SUB_GET
) {
9265 if (get_tier_type(store
) != "pubsub") {
9266 cerr
<< "ERROR: only pubsub tier type supports this command" << std::endl
;
9269 if (sub_name
.empty()) {
9270 cerr
<< "ERROR: subscription name was not provided (via --subscription)" << std::endl
;
9274 RGWPubSub
ps(store
, tenant
);
9276 rgw_pubsub_sub_config sub_conf
;
9278 auto sub
= ps
.get_sub(sub_name
);
9279 ret
= sub
->get_conf(&sub_conf
);
9281 cerr
<< "ERROR: could not get subscription info: " << cpp_strerror(-ret
) << std::endl
;
9284 encode_json("sub", sub_conf
, formatter
.get());
9285 formatter
->flush(cout
);
9288 if (opt_cmd
== OPT::PUBSUB_SUB_RM
) {
9289 if (get_tier_type(store
) != "pubsub") {
9290 cerr
<< "ERROR: only pubsub tier type supports this command" << std::endl
;
9293 if (sub_name
.empty()) {
9294 cerr
<< "ERROR: subscription name was not provided (via --subscription)" << std::endl
;
9298 RGWPubSub
ps(store
, tenant
);
9300 auto sub
= ps
.get_sub(sub_name
);
9301 ret
= sub
->unsubscribe(topic_name
, null_yield
);
9303 cerr
<< "ERROR: could not get subscription info: " << cpp_strerror(-ret
) << std::endl
;
9308 if (opt_cmd
== OPT::PUBSUB_SUB_PULL
) {
9309 if (get_tier_type(store
) != "pubsub") {
9310 cerr
<< "ERROR: only pubsub tier type supports this command" << std::endl
;
9313 if (sub_name
.empty()) {
9314 cerr
<< "ERROR: subscription name was not provided (via --subscription)" << std::endl
;
9318 RGWPubSub
ps(store
, tenant
);
9320 if (!max_entries_specified
) {
9321 max_entries
= RGWPubSub::Sub::DEFAULT_MAX_EVENTS
;
9323 auto sub
= ps
.get_sub_with_events(sub_name
);
9324 ret
= sub
->list_events(marker
, max_entries
);
9326 cerr
<< "ERROR: could not list events: " << cpp_strerror(-ret
) << std::endl
;
9329 encode_json("result", *sub
, formatter
.get());
9330 formatter
->flush(cout
);
9333 if (opt_cmd
== OPT::PUBSUB_EVENT_RM
) {
9334 if (get_tier_type(store
) != "pubsub") {
9335 cerr
<< "ERROR: only pubsub tier type supports this command" << std::endl
;
9338 if (sub_name
.empty()) {
9339 cerr
<< "ERROR: subscription name was not provided (via --subscription)" << std::endl
;
9342 if (event_id
.empty()) {
9343 cerr
<< "ERROR: event id was not provided (via --event-id)" << std::endl
;
9347 RGWPubSub
ps(store
, tenant
);
9349 auto sub
= ps
.get_sub_with_events(sub_name
);
9350 ret
= sub
->remove_event(event_id
);
9352 cerr
<< "ERROR: could not remove event: " << cpp_strerror(-ret
) << std::endl
;
9357 if (opt_cmd
== OPT::SCRIPT_PUT
) {
9358 if (!str_script_ctx
) {
9359 cerr
<< "ERROR: context was not provided (via --context)" << std::endl
;
9362 if (infile
.empty()) {
9363 cerr
<< "ERROR: infile was not provided (via --infile)" << std::endl
;
9367 auto rc
= read_input(infile
, bl
);
9369 cerr
<< "ERROR: failed to read script: '" << infile
<< "'. error: " << rc
<< std::endl
;
9372 const std::string script
= bl
.to_str();
9373 std::string err_msg
;
9374 if (!rgw::lua::verify(script
, err_msg
)) {
9375 cerr
<< "ERROR: script: '" << infile
<< "' has error: " << std::endl
<< err_msg
<< std::endl
;
9378 const rgw::lua::context script_ctx
= rgw::lua::to_context(*str_script_ctx
);
9379 if (script_ctx
== rgw::lua::context::none
) {
9380 cerr
<< "ERROR: invalid script context: " << *str_script_ctx
<< ". must be one of: preRequest, postRequest" << std::endl
;
9383 rc
= rgw::lua::write_script(store
, tenant
, null_yield
, script_ctx
, script
);
9385 cerr
<< "ERROR: failed to put script. error: " << rc
<< std::endl
;
9390 if (opt_cmd
== OPT::SCRIPT_GET
) {
9391 if (!str_script_ctx
) {
9392 cerr
<< "ERROR: context was not provided (via --context)" << std::endl
;
9395 const rgw::lua::context script_ctx
= rgw::lua::to_context(*str_script_ctx
);
9396 if (script_ctx
== rgw::lua::context::none
) {
9397 cerr
<< "ERROR: invalid script context: " << *str_script_ctx
<< ". must be one of: preRequest, postRequest" << std::endl
;
9401 const auto rc
= rgw::lua::read_script(store
, tenant
, null_yield
, script_ctx
, script
);
9402 if (rc
== -ENOENT
) {
9403 std::cout
<< "no script exists for context: " << *str_script_ctx
<<
9404 (tenant
.empty() ? "" : (" in tenant: " + tenant
)) << std::endl
;
9405 } else if (rc
< 0) {
9406 cerr
<< "ERROR: failed to read script. error: " << rc
<< std::endl
;
9409 std::cout
<< script
<< std::endl
;
9413 if (opt_cmd
== OPT::SCRIPT_RM
) {
9414 if (!str_script_ctx
) {
9415 cerr
<< "ERROR: context was not provided (via --context)" << std::endl
;
9418 const rgw::lua::context script_ctx
= rgw::lua::to_context(*str_script_ctx
);
9419 if (script_ctx
== rgw::lua::context::none
) {
9420 cerr
<< "ERROR: invalid script context: " << *str_script_ctx
<< ". must be one of: preRequest, postRequest" << std::endl
;
9423 const auto rc
= rgw::lua::delete_script(store
, tenant
, null_yield
, script_ctx
);
9425 cerr
<< "ERROR: failed to remove script. error: " << rc
<< std::endl
;
9430 if (opt_cmd
== OPT::SCRIPT_PACKAGE_ADD
) {
9431 #ifdef WITH_RADOSGW_LUA_PACKAGES
9432 if (!script_package
) {
9433 cerr
<< "ERROR: lua package name was not provided (via --package)" << std::endl
;
9436 const auto rc
= rgw::lua::add_package(store
, null_yield
, *script_package
, bool(allow_compilation
));
9438 cerr
<< "ERROR: failed to add lua package: " << script_package
<< " .error: " << rc
<< std::endl
;
9442 cerr
<< "ERROR: adding lua packages is not permitted" << std::endl
;
9447 if (opt_cmd
== OPT::SCRIPT_PACKAGE_RM
) {
9448 #ifdef WITH_RADOSGW_LUA_PACKAGES
9449 if (!script_package
) {
9450 cerr
<< "ERROR: lua package name was not provided (via --package)" << std::endl
;
9453 const auto rc
= rgw::lua::remove_package(store
, null_yield
, *script_package
);
9454 if (rc
== -ENOENT
) {
9455 cerr
<< "WARNING: package " << script_package
<< " did not exists or already removed" << std::endl
;
9459 cerr
<< "ERROR: failed to remove lua package: " << script_package
<< " .error: " << rc
<< std::endl
;
9463 cerr
<< "ERROR: removing lua packages in not permitted" << std::endl
;
9468 if (opt_cmd
== OPT::SCRIPT_PACKAGE_LIST
) {
9469 #ifdef WITH_RADOSGW_LUA_PACKAGES
9470 rgw::lua::packages_t packages
;
9471 const auto rc
= rgw::lua::list_packages(store
, null_yield
, packages
);
9472 if (rc
== -ENOENT
) {
9473 std::cout
<< "no lua packages in allowlist" << std::endl
;
9474 } else if (rc
< 0) {
9475 cerr
<< "ERROR: failed to read lua packages allowlist. error: " << rc
<< std::endl
;
9478 for (const auto& package
: packages
) {
9479 std::cout
<< package
<< std::endl
;
9483 cerr
<< "ERROR: listing lua packages in not permitted" << std::endl
;