1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
9 #include <boost/asio.hpp>
10 #include <boost/optional.hpp>
13 #include <liboath/oath.h>
16 #include <fmt/format.h>
18 #include "auth/Crypto.h"
19 #include "compressor/Compressor.h"
21 #include "common/armor.h"
22 #include "common/ceph_json.h"
23 #include "common/config.h"
24 #include "common/ceph_argparse.h"
25 #include "common/Formatter.h"
26 #include "common/errno.h"
27 #include "common/safe_io.h"
28 #include "common/fault_injector.h"
30 #include "include/util.h"
32 #include "cls/rgw/cls_rgw_types.h"
33 #include "cls/rgw/cls_rgw_client.h"
35 #include "include/utime.h"
36 #include "include/str_list.h"
40 #include "rgw_rados.h"
42 #include "rgw_acl_s3.h"
43 #include "rgw_datalog.h"
46 #include "rgw_formats.h"
47 #include "rgw_usage.h"
48 #include "rgw_orphan.h"
50 #include "rgw_trim_bilog.h"
51 #include "rgw_trim_datalog.h"
52 #include "rgw_trim_mdlog.h"
53 #include "rgw_data_sync.h"
54 #include "rgw_rest_conn.h"
55 #include "rgw_realm_watcher.h"
57 #include "rgw_reshard.h"
58 #include "rgw_http_client_curl.h"
60 #include "rgw_pubsub.h"
61 #include "rgw_bucket_sync.h"
62 #include "rgw_sync_checkpoint.h"
65 #include "rgw_sal_config.h"
67 #include "services/svc_sync_modules.h"
68 #include "services/svc_cls.h"
69 #include "services/svc_bilog_rados.h"
70 #include "services/svc_mdlog.h"
71 #include "services/svc_meta_be_otp.h"
72 #include "services/svc_user.h"
73 #include "services/svc_zone.h"
75 #include "driver/rados/rgw_bucket.h"
76 #include "driver/rados/rgw_sal_rados.h"
78 #define dout_context g_ceph_context
80 #define SECRET_KEY_LEN 40
81 #define PUBLIC_ID_LEN 20
85 static rgw::sal::Driver
* driver
= NULL
;
86 static constexpr auto dout_subsys
= ceph_subsys_rgw
;
89 static const DoutPrefixProvider
* dpp() {
90 struct GlobalPrefix
: public DoutPrefixProvider
{
91 CephContext
*get_cct() const override
{ return dout_context
; }
92 unsigned get_subsys() const override
{ return dout_subsys
; }
93 std::ostream
& gen_prefix(std::ostream
& out
) const override
{ return out
; }
95 static GlobalPrefix global_dpp
;
99 #define CHECK_TRUE(x, msg, err) \
102 cerr << msg << std::endl; \
107 #define CHECK_SUCCESS(x, msg) \
111 cerr << msg << ": " << cpp_strerror(-_x_val) << std::endl; \
116 static inline int posix_errortrans(int r
)
119 case ERR_NO_SUCH_BUCKET
:
129 static const std::string
LUA_CONTEXT_LIST("prerequest, postrequest, background, getdata, putdata");
133 cout
<< "usage: radosgw-admin <cmd> [options...]" << std::endl
;
134 cout
<< "commands:\n";
135 cout
<< " user create create a new user\n" ;
136 cout
<< " user modify modify user\n";
137 cout
<< " user info get user info\n";
138 cout
<< " user rename rename user\n";
139 cout
<< " user rm remove user\n";
140 cout
<< " user suspend suspend a user\n";
141 cout
<< " user enable re-enable user after suspension\n";
142 cout
<< " user check check user info\n";
143 cout
<< " user stats show user stats as accounted by quota subsystem\n";
144 cout
<< " user list list users\n";
145 cout
<< " caps add add user capabilities\n";
146 cout
<< " caps rm remove user capabilities\n";
147 cout
<< " subuser create create a new subuser\n" ;
148 cout
<< " subuser modify modify subuser\n";
149 cout
<< " subuser rm remove subuser\n";
150 cout
<< " key create create access key\n";
151 cout
<< " key rm remove access key\n";
152 cout
<< " bucket list list buckets (specify --allow-unordered for\n";
153 cout
<< " faster, unsorted listing)\n";
154 cout
<< " bucket limit check show bucket sharding stats\n";
155 cout
<< " bucket link link bucket to specified user\n";
156 cout
<< " bucket unlink unlink bucket from specified user\n";
157 cout
<< " bucket stats returns bucket statistics\n";
158 cout
<< " bucket rm remove bucket\n";
159 cout
<< " bucket check check bucket index by verifying size and object count stats\n";
160 cout
<< " bucket check olh check for olh index entries and objects that are pending removal\n";
161 cout
<< " bucket check unlinked check for object versions that are not visible in a bucket listing \n";
162 cout
<< " bucket chown link bucket to specified user and update its object ACLs\n";
163 cout
<< " bucket reshard reshard bucket\n";
164 cout
<< " bucket rewrite rewrite all objects in the specified bucket\n";
165 cout
<< " bucket sync checkpoint poll a bucket's sync status until it catches up to its remote\n";
166 cout
<< " bucket sync disable disable bucket sync\n";
167 cout
<< " bucket sync enable enable bucket sync\n";
168 cout
<< " bucket radoslist list rados objects backing bucket's objects\n";
169 cout
<< " bi get retrieve bucket index object entries\n";
170 cout
<< " bi put store bucket index object entries\n";
171 cout
<< " bi list list raw bucket index entries\n";
172 cout
<< " bi purge purge bucket index entries\n";
173 cout
<< " object rm remove object\n";
174 cout
<< " object put put object\n";
175 cout
<< " object stat stat an object for its metadata\n";
176 cout
<< " object unlink unlink object from bucket index\n";
177 cout
<< " object rewrite rewrite the specified object\n";
178 cout
<< " object reindex reindex the object(s) indicated by --bucket and either --object or --objects-file\n";
179 cout
<< " objects expire run expired objects cleanup\n";
180 cout
<< " objects expire-stale list list stale expired objects (caused by reshard)\n";
181 cout
<< " objects expire-stale rm remove stale expired objects\n";
182 cout
<< " period rm remove a period\n";
183 cout
<< " period get get period info\n";
184 cout
<< " period get-current get current period info\n";
185 cout
<< " period pull pull a period\n";
186 cout
<< " period push push a period\n";
187 cout
<< " period list list all periods\n";
188 cout
<< " period update update the staging period\n";
189 cout
<< " period commit commit the staging period\n";
190 cout
<< " quota set set quota params\n";
191 cout
<< " quota enable enable quota\n";
192 cout
<< " quota disable disable quota\n";
193 cout
<< " ratelimit get get ratelimit params\n";
194 cout
<< " ratelimit set set ratelimit params\n";
195 cout
<< " ratelimit enable enable ratelimit\n";
196 cout
<< " ratelimit disable disable ratelimit\n";
197 cout
<< " global quota get view global quota params\n";
198 cout
<< " global quota set set global quota params\n";
199 cout
<< " global quota enable enable a global quota\n";
200 cout
<< " global quota disable disable a global quota\n";
201 cout
<< " global ratelimit get view global ratelimit params\n";
202 cout
<< " global ratelimit set set global ratelimit params\n";
203 cout
<< " global ratelimit enable enable a ratelimit quota\n";
204 cout
<< " global ratelimit disable disable a ratelimit quota\n";
205 cout
<< " realm create create a new realm\n";
206 cout
<< " realm rm remove a realm\n";
207 cout
<< " realm get show realm info\n";
208 cout
<< " realm get-default get default realm name\n";
209 cout
<< " realm list list realms\n";
210 cout
<< " realm list-periods list all realm periods\n";
211 cout
<< " realm rename rename a realm\n";
212 cout
<< " realm set set realm info (requires infile)\n";
213 cout
<< " realm default set realm as default\n";
214 cout
<< " realm pull pull a realm and its current period\n";
215 cout
<< " zonegroup add add a zone to a zonegroup\n";
216 cout
<< " zonegroup create create a new zone group info\n";
217 cout
<< " zonegroup default set default zone group\n";
218 cout
<< " zonegroup delete delete a zone group info\n";
219 cout
<< " zonegroup get show zone group info\n";
220 cout
<< " zonegroup modify modify an existing zonegroup\n";
221 cout
<< " zonegroup set set zone group info (requires infile)\n";
222 cout
<< " zonegroup rm remove a zone from a zonegroup\n";
223 cout
<< " zonegroup rename rename a zone group\n";
224 cout
<< " zonegroup list list all zone groups set on this cluster\n";
225 cout
<< " zonegroup placement list list zonegroup's placement targets\n";
226 cout
<< " zonegroup placement get get a placement target of a specific zonegroup\n";
227 cout
<< " zonegroup placement add add a placement target id to a zonegroup\n";
228 cout
<< " zonegroup placement modify modify a placement target of a specific zonegroup\n";
229 cout
<< " zonegroup placement rm remove a placement target from a zonegroup\n";
230 cout
<< " zonegroup placement default set a zonegroup's default placement target\n";
231 cout
<< " zone create create a new zone\n";
232 cout
<< " zone rm remove a zone\n";
233 cout
<< " zone get show zone cluster params\n";
234 cout
<< " zone modify modify an existing zone\n";
235 cout
<< " zone set set zone cluster params (requires infile)\n";
236 cout
<< " zone list list all zones set on this cluster\n";
237 cout
<< " zone rename rename a zone\n";
238 cout
<< " zone placement list list zone's placement targets\n";
239 cout
<< " zone placement get get a zone placement target\n";
240 cout
<< " zone placement add add a zone placement target\n";
241 cout
<< " zone placement modify modify a zone placement target\n";
242 cout
<< " zone placement rm remove a zone placement target\n";
243 cout
<< " metadata sync status get metadata sync status\n";
244 cout
<< " metadata sync init init metadata sync\n";
245 cout
<< " metadata sync run run metadata sync\n";
246 cout
<< " data sync status get data sync status of the specified source zone\n";
247 cout
<< " data sync init init data sync for the specified source zone\n";
248 cout
<< " data sync run run data sync for the specified source zone\n";
249 cout
<< " pool add add an existing pool for data placement\n";
250 cout
<< " pool rm remove an existing pool from data placement set\n";
251 cout
<< " pools list list placement active set\n";
252 cout
<< " policy read bucket/object policy\n";
253 cout
<< " log list list log objects\n";
254 cout
<< " log show dump a log from specific object or (bucket + date\n";
255 cout
<< " + bucket-id)\n";
256 cout
<< " (NOTE: required to specify formatting of date\n";
257 cout
<< " to \"YYYY-MM-DD-hh\")\n";
258 cout
<< " log rm remove log object\n";
259 cout
<< " usage show show usage (by user, by bucket, date range)\n";
260 cout
<< " usage trim trim usage (by user, by bucket, date range)\n";
261 cout
<< " usage clear reset all the usage stats for the cluster\n";
262 cout
<< " gc list dump expired garbage collection objects (specify\n";
263 cout
<< " --include-all to list all entries, including unexpired)\n";
264 cout
<< " gc process manually process garbage (specify\n";
265 cout
<< " --include-all to process all entries, including unexpired)\n";
266 cout
<< " lc list list all bucket lifecycle progress\n";
267 cout
<< " lc get get a lifecycle bucket configuration\n";
268 cout
<< " lc process manually process lifecycle\n";
269 cout
<< " lc reshard fix fix LC for a resharded bucket\n";
270 cout
<< " metadata get get metadata info\n";
271 cout
<< " metadata put put metadata info\n";
272 cout
<< " metadata rm remove metadata info\n";
273 cout
<< " metadata list list metadata info\n";
274 cout
<< " mdlog list list metadata log\n";
275 cout
<< " mdlog autotrim auto trim metadata log\n";
276 cout
<< " mdlog trim trim metadata log (use marker)\n";
277 cout
<< " mdlog status read metadata log status\n";
278 cout
<< " bilog list list bucket index log\n";
279 cout
<< " bilog trim trim bucket index log (use start-marker, end-marker)\n";
280 cout
<< " bilog status read bucket index log status\n";
281 cout
<< " bilog autotrim auto trim bucket index log\n";
282 cout
<< " datalog list list data log\n";
283 cout
<< " datalog trim trim data log\n";
284 cout
<< " datalog status read data log status\n";
285 cout
<< " datalog type change datalog type to --log_type={fifo,omap}\n";
286 cout
<< " orphans find deprecated -- init and run search for leaked rados objects (use job-id, pool)\n";
287 cout
<< " orphans finish deprecated -- clean up search for leaked rados objects\n";
288 cout
<< " orphans list-jobs deprecated -- list the current job-ids for orphans search\n";
289 cout
<< " * the three 'orphans' sub-commands are now deprecated; consider using the `rgw-orphan-list` tool\n";
290 cout
<< " role create create a AWS role for use with STS\n";
291 cout
<< " role delete remove a role\n";
292 cout
<< " role get get a role\n";
293 cout
<< " role list list roles with specified path prefix\n";
294 cout
<< " role-trust-policy modify modify the assume role policy of an existing role\n";
295 cout
<< " role-policy put add/update permission policy to role\n";
296 cout
<< " role-policy list list policies attached to a role\n";
297 cout
<< " role-policy get get the specified inline policy document embedded with the given role\n";
298 cout
<< " role-policy delete remove policy attached to a role\n";
299 cout
<< " role update update max_session_duration of a role\n";
300 cout
<< " reshard add schedule a resharding of a bucket\n";
301 cout
<< " reshard list list all bucket resharding or scheduled to be resharded\n";
302 cout
<< " reshard status read bucket resharding status\n";
303 cout
<< " reshard process process of scheduled reshard jobs\n";
304 cout
<< " reshard cancel cancel resharding a bucket\n";
305 cout
<< " reshard stale-instances list list stale-instances from bucket resharding\n";
306 cout
<< " reshard stale-instances delete cleanup stale-instances from bucket resharding\n";
307 cout
<< " sync error list list sync error\n";
308 cout
<< " sync error trim trim sync error\n";
309 cout
<< " mfa create create a new MFA TOTP token\n";
310 cout
<< " mfa list list MFA TOTP tokens\n";
311 cout
<< " mfa get show MFA TOTP token\n";
312 cout
<< " mfa remove delete MFA TOTP token\n";
313 cout
<< " mfa check check MFA TOTP token\n";
314 cout
<< " mfa resync re-sync MFA TOTP token\n";
315 cout
<< " topic list list bucket notifications topics\n";
316 cout
<< " topic get get a bucket notifications topic\n";
317 cout
<< " topic rm remove a bucket notifications topic\n";
318 cout
<< " script put upload a lua script to a context\n";
319 cout
<< " script get get the lua script of a context\n";
320 cout
<< " script rm remove the lua scripts of a context\n";
321 cout
<< " script-package add add a lua package to the scripts allowlist\n";
322 cout
<< " script-package rm remove a lua package from the scripts allowlist\n";
323 cout
<< " script-package list get the lua packages allowlist\n";
324 cout
<< " notification list list bucket notifications configuration\n";
325 cout
<< " notification get get a bucket notifications configuration\n";
326 cout
<< " notification rm remove a bucket notifications configuration\n";
327 cout
<< "options:\n";
328 cout
<< " --tenant=<tenant> tenant name\n";
329 cout
<< " --user_ns=<namespace> namespace of user (oidc in case of users authenticated with oidc provider)\n";
330 cout
<< " --uid=<id> user id\n";
331 cout
<< " --new-uid=<id> new user id\n";
332 cout
<< " --subuser=<name> subuser name\n";
333 cout
<< " --access-key=<key> S3 access key\n";
334 cout
<< " --email=<email> user's email address\n";
335 cout
<< " --secret/--secret-key=<key>\n";
336 cout
<< " specify secret key\n";
337 cout
<< " --gen-access-key generate random access key (for S3)\n";
338 cout
<< " --gen-secret generate random secret key\n";
339 cout
<< " --key-type=<type> key type, options are: swift, s3\n";
340 cout
<< " --temp-url-key[-2]=<key> temp url key\n";
341 cout
<< " --access=<access> Set access permissions for sub-user, should be one\n";
342 cout
<< " of read, write, readwrite, full\n";
343 cout
<< " --display-name=<name> user's display name\n";
344 cout
<< " --max-buckets max number of buckets for a user\n";
345 cout
<< " --admin set the admin flag on the user\n";
346 cout
<< " --system set the system flag on the user\n";
347 cout
<< " --op-mask set the op mask on the user\n";
348 cout
<< " --bucket=<bucket> Specify the bucket name. Also used by the quota command.\n";
349 cout
<< " --pool=<pool> Specify the pool name. Also used to scan for leaked rados objects.\n";
350 cout
<< " --object=<object> object name\n";
351 cout
<< " --objects-file=<file> file containing a list of object names to process\n";
352 cout
<< " --object-version=<version> object version\n";
353 cout
<< " --date=<date> date in the format yyyy-mm-dd\n";
354 cout
<< " --start-date=<date> start date in the format yyyy-mm-dd\n";
355 cout
<< " --end-date=<date> end date in the format yyyy-mm-dd\n";
356 cout
<< " --bucket-id=<bucket-id> bucket id\n";
357 cout
<< " --bucket-new-name=<bucket>\n";
358 cout
<< " for bucket link: optional new name\n";
359 cout
<< " --shard-id=<shard-id> optional for: \n";
360 cout
<< " mdlog list\n";
361 cout
<< " data sync status\n";
362 cout
<< " required for: \n";
363 cout
<< " mdlog trim\n";
364 cout
<< " --gen=<gen-id> optional for: \n";
365 cout
<< " bilog list\n";
366 cout
<< " bilog trim\n";
367 cout
<< " bilog status\n";
368 cout
<< " --max-entries=<entries> max entries for listing operations\n";
369 cout
<< " --metadata-key=<key> key to retrieve metadata from with metadata get\n";
370 cout
<< " --remote=<remote> zone or zonegroup id of remote gateway\n";
371 cout
<< " --period=<id> period id\n";
372 cout
<< " --url=<url> url for pushing/pulling period/realm\n";
373 cout
<< " --epoch=<number> period epoch\n";
374 cout
<< " --commit commit the period during 'period update'\n";
375 cout
<< " --staging get staging period info\n";
376 cout
<< " --master set as master\n";
377 cout
<< " --master-zone=<id> master zone id\n";
378 cout
<< " --rgw-realm=<name> realm name\n";
379 cout
<< " --realm-id=<id> realm id\n";
380 cout
<< " --realm-new-name=<name> realm new name\n";
381 cout
<< " --rgw-zonegroup=<name> zonegroup name\n";
382 cout
<< " --zonegroup-id=<id> zonegroup id\n";
383 cout
<< " --zonegroup-new-name=<name>\n";
384 cout
<< " zonegroup new name\n";
385 cout
<< " --rgw-zone=<name> name of zone in which radosgw is running\n";
386 cout
<< " --zone-id=<id> zone id\n";
387 cout
<< " --zone-new-name=<name> zone new name\n";
388 cout
<< " --source-zone specify the source zone (for data sync)\n";
389 cout
<< " --default set entity (realm, zonegroup, zone) as default\n";
390 cout
<< " --read-only set zone as read-only (when adding to zonegroup)\n";
391 cout
<< " --redirect-zone specify zone id to redirect when response is 404 (not found)\n";
392 cout
<< " --placement-id placement id for zonegroup placement commands\n";
393 cout
<< " --storage-class storage class for zonegroup placement commands\n";
394 cout
<< " --tags=<list> list of tags for zonegroup placement add and modify commands\n";
395 cout
<< " --tags-add=<list> list of tags to add for zonegroup placement modify command\n";
396 cout
<< " --tags-rm=<list> list of tags to remove for zonegroup placement modify command\n";
397 cout
<< " --endpoints=<list> zone endpoints\n";
398 cout
<< " --index-pool=<pool> placement target index pool\n";
399 cout
<< " --data-pool=<pool> placement target data pool\n";
400 cout
<< " --data-extra-pool=<pool> placement target data extra (non-ec) pool\n";
401 cout
<< " --placement-index-type=<type>\n";
402 cout
<< " placement target index type (normal, indexless, or #id)\n";
403 cout
<< " --placement-inline-data=<true>\n";
404 cout
<< " set whether the placement target is configured to store a data\n";
405 cout
<< " chunk inline in head objects\n";
406 cout
<< " --compression=<type> placement target compression type (plugin name or empty/none)\n";
407 cout
<< " --tier-type=<type> zone tier type\n";
408 cout
<< " --tier-config=<k>=<v>[,...]\n";
409 cout
<< " set zone tier config keys, values\n";
410 cout
<< " --tier-config-rm=<k>[,...]\n";
411 cout
<< " unset zone tier config keys\n";
412 cout
<< " --sync-from-all[=false] set/reset whether zone syncs from all zonegroup peers\n";
413 cout
<< " --sync-from=[zone-name][,...]\n";
414 cout
<< " set list of zones to sync from\n";
415 cout
<< " --sync-from-rm=[zone-name][,...]\n";
416 cout
<< " remove zones from list of zones to sync from\n";
417 cout
<< " --bucket-index-max-shards override a zone/zonegroup's default bucket index shard count\n";
418 cout
<< " --fix besides checking bucket index, will also fix it\n";
419 cout
<< " --check-objects bucket check: rebuilds bucket index according to\n";
420 cout
<< " actual objects state\n";
421 cout
<< " --format=<format> specify output format for certain operations: xml,\n";
423 cout
<< " --purge-data when specified, user removal will also purge all the\n";
424 cout
<< " user data\n";
425 cout
<< " --purge-keys when specified, subuser removal will also purge all the\n";
426 cout
<< " subuser keys\n";
427 cout
<< " --purge-objects remove a bucket's objects before deleting it\n";
428 cout
<< " (NOTE: required to delete a non-empty bucket)\n";
429 cout
<< " --sync-stats option to 'user stats', update user stats with current\n";
430 cout
<< " stats reported by user's buckets indexes\n";
431 cout
<< " --reset-stats option to 'user stats', reset stats in accordance with user buckets\n";
432 cout
<< " --show-config show configuration\n";
433 cout
<< " --show-log-entries=<flag> enable/disable dump of log entries on log show\n";
434 cout
<< " --show-log-sum=<flag> enable/disable dump of log summation on log show\n";
435 cout
<< " --skip-zero-entries log show only dumps entries that don't have zero value\n";
436 cout
<< " in one of the numeric field\n";
437 cout
<< " --infile=<file> file to read in when setting data\n";
438 cout
<< " --categories=<list> comma separated list of categories, used in usage show\n";
439 cout
<< " --caps=<caps> list of caps (e.g., \"usage=read, write; user=read\")\n";
440 cout
<< " --op-mask=<op-mask> permission of user's operations (e.g., \"read, write, delete, *\")\n";
441 cout
<< " --yes-i-really-mean-it required for certain operations\n";
442 cout
<< " --warnings-only when specified with bucket limit check, list\n";
443 cout
<< " only buckets nearing or over the current max\n";
444 cout
<< " objects per shard value\n";
445 cout
<< " --bypass-gc when specified with bucket deletion, triggers\n";
446 cout
<< " object deletions by not involving GC\n";
447 cout
<< " --inconsistent-index when specified with bucket deletion and bypass-gc set to true,\n";
448 cout
<< " ignores bucket index consistency\n";
449 cout
<< " --min-rewrite-size min object size for bucket rewrite (default 4M)\n";
450 cout
<< " --max-rewrite-size max object size for bucket rewrite (default ULLONG_MAX)\n";
451 cout
<< " --min-rewrite-stripe-size min stripe size for object rewrite (default 0)\n";
452 cout
<< " --trim-delay-ms time interval in msec to limit the frequency of sync error log entries trimming operations,\n";
453 cout
<< " the trimming process will sleep the specified msec for every 1000 entries trimmed\n";
454 cout
<< " --max-concurrent-ios maximum concurrent ios for bucket operations (default: 32)\n";
455 cout
<< " --enable-feature enable a zone/zonegroup feature\n";
456 cout
<< " --disable-feature disable a zone/zonegroup feature\n";
458 cout
<< "<date> := \"YYYY-MM-DD[ hh:mm:ss]\"\n";
459 cout
<< "\nQuota options:\n";
460 cout
<< " --max-objects specify max objects (negative value to disable)\n";
461 cout
<< " --max-size specify max size (in B/K/M/G/T, negative value to disable)\n";
462 cout
<< " --quota-scope scope of quota (bucket, user)\n";
463 cout
<< "\nRate limiting options:\n";
464 cout
<< " --max-read-ops specify max requests per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n";
465 cout
<< " --max-read-bytes specify max bytes per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n";
466 cout
<< " --max-write-ops specify max requests per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n";
467 cout
<< " --max-write-bytes specify max bytes per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n";
468 cout
<< " --ratelimit-scope scope of rate limiting: bucket, user, anonymous\n";
469 cout
<< " anonymous can be configured only with global rate limit\n";
470 cout
<< "\nOrphans search options:\n";
471 cout
<< " --num-shards num of shards to use for keeping the temporary scan info\n";
472 cout
<< " --orphan-stale-secs num of seconds to wait before declaring an object to be an orphan (default: 86400)\n";
473 cout
<< " --job-id set the job id (for orphans find)\n";
474 cout
<< " --detail detailed mode, log and stat head objects as well\n";
475 cout
<< "\nOrphans list-jobs options:\n";
476 cout
<< " --extra-info provide extra info in job list\n";
477 cout
<< "\nRole options:\n";
478 cout
<< " --role-name name of the role to create\n";
479 cout
<< " --path path to the role\n";
480 cout
<< " --assume-role-policy-doc the trust relationship policy document that grants an entity permission to assume the role\n";
481 cout
<< " --policy-name name of the policy document\n";
482 cout
<< " --policy-doc permission policy document\n";
483 cout
<< " --path-prefix path prefix for filtering roles\n";
484 cout
<< "\nMFA options:\n";
485 cout
<< " --totp-serial a string that represents the ID of a TOTP token\n";
486 cout
<< " --totp-seed the secret seed that is used to calculate the TOTP\n";
487 cout
<< " --totp-seconds the time resolution that is being used for TOTP generation\n";
488 cout
<< " --totp-window the number of TOTP tokens that are checked before and after the current token when validating token\n";
489 cout
<< " --totp-pin the valid value of a TOTP token at a certain time\n";
490 cout
<< "\nBucket notifications options:\n";
491 cout
<< " --topic bucket notifications topic name\n";
492 cout
<< " --notification-id bucket notifications id\n";
493 cout
<< "\nScript options:\n";
494 cout
<< " --context context in which the script runs. one of: "+LUA_CONTEXT_LIST
+"\n";
495 cout
<< " --package name of the lua package that should be added/removed to/from the allowlist\n";
496 cout
<< " --allow-compilation package is allowed to compile C code as part of its installation\n";
497 cout
<< "\nBucket check olh/unlinked options:\n";
498 cout
<< " --min-age-hours minimum age of unlinked objects to consider for bucket check unlinked (default: 1)\n";
499 cout
<< " --dump-keys when specified, all keys identified as problematic are printed to stdout\n";
500 cout
<< " --hide-progress when specified, per-shard progress details are not printed to stderr\n";
501 cout
<< "\nradoslist options:\n";
502 cout
<< " --rgw-obj-fs the field separator that will separate the rados\n";
503 cout
<< " object name from the rgw object name;\n";
504 cout
<< " additionally rados objects for incomplete\n";
505 cout
<< " multipart uploads will not be output\n";
507 generic_client_usage();
518 using Aliases
= std::vector
<std::set
<string
> >;
519 using Commands
= std::vector
<Def
>;
523 map
<string
, Node
> next
;
524 set
<string
> expected
; /* separate un-normalized list */
529 map
<string
, string
> alias_map
;
531 string
normalize_alias(const string
& s
) const {
532 auto iter
= alias_map
.find(s
);
533 if (iter
== alias_map
.end()) {
539 void init_alias_map(Aliases
& aliases
) {
540 for (auto& alias_set
: aliases
) {
541 std::optional
<string
> first
;
543 for (auto& alias
: alias_set
) {
547 alias_map
[alias
] = *first
;
553 bool gen_next_expected(Node
*node
, vector
<string
> *expected
, bool ret
) {
554 for (auto& next_cmd
: node
->expected
) {
555 expected
->push_back(next_cmd
);
565 SimpleCmd(std::optional
<Commands
> cmds
,
566 std::optional
<Aliases
> aliases
) {
568 add_aliases(*aliases
);
576 void add_aliases(Aliases
& aliases
) {
577 init_alias_map(aliases
);
580 void add_commands(std::vector
<Def
>& cmds
) {
581 for (auto& cmd
: cmds
) {
582 vector
<string
> words
;
583 get_str_vec(cmd
.cmd
, " ", words
);
585 auto node
= &cmd_root
;
586 for (auto& word
: words
) {
587 auto norm
= normalize_alias(word
);
590 node
->expected
.insert(word
);
592 node
= &node
->next
[norm
];
594 if (norm
== "[*]") { /* optional param at the end */
595 parent
->next
["*"] = *node
; /* can be also looked up by '*' */
596 parent
->opt
= cmd
.opt
;
604 template <class Container
>
605 bool find_command(Container
& args
,
607 vector
<string
> *extra_args
,
609 vector
<string
> *expected
) {
610 auto node
= &cmd_root
;
612 std::optional
<std::any
> found_opt
;
614 for (auto& arg
: args
) {
615 string norm
= normalize_alias(arg
);
616 auto iter
= node
->next
.find(norm
);
617 if (iter
== node
->next
.end()) {
618 iter
= node
->next
.find("*");
619 if (iter
== node
->next
.end()) {
620 *error
= string("ERROR: Unrecognized argument: '") + arg
+ "'";
621 return gen_next_expected(node
, expected
, false);
623 extra_args
->push_back(arg
);
625 found_opt
= node
->opt
;
628 node
= &(iter
->second
);
631 *opt_cmd
= found_opt
.value_or(node
->opt
);
633 if (!opt_cmd
->has_value()) {
634 *error
="ERROR: Unknown command";
635 return gen_next_expected(node
, expected
, false);
643 namespace rgw_admin
{
670 BUCKET_CHECK_UNLINKED
,
671 BUCKET_SYNC_CHECKPOINT
,
684 BUCKET_SHARD_OBJECTS
,
686 BUCKET_RESYNC_ENCRYPTED_MULTIPART
,
704 OBJECTS_EXPIRE_STALE_LIST
,
705 OBJECTS_EXPIRE_STALE_RM
,
738 ZONEGROUP_PLACEMENT_ADD
,
739 ZONEGROUP_PLACEMENT_MODIFY
,
740 ZONEGROUP_PLACEMENT_RM
,
741 ZONEGROUP_PLACEMENT_LIST
,
742 ZONEGROUP_PLACEMENT_GET
,
743 ZONEGROUP_PLACEMENT_DEFAULT
,
753 ZONE_PLACEMENT_MODIFY
,
763 METADATA_SYNC_STATUS
,
777 SYNC_GROUP_FLOW_CREATE
,
778 SYNC_GROUP_FLOW_REMOVE
,
779 SYNC_GROUP_PIPE_CREATE
,
780 SYNC_GROUP_PIPE_MODIFY
,
781 SYNC_GROUP_PIPE_REMOVE
,
817 GLOBAL_QUOTA_DISABLE
,
818 GLOBAL_RATELIMIT_GET
,
819 GLOBAL_RATELIMIT_SET
,
820 GLOBAL_RATELIMIT_ENABLE
,
821 GLOBAL_RATELIMIT_DISABLE
,
827 ROLE_TRUST_POLICY_MODIFY
,
845 RESHARD_STALE_INSTANCES_LIST
,
846 RESHARD_STALE_INSTANCES_DELETE
,
850 PUBSUB_NOTIFICATION_LIST
,
851 PUBSUB_NOTIFICATION_GET
,
852 PUBSUB_NOTIFICATION_RM
,
863 using namespace rgw_admin
;
865 static SimpleCmd::Commands all_cmds
= {
866 { "user create", OPT::USER_CREATE
},
867 { "user info", OPT::USER_INFO
},
868 { "user modify", OPT::USER_MODIFY
},
869 { "user rename", OPT::USER_RENAME
},
870 { "user rm", OPT::USER_RM
},
871 { "user suspend", OPT::USER_SUSPEND
},
872 { "user enable", OPT::USER_ENABLE
},
873 { "user check", OPT::USER_CHECK
},
874 { "user stats", OPT::USER_STATS
},
875 { "user list", OPT::USER_LIST
},
876 { "subuser create", OPT::SUBUSER_CREATE
},
877 { "subuser modify", OPT::SUBUSER_MODIFY
},
878 { "subuser rm", OPT::SUBUSER_RM
},
879 { "key create", OPT::KEY_CREATE
},
880 { "key rm", OPT::KEY_RM
},
881 { "buckets list", OPT::BUCKETS_LIST
},
882 { "bucket list", OPT::BUCKETS_LIST
},
883 { "bucket limit check", OPT::BUCKET_LIMIT_CHECK
},
884 { "bucket link", OPT::BUCKET_LINK
},
885 { "bucket unlink", OPT::BUCKET_UNLINK
},
886 { "bucket layout", OPT::BUCKET_LAYOUT
},
887 { "bucket stats", OPT::BUCKET_STATS
},
888 { "bucket check", OPT::BUCKET_CHECK
},
889 { "bucket check olh", OPT::BUCKET_CHECK_OLH
},
890 { "bucket check unlinked", OPT::BUCKET_CHECK_UNLINKED
},
891 { "bucket sync checkpoint", OPT::BUCKET_SYNC_CHECKPOINT
},
892 { "bucket sync info", OPT::BUCKET_SYNC_INFO
},
893 { "bucket sync status", OPT::BUCKET_SYNC_STATUS
},
894 { "bucket sync markers", OPT::BUCKET_SYNC_MARKERS
},
895 { "bucket sync init", OPT::BUCKET_SYNC_INIT
},
896 { "bucket sync run", OPT::BUCKET_SYNC_RUN
},
897 { "bucket sync disable", OPT::BUCKET_SYNC_DISABLE
},
898 { "bucket sync enable", OPT::BUCKET_SYNC_ENABLE
},
899 { "bucket rm", OPT::BUCKET_RM
},
900 { "bucket rewrite", OPT::BUCKET_REWRITE
},
901 { "bucket reshard", OPT::BUCKET_RESHARD
},
902 { "bucket chown", OPT::BUCKET_CHOWN
},
903 { "bucket radoslist", OPT::BUCKET_RADOS_LIST
},
904 { "bucket rados list", OPT::BUCKET_RADOS_LIST
},
905 { "bucket shard objects", OPT::BUCKET_SHARD_OBJECTS
},
906 { "bucket shard object", OPT::BUCKET_SHARD_OBJECTS
},
907 { "bucket object shard", OPT::BUCKET_OBJECT_SHARD
},
908 { "bucket resync encrypted multipart", OPT::BUCKET_RESYNC_ENCRYPTED_MULTIPART
},
909 { "policy", OPT::POLICY
},
910 { "pool add", OPT::POOL_ADD
},
911 { "pool rm", OPT::POOL_RM
},
912 { "pool list", OPT::POOLS_LIST
},
913 { "pools list", OPT::POOLS_LIST
},
914 { "log list", OPT::LOG_LIST
},
915 { "log show", OPT::LOG_SHOW
},
916 { "log rm", OPT::LOG_RM
},
917 { "usage show", OPT::USAGE_SHOW
},
918 { "usage trim", OPT::USAGE_TRIM
},
919 { "usage clear", OPT::USAGE_CLEAR
},
920 { "object put", OPT::OBJECT_PUT
},
921 { "object rm", OPT::OBJECT_RM
},
922 { "object unlink", OPT::OBJECT_UNLINK
},
923 { "object stat", OPT::OBJECT_STAT
},
924 { "object rewrite", OPT::OBJECT_REWRITE
},
925 { "object reindex", OPT::OBJECT_REINDEX
},
926 { "objects expire", OPT::OBJECTS_EXPIRE
},
927 { "objects expire-stale list", OPT::OBJECTS_EXPIRE_STALE_LIST
},
928 { "objects expire-stale rm", OPT::OBJECTS_EXPIRE_STALE_RM
},
929 { "bi get", OPT::BI_GET
},
930 { "bi put", OPT::BI_PUT
},
931 { "bi list", OPT::BI_LIST
},
932 { "bi purge", OPT::BI_PURGE
},
933 { "olh get", OPT::OLH_GET
},
934 { "olh readlog", OPT::OLH_READLOG
},
935 { "quota set", OPT::QUOTA_SET
},
936 { "quota enable", OPT::QUOTA_ENABLE
},
937 { "quota disable", OPT::QUOTA_DISABLE
},
938 { "ratelimit get", OPT::RATELIMIT_GET
},
939 { "ratelimit set", OPT::RATELIMIT_SET
},
940 { "ratelimit enable", OPT::RATELIMIT_ENABLE
},
941 { "ratelimit disable", OPT::RATELIMIT_DISABLE
},
942 { "gc list", OPT::GC_LIST
},
943 { "gc process", OPT::GC_PROCESS
},
944 { "lc list", OPT::LC_LIST
},
945 { "lc get", OPT::LC_GET
},
946 { "lc process", OPT::LC_PROCESS
},
947 { "lc reshard fix", OPT::LC_RESHARD_FIX
},
948 { "orphans find", OPT::ORPHANS_FIND
},
949 { "orphans finish", OPT::ORPHANS_FINISH
},
950 { "orphans list jobs", OPT::ORPHANS_LIST_JOBS
},
951 { "orphans list-jobs", OPT::ORPHANS_LIST_JOBS
},
952 { "zonegroup add", OPT::ZONEGROUP_ADD
},
953 { "zonegroup create", OPT::ZONEGROUP_CREATE
},
954 { "zonegroup default", OPT::ZONEGROUP_DEFAULT
},
955 { "zonegroup delete", OPT::ZONEGROUP_DELETE
},
956 { "zonegroup get", OPT::ZONEGROUP_GET
},
957 { "zonegroup modify", OPT::ZONEGROUP_MODIFY
},
958 { "zonegroup set", OPT::ZONEGROUP_SET
},
959 { "zonegroup list", OPT::ZONEGROUP_LIST
},
960 { "zonegroups list", OPT::ZONEGROUP_LIST
},
961 { "zonegroup remove", OPT::ZONEGROUP_REMOVE
},
962 { "zonegroup remove zone", OPT::ZONEGROUP_REMOVE
},
963 { "zonegroup rename", OPT::ZONEGROUP_RENAME
},
964 { "zonegroup placement add", OPT::ZONEGROUP_PLACEMENT_ADD
},
965 { "zonegroup placement modify", OPT::ZONEGROUP_PLACEMENT_MODIFY
},
966 { "zonegroup placement rm", OPT::ZONEGROUP_PLACEMENT_RM
},
967 { "zonegroup placement list", OPT::ZONEGROUP_PLACEMENT_LIST
},
968 { "zonegroup placement get", OPT::ZONEGROUP_PLACEMENT_GET
},
969 { "zonegroup placement default", OPT::ZONEGROUP_PLACEMENT_DEFAULT
},
970 { "zone create", OPT::ZONE_CREATE
},
971 { "zone delete", OPT::ZONE_DELETE
},
972 { "zone get", OPT::ZONE_GET
},
973 { "zone modify", OPT::ZONE_MODIFY
},
974 { "zone set", OPT::ZONE_SET
},
975 { "zone list", OPT::ZONE_LIST
},
976 { "zones list", OPT::ZONE_LIST
},
977 { "zone rename", OPT::ZONE_RENAME
},
978 { "zone default", OPT::ZONE_DEFAULT
},
979 { "zone placement add", OPT::ZONE_PLACEMENT_ADD
},
980 { "zone placement modify", OPT::ZONE_PLACEMENT_MODIFY
},
981 { "zone placement rm", OPT::ZONE_PLACEMENT_RM
},
982 { "zone placement list", OPT::ZONE_PLACEMENT_LIST
},
983 { "zone placement get", OPT::ZONE_PLACEMENT_GET
},
984 { "caps add", OPT::CAPS_ADD
},
985 { "caps rm", OPT::CAPS_RM
},
986 { "metadata get [*]", OPT::METADATA_GET
},
987 { "metadata put [*]", OPT::METADATA_PUT
},
988 { "metadata rm [*]", OPT::METADATA_RM
},
989 { "metadata list [*]", OPT::METADATA_LIST
},
990 { "metadata sync status", OPT::METADATA_SYNC_STATUS
},
991 { "metadata sync init", OPT::METADATA_SYNC_INIT
},
992 { "metadata sync run", OPT::METADATA_SYNC_RUN
},
993 { "mdlog list", OPT::MDLOG_LIST
},
994 { "mdlog autotrim", OPT::MDLOG_AUTOTRIM
},
995 { "mdlog trim", OPT::MDLOG_TRIM
},
996 { "mdlog fetch", OPT::MDLOG_FETCH
},
997 { "mdlog status", OPT::MDLOG_STATUS
},
998 { "sync error list", OPT::SYNC_ERROR_LIST
},
999 { "sync error trim", OPT::SYNC_ERROR_TRIM
},
1000 { "sync policy get", OPT::SYNC_POLICY_GET
},
1001 { "sync group create", OPT::SYNC_GROUP_CREATE
},
1002 { "sync group modify", OPT::SYNC_GROUP_MODIFY
},
1003 { "sync group get", OPT::SYNC_GROUP_GET
},
1004 { "sync group remove", OPT::SYNC_GROUP_REMOVE
},
1005 { "sync group flow create", OPT::SYNC_GROUP_FLOW_CREATE
},
1006 { "sync group flow remove", OPT::SYNC_GROUP_FLOW_REMOVE
},
1007 { "sync group pipe create", OPT::SYNC_GROUP_PIPE_CREATE
},
1008 { "sync group pipe modify", OPT::SYNC_GROUP_PIPE_MODIFY
},
1009 { "sync group pipe remove", OPT::SYNC_GROUP_PIPE_REMOVE
},
1010 { "bilog list", OPT::BILOG_LIST
},
1011 { "bilog trim", OPT::BILOG_TRIM
},
1012 { "bilog status", OPT::BILOG_STATUS
},
1013 { "bilog autotrim", OPT::BILOG_AUTOTRIM
},
1014 { "data sync status", OPT::DATA_SYNC_STATUS
},
1015 { "data sync init", OPT::DATA_SYNC_INIT
},
1016 { "data sync run", OPT::DATA_SYNC_RUN
},
1017 { "datalog list", OPT::DATALOG_LIST
},
1018 { "datalog status", OPT::DATALOG_STATUS
},
1019 { "datalog autotrim", OPT::DATALOG_AUTOTRIM
},
1020 { "datalog trim", OPT::DATALOG_TRIM
},
1021 { "datalog type", OPT::DATALOG_TYPE
},
1022 { "datalog prune", OPT::DATALOG_PRUNE
},
1023 { "realm create", OPT::REALM_CREATE
},
1024 { "realm rm", OPT::REALM_DELETE
},
1025 { "realm get", OPT::REALM_GET
},
1026 { "realm get default", OPT::REALM_GET_DEFAULT
},
1027 { "realm get-default", OPT::REALM_GET_DEFAULT
},
1028 { "realm list", OPT::REALM_LIST
},
1029 { "realm list periods", OPT::REALM_LIST_PERIODS
},
1030 { "realm list-periods", OPT::REALM_LIST_PERIODS
},
1031 { "realm rename", OPT::REALM_RENAME
},
1032 { "realm set", OPT::REALM_SET
},
1033 { "realm default", OPT::REALM_DEFAULT
},
1034 { "realm pull", OPT::REALM_PULL
},
1035 { "period delete", OPT::PERIOD_DELETE
},
1036 { "period get", OPT::PERIOD_GET
},
1037 { "period get-current", OPT::PERIOD_GET_CURRENT
},
1038 { "period get current", OPT::PERIOD_GET_CURRENT
},
1039 { "period pull", OPT::PERIOD_PULL
},
1040 { "period push", OPT::PERIOD_PUSH
},
1041 { "period list", OPT::PERIOD_LIST
},
1042 { "period update", OPT::PERIOD_UPDATE
},
1043 { "period commit", OPT::PERIOD_COMMIT
},
1044 { "global quota get", OPT::GLOBAL_QUOTA_GET
},
1045 { "global quota set", OPT::GLOBAL_QUOTA_SET
},
1046 { "global quota enable", OPT::GLOBAL_QUOTA_ENABLE
},
1047 { "global quota disable", OPT::GLOBAL_QUOTA_DISABLE
},
1048 { "global ratelimit get", OPT::GLOBAL_RATELIMIT_GET
},
1049 { "global ratelimit set", OPT::GLOBAL_RATELIMIT_SET
},
1050 { "global ratelimit enable", OPT::GLOBAL_RATELIMIT_ENABLE
},
1051 { "global ratelimit disable", OPT::GLOBAL_RATELIMIT_DISABLE
},
1052 { "sync info", OPT::SYNC_INFO
},
1053 { "sync status", OPT::SYNC_STATUS
},
1054 { "role create", OPT::ROLE_CREATE
},
1055 { "role delete", OPT::ROLE_DELETE
},
1056 { "role get", OPT::ROLE_GET
},
1057 { "role-trust-policy modify", OPT::ROLE_TRUST_POLICY_MODIFY
},
1058 { "role list", OPT::ROLE_LIST
},
1059 { "role policy put", OPT::ROLE_POLICY_PUT
},
1060 { "role-policy put", OPT::ROLE_POLICY_PUT
},
1061 { "role policy list", OPT::ROLE_POLICY_LIST
},
1062 { "role-policy list", OPT::ROLE_POLICY_LIST
},
1063 { "role policy get", OPT::ROLE_POLICY_GET
},
1064 { "role-policy get", OPT::ROLE_POLICY_GET
},
1065 { "role policy delete", OPT::ROLE_POLICY_DELETE
},
1066 { "role-policy delete", OPT::ROLE_POLICY_DELETE
},
1067 { "role update", OPT::ROLE_UPDATE
},
1068 { "reshard bucket", OPT::BUCKET_RESHARD
},
1069 { "reshard add", OPT::RESHARD_ADD
},
1070 { "reshard list", OPT::RESHARD_LIST
},
1071 { "reshard status", OPT::RESHARD_STATUS
},
1072 { "reshard process", OPT::RESHARD_PROCESS
},
1073 { "reshard cancel", OPT::RESHARD_CANCEL
},
1074 { "mfa create", OPT::MFA_CREATE
},
1075 { "mfa remove", OPT::MFA_REMOVE
},
1076 { "mfa get", OPT::MFA_GET
},
1077 { "mfa list", OPT::MFA_LIST
},
1078 { "mfa check", OPT::MFA_CHECK
},
1079 { "mfa resync", OPT::MFA_RESYNC
},
1080 { "reshard stale-instances list", OPT::RESHARD_STALE_INSTANCES_LIST
},
1081 { "reshard stale list", OPT::RESHARD_STALE_INSTANCES_LIST
},
1082 { "reshard stale-instances delete", OPT::RESHARD_STALE_INSTANCES_DELETE
},
1083 { "reshard stale delete", OPT::RESHARD_STALE_INSTANCES_DELETE
},
1084 { "topic list", OPT::PUBSUB_TOPIC_LIST
},
1085 { "topic get", OPT::PUBSUB_TOPIC_GET
},
1086 { "topic rm", OPT::PUBSUB_TOPIC_RM
},
1087 { "notification list", OPT::PUBSUB_NOTIFICATION_LIST
},
1088 { "notification get", OPT::PUBSUB_NOTIFICATION_GET
},
1089 { "notification rm", OPT::PUBSUB_NOTIFICATION_RM
},
1090 { "script put", OPT::SCRIPT_PUT
},
1091 { "script get", OPT::SCRIPT_GET
},
1092 { "script rm", OPT::SCRIPT_RM
},
1093 { "script-package add", OPT::SCRIPT_PACKAGE_ADD
},
1094 { "script-package rm", OPT::SCRIPT_PACKAGE_RM
},
1095 { "script-package list", OPT::SCRIPT_PACKAGE_LIST
},
1098 static SimpleCmd::Aliases cmd_aliases
= {
1099 { "delete", "del" },
1106 BIIndexType
get_bi_index_type(const string
& type_str
) {
1107 if (type_str
== "plain")
1108 return BIIndexType::Plain
;
1109 if (type_str
== "instance")
1110 return BIIndexType::Instance
;
1111 if (type_str
== "olh")
1112 return BIIndexType::OLH
;
1114 return BIIndexType::Invalid
;
1117 log_type
get_log_type(const string
& type_str
) {
1118 if (strcasecmp(type_str
.c_str(), "fifo") == 0)
1119 return log_type::fifo
;
1120 if (strcasecmp(type_str
.c_str(), "omap") == 0)
1121 return log_type::omap
;
1123 return static_cast<log_type
>(0xff);
1126 void dump_bi_entry(bufferlist
& bl
, BIIndexType index_type
, Formatter
*formatter
)
1128 auto iter
= bl
.cbegin();
1129 switch (index_type
) {
1130 case BIIndexType::Plain
:
1131 case BIIndexType::Instance
:
1133 rgw_bucket_dir_entry entry
;
1134 decode(entry
, iter
);
1135 encode_json("entry", entry
, formatter
);
1138 case BIIndexType::OLH
:
1140 rgw_bucket_olh_entry entry
;
1141 decode(entry
, iter
);
1142 encode_json("entry", entry
, formatter
);
1151 static void show_user_info(RGWUserInfo
& info
, Formatter
*formatter
)
1153 encode_json("user_info", info
, formatter
);
1154 formatter
->flush(cout
);
1158 static void show_perm_policy(string perm_policy
, Formatter
* formatter
)
1160 formatter
->open_object_section("role");
1161 formatter
->dump_string("Permission policy", perm_policy
);
1162 formatter
->close_section();
1163 formatter
->flush(cout
);
1166 static void show_policy_names(std::vector
<string
> policy_names
, Formatter
* formatter
)
1168 formatter
->open_array_section("PolicyNames");
1169 for (const auto& it
: policy_names
) {
1170 formatter
->dump_string("policyname", it
);
1172 formatter
->close_section();
1173 formatter
->flush(cout
);
1176 static void show_role_info(rgw::sal::RGWRole
* role
, Formatter
* formatter
)
1178 formatter
->open_object_section("role");
1179 role
->dump(formatter
);
1180 formatter
->close_section();
1181 formatter
->flush(cout
);
1184 static void show_roles_info(vector
<std::unique_ptr
<rgw::sal::RGWRole
>>& roles
, Formatter
* formatter
)
1186 formatter
->open_array_section("Roles");
1187 for (const auto& it
: roles
) {
1188 formatter
->open_object_section("role");
1189 it
->dump(formatter
);
1190 formatter
->close_section();
1192 formatter
->close_section();
1193 formatter
->flush(cout
);
1196 static void show_reshard_status(
1197 const list
<cls_rgw_bucket_instance_entry
>& status
, Formatter
*formatter
)
1199 formatter
->open_array_section("status");
1200 for (const auto& entry
: status
) {
1201 formatter
->open_object_section("entry");
1202 formatter
->dump_string("reshard_status", to_string(entry
.reshard_status
));
1203 formatter
->close_section();
1205 formatter
->close_section();
1206 formatter
->flush(cout
);
1209 class StoreDestructor
{
1210 rgw::sal::Driver
* driver
;
1212 explicit StoreDestructor(rgw::sal::Driver
* _s
) : driver(_s
) {}
1213 ~StoreDestructor() {
1214 DriverManager::close_storage(driver
);
1215 rgw_http_client_cleanup();
1219 static int init_bucket(rgw::sal::User
* user
, const rgw_bucket
& b
,
1220 std::unique_ptr
<rgw::sal::Bucket
>* bucket
)
1222 return driver
->get_bucket(dpp(), user
, b
, bucket
, null_yield
);
1225 static int init_bucket(rgw::sal::User
* user
,
1226 const string
& tenant_name
,
1227 const string
& bucket_name
,
1228 const string
& bucket_id
,
1229 std::unique_ptr
<rgw::sal::Bucket
>* bucket
)
1231 rgw_bucket b
{tenant_name
, bucket_name
, bucket_id
};
1232 return init_bucket(user
, b
, bucket
);
1235 static int read_input(const string
& infile
, bufferlist
& bl
)
1238 if (infile
.size()) {
1239 fd
= open(infile
.c_str(), O_RDONLY
);
1242 cerr
<< "error reading input file " << infile
<< std::endl
;
1247 #define READ_CHUNK 8196
1252 char buf
[READ_CHUNK
];
1254 r
= safe_read(fd
, buf
, READ_CHUNK
);
1257 cerr
<< "error while reading input" << std::endl
;
1265 if (infile
.size()) {
1272 static int read_decode_json(const string
& infile
, T
& t
)
1275 int ret
= read_input(infile
, bl
);
1277 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
1281 if (!p
.parse(bl
.c_str(), bl
.length())) {
1282 cout
<< "failed to parse JSON" << std::endl
;
1287 decode_json_obj(t
, &p
);
1288 } catch (const JSONDecoder::err
& e
) {
1289 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
1295 template <class T
, class K
>
1296 static int read_decode_json(const string
& infile
, T
& t
, K
*k
)
1299 int ret
= read_input(infile
, bl
);
1301 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
1305 if (!p
.parse(bl
.c_str(), bl
.length())) {
1306 cout
<< "failed to parse JSON" << std::endl
;
1311 t
.decode_json(&p
, k
);
1312 } catch (const JSONDecoder::err
& e
) {
1313 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
1320 static bool decode_dump(const char *field_name
, bufferlist
& bl
, Formatter
*f
)
1324 auto iter
= bl
.cbegin();
1328 } catch (buffer::error
& err
) {
1332 encode_json(field_name
, t
, f
);
1337 static bool dump_string(const char *field_name
, bufferlist
& bl
, Formatter
*f
)
1339 string val
= bl
.to_str();
1340 f
->dump_string(field_name
, val
.c_str() /* hide encoded null termination chars */);
1345 bool set_ratelimit_info(RGWRateLimitInfo
& ratelimit
, OPT opt_cmd
, int64_t max_read_ops
, int64_t max_write_ops
,
1346 int64_t max_read_bytes
, int64_t max_write_bytes
,
1347 bool have_max_read_ops
, bool have_max_write_ops
,
1348 bool have_max_read_bytes
, bool have_max_write_bytes
)
1350 bool ratelimit_configured
= true;
1352 case OPT::RATELIMIT_ENABLE
:
1353 case OPT::GLOBAL_RATELIMIT_ENABLE
:
1354 ratelimit
.enabled
= true;
1357 case OPT::RATELIMIT_SET
:
1358 case OPT::GLOBAL_RATELIMIT_SET
:
1359 ratelimit_configured
= false;
1360 if (have_max_read_ops
) {
1361 if (max_read_ops
>= 0) {
1362 ratelimit
.max_read_ops
= max_read_ops
;
1363 ratelimit_configured
= true;
1366 if (have_max_write_ops
) {
1367 if (max_write_ops
>= 0) {
1368 ratelimit
.max_write_ops
= max_write_ops
;
1369 ratelimit_configured
= true;
1372 if (have_max_read_bytes
) {
1373 if (max_read_bytes
>= 0) {
1374 ratelimit
.max_read_bytes
= max_read_bytes
;
1375 ratelimit_configured
= true;
1378 if (have_max_write_bytes
) {
1379 if (max_write_bytes
>= 0) {
1380 ratelimit
.max_write_bytes
= max_write_bytes
;
1381 ratelimit_configured
= true;
1385 case OPT::RATELIMIT_DISABLE
:
1386 case OPT::GLOBAL_RATELIMIT_DISABLE
:
1387 ratelimit
.enabled
= false;
1392 return ratelimit_configured
;
1395 void set_quota_info(RGWQuotaInfo
& quota
, OPT opt_cmd
, int64_t max_size
, int64_t max_objects
,
1396 bool have_max_size
, bool have_max_objects
)
1399 case OPT::QUOTA_ENABLE
:
1400 case OPT::GLOBAL_QUOTA_ENABLE
:
1401 quota
.enabled
= true;
1403 // falling through on purpose
1405 case OPT::QUOTA_SET
:
1406 case OPT::GLOBAL_QUOTA_SET
:
1407 if (have_max_objects
) {
1408 if (max_objects
< 0) {
1409 quota
.max_objects
= -1;
1411 quota
.max_objects
= max_objects
;
1414 if (have_max_size
) {
1416 quota
.max_size
= -1;
1418 quota
.max_size
= rgw_rounded_kb(max_size
) * 1024;
1422 case OPT::QUOTA_DISABLE
:
1423 case OPT::GLOBAL_QUOTA_DISABLE
:
1424 quota
.enabled
= false;
1431 int set_bucket_quota(rgw::sal::Driver
* driver
, OPT opt_cmd
,
1432 const string
& tenant_name
, const string
& bucket_name
,
1433 int64_t max_size
, int64_t max_objects
,
1434 bool have_max_size
, bool have_max_objects
)
1436 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1437 int r
= driver
->get_bucket(dpp(), nullptr, tenant_name
, bucket_name
, &bucket
, null_yield
);
1439 cerr
<< "could not get bucket info for bucket=" << bucket_name
<< ": " << cpp_strerror(-r
) << std::endl
;
1443 set_quota_info(bucket
->get_info().quota
, opt_cmd
, max_size
, max_objects
, have_max_size
, have_max_objects
);
1445 r
= bucket
->put_info(dpp(), false, real_time());
1447 cerr
<< "ERROR: failed writing bucket instance info: " << cpp_strerror(-r
) << std::endl
;
1453 int set_bucket_ratelimit(rgw::sal::Driver
* driver
, OPT opt_cmd
,
1454 const string
& tenant_name
, const string
& bucket_name
,
1455 int64_t max_read_ops
, int64_t max_write_ops
,
1456 int64_t max_read_bytes
, int64_t max_write_bytes
,
1457 bool have_max_read_ops
, bool have_max_write_ops
,
1458 bool have_max_read_bytes
, bool have_max_write_bytes
)
1460 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1461 int r
= driver
->get_bucket(dpp(), nullptr, tenant_name
, bucket_name
, &bucket
, null_yield
);
1463 cerr
<< "could not get bucket info for bucket=" << bucket_name
<< ": " << cpp_strerror(-r
) << std::endl
;
1466 RGWRateLimitInfo ratelimit_info
;
1467 auto iter
= bucket
->get_attrs().find(RGW_ATTR_RATELIMIT
);
1468 if(iter
!= bucket
->get_attrs().end()) {
1470 bufferlist
& bl
= iter
->second
;
1471 auto biter
= bl
.cbegin();
1472 decode(ratelimit_info
, biter
);
1473 } catch (buffer::error
& err
) {
1474 ldpp_dout(dpp(), 0) << "ERROR: failed to decode rate limit" << dendl
;
1478 bool ratelimit_configured
= set_ratelimit_info(ratelimit_info
, opt_cmd
, max_read_ops
, max_write_ops
,
1479 max_read_bytes
, max_write_bytes
,
1480 have_max_read_ops
, have_max_write_ops
,
1481 have_max_read_bytes
, have_max_write_bytes
);
1482 if (!ratelimit_configured
) {
1483 ldpp_dout(dpp(), 0) << "ERROR: no rate limit values have been specified" << dendl
;
1487 ratelimit_info
.encode(bl
);
1488 rgw::sal::Attrs attr
;
1489 attr
[RGW_ATTR_RATELIMIT
] = bl
;
1490 r
= bucket
->merge_and_store_attrs(dpp(), attr
, null_yield
);
1492 cerr
<< "ERROR: failed writing bucket instance info: " << cpp_strerror(-r
) << std::endl
;
1498 int set_user_ratelimit(OPT opt_cmd
, std::unique_ptr
<rgw::sal::User
>& user
,
1499 int64_t max_read_ops
, int64_t max_write_ops
,
1500 int64_t max_read_bytes
, int64_t max_write_bytes
,
1501 bool have_max_read_ops
, bool have_max_write_ops
,
1502 bool have_max_read_bytes
, bool have_max_write_bytes
)
1504 RGWRateLimitInfo ratelimit_info
;
1505 user
->load_user(dpp(), null_yield
);
1506 auto iter
= user
->get_attrs().find(RGW_ATTR_RATELIMIT
);
1507 if(iter
!= user
->get_attrs().end()) {
1509 bufferlist
& bl
= iter
->second
;
1510 auto biter
= bl
.cbegin();
1511 decode(ratelimit_info
, biter
);
1512 } catch (buffer::error
& err
) {
1513 ldpp_dout(dpp(), 0) << "ERROR: failed to decode rate limit" << dendl
;
1517 bool ratelimit_configured
= set_ratelimit_info(ratelimit_info
, opt_cmd
, max_read_ops
, max_write_ops
,
1518 max_read_bytes
, max_write_bytes
,
1519 have_max_read_ops
, have_max_write_ops
,
1520 have_max_read_bytes
, have_max_write_bytes
);
1521 if (!ratelimit_configured
) {
1522 ldpp_dout(dpp(), 0) << "ERROR: no rate limit values have been specified" << dendl
;
1526 ratelimit_info
.encode(bl
);
1527 rgw::sal::Attrs attr
;
1528 attr
[RGW_ATTR_RATELIMIT
] = bl
;
1529 int r
= user
->merge_and_store_attrs(dpp(), attr
, null_yield
);
1531 cerr
<< "ERROR: failed writing user instance info: " << cpp_strerror(-r
) << std::endl
;
1537 int show_user_ratelimit(std::unique_ptr
<rgw::sal::User
>& user
, Formatter
*formatter
)
1539 RGWRateLimitInfo ratelimit_info
;
1540 user
->load_user(dpp(), null_yield
);
1541 auto iter
= user
->get_attrs().find(RGW_ATTR_RATELIMIT
);
1542 if(iter
!= user
->get_attrs().end()) {
1544 bufferlist
& bl
= iter
->second
;
1545 auto biter
= bl
.cbegin();
1546 decode(ratelimit_info
, biter
);
1547 } catch (buffer::error
& err
) {
1548 ldpp_dout(dpp(), 0) << "ERROR: failed to decode rate limit" << dendl
;
1552 formatter
->open_object_section("user_ratelimit");
1553 encode_json("user_ratelimit", ratelimit_info
, formatter
);
1554 formatter
->close_section();
1555 formatter
->flush(cout
);
1560 int show_bucket_ratelimit(rgw::sal::Driver
* driver
, const string
& tenant_name
,
1561 const string
& bucket_name
, Formatter
*formatter
)
1563 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1564 int r
= driver
->get_bucket(dpp(), nullptr, tenant_name
, bucket_name
, &bucket
, null_yield
);
1566 cerr
<< "could not get bucket info for bucket=" << bucket_name
<< ": " << cpp_strerror(-r
) << std::endl
;
1569 RGWRateLimitInfo ratelimit_info
;
1570 auto iter
= bucket
->get_attrs().find(RGW_ATTR_RATELIMIT
);
1571 if (iter
!= bucket
->get_attrs().end()) {
1573 bufferlist
& bl
= iter
->second
;
1574 auto biter
= bl
.cbegin();
1575 decode(ratelimit_info
, biter
);
1576 } catch (buffer::error
& err
) {
1577 ldpp_dout(dpp(), 0) << "ERROR: failed to decode rate limit" << dendl
;
1581 formatter
->open_object_section("bucket_ratelimit");
1582 encode_json("bucket_ratelimit", ratelimit_info
, formatter
);
1583 formatter
->close_section();
1584 formatter
->flush(cout
);
1588 int set_user_bucket_quota(OPT opt_cmd
, RGWUser
& user
, RGWUserAdminOpState
& op_state
, int64_t max_size
, int64_t max_objects
,
1589 bool have_max_size
, bool have_max_objects
)
1591 RGWUserInfo
& user_info
= op_state
.get_user_info();
1593 set_quota_info(user_info
.quota
.bucket_quota
, opt_cmd
, max_size
, max_objects
, have_max_size
, have_max_objects
);
1595 op_state
.set_bucket_quota(user_info
.quota
.bucket_quota
);
1598 int r
= user
.modify(dpp(), op_state
, null_yield
, &err
);
1600 cerr
<< "ERROR: failed updating user info: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
1606 int set_user_quota(OPT opt_cmd
, RGWUser
& user
, RGWUserAdminOpState
& op_state
, int64_t max_size
, int64_t max_objects
,
1607 bool have_max_size
, bool have_max_objects
)
1609 RGWUserInfo
& user_info
= op_state
.get_user_info();
1611 set_quota_info(user_info
.quota
.user_quota
, opt_cmd
, max_size
, max_objects
, have_max_size
, have_max_objects
);
1613 op_state
.set_user_quota(user_info
.quota
.user_quota
);
1616 int r
= user
.modify(dpp(), op_state
, null_yield
, &err
);
1618 cerr
<< "ERROR: failed updating user info: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
1624 int check_min_obj_stripe_size(rgw::sal::Driver
* driver
, rgw::sal::Object
* obj
, uint64_t min_stripe_size
, bool *need_rewrite
)
1626 int ret
= obj
->get_obj_attrs(null_yield
, dpp());
1628 ldpp_dout(dpp(), -1) << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret
) << dendl
;
1632 map
<string
, bufferlist
>::iterator iter
;
1633 iter
= obj
->get_attrs().find(RGW_ATTR_MANIFEST
);
1634 if (iter
== obj
->get_attrs().end()) {
1635 *need_rewrite
= (obj
->get_obj_size() >= min_stripe_size
);
1639 RGWObjManifest manifest
;
1642 bufferlist
& bl
= iter
->second
;
1643 auto biter
= bl
.cbegin();
1644 decode(manifest
, biter
);
1645 } catch (buffer::error
& err
) {
1646 ldpp_dout(dpp(), 0) << "ERROR: failed to decode manifest" << dendl
;
1650 map
<uint64_t, RGWObjManifestPart
>& objs
= manifest
.get_explicit_objs();
1651 map
<uint64_t, RGWObjManifestPart
>::iterator oiter
;
1652 for (oiter
= objs
.begin(); oiter
!= objs
.end(); ++oiter
) {
1653 RGWObjManifestPart
& part
= oiter
->second
;
1655 if (part
.size
>= min_stripe_size
) {
1656 *need_rewrite
= true;
1660 *need_rewrite
= false;
1666 int check_obj_locator_underscore(rgw::sal::Object
* obj
, bool fix
, bool remove_bad
, Formatter
*f
) {
1667 f
->open_object_section("object");
1668 f
->open_object_section("key");
1669 f
->dump_string("type", "head");
1670 f
->dump_string("name", obj
->get_name());
1671 f
->dump_string("instance", obj
->get_instance());
1677 get_obj_bucket_and_oid_loc(obj
->get_obj(), oid
, locator
);
1679 f
->dump_string("oid", oid
);
1680 f
->dump_string("locator", locator
);
1682 std::unique_ptr
<rgw::sal::Object::ReadOp
> read_op
= obj
->get_read_op();
1684 int ret
= read_op
->prepare(null_yield
, dpp());
1685 bool needs_fixing
= (ret
== -ENOENT
);
1687 f
->dump_bool("needs_fixing", needs_fixing
);
1689 string status
= (needs_fixing
? "needs_fixing" : "ok");
1691 if ((needs_fixing
|| remove_bad
) && fix
) {
1692 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->fix_head_obj_locator(dpp(), obj
->get_bucket()->get_info(), needs_fixing
, remove_bad
, obj
->get_key());
1694 cerr
<< "ERROR: fix_head_object_locator() returned ret=" << ret
<< std::endl
;
1701 f
->dump_string("status", status
);
1708 int check_obj_tail_locator_underscore(RGWBucketInfo
& bucket_info
, rgw_obj_key
& key
, bool fix
, Formatter
*f
) {
1709 f
->open_object_section("object");
1710 f
->open_object_section("key");
1711 f
->dump_string("type", "tail");
1712 f
->dump_string("name", key
.name
);
1713 f
->dump_string("instance", key
.instance
);
1719 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->fix_tail_obj_locator(dpp(), bucket_info
, key
, fix
, &needs_fixing
, null_yield
);
1721 cerr
<< "ERROR: fix_tail_object_locator_underscore() returned ret=" << ret
<< std::endl
;
1724 status
= (needs_fixing
&& !fix
? "needs_fixing" : "ok");
1727 f
->dump_bool("needs_fixing", needs_fixing
);
1728 f
->dump_string("status", status
);
1735 int do_check_object_locator(const string
& tenant_name
, const string
& bucket_name
,
1736 bool fix
, bool remove_bad
, Formatter
*f
)
1738 if (remove_bad
&& !fix
) {
1739 cerr
<< "ERROR: can't have remove_bad specified without fix" << std::endl
;
1743 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1746 f
->open_object_section("bucket");
1747 f
->dump_string("bucket", bucket_name
);
1748 int ret
= init_bucket(nullptr, tenant_name
, bucket_name
, bucket_id
, &bucket
);
1750 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
1755 int max_entries
= 1000;
1760 vector
<rgw_bucket_dir_entry
> result
;
1763 rgw::sal::Bucket::ListParams params
;
1764 rgw::sal::Bucket::ListResults results
;
1766 params
.prefix
= prefix
;
1767 params
.delim
= delim
;
1768 params
.marker
= rgw_obj_key(marker
);
1770 params
.enforce_ns
= true;
1771 params
.list_versions
= true;
1773 f
->open_array_section("check_objects");
1775 ret
= bucket
->list(dpp(), params
, max_entries
- count
, results
, null_yield
);
1777 cerr
<< "ERROR: driver->list_objects(): " << cpp_strerror(-ret
) << std::endl
;
1781 count
+= results
.objs
.size();
1783 for (vector
<rgw_bucket_dir_entry
>::iterator iter
= results
.objs
.begin(); iter
!= results
.objs
.end(); ++iter
) {
1784 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(iter
->key
);
1786 if (obj
->get_name()[0] == '_') {
1787 ret
= check_obj_locator_underscore(obj
.get(), fix
, remove_bad
, f
);
1790 ret
= check_obj_tail_locator_underscore(bucket
->get_info(), obj
->get_key(), fix
, f
);
1792 cerr
<< "ERROR: check_obj_tail_locator_underscore(): " << cpp_strerror(-ret
) << std::endl
;
1799 } while (results
.is_truncated
&& count
< max_entries
);
1808 /// search for a matching zone/zonegroup id and return a connection if found
1809 static boost::optional
<RGWRESTConn
> get_remote_conn(rgw::sal::RadosStore
* driver
,
1810 const RGWZoneGroup
& zonegroup
,
1811 const std::string
& remote
)
1813 boost::optional
<RGWRESTConn
> conn
;
1814 if (remote
== zonegroup
.get_id()) {
1815 conn
.emplace(driver
->ctx(), driver
, remote
, zonegroup
.endpoints
, zonegroup
.api_name
);
1817 for (const auto& z
: zonegroup
.zones
) {
1818 const auto& zone
= z
.second
;
1819 if (remote
== zone
.id
) {
1820 conn
.emplace(driver
->ctx(), driver
, remote
, zone
.endpoints
, zonegroup
.api_name
);
1828 /// search each zonegroup for a connection
1829 static boost::optional
<RGWRESTConn
> get_remote_conn(rgw::sal::RadosStore
* driver
,
1830 const RGWPeriodMap
& period_map
,
1831 const std::string
& remote
)
1833 boost::optional
<RGWRESTConn
> conn
;
1834 for (const auto& zg
: period_map
.zonegroups
) {
1835 conn
= get_remote_conn(driver
, zg
.second
, remote
);
1843 // we expect a very small response
1844 static constexpr size_t MAX_REST_RESPONSE
= 128 * 1024;
1846 static int send_to_remote_gateway(RGWRESTConn
* conn
, req_info
& info
,
1847 bufferlist
& in_data
, JSONParser
& parser
)
1853 ceph::bufferlist response
;
1855 int ret
= conn
->forward(dpp(), user
, info
, nullptr, MAX_REST_RESPONSE
, &in_data
, &response
, null_yield
);
1857 int parse_ret
= parser
.parse(response
.c_str(), response
.length());
1858 if (parse_ret
< 0) {
1859 cerr
<< "failed to parse response" << std::endl
;
1865 static int send_to_url(const string
& url
,
1866 std::optional
<string
> opt_region
,
1867 const string
& access
,
1868 const string
& secret
, req_info
& info
,
1869 bufferlist
& in_data
, JSONParser
& parser
)
1871 if (access
.empty() || secret
.empty()) {
1872 cerr
<< "An --access-key and --secret must be provided with --url." << std::endl
;
1880 RGWRESTSimpleRequest
req(g_ceph_context
, info
.method
, url
, NULL
, ¶ms
, opt_region
);
1882 bufferlist response
;
1883 int ret
= req
.forward_request(dpp(), key
, info
, MAX_REST_RESPONSE
, &in_data
, &response
, null_yield
);
1885 int parse_ret
= parser
.parse(response
.c_str(), response
.length());
1886 if (parse_ret
< 0) {
1887 cout
<< "failed to parse response" << std::endl
;
1893 static int send_to_remote_or_url(RGWRESTConn
*conn
, const string
& url
,
1894 std::optional
<string
> opt_region
,
1895 const string
& access
, const string
& secret
,
1896 req_info
& info
, bufferlist
& in_data
,
1900 return send_to_remote_gateway(conn
, info
, in_data
, parser
);
1902 return send_to_url(url
, opt_region
, access
, secret
, info
, in_data
, parser
);
1905 static int commit_period(rgw::sal::ConfigStore
* cfgstore
,
1906 RGWRealm
& realm
, rgw::sal::RealmWriter
& realm_writer
,
1907 RGWPeriod
& period
, string remote
, const string
& url
,
1908 std::optional
<string
> opt_region
,
1909 const string
& access
, const string
& secret
,
1912 auto& master_zone
= period
.get_master_zone().id
;
1913 if (master_zone
.empty()) {
1914 cerr
<< "cannot commit period: period does not have a master zone of a master zonegroup" << std::endl
;
1917 // are we the period's master zone?
1918 if (driver
->get_zone()->get_id() == master_zone
) {
1919 // read the current period
1920 RGWPeriod current_period
;
1921 int ret
= cfgstore
->read_period(dpp(), null_yield
, realm
.current_period
,
1922 std::nullopt
, current_period
);
1924 cerr
<< "failed to load current period: " << cpp_strerror(ret
) << std::endl
;
1927 // the master zone can commit locally
1928 ret
= rgw::commit_period(dpp(), null_yield
, cfgstore
, driver
,
1929 realm
, realm_writer
, current_period
,
1930 period
, cerr
, force
);
1932 cerr
<< "failed to commit period: " << cpp_strerror(-ret
) << std::endl
;
1937 if (remote
.empty() && url
.empty()) {
1938 // use the new master zone's connection
1939 remote
= master_zone
;
1940 cerr
<< "Sending period to new master zone " << remote
<< std::endl
;
1942 boost::optional
<RGWRESTConn
> conn
;
1943 RGWRESTConn
*remote_conn
= nullptr;
1944 if (!remote
.empty()) {
1945 conn
= get_remote_conn(static_cast<rgw::sal::RadosStore
*>(driver
), period
.get_map(), remote
);
1947 cerr
<< "failed to find a zone or zonegroup for remote "
1948 << remote
<< std::endl
;
1951 remote_conn
= &*conn
;
1954 // push period to the master with an empty period id
1955 period
.set_id(string());
1958 req_info
info(g_ceph_context
, &env
);
1959 info
.method
= "POST";
1960 info
.request_uri
= "/admin/realm/period";
1962 // json format into a bufferlist
1963 JSONFormatter
jf(false);
1964 encode_json("period", period
, &jf
);
1969 int ret
= send_to_remote_or_url(remote_conn
, url
, opt_region
, access
, secret
, info
, bl
, p
);
1971 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
1973 // did we parse an error message?
1974 auto message
= p
.find_obj("Message");
1976 cerr
<< "Reason: " << message
->get_data() << std::endl
;
1981 // decode the response and driver it back
1983 decode_json_obj(period
, &p
);
1984 } catch (const JSONDecoder::err
& e
) {
1985 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
1988 if (period
.get_id().empty()) {
1989 cerr
<< "Period commit got back an empty period id" << std::endl
;
1992 // the master zone gave us back the period that it committed, so it's
1993 // safe to save it as our latest epoch
1994 constexpr bool exclusive
= false;
1995 ret
= cfgstore
->create_period(dpp(), null_yield
, exclusive
, period
);
1997 cerr
<< "Error storing committed period " << period
.get_id() << ": "
1998 << cpp_strerror(ret
) << std::endl
;
2001 ret
= rgw::reflect_period(dpp(), null_yield
, cfgstore
, period
);
2003 cerr
<< "Error updating local objects: " << cpp_strerror(ret
) << std::endl
;
2006 (void) cfgstore
->realm_notify_new_period(dpp(), null_yield
, period
);
2010 static int update_period(rgw::sal::ConfigStore
* cfgstore
,
2011 const string
& realm_id
, const string
& realm_name
,
2012 const string
& period_epoch
, bool commit
,
2013 const string
& remote
, const string
& url
,
2014 std::optional
<string
> opt_region
,
2015 const string
& access
, const string
& secret
,
2016 Formatter
*formatter
, bool force
)
2019 std::unique_ptr
<rgw::sal::RealmWriter
> realm_writer
;
2020 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
,
2021 realm_id
, realm_name
,
2022 realm
, &realm_writer
);
2024 cerr
<< "failed to load realm " << cpp_strerror(-ret
) << std::endl
;
2027 std::optional
<epoch_t
> epoch
;
2028 if (!period_epoch
.empty()) {
2029 epoch
= atoi(period_epoch
.c_str());
2032 ret
= cfgstore
->read_period(dpp(), null_yield
, realm
.current_period
,
2035 cerr
<< "failed to load current period: " << cpp_strerror(-ret
) << std::endl
;
2038 // convert to the realm's staging period
2039 rgw::fork_period(dpp(), period
);
2040 // update the staging period with all of the realm's zonegroups
2041 ret
= rgw::update_period(dpp(), null_yield
, cfgstore
, period
);
2046 constexpr bool exclusive
= false;
2047 ret
= cfgstore
->create_period(dpp(), null_yield
, exclusive
, period
);
2049 cerr
<< "failed to driver period: " << cpp_strerror(-ret
) << std::endl
;
2053 ret
= commit_period(cfgstore
, realm
, *realm_writer
, period
, remote
, url
,
2054 opt_region
, access
, secret
, force
);
2056 cerr
<< "failed to commit period: " << cpp_strerror(-ret
) << std::endl
;
2060 encode_json("period", period
, formatter
);
2061 formatter
->flush(cout
);
2065 static int init_bucket_for_sync(rgw::sal::User
* user
,
2066 const string
& tenant
, const string
& bucket_name
,
2067 const string
& bucket_id
,
2068 std::unique_ptr
<rgw::sal::Bucket
>* bucket
)
2070 int ret
= init_bucket(user
, tenant
, bucket_name
, bucket_id
, bucket
);
2072 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
2079 static int do_period_pull(rgw::sal::ConfigStore
* cfgstore
,
2080 RGWRESTConn
*remote_conn
, const string
& url
,
2081 std::optional
<string
> opt_region
,
2082 const string
& access_key
, const string
& secret_key
,
2083 const string
& realm_id
, const string
& realm_name
,
2084 const string
& period_id
, const string
& period_epoch
,
2088 req_info
info(g_ceph_context
, &env
);
2089 info
.method
= "GET";
2090 info
.request_uri
= "/admin/realm/period";
2092 map
<string
, string
> ¶ms
= info
.args
.get_params();
2093 if (!realm_id
.empty())
2094 params
["realm_id"] = realm_id
;
2095 if (!realm_name
.empty())
2096 params
["realm_name"] = realm_name
;
2097 if (!period_id
.empty())
2098 params
["period_id"] = period_id
;
2099 if (!period_epoch
.empty())
2100 params
["epoch"] = period_epoch
;
2104 int ret
= send_to_remote_or_url(remote_conn
, url
, opt_region
, access_key
, secret_key
,
2107 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
2111 decode_json_obj(*period
, &p
);
2112 } catch (const JSONDecoder::err
& e
) {
2113 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
2116 constexpr bool exclusive
= false;
2117 ret
= cfgstore
->create_period(dpp(), null_yield
, exclusive
, *period
);
2119 cerr
<< "Error storing period " << period
->get_id() << ": " << cpp_strerror(ret
) << std::endl
;
2124 void flush_ss(stringstream
& ss
, list
<string
>& l
)
2126 if (!ss
.str().empty()) {
2127 l
.push_back(ss
.str());
2132 stringstream
& push_ss(stringstream
& ss
, list
<string
>& l
, int tab
= 0)
2136 ss
<< setw(tab
) << "" << setw(1);
2141 static void get_md_sync_status(list
<string
>& status
)
2143 RGWMetaSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor());
2145 int ret
= sync
.init(dpp());
2147 status
.push_back(string("failed to retrieve sync info: sync.init() failed: ") + cpp_strerror(-ret
));
2151 rgw_meta_sync_status sync_status
;
2152 ret
= sync
.read_sync_status(dpp(), &sync_status
);
2154 status
.push_back(string("failed to read sync status: ") + cpp_strerror(-ret
));
2159 switch (sync_status
.sync_info
.state
) {
2160 case rgw_meta_sync_info::StateInit
:
2161 status_str
= "init";
2163 case rgw_meta_sync_info::StateBuildingFullSyncMaps
:
2164 status_str
= "preparing for full sync";
2166 case rgw_meta_sync_info::StateSync
:
2167 status_str
= "syncing";
2170 status_str
= "unknown";
2173 status
.push_back(status_str
);
2175 uint64_t full_total
= 0;
2176 uint64_t full_complete
= 0;
2180 int total_shards
= 0;
2181 set
<int> shards_behind_set
;
2183 for (auto marker_iter
: sync_status
.sync_markers
) {
2184 full_total
+= marker_iter
.second
.total_entries
;
2186 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::FullSync
) {
2188 full_complete
+= marker_iter
.second
.pos
;
2189 int shard_id
= marker_iter
.first
;
2190 shards_behind_set
.insert(shard_id
);
2192 full_complete
+= marker_iter
.second
.total_entries
;
2194 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::IncrementalSync
) {
2200 push_ss(ss
, status
) << "full sync: " << num_full
<< "/" << total_shards
<< " shards";
2203 push_ss(ss
, status
) << "full sync: " << full_total
- full_complete
<< " entries to sync";
2206 push_ss(ss
, status
) << "incremental sync: " << num_inc
<< "/" << total_shards
<< " shards";
2208 map
<int, RGWMetadataLogInfo
> master_shards_info
;
2209 string master_period
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_current_period_id();
2211 ret
= sync
.read_master_log_shards_info(dpp(), master_period
, &master_shards_info
);
2213 status
.push_back(string("failed to fetch master sync status: ") + cpp_strerror(-ret
));
2217 map
<int, string
> shards_behind
;
2218 if (sync_status
.sync_info
.period
!= master_period
) {
2219 status
.push_back(string("master is on a different period: master_period=" +
2220 master_period
+ " local_period=" + sync_status
.sync_info
.period
));
2222 for (auto local_iter
: sync_status
.sync_markers
) {
2223 int shard_id
= local_iter
.first
;
2224 auto iter
= master_shards_info
.find(shard_id
);
2226 if (iter
== master_shards_info
.end()) {
2228 derr
<< "ERROR: could not find remote sync shard status for shard_id=" << shard_id
<< dendl
;
2231 auto master_marker
= iter
->second
.marker
;
2232 if (local_iter
.second
.state
== rgw_meta_sync_marker::SyncState::IncrementalSync
&&
2233 master_marker
> local_iter
.second
.marker
) {
2234 shards_behind
[shard_id
] = local_iter
.second
.marker
;
2235 shards_behind_set
.insert(shard_id
);
2240 // fetch remote log entries to determine the oldest change
2241 std::optional
<std::pair
<int, ceph::real_time
>> oldest
;
2242 if (!shards_behind
.empty()) {
2243 map
<int, rgw_mdlog_shard_data
> master_pos
;
2244 ret
= sync
.read_master_log_shards_next(dpp(), sync_status
.sync_info
.period
, shards_behind
, &master_pos
);
2246 derr
<< "ERROR: failed to fetch master next positions (" << cpp_strerror(-ret
) << ")" << dendl
;
2248 for (auto iter
: master_pos
) {
2249 rgw_mdlog_shard_data
& shard_data
= iter
.second
;
2251 if (shard_data
.entries
.empty()) {
2252 // there aren't any entries in this shard, so we're not really behind
2253 shards_behind
.erase(iter
.first
);
2254 shards_behind_set
.erase(iter
.first
);
2256 rgw_mdlog_entry
& entry
= shard_data
.entries
.front();
2258 oldest
.emplace(iter
.first
, entry
.timestamp
);
2259 } else if (!ceph::real_clock::is_zero(entry
.timestamp
) && entry
.timestamp
< oldest
->second
) {
2260 oldest
.emplace(iter
.first
, entry
.timestamp
);
2267 int total_behind
= shards_behind
.size() + (sync_status
.sync_info
.num_shards
- num_inc
);
2268 if (total_behind
== 0) {
2269 push_ss(ss
, status
) << "metadata is caught up with master";
2271 push_ss(ss
, status
) << "metadata is behind on " << total_behind
<< " shards";
2272 push_ss(ss
, status
) << "behind shards: " << "[" << shards_behind_set
<< "]";
2274 push_ss(ss
, status
) << "oldest incremental change not applied: "
2275 << oldest
->second
<< " [" << oldest
->first
<< ']';
2279 flush_ss(ss
, status
);
2282 static void get_data_sync_status(const rgw_zone_id
& source_zone
, list
<string
>& status
, int tab
)
2288 if (!(sz
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->find_zone(source_zone
))) {
2289 push_ss(ss
, status
, tab
) << string("zone not found");
2290 flush_ss(ss
, status
);
2294 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->zone_syncs_from(*sz
)) {
2295 push_ss(ss
, status
, tab
) << string("not syncing from zone");
2296 flush_ss(ss
, status
);
2299 RGWDataSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor(), source_zone
, nullptr);
2301 int ret
= sync
.init(dpp());
2303 push_ss(ss
, status
, tab
) << string("failed to retrieve sync info: ") + cpp_strerror(-ret
);
2304 flush_ss(ss
, status
);
2308 rgw_data_sync_status sync_status
;
2309 ret
= sync
.read_sync_status(dpp(), &sync_status
);
2310 if (ret
< 0 && ret
!= -ENOENT
) {
2311 push_ss(ss
, status
, tab
) << string("failed read sync status: ") + cpp_strerror(-ret
);
2315 set
<int> recovering_shards
;
2316 ret
= sync
.read_recovering_shards(dpp(), sync_status
.sync_info
.num_shards
, recovering_shards
);
2317 if (ret
< 0 && ret
!= ENOENT
) {
2318 push_ss(ss
, status
, tab
) << string("failed read recovering shards: ") + cpp_strerror(-ret
);
2323 switch (sync_status
.sync_info
.state
) {
2324 case rgw_data_sync_info::StateInit
:
2325 status_str
= "init";
2327 case rgw_data_sync_info::StateBuildingFullSyncMaps
:
2328 status_str
= "preparing for full sync";
2330 case rgw_data_sync_info::StateSync
:
2331 status_str
= "syncing";
2334 status_str
= "unknown";
2337 push_ss(ss
, status
, tab
) << status_str
;
2339 uint64_t full_total
= 0;
2340 uint64_t full_complete
= 0;
2344 int total_shards
= 0;
2345 set
<int> shards_behind_set
;
2347 for (auto marker_iter
: sync_status
.sync_markers
) {
2348 full_total
+= marker_iter
.second
.total_entries
;
2350 if (marker_iter
.second
.state
== rgw_data_sync_marker::SyncState::FullSync
) {
2352 full_complete
+= marker_iter
.second
.pos
;
2353 int shard_id
= marker_iter
.first
;
2354 shards_behind_set
.insert(shard_id
);
2356 full_complete
+= marker_iter
.second
.total_entries
;
2358 if (marker_iter
.second
.state
== rgw_data_sync_marker::SyncState::IncrementalSync
) {
2363 push_ss(ss
, status
, tab
) << "full sync: " << num_full
<< "/" << total_shards
<< " shards";
2366 push_ss(ss
, status
, tab
) << "full sync: " << full_total
- full_complete
<< " buckets to sync";
2369 push_ss(ss
, status
, tab
) << "incremental sync: " << num_inc
<< "/" << total_shards
<< " shards";
2371 map
<int, RGWDataChangesLogInfo
> source_shards_info
;
2373 ret
= sync
.read_source_log_shards_info(dpp(), &source_shards_info
);
2375 push_ss(ss
, status
, tab
) << string("failed to fetch source sync status: ") + cpp_strerror(-ret
);
2379 map
<int, string
> shards_behind
;
2381 for (auto local_iter
: sync_status
.sync_markers
) {
2382 int shard_id
= local_iter
.first
;
2383 auto iter
= source_shards_info
.find(shard_id
);
2385 if (iter
== source_shards_info
.end()) {
2387 derr
<< "ERROR: could not find remote sync shard status for shard_id=" << shard_id
<< dendl
;
2390 auto master_marker
= iter
->second
.marker
;
2391 if (local_iter
.second
.state
== rgw_data_sync_marker::SyncState::IncrementalSync
&&
2392 master_marker
> local_iter
.second
.marker
) {
2393 shards_behind
[shard_id
] = local_iter
.second
.marker
;
2394 shards_behind_set
.insert(shard_id
);
2398 std::optional
<std::pair
<int, ceph::real_time
>> oldest
;
2399 if (!shards_behind
.empty()) {
2400 map
<int, rgw_datalog_shard_data
> master_pos
;
2401 ret
= sync
.read_source_log_shards_next(dpp(), shards_behind
, &master_pos
);
2404 derr
<< "ERROR: failed to fetch next positions (" << cpp_strerror(-ret
) << ")" << dendl
;
2406 for (auto iter
: master_pos
) {
2407 rgw_datalog_shard_data
& shard_data
= iter
.second
;
2408 if (shard_data
.entries
.empty()) {
2409 // there aren't any entries in this shard, so we're not really behind
2410 shards_behind
.erase(iter
.first
);
2411 shards_behind_set
.erase(iter
.first
);
2413 rgw_datalog_entry
& entry
= shard_data
.entries
.front();
2415 oldest
.emplace(iter
.first
, entry
.timestamp
);
2416 } else if (!ceph::real_clock::is_zero(entry
.timestamp
) && entry
.timestamp
< oldest
->second
) {
2417 oldest
.emplace(iter
.first
, entry
.timestamp
);
2424 int total_behind
= shards_behind
.size() + (sync_status
.sync_info
.num_shards
- num_inc
);
2425 int total_recovering
= recovering_shards
.size();
2427 if (total_behind
== 0 && total_recovering
== 0) {
2428 push_ss(ss
, status
, tab
) << "data is caught up with source";
2429 } else if (total_behind
> 0) {
2430 push_ss(ss
, status
, tab
) << "data is behind on " << total_behind
<< " shards";
2431 push_ss(ss
, status
, tab
) << "behind shards: " << "[" << shards_behind_set
<< "]" ;
2433 push_ss(ss
, status
, tab
) << "oldest incremental change not applied: "
2434 << oldest
->second
<< " [" << oldest
->first
<< ']';
2438 if (total_recovering
> 0) {
2439 push_ss(ss
, status
, tab
) << total_recovering
<< " shards are recovering";
2440 push_ss(ss
, status
, tab
) << "recovering shards: " << "[" << recovering_shards
<< "]";
2443 flush_ss(ss
, status
);
2446 static void tab_dump(const string
& header
, int width
, const list
<string
>& entries
)
2450 for (auto e
: entries
) {
2451 cout
<< std::setw(width
) << s
<< std::setw(1) << " " << e
<< std::endl
;
2456 // return features that are supported but not enabled
2457 static auto get_disabled_features(const rgw::zone_features::set
& enabled
) {
2458 auto features
= rgw::zone_features::set
{rgw::zone_features::supported
.begin(),
2459 rgw::zone_features::supported
.end()};
2460 for (const auto& feature
: enabled
) {
2461 features
.erase(feature
);
2467 static void sync_status(Formatter
*formatter
)
2469 const rgw::sal::ZoneGroup
& zonegroup
= driver
->get_zone()->get_zonegroup();
2470 rgw::sal::Zone
* zone
= driver
->get_zone();
2474 cout
<< std::setw(width
) << "realm" << std::setw(1) << " " << zone
->get_realm_id() << " (" << zone
->get_realm_name() << ")" << std::endl
;
2475 cout
<< std::setw(width
) << "zonegroup" << std::setw(1) << " " << zonegroup
.get_id() << " (" << zonegroup
.get_name() << ")" << std::endl
;
2476 cout
<< std::setw(width
) << "zone" << std::setw(1) << " " << zone
->get_id() << " (" << zone
->get_name() << ")" << std::endl
;
2477 cout
<< std::setw(width
) << "current time" << std::setw(1) << " "
2478 << to_iso_8601(ceph::real_clock::now(), iso_8601_format::YMDhms
) << std::endl
;
2481 static_cast<const rgw::sal::RadosZoneGroup
&>(zonegroup
).get_group();
2483 cout
<< std::setw(width
) << "zonegroup features enabled: " << rzg
.enabled_features
<< std::endl
;
2484 if (auto d
= get_disabled_features(rzg
.enabled_features
); !d
.empty()) {
2485 cout
<< std::setw(width
) << " disabled: " << d
<< std::endl
;
2488 list
<string
> md_status
;
2490 if (driver
->is_meta_master()) {
2491 md_status
.push_back("no sync (zone is master)");
2493 get_md_sync_status(md_status
);
2496 tab_dump("metadata sync", width
, md_status
);
2498 list
<string
> data_status
;
2500 auto& zone_conn_map
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zone_conn_map();
2502 for (auto iter
: zone_conn_map
) {
2503 const rgw_zone_id
& source_id
= iter
.first
;
2504 string source_str
= "source: ";
2505 string s
= source_str
+ source_id
.id
;
2506 std::unique_ptr
<rgw::sal::Zone
> sz
;
2507 if (driver
->get_zone()->get_zonegroup().get_zone_by_id(source_id
.id
, &sz
) == 0) {
2508 s
+= string(" (") + sz
->get_name() + ")";
2510 data_status
.push_back(s
);
2511 get_data_sync_status(source_id
, data_status
, source_str
.size());
2514 tab_dump("data sync", width
, data_status
);
2518 int w
; // indent width
2519 std::string_view header
;
2520 indented(int w
, std::string_view header
= "") : w(w
), header(header
) {}
2522 std::ostream
& operator<<(std::ostream
& out
, const indented
& h
) {
2523 return out
<< std::setw(h
.w
) << h
.header
<< std::setw(1) << ' ';
2526 static int bucket_source_sync_status(const DoutPrefixProvider
*dpp
, rgw::sal::RadosStore
* driver
, const RGWZone
& zone
,
2527 const RGWZone
& source
, RGWRESTConn
*conn
,
2528 const RGWBucketInfo
& bucket_info
,
2529 rgw_sync_bucket_pipe pipe
,
2530 int width
, std::ostream
& out
)
2532 out
<< indented
{width
, "source zone"} << source
.id
<< " (" << source
.name
<< ")" << std::endl
;
2534 // syncing from this zone?
2535 if (!driver
->svc()->zone
->zone_syncs_from(zone
, source
)) {
2536 out
<< indented
{width
} << "does not sync from zone\n";
2540 if (!pipe
.source
.bucket
) {
2541 ldpp_dout(dpp
, -1) << __func__
<< "(): missing source bucket" << dendl
;
2545 std::unique_ptr
<rgw::sal::Bucket
> source_bucket
;
2546 int r
= init_bucket(nullptr, *pipe
.source
.bucket
, &source_bucket
);
2548 ldpp_dout(dpp
, -1) << "failed to read source bucket info: " << cpp_strerror(r
) << dendl
;
2552 out
<< indented
{width
, "source bucket"} << source_bucket
->get_key() << std::endl
;
2553 pipe
.source
.bucket
= source_bucket
->get_key();
2555 pipe
.dest
.bucket
= bucket_info
.bucket
;
2558 std::vector
<rgw_bucket_shard_sync_info
> shard_status
;
2560 // check for full sync status
2561 rgw_bucket_sync_status full_status
;
2562 r
= rgw_read_bucket_full_sync_status(dpp
, driver
, pipe
, &full_status
, null_yield
);
2564 if (full_status
.state
== BucketSyncState::Init
) {
2565 out
<< indented
{width
} << "init: bucket sync has not started\n";
2568 if (full_status
.state
== BucketSyncState::Stopped
) {
2569 out
<< indented
{width
} << "stopped: bucket sync is disabled\n";
2572 if (full_status
.state
== BucketSyncState::Full
) {
2573 out
<< indented
{width
} << "full sync: " << full_status
.full
.count
<< " objects completed\n";
2576 gen
= full_status
.incremental_gen
;
2577 shard_status
.resize(full_status
.shards_done_with_gen
.size());
2578 } else if (r
== -ENOENT
) {
2579 // no full status, but there may be per-shard status from before upgrade
2580 const auto& logs
= source_bucket
->get_info().layout
.logs
;
2582 out
<< indented
{width
} << "init: bucket sync has not started\n";
2585 const auto& log
= logs
.front();
2587 // this isn't the backward-compatible case, so we just haven't started yet
2588 out
<< indented
{width
} << "init: bucket sync has not started\n";
2591 if (log
.layout
.type
!= rgw::BucketLogType::InIndex
) {
2592 ldpp_dout(dpp
, -1) << "unrecognized log layout type " << log
.layout
.type
<< dendl
;
2595 // use shard count from our log gen=0
2596 shard_status
.resize(rgw::num_shards(log
.layout
.in_index
));
2598 lderr(driver
->ctx()) << "failed to read bucket full sync status: " << cpp_strerror(r
) << dendl
;
2602 r
= rgw_read_bucket_inc_sync_status(dpp
, driver
, pipe
, gen
, &shard_status
);
2604 lderr(driver
->ctx()) << "failed to read bucket incremental sync status: " << cpp_strerror(r
) << dendl
;
2608 const int total_shards
= shard_status
.size();
2610 out
<< indented
{width
} << "incremental sync on " << total_shards
<< " shards\n";
2612 rgw_bucket_index_marker_info remote_info
;
2613 BucketIndexShardsManager remote_markers
;
2614 r
= rgw_read_remote_bilog_info(dpp
, conn
, source_bucket
->get_key(),
2615 remote_info
, remote_markers
, null_yield
);
2617 ldpp_dout(dpp
, -1) << "failed to read remote log: " << cpp_strerror(r
) << dendl
;
2621 std::set
<int> shards_behind
;
2622 for (const auto& r
: remote_markers
.get()) {
2623 auto shard_id
= r
.first
;
2624 if (r
.second
.empty()) {
2625 continue; // empty bucket index shard
2627 if (shard_id
>= total_shards
) {
2628 // unexpected shard id. we don't have status for it, so we're behind
2629 shards_behind
.insert(shard_id
);
2632 auto& m
= shard_status
[shard_id
];
2633 const auto pos
= BucketIndexShardsManager::get_shard_marker(m
.inc_marker
.position
);
2634 if (pos
< r
.second
) {
2635 shards_behind
.insert(shard_id
);
2638 if (!shards_behind
.empty()) {
2639 out
<< indented
{width
} << "bucket is behind on " << shards_behind
.size() << " shards\n";
2640 out
<< indented
{width
} << "behind shards: [" << shards_behind
<< "]\n" ;
2642 out
<< indented
{width
} << "bucket is caught up with source\n";
2647 void encode_json(const char *name
, const RGWBucketSyncFlowManager::pipe_set
& pset
, Formatter
*f
)
2649 Formatter::ObjectSection
top_section(*f
, name
);
2650 Formatter::ArraySection
as(*f
, "entries");
2652 for (auto& pipe_handler
: pset
) {
2653 Formatter::ObjectSection
hs(*f
, "handler");
2654 encode_json("source", pipe_handler
.source
, f
);
2655 encode_json("dest", pipe_handler
.dest
, f
);
2659 static std::vector
<string
> convert_bucket_set_to_str_vec(const std::set
<rgw_bucket
>& bs
)
2661 std::vector
<string
> result
;
2662 result
.reserve(bs
.size());
2663 for (auto& b
: bs
) {
2664 result
.push_back(b
.get_key());
2669 static void get_hint_entities(const std::set
<rgw_zone_id
>& zones
, const std::set
<rgw_bucket
>& buckets
,
2670 std::set
<rgw_sync_bucket_entity
> *hint_entities
)
2672 for (auto& zone_id
: zones
) {
2673 for (auto& b
: buckets
) {
2674 std::unique_ptr
<rgw::sal::Bucket
> hint_bucket
;
2675 int ret
= init_bucket(nullptr, b
, &hint_bucket
);
2677 ldpp_dout(dpp(), 20) << "could not init bucket info for hint bucket=" << b
<< " ... skipping" << dendl
;
2681 hint_entities
->insert(rgw_sync_bucket_entity(zone_id
, hint_bucket
->get_key()));
2686 static rgw_zone_id
resolve_zone_id(const string
& s
)
2688 std::unique_ptr
<rgw::sal::Zone
> zone
;
2689 int ret
= driver
->get_zone()->get_zonegroup().get_zone_by_id(s
, &zone
);
2691 ret
= driver
->get_zone()->get_zonegroup().get_zone_by_name(s
, &zone
);
2693 return rgw_zone_id(s
);
2695 return rgw_zone_id(zone
->get_id());
2698 rgw_zone_id
validate_zone_id(const rgw_zone_id
& zone_id
)
2700 return resolve_zone_id(zone_id
.id
);
2703 static int sync_info(std::optional
<rgw_zone_id
> opt_target_zone
, std::optional
<rgw_bucket
> opt_bucket
, Formatter
*formatter
)
2705 rgw_zone_id zone_id
= opt_target_zone
.value_or(driver
->get_zone()->get_id());
2707 auto zone_policy_handler
= driver
->get_zone()->get_sync_policy_handler();
2709 RGWBucketSyncPolicyHandlerRef bucket_handler
;
2711 std::optional
<rgw_bucket
> eff_bucket
= opt_bucket
;
2713 auto handler
= zone_policy_handler
;
2716 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
2718 int ret
= init_bucket(nullptr, *eff_bucket
, &bucket
);
2719 if (ret
< 0 && ret
!= -ENOENT
) {
2720 cerr
<< "ERROR: init_bucket failed: " << cpp_strerror(-ret
) << std::endl
;
2725 rgw::sal::Attrs attrs
= bucket
->get_attrs();
2726 bucket_handler
.reset(handler
->alloc_child(bucket
->get_info(), std::move(attrs
)));
2728 cerr
<< "WARNING: bucket not found, simulating result" << std::endl
;
2729 bucket_handler
.reset(handler
->alloc_child(*eff_bucket
, nullopt
));
2732 ret
= bucket_handler
->init(dpp(), null_yield
);
2734 cerr
<< "ERROR: failed to init bucket sync policy handler: " << cpp_strerror(-ret
) << " (ret=" << ret
<< ")" << std::endl
;
2738 handler
= bucket_handler
;
2741 std::set
<rgw_sync_bucket_pipe
> sources
;
2742 std::set
<rgw_sync_bucket_pipe
> dests
;
2744 handler
->get_pipes(&sources
, &dests
, std::nullopt
);
2746 auto source_hints_vec
= convert_bucket_set_to_str_vec(handler
->get_source_hints());
2747 auto target_hints_vec
= convert_bucket_set_to_str_vec(handler
->get_target_hints());
2749 std::set
<rgw_sync_bucket_pipe
> resolved_sources
;
2750 std::set
<rgw_sync_bucket_pipe
> resolved_dests
;
2752 rgw_sync_bucket_entity
self_entity(zone_id
, opt_bucket
);
2754 set
<rgw_zone_id
> source_zones
;
2755 set
<rgw_zone_id
> target_zones
;
2757 zone_policy_handler
->reflect(dpp(), nullptr, nullptr,
2761 false); /* relaxed: also get all zones that we allow to sync to/from */
2763 std::set
<rgw_sync_bucket_entity
> hint_entities
;
2765 get_hint_entities(source_zones
, handler
->get_source_hints(), &hint_entities
);
2766 get_hint_entities(target_zones
, handler
->get_target_hints(), &hint_entities
);
2768 for (auto& hint_entity
: hint_entities
) {
2769 if (!hint_entity
.zone
||
2770 !hint_entity
.bucket
) {
2771 continue; /* shouldn't really happen */
2774 auto zid
= validate_zone_id(*hint_entity
.zone
);
2775 auto& hint_bucket
= *hint_entity
.bucket
;
2777 RGWBucketSyncPolicyHandlerRef hint_bucket_handler
;
2778 int r
= driver
->get_sync_policy_handler(dpp(), zid
, hint_bucket
, &hint_bucket_handler
, null_yield
);
2780 ldpp_dout(dpp(), 20) << "could not get bucket sync policy handler for hint bucket=" << hint_bucket
<< " ... skipping" << dendl
;
2784 hint_bucket_handler
->get_pipes(&resolved_dests
,
2786 self_entity
); /* flipping resolved dests and sources as these are
2787 relative to the remote entity */
2791 Formatter::ObjectSection
os(*formatter
, "result");
2792 encode_json("sources", sources
, formatter
);
2793 encode_json("dests", dests
, formatter
);
2795 Formatter::ObjectSection
hints_section(*formatter
, "hints");
2796 encode_json("sources", source_hints_vec
, formatter
);
2797 encode_json("dests", target_hints_vec
, formatter
);
2800 Formatter::ObjectSection
resolved_hints_section(*formatter
, "resolved-hints-1");
2801 encode_json("sources", resolved_sources
, formatter
);
2802 encode_json("dests", resolved_dests
, formatter
);
2805 Formatter::ObjectSection
resolved_hints_section(*formatter
, "resolved-hints");
2806 encode_json("sources", handler
->get_resolved_source_hints(), formatter
);
2807 encode_json("dests", handler
->get_resolved_dest_hints(), formatter
);
2811 formatter
->flush(cout
);
2816 static int bucket_sync_info(rgw::sal::Driver
* driver
, const RGWBucketInfo
& info
,
2819 const rgw::sal::ZoneGroup
& zonegroup
= driver
->get_zone()->get_zonegroup();
2820 rgw::sal::Zone
* zone
= driver
->get_zone();
2821 constexpr int width
= 15;
2823 out
<< indented
{width
, "realm"} << zone
->get_realm_id() << " (" << zone
->get_realm_name() << ")\n";
2824 out
<< indented
{width
, "zonegroup"} << zonegroup
.get_id() << " (" << zonegroup
.get_name() << ")\n";
2825 out
<< indented
{width
, "zone"} << zone
->get_id() << " (" << zone
->get_name() << ")\n";
2826 out
<< indented
{width
, "bucket"} << info
.bucket
<< "\n\n";
2828 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->bucket
->bucket_imports_data(info
.bucket
, null_yield
, dpp())) {
2829 out
<< "Sync is disabled for bucket " << info
.bucket
.name
<< '\n';
2833 RGWBucketSyncPolicyHandlerRef handler
;
2835 int r
= driver
->get_sync_policy_handler(dpp(), std::nullopt
, info
.bucket
, &handler
, null_yield
);
2837 ldpp_dout(dpp(), -1) << "ERROR: failed to get policy handler for bucket (" << info
.bucket
<< "): r=" << r
<< ": " << cpp_strerror(-r
) << dendl
;
2841 auto& sources
= handler
->get_sources();
2843 for (auto& m
: sources
) {
2844 auto& zone
= m
.first
;
2845 out
<< indented
{width
, "source zone"} << zone
<< std::endl
;
2846 for (auto& pipe_handler
: m
.second
) {
2847 out
<< indented
{width
, "bucket"} << *pipe_handler
.source
.bucket
<< std::endl
;
2854 static int bucket_sync_status(rgw::sal::Driver
* driver
, const RGWBucketInfo
& info
,
2855 const rgw_zone_id
& source_zone_id
,
2856 std::optional
<rgw_bucket
>& opt_source_bucket
,
2859 const rgw::sal::ZoneGroup
& zonegroup
= driver
->get_zone()->get_zonegroup();
2860 rgw::sal::Zone
* zone
= driver
->get_zone();
2861 constexpr int width
= 15;
2863 out
<< indented
{width
, "realm"} << zone
->get_realm_id() << " (" << zone
->get_realm_name() << ")\n";
2864 out
<< indented
{width
, "zonegroup"} << zonegroup
.get_id() << " (" << zonegroup
.get_name() << ")\n";
2865 out
<< indented
{width
, "zone"} << zone
->get_id() << " (" << zone
->get_name() << ")\n";
2866 out
<< indented
{width
, "bucket"} << info
.bucket
<< "\n";
2867 out
<< indented
{width
, "current time"}
2868 << to_iso_8601(ceph::real_clock::now(), iso_8601_format::YMDhms
) << "\n\n";
2871 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->bucket
->bucket_imports_data(info
.bucket
, null_yield
, dpp())) {
2872 out
<< "Sync is disabled for bucket " << info
.bucket
.name
<< " or bucket has no sync sources" << std::endl
;
2876 RGWBucketSyncPolicyHandlerRef handler
;
2878 int r
= driver
->get_sync_policy_handler(dpp(), std::nullopt
, info
.bucket
, &handler
, null_yield
);
2880 ldpp_dout(dpp(), -1) << "ERROR: failed to get policy handler for bucket (" << info
.bucket
<< "): r=" << r
<< ": " << cpp_strerror(-r
) << dendl
;
2884 auto sources
= handler
->get_all_sources();
2886 auto& zone_conn_map
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zone_conn_map();
2887 set
<rgw_zone_id
> zone_ids
;
2889 if (!source_zone_id
.empty()) {
2890 std::unique_ptr
<rgw::sal::Zone
> zone
;
2891 int ret
= driver
->get_zone()->get_zonegroup().get_zone_by_id(source_zone_id
.id
, &zone
);
2893 ldpp_dout(dpp(), -1) << "Source zone not found in zonegroup "
2894 << zonegroup
.get_name() << dendl
;
2897 auto c
= zone_conn_map
.find(source_zone_id
);
2898 if (c
== zone_conn_map
.end()) {
2899 ldpp_dout(dpp(), -1) << "No connection to zone " << zone
->get_name() << dendl
;
2902 zone_ids
.insert(source_zone_id
);
2904 std::list
<std::string
> ids
;
2905 int ret
= driver
->get_zone()->get_zonegroup().list_zones(ids
);
2907 for (const auto& entry
: ids
) {
2908 zone_ids
.insert(entry
);
2913 for (auto& zone_id
: zone_ids
) {
2914 auto z
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zonegroup().zones
.find(zone_id
.id
);
2915 if (z
== static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zonegroup().zones
.end()) { /* should't happen */
2918 auto c
= zone_conn_map
.find(zone_id
.id
);
2919 if (c
== zone_conn_map
.end()) { /* should't happen */
2923 for (auto& entry
: sources
) {
2924 auto& pipe
= entry
.second
;
2925 if (opt_source_bucket
&&
2926 pipe
.source
.bucket
!= opt_source_bucket
) {
2929 if (pipe
.source
.zone
.value_or(rgw_zone_id()) == z
->second
.id
) {
2930 bucket_source_sync_status(dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zone(), z
->second
,
2941 static void parse_tier_config_param(const string
& s
, map
<string
, string
, ltstr_nocase
>& out
)
2949 confs
.push_back(cur_conf
);
2956 } else if (c
== '}') {
2961 if (!cur_conf
.empty()) {
2962 confs
.push_back(cur_conf
);
2965 for (auto c
: confs
) {
2966 ssize_t pos
= c
.find("=");
2970 out
[c
.substr(0, pos
)] = c
.substr(pos
+ 1);
2975 static int check_pool_support_omap(const rgw_pool
& pool
)
2977 librados::IoCtx io_ctx
;
2978 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->get_rados_handle()->ioctx_create(pool
.to_str().c_str(), io_ctx
);
2980 // the pool may not exist at this moment, we have no way to check if it supports omap.
2984 ret
= io_ctx
.omap_clear("__omap_test_not_exist_oid__");
2985 if (ret
== -EOPNOTSUPP
) {
2993 int check_reshard_bucket_params(rgw::sal::Driver
* driver
,
2994 const string
& bucket_name
,
2995 const string
& tenant
,
2996 const string
& bucket_id
,
2997 bool num_shards_specified
,
2999 int yes_i_really_mean_it
,
3000 std::unique_ptr
<rgw::sal::Bucket
>* bucket
)
3002 if (bucket_name
.empty()) {
3003 cerr
<< "ERROR: bucket not specified" << std::endl
;
3007 if (!num_shards_specified
) {
3008 cerr
<< "ERROR: --num-shards not specified" << std::endl
;
3012 if (num_shards
> (int)static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->get_max_bucket_shards()) {
3013 cerr
<< "ERROR: num_shards too high, max value: " << static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->get_max_bucket_shards() << std::endl
;
3017 if (num_shards
< 0) {
3018 cerr
<< "ERROR: num_shards must be non-negative integer" << std::endl
;
3022 int ret
= init_bucket(nullptr, tenant
, bucket_name
, bucket_id
, bucket
);
3024 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
3028 int num_source_shards
= rgw::current_num_shards((*bucket
)->get_info().layout
);
3030 if (num_shards
<= num_source_shards
&& !yes_i_really_mean_it
) {
3031 cerr
<< "num shards is less or equal to current shards count" << std::endl
3032 << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
3038 static int scan_totp(CephContext
*cct
, ceph::real_time
& now
, rados::cls::otp::otp_info_t
& totp
, vector
<string
>& pins
,
3041 #define MAX_TOTP_SKEW_HOURS (24 * 7)
3042 time_t start_time
= ceph::real_clock::to_time_t(now
);
3043 time_t time_ofs
= 0, time_ofs_abs
= 0;
3044 time_t step_size
= totp
.step_size
;
3045 if (step_size
== 0) {
3046 step_size
= OATH_TOTP_DEFAULT_TIME_STEP_SIZE
;
3051 uint32_t max_skew
= MAX_TOTP_SKEW_HOURS
* 3600;
3053 while (time_ofs_abs
< max_skew
) {
3054 int rc
= oath_totp_validate2(totp
.seed_bin
.c_str(), totp
.seed_bin
.length(),
3061 if (rc
!= OATH_INVALID_OTP
) {
3062 rc
= oath_totp_validate2(totp
.seed_bin
.c_str(), totp
.seed_bin
.length(),
3065 time_ofs
- step_size
, /* smaller time_ofs moves time forward */
3069 if (rc
!= OATH_INVALID_OTP
) {
3070 *pofs
= time_ofs
- step_size
+ step_size
* totp
.window
/ 2;
3071 ldpp_dout(dpp(), 20) << "found at time=" << start_time
- time_ofs
<< " time_ofs=" << time_ofs
<< dendl
;
3076 time_ofs_abs
= (++count
) * step_size
;
3077 time_ofs
= sign
* time_ofs_abs
;
3083 static int trim_sync_error_log(int shard_id
, const string
& marker
, int delay_ms
)
3085 auto oid
= RGWSyncErrorLogger::get_shard_oid(RGW_SYNC_ERROR_LOG_SHARD_PREFIX
,
3087 // call cls_log_trim() until it returns -ENODATA
3089 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->timelog
.trim(dpp(), oid
, {}, {}, {}, marker
, nullptr,
3091 if (ret
== -ENODATA
) {
3098 std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms
));
3104 static bool symmetrical_flow_opt(const string
& opt
)
3106 return (opt
== "symmetrical" || opt
== "symmetric");
3109 static bool directional_flow_opt(const string
& opt
)
3111 return (opt
== "directional" || opt
== "direction");
3115 static bool require_opt(std::optional
<T
> opt
, bool extra_check
= true)
3117 if (!opt
|| !extra_check
) {
3124 static bool require_non_empty_opt(std::optional
<T
> opt
, bool extra_check
= true)
3126 if (!opt
|| opt
->empty() || !extra_check
) {
3133 static void show_result(T
& obj
,
3134 Formatter
*formatter
,
3137 encode_json("obj", obj
, formatter
);
3139 formatter
->flush(cout
);
3142 void init_optional_bucket(std::optional
<rgw_bucket
>& opt_bucket
,
3143 std::optional
<string
>& opt_tenant
,
3144 std::optional
<string
>& opt_bucket_name
,
3145 std::optional
<string
>& opt_bucket_id
)
3147 if (opt_tenant
|| opt_bucket_name
|| opt_bucket_id
) {
3148 opt_bucket
.emplace();
3150 opt_bucket
->tenant
= *opt_tenant
;
3152 if (opt_bucket_name
) {
3153 opt_bucket
->name
= *opt_bucket_name
;
3155 if (opt_bucket_id
) {
3156 opt_bucket
->bucket_id
= *opt_bucket_id
;
3161 class SyncPolicyContext
3163 rgw::sal::ConfigStore
* cfgstore
;
3164 RGWZoneGroup zonegroup
;
3165 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> zonegroup_writer
;
3167 std::optional
<rgw_bucket
> b
;
3168 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
3170 rgw_sync_policy_info
*policy
{nullptr};
3172 std::optional
<rgw_user
> owner
;
3175 SyncPolicyContext(rgw::sal::ConfigStore
* cfgstore
,
3176 std::optional
<rgw_bucket
> _bucket
)
3177 : cfgstore(cfgstore
), b(std::move(_bucket
)) {}
3179 int init(const string
& zonegroup_id
, const string
& zonegroup_name
) {
3180 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
,
3181 zonegroup_id
, zonegroup_name
,
3182 zonegroup
, &zonegroup_writer
);
3184 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
3189 policy
= &zonegroup
.sync_policy
;
3193 ret
= init_bucket(nullptr, *b
, &bucket
);
3195 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
3199 owner
= bucket
->get_info().owner
;
3201 if (!bucket
->get_info().sync_policy
) {
3202 rgw_sync_policy_info new_policy
;
3203 bucket
->get_info().set_sync_policy(std::move(new_policy
));
3206 policy
= &(*bucket
->get_info().sync_policy
);
3211 int write_policy() {
3213 int ret
= zonegroup_writer
->write(dpp(), null_yield
, zonegroup
);
3215 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
3221 int ret
= bucket
->put_info(dpp(), false, real_time());
3223 cerr
<< "failed to driver bucket info: " << cpp_strerror(-ret
) << std::endl
;
3230 rgw_sync_policy_info
& get_policy() {
3234 std::optional
<rgw_user
>& get_owner() {
3239 void resolve_zone_id_opt(std::optional
<string
>& zone_name
, std::optional
<rgw_zone_id
>& zone_id
)
3241 if (!zone_name
|| zone_id
) {
3245 std::unique_ptr
<rgw::sal::Zone
> zone
;
3246 int ret
= driver
->get_zone()->get_zonegroup().get_zone_by_name(*zone_name
, &zone
);
3248 cerr
<< "WARNING: cannot find source zone id for name=" << *zone_name
<< std::endl
;
3249 zone_id
= rgw_zone_id(*zone_name
);
3251 zone_id
->id
= zone
->get_id();
3254 void resolve_zone_ids_opt(std::optional
<vector
<string
> >& names
, std::optional
<vector
<rgw_zone_id
> >& ids
)
3256 if (!names
|| ids
) {
3260 for (auto& name
: *names
) {
3262 std::unique_ptr
<rgw::sal::Zone
> zone
;
3263 int ret
= driver
->get_zone()->get_zonegroup().get_zone_by_name(name
, &zone
);
3265 cerr
<< "WARNING: cannot find source zone id for name=" << name
<< std::endl
;
3266 zid
= rgw_zone_id(name
);
3268 zid
.id
= zone
->get_id();
3270 ids
->push_back(zid
);
3274 static vector
<rgw_zone_id
> zone_ids_from_str(const string
& val
)
3276 vector
<rgw_zone_id
> result
;
3278 get_str_vec(val
, v
);
3280 result
.push_back(rgw_zone_id(z
));
3285 class JSONFormatter_PrettyZone
: public JSONFormatter
{
3286 class Handler
: public JSONEncodeFilter::Handler
<rgw_zone_id
> {
3287 void encode_json(const char *name
, const void *pval
, ceph::Formatter
*f
) const override
{
3288 auto zone_id
= *(static_cast<const rgw_zone_id
*>(pval
));
3290 std::unique_ptr
<rgw::sal::Zone
> zone
;
3291 if (driver
->get_zone()->get_zonegroup().get_zone_by_id(zone_id
.id
, &zone
) == 0) {
3292 zone_name
= zone
->get_name();
3294 cerr
<< "WARNING: cannot find zone name for id=" << zone_id
<< std::endl
;
3295 zone_name
= zone_id
.id
;
3298 ::encode_json(name
, zone_name
, f
);
3300 } zone_id_type_handler
;
3302 JSONEncodeFilter encode_filter
;
3304 JSONFormatter_PrettyZone(bool pretty_format
) : JSONFormatter(pretty_format
) {
3305 encode_filter
.register_type(&zone_id_type_handler
);
3308 void *get_external_feature_handler(const std::string
& feature
) override
{
3309 if (feature
!= "JSONEncodeFilter") {
3312 return &encode_filter
;
3316 void init_realm_param(CephContext
*cct
, string
& var
, std::optional
<string
>& opt_var
, const string
& conf_name
)
3318 var
= cct
->_conf
.get_val
<string
>(conf_name
);
3324 int main(int argc
, const char **argv
)
3326 auto args
= argv_to_vec(argc
, argv
);
3328 cerr
<< argv
[0] << ": -h or --help for usage" << std::endl
;
3331 if (ceph_argparse_need_usage(args
)) {
3336 auto cct
= rgw_global_init(NULL
, args
, CEPH_ENTITY_TYPE_CLIENT
,
3337 CODE_ENVIRONMENT_UTILITY
, 0);
3339 // for region -> zonegroup conversion (must happen before common_init_finish())
3340 if (!g_conf()->rgw_region
.empty() && g_conf()->rgw_zonegroup
.empty()) {
3341 g_conf().set_val_or_die("rgw_zonegroup", g_conf()->rgw_region
.c_str());
3344 rgw_user user_id_arg
;
3345 std::unique_ptr
<rgw::sal::User
> user
;
3348 rgw_user new_user_id
;
3349 std::string access_key
, secret_key
, user_email
, display_name
;
3350 std::string bucket_name
, pool_name
, object
;
3352 std::string date
, subuser
, access
, format
;
3353 std::string start_date
, end_date
;
3354 std::string key_type_str
;
3355 std::string period_id
, period_epoch
, remote
, url
;
3356 std::optional
<string
> opt_region
;
3357 std::string master_zone
;
3358 std::string realm_name
, realm_id
, realm_new_name
;
3359 std::optional
<string
> opt_realm_name
, opt_realm_id
;
3360 std::string zone_name
, zone_id
, zone_new_name
;
3361 std::optional
<string
> opt_zone_name
, opt_zone_id
;
3362 std::string zonegroup_name
, zonegroup_id
, zonegroup_new_name
;
3363 std::optional
<string
> opt_zonegroup_name
, opt_zonegroup_id
;
3364 std::string api_name
;
3365 std::string role_name
, path
, assume_role_doc
, policy_name
, perm_policy_doc
, path_prefix
, max_session_duration
;
3366 std::string redirect_zone
;
3367 bool redirect_zone_set
= false;
3368 list
<string
> endpoints
;
3370 int sync_from_all_specified
= false;
3371 bool sync_from_all
= false;
3372 list
<string
> sync_from
;
3373 list
<string
> sync_from_rm
;
3375 int set_default
= 0;
3376 bool is_master
= false;
3377 bool is_master_set
= false;
3379 bool read_only
= false;
3380 int is_read_only_set
= false;
3382 int staging
= false;
3383 int key_type
= KEY_TYPE_UNDEFINED
;
3384 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
3385 uint32_t perm_mask
= 0;
3387 OPT opt_cmd
= OPT::NO_CMD
;
3388 int gen_access_key
= 0;
3389 int gen_secret_key
= 0;
3390 bool set_perm
= false;
3391 bool set_temp_url_key
= false;
3392 map
<int, string
> temp_url_keys
;
3394 string new_bucket_name
;
3395 std::unique_ptr
<Formatter
> formatter
;
3396 std::unique_ptr
<Formatter
> zone_formatter
;
3397 int purge_data
= false;
3398 int pretty_format
= false;
3399 int show_log_entries
= true;
3400 int show_log_sum
= true;
3401 int skip_zero_entries
= false; // log show
3402 int purge_keys
= false;
3403 int yes_i_really_mean_it
= false;
3404 int delete_child_objects
= false;
3406 int remove_bad
= false;
3407 int check_head_obj_locator
= false;
3408 int max_buckets
= -1;
3409 bool max_buckets_specified
= false;
3410 map
<string
, bool> categories
;
3412 int check_objects
= false;
3413 RGWBucketAdminOpState bucket_op
;
3415 string metadata_key
;
3416 RGWObjVersionTracker objv_tracker
;
3418 string start_marker
;
3420 int max_entries
= -1;
3421 bool max_entries_specified
= false;
3423 bool admin_specified
= false;
3425 bool system_specified
= false;
3427 bool specified_shard_id
= false;
3432 string ratelimit_scope
;
3433 std::string objects_file
;
3434 string object_version
;
3435 string placement_id
;
3436 std::optional
<string
> opt_storage_class
;
3438 list
<string
> tags_add
;
3439 list
<string
> tags_rm
;
3440 int placement_inline_data
= true;
3441 bool placement_inline_data_specified
= false;
3443 int64_t max_objects
= -1;
3444 int64_t max_size
= -1;
3445 int64_t max_read_ops
= 0;
3446 int64_t max_write_ops
= 0;
3447 int64_t max_read_bytes
= 0;
3448 int64_t max_write_bytes
= 0;
3449 bool have_max_objects
= false;
3450 bool have_max_size
= false;
3451 bool have_max_write_ops
= false;
3452 bool have_max_read_ops
= false;
3453 bool have_max_write_bytes
= false;
3454 bool have_max_read_bytes
= false;
3455 int include_all
= false;
3456 int allow_unordered
= false;
3458 int sync_stats
= false;
3459 int reset_stats
= false;
3460 int bypass_gc
= false;
3461 int warnings_only
= false;
3462 int inconsistent_index
= false;
3464 int verbose
= false;
3466 int extra_info
= false;
3468 uint64_t min_rewrite_size
= 4 * 1024 * 1024;
3469 uint64_t max_rewrite_size
= ULLONG_MAX
;
3470 uint64_t min_rewrite_stripe_size
= 0;
3472 BIIndexType bi_index_type
= BIIndexType::Plain
;
3473 std::optional
<log_type
> opt_log_type
;
3477 bool num_shards_specified
= false;
3478 std::optional
<int> bucket_index_max_shards
;
3480 int max_concurrent_ios
= 32;
3481 ceph::timespan min_age
= std::chrono::hours(1);
3482 bool hide_progress
= false;
3483 bool dump_keys
= false;
3484 uint64_t orphan_stale_secs
= (24 * 3600);
3488 std::ostringstream errs
;
3491 string source_zone_name
;
3492 rgw_zone_id source_zone
; /* zone id */
3495 bool tier_type_specified
= false;
3497 map
<string
, string
, ltstr_nocase
> tier_config_add
;
3498 map
<string
, string
, ltstr_nocase
> tier_config_rm
;
3500 boost::optional
<string
> index_pool
;
3501 boost::optional
<string
> data_pool
;
3502 boost::optional
<string
> data_extra_pool
;
3503 rgw::BucketIndexType placement_index_type
= rgw::BucketIndexType::Normal
;
3504 bool index_type_specified
= false;
3506 boost::optional
<std::string
> compression_type
;
3510 string totp_seed_type
= "hex";
3511 vector
<string
> totp_pin
;
3512 int totp_seconds
= 0;
3513 int totp_window
= 0;
3514 int trim_delay_ms
= 0;
3517 string notification_id
;
3521 std::optional
<uint64_t> gen
;
3522 std::optional
<std::string
> str_script_ctx
;
3523 std::optional
<std::string
> script_package
;
3524 int allow_compilation
= false;
3526 std::optional
<string
> opt_group_id
;
3527 std::optional
<string
> opt_status
;
3528 std::optional
<string
> opt_flow_type
;
3529 std::optional
<vector
<string
> > opt_zone_names
;
3530 std::optional
<vector
<rgw_zone_id
> > opt_zone_ids
;
3531 std::optional
<string
> opt_flow_id
;
3532 std::optional
<string
> opt_source_zone_name
;
3533 std::optional
<rgw_zone_id
> opt_source_zone_id
;
3534 std::optional
<string
> opt_dest_zone_name
;
3535 std::optional
<rgw_zone_id
> opt_dest_zone_id
;
3536 std::optional
<vector
<string
> > opt_source_zone_names
;
3537 std::optional
<vector
<rgw_zone_id
> > opt_source_zone_ids
;
3538 std::optional
<vector
<string
> > opt_dest_zone_names
;
3539 std::optional
<vector
<rgw_zone_id
> > opt_dest_zone_ids
;
3540 std::optional
<string
> opt_pipe_id
;
3541 std::optional
<rgw_bucket
> opt_bucket
;
3542 std::optional
<string
> opt_tenant
;
3543 std::optional
<string
> opt_bucket_name
;
3544 std::optional
<string
> opt_bucket_id
;
3545 std::optional
<rgw_bucket
> opt_source_bucket
;
3546 std::optional
<string
> opt_source_tenant
;
3547 std::optional
<string
> opt_source_bucket_name
;
3548 std::optional
<string
> opt_source_bucket_id
;
3549 std::optional
<rgw_bucket
> opt_dest_bucket
;
3550 std::optional
<string
> opt_dest_tenant
;
3551 std::optional
<string
> opt_dest_bucket_name
;
3552 std::optional
<string
> opt_dest_bucket_id
;
3553 std::optional
<string
> opt_effective_zone_name
;
3554 std::optional
<rgw_zone_id
> opt_effective_zone_id
;
3556 std::optional
<string
> opt_prefix
;
3557 std::optional
<string
> opt_prefix_rm
;
3559 std::optional
<int> opt_priority
;
3560 std::optional
<string
> opt_mode
;
3561 std::optional
<rgw_user
> opt_dest_owner
;
3562 ceph::timespan opt_retry_delay_ms
= std::chrono::milliseconds(2000);
3563 ceph::timespan opt_timeout_sec
= std::chrono::seconds(60);
3565 std::optional
<std::string
> inject_error_at
;
3566 std::optional
<int> inject_error_code
;
3567 std::optional
<std::string
> inject_abort_at
;
3568 std::optional
<std::string
> inject_delay_at
;
3569 ceph::timespan inject_delay
= std::chrono::milliseconds(2000);
3571 rgw::zone_features::set enable_features
;
3572 rgw::zone_features::set disable_features
;
3574 SimpleCmd
cmd(all_cmds
, cmd_aliases
);
3575 bool raw_storage_op
= false;
3577 std::optional
<std::string
> rgw_obj_fs
; // radoslist field separator
3579 init_realm_param(cct
.get(), realm_id
, opt_realm_id
, "rgw_realm_id");
3580 init_realm_param(cct
.get(), zonegroup_id
, opt_zonegroup_id
, "rgw_zonegroup_id");
3581 init_realm_param(cct
.get(), zone_id
, opt_zone_id
, "rgw_zone_id");
3583 for (std::vector
<const char*>::iterator i
= args
.begin(); i
!= args
.end(); ) {
3584 if (ceph_argparse_double_dash(args
, i
)) {
3586 } else if (ceph_argparse_witharg(args
, i
, &val
, "-i", "--uid", (char*)NULL
)) {
3587 user_id_arg
.from_str(val
);
3588 if (user_id_arg
.empty()) {
3589 cerr
<< "no value for uid" << std::endl
;
3592 } else if (ceph_argparse_witharg(args
, i
, &val
, "--new-uid", (char*)NULL
)) {
3593 new_user_id
.from_str(val
);
3594 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tenant", (char*)NULL
)) {
3597 } else if (ceph_argparse_witharg(args
, i
, &val
, "--user_ns", (char*)NULL
)) {
3599 } else if (ceph_argparse_witharg(args
, i
, &val
, "--access-key", (char*)NULL
)) {
3601 } else if (ceph_argparse_witharg(args
, i
, &val
, "--subuser", (char*)NULL
)) {
3603 } else if (ceph_argparse_witharg(args
, i
, &val
, "--secret", "--secret-key", (char*)NULL
)) {
3605 } else if (ceph_argparse_witharg(args
, i
, &val
, "-e", "--email", (char*)NULL
)) {
3607 } else if (ceph_argparse_witharg(args
, i
, &val
, "-n", "--display-name", (char*)NULL
)) {
3609 } else if (ceph_argparse_witharg(args
, i
, &val
, "-b", "--bucket", (char*)NULL
)) {
3611 opt_bucket_name
= val
;
3612 } else if (ceph_argparse_witharg(args
, i
, &val
, "-p", "--pool", (char*)NULL
)) {
3614 pool
= rgw_pool(pool_name
);
3615 } else if (ceph_argparse_witharg(args
, i
, &val
, "-o", "--object", (char*)NULL
)) {
3617 } else if (ceph_argparse_witharg(args
, i
, &val
, "--objects-file", (char*)NULL
)) {
3619 } else if (ceph_argparse_witharg(args
, i
, &val
, "--object-version", (char*)NULL
)) {
3620 object_version
= val
;
3621 } else if (ceph_argparse_witharg(args
, i
, &val
, "--client-id", (char*)NULL
)) {
3623 } else if (ceph_argparse_witharg(args
, i
, &val
, "--op-id", (char*)NULL
)) {
3625 } else if (ceph_argparse_witharg(args
, i
, &val
, "--op-mask", (char*)NULL
)) {
3627 } else if (ceph_argparse_witharg(args
, i
, &val
, "--key-type", (char*)NULL
)) {
3629 if (key_type_str
.compare("swift") == 0) {
3630 key_type
= KEY_TYPE_SWIFT
;
3631 } else if (key_type_str
.compare("s3") == 0) {
3632 key_type
= KEY_TYPE_S3
;
3634 cerr
<< "bad key type: " << key_type_str
<< std::endl
;
3637 } else if (ceph_argparse_witharg(args
, i
, &val
, "--job-id", (char*)NULL
)) {
3639 } else if (ceph_argparse_binary_flag(args
, i
, &gen_access_key
, NULL
, "--gen-access-key", (char*)NULL
)) {
3641 } else if (ceph_argparse_binary_flag(args
, i
, &gen_secret_key
, NULL
, "--gen-secret", (char*)NULL
)) {
3643 } else if (ceph_argparse_binary_flag(args
, i
, &show_log_entries
, NULL
, "--show-log-entries", (char*)NULL
)) {
3645 } else if (ceph_argparse_binary_flag(args
, i
, &show_log_sum
, NULL
, "--show-log-sum", (char*)NULL
)) {
3647 } else if (ceph_argparse_binary_flag(args
, i
, &skip_zero_entries
, NULL
, "--skip-zero-entries", (char*)NULL
)) {
3649 } else if (ceph_argparse_binary_flag(args
, i
, &admin
, NULL
, "--admin", (char*)NULL
)) {
3650 admin_specified
= true;
3651 } else if (ceph_argparse_binary_flag(args
, i
, &system
, NULL
, "--system", (char*)NULL
)) {
3652 system_specified
= true;
3653 } else if (ceph_argparse_binary_flag(args
, i
, &verbose
, NULL
, "--verbose", (char*)NULL
)) {
3655 } else if (ceph_argparse_binary_flag(args
, i
, &staging
, NULL
, "--staging", (char*)NULL
)) {
3657 } else if (ceph_argparse_binary_flag(args
, i
, &commit
, NULL
, "--commit", (char*)NULL
)) {
3659 } else if (ceph_argparse_witharg(args
, i
, &val
, "--min-rewrite-size", (char*)NULL
)) {
3660 min_rewrite_size
= (uint64_t)atoll(val
.c_str());
3661 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-rewrite-size", (char*)NULL
)) {
3662 max_rewrite_size
= (uint64_t)atoll(val
.c_str());
3663 } else if (ceph_argparse_witharg(args
, i
, &val
, "--min-rewrite-stripe-size", (char*)NULL
)) {
3664 min_rewrite_stripe_size
= (uint64_t)atoll(val
.c_str());
3665 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-buckets", (char*)NULL
)) {
3666 max_buckets
= (int)strict_strtol(val
.c_str(), 10, &err
);
3668 cerr
<< "ERROR: failed to parse max buckets: " << err
<< std::endl
;
3671 max_buckets_specified
= true;
3672 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-entries", (char*)NULL
)) {
3673 max_entries
= (int)strict_strtol(val
.c_str(), 10, &err
);
3674 max_entries_specified
= true;
3676 cerr
<< "ERROR: failed to parse max entries: " << err
<< std::endl
;
3679 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-size", (char*)NULL
)) {
3680 max_size
= strict_iec_cast
<long long>(val
, &err
);
3682 cerr
<< "ERROR: failed to parse max size: " << err
<< std::endl
;
3685 have_max_size
= true;
3686 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-objects", (char*)NULL
)) {
3687 max_objects
= (int64_t)strict_strtoll(val
.c_str(), 10, &err
);
3689 cerr
<< "ERROR: failed to parse max objects: " << err
<< std::endl
;
3692 have_max_objects
= true;
3693 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-read-ops", (char*)NULL
)) {
3694 max_read_ops
= (int64_t)strict_strtoll(val
.c_str(), 10, &err
);
3696 cerr
<< "ERROR: failed to parse max read requests: " << err
<< std::endl
;
3699 have_max_read_ops
= true;
3700 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-write-ops", (char*)NULL
)) {
3701 max_write_ops
= (int64_t)strict_strtoll(val
.c_str(), 10, &err
);
3703 cerr
<< "ERROR: failed to parse max write requests: " << err
<< std::endl
;
3706 have_max_write_ops
= true;
3707 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-read-bytes", (char*)NULL
)) {
3708 max_read_bytes
= (int64_t)strict_strtoll(val
.c_str(), 10, &err
);
3710 cerr
<< "ERROR: failed to parse max read bytes: " << err
<< std::endl
;
3713 have_max_read_bytes
= true;
3714 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-write-bytes", (char*)NULL
)) {
3715 max_write_bytes
= (int64_t)strict_strtoll(val
.c_str(), 10, &err
);
3717 cerr
<< "ERROR: failed to parse max write bytes: " << err
<< std::endl
;
3720 have_max_write_bytes
= true;
3721 } else if (ceph_argparse_witharg(args
, i
, &val
, "--date", "--time", (char*)NULL
)) {
3723 if (end_date
.empty())
3725 } else if (ceph_argparse_witharg(args
, i
, &val
, "--start-date", "--start-time", (char*)NULL
)) {
3727 } else if (ceph_argparse_witharg(args
, i
, &val
, "--end-date", "--end-time", (char*)NULL
)) {
3729 } else if (ceph_argparse_witharg(args
, i
, &val
, "--num-shards", (char*)NULL
)) {
3730 num_shards
= (int)strict_strtol(val
.c_str(), 10, &err
);
3732 cerr
<< "ERROR: failed to parse num shards: " << err
<< std::endl
;
3735 num_shards_specified
= true;
3736 } else if (ceph_argparse_witharg(args
, i
, &val
, "--bucket-index-max-shards", (char*)NULL
)) {
3737 bucket_index_max_shards
= (int)strict_strtol(val
.c_str(), 10, &err
);
3739 cerr
<< "ERROR: failed to parse bucket-index-max-shards: " << err
<< std::endl
;
3742 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-concurrent-ios", (char*)NULL
)) {
3743 max_concurrent_ios
= (int)strict_strtol(val
.c_str(), 10, &err
);
3745 cerr
<< "ERROR: failed to parse max concurrent ios: " << err
<< std::endl
;
3748 } else if (ceph_argparse_witharg(args
, i
, &val
, "--min-age-hours", (char*)NULL
)) {
3749 min_age
= std::chrono::hours(atoi(val
.c_str()));
3750 } else if (ceph_argparse_witharg(args
, i
, &val
, "--orphan-stale-secs", (char*)NULL
)) {
3751 orphan_stale_secs
= (uint64_t)strict_strtoll(val
.c_str(), 10, &err
);
3753 cerr
<< "ERROR: failed to parse orphan stale secs: " << err
<< std::endl
;
3756 } else if (ceph_argparse_witharg(args
, i
, &val
, "--shard-id", (char*)NULL
)) {
3757 shard_id
= (int)strict_strtol(val
.c_str(), 10, &err
);
3759 cerr
<< "ERROR: failed to parse shard id: " << err
<< std::endl
;
3762 specified_shard_id
= true;
3763 } else if (ceph_argparse_witharg(args
, i
, &val
, "--gen", (char*)NULL
)) {
3764 gen
= strict_strtoll(val
.c_str(), 10, &err
);
3766 cerr
<< "ERROR: failed to parse gen id: " << err
<< std::endl
;
3769 } else if (ceph_argparse_witharg(args
, i
, &val
, "--access", (char*)NULL
)) {
3771 perm_mask
= rgw_str_to_perm(access
.c_str());
3773 } else if (ceph_argparse_witharg(args
, i
, &val
, "--temp-url-key", (char*)NULL
)) {
3774 temp_url_keys
[0] = val
;
3775 set_temp_url_key
= true;
3776 } else if (ceph_argparse_witharg(args
, i
, &val
, "--temp-url-key2", "--temp-url-key-2", (char*)NULL
)) {
3777 temp_url_keys
[1] = val
;
3778 set_temp_url_key
= true;
3779 } else if (ceph_argparse_witharg(args
, i
, &val
, "--bucket-id", (char*)NULL
)) {
3781 opt_bucket_id
= val
;
3782 if (bucket_id
.empty()) {
3783 cerr
<< "no value for bucket-id" << std::endl
;
3786 } else if (ceph_argparse_witharg(args
, i
, &val
, "--bucket-new-name", (char*)NULL
)) {
3787 new_bucket_name
= val
;
3788 } else if (ceph_argparse_witharg(args
, i
, &val
, "--format", (char*)NULL
)) {
3790 } else if (ceph_argparse_witharg(args
, i
, &val
, "--categories", (char*)NULL
)) {
3791 string cat_str
= val
;
3792 list
<string
> cat_list
;
3793 list
<string
>::iterator iter
;
3794 get_str_list(cat_str
, cat_list
);
3795 for (iter
= cat_list
.begin(); iter
!= cat_list
.end(); ++iter
) {
3796 categories
[*iter
] = true;
3798 } else if (ceph_argparse_binary_flag(args
, i
, &delete_child_objects
, NULL
, "--purge-objects", (char*)NULL
)) {
3800 } else if (ceph_argparse_binary_flag(args
, i
, &pretty_format
, NULL
, "--pretty-format", (char*)NULL
)) {
3802 } else if (ceph_argparse_binary_flag(args
, i
, &purge_data
, NULL
, "--purge-data", (char*)NULL
)) {
3803 delete_child_objects
= purge_data
;
3804 } else if (ceph_argparse_binary_flag(args
, i
, &purge_keys
, NULL
, "--purge-keys", (char*)NULL
)) {
3806 } else if (ceph_argparse_binary_flag(args
, i
, &yes_i_really_mean_it
, NULL
, "--yes-i-really-mean-it", (char*)NULL
)) {
3808 } else if (ceph_argparse_binary_flag(args
, i
, &fix
, NULL
, "--fix", (char*)NULL
)) {
3810 } else if (ceph_argparse_binary_flag(args
, i
, &remove_bad
, NULL
, "--remove-bad", (char*)NULL
)) {
3812 } else if (ceph_argparse_binary_flag(args
, i
, &check_head_obj_locator
, NULL
, "--check-head-obj-locator", (char*)NULL
)) {
3814 } else if (ceph_argparse_binary_flag(args
, i
, &check_objects
, NULL
, "--check-objects", (char*)NULL
)) {
3816 } else if (ceph_argparse_binary_flag(args
, i
, &sync_stats
, NULL
, "--sync-stats", (char*)NULL
)) {
3818 } else if (ceph_argparse_binary_flag(args
, i
, &reset_stats
, NULL
, "--reset-stats", (char*)NULL
)) {
3820 } else if (ceph_argparse_binary_flag(args
, i
, &include_all
, NULL
, "--include-all", (char*)NULL
)) {
3822 } else if (ceph_argparse_binary_flag(args
, i
, &allow_unordered
, NULL
, "--allow-unordered", (char*)NULL
)) {
3824 } else if (ceph_argparse_binary_flag(args
, i
, &extra_info
, NULL
, "--extra-info", (char*)NULL
)) {
3826 } else if (ceph_argparse_binary_flag(args
, i
, &bypass_gc
, NULL
, "--bypass-gc", (char*)NULL
)) {
3828 } else if (ceph_argparse_binary_flag(args
, i
, &warnings_only
, NULL
, "--warnings-only", (char*)NULL
)) {
3830 } else if (ceph_argparse_binary_flag(args
, i
, &inconsistent_index
, NULL
, "--inconsistent-index", (char*)NULL
)) {
3832 } else if (ceph_argparse_flag(args
, i
, "--hide-progress", (char*)NULL
)) {
3833 hide_progress
= true;
3834 } else if (ceph_argparse_flag(args
, i
, "--dump-keys", (char*)NULL
)) {
3836 } else if (ceph_argparse_binary_flag(args
, i
, &placement_inline_data
, NULL
, "--placement-inline-data", (char*)NULL
)) {
3837 placement_inline_data_specified
= true;
3839 } else if (ceph_argparse_witharg(args
, i
, &val
, "--caps", (char*)NULL
)) {
3841 } else if (ceph_argparse_witharg(args
, i
, &val
, "--infile", (char*)NULL
)) {
3843 } else if (ceph_argparse_witharg(args
, i
, &val
, "--metadata-key", (char*)NULL
)) {
3845 } else if (ceph_argparse_witharg(args
, i
, &val
, "--marker", (char*)NULL
)) {
3847 } else if (ceph_argparse_witharg(args
, i
, &val
, "--start-marker", (char*)NULL
)) {
3849 } else if (ceph_argparse_witharg(args
, i
, &val
, "--end-marker", (char*)NULL
)) {
3851 } else if (ceph_argparse_witharg(args
, i
, &val
, "--quota-scope", (char*)NULL
)) {
3853 } else if (ceph_argparse_witharg(args
, i
, &val
, "--ratelimit-scope", (char*)NULL
)) {
3854 ratelimit_scope
= val
;
3855 } else if (ceph_argparse_witharg(args
, i
, &val
, "--index-type", (char*)NULL
)) {
3856 string index_type_str
= val
;
3857 bi_index_type
= get_bi_index_type(index_type_str
);
3858 if (bi_index_type
== BIIndexType::Invalid
) {
3859 cerr
<< "ERROR: invalid bucket index entry type" << std::endl
;
3862 } else if (ceph_argparse_witharg(args
, i
, &val
, "--log-type", (char*)NULL
)) {
3863 string log_type_str
= val
;
3864 auto l
= get_log_type(log_type_str
);
3865 if (l
== static_cast<log_type
>(0xff)) {
3866 cerr
<< "ERROR: invalid log type" << std::endl
;
3870 } else if (ceph_argparse_binary_flag(args
, i
, &is_master_int
, NULL
, "--master", (char*)NULL
)) {
3871 is_master
= (bool)is_master_int
;
3872 is_master_set
= true;
3873 } else if (ceph_argparse_binary_flag(args
, i
, &set_default
, NULL
, "--default", (char*)NULL
)) {
3875 } else if (ceph_argparse_witharg(args
, i
, &val
, "--redirect-zone", (char*)NULL
)) {
3876 redirect_zone
= val
;
3877 redirect_zone_set
= true;
3878 } else if (ceph_argparse_binary_flag(args
, i
, &read_only_int
, NULL
, "--read-only", (char*)NULL
)) {
3879 read_only
= (bool)read_only_int
;
3880 is_read_only_set
= true;
3881 } else if (ceph_argparse_witharg(args
, i
, &val
, "--master-zone", (char*)NULL
)) {
3883 } else if (ceph_argparse_witharg(args
, i
, &val
, "--period", (char*)NULL
)) {
3885 } else if (ceph_argparse_witharg(args
, i
, &val
, "--epoch", (char*)NULL
)) {
3887 } else if (ceph_argparse_witharg(args
, i
, &val
, "--remote", (char*)NULL
)) {
3889 } else if (ceph_argparse_witharg(args
, i
, &val
, "--url", (char*)NULL
)) {
3891 } else if (ceph_argparse_witharg(args
, i
, &val
, "--region", (char*)NULL
)) {
3893 } else if (ceph_argparse_witharg(args
, i
, &val
, "--realm-id", (char*)NULL
)) {
3896 g_conf().set_val("rgw_realm_id", val
);
3897 } else if (ceph_argparse_witharg(args
, i
, &val
, "--realm-new-name", (char*)NULL
)) {
3898 realm_new_name
= val
;
3899 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zonegroup-id", (char*)NULL
)) {
3901 opt_zonegroup_id
= val
;
3902 g_conf().set_val("rgw_zonegroup_id", val
);
3903 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zonegroup-new-name", (char*)NULL
)) {
3904 zonegroup_new_name
= val
;
3905 } else if (ceph_argparse_witharg(args
, i
, &val
, "--placement-id", (char*)NULL
)) {
3907 } else if (ceph_argparse_witharg(args
, i
, &val
, "--storage-class", (char*)NULL
)) {
3908 opt_storage_class
= val
;
3909 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tags", (char*)NULL
)) {
3910 get_str_list(val
, ",", tags
);
3911 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tags-add", (char*)NULL
)) {
3912 get_str_list(val
, ",", tags_add
);
3913 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tags-rm", (char*)NULL
)) {
3914 get_str_list(val
, ",", tags_rm
);
3915 } else if (ceph_argparse_witharg(args
, i
, &val
, "--api-name", (char*)NULL
)) {
3917 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zone-id", (char*)NULL
)) {
3920 g_conf().set_val("rgw_zone_id", val
);
3921 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zone-new-name", (char*)NULL
)) {
3922 zone_new_name
= val
;
3923 } else if (ceph_argparse_witharg(args
, i
, &val
, "--endpoints", (char*)NULL
)) {
3924 get_str_list(val
, endpoints
);
3925 } else if (ceph_argparse_witharg(args
, i
, &val
, "--sync-from", (char*)NULL
)) {
3926 get_str_list(val
, sync_from
);
3927 } else if (ceph_argparse_witharg(args
, i
, &val
, "--sync-from-rm", (char*)NULL
)) {
3928 get_str_list(val
, sync_from_rm
);
3929 } else if (ceph_argparse_binary_flag(args
, i
, &tmp_int
, NULL
, "--sync-from-all", (char*)NULL
)) {
3930 sync_from_all
= (bool)tmp_int
;
3931 sync_from_all_specified
= true;
3932 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zone", (char*)NULL
)) {
3933 source_zone_name
= val
;
3934 opt_source_zone_name
= val
;
3935 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zone-id", (char*)NULL
)) {
3936 opt_source_zone_id
= val
;
3937 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zone", (char*)NULL
)) {
3938 opt_dest_zone_name
= val
;
3939 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zone-id", (char*)NULL
)) {
3940 opt_dest_zone_id
= val
;
3941 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tier-type", (char*)NULL
)) {
3943 tier_type_specified
= true;
3944 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tier-config", (char*)NULL
)) {
3945 parse_tier_config_param(val
, tier_config_add
);
3946 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tier-config-rm", (char*)NULL
)) {
3947 parse_tier_config_param(val
, tier_config_rm
);
3948 } else if (ceph_argparse_witharg(args
, i
, &val
, "--index-pool", (char*)NULL
)) {
3950 } else if (ceph_argparse_witharg(args
, i
, &val
, "--data-pool", (char*)NULL
)) {
3952 } else if (ceph_argparse_witharg(args
, i
, &val
, "--data-extra-pool", (char*)NULL
)) {
3953 data_extra_pool
= val
;
3954 } else if (ceph_argparse_witharg(args
, i
, &val
, "--placement-index-type", (char*)NULL
)) {
3955 if (val
== "normal") {
3956 placement_index_type
= rgw::BucketIndexType::Normal
;
3957 } else if (val
== "indexless") {
3958 placement_index_type
= rgw::BucketIndexType::Indexless
;
3960 placement_index_type
= (rgw::BucketIndexType
)strict_strtol(val
.c_str(), 10, &err
);
3962 cerr
<< "ERROR: failed to parse index type index: " << err
<< std::endl
;
3966 index_type_specified
= true;
3967 } else if (ceph_argparse_witharg(args
, i
, &val
, "--compression", (char*)NULL
)) {
3968 compression_type
= val
;
3969 } else if (ceph_argparse_witharg(args
, i
, &val
, "--role-name", (char*)NULL
)) {
3971 } else if (ceph_argparse_witharg(args
, i
, &val
, "--path", (char*)NULL
)) {
3973 } else if (ceph_argparse_witharg(args
, i
, &val
, "--assume-role-policy-doc", (char*)NULL
)) {
3974 assume_role_doc
= val
;
3975 } else if (ceph_argparse_witharg(args
, i
, &val
, "--policy-name", (char*)NULL
)) {
3977 } else if (ceph_argparse_witharg(args
, i
, &val
, "--policy-doc", (char*)NULL
)) {
3978 perm_policy_doc
= val
;
3979 } else if (ceph_argparse_witharg(args
, i
, &val
, "--path-prefix", (char*)NULL
)) {
3981 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-session-duration", (char*)NULL
)) {
3982 max_session_duration
= val
;
3983 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-serial", (char*)NULL
)) {
3985 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-pin", (char*)NULL
)) {
3986 totp_pin
.push_back(val
);
3987 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-seed", (char*)NULL
)) {
3989 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-seed-type", (char*)NULL
)) {
3990 totp_seed_type
= val
;
3991 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-seconds", (char*)NULL
)) {
3992 totp_seconds
= atoi(val
.c_str());
3993 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-window", (char*)NULL
)) {
3994 totp_window
= atoi(val
.c_str());
3995 } else if (ceph_argparse_witharg(args
, i
, &val
, "--trim-delay-ms", (char*)NULL
)) {
3996 trim_delay_ms
= atoi(val
.c_str());
3997 } else if (ceph_argparse_witharg(args
, i
, &val
, "--topic", (char*)NULL
)) {
3999 } else if (ceph_argparse_witharg(args
, i
, &val
, "--notification-id", (char*)NULL
)) {
4000 notification_id
= val
;
4001 } else if (ceph_argparse_witharg(args
, i
, &val
, "--subscription", (char*)NULL
)) {
4003 } else if (ceph_argparse_witharg(args
, i
, &val
, "--event-id", (char*)NULL
)) {
4005 } else if (ceph_argparse_witharg(args
, i
, &val
, "--group-id", (char*)NULL
)) {
4007 } else if (ceph_argparse_witharg(args
, i
, &val
, "--status", (char*)NULL
)) {
4009 } else if (ceph_argparse_witharg(args
, i
, &val
, "--flow-type", (char*)NULL
)) {
4010 opt_flow_type
= val
;
4011 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zones", "--zone-names", (char*)NULL
)) {
4013 get_str_vec(val
, v
);
4014 opt_zone_names
= std::move(v
);
4015 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zone-ids", (char*)NULL
)) {
4016 opt_zone_ids
= zone_ids_from_str(val
);
4017 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zones", "--source-zone-names", (char*)NULL
)) {
4019 get_str_vec(val
, v
);
4020 opt_source_zone_names
= std::move(v
);
4021 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zone-ids", (char*)NULL
)) {
4022 opt_source_zone_ids
= zone_ids_from_str(val
);
4023 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zones", "--dest-zone-names", (char*)NULL
)) {
4025 get_str_vec(val
, v
);
4026 opt_dest_zone_names
= std::move(v
);
4027 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zone-ids", (char*)NULL
)) {
4028 opt_dest_zone_ids
= zone_ids_from_str(val
);
4029 } else if (ceph_argparse_witharg(args
, i
, &val
, "--flow-id", (char*)NULL
)) {
4031 } else if (ceph_argparse_witharg(args
, i
, &val
, "--pipe-id", (char*)NULL
)) {
4033 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-tenant", (char*)NULL
)) {
4034 opt_source_tenant
= val
;
4035 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-bucket", (char*)NULL
)) {
4036 opt_source_bucket_name
= val
;
4037 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-bucket-id", (char*)NULL
)) {
4038 opt_source_bucket_id
= val
;
4039 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-tenant", (char*)NULL
)) {
4040 opt_dest_tenant
= val
;
4041 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-bucket", (char*)NULL
)) {
4042 opt_dest_bucket_name
= val
;
4043 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-bucket-id", (char*)NULL
)) {
4044 opt_dest_bucket_id
= val
;
4045 } else if (ceph_argparse_witharg(args
, i
, &val
, "--effective-zone-name", "--effective-zone", (char*)NULL
)) {
4046 opt_effective_zone_name
= val
;
4047 } else if (ceph_argparse_witharg(args
, i
, &val
, "--effective-zone-id", (char*)NULL
)) {
4048 opt_effective_zone_id
= rgw_zone_id(val
);
4049 } else if (ceph_argparse_witharg(args
, i
, &val
, "--prefix", (char*)NULL
)) {
4051 } else if (ceph_argparse_witharg(args
, i
, &val
, "--prefix-rm", (char*)NULL
)) {
4052 opt_prefix_rm
= val
;
4053 } else if (ceph_argparse_witharg(args
, i
, &val
, "--priority", (char*)NULL
)) {
4054 opt_priority
= atoi(val
.c_str());
4055 } else if (ceph_argparse_witharg(args
, i
, &val
, "--mode", (char*)NULL
)) {
4057 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-owner", (char*)NULL
)) {
4058 opt_dest_owner
.emplace(val
);
4059 opt_dest_owner
= val
;
4060 } else if (ceph_argparse_witharg(args
, i
, &val
, "--retry-delay-ms", (char*)NULL
)) {
4061 opt_retry_delay_ms
= std::chrono::milliseconds(atoi(val
.c_str()));
4062 } else if (ceph_argparse_witharg(args
, i
, &val
, "--timeout-sec", (char*)NULL
)) {
4063 opt_timeout_sec
= std::chrono::seconds(atoi(val
.c_str()));
4064 } else if (ceph_argparse_witharg(args
, i
, &val
, "--inject-error-at", (char*)NULL
)) {
4065 inject_error_at
= val
;
4066 } else if (ceph_argparse_witharg(args
, i
, &val
, "--inject-error-code", (char*)NULL
)) {
4067 inject_error_code
= atoi(val
.c_str());
4068 } else if (ceph_argparse_witharg(args
, i
, &val
, "--inject-abort-at", (char*)NULL
)) {
4069 inject_abort_at
= val
;
4070 } else if (ceph_argparse_witharg(args
, i
, &val
, "--inject-delay-at", (char*)NULL
)) {
4071 inject_delay_at
= val
;
4072 } else if (ceph_argparse_witharg(args
, i
, &val
, "--inject-delay-ms", (char*)NULL
)) {
4073 inject_delay
= std::chrono::milliseconds(atoi(val
.c_str()));
4074 } else if (ceph_argparse_binary_flag(args
, i
, &detail
, NULL
, "--detail", (char*)NULL
)) {
4076 } else if (ceph_argparse_witharg(args
, i
, &val
, "--context", (char*)NULL
)) {
4077 str_script_ctx
= val
;
4078 } else if (ceph_argparse_witharg(args
, i
, &val
, "--package", (char*)NULL
)) {
4079 script_package
= val
;
4080 } else if (ceph_argparse_binary_flag(args
, i
, &allow_compilation
, NULL
, "--allow-compilation", (char*)NULL
)) {
4082 } else if (ceph_argparse_witharg(args
, i
, &val
, "--rgw-obj-fs", (char*)NULL
)) {
4084 } else if (ceph_argparse_witharg(args
, i
, &val
, "--enable-feature", (char*)NULL
)) {
4085 if (!rgw::zone_features::supports(val
)) {
4086 std::cerr
<< "ERROR: Cannot enable unrecognized zone feature \"" << val
<< "\"" << std::endl
;
4089 enable_features
.insert(val
);
4090 } else if (ceph_argparse_witharg(args
, i
, &val
, "--disable-feature", (char*)NULL
)) {
4091 disable_features
.insert(val
);
4092 } else if (strncmp(*i
, "-", 1) == 0) {
4093 cerr
<< "ERROR: invalid flag " << *i
<< std::endl
;
4100 /* common_init_finish needs to be called after g_conf().set_val() */
4101 common_init_finish(g_ceph_context
);
4103 std::unique_ptr
<rgw::sal::ConfigStore
> cfgstore
;
4110 std::vector
<string
> extra_args
;
4111 std::vector
<string
> expected
;
4115 if (!cmd
.find_command(args
, &_opt_cmd
, &extra_args
, &err
, &expected
)) {
4116 if (!expected
.empty()) {
4117 cerr
<< err
<< std::endl
;
4118 cerr
<< "Expected one of the following:" << std::endl
;
4119 for (auto& exp
: expected
) {
4120 if (exp
== "*" || exp
== "[*]") {
4123 cerr
<< " " << exp
<< std::endl
;
4126 cerr
<< "Command not found:";
4127 for (auto& arg
: args
) {
4135 opt_cmd
= std::any_cast
<OPT
>(_opt_cmd
);
4137 /* some commands may have an optional extra param */
4138 if (!extra_args
.empty()) {
4140 case OPT::METADATA_GET
:
4141 case OPT::METADATA_PUT
:
4142 case OPT::METADATA_RM
:
4143 case OPT::METADATA_LIST
:
4144 metadata_key
= extra_args
[0];
4151 // not a raw op if 'period update' needs to commit to master
4152 bool raw_period_update
= opt_cmd
== OPT::PERIOD_UPDATE
&& !commit
;
4153 // not a raw op if 'period pull' needs to read zone/period configuration
4154 bool raw_period_pull
= opt_cmd
== OPT::PERIOD_PULL
&& !url
.empty();
4156 std::set
<OPT
> raw_storage_ops_list
= {OPT::ZONEGROUP_ADD
, OPT::ZONEGROUP_CREATE
,
4157 OPT::ZONEGROUP_DELETE
,
4158 OPT::ZONEGROUP_GET
, OPT::ZONEGROUP_LIST
,
4159 OPT::ZONEGROUP_SET
, OPT::ZONEGROUP_DEFAULT
,
4160 OPT::ZONEGROUP_RENAME
, OPT::ZONEGROUP_MODIFY
,
4161 OPT::ZONEGROUP_REMOVE
,
4162 OPT::ZONEGROUP_PLACEMENT_ADD
, OPT::ZONEGROUP_PLACEMENT_RM
,
4163 OPT::ZONEGROUP_PLACEMENT_MODIFY
, OPT::ZONEGROUP_PLACEMENT_LIST
,
4164 OPT::ZONEGROUP_PLACEMENT_GET
,
4165 OPT::ZONEGROUP_PLACEMENT_DEFAULT
,
4166 OPT::ZONE_CREATE
, OPT::ZONE_DELETE
,
4167 OPT::ZONE_GET
, OPT::ZONE_SET
, OPT::ZONE_RENAME
,
4168 OPT::ZONE_LIST
, OPT::ZONE_MODIFY
, OPT::ZONE_DEFAULT
,
4169 OPT::ZONE_PLACEMENT_ADD
, OPT::ZONE_PLACEMENT_RM
,
4170 OPT::ZONE_PLACEMENT_MODIFY
, OPT::ZONE_PLACEMENT_LIST
,
4171 OPT::ZONE_PLACEMENT_GET
,
4173 OPT::PERIOD_DELETE
, OPT::PERIOD_GET
,
4174 OPT::PERIOD_GET_CURRENT
, OPT::PERIOD_LIST
,
4175 OPT::GLOBAL_QUOTA_GET
, OPT::GLOBAL_QUOTA_SET
,
4176 OPT::GLOBAL_QUOTA_ENABLE
, OPT::GLOBAL_QUOTA_DISABLE
,
4177 OPT::GLOBAL_RATELIMIT_GET
, OPT::GLOBAL_RATELIMIT_SET
,
4178 OPT::GLOBAL_RATELIMIT_ENABLE
, OPT::GLOBAL_RATELIMIT_DISABLE
,
4179 OPT::REALM_DELETE
, OPT::REALM_GET
, OPT::REALM_LIST
,
4180 OPT::REALM_LIST_PERIODS
,
4181 OPT::REALM_GET_DEFAULT
,
4182 OPT::REALM_RENAME
, OPT::REALM_SET
,
4183 OPT::REALM_DEFAULT
, OPT::REALM_PULL
};
4185 std::set
<OPT
> readonly_ops_list
= {
4189 OPT::BUCKET_LIMIT_CHECK
,
4192 OPT::BUCKET_SYNC_CHECKPOINT
,
4193 OPT::BUCKET_SYNC_INFO
,
4194 OPT::BUCKET_SYNC_STATUS
,
4195 OPT::BUCKET_SYNC_MARKERS
,
4196 OPT::BUCKET_SHARD_OBJECTS
,
4197 OPT::BUCKET_OBJECT_SHARD
,
4208 OPT::ORPHANS_LIST_JOBS
,
4210 OPT::ZONEGROUP_LIST
,
4211 OPT::ZONEGROUP_PLACEMENT_LIST
,
4212 OPT::ZONEGROUP_PLACEMENT_GET
,
4215 OPT::ZONE_PLACEMENT_LIST
,
4216 OPT::ZONE_PLACEMENT_GET
,
4219 OPT::METADATA_SYNC_STATUS
,
4222 OPT::SYNC_ERROR_LIST
,
4223 OPT::SYNC_GROUP_GET
,
4224 OPT::SYNC_POLICY_GET
,
4227 OPT::DATA_SYNC_STATUS
,
4229 OPT::DATALOG_STATUS
,
4231 OPT::REALM_GET_DEFAULT
,
4233 OPT::REALM_LIST_PERIODS
,
4235 OPT::PERIOD_GET_CURRENT
,
4237 OPT::GLOBAL_QUOTA_GET
,
4238 OPT::GLOBAL_RATELIMIT_GET
,
4243 OPT::ROLE_POLICY_LIST
,
4244 OPT::ROLE_POLICY_GET
,
4246 OPT::RESHARD_STATUS
,
4247 OPT::PUBSUB_TOPIC_LIST
,
4248 OPT::PUBSUB_NOTIFICATION_LIST
,
4249 OPT::PUBSUB_TOPIC_GET
,
4250 OPT::PUBSUB_NOTIFICATION_GET
,
4254 std::set
<OPT
> gc_ops_list
= {
4258 OPT::BUCKET_RM
, // --purge-objects
4259 OPT::USER_RM
, // --purge-data
4260 OPT::OBJECTS_EXPIRE
,
4261 OPT::OBJECTS_EXPIRE_STALE_RM
,
4263 OPT::BUCKET_SYNC_RUN
,
4265 OPT::BUCKET_REWRITE
,
4269 raw_storage_op
= (raw_storage_ops_list
.find(opt_cmd
) != raw_storage_ops_list
.end() ||
4270 raw_period_update
|| raw_period_pull
);
4271 bool need_cache
= readonly_ops_list
.find(opt_cmd
) == readonly_ops_list
.end();
4272 bool need_gc
= (gc_ops_list
.find(opt_cmd
) != gc_ops_list
.end()) && !bypass_gc
;
4274 DriverManager::Config cfg
= DriverManager::get_config(true, g_ceph_context
);
4276 auto config_store_type
= g_conf().get_val
<std::string
>("rgw_config_store");
4277 cfgstore
= DriverManager::create_config_store(dpp(), config_store_type
);
4279 cerr
<< "couldn't init config storage provider" << std::endl
;
4283 if (raw_storage_op
) {
4284 driver
= DriverManager::get_raw_storage(dpp(),
4288 driver
= DriverManager::get_storage(dpp(),
4296 need_cache
&& g_conf()->rgw_cache_enabled
,
4300 cerr
<< "couldn't init storage provider" << std::endl
;
4304 /* Needs to be after the driver is initialized. Note, user could be empty here. */
4305 user
= driver
->get_user(user_id_arg
);
4307 init_optional_bucket(opt_bucket
, opt_tenant
,
4308 opt_bucket_name
, opt_bucket_id
);
4309 init_optional_bucket(opt_source_bucket
, opt_source_tenant
,
4310 opt_source_bucket_name
, opt_source_bucket_id
);
4311 init_optional_bucket(opt_dest_bucket
, opt_dest_tenant
,
4312 opt_dest_bucket_name
, opt_dest_bucket_id
);
4314 if (tenant
.empty()) {
4315 tenant
= user
->get_tenant();
4317 if (rgw::sal::User::empty(user
) && opt_cmd
!= OPT::ROLE_CREATE
4318 && opt_cmd
!= OPT::ROLE_DELETE
4319 && opt_cmd
!= OPT::ROLE_GET
4320 && opt_cmd
!= OPT::ROLE_TRUST_POLICY_MODIFY
4321 && opt_cmd
!= OPT::ROLE_LIST
4322 && opt_cmd
!= OPT::ROLE_POLICY_PUT
4323 && opt_cmd
!= OPT::ROLE_POLICY_LIST
4324 && opt_cmd
!= OPT::ROLE_POLICY_GET
4325 && opt_cmd
!= OPT::ROLE_POLICY_DELETE
4326 && opt_cmd
!= OPT::ROLE_UPDATE
4327 && opt_cmd
!= OPT::RESHARD_ADD
4328 && opt_cmd
!= OPT::RESHARD_CANCEL
4329 && opt_cmd
!= OPT::RESHARD_STATUS
4330 && opt_cmd
!= OPT::PUBSUB_TOPIC_LIST
4331 && opt_cmd
!= OPT::PUBSUB_NOTIFICATION_LIST
4332 && opt_cmd
!= OPT::PUBSUB_TOPIC_GET
4333 && opt_cmd
!= OPT::PUBSUB_NOTIFICATION_GET
4334 && opt_cmd
!= OPT::PUBSUB_TOPIC_RM
4335 && opt_cmd
!= OPT::PUBSUB_NOTIFICATION_RM
) {
4336 cerr
<< "ERROR: --tenant is set, but there's no user ID" << std::endl
;
4339 user
->set_tenant(tenant
);
4341 if (user_ns
.empty()) {
4342 user_ns
= user
->get_id().ns
;
4344 user
->set_ns(user_ns
);
4347 if (!new_user_id
.empty() && !tenant
.empty()) {
4348 new_user_id
.tenant
= tenant
;
4351 /* check key parameter conflict */
4352 if ((!access_key
.empty()) && gen_access_key
) {
4353 cerr
<< "ERROR: key parameter conflict, --access-key & --gen-access-key" << std::endl
;
4356 if ((!secret_key
.empty()) && gen_secret_key
) {
4357 cerr
<< "ERROR: key parameter conflict, --secret & --gen-secret" << std::endl
;
4362 // default to pretty json
4363 if (format
.empty()) {
4365 pretty_format
= true;
4368 if (format
== "xml")
4369 formatter
= make_unique
<XMLFormatter
>(new XMLFormatter(pretty_format
));
4370 else if (format
== "json")
4371 formatter
= make_unique
<JSONFormatter
>(new JSONFormatter(pretty_format
));
4373 cerr
<< "unrecognized format: " << format
<< std::endl
;
4377 zone_formatter
= std::make_unique
<JSONFormatter_PrettyZone
>(pretty_format
);
4379 realm_name
= g_conf()->rgw_realm
;
4380 zone_name
= g_conf()->rgw_zone
;
4381 zonegroup_name
= g_conf()->rgw_zonegroup
;
4383 if (!realm_name
.empty()) {
4384 opt_realm_name
= realm_name
;
4387 if (!zone_name
.empty()) {
4388 opt_zone_name
= zone_name
;
4391 if (!zonegroup_name
.empty()) {
4392 opt_zonegroup_name
= zonegroup_name
;
4395 RGWStreamFlusher
stream_flusher(formatter
.get(), cout
);
4397 RGWUserAdminOpState
user_op(driver
);
4398 if (!user_email
.empty()) {
4399 user_op
.user_email_specified
=true;
4402 if (!source_zone_name
.empty()) {
4403 std::unique_ptr
<rgw::sal::Zone
> zone
;
4404 if (driver
->get_zone()->get_zonegroup().get_zone_by_name(source_zone_name
, &zone
) < 0) {
4405 cerr
<< "WARNING: cannot find source zone id for name=" << source_zone_name
<< std::endl
;
4406 source_zone
= source_zone_name
;
4408 source_zone
.id
= zone
->get_id();
4412 rgw_http_client_init(g_ceph_context
);
4414 struct rgw_curl_setup
{
4416 rgw::curl::setup_curl(boost::none
);
4419 rgw::curl::cleanup_curl();
4425 StoreDestructor
store_destructor(driver
);
4427 if (raw_storage_op
) {
4429 case OPT::PERIOD_DELETE
:
4431 if (period_id
.empty()) {
4432 cerr
<< "missing period id" << std::endl
;
4435 int ret
= cfgstore
->delete_period(dpp(), null_yield
, period_id
);
4437 cerr
<< "ERROR: couldn't delete period: " << cpp_strerror(-ret
) << std::endl
;
4443 case OPT::PERIOD_GET
:
4445 std::optional
<epoch_t
> epoch
;
4446 if (!period_epoch
.empty()) {
4447 epoch
= atoi(period_epoch
.c_str());
4451 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4452 realm_id
, realm_name
, realm
);
4454 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4457 realm_id
= realm
.get_id();
4458 realm_name
= realm
.get_name();
4459 period_id
= RGWPeriod::get_staging_id(realm_id
);
4462 if (period_id
.empty()) {
4463 // use realm's current period
4465 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4466 realm_id
, realm_name
, realm
);
4468 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4471 period_id
= realm
.current_period
;
4475 int ret
= cfgstore
->read_period(dpp(), null_yield
, period_id
,
4478 cerr
<< "failed to load period: " << cpp_strerror(-ret
) << std::endl
;
4481 encode_json("period", period
, formatter
.get());
4482 formatter
->flush(cout
);
4485 case OPT::PERIOD_GET_CURRENT
:
4488 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4489 realm_id
, realm_name
, realm
);
4491 std::cerr
<< "failed to load realm: " << cpp_strerror(ret
) << std::endl
;
4495 formatter
->open_object_section("period_get_current");
4496 encode_json("current_period", realm
.current_period
, formatter
.get());
4497 formatter
->close_section();
4498 formatter
->flush(cout
);
4501 case OPT::PERIOD_LIST
:
4503 Formatter::ObjectSection periods_list
{*formatter
, "periods_list"};
4504 Formatter::ArraySection periods
{*formatter
, "periods"};
4505 rgw::sal::ListResult
<std::string
> listing
;
4506 std::array
<std::string
, 1000> period_ids
; // list in pages of 1000
4508 int ret
= cfgstore
->list_period_ids(dpp(), null_yield
, listing
.next
,
4509 period_ids
, listing
);
4511 std::cerr
<< "failed to list periods: " << cpp_strerror(-ret
) << std::endl
;
4514 for (const auto& id
: listing
.entries
) {
4515 encode_json("id", id
, formatter
.get());
4517 } while (!listing
.next
.empty());
4518 } // close sections periods and periods_list
4519 formatter
->flush(cout
);
4521 case OPT::PERIOD_UPDATE
:
4523 int ret
= update_period(cfgstore
.get(), realm_id
, realm_name
,
4524 period_epoch
, commit
, remote
, url
,
4525 opt_region
, access_key
, secret_key
,
4526 formatter
.get(), yes_i_really_mean_it
);
4532 case OPT::PERIOD_PULL
:
4534 boost::optional
<RGWRESTConn
> conn
;
4535 RGWRESTConn
*remote_conn
= nullptr;
4537 // load current period for endpoints
4539 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4540 realm_id
, realm_name
, realm
);
4542 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4545 period_id
= realm
.current_period
;
4547 RGWPeriod current_period
;
4548 ret
= cfgstore
->read_period(dpp(), null_yield
, period_id
,
4549 std::nullopt
, current_period
);
4551 cerr
<< "failed to load current period: " << cpp_strerror(-ret
) << std::endl
;
4554 if (remote
.empty()) {
4555 // use realm master zone as remote
4556 remote
= current_period
.get_master_zone().id
;
4558 conn
= get_remote_conn(static_cast<rgw::sal::RadosStore
*>(driver
), current_period
.get_map(), remote
);
4560 cerr
<< "failed to find a zone or zonegroup for remote "
4561 << remote
<< std::endl
;
4564 remote_conn
= &*conn
;
4568 int ret
= do_period_pull(cfgstore
.get(), remote_conn
, url
,
4569 opt_region
, access_key
, secret_key
,
4570 realm_id
, realm_name
, period_id
, period_epoch
,
4573 cerr
<< "period pull failed: " << cpp_strerror(-ret
) << std::endl
;
4577 encode_json("period", period
, formatter
.get());
4578 formatter
->flush(cout
);
4581 case OPT::GLOBAL_RATELIMIT_GET
:
4582 case OPT::GLOBAL_RATELIMIT_SET
:
4583 case OPT::GLOBAL_RATELIMIT_ENABLE
:
4584 case OPT::GLOBAL_RATELIMIT_DISABLE
:
4586 if (realm_id
.empty()) {
4587 if (!realm_name
.empty()) {
4588 // look up realm_id for the given realm_name
4589 int ret
= cfgstore
->read_realm_id(dpp(), null_yield
,
4590 realm_name
, realm_id
);
4592 cerr
<< "ERROR: failed to read realm for " << realm_name
4593 << ": " << cpp_strerror(-ret
) << std::endl
;
4597 // use default realm_id when none is given
4598 int ret
= cfgstore
->read_default_realm_id(dpp(), null_yield
,
4600 if (ret
< 0 && ret
!= -ENOENT
) { // on ENOENT, use empty realm_id
4601 cerr
<< "ERROR: failed to read default realm: "
4602 << cpp_strerror(-ret
) << std::endl
;
4608 RGWPeriodConfig period_config
;
4609 int ret
= cfgstore
->read_period_config(dpp(), null_yield
, realm_id
,
4611 if (ret
< 0 && ret
!= -ENOENT
) {
4612 cerr
<< "ERROR: failed to read period config: "
4613 << cpp_strerror(-ret
) << std::endl
;
4616 bool ratelimit_configured
= true;
4617 formatter
->open_object_section("period_config");
4618 if (ratelimit_scope
== "bucket") {
4619 ratelimit_configured
= set_ratelimit_info(period_config
.bucket_ratelimit
, opt_cmd
,
4620 max_read_ops
, max_write_ops
,
4621 max_read_bytes
, max_write_bytes
,
4622 have_max_read_ops
, have_max_write_ops
,
4623 have_max_read_bytes
, have_max_write_bytes
);
4624 encode_json("bucket_ratelimit", period_config
.bucket_ratelimit
, formatter
.get());
4625 } else if (ratelimit_scope
== "user") {
4626 ratelimit_configured
= set_ratelimit_info(period_config
.user_ratelimit
, opt_cmd
,
4627 max_read_ops
, max_write_ops
,
4628 max_read_bytes
, max_write_bytes
,
4629 have_max_read_ops
, have_max_write_ops
,
4630 have_max_read_bytes
, have_max_write_bytes
);
4631 encode_json("user_ratelimit", period_config
.user_ratelimit
, formatter
.get());
4632 } else if (ratelimit_scope
== "anonymous") {
4633 ratelimit_configured
= set_ratelimit_info(period_config
.anon_ratelimit
, opt_cmd
,
4634 max_read_ops
, max_write_ops
,
4635 max_read_bytes
, max_write_bytes
,
4636 have_max_read_ops
, have_max_write_ops
,
4637 have_max_read_bytes
, have_max_write_bytes
);
4638 encode_json("anonymous_ratelimit", period_config
.anon_ratelimit
, formatter
.get());
4639 } else if (ratelimit_scope
.empty() && opt_cmd
== OPT::GLOBAL_RATELIMIT_GET
) {
4640 // if no scope is given for GET, print both
4641 encode_json("bucket_ratelimit", period_config
.bucket_ratelimit
, formatter
.get());
4642 encode_json("user_ratelimit", period_config
.user_ratelimit
, formatter
.get());
4643 encode_json("anonymous_ratelimit", period_config
.anon_ratelimit
, formatter
.get());
4645 cerr
<< "ERROR: invalid rate limit scope specification. Please specify "
4646 "either --ratelimit-scope=bucket, or --ratelimit-scope=user or --ratelimit-scope=anonymous" << std::endl
;
4649 if (!ratelimit_configured
) {
4650 cerr
<< "ERROR: no rate limit values have been specified" << std::endl
;
4654 formatter
->close_section();
4656 if (opt_cmd
!= OPT::GLOBAL_RATELIMIT_GET
) {
4657 // write the modified period config
4658 constexpr bool exclusive
= false;
4659 ret
= cfgstore
->write_period_config(dpp(), null_yield
, exclusive
,
4660 realm_id
, period_config
);
4662 cerr
<< "ERROR: failed to write period config: "
4663 << cpp_strerror(-ret
) << std::endl
;
4666 if (!realm_id
.empty()) {
4667 cout
<< "Global ratelimit changes saved. Use 'period update' to apply "
4668 "them to the staging period, and 'period commit' to commit the "
4669 "new period." << std::endl
;
4671 cout
<< "Global ratelimit changes saved. They will take effect as "
4672 "the gateways are restarted." << std::endl
;
4676 formatter
->flush(cout
);
4679 case OPT::GLOBAL_QUOTA_GET
:
4680 case OPT::GLOBAL_QUOTA_SET
:
4681 case OPT::GLOBAL_QUOTA_ENABLE
:
4682 case OPT::GLOBAL_QUOTA_DISABLE
:
4684 if (realm_id
.empty()) {
4685 if (!realm_name
.empty()) {
4686 // look up realm_id for the given realm_name
4687 int ret
= cfgstore
->read_realm_id(dpp(), null_yield
,
4688 realm_name
, realm_id
);
4690 cerr
<< "ERROR: failed to read realm for " << realm_name
4691 << ": " << cpp_strerror(-ret
) << std::endl
;
4695 // use default realm_id when none is given
4696 int ret
= cfgstore
->read_default_realm_id(dpp(), null_yield
,
4698 if (ret
< 0 && ret
!= -ENOENT
) { // on ENOENT, use empty realm_id
4699 cerr
<< "ERROR: failed to read default realm: "
4700 << cpp_strerror(-ret
) << std::endl
;
4706 RGWPeriodConfig period_config
;
4707 int ret
= cfgstore
->read_period_config(dpp(), null_yield
, realm_id
,
4709 if (ret
< 0 && ret
!= -ENOENT
) {
4710 cerr
<< "ERROR: failed to read period config: "
4711 << cpp_strerror(-ret
) << std::endl
;
4715 formatter
->open_object_section("period_config");
4716 if (quota_scope
== "bucket") {
4717 set_quota_info(period_config
.quota
.bucket_quota
, opt_cmd
,
4718 max_size
, max_objects
,
4719 have_max_size
, have_max_objects
);
4720 encode_json("bucket quota", period_config
.quota
.bucket_quota
, formatter
.get());
4721 } else if (quota_scope
== "user") {
4722 set_quota_info(period_config
.quota
.user_quota
, opt_cmd
,
4723 max_size
, max_objects
,
4724 have_max_size
, have_max_objects
);
4725 encode_json("user quota", period_config
.quota
.user_quota
, formatter
.get());
4726 } else if (quota_scope
.empty() && opt_cmd
== OPT::GLOBAL_QUOTA_GET
) {
4727 // if no scope is given for GET, print both
4728 encode_json("bucket quota", period_config
.quota
.bucket_quota
, formatter
.get());
4729 encode_json("user quota", period_config
.quota
.user_quota
, formatter
.get());
4731 cerr
<< "ERROR: invalid quota scope specification. Please specify "
4732 "either --quota-scope=bucket, or --quota-scope=user" << std::endl
;
4735 formatter
->close_section();
4737 if (opt_cmd
!= OPT::GLOBAL_QUOTA_GET
) {
4738 // write the modified period config
4739 constexpr bool exclusive
= false;
4740 ret
= cfgstore
->write_period_config(dpp(), null_yield
, exclusive
,
4741 realm_id
, period_config
);
4743 cerr
<< "ERROR: failed to write period config: "
4744 << cpp_strerror(-ret
) << std::endl
;
4747 if (!realm_id
.empty()) {
4748 cout
<< "Global quota changes saved. Use 'period update' to apply "
4749 "them to the staging period, and 'period commit' to commit the "
4750 "new period." << std::endl
;
4752 cout
<< "Global quota changes saved. They will take effect as "
4753 "the gateways are restarted." << std::endl
;
4757 formatter
->flush(cout
);
4760 case OPT::REALM_CREATE
:
4762 if (realm_name
.empty()) {
4763 cerr
<< "missing realm name" << std::endl
;
4768 realm
.name
= realm_name
;
4770 constexpr bool exclusive
= true;
4771 int ret
= rgw::create_realm(dpp(), null_yield
, cfgstore
.get(),
4774 cerr
<< "ERROR: couldn't create realm " << realm_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
4779 ret
= rgw::set_default_realm(dpp(), null_yield
, cfgstore
.get(), realm
);
4781 cerr
<< "failed to set realm " << realm_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
4785 encode_json("realm", realm
, formatter
.get());
4786 formatter
->flush(cout
);
4789 case OPT::REALM_DELETE
:
4791 if (realm_id
.empty() && realm_name
.empty()) {
4792 cerr
<< "missing realm name or id" << std::endl
;
4796 std::unique_ptr
<rgw::sal::RealmWriter
> writer
;
4797 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4798 realm_id
, realm_name
, realm
, &writer
);
4800 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4803 ret
= writer
->remove(dpp(), null_yield
);
4805 cerr
<< "failed to remove realm: " << cpp_strerror(-ret
) << std::endl
;
4811 case OPT::REALM_GET
:
4814 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4815 realm_id
, realm_name
, realm
);
4817 if (ret
== -ENOENT
&& realm_name
.empty() && realm_id
.empty()) {
4818 cerr
<< "missing realm name or id, or default realm not found" << std::endl
;
4820 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4824 encode_json("realm", realm
, formatter
.get());
4825 formatter
->flush(cout
);
4828 case OPT::REALM_GET_DEFAULT
:
4831 int ret
= cfgstore
->read_default_realm_id(dpp(), null_yield
, default_id
);
4832 if (ret
== -ENOENT
) {
4833 cout
<< "No default realm is set" << std::endl
;
4835 } else if (ret
< 0) {
4836 cerr
<< "Error reading default realm: " << cpp_strerror(-ret
) << std::endl
;
4839 cout
<< "default realm: " << default_id
<< std::endl
;
4842 case OPT::REALM_LIST
:
4844 std::string default_id
;
4845 int ret
= cfgstore
->read_default_realm_id(dpp(), null_yield
,
4847 if (ret
< 0 && ret
!= -ENOENT
) {
4848 cerr
<< "could not determine default realm: " << cpp_strerror(-ret
) << std::endl
;
4851 Formatter::ObjectSection realms_list
{*formatter
, "realms_list"};
4852 encode_json("default_info", default_id
, formatter
.get());
4854 Formatter::ArraySection realms
{*formatter
, "realms"};
4855 rgw::sal::ListResult
<std::string
> listing
;
4856 std::array
<std::string
, 1000> names
; // list in pages of 1000
4858 ret
= cfgstore
->list_realm_names(dpp(), null_yield
, listing
.next
,
4861 std::cerr
<< "failed to list realms: " << cpp_strerror(-ret
) << std::endl
;
4864 for (const auto& name
: listing
.entries
) {
4865 encode_json("name", name
, formatter
.get());
4867 } while (!listing
.next
.empty());
4868 } // close sections realms and realms_list
4869 formatter
->flush(cout
);
4871 case OPT::REALM_LIST_PERIODS
:
4873 // use realm's current period
4875 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4876 realm_id
, realm_name
, realm
);
4878 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4881 period_id
= realm
.current_period
;
4883 Formatter::ObjectSection periods_list
{*formatter
, "realm_periods_list"};
4884 encode_json("current_period", period_id
, formatter
.get());
4886 Formatter::ArraySection periods
{*formatter
, "periods"};
4888 while (!period_id
.empty()) {
4890 ret
= cfgstore
->read_period(dpp(), null_yield
, period_id
,
4891 std::nullopt
, period
);
4893 cerr
<< "failed to load period id " << period_id
4894 << ": " << cpp_strerror(-ret
) << std::endl
;
4897 encode_json("id", period_id
, formatter
.get());
4898 period_id
= period
.predecessor_uuid
;
4900 } // close sections periods and realm_periods_list
4901 formatter
->flush(cout
);
4904 case OPT::REALM_RENAME
:
4906 if (realm_new_name
.empty()) {
4907 cerr
<< "missing realm new name" << std::endl
;
4910 if (realm_name
.empty() && realm_id
.empty()) {
4911 cerr
<< "missing realm name or id" << std::endl
;
4916 std::unique_ptr
<rgw::sal::RealmWriter
> writer
;
4917 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4918 realm_id
, realm_name
, realm
, &writer
);
4920 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4923 ret
= writer
->rename(dpp(), null_yield
, realm
, realm_new_name
);
4925 cerr
<< "rename failed: " << cpp_strerror(-ret
) << std::endl
;
4928 cout
<< "Realm name updated. Note that this change only applies to "
4929 "the current cluster, so this command must be run separately "
4930 "on each of the realm's other clusters." << std::endl
;
4933 case OPT::REALM_SET
:
4935 if (realm_id
.empty() && realm_name
.empty()) {
4936 cerr
<< "no realm name or id provided" << std::endl
;
4939 bool new_realm
= false;
4941 std::unique_ptr
<rgw::sal::RealmWriter
> writer
;
4942 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4943 realm_id
, realm_name
, realm
, &writer
);
4944 if (ret
< 0 && ret
!= -ENOENT
) {
4945 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
4947 } else if (ret
== -ENOENT
) {
4950 ret
= read_decode_json(infile
, realm
);
4954 if (!realm_name
.empty() && realm
.get_name() != realm_name
) {
4955 cerr
<< "mismatch between --rgw-realm " << realm_name
<< " and json input file name " <<
4956 realm
.get_name() << std::endl
;
4961 cout
<< "clearing period and epoch for new realm" << std::endl
;
4962 realm
.clear_current_period_and_epoch();
4963 constexpr bool exclusive
= true;
4964 ret
= rgw::create_realm(dpp(), null_yield
, cfgstore
.get(),
4967 cerr
<< "ERROR: couldn't create new realm: " << cpp_strerror(-ret
) << std::endl
;
4971 ret
= writer
->write(dpp(), null_yield
, realm
);
4973 cerr
<< "ERROR: couldn't driver realm info: " << cpp_strerror(-ret
) << std::endl
;
4979 ret
= rgw::set_default_realm(dpp(), null_yield
, cfgstore
.get(), realm
);
4981 cerr
<< "failed to set realm " << realm_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
4984 encode_json("realm", realm
, formatter
.get());
4985 formatter
->flush(cout
);
4989 case OPT::REALM_DEFAULT
:
4992 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4993 realm_id
, realm_name
, realm
);
4995 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4998 ret
= rgw::set_default_realm(dpp(), null_yield
, cfgstore
.get(), realm
);
5000 cerr
<< "failed to set realm as default: " << cpp_strerror(-ret
) << std::endl
;
5005 case OPT::REALM_PULL
:
5008 cerr
<< "A --url must be provided." << std::endl
;
5012 req_info
info(g_ceph_context
, &env
);
5013 info
.method
= "GET";
5014 info
.request_uri
= "/admin/realm";
5016 map
<string
, string
> ¶ms
= info
.args
.get_params();
5017 if (!realm_id
.empty())
5018 params
["id"] = realm_id
;
5019 if (!realm_name
.empty())
5020 params
["name"] = realm_name
;
5024 int ret
= send_to_url(url
, opt_region
, access_key
, secret_key
, info
, bl
, p
);
5026 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
5027 if (ret
== -EACCES
) {
5028 cerr
<< "If the realm has been changed on the master zone, the "
5029 "master zone's gateway may need to be restarted to recognize "
5030 "this user." << std::endl
;
5036 decode_json_obj(realm
, &p
);
5037 } catch (const JSONDecoder::err
& e
) {
5038 cerr
<< "failed to decode JSON response: " << e
.what() << std::endl
;
5042 auto& current_period
= realm
.get_current_period();
5043 if (!current_period
.empty()) {
5044 // pull the latest epoch of the realm's current period
5045 ret
= do_period_pull(cfgstore
.get(), nullptr, url
, opt_region
,
5046 access_key
, secret_key
,
5047 realm_id
, realm_name
, current_period
, "",
5050 cerr
<< "could not fetch period " << current_period
<< std::endl
;
5054 constexpr bool exclusive
= false;
5055 ret
= rgw::create_realm(dpp(), null_yield
, cfgstore
.get(),
5058 cerr
<< "Error storing realm " << realm
.get_id() << ": "
5059 << cpp_strerror(ret
) << std::endl
;
5064 ret
= rgw::set_default_realm(dpp(), null_yield
, cfgstore
.get(), realm
);
5066 cerr
<< "failed to set realm " << realm_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5070 encode_json("realm", realm
, formatter
.get());
5071 formatter
->flush(cout
);
5075 case OPT::ZONEGROUP_ADD
:
5077 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
5078 cerr
<< "no zonegroup name or id provided" << std::endl
;
5082 // load the zonegroup and zone params
5083 RGWZoneGroup zonegroup
;
5084 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> zonegroup_writer
;
5085 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5086 zonegroup_id
, zonegroup_name
,
5087 zonegroup
, &zonegroup_writer
);
5089 cerr
<< "failed to load zonegroup " << zonegroup_name
<< " id "
5090 << zonegroup_id
<< ": " << cpp_strerror(-ret
) << std::endl
;
5094 RGWZoneParams zone_params
;
5095 std::unique_ptr
<rgw::sal::ZoneWriter
> zone_writer
;
5096 ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5097 zone_id
, zone_name
, zone_params
, &zone_writer
);
5099 cerr
<< "unable to load zone: " << cpp_strerror(-ret
) << std::endl
;
5103 // update zone_params if necessary
5104 bool need_zone_update
= false;
5106 if (zone_params
.realm_id
!= zonegroup
.realm_id
) {
5107 if (!zone_params
.realm_id
.empty()) {
5108 cerr
<< "WARNING: overwriting zone realm_id=" << zone_params
.realm_id
5109 << " to match zonegroup realm_id=" << zonegroup
.realm_id
<< std::endl
;
5111 zone_params
.realm_id
= zonegroup
.realm_id
;
5112 need_zone_update
= true;
5115 for (auto a
: tier_config_add
) {
5116 ret
= zone_params
.tier_config
.set(a
.first
, a
.second
);
5118 cerr
<< "ERROR: failed to set configurable: " << a
<< std::endl
;
5121 need_zone_update
= true;
5124 if (need_zone_update
) {
5125 ret
= zone_writer
->write(dpp(), null_yield
, zone_params
);
5127 cerr
<< "failed to save zone info: " << cpp_strerror(-ret
) << std::endl
;
5132 const bool *pis_master
= (is_master_set
? &is_master
: nullptr);
5133 const bool *pread_only
= (is_read_only_set
? &read_only
: nullptr);
5134 const bool *psync_from_all
= (sync_from_all_specified
? &sync_from_all
: nullptr);
5135 const string
*predirect_zone
= (redirect_zone_set
? &redirect_zone
: nullptr);
5137 // validate --tier-type if specified
5138 const string
*ptier_type
= (tier_type_specified
? &tier_type
: nullptr);
5140 auto sync_mgr
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->sync_modules
->get_manager();
5141 if (!sync_mgr
->get_module(*ptier_type
, nullptr)) {
5142 ldpp_dout(dpp(), -1) << "ERROR: could not find sync module: "
5143 << *ptier_type
<< ", valid sync modules: "
5144 << sync_mgr
->get_registered_module_names() << dendl
;
5149 if (enable_features
.empty()) { // enable all features by default
5150 enable_features
.insert(rgw::zone_features::supported
.begin(),
5151 rgw::zone_features::supported
.end());
5154 // add/update the public zone information stored in the zonegroup
5155 ret
= rgw::add_zone_to_group(dpp(), zonegroup
, zone_params
,
5156 pis_master
, pread_only
, endpoints
,
5157 ptier_type
, psync_from_all
,
5158 sync_from
, sync_from_rm
,
5159 predirect_zone
, bucket_index_max_shards
,
5160 enable_features
, disable_features
);
5165 // write the updated zonegroup
5166 ret
= zonegroup_writer
->write(dpp(), null_yield
, zonegroup
);
5168 cerr
<< "failed to write updated zonegroup " << zonegroup
.get_name()
5169 << ": " << cpp_strerror(-ret
) << std::endl
;
5173 encode_json("zonegroup", zonegroup
, formatter
.get());
5174 formatter
->flush(cout
);
5177 case OPT::ZONEGROUP_CREATE
:
5179 if (zonegroup_name
.empty()) {
5180 cerr
<< "Missing zonegroup name" << std::endl
;
5184 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
5185 realm_id
, realm_name
, realm
);
5187 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
5191 RGWZoneGroup zonegroup
;
5192 zonegroup
.name
= zonegroup_name
;
5193 zonegroup
.is_master
= is_master
;
5194 zonegroup
.realm_id
= realm
.get_id();
5195 zonegroup
.endpoints
= endpoints
;
5196 zonegroup
.api_name
= (api_name
.empty() ? zonegroup_name
: api_name
);
5198 zonegroup
.enabled_features
= enable_features
;
5199 if (zonegroup
.enabled_features
.empty()) { // enable features by default
5200 zonegroup
.enabled_features
.insert(rgw::zone_features::enabled
.begin(),
5201 rgw::zone_features::enabled
.end());
5203 for (const auto& feature
: disable_features
) {
5204 auto i
= zonegroup
.enabled_features
.find(feature
);
5205 if (i
== zonegroup
.enabled_features
.end()) {
5206 ldout(cct
, 1) << "WARNING: zone feature \"" << feature
5207 << "\" was not enabled in zonegroup " << zonegroup_name
<< dendl
;
5210 zonegroup
.enabled_features
.erase(i
);
5213 constexpr bool exclusive
= true;
5214 ret
= rgw::create_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5215 exclusive
, zonegroup
);
5217 cerr
<< "failed to create zonegroup " << zonegroup_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
5222 ret
= rgw::set_default_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5225 cerr
<< "failed to set zonegroup " << zonegroup_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5229 encode_json("zonegroup", zonegroup
, formatter
.get());
5230 formatter
->flush(cout
);
5233 case OPT::ZONEGROUP_DEFAULT
:
5235 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
5236 cerr
<< "no zonegroup name or id provided" << std::endl
;
5240 RGWZoneGroup zonegroup
;
5241 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5242 zonegroup_id
, zonegroup_name
,
5245 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5249 ret
= rgw::set_default_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5252 cerr
<< "failed to set zonegroup as default: " << cpp_strerror(-ret
) << std::endl
;
5257 case OPT::ZONEGROUP_DELETE
:
5259 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
5260 cerr
<< "no zonegroup name or id provided" << std::endl
;
5263 RGWZoneGroup zonegroup
;
5264 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> writer
;
5265 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5266 zonegroup_id
, zonegroup_name
,
5267 zonegroup
, &writer
);
5269 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5272 ret
= writer
->remove(dpp(), null_yield
);
5274 cerr
<< "ERROR: couldn't delete zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5279 case OPT::ZONEGROUP_GET
:
5281 RGWZoneGroup zonegroup
;
5282 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5283 zonegroup_id
, zonegroup_name
, zonegroup
);
5285 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5289 encode_json("zonegroup", zonegroup
, formatter
.get());
5290 formatter
->flush(cout
);
5293 case OPT::ZONEGROUP_LIST
:
5295 RGWZoneGroup default_zonegroup
;
5296 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5297 {}, {}, default_zonegroup
);
5298 if (ret
< 0 && ret
!= -ENOENT
) {
5299 cerr
<< "could not determine default zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5302 Formatter::ObjectSection zonegroups_list
{*formatter
, "zonegroups_list"};
5303 encode_json("default_info", default_zonegroup
.id
, formatter
.get());
5305 Formatter::ArraySection zonegroups
{*formatter
, "zonegroups"};
5306 rgw::sal::ListResult
<std::string
> listing
;
5307 std::array
<std::string
, 1000> names
; // list in pages of 1000
5309 ret
= cfgstore
->list_zonegroup_names(dpp(), null_yield
, listing
.next
,
5312 std::cerr
<< "failed to list zonegroups: " << cpp_strerror(-ret
) << std::endl
;
5315 for (const auto& name
: listing
.entries
) {
5316 encode_json("name", name
, formatter
.get());
5318 } while (!listing
.next
.empty());
5319 } // close sections zonegroups and zonegroups_list
5320 formatter
->flush(cout
);
5322 case OPT::ZONEGROUP_MODIFY
:
5324 RGWZoneGroup zonegroup
;
5325 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> writer
;
5326 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5327 zonegroup_id
, zonegroup_name
,
5328 zonegroup
, &writer
);
5330 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5334 bool need_update
= false;
5336 if (!master_zone
.empty()) {
5337 zonegroup
.master_zone
= master_zone
;
5341 if (is_master_set
) {
5342 zonegroup
.is_master
= is_master
;
5346 if (!endpoints
.empty()) {
5347 zonegroup
.endpoints
= endpoints
;
5351 if (!api_name
.empty()) {
5352 zonegroup
.api_name
= api_name
;
5356 if (!realm_id
.empty()) {
5357 zonegroup
.realm_id
= realm_id
;
5359 } else if (!realm_name
.empty()) {
5360 // get realm id from name
5361 ret
= cfgstore
->read_realm_id(dpp(), null_yield
, realm_name
,
5362 zonegroup
.realm_id
);
5364 cerr
<< "failed to find realm by name " << realm_name
<< std::endl
;
5370 if (bucket_index_max_shards
) {
5371 for (auto& [name
, zone
] : zonegroup
.zones
) {
5372 zone
.bucket_index_max_shards
= *bucket_index_max_shards
;
5377 for (const auto& feature
: enable_features
) {
5378 zonegroup
.enabled_features
.insert(feature
);
5381 for (const auto& feature
: disable_features
) {
5382 auto i
= zonegroup
.enabled_features
.find(feature
);
5383 if (i
== zonegroup
.enabled_features
.end()) {
5384 ldout(cct
, 1) << "WARNING: zone feature \"" << feature
5385 << "\" was not enabled in zonegroup "
5386 << zonegroup
.get_name() << dendl
;
5389 zonegroup
.enabled_features
.erase(i
);
5394 ret
= writer
->write(dpp(), null_yield
, zonegroup
);
5396 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5402 ret
= rgw::set_default_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5405 cerr
<< "failed to set zonegroup " << zonegroup_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5409 encode_json("zonegroup", zonegroup
, formatter
.get());
5410 formatter
->flush(cout
);
5413 case OPT::ZONEGROUP_SET
:
5416 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
5417 realm_id
, realm_name
, realm
);
5418 bool default_realm_not_exist
= (ret
== -ENOENT
&& realm_id
.empty() && realm_name
.empty());
5420 if (ret
< 0 && !default_realm_not_exist
) {
5421 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
5425 RGWZoneGroup zonegroup
;
5426 ret
= read_decode_json(infile
, zonegroup
);
5430 if (zonegroup
.realm_id
.empty() && !default_realm_not_exist
) {
5431 zonegroup
.realm_id
= realm
.get_id();
5433 // validate zonegroup features
5434 for (const auto& feature
: zonegroup
.enabled_features
) {
5435 if (!rgw::zone_features::supports(feature
)) {
5436 std::cerr
<< "ERROR: Unrecognized zonegroup feature \""
5437 << feature
<< "\"" << std::endl
;
5441 for (const auto& [name
, zone
] : zonegroup
.zones
) {
5442 // validate zone features
5443 for (const auto& feature
: zone
.supported_features
) {
5444 if (!rgw::zone_features::supports(feature
)) {
5445 std::cerr
<< "ERROR: Unrecognized zone feature \""
5446 << feature
<< "\" in zone " << zone
.name
<< std::endl
;
5450 // zone must support everything zonegroup does
5451 for (const auto& feature
: zonegroup
.enabled_features
) {
5452 if (!zone
.supports(feature
)) {
5453 std::cerr
<< "ERROR: Zone " << name
<< " does not support feature \""
5454 << feature
<< "\" required by zonegroup" << std::endl
;
5460 // create/overwrite the zonegroup info
5461 constexpr bool exclusive
= false;
5462 ret
= rgw::create_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5463 exclusive
, zonegroup
);
5465 cerr
<< "ERROR: couldn't create zonegroup info: " << cpp_strerror(-ret
) << std::endl
;
5470 ret
= rgw::set_default_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5473 cerr
<< "failed to set zonegroup " << zonegroup_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5477 encode_json("zonegroup", zonegroup
, formatter
.get());
5478 formatter
->flush(cout
);
5481 case OPT::ZONEGROUP_REMOVE
:
5483 RGWZoneGroup zonegroup
;
5484 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> writer
;
5485 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5486 zonegroup_id
, zonegroup_name
,
5487 zonegroup
, &writer
);
5489 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5493 if (zone_id
.empty()) {
5494 if (zone_name
.empty()) {
5495 cerr
<< "no --zone-id or --rgw-zone name provided" << std::endl
;
5498 // look up zone id by name
5499 for (auto& z
: zonegroup
.zones
) {
5500 if (zone_name
== z
.second
.name
) {
5501 zone_id
= z
.second
.id
;
5505 if (zone_id
.empty()) {
5506 cerr
<< "zone name " << zone_name
<< " not found in zonegroup "
5507 << zonegroup
.get_name() << std::endl
;
5512 ret
= rgw::remove_zone_from_group(dpp(), zonegroup
, zone_id
);
5514 cerr
<< "failed to remove zone: " << cpp_strerror(-ret
) << std::endl
;
5518 ret
= writer
->write(dpp(), null_yield
, zonegroup
);
5520 cerr
<< "failed to write zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5524 encode_json("zonegroup", zonegroup
, formatter
.get());
5525 formatter
->flush(cout
);
5528 case OPT::ZONEGROUP_RENAME
:
5530 if (zonegroup_new_name
.empty()) {
5531 cerr
<< " missing zonegroup new name" << std::endl
;
5534 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
5535 cerr
<< "no zonegroup name or id provided" << std::endl
;
5538 RGWZoneGroup zonegroup
;
5539 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> writer
;
5540 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5541 zonegroup_id
, zonegroup_name
,
5542 zonegroup
, &writer
);
5544 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5547 ret
= writer
->rename(dpp(), null_yield
, zonegroup
, zonegroup_new_name
);
5549 cerr
<< "failed to rename zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5554 case OPT::ZONEGROUP_PLACEMENT_LIST
:
5556 RGWZoneGroup zonegroup
;
5557 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5558 zonegroup_id
, zonegroup_name
, zonegroup
);
5560 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5564 encode_json("placement_targets", zonegroup
.placement_targets
, formatter
.get());
5565 formatter
->flush(cout
);
5568 case OPT::ZONEGROUP_PLACEMENT_GET
:
5570 if (placement_id
.empty()) {
5571 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
5575 RGWZoneGroup zonegroup
;
5576 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5577 zonegroup_id
, zonegroup_name
, zonegroup
);
5579 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5583 auto p
= zonegroup
.placement_targets
.find(placement_id
);
5584 if (p
== zonegroup
.placement_targets
.end()) {
5585 cerr
<< "failed to find a zonegroup placement target named '" << placement_id
<< "'" << std::endl
;
5588 encode_json("placement_targets", p
->second
, formatter
.get());
5589 formatter
->flush(cout
);
5592 case OPT::ZONEGROUP_PLACEMENT_ADD
:
5593 case OPT::ZONEGROUP_PLACEMENT_MODIFY
:
5594 case OPT::ZONEGROUP_PLACEMENT_RM
:
5595 case OPT::ZONEGROUP_PLACEMENT_DEFAULT
:
5597 if (placement_id
.empty()) {
5598 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
5602 rgw_placement_rule rule
;
5603 rule
.from_str(placement_id
);
5605 if (!rule
.storage_class
.empty() && opt_storage_class
&&
5606 rule
.storage_class
!= *opt_storage_class
) {
5607 cerr
<< "ERROR: provided contradicting storage class configuration" << std::endl
;
5609 } else if (rule
.storage_class
.empty()) {
5610 rule
.storage_class
= opt_storage_class
.value_or(string());
5613 RGWZoneGroup zonegroup
;
5614 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> writer
;
5615 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5616 zonegroup_id
, zonegroup_name
,
5617 zonegroup
, &writer
);
5619 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5623 if (opt_cmd
== OPT::ZONEGROUP_PLACEMENT_ADD
||
5624 opt_cmd
== OPT::ZONEGROUP_PLACEMENT_MODIFY
) {
5625 RGWZoneGroupPlacementTarget
& target
= zonegroup
.placement_targets
[placement_id
];
5626 if (!tags
.empty()) {
5627 target
.tags
.clear();
5628 for (auto& t
: tags
) {
5629 target
.tags
.insert(t
);
5633 target
.name
= placement_id
;
5634 for (auto& t
: tags_rm
) {
5635 target
.tags
.erase(t
);
5637 for (auto& t
: tags_add
) {
5638 target
.tags
.insert(t
);
5640 target
.storage_classes
.insert(rule
.get_storage_class());
5643 bool tier_class
= false;
5644 std::string storage_class
= rule
.get_storage_class();
5645 RGWZoneGroupPlacementTier t
{storage_class
};
5646 RGWZoneGroupPlacementTier
*pt
= &t
;
5648 auto ptiter
= target
.tier_targets
.find(storage_class
);
5649 if (ptiter
!= target
.tier_targets
.end()) {
5650 pt
= &ptiter
->second
;
5652 } else if (tier_type_specified
) {
5653 if (tier_type
== "cloud-s3") {
5654 /* we support only cloud-s3 tier-type for now.
5655 * Once set cant be reset. */
5657 pt
->tier_type
= tier_type
;
5658 pt
->storage_class
= storage_class
;
5660 cerr
<< "ERROR: Invalid tier-type specified" << std::endl
;
5666 if (tier_config_add
.size() > 0) {
5667 JSONFormattable tconfig
;
5668 for (auto add
: tier_config_add
) {
5669 int r
= tconfig
.set(add
.first
, add
.second
);
5671 cerr
<< "ERROR: failed to set configurable: " << add
<< std::endl
;
5675 int r
= pt
->update_params(tconfig
);
5677 cerr
<< "ERROR: failed to update tier_config options"<< std::endl
;
5680 if (tier_config_rm
.size() > 0) {
5681 JSONFormattable tconfig
;
5682 for (auto add
: tier_config_rm
) {
5683 int r
= tconfig
.set(add
.first
, add
.second
);
5685 cerr
<< "ERROR: failed to set configurable: " << add
<< std::endl
;
5689 int r
= pt
->clear_params(tconfig
);
5691 cerr
<< "ERROR: failed to update tier_config options"<< std::endl
;
5695 target
.tier_targets
.emplace(std::make_pair(storage_class
, *pt
));
5698 if (zonegroup
.default_placement
.empty()) {
5699 zonegroup
.default_placement
.init(rule
.name
, RGW_STORAGE_CLASS_STANDARD
);
5701 } else if (opt_cmd
== OPT::ZONEGROUP_PLACEMENT_RM
) {
5702 if (!opt_storage_class
|| opt_storage_class
->empty()) {
5703 zonegroup
.placement_targets
.erase(placement_id
);
5704 if (zonegroup
.default_placement
.name
== placement_id
) {
5705 // clear default placement
5706 zonegroup
.default_placement
.clear();
5709 auto iter
= zonegroup
.placement_targets
.find(placement_id
);
5710 if (iter
!= zonegroup
.placement_targets
.end()) {
5711 RGWZoneGroupPlacementTarget
& info
= zonegroup
.placement_targets
[placement_id
];
5712 info
.storage_classes
.erase(*opt_storage_class
);
5714 if (zonegroup
.default_placement
== rule
) {
5715 // clear default storage class
5716 zonegroup
.default_placement
.storage_class
.clear();
5719 auto ptiter
= info
.tier_targets
.find(*opt_storage_class
);
5720 if (ptiter
!= info
.tier_targets
.end()) {
5721 info
.tier_targets
.erase(ptiter
);
5725 } else if (opt_cmd
== OPT::ZONEGROUP_PLACEMENT_DEFAULT
) {
5726 if (!zonegroup
.placement_targets
.count(placement_id
)) {
5727 cerr
<< "failed to find a zonegroup placement target named '"
5728 << placement_id
<< "'" << std::endl
;
5731 zonegroup
.default_placement
= rule
;
5734 ret
= writer
->write(dpp(), null_yield
, zonegroup
);
5736 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5740 encode_json("placement_targets", zonegroup
.placement_targets
, formatter
.get());
5741 formatter
->flush(cout
);
5744 case OPT::ZONE_CREATE
:
5746 if (zone_name
.empty()) {
5747 cerr
<< "zone name not provided" << std::endl
;
5751 RGWZoneGroup zonegroup
;
5752 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> zonegroup_writer
;
5753 /* if the user didn't provide zonegroup info , create stand alone zone */
5754 if (!zonegroup_id
.empty() || !zonegroup_name
.empty()) {
5755 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5756 zonegroup_id
, zonegroup_name
,
5757 zonegroup
, &zonegroup_writer
);
5759 cerr
<< "failed to load zonegroup " << zonegroup_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
5762 if (realm_id
.empty() && realm_name
.empty()) {
5763 realm_id
= zonegroup
.realm_id
;
5767 // create the local zone params
5768 RGWZoneParams zone_params
;
5769 zone_params
.id
= zone_id
;
5770 zone_params
.name
= zone_name
;
5772 zone_params
.system_key
.id
= access_key
;
5773 zone_params
.system_key
.key
= secret_key
;
5774 zone_params
.realm_id
= realm_id
;
5775 for (const auto& a
: tier_config_add
) {
5776 int r
= zone_params
.tier_config
.set(a
.first
, a
.second
);
5778 cerr
<< "ERROR: failed to set configurable: " << a
<< std::endl
;
5783 if (zone_params
.realm_id
.empty()) {
5785 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
5786 realm_id
, realm_name
, realm
);
5787 if (ret
< 0 && ret
!= -ENOENT
) {
5788 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
5791 zone_params
.realm_id
= realm
.id
;
5792 cerr
<< "NOTICE: set zone's realm_id=" << realm
.id
<< std::endl
;
5795 constexpr bool exclusive
= true;
5796 int ret
= rgw::create_zone(dpp(), null_yield
, cfgstore
.get(),
5797 exclusive
, zone_params
);
5799 cerr
<< "failed to create zone " << zone_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
5803 if (zonegroup_writer
) {
5804 const bool *pis_master
= (is_master_set
? &is_master
: nullptr);
5805 const bool *pread_only
= (is_read_only_set
? &read_only
: nullptr);
5806 const bool *psync_from_all
= (sync_from_all_specified
? &sync_from_all
: nullptr);
5807 const string
*predirect_zone
= (redirect_zone_set
? &redirect_zone
: nullptr);
5809 // validate --tier-type if specified
5810 const string
*ptier_type
= (tier_type_specified
? &tier_type
: nullptr);
5812 auto sync_mgr
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->sync_modules
->get_manager();
5813 if (!sync_mgr
->get_module(*ptier_type
, nullptr)) {
5814 ldpp_dout(dpp(), -1) << "ERROR: could not find sync module: "
5815 << *ptier_type
<< ", valid sync modules: "
5816 << sync_mgr
->get_registered_module_names() << dendl
;
5821 if (enable_features
.empty()) { // enable all features by default
5822 enable_features
.insert(rgw::zone_features::supported
.begin(),
5823 rgw::zone_features::supported
.end());
5826 // add/update the public zone information stored in the zonegroup
5827 ret
= rgw::add_zone_to_group(dpp(), zonegroup
, zone_params
,
5828 pis_master
, pread_only
, endpoints
,
5829 ptier_type
, psync_from_all
,
5830 sync_from
, sync_from_rm
,
5831 predirect_zone
, bucket_index_max_shards
,
5832 enable_features
, disable_features
);
5837 // write the updated zonegroup
5838 ret
= zonegroup_writer
->write(dpp(), null_yield
, zonegroup
);
5840 cerr
<< "failed to add zone " << zone_name
<< " to zonegroup " << zonegroup
.get_name()
5841 << ": " << cpp_strerror(-ret
) << std::endl
;
5847 ret
= rgw::set_default_zone(dpp(), null_yield
, cfgstore
.get(),
5850 cerr
<< "failed to set zone " << zone_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5854 encode_json("zone", zone_params
, formatter
.get());
5855 formatter
->flush(cout
);
5858 case OPT::ZONE_DEFAULT
:
5860 if (zone_id
.empty() && zone_name
.empty()) {
5861 cerr
<< "no zone name or id provided" << std::endl
;
5864 RGWZoneParams zone_params
;
5865 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5866 zone_id
, zone_name
, zone_params
);
5868 cerr
<< "unable to load zone: " << cpp_strerror(-ret
) << std::endl
;
5872 ret
= rgw::set_default_zone(dpp(), null_yield
, cfgstore
.get(),
5875 cerr
<< "failed to set zone as default: " << cpp_strerror(-ret
) << std::endl
;
5880 case OPT::ZONE_DELETE
:
5882 if (zone_id
.empty() && zone_name
.empty()) {
5883 cerr
<< "no zone name or id provided" << std::endl
;
5886 RGWZoneParams zone_params
;
5887 std::unique_ptr
<rgw::sal::ZoneWriter
> writer
;
5888 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5889 zone_id
, zone_name
, zone_params
, &writer
);
5891 cerr
<< "failed to load zone: " << cpp_strerror(-ret
) << std::endl
;
5895 ret
= rgw::delete_zone(dpp(), null_yield
, cfgstore
.get(),
5896 zone_params
, *writer
);
5898 cerr
<< "failed to delete zone " << zone_params
.get_name()
5899 << ": " << cpp_strerror(-ret
) << std::endl
;
5906 RGWZoneParams zone_params
;
5907 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5908 zone_id
, zone_name
, zone_params
);
5910 cerr
<< "failed to load zone: " << cpp_strerror(-ret
) << std::endl
;
5913 encode_json("zone", zone_params
, formatter
.get());
5914 formatter
->flush(cout
);
5920 std::unique_ptr
<rgw::sal::ZoneWriter
> writer
;
5921 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5922 zone_id
, zone_name
, zone
, &writer
);
5923 if (ret
< 0 && ret
!= -ENOENT
) {
5924 cerr
<< "failed to load zone: " << cpp_strerror(ret
) << std::endl
;
5928 string orig_id
= zone
.get_id();
5930 ret
= read_decode_json(infile
, zone
);
5935 if (zone
.realm_id
.empty()) {
5937 ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
5938 realm_id
, realm_name
, realm
);
5939 if (ret
< 0 && ret
!= -ENOENT
) {
5940 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
5943 zone
.realm_id
= realm
.get_id();
5944 cerr
<< "NOTICE: set zone's realm_id=" << zone
.realm_id
<< std::endl
;
5947 if (!zone_name
.empty() && !zone
.get_name().empty() && zone
.get_name() != zone_name
) {
5948 cerr
<< "Error: zone name " << zone_name
<< " is different than the zone name " << zone
.get_name() << " in the provided json " << std::endl
;
5952 if (zone
.get_name().empty()) {
5953 zone
.set_name(zone_name
);
5954 if (zone
.get_name().empty()) {
5955 cerr
<< "no zone name specified" << std::endl
;
5960 zone_name
= zone
.get_name();
5962 if (zone
.get_id().empty()) {
5963 zone
.set_id(orig_id
);
5966 constexpr bool exclusive
= false;
5967 ret
= rgw::create_zone(dpp(), null_yield
, cfgstore
.get(),
5970 cerr
<< "ERROR: couldn't create zone: " << cpp_strerror(-ret
) << std::endl
;
5975 ret
= rgw::set_default_zone(dpp(), null_yield
, cfgstore
.get(), zone
);
5977 cerr
<< "failed to set zone " << zone_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5981 encode_json("zone", zone
, formatter
.get());
5982 formatter
->flush(cout
);
5985 case OPT::ZONE_LIST
:
5987 RGWZoneParams default_zone_params
;
5988 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5989 {}, {}, default_zone_params
);
5990 if (ret
< 0 && ret
!= -ENOENT
) {
5991 cerr
<< "could not determine default zone: " << cpp_strerror(-ret
) << std::endl
;
5994 Formatter::ObjectSection zones_list
{*formatter
, "zones_list"};
5995 encode_json("default_info", default_zone_params
.id
, formatter
.get());
5997 Formatter::ArraySection zones
{*formatter
, "zones"};
5998 rgw::sal::ListResult
<std::string
> listing
;
5999 std::array
<std::string
, 1000> names
; // list in pages of 1000
6001 ret
= cfgstore
->list_zone_names(dpp(), null_yield
, listing
.next
,
6004 std::cerr
<< "failed to list zones: " << cpp_strerror(-ret
) << std::endl
;
6007 for (const auto& name
: listing
.entries
) {
6008 encode_json("name", name
, formatter
.get());
6010 } while (!listing
.next
.empty());
6011 } // close sections zones and zones_list
6012 formatter
->flush(cout
);
6014 case OPT::ZONE_MODIFY
:
6016 RGWZoneParams zone_params
;
6017 std::unique_ptr
<rgw::sal::ZoneWriter
> zone_writer
;
6018 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
6019 zone_id
, zone_name
, zone_params
, &zone_writer
);
6021 cerr
<< "failed to load zone: " << cpp_strerror(-ret
) << std::endl
;
6025 bool need_zone_update
= false;
6026 if (!access_key
.empty()) {
6027 zone_params
.system_key
.id
= access_key
;
6028 need_zone_update
= true;
6031 if (!secret_key
.empty()) {
6032 zone_params
.system_key
.key
= secret_key
;
6033 need_zone_update
= true;
6036 if (!realm_id
.empty()) {
6037 zone_params
.realm_id
= realm_id
;
6038 need_zone_update
= true;
6039 } else if (!realm_name
.empty()) {
6040 // get realm id from name
6041 ret
= cfgstore
->read_realm_id(dpp(), null_yield
,
6042 realm_name
, zone_params
.realm_id
);
6044 cerr
<< "failed to find realm by name " << realm_name
<< std::endl
;
6047 need_zone_update
= true;
6050 for (const auto& add
: tier_config_add
) {
6051 ret
= zone_params
.tier_config
.set(add
.first
, add
.second
);
6053 cerr
<< "ERROR: failed to set configurable: " << add
<< std::endl
;
6056 need_zone_update
= true;
6059 for (const auto& rm
: tier_config_rm
) {
6060 if (!rm
.first
.empty()) { /* otherwise will remove the entire config */
6061 zone_params
.tier_config
.erase(rm
.first
);
6062 need_zone_update
= true;
6066 if (need_zone_update
) {
6067 ret
= zone_writer
->write(dpp(), null_yield
, zone_params
);
6069 cerr
<< "failed to save zone info: " << cpp_strerror(-ret
) << std::endl
;
6074 RGWZoneGroup zonegroup
;
6075 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> zonegroup_writer
;
6076 ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
6077 zonegroup_id
, zonegroup_name
,
6078 zonegroup
, &zonegroup_writer
);
6080 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
6084 const bool *pis_master
= (is_master_set
? &is_master
: nullptr);
6085 const bool *pread_only
= (is_read_only_set
? &read_only
: nullptr);
6086 const bool *psync_from_all
= (sync_from_all_specified
? &sync_from_all
: nullptr);
6087 const string
*predirect_zone
= (redirect_zone_set
? &redirect_zone
: nullptr);
6089 // validate --tier-type if specified
6090 const string
*ptier_type
= (tier_type_specified
? &tier_type
: nullptr);
6092 auto sync_mgr
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->sync_modules
->get_manager();
6093 if (!sync_mgr
->get_module(*ptier_type
, nullptr)) {
6094 ldpp_dout(dpp(), -1) << "ERROR: could not find sync module: "
6095 << *ptier_type
<< ", valid sync modules: "
6096 << sync_mgr
->get_registered_module_names() << dendl
;
6101 if (enable_features
.empty()) { // enable all features by default
6102 enable_features
.insert(rgw::zone_features::supported
.begin(),
6103 rgw::zone_features::supported
.end());
6106 // add/update the public zone information stored in the zonegroup
6107 ret
= rgw::add_zone_to_group(dpp(), zonegroup
, zone_params
,
6108 pis_master
, pread_only
, endpoints
,
6109 ptier_type
, psync_from_all
,
6110 sync_from
, sync_from_rm
,
6111 predirect_zone
, bucket_index_max_shards
,
6112 enable_features
, disable_features
);
6117 // write the updated zonegroup
6118 ret
= zonegroup_writer
->write(dpp(), null_yield
, zonegroup
);
6120 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
6125 ret
= rgw::set_default_zone(dpp(), null_yield
, cfgstore
.get(),
6128 cerr
<< "failed to set zone " << zone_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
6132 encode_json("zone", zone_params
, formatter
.get());
6133 formatter
->flush(cout
);
6136 case OPT::ZONE_RENAME
:
6138 if (zone_new_name
.empty()) {
6139 cerr
<< " missing zone new name" << std::endl
;
6142 if (zone_id
.empty() && zone_name
.empty()) {
6143 cerr
<< "no zone name or id provided" << std::endl
;
6147 RGWZoneParams zone_params
;
6148 std::unique_ptr
<rgw::sal::ZoneWriter
> zone_writer
;
6149 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
6150 zone_id
, zone_name
, zone_params
, &zone_writer
);
6152 cerr
<< "failed to load zone: " << cpp_strerror(-ret
) << std::endl
;
6156 ret
= zone_writer
->rename(dpp(), null_yield
, zone_params
, zone_new_name
);
6158 cerr
<< "failed to rename zone " << zone_name
<< " to " << zone_new_name
<< ": " << cpp_strerror(-ret
)
6163 RGWZoneGroup zonegroup
;
6164 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> zonegroup_writer
;
6165 ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
6166 zonegroup_id
, zonegroup_name
,
6167 zonegroup
, &zonegroup_writer
);
6169 cerr
<< "WARNING: failed to load zonegroup " << zonegroup_name
<< std::endl
;
6170 return EXIT_SUCCESS
;
6173 auto z
= zonegroup
.zones
.find(zone_params
.id
);
6174 if (z
== zonegroup
.zones
.end()) {
6175 return EXIT_SUCCESS
;
6177 z
->second
.name
= zone_params
.name
;
6179 ret
= zonegroup_writer
->write(dpp(), null_yield
, zonegroup
);
6181 cerr
<< "Error in zonegroup rename for " << zone_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
6186 case OPT::ZONE_PLACEMENT_ADD
:
6187 case OPT::ZONE_PLACEMENT_MODIFY
:
6188 case OPT::ZONE_PLACEMENT_RM
:
6190 if (placement_id
.empty()) {
6191 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
6194 // validate compression type
6195 if (compression_type
&& *compression_type
!= "random"
6196 && !Compressor::get_comp_alg_type(*compression_type
)) {
6197 std::cerr
<< "Unrecognized compression type" << std::endl
;
6202 std::unique_ptr
<rgw::sal::ZoneWriter
> writer
;
6203 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
6204 zone_id
, zone_name
, zone
, &writer
);
6206 cerr
<< "failed to init zone: " << cpp_strerror(-ret
) << std::endl
;
6210 if (opt_cmd
== OPT::ZONE_PLACEMENT_ADD
||
6211 opt_cmd
== OPT::ZONE_PLACEMENT_MODIFY
) {
6212 RGWZoneGroup zonegroup
;
6213 ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
6214 zonegroup_id
, zonegroup_name
, zonegroup
);
6216 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
6220 auto ptiter
= zonegroup
.placement_targets
.find(placement_id
);
6221 if (ptiter
== zonegroup
.placement_targets
.end()) {
6222 cerr
<< "ERROR: placement id '" << placement_id
<< "' is not configured in zonegroup placement targets" << std::endl
;
6226 string storage_class
= rgw_placement_rule::get_canonical_storage_class(opt_storage_class
.value_or(string()));
6227 if (ptiter
->second
.storage_classes
.find(storage_class
) == ptiter
->second
.storage_classes
.end()) {
6228 cerr
<< "ERROR: storage class '" << storage_class
<< "' is not defined in zonegroup '" << placement_id
<< "' placement target" << std::endl
;
6231 if (ptiter
->second
.tier_targets
.find(storage_class
) != ptiter
->second
.tier_targets
.end()) {
6232 cerr
<< "ERROR: storage class '" << storage_class
<< "' is of tier type in zonegroup '" << placement_id
<< "' placement target" << std::endl
;
6236 RGWZonePlacementInfo
& info
= zone
.placement_pools
[placement_id
];
6238 string opt_index_pool
= index_pool
.value_or(string());
6239 string opt_data_pool
= data_pool
.value_or(string());
6241 if (!opt_index_pool
.empty()) {
6242 info
.index_pool
= opt_index_pool
;
6245 if (info
.index_pool
.empty()) {
6246 cerr
<< "ERROR: index pool not configured, need to specify --index-pool" << std::endl
;
6250 if (opt_data_pool
.empty()) {
6251 const RGWZoneStorageClass
*porig_sc
{nullptr};
6252 if (info
.storage_classes
.find(storage_class
, &porig_sc
)) {
6253 if (porig_sc
->data_pool
) {
6254 opt_data_pool
= porig_sc
->data_pool
->to_str();
6257 if (opt_data_pool
.empty()) {
6258 cerr
<< "ERROR: data pool not configured, need to specify --data-pool" << std::endl
;
6263 rgw_pool dp
= opt_data_pool
;
6264 info
.storage_classes
.set_storage_class(storage_class
, &dp
, compression_type
.get_ptr());
6266 if (data_extra_pool
) {
6267 info
.data_extra_pool
= *data_extra_pool
;
6269 if (index_type_specified
) {
6270 info
.index_type
= placement_index_type
;
6272 if (placement_inline_data_specified
) {
6273 info
.inline_data
= placement_inline_data
;
6276 ret
= check_pool_support_omap(info
.get_data_extra_pool());
6278 cerr
<< "ERROR: the data extra (non-ec) pool '" << info
.get_data_extra_pool()
6279 << "' does not support omap" << std::endl
;
6282 } else if (opt_cmd
== OPT::ZONE_PLACEMENT_RM
) {
6283 if (!opt_storage_class
||
6284 opt_storage_class
->empty()) {
6285 zone
.placement_pools
.erase(placement_id
);
6287 auto iter
= zone
.placement_pools
.find(placement_id
);
6288 if (iter
!= zone
.placement_pools
.end()) {
6289 RGWZonePlacementInfo
& info
= zone
.placement_pools
[placement_id
];
6290 info
.storage_classes
.remove_storage_class(*opt_storage_class
);
6295 ret
= writer
->write(dpp(), null_yield
, zone
);
6297 cerr
<< "failed to save zone info: " << cpp_strerror(-ret
) << std::endl
;
6301 encode_json("zone", zone
, formatter
.get());
6302 formatter
->flush(cout
);
6305 case OPT::ZONE_PLACEMENT_LIST
:
6308 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
6309 zone_id
, zone_name
, zone
);
6311 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
6314 encode_json("placement_pools", zone
.placement_pools
, formatter
.get());
6315 formatter
->flush(cout
);
6318 case OPT::ZONE_PLACEMENT_GET
:
6320 if (placement_id
.empty()) {
6321 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
6326 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
6327 zone_id
, zone_name
, zone
);
6329 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
6332 auto p
= zone
.placement_pools
.find(placement_id
);
6333 if (p
== zone
.placement_pools
.end()) {
6334 cerr
<< "ERROR: zone placement target '" << placement_id
<< "' not found" << std::endl
;
6337 encode_json("placement_pools", p
->second
, formatter
.get());
6338 formatter
->flush(cout
);
6346 resolve_zone_id_opt(opt_effective_zone_name
, opt_effective_zone_id
);
6347 resolve_zone_id_opt(opt_source_zone_name
, opt_source_zone_id
);
6348 resolve_zone_id_opt(opt_dest_zone_name
, opt_dest_zone_id
);
6349 resolve_zone_ids_opt(opt_zone_names
, opt_zone_ids
);
6350 resolve_zone_ids_opt(opt_source_zone_names
, opt_source_zone_ids
);
6351 resolve_zone_ids_opt(opt_dest_zone_names
, opt_dest_zone_ids
);
6353 bool non_master_cmd
= (!driver
->is_meta_master() && !yes_i_really_mean_it
);
6354 std::set
<OPT
> non_master_ops_list
= {OPT::USER_CREATE
, OPT::USER_RM
,
6355 OPT::USER_MODIFY
, OPT::USER_ENABLE
,
6356 OPT::USER_SUSPEND
, OPT::SUBUSER_CREATE
,
6357 OPT::SUBUSER_MODIFY
, OPT::SUBUSER_RM
,
6358 OPT::BUCKET_LINK
, OPT::BUCKET_UNLINK
,
6360 OPT::BUCKET_CHOWN
, OPT::METADATA_PUT
,
6361 OPT::METADATA_RM
, OPT::MFA_CREATE
,
6362 OPT::MFA_REMOVE
, OPT::MFA_RESYNC
,
6363 OPT::CAPS_ADD
, OPT::CAPS_RM
,
6364 OPT::ROLE_CREATE
, OPT::ROLE_DELETE
,
6365 OPT::ROLE_POLICY_PUT
, OPT::ROLE_POLICY_DELETE
};
6367 bool print_warning_message
= (non_master_ops_list
.find(opt_cmd
) != non_master_ops_list
.end() &&
6370 if (print_warning_message
) {
6371 cerr
<< "Please run the command on master zone. Performing this operation on non-master zone leads to inconsistent metadata between zones" << std::endl
;
6372 cerr
<< "Are you sure you want to go ahead? (requires --yes-i-really-mean-it)" << std::endl
;
6376 if (!rgw::sal::User::empty(user
)) {
6377 user_op
.set_user_id(user
->get_id());
6378 bucket_op
.set_user_id(user
->get_id());
6381 if (!display_name
.empty())
6382 user_op
.set_display_name(display_name
);
6384 if (!user_email
.empty())
6385 user_op
.set_user_email(user_email
);
6387 if (!rgw::sal::User::empty(user
)) {
6388 user_op
.set_new_user_id(new_user_id
);
6391 if (!access_key
.empty())
6392 user_op
.set_access_key(access_key
);
6394 if (!secret_key
.empty())
6395 user_op
.set_secret_key(secret_key
);
6397 if (!subuser
.empty())
6398 user_op
.set_subuser(subuser
);
6401 user_op
.set_caps(caps
);
6403 user_op
.set_purge_data(purge_data
);
6406 user_op
.set_purge_keys();
6409 user_op
.set_generate_key();
6412 user_op
.set_gen_secret(); // assume that a key pair should be created
6414 if (max_buckets_specified
)
6415 user_op
.set_max_buckets(max_buckets
);
6417 if (admin_specified
)
6418 user_op
.set_admin(admin
);
6420 if (system_specified
)
6421 user_op
.set_system(system
);
6424 user_op
.set_perm(perm_mask
);
6426 if (set_temp_url_key
) {
6427 map
<int, string
>::iterator iter
= temp_url_keys
.begin();
6428 for (; iter
!= temp_url_keys
.end(); ++iter
) {
6429 user_op
.set_temp_url_key(iter
->second
, iter
->first
);
6433 if (!op_mask_str
.empty()) {
6435 int ret
= rgw_parse_op_type_list(op_mask_str
, &op_mask
);
6437 cerr
<< "failed to parse op_mask: " << cpp_strerror(-ret
) << std::endl
;
6441 user_op
.set_op_mask(op_mask
);
6444 if (key_type
!= KEY_TYPE_UNDEFINED
)
6445 user_op
.set_key_type(key_type
);
6447 // set suspension operation parameters
6448 if (opt_cmd
== OPT::USER_ENABLE
)
6449 user_op
.set_suspension(false);
6450 else if (opt_cmd
== OPT::USER_SUSPEND
)
6451 user_op
.set_suspension(true);
6453 if (!placement_id
.empty()) {
6454 rgw_placement_rule target_rule
;
6455 target_rule
.name
= placement_id
;
6456 target_rule
.storage_class
= opt_storage_class
.value_or("");
6457 if (!driver
->valid_placement(target_rule
)) {
6458 cerr
<< "NOTICE: invalid dest placement: " << target_rule
.to_str() << std::endl
;
6461 user_op
.set_default_placement(target_rule
);
6464 if (!tags
.empty()) {
6465 user_op
.set_placement_tags(tags
);
6468 // RGWUser to use for user operations
6471 if (!(rgw::sal::User::empty(user
) && access_key
.empty()) || !subuser
.empty()) {
6472 ret
= ruser
.init(dpp(), driver
, user_op
, null_yield
);
6474 cerr
<< "user.init failed: " << cpp_strerror(-ret
) << std::endl
;
6479 /* populate bucket operation */
6480 bucket_op
.set_bucket_name(bucket_name
);
6481 bucket_op
.set_object(object
);
6482 bucket_op
.set_check_objects(check_objects
);
6483 bucket_op
.set_delete_children(delete_child_objects
);
6484 bucket_op
.set_fix_index(fix
);
6485 bucket_op
.set_max_aio(max_concurrent_ios
);
6486 bucket_op
.set_min_age(min_age
);
6487 bucket_op
.set_dump_keys(dump_keys
);
6488 bucket_op
.set_hide_progress(hide_progress
);
6490 // required to gather errors from operations
6491 std::string err_msg
;
6493 bool output_user_info
= true;
6496 case OPT::USER_INFO
:
6497 if (rgw::sal::User::empty(user
) && access_key
.empty()) {
6498 cerr
<< "ERROR: --uid or --access-key required" << std::endl
;
6502 case OPT::USER_CREATE
:
6503 if (!user_op
.has_existing_user()) {
6504 user_op
.set_generate_key(); // generate a new key by default
6506 ret
= ruser
.add(dpp(), user_op
, null_yield
, &err_msg
);
6508 cerr
<< "could not create user: " << err_msg
<< std::endl
;
6509 if (ret
== -ERR_INVALID_TENANT_NAME
)
6514 if (!subuser
.empty()) {
6515 ret
= ruser
.subusers
.add(dpp(),user_op
, null_yield
, &err_msg
);
6517 cerr
<< "could not create subuser: " << err_msg
<< std::endl
;
6523 ret
= ruser
.remove(dpp(), user_op
, null_yield
, &err_msg
);
6525 cerr
<< "could not remove user: " << err_msg
<< std::endl
;
6529 output_user_info
= false;
6531 case OPT::USER_RENAME
:
6532 if (yes_i_really_mean_it
) {
6533 user_op
.set_overwrite_new_user(true);
6535 ret
= ruser
.rename(user_op
, null_yield
, dpp(), &err_msg
);
6537 if (ret
== -EEXIST
) {
6538 err_msg
+= ". to overwrite this user, add --yes-i-really-mean-it";
6540 cerr
<< "could not rename user: " << err_msg
<< std::endl
;
6545 case OPT::USER_ENABLE
:
6546 case OPT::USER_SUSPEND
:
6547 case OPT::USER_MODIFY
:
6548 ret
= ruser
.modify(dpp(), user_op
, null_yield
, &err_msg
);
6550 cerr
<< "could not modify user: " << err_msg
<< std::endl
;
6555 case OPT::SUBUSER_CREATE
:
6556 ret
= ruser
.subusers
.add(dpp(), user_op
, null_yield
, &err_msg
);
6558 cerr
<< "could not create subuser: " << err_msg
<< std::endl
;
6563 case OPT::SUBUSER_MODIFY
:
6564 ret
= ruser
.subusers
.modify(dpp(), user_op
, null_yield
, &err_msg
);
6566 cerr
<< "could not modify subuser: " << err_msg
<< std::endl
;
6571 case OPT::SUBUSER_RM
:
6572 ret
= ruser
.subusers
.remove(dpp(), user_op
, null_yield
, &err_msg
);
6574 cerr
<< "could not remove subuser: " << err_msg
<< std::endl
;
6580 ret
= ruser
.caps
.add(dpp(), user_op
, null_yield
, &err_msg
);
6582 cerr
<< "could not add caps: " << err_msg
<< std::endl
;
6588 ret
= ruser
.caps
.remove(dpp(), user_op
, null_yield
, &err_msg
);
6590 cerr
<< "could not remove caps: " << err_msg
<< std::endl
;
6595 case OPT::KEY_CREATE
:
6596 ret
= ruser
.keys
.add(dpp(), user_op
, null_yield
, &err_msg
);
6598 cerr
<< "could not create key: " << err_msg
<< std::endl
;
6604 ret
= ruser
.keys
.remove(dpp(), user_op
, null_yield
, &err_msg
);
6606 cerr
<< "could not remove key: " << err_msg
<< std::endl
;
6610 case OPT::PERIOD_PUSH
:
6613 req_info
info(g_ceph_context
, &env
);
6614 info
.method
= "POST";
6615 info
.request_uri
= "/admin/realm/period";
6617 map
<string
, string
> ¶ms
= info
.args
.get_params();
6618 if (!realm_id
.empty())
6619 params
["realm_id"] = realm_id
;
6620 if (!realm_name
.empty())
6621 params
["realm_name"] = realm_name
;
6622 if (!period_id
.empty())
6623 params
["period_id"] = period_id
;
6624 if (!period_epoch
.empty())
6625 params
["epoch"] = period_epoch
;
6629 int ret
= cfgstore
->read_period(dpp(), null_yield
, period_id
,
6630 std::nullopt
, period
);
6632 cerr
<< "failed to load period: " << cpp_strerror(-ret
) << std::endl
;
6635 // json format into a bufferlist
6636 JSONFormatter
jf(false);
6637 encode_json("period", period
, &jf
);
6642 ret
= send_to_remote_or_url(nullptr, url
, opt_region
,
6643 access_key
, secret_key
,
6646 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
6651 case OPT::PERIOD_UPDATE
:
6653 int ret
= update_period(cfgstore
.get(), realm_id
, realm_name
,
6654 period_epoch
, commit
, remote
, url
,
6655 opt_region
, access_key
, secret_key
,
6656 formatter
.get(), yes_i_really_mean_it
);
6662 case OPT::PERIOD_COMMIT
:
6664 // read realm and staging period
6666 std::unique_ptr
<rgw::sal::RealmWriter
> realm_writer
;
6667 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
6668 realm_id
, realm_name
,
6669 realm
, &realm_writer
);
6671 cerr
<< "Error initializing realm: " << cpp_strerror(-ret
) << std::endl
;
6674 period_id
= rgw::get_staging_period_id(realm
.id
);
6678 ret
= cfgstore
->read_period(dpp(), null_yield
, period_id
, epoch
, period
);
6680 cerr
<< "failed to load period: " << cpp_strerror(-ret
) << std::endl
;
6683 ret
= commit_period(cfgstore
.get(), realm
, *realm_writer
, period
,
6684 remote
, url
, opt_region
, access_key
, secret_key
,
6685 yes_i_really_mean_it
);
6687 cerr
<< "failed to commit period: " << cpp_strerror(-ret
) << std::endl
;
6691 encode_json("period", period
, formatter
.get());
6692 formatter
->flush(cout
);
6695 case OPT::ROLE_CREATE
:
6697 if (role_name
.empty()) {
6698 cerr
<< "ERROR: role name is empty" << std::endl
;
6702 if (assume_role_doc
.empty()) {
6703 cerr
<< "ERROR: assume role policy document is empty" << std::endl
;
6706 bufferlist bl
= bufferlist::static_from_string(assume_role_doc
);
6708 const rgw::IAM::Policy
p(
6709 g_ceph_context
, tenant
, bl
,
6710 g_ceph_context
->_conf
.get_val
<bool>(
6711 "rgw_policy_reject_invalid_principals"));
6712 } catch (rgw::IAM::PolicyParseException
& e
) {
6713 cerr
<< "failed to parse policy: " << e
.what() << std::endl
;
6716 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
, path
, assume_role_doc
);
6717 ret
= role
->create(dpp(), true, "", null_yield
);
6721 show_role_info(role
.get(), formatter
.get());
6724 case OPT::ROLE_DELETE
:
6726 if (role_name
.empty()) {
6727 cerr
<< "ERROR: empty role name" << std::endl
;
6730 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6731 ret
= role
->delete_obj(dpp(), null_yield
);
6735 cout
<< "role: " << role_name
<< " successfully deleted" << std::endl
;
6740 if (role_name
.empty()) {
6741 cerr
<< "ERROR: empty role name" << std::endl
;
6744 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6745 ret
= role
->get(dpp(), null_yield
);
6749 show_role_info(role
.get(), formatter
.get());
6752 case OPT::ROLE_TRUST_POLICY_MODIFY
:
6754 if (role_name
.empty()) {
6755 cerr
<< "ERROR: role name is empty" << std::endl
;
6759 if (assume_role_doc
.empty()) {
6760 cerr
<< "ERROR: assume role policy document is empty" << std::endl
;
6764 bufferlist bl
= bufferlist::static_from_string(assume_role_doc
);
6766 const rgw::IAM::Policy
p(g_ceph_context
, tenant
, bl
,
6767 g_ceph_context
->_conf
.get_val
<bool>(
6768 "rgw_policy_reject_invalid_principals"));
6769 } catch (rgw::IAM::PolicyParseException
& e
) {
6770 cerr
<< "failed to parse policy: " << e
.what() << std::endl
;
6774 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6775 ret
= role
->get(dpp(), null_yield
);
6779 role
->update_trust_policy(assume_role_doc
);
6780 ret
= role
->update(dpp(), null_yield
);
6784 cout
<< "Assume role policy document updated successfully for role: " << role_name
<< std::endl
;
6787 case OPT::ROLE_LIST
:
6789 vector
<std::unique_ptr
<rgw::sal::RGWRole
>> result
;
6790 ret
= driver
->get_roles(dpp(), null_yield
, path_prefix
, tenant
, result
);
6794 show_roles_info(result
, formatter
.get());
6797 case OPT::ROLE_POLICY_PUT
:
6799 if (role_name
.empty()) {
6800 cerr
<< "role name is empty" << std::endl
;
6804 if (policy_name
.empty()) {
6805 cerr
<< "policy name is empty" << std::endl
;
6809 if (perm_policy_doc
.empty() && infile
.empty()) {
6810 cerr
<< "permission policy document is empty" << std::endl
;
6815 if (!infile
.empty()) {
6816 int ret
= read_input(infile
, bl
);
6818 cerr
<< "ERROR: failed to read input policy document: " << cpp_strerror(-ret
) << std::endl
;
6821 perm_policy_doc
= bl
.to_str();
6823 bl
= bufferlist::static_from_string(perm_policy_doc
);
6826 const rgw::IAM::Policy
p(g_ceph_context
, tenant
, bl
,
6827 g_ceph_context
->_conf
.get_val
<bool>(
6828 "rgw_policy_reject_invalid_principals"));
6829 } catch (rgw::IAM::PolicyParseException
& e
) {
6830 cerr
<< "failed to parse perm policy: " << e
.what() << std::endl
;
6834 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6835 ret
= role
->get(dpp(), null_yield
);
6839 role
->set_perm_policy(policy_name
, perm_policy_doc
);
6840 ret
= role
->update(dpp(), null_yield
);
6844 cout
<< "Permission policy attached successfully" << std::endl
;
6847 case OPT::ROLE_POLICY_LIST
:
6849 if (role_name
.empty()) {
6850 cerr
<< "ERROR: Role name is empty" << std::endl
;
6853 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6854 ret
= role
->get(dpp(), null_yield
);
6858 std::vector
<string
> policy_names
= role
->get_role_policy_names();
6859 show_policy_names(policy_names
, formatter
.get());
6862 case OPT::ROLE_POLICY_GET
:
6864 if (role_name
.empty()) {
6865 cerr
<< "ERROR: role name is empty" << std::endl
;
6869 if (policy_name
.empty()) {
6870 cerr
<< "ERROR: policy name is empty" << std::endl
;
6873 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6874 int ret
= role
->get(dpp(), null_yield
);
6879 ret
= role
->get_role_policy(dpp(), policy_name
, perm_policy
);
6883 show_perm_policy(perm_policy
, formatter
.get());
6886 case OPT::ROLE_POLICY_DELETE
:
6888 if (role_name
.empty()) {
6889 cerr
<< "ERROR: role name is empty" << std::endl
;
6893 if (policy_name
.empty()) {
6894 cerr
<< "ERROR: policy name is empty" << std::endl
;
6897 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6898 ret
= role
->get(dpp(), null_yield
);
6902 ret
= role
->delete_policy(dpp(), policy_name
);
6906 ret
= role
->update(dpp(), null_yield
);
6910 cout
<< "Policy: " << policy_name
<< " successfully deleted for role: "
6911 << role_name
<< std::endl
;
6914 case OPT::ROLE_UPDATE
:
6916 if (role_name
.empty()) {
6917 cerr
<< "ERROR: role name is empty" << std::endl
;
6921 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6922 ret
= role
->get(dpp(), null_yield
);
6926 if (!role
->validate_max_session_duration(dpp())) {
6930 role
->update_max_session_duration(max_session_duration
);
6931 ret
= role
->update(dpp(), null_yield
);
6935 cout
<< "Max session duration updated successfully for role: " << role_name
<< std::endl
;
6939 output_user_info
= false;
6942 // output the result of a user operation
6943 if (output_user_info
) {
6944 ret
= ruser
.info(info
, &err_msg
);
6946 cerr
<< "could not fetch user info: " << err_msg
<< std::endl
;
6949 show_user_info(info
, formatter
.get());
6952 if (opt_cmd
== OPT::POLICY
) {
6953 if (format
== "xml") {
6954 int ret
= RGWBucketAdminOp::dump_s3_policy(driver
, bucket_op
, cout
, dpp());
6956 cerr
<< "ERROR: failed to get policy: " << cpp_strerror(-ret
) << std::endl
;
6960 int ret
= RGWBucketAdminOp::get_policy(driver
, bucket_op
, stream_flusher
, dpp());
6962 cerr
<< "ERROR: failed to get policy: " << cpp_strerror(-ret
) << std::endl
;
6968 if (opt_cmd
== OPT::BUCKET_LIMIT_CHECK
) {
6970 std::list
<std::string
> user_ids
;
6971 metadata_key
= "user";
6976 if (!rgw::sal::User::empty(user
)) {
6977 user_ids
.push_back(user
->get_id().id
);
6979 RGWBucketAdminOp::limit_check(driver
, bucket_op
, user_ids
, stream_flusher
,
6980 null_yield
, dpp(), warnings_only
);
6982 /* list users in groups of max-keys, then perform user-bucket
6983 * limit-check on each group */
6984 ret
= driver
->meta_list_keys_init(dpp(), metadata_key
, string(), &handle
);
6986 cerr
<< "ERROR: buckets limit check can't get user metadata_key: "
6987 << cpp_strerror(-ret
) << std::endl
;
6992 ret
= driver
->meta_list_keys_next(dpp(), handle
, max
, user_ids
,
6994 if (ret
< 0 && ret
!= -ENOENT
) {
6995 cerr
<< "ERROR: buckets limit check lists_keys_next(): "
6996 << cpp_strerror(-ret
) << std::endl
;
6999 /* ok, do the limit checks for this group */
7001 RGWBucketAdminOp::limit_check(driver
, bucket_op
, user_ids
, stream_flusher
,
7002 null_yield
, dpp(), warnings_only
);
7007 } while (truncated
);
7008 driver
->meta_list_keys_complete(handle
);
7011 } /* OPT::BUCKET_LIMIT_CHECK */
7013 if (opt_cmd
== OPT::BUCKETS_LIST
) {
7014 if (bucket_name
.empty()) {
7015 if (!rgw::sal::User::empty(user
)) {
7016 if (!user_op
.has_existing_user()) {
7017 cerr
<< "ERROR: could not find user: " << user
<< std::endl
;
7021 RGWBucketAdminOp::info(driver
, bucket_op
, stream_flusher
, null_yield
, dpp());
7023 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7025 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7028 formatter
->open_array_section("entries");
7032 static constexpr int MAX_PAGINATE_SIZE
= 10000;
7033 static constexpr int DEFAULT_MAX_ENTRIES
= 1000;
7035 if (max_entries
< 0) {
7036 max_entries
= DEFAULT_MAX_ENTRIES
;
7038 const int paginate_size
= std::min(max_entries
, MAX_PAGINATE_SIZE
);
7044 rgw::sal::Bucket::ListParams params
;
7045 rgw::sal::Bucket::ListResults results
;
7047 params
.prefix
= prefix
;
7048 params
.delim
= delim
;
7049 params
.marker
= rgw_obj_key(marker
);
7051 params
.enforce_ns
= false;
7052 params
.list_versions
= true;
7053 params
.allow_unordered
= bool(allow_unordered
);
7056 const int remaining
= max_entries
- count
;
7057 ret
= bucket
->list(dpp(), params
, std::min(remaining
, paginate_size
), results
,
7060 cerr
<< "ERROR: driver->list_objects(): " << cpp_strerror(-ret
) << std::endl
;
7063 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<<
7064 ": list() returned without error; results.objs.sizie()=" <<
7065 results
.objs
.size() << "results.is_truncated=" << results
.is_truncated
<< ", marker=" <<
7066 params
.marker
<< dendl
;
7068 count
+= results
.objs
.size();
7070 for (const auto& entry
: results
.objs
) {
7071 encode_json("entry", entry
, formatter
.get());
7073 formatter
->flush(cout
);
7074 } while (results
.is_truncated
&& count
< max_entries
);
7075 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<< ": done" << dendl
;
7077 formatter
->close_section();
7078 formatter
->flush(cout
);
7079 } /* have bucket_name */
7080 } /* OPT::BUCKETS_LIST */
7082 if (opt_cmd
== OPT::BUCKET_RADOS_LIST
) {
7083 RGWRadosList
lister(static_cast<rgw::sal::RadosStore
*>(driver
),
7084 max_concurrent_ios
, orphan_stale_secs
, tenant
);
7086 lister
.set_field_separator(*rgw_obj_fs
);
7089 if (bucket_name
.empty()) {
7090 // yes_i_really_mean_it means continue with listing even if
7091 // there are indexless buckets
7092 ret
= lister
.run(dpp(), yes_i_really_mean_it
);
7094 ret
= lister
.run(dpp(), bucket_name
);
7099 "ERROR: bucket radoslist failed to finish before " <<
7100 "encountering error: " << cpp_strerror(-ret
) << std::endl
;
7101 std::cerr
<< "************************************"
7102 "************************************" << std::endl
;
7103 std::cerr
<< "WARNING: THE RESULTS ARE NOT RELIABLE AND SHOULD NOT " <<
7104 "BE USED IN DELETING ORPHANS" << std::endl
;
7105 std::cerr
<< "************************************"
7106 "************************************" << std::endl
;
7111 if (opt_cmd
== OPT::BUCKET_LAYOUT
) {
7112 if (bucket_name
.empty()) {
7113 cerr
<< "ERROR: bucket not specified" << std::endl
;
7116 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7120 const auto& bucket_info
= bucket
->get_info();
7121 formatter
->open_object_section("layout");
7122 encode_json("layout", bucket_info
.layout
, formatter
.get());
7123 formatter
->close_section();
7124 formatter
->flush(cout
);
7127 if (opt_cmd
== OPT::BUCKET_STATS
) {
7128 if (bucket_name
.empty() && !bucket_id
.empty()) {
7130 if (!rgw_find_bucket_by_id(dpp(), driver
->ctx(), driver
, marker
, bucket_id
, &bucket
)) {
7131 cerr
<< "failure: no such bucket id" << std::endl
;
7134 bucket_op
.set_tenant(bucket
.tenant
);
7135 bucket_op
.set_bucket_name(bucket
.name
);
7137 bucket_op
.set_fetch_stats(true);
7139 int r
= RGWBucketAdminOp::info(driver
, bucket_op
, stream_flusher
, null_yield
, dpp());
7141 cerr
<< "failure: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
7142 return posix_errortrans(-r
);
7146 if (opt_cmd
== OPT::BUCKET_LINK
) {
7147 bucket_op
.set_bucket_id(bucket_id
);
7148 bucket_op
.set_new_bucket_name(new_bucket_name
);
7150 int r
= RGWBucketAdminOp::link(driver
, bucket_op
, dpp(), &err
);
7152 cerr
<< "failure: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
7157 if (opt_cmd
== OPT::BUCKET_UNLINK
) {
7158 int r
= RGWBucketAdminOp::unlink(driver
, bucket_op
, dpp());
7160 cerr
<< "failure: " << cpp_strerror(-r
) << std::endl
;
7165 if (opt_cmd
== OPT::BUCKET_SHARD_OBJECTS
) {
7166 const auto prefix
= opt_prefix
? *opt_prefix
: "obj"s
;
7167 if (!num_shards_specified
) {
7168 cerr
<< "ERROR: num-shards must be specified."
7173 if (specified_shard_id
) {
7174 if (shard_id
>= num_shards
) {
7175 cerr
<< "ERROR: shard-id must be less than num-shards."
7183 obj
= fmt::format("{}{:0>20}", prefix
, ctr
);
7184 shard
= RGWSI_BucketIndex_RADOS::bucket_shard_index(obj
, num_shards
);
7186 } while (shard
!= shard_id
);
7188 formatter
->open_object_section("shard_obj");
7189 encode_json("obj", obj
, formatter
.get());
7190 formatter
->close_section();
7191 formatter
->flush(cout
);
7193 std::vector
<std::string
> objs(num_shards
);
7194 for (uint64_t ctr
= 0, shardsleft
= num_shards
; shardsleft
> 0; ++ctr
) {
7195 auto key
= fmt::format("{}{:0>20}", prefix
, ctr
);
7196 auto shard
= RGWSI_BucketIndex_RADOS::bucket_shard_index(key
, num_shards
);
7197 if (objs
[shard
].empty()) {
7198 objs
[shard
] = std::move(key
);
7203 formatter
->open_object_section("shard_objs");
7204 encode_json("objs", objs
, formatter
.get());
7205 formatter
->close_section();
7206 formatter
->flush(cout
);
7210 if (opt_cmd
== OPT::BUCKET_OBJECT_SHARD
) {
7211 if (!num_shards_specified
|| object
.empty()) {
7212 cerr
<< "ERROR: num-shards and object must be specified."
7216 auto shard
= RGWSI_BucketIndex_RADOS::bucket_shard_index(object
, num_shards
);
7217 formatter
->open_object_section("obj_shard");
7218 encode_json("shard", shard
, formatter
.get());
7219 formatter
->close_section();
7220 formatter
->flush(cout
);
7223 if (opt_cmd
== OPT::BUCKET_RESYNC_ENCRYPTED_MULTIPART
) {
7224 // repair logic for replication of encrypted multipart uploads:
7225 // https://tracker.ceph.com/issues/46062
7226 if (bucket_name
.empty()) {
7227 cerr
<< "ERROR: bucket not specified" << std::endl
;
7230 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7235 auto rados_driver
= dynamic_cast<rgw::sal::RadosStore
*>(driver
);
7236 if (!rados_driver
) {
7237 cerr
<< "ERROR: this command can only work when the cluster "
7238 "has a RADOS backing store." << std::endl
;
7242 // fail if recovery wouldn't generate replication log entries
7243 if (!rados_driver
->svc()->zone
->need_to_log_data() && !yes_i_really_mean_it
) {
7244 cerr
<< "This command is only necessary for replicated buckets." << std::endl
;
7245 cerr
<< "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
7249 formatter
->open_object_section("modified");
7250 encode_json("bucket", bucket
->get_name(), formatter
.get());
7251 encode_json("bucket_id", bucket
->get_bucket_id(), formatter
.get());
7253 ret
= rados_driver
->getRados()->bucket_resync_encrypted_multipart(
7254 dpp(), null_yield
, rados_driver
, bucket
->get_info(),
7255 marker
, stream_flusher
);
7259 formatter
->close_section();
7260 formatter
->flush(cout
);
7264 if (opt_cmd
== OPT::BUCKET_CHOWN
) {
7265 if (bucket_name
.empty()) {
7266 cerr
<< "ERROR: bucket name not specified" << std::endl
;
7270 bucket_op
.set_bucket_name(bucket_name
);
7271 bucket_op
.set_new_bucket_name(new_bucket_name
);
7274 int r
= RGWBucketAdminOp::chown(driver
, bucket_op
, marker
, dpp(), &err
);
7276 cerr
<< "failure: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
7281 if (opt_cmd
== OPT::LOG_LIST
) {
7283 if (date
.size() && date
.size() != 10) {
7284 cerr
<< "bad date format for '" << date
<< "', expect YYYY-MM-DD" << std::endl
;
7289 formatter
->open_array_section("logs");
7291 int r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_list_init(dpp(), date
, &h
);
7296 cerr
<< "log list: error " << r
<< std::endl
;
7301 int r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_list_next(h
, &name
);
7305 cerr
<< "log list: error " << r
<< std::endl
;
7308 formatter
->dump_string("object", name
);
7311 formatter
->close_section();
7312 formatter
->flush(cout
);
7316 if (opt_cmd
== OPT::LOG_SHOW
|| opt_cmd
== OPT::LOG_RM
) {
7317 if (object
.empty() && (date
.empty() || bucket_name
.empty() || bucket_id
.empty())) {
7318 cerr
<< "specify an object or a date, bucket and bucket-id" << std::endl
;
7323 if (!object
.empty()) {
7333 if (opt_cmd
== OPT::LOG_SHOW
) {
7336 int r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_show_init(dpp(), oid
, &h
);
7338 cerr
<< "error opening log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
7343 formatter
->open_object_section("log");
7345 struct rgw_log_entry entry
;
7347 // peek at first entry to get bucket metadata
7348 r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_show_next(dpp(), h
, &entry
);
7350 cerr
<< "error reading log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
7353 formatter
->dump_string("bucket_id", entry
.bucket_id
);
7354 formatter
->dump_string("bucket_owner", entry
.bucket_owner
.to_str());
7355 formatter
->dump_string("bucket", entry
.bucket
);
7357 uint64_t agg_time
= 0;
7358 uint64_t agg_bytes_sent
= 0;
7359 uint64_t agg_bytes_received
= 0;
7360 uint64_t total_entries
= 0;
7362 if (show_log_entries
)
7363 formatter
->open_array_section("log_entries");
7366 using namespace std::chrono
;
7367 uint64_t total_time
= duration_cast
<milliseconds
>(entry
.total_time
).count();
7369 agg_time
+= total_time
;
7370 agg_bytes_sent
+= entry
.bytes_sent
;
7371 agg_bytes_received
+= entry
.bytes_received
;
7374 if (skip_zero_entries
&& entry
.bytes_sent
== 0 &&
7375 entry
.bytes_received
== 0)
7378 if (show_log_entries
) {
7380 rgw_format_ops_log_entry(entry
, formatter
.get());
7381 formatter
->flush(cout
);
7384 r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_show_next(dpp(), h
, &entry
);
7388 cerr
<< "error reading log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
7391 if (show_log_entries
)
7392 formatter
->close_section();
7395 formatter
->open_object_section("log_sum");
7396 formatter
->dump_int("bytes_sent", agg_bytes_sent
);
7397 formatter
->dump_int("bytes_received", agg_bytes_received
);
7398 formatter
->dump_int("total_time", agg_time
);
7399 formatter
->dump_int("total_entries", total_entries
);
7400 formatter
->close_section();
7402 formatter
->close_section();
7403 formatter
->flush(cout
);
7406 if (opt_cmd
== OPT::LOG_RM
) {
7407 int r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_remove(dpp(), oid
);
7409 cerr
<< "error removing log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
7415 if (opt_cmd
== OPT::POOL_ADD
) {
7416 if (pool_name
.empty()) {
7417 cerr
<< "need to specify pool to add!" << std::endl
;
7421 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->add_bucket_placement(dpp(), pool
, null_yield
);
7423 cerr
<< "failed to add bucket placement: " << cpp_strerror(-ret
) << std::endl
;
7426 if (opt_cmd
== OPT::POOL_RM
) {
7427 if (pool_name
.empty()) {
7428 cerr
<< "need to specify pool to remove!" << std::endl
;
7432 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->remove_bucket_placement(dpp(), pool
, null_yield
);
7434 cerr
<< "failed to remove bucket placement: " << cpp_strerror(-ret
) << std::endl
;
7437 if (opt_cmd
== OPT::POOLS_LIST
) {
7438 set
<rgw_pool
> pools
;
7439 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->list_placement_set(dpp(), pools
, null_yield
);
7441 cerr
<< "could not list placement set: " << cpp_strerror(-ret
) << std::endl
;
7445 formatter
->open_array_section("pools");
7446 for (auto siter
= pools
.begin(); siter
!= pools
.end(); ++siter
) {
7447 formatter
->open_object_section("pool");
7448 formatter
->dump_string("name", siter
->to_str());
7449 formatter
->close_section();
7451 formatter
->close_section();
7452 formatter
->flush(cout
);
7456 if (opt_cmd
== OPT::USAGE_SHOW
) {
7457 uint64_t start_epoch
= 0;
7458 uint64_t end_epoch
= (uint64_t)-1;
7462 if (!start_date
.empty()) {
7463 ret
= utime_t::parse_date(start_date
, &start_epoch
, NULL
);
7465 cerr
<< "ERROR: failed to parse start date" << std::endl
;
7469 if (!end_date
.empty()) {
7470 ret
= utime_t::parse_date(end_date
, &end_epoch
, NULL
);
7472 cerr
<< "ERROR: failed to parse end date" << std::endl
;
7478 if (!bucket_name
.empty()) {
7479 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7481 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7485 ret
= RGWUsage::show(dpp(), driver
, user
.get(), bucket
.get(), start_epoch
,
7486 end_epoch
, show_log_entries
, show_log_sum
, &categories
,
7489 cerr
<< "ERROR: failed to show usage" << std::endl
;
7494 if (opt_cmd
== OPT::USAGE_TRIM
) {
7495 if (rgw::sal::User::empty(user
) && bucket_name
.empty() &&
7496 start_date
.empty() && end_date
.empty() && !yes_i_really_mean_it
) {
7497 cerr
<< "usage trim without user/date/bucket specified will remove *all* users data" << std::endl
;
7498 cerr
<< "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
7502 uint64_t start_epoch
= 0;
7503 uint64_t end_epoch
= (uint64_t)-1;
7506 if (!start_date
.empty()) {
7507 ret
= utime_t::parse_date(start_date
, &start_epoch
, NULL
);
7509 cerr
<< "ERROR: failed to parse start date" << std::endl
;
7514 if (!end_date
.empty()) {
7515 ret
= utime_t::parse_date(end_date
, &end_epoch
, NULL
);
7517 cerr
<< "ERROR: failed to parse end date" << std::endl
;
7522 if (!bucket_name
.empty()) {
7523 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7525 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7529 ret
= RGWUsage::trim(dpp(), driver
, user
.get(), bucket
.get(), start_epoch
, end_epoch
);
7531 cerr
<< "ERROR: read_usage() returned ret=" << ret
<< std::endl
;
7536 if (opt_cmd
== OPT::USAGE_CLEAR
) {
7537 if (!yes_i_really_mean_it
) {
7538 cerr
<< "usage clear would remove *all* users usage data for all time" << std::endl
;
7539 cerr
<< "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
7543 ret
= RGWUsage::clear(dpp(), driver
);
7550 if (opt_cmd
== OPT::OLH_GET
|| opt_cmd
== OPT::OLH_READLOG
) {
7551 if (bucket_name
.empty()) {
7552 cerr
<< "ERROR: bucket not specified" << std::endl
;
7555 if (object
.empty()) {
7556 cerr
<< "ERROR: object not specified" << std::endl
;
7561 if (opt_cmd
== OPT::OLH_GET
) {
7562 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7564 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7568 rgw_obj
obj(bucket
->get_key(), object
);
7569 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->get_olh(dpp(), bucket
->get_info(), obj
, &olh
);
7571 cerr
<< "ERROR: failed reading olh: " << cpp_strerror(-ret
) << std::endl
;
7574 encode_json("olh", olh
, formatter
.get());
7575 formatter
->flush(cout
);
7578 if (opt_cmd
== OPT::OLH_READLOG
) {
7579 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7581 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7584 map
<uint64_t, vector
<rgw_bucket_olh_log_entry
> > log
;
7587 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(object
);
7591 ret
= obj
->get_obj_state(dpp(), &state
, null_yield
);
7596 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->bucket_index_read_olh_log(dpp(), bucket
->get_info(), *state
, obj
->get_obj(), 0, &log
, &is_truncated
);
7598 cerr
<< "ERROR: failed reading olh: " << cpp_strerror(-ret
) << std::endl
;
7601 formatter
->open_object_section("result");
7602 encode_json("is_truncated", is_truncated
, formatter
.get());
7603 encode_json("log", log
, formatter
.get());
7604 formatter
->close_section();
7605 formatter
->flush(cout
);
7608 if (opt_cmd
== OPT::BI_GET
) {
7609 if (bucket_name
.empty()) {
7610 cerr
<< "ERROR: bucket name not specified" << std::endl
;
7613 if (object
.empty()) {
7614 cerr
<< "ERROR: object not specified" << std::endl
;
7617 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7619 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7622 rgw_obj
obj(bucket
->get_key(), object
);
7623 if (!object_version
.empty()) {
7624 obj
.key
.set_instance(object_version
);
7627 rgw_cls_bi_entry entry
;
7628 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->bi_get(dpp(), bucket
->get_info(), obj
, bi_index_type
, &entry
);
7630 cerr
<< "ERROR: bi_get(): " << cpp_strerror(-ret
) << std::endl
;
7634 encode_json("entry", entry
, formatter
.get());
7635 formatter
->flush(cout
);
7638 if (opt_cmd
== OPT::BI_PUT
) {
7639 if (bucket_name
.empty()) {
7640 cerr
<< "ERROR: bucket name not specified" << std::endl
;
7643 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7645 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7649 rgw_cls_bi_entry entry
;
7650 cls_rgw_obj_key key
;
7651 ret
= read_decode_json(infile
, entry
, &key
);
7656 rgw_obj
obj(bucket
->get_key(), key
);
7658 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->bi_put(dpp(), bucket
->get_key(), obj
, entry
);
7660 cerr
<< "ERROR: bi_put(): " << cpp_strerror(-ret
) << std::endl
;
7665 if (opt_cmd
== OPT::BI_LIST
) {
7666 if (bucket_name
.empty()) {
7667 cerr
<< "ERROR: bucket name not specified" << std::endl
;
7671 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7673 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7677 std::list
<rgw_cls_bi_entry
> entries
;
7679 const auto& index
= bucket
->get_info().layout
.current_index
;
7680 const int max_shards
= rgw::num_shards(index
);
7681 if (max_entries
< 0) {
7685 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<< ": max_entries=" << max_entries
<<
7686 ", index=" << index
<< ", max_shards=" << max_shards
<< dendl
;
7688 formatter
->open_array_section("entries");
7690 int i
= (specified_shard_id
? shard_id
: 0);
7691 for (; i
< max_shards
; i
++) {
7692 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<< ": starting shard=" << i
<< dendl
;
7694 RGWRados::BucketShard
bs(static_cast<rgw::sal::RadosStore
*>(driver
)->getRados());
7695 int ret
= bs
.init(dpp(), bucket
->get_info(), index
, i
);
7699 cerr
<< "ERROR: bs.init(bucket=" << bucket
<< ", shard=" << i
<< "): " << cpp_strerror(-ret
) << std::endl
;
7705 // if object is specified, we use that as a filter to only retrieve some some entries
7706 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->bi_list(bs
, object
, marker
, max_entries
, &entries
, &is_truncated
);
7708 cerr
<< "ERROR: bi_list(): " << cpp_strerror(-ret
) << std::endl
;
7711 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<<
7712 ": bi_list() returned without error; entries.size()=" <<
7713 entries
.size() << ", is_truncated=" << is_truncated
<<
7714 ", marker=" << marker
<< dendl
;
7716 for (const auto& entry
: entries
) {
7717 encode_json("entry", entry
, formatter
.get());
7720 formatter
->flush(cout
);
7721 } while (is_truncated
);
7723 formatter
->flush(cout
);
7725 if (specified_shard_id
) {
7729 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<< ": done" << dendl
;
7731 formatter
->close_section();
7732 formatter
->flush(cout
);
7735 if (opt_cmd
== OPT::BI_PURGE
) {
7736 if (bucket_name
.empty()) {
7737 cerr
<< "ERROR: bucket name not specified" << std::endl
;
7740 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7742 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7746 std::unique_ptr
<rgw::sal::Bucket
> cur_bucket
;
7747 ret
= init_bucket(user
.get(), tenant
, bucket_name
, string(), &cur_bucket
);
7748 if (ret
== -ENOENT
) {
7749 // no bucket entrypoint
7750 } else if (ret
< 0) {
7751 cerr
<< "ERROR: could not init current bucket info for bucket_name=" << bucket_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
7753 } else if (cur_bucket
->get_bucket_id() == bucket
->get_bucket_id() &&
7754 !yes_i_really_mean_it
) {
7755 cerr
<< "specified bucket instance points to a current bucket instance" << std::endl
;
7756 cerr
<< "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
7760 const auto& index
= bucket
->get_info().layout
.current_index
;
7761 if (index
.layout
.type
== rgw::BucketIndexType::Indexless
) {
7762 cerr
<< "ERROR: indexless bucket has no index to purge" << std::endl
;
7766 const int max_shards
= rgw::num_shards(index
);
7767 for (int i
= 0; i
< max_shards
; i
++) {
7768 RGWRados::BucketShard
bs(static_cast<rgw::sal::RadosStore
*>(driver
)->getRados());
7769 int ret
= bs
.init(dpp(), bucket
->get_info(), index
, i
);
7771 cerr
<< "ERROR: bs.init(bucket=" << bucket
<< ", shard=" << i
<< "): " << cpp_strerror(-ret
) << std::endl
;
7775 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->bi_remove(dpp(), bs
);
7777 cerr
<< "ERROR: failed to remove bucket index object: " << cpp_strerror(-ret
) << std::endl
;
7783 if (opt_cmd
== OPT::OBJECT_PUT
) {
7784 if (bucket_name
.empty()) {
7785 cerr
<< "ERROR: bucket not specified" << std::endl
;
7788 if (object
.empty()) {
7789 cerr
<< "ERROR: object not specified" << std::endl
;
7793 RGWDataAccess
data_access(driver
);
7794 rgw_obj_key
key(object
, object_version
);
7796 RGWDataAccess::BucketRef b
;
7797 RGWDataAccess::ObjectRef obj
;
7799 int ret
= data_access
.get_bucket(dpp(), tenant
, bucket_name
, bucket_id
, &b
, null_yield
);
7801 cerr
<< "ERROR: failed to init bucket: " << cpp_strerror(-ret
) << std::endl
;
7805 ret
= b
->get_object(key
, &obj
);
7807 cerr
<< "ERROR: failed to get object: " << cpp_strerror(-ret
) << std::endl
;
7812 ret
= read_input(infile
, bl
);
7814 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
7817 map
<string
, bufferlist
> attrs
;
7818 ret
= obj
->put(bl
, attrs
, dpp(), null_yield
);
7820 cerr
<< "ERROR: put object returned error: " << cpp_strerror(-ret
) << std::endl
;
7824 if (opt_cmd
== OPT::OBJECT_RM
) {
7825 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7827 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7830 rgw_obj_key
key(object
, object_version
);
7831 ret
= rgw_remove_object(dpp(), driver
, bucket
.get(), key
);
7834 cerr
<< "ERROR: object remove returned: " << cpp_strerror(-ret
) << std::endl
;
7839 if (opt_cmd
== OPT::OBJECT_REWRITE
) {
7840 if (bucket_name
.empty()) {
7841 cerr
<< "ERROR: bucket not specified" << std::endl
;
7844 if (object
.empty()) {
7845 cerr
<< "ERROR: object not specified" << std::endl
;
7849 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7851 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7855 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(object
);
7856 obj
->set_instance(object_version
);
7857 bool need_rewrite
= true;
7858 if (min_rewrite_stripe_size
> 0) {
7859 ret
= check_min_obj_stripe_size(driver
, obj
.get(), min_rewrite_stripe_size
, &need_rewrite
);
7861 ldpp_dout(dpp(), 0) << "WARNING: check_min_obj_stripe_size failed, r=" << ret
<< dendl
;
7865 RGWRados
* store
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados();
7866 ret
= store
->rewrite_obj(bucket
->get_info(), obj
->get_obj(), dpp(), null_yield
);
7868 cerr
<< "ERROR: object rewrite returned: " << cpp_strerror(-ret
) << std::endl
;
7872 ldpp_dout(dpp(), 20) << "skipped object" << dendl
;
7874 } // OPT::OBJECT_REWRITE
7876 if (opt_cmd
== OPT::OBJECT_REINDEX
) {
7877 if (bucket_name
.empty()) {
7878 cerr
<< "ERROR: --bucket not specified." << std::endl
;
7881 if (object
.empty() && objects_file
.empty()) {
7882 cerr
<< "ERROR: neither --object nor --objects-file specified." << std::endl
;
7884 } else if (!object
.empty() && !objects_file
.empty()) {
7885 cerr
<< "ERROR: both --object and --objects-file specified and only one is allowed." << std::endl
;
7887 } else if (!objects_file
.empty() && !object_version
.empty()) {
7888 cerr
<< "ERROR: cannot specify --object_version when --objects-file specified." << std::endl
;
7892 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7894 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) <<
7899 rgw::sal::RadosStore
* rados_store
= dynamic_cast<rgw::sal::RadosStore
*>(driver
);
7902 "ERROR: this command can only work when the cluster has a RADOS backing store." <<
7906 RGWRados
* store
= rados_store
->getRados();
7908 auto process
= [&](const std::string
& p_object
, const std::string
& p_object_version
) -> int {
7909 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(p_object
);
7910 obj
->set_instance(p_object_version
);
7911 ret
= store
->reindex_obj(bucket
->get_info(), obj
->get_obj(), dpp(), null_yield
);
7918 if (!object
.empty()) {
7919 ret
= process(object
, object_version
);
7925 file
.open(objects_file
);
7926 if (!file
.is_open()) {
7927 std::cerr
<< "ERROR: unable to open objects-file \"" <<
7928 objects_file
<< "\"." << std::endl
;
7932 std::string obj_name
;
7933 const std::string empty_version
;
7934 while (std::getline(file
, obj_name
)) {
7935 ret
= process(obj_name
, empty_version
);
7937 std::cerr
<< "ERROR: while processing \"" << obj_name
<<
7938 "\", received " << cpp_strerror(-ret
) << "." << std::endl
;
7939 if (!yes_i_really_mean_it
) {
7941 "NOTE: with *caution* you can use --yes-i-really-mean-it to push through errors and continue processing." <<
7948 } // OPT::OBJECT_REINDEX
7950 if (opt_cmd
== OPT::OBJECTS_EXPIRE
) {
7951 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->process_expire_objects(dpp())) {
7952 cerr
<< "ERROR: process_expire_objects() processing returned error." << std::endl
;
7957 if (opt_cmd
== OPT::OBJECTS_EXPIRE_STALE_LIST
) {
7958 ret
= RGWBucketAdminOp::fix_obj_expiry(driver
, bucket_op
, stream_flusher
, dpp(), true);
7960 cerr
<< "ERROR: listing returned " << cpp_strerror(-ret
) << std::endl
;
7965 if (opt_cmd
== OPT::OBJECTS_EXPIRE_STALE_RM
) {
7966 ret
= RGWBucketAdminOp::fix_obj_expiry(driver
, bucket_op
, stream_flusher
, dpp(), false);
7968 cerr
<< "ERROR: removing returned " << cpp_strerror(-ret
) << std::endl
;
7973 if (opt_cmd
== OPT::BUCKET_REWRITE
) {
7974 if (bucket_name
.empty()) {
7975 cerr
<< "ERROR: bucket not specified" << std::endl
;
7979 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7981 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7985 uint64_t start_epoch
= 0;
7986 uint64_t end_epoch
= 0;
7988 if (!end_date
.empty()) {
7989 int ret
= utime_t::parse_date(end_date
, &end_epoch
, NULL
);
7991 cerr
<< "ERROR: failed to parse end date" << std::endl
;
7995 if (!start_date
.empty()) {
7996 int ret
= utime_t::parse_date(start_date
, &start_epoch
, NULL
);
7998 cerr
<< "ERROR: failed to parse start date" << std::endl
;
8003 bool is_truncated
= true;
8004 bool cls_filtered
= true;
8006 rgw_obj_index_key marker
;
8007 string empty_prefix
;
8008 string empty_delimiter
;
8010 formatter
->open_object_section("result");
8011 formatter
->dump_string("bucket", bucket_name
);
8012 formatter
->open_array_section("objects");
8014 constexpr uint32_t NUM_ENTRIES
= 1000;
8015 uint16_t expansion_factor
= 1;
8016 while (is_truncated
) {
8017 RGWRados::ent_map_t result
;
8018 result
.reserve(NUM_ENTRIES
);
8020 const auto& current_index
= bucket
->get_info().layout
.current_index
;
8021 int r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->cls_bucket_list_ordered(
8022 dpp(), bucket
->get_info(), current_index
, RGW_NO_SHARD
,
8023 marker
, empty_prefix
, empty_delimiter
,
8024 NUM_ENTRIES
, true, expansion_factor
,
8025 result
, &is_truncated
, &cls_filtered
, &marker
,
8027 rgw_bucket_object_check_filter
);
8028 if (r
< 0 && r
!= -ENOENT
) {
8029 cerr
<< "ERROR: failed operation r=" << r
<< std::endl
;
8030 } else if (r
== -ENOENT
) {
8034 if (result
.size() < NUM_ENTRIES
/ 8) {
8036 } else if (result
.size() > NUM_ENTRIES
* 7 / 8 &&
8037 expansion_factor
> 1) {
8041 for (auto iter
= result
.begin(); iter
!= result
.end(); ++iter
) {
8042 rgw_obj_key key
= iter
->second
.key
;
8043 rgw_bucket_dir_entry
& entry
= iter
->second
;
8045 formatter
->open_object_section("object");
8046 formatter
->dump_string("name", key
.name
);
8047 formatter
->dump_string("instance", key
.instance
);
8048 formatter
->dump_int("size", entry
.meta
.size
);
8049 utime_t
ut(entry
.meta
.mtime
);
8050 ut
.gmtime(formatter
->dump_stream("mtime"));
8052 if ((entry
.meta
.size
< min_rewrite_size
) ||
8053 (entry
.meta
.size
> max_rewrite_size
) ||
8054 (start_epoch
> 0 && start_epoch
> (uint64_t)ut
.sec()) ||
8055 (end_epoch
> 0 && end_epoch
< (uint64_t)ut
.sec())) {
8056 formatter
->dump_string("status", "Skipped");
8058 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(key
);
8060 bool need_rewrite
= true;
8061 if (min_rewrite_stripe_size
> 0) {
8062 r
= check_min_obj_stripe_size(driver
, obj
.get(), min_rewrite_stripe_size
, &need_rewrite
);
8064 ldpp_dout(dpp(), 0) << "WARNING: check_min_obj_stripe_size failed, r=" << r
<< dendl
;
8067 if (!need_rewrite
) {
8068 formatter
->dump_string("status", "Skipped");
8070 RGWRados
* store
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados();
8071 r
= store
->rewrite_obj(bucket
->get_info(), obj
->get_obj(), dpp(), null_yield
);
8073 formatter
->dump_string("status", "Success");
8075 formatter
->dump_string("status", cpp_strerror(-r
));
8079 formatter
->dump_int("flags", entry
.flags
);
8081 formatter
->close_section();
8082 formatter
->flush(cout
);
8085 formatter
->close_section();
8086 formatter
->close_section();
8087 formatter
->flush(cout
);
8090 if (opt_cmd
== OPT::BUCKET_RESHARD
) {
8091 int ret
= check_reshard_bucket_params(driver
,
8095 num_shards_specified
,
8097 yes_i_really_mean_it
,
8103 auto zone_svc
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
;
8104 if (!zone_svc
->can_reshard()) {
8105 const auto& zonegroup
= zone_svc
->get_zonegroup();
8106 std::cerr
<< "The zonegroup '" << zonegroup
.get_name() << "' does not "
8107 "have the resharding feature enabled." << std::endl
;
8110 if (!RGWBucketReshard::can_reshard(bucket
->get_info(), zone_svc
) &&
8111 !yes_i_really_mean_it
) {
8112 std::cerr
<< "Bucket '" << bucket
->get_name() << "' already has too many "
8113 "log generations (" << bucket
->get_info().layout
.logs
.size() << ") "
8114 "from previous reshards that peer zones haven't finished syncing. "
8115 "Resharding is not recommended until the old generations sync, but "
8116 "you can force a reshard with --yes-i-really-mean-it." << std::endl
;
8120 RGWBucketReshard
br(static_cast<rgw::sal::RadosStore
*>(driver
),
8121 bucket
->get_info(), bucket
->get_attrs(),
8122 nullptr /* no callback */);
8124 #define DEFAULT_RESHARD_MAX_ENTRIES 1000
8125 if (max_entries
< 1) {
8126 max_entries
= DEFAULT_RESHARD_MAX_ENTRIES
;
8129 ReshardFaultInjector fault
;
8130 if (inject_error_at
) {
8131 const int code
= -inject_error_code
.value_or(EIO
);
8132 fault
.inject(*inject_error_at
, InjectError
{code
, dpp()});
8133 } else if (inject_abort_at
) {
8134 fault
.inject(*inject_abort_at
, InjectAbort
{});
8135 } else if (inject_delay_at
) {
8136 fault
.inject(*inject_delay_at
, InjectDelay
{inject_delay
, dpp()});
8138 ret
= br
.execute(num_shards
, fault
, max_entries
, dpp(),
8139 verbose
, &cout
, formatter
.get());
8143 if (opt_cmd
== OPT::RESHARD_ADD
) {
8144 int ret
= check_reshard_bucket_params(driver
,
8148 num_shards_specified
,
8150 yes_i_really_mean_it
,
8156 int num_source_shards
= rgw::current_num_shards(bucket
->get_info().layout
);
8158 RGWReshard
reshard(static_cast<rgw::sal::RadosStore
*>(driver
), dpp());
8159 cls_rgw_reshard_entry entry
;
8160 entry
.time
= real_clock::now();
8161 entry
.tenant
= tenant
;
8162 entry
.bucket_name
= bucket_name
;
8163 entry
.bucket_id
= bucket
->get_info().bucket
.bucket_id
;
8164 entry
.old_num_shards
= num_source_shards
;
8165 entry
.new_num_shards
= num_shards
;
8167 return reshard
.add(dpp(), entry
);
8170 if (opt_cmd
== OPT::RESHARD_LIST
) {
8173 if (max_entries
< 0) {
8178 driver
->ctx()->_conf
.get_val
<uint64_t>("rgw_reshard_num_logs");
8180 RGWReshard
reshard(static_cast<rgw::sal::RadosStore
*>(driver
), dpp());
8182 formatter
->open_array_section("reshard");
8183 for (int i
= 0; i
< num_logshards
; i
++) {
8184 bool is_truncated
= true;
8187 std::list
<cls_rgw_reshard_entry
> entries
;
8188 ret
= reshard
.list(dpp(), i
, marker
, max_entries
- count
, entries
, &is_truncated
);
8190 cerr
<< "Error listing resharding buckets: " << cpp_strerror(-ret
) << std::endl
;
8193 for (const auto& entry
: entries
) {
8194 encode_json("entry", entry
, formatter
.get());
8197 entries
.crbegin()->get_key(&marker
); // last entry's key becomes marker
8199 count
+= entries
.size();
8200 formatter
->flush(cout
);
8201 } while (is_truncated
&& count
< max_entries
);
8203 if (count
>= max_entries
) {
8208 formatter
->close_section();
8209 formatter
->flush(cout
);
8214 if (opt_cmd
== OPT::RESHARD_STATUS
) {
8215 if (bucket_name
.empty()) {
8216 cerr
<< "ERROR: bucket not specified" << std::endl
;
8220 ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8222 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8226 RGWBucketReshard
br(static_cast<rgw::sal::RadosStore
*>(driver
),
8227 bucket
->get_info(), bucket
->get_attrs(),
8228 nullptr /* no callback */);
8229 list
<cls_rgw_bucket_instance_entry
> status
;
8230 int r
= br
.get_status(dpp(), &status
);
8232 cerr
<< "ERROR: could not get resharding status for bucket " <<
8233 bucket_name
<< std::endl
;
8237 show_reshard_status(status
, formatter
.get());
8240 if (opt_cmd
== OPT::RESHARD_PROCESS
) {
8241 RGWReshard
reshard(static_cast<rgw::sal::RadosStore
*>(driver
), true, &cout
);
8243 int ret
= reshard
.process_all_logshards(dpp());
8245 cerr
<< "ERROR: failed to process reshard logs, error=" << cpp_strerror(-ret
) << std::endl
;
8250 if (opt_cmd
== OPT::RESHARD_CANCEL
) {
8251 if (bucket_name
.empty()) {
8252 cerr
<< "ERROR: bucket not specified" << std::endl
;
8256 bool bucket_initable
= true;
8257 ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8259 if (yes_i_really_mean_it
) {
8260 bucket_initable
= false;
8262 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) <<
8263 "; if you want to cancel the reshard request nonetheless, please "
8264 "use the --yes-i-really-mean-it option" << std::endl
;
8269 bool resharding_underway
= true;
8271 if (bucket_initable
) {
8272 // we did not encounter an error, so let's work with the bucket
8273 RGWBucketReshard
br(static_cast<rgw::sal::RadosStore
*>(driver
),
8274 bucket
->get_info(), bucket
->get_attrs(),
8275 nullptr /* no callback */);
8276 int ret
= br
.cancel(dpp());
8278 if (ret
== -EBUSY
) {
8279 cerr
<< "There is ongoing resharding, please retry after " <<
8280 driver
->ctx()->_conf
.get_val
<uint64_t>("rgw_reshard_bucket_lock_duration") <<
8281 " seconds." << std::endl
;
8283 } else if (ret
== -EINVAL
) {
8284 resharding_underway
= false;
8285 // we can continue and try to unschedule
8287 cerr
<< "Error cancelling bucket \"" << bucket_name
<<
8288 "\" resharding: " << cpp_strerror(-ret
) << std::endl
;
8294 RGWReshard
reshard(static_cast<rgw::sal::RadosStore
*>(driver
), dpp());
8296 cls_rgw_reshard_entry entry
;
8297 entry
.tenant
= tenant
;
8298 entry
.bucket_name
= bucket_name
;
8300 ret
= reshard
.remove(dpp(), entry
);
8301 if (ret
== -ENOENT
) {
8302 if (!resharding_underway
) {
8303 cerr
<< "Error, bucket \"" << bucket_name
<<
8304 "\" is neither undergoing resharding nor scheduled to undergo "
8305 "resharding." << std::endl
;
8308 // we cancelled underway resharding above, so we're good
8311 } else if (ret
< 0) {
8312 cerr
<< "Error in updating reshard log with bucket \"" <<
8313 bucket_name
<< "\": " << cpp_strerror(-ret
) << std::endl
;
8316 } // OPT_RESHARD_CANCEL
8318 if (opt_cmd
== OPT::OBJECT_UNLINK
) {
8319 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8321 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8324 list
<rgw_obj_index_key
> oid_list
;
8325 rgw_obj_key
key(object
, object_version
);
8326 rgw_obj_index_key index_key
;
8327 key
.get_index_key(&index_key
);
8328 oid_list
.push_back(index_key
);
8330 // note: under rados this removes directly from rados index objects
8331 ret
= bucket
->remove_objs_from_index(dpp(), oid_list
);
8333 cerr
<< "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret
) << std::endl
;
8338 if (opt_cmd
== OPT::OBJECT_STAT
) {
8339 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8341 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8344 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(object
);
8345 obj
->set_instance(object_version
);
8347 ret
= obj
->get_obj_attrs(null_yield
, dpp());
8349 cerr
<< "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret
) << std::endl
;
8352 formatter
->open_object_section("object_metadata");
8353 formatter
->dump_string("name", object
);
8354 formatter
->dump_unsigned("size", obj
->get_obj_size());
8356 map
<string
, bufferlist
>::iterator iter
;
8357 map
<string
, bufferlist
> other_attrs
;
8358 for (iter
= obj
->get_attrs().begin(); iter
!= obj
->get_attrs().end(); ++iter
) {
8359 bufferlist
& bl
= iter
->second
;
8360 bool handled
= false;
8361 if (iter
->first
== RGW_ATTR_MANIFEST
) {
8362 handled
= decode_dump
<RGWObjManifest
>("manifest", bl
, formatter
.get());
8363 } else if (iter
->first
== RGW_ATTR_ACL
) {
8364 handled
= decode_dump
<RGWAccessControlPolicy
>("policy", bl
, formatter
.get());
8365 } else if (iter
->first
== RGW_ATTR_ID_TAG
) {
8366 handled
= dump_string("tag", bl
, formatter
.get());
8367 } else if (iter
->first
== RGW_ATTR_ETAG
) {
8368 handled
= dump_string("etag", bl
, formatter
.get());
8369 } else if (iter
->first
== RGW_ATTR_COMPRESSION
) {
8370 handled
= decode_dump
<RGWCompressionInfo
>("compression", bl
, formatter
.get());
8371 } else if (iter
->first
== RGW_ATTR_DELETE_AT
) {
8372 handled
= decode_dump
<utime_t
>("delete_at", bl
, formatter
.get());
8376 other_attrs
[iter
->first
] = bl
;
8379 formatter
->open_object_section("attrs");
8380 for (iter
= other_attrs
.begin(); iter
!= other_attrs
.end(); ++iter
) {
8381 dump_string(iter
->first
.c_str(), iter
->second
, formatter
.get());
8383 formatter
->close_section();
8384 formatter
->close_section();
8385 formatter
->flush(cout
);
8388 if (opt_cmd
== OPT::BUCKET_CHECK
) {
8389 if (check_head_obj_locator
) {
8390 if (bucket_name
.empty()) {
8391 cerr
<< "ERROR: need to specify bucket name" << std::endl
;
8394 do_check_object_locator(tenant
, bucket_name
, fix
, remove_bad
, formatter
.get());
8396 RGWBucketAdminOp::check_index(driver
, bucket_op
, stream_flusher
, null_yield
, dpp());
8400 if (opt_cmd
== OPT::BUCKET_CHECK_OLH
) {
8401 rgw::sal::RadosStore
* store
= dynamic_cast<rgw::sal::RadosStore
*>(driver
);
8404 "WARNING: this command is only relevant when the cluster has a RADOS backing store." <<
8408 RGWBucketAdminOp::check_index_olh(store
, bucket_op
, stream_flusher
, dpp());
8411 if (opt_cmd
== OPT::BUCKET_CHECK_UNLINKED
) {
8412 rgw::sal::RadosStore
* store
= dynamic_cast<rgw::sal::RadosStore
*>(driver
);
8415 "WARNING: this command is only relevant when the cluster has a RADOS backing store." <<
8419 RGWBucketAdminOp::check_index_unlinked(store
, bucket_op
, stream_flusher
, dpp());
8422 if (opt_cmd
== OPT::BUCKET_RM
) {
8423 if (!inconsistent_index
) {
8424 RGWBucketAdminOp::remove_bucket(driver
, bucket_op
, null_yield
, dpp(), bypass_gc
, true);
8426 if (!yes_i_really_mean_it
) {
8427 cerr
<< "using --inconsistent_index can corrupt the bucket index " << std::endl
8428 << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
8431 RGWBucketAdminOp::remove_bucket(driver
, bucket_op
, null_yield
, dpp(), bypass_gc
, false);
8435 if (opt_cmd
== OPT::GC_LIST
) {
8438 bool processing_queue
= false;
8439 formatter
->open_array_section("entries");
8442 list
<cls_rgw_gc_obj_info
> result
;
8443 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->list_gc_objs(&index
, marker
, 1000, !include_all
, result
, &truncated
, processing_queue
);
8445 cerr
<< "ERROR: failed to list objs: " << cpp_strerror(-ret
) << std::endl
;
8450 list
<cls_rgw_gc_obj_info
>::iterator iter
;
8451 for (iter
= result
.begin(); iter
!= result
.end(); ++iter
) {
8452 cls_rgw_gc_obj_info
& info
= *iter
;
8453 formatter
->open_object_section("chain_info");
8454 formatter
->dump_string("tag", info
.tag
);
8455 formatter
->dump_stream("time") << info
.time
;
8456 formatter
->open_array_section("objs");
8457 list
<cls_rgw_obj
>::iterator liter
;
8458 cls_rgw_obj_chain
& chain
= info
.chain
;
8459 for (liter
= chain
.objs
.begin(); liter
!= chain
.objs
.end(); ++liter
) {
8460 cls_rgw_obj
& obj
= *liter
;
8461 encode_json("obj", obj
, formatter
.get());
8463 formatter
->close_section(); // objs
8464 formatter
->close_section(); // obj_chain
8465 formatter
->flush(cout
);
8467 } while (truncated
);
8468 formatter
->close_section();
8469 formatter
->flush(cout
);
8472 if (opt_cmd
== OPT::GC_PROCESS
) {
8473 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->process_gc(!include_all
);
8475 cerr
<< "ERROR: gc processing returned error: " << cpp_strerror(-ret
) << std::endl
;
8480 if (opt_cmd
== OPT::LC_LIST
) {
8481 formatter
->open_array_section("lifecycle_list");
8482 vector
<std::unique_ptr
<rgw::sal::Lifecycle::LCEntry
>> bucket_lc_map
;
8485 #define MAX_LC_LIST_ENTRIES 100
8486 if (max_entries
< 0) {
8487 max_entries
= MAX_LC_LIST_ENTRIES
;
8490 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->list_lc_progress(marker
, max_entries
,
8491 bucket_lc_map
, index
);
8493 cerr
<< "ERROR: failed to list objs: " << cpp_strerror(-ret
)
8497 for (const auto& entry
: bucket_lc_map
) {
8498 formatter
->open_object_section("bucket_lc_info");
8499 formatter
->dump_string("bucket", entry
->get_bucket());
8500 formatter
->dump_string("shard", entry
->get_oid());
8502 time_t t
{time_t(entry
->get_start_time())};
8504 exp_buf
, sizeof(exp_buf
),
8505 "%a, %d %b %Y %T %Z", std::gmtime(&t
))) {
8506 formatter
->dump_string("started", exp_buf
);
8508 string lc_status
= LC_STATUS
[entry
->get_status()];
8509 formatter
->dump_string("status", lc_status
);
8510 formatter
->close_section(); // objs
8511 formatter
->flush(cout
);
8513 } while (!bucket_lc_map
.empty());
8515 formatter
->close_section(); //lifecycle list
8516 formatter
->flush(cout
);
8520 if (opt_cmd
== OPT::LC_GET
) {
8521 if (bucket_name
.empty()) {
8522 cerr
<< "ERROR: bucket not specified" << std::endl
;
8526 RGWLifecycleConfiguration config
;
8527 ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8529 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8533 auto aiter
= bucket
->get_attrs().find(RGW_ATTR_LC
);
8534 if (aiter
== bucket
->get_attrs().end()) {
8538 bufferlist::const_iterator iter
{&aiter
->second
};
8540 config
.decode(iter
);
8541 } catch (const buffer::error
& e
) {
8542 cerr
<< "ERROR: decode life cycle config failed" << std::endl
;
8546 encode_json("result", config
, formatter
.get());
8547 formatter
->flush(cout
);
8550 if (opt_cmd
== OPT::LC_PROCESS
) {
8551 if ((! bucket_name
.empty()) ||
8552 (! bucket_id
.empty())) {
8553 int ret
= init_bucket(nullptr, tenant
, bucket_name
, bucket_id
, &bucket
);
8555 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
)
8562 static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->process_lc(bucket
);
8564 cerr
<< "ERROR: lc processing returned error: " << cpp_strerror(-ret
) << std::endl
;
8569 if (opt_cmd
== OPT::LC_RESHARD_FIX
) {
8570 ret
= RGWBucketAdminOp::fix_lc_shards(driver
, bucket_op
, stream_flusher
, dpp());
8572 cerr
<< "ERROR: fixing lc shards: " << cpp_strerror(-ret
) << std::endl
;
8577 if (opt_cmd
== OPT::ORPHANS_FIND
) {
8578 if (!yes_i_really_mean_it
) {
8579 cerr
<< "this command is now deprecated; please consider using the rgw-orphan-list tool; "
8580 << "accidental removal of active objects cannot be reversed; "
8581 << "do you really mean it? (requires --yes-i-really-mean-it)"
8585 cerr
<< "IMPORTANT: this command is now deprecated; please consider using the rgw-orphan-list tool"
8589 RGWOrphanSearch
search(static_cast<rgw::sal::RadosStore
*>(driver
), max_concurrent_ios
, orphan_stale_secs
);
8591 if (job_id
.empty()) {
8592 cerr
<< "ERROR: --job-id not specified" << std::endl
;
8595 if (pool_name
.empty()) {
8596 cerr
<< "ERROR: --pool not specified" << std::endl
;
8600 RGWOrphanSearchInfo info
;
8603 info
.job_name
= job_id
;
8604 info
.num_shards
= num_shards
;
8606 int ret
= search
.init(dpp(), job_id
, &info
, detail
);
8608 cerr
<< "could not init search, ret=" << ret
<< std::endl
;
8611 ret
= search
.run(dpp());
8617 if (opt_cmd
== OPT::ORPHANS_FINISH
) {
8618 if (!yes_i_really_mean_it
) {
8619 cerr
<< "this command is now deprecated; please consider using the rgw-orphan-list tool; "
8620 << "accidental removal of active objects cannot be reversed; "
8621 << "do you really mean it? (requires --yes-i-really-mean-it)"
8625 cerr
<< "IMPORTANT: this command is now deprecated; please consider using the rgw-orphan-list tool"
8629 RGWOrphanSearch
search(static_cast<rgw::sal::RadosStore
*>(driver
), max_concurrent_ios
, orphan_stale_secs
);
8631 if (job_id
.empty()) {
8632 cerr
<< "ERROR: --job-id not specified" << std::endl
;
8635 int ret
= search
.init(dpp(), job_id
, NULL
);
8637 if (ret
== -ENOENT
) {
8638 cerr
<< "job not found" << std::endl
;
8642 ret
= search
.finish();
8648 if (opt_cmd
== OPT::ORPHANS_LIST_JOBS
){
8649 if (!yes_i_really_mean_it
) {
8650 cerr
<< "this command is now deprecated; please consider using the rgw-orphan-list tool; "
8651 << "do you really mean it? (requires --yes-i-really-mean-it)"
8655 cerr
<< "IMPORTANT: this command is now deprecated; please consider using the rgw-orphan-list tool"
8659 RGWOrphanStore
orphan_store(static_cast<rgw::sal::RadosStore
*>(driver
));
8660 int ret
= orphan_store
.init(dpp());
8662 cerr
<< "connection to cluster failed!" << std::endl
;
8666 map
<string
,RGWOrphanSearchState
> m
;
8667 ret
= orphan_store
.list_jobs(m
);
8669 cerr
<< "job list failed" << std::endl
;
8672 formatter
->open_array_section("entries");
8673 for (const auto &it
: m
){
8675 formatter
->dump_string("job-id",it
.first
);
8677 encode_json("orphan_search_state", it
.second
, formatter
.get());
8680 formatter
->close_section();
8681 formatter
->flush(cout
);
8684 if (opt_cmd
== OPT::USER_CHECK
) {
8685 check_bad_user_bucket_mapping(driver
, *user
.get(), fix
, null_yield
, dpp());
8688 if (opt_cmd
== OPT::USER_STATS
) {
8689 if (rgw::sal::User::empty(user
)) {
8690 cerr
<< "ERROR: uid not specified" << std::endl
;
8694 if (!bucket_name
.empty()) {
8695 cerr
<< "ERROR: --reset-stats does not work on buckets and "
8696 "bucket specified" << std::endl
;
8700 cerr
<< "ERROR: sync-stats includes the reset-stats functionality, "
8701 "so at most one of the two should be specified" << std::endl
;
8704 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->user
->reset_bucket_stats(dpp(), user
->get_id(), null_yield
);
8706 cerr
<< "ERROR: could not reset user stats: " << cpp_strerror(-ret
) <<
8713 if (!bucket_name
.empty()) {
8714 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8716 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8719 ret
= bucket
->sync_user_stats(dpp(), null_yield
);
8721 cerr
<< "ERROR: could not sync bucket stats: " <<
8722 cpp_strerror(-ret
) << std::endl
;
8726 int ret
= rgw_user_sync_all_stats(dpp(), driver
, user
.get(), null_yield
);
8728 cerr
<< "ERROR: could not sync user stats: " <<
8729 cpp_strerror(-ret
) << std::endl
;
8735 constexpr bool omit_utilized_stats
= false;
8736 RGWStorageStats
stats(omit_utilized_stats
);
8737 ceph::real_time last_stats_sync
;
8738 ceph::real_time last_stats_update
;
8739 int ret
= user
->read_stats(dpp(), null_yield
, &stats
, &last_stats_sync
, &last_stats_update
);
8741 if (ret
== -ENOENT
) { /* in case of ENOENT */
8742 cerr
<< "User has not been initialized or user does not exist" << std::endl
;
8744 cerr
<< "ERROR: can't read user: " << cpp_strerror(ret
) << std::endl
;
8751 Formatter::ObjectSection
os(*formatter
, "result");
8752 encode_json("stats", stats
, formatter
.get());
8753 utime_t
last_sync_ut(last_stats_sync
);
8754 encode_json("last_stats_sync", last_sync_ut
, formatter
.get());
8755 utime_t
last_update_ut(last_stats_update
);
8756 encode_json("last_stats_update", last_update_ut
, formatter
.get());
8758 formatter
->flush(cout
);
8761 if (opt_cmd
== OPT::METADATA_GET
) {
8762 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->get(metadata_key
, formatter
.get(), null_yield
, dpp());
8764 cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
8768 formatter
->flush(cout
);
8771 if (opt_cmd
== OPT::METADATA_PUT
) {
8773 int ret
= read_input(infile
, bl
);
8775 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
8778 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->put(metadata_key
, bl
, null_yield
, dpp(), RGWMDLogSyncType::APPLY_ALWAYS
, false);
8780 cerr
<< "ERROR: can't put key: " << cpp_strerror(-ret
) << std::endl
;
8785 if (opt_cmd
== OPT::METADATA_RM
) {
8786 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->remove(metadata_key
, null_yield
, dpp());
8788 cerr
<< "ERROR: can't remove key: " << cpp_strerror(-ret
) << std::endl
;
8793 if (opt_cmd
== OPT::METADATA_LIST
|| opt_cmd
== OPT::USER_LIST
) {
8794 if (opt_cmd
== OPT::USER_LIST
) {
8795 metadata_key
= "user";
8799 int ret
= driver
->meta_list_keys_init(dpp(), metadata_key
, marker
, &handle
);
8801 cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
8808 if (max_entries_specified
) {
8809 formatter
->open_object_section("result");
8811 formatter
->open_array_section("keys");
8816 left
= (max_entries_specified
? max_entries
- count
: max
);
8817 ret
= driver
->meta_list_keys_next(dpp(), handle
, left
, keys
, &truncated
);
8818 if (ret
< 0 && ret
!= -ENOENT
) {
8819 cerr
<< "ERROR: lists_keys_next(): " << cpp_strerror(-ret
) << std::endl
;
8821 } if (ret
!= -ENOENT
) {
8822 for (list
<string
>::iterator iter
= keys
.begin(); iter
!= keys
.end(); ++iter
) {
8823 formatter
->dump_string("key", *iter
);
8826 formatter
->flush(cout
);
8828 } while (truncated
&& left
> 0);
8830 formatter
->close_section();
8832 if (max_entries_specified
) {
8833 encode_json("truncated", truncated
, formatter
.get());
8834 encode_json("count", count
, formatter
.get());
8836 encode_json("marker", driver
->meta_get_marker(handle
), formatter
.get());
8838 formatter
->close_section();
8840 formatter
->flush(cout
);
8842 driver
->meta_list_keys_complete(handle
);
8845 if (opt_cmd
== OPT::MDLOG_LIST
) {
8846 if (!start_date
.empty()) {
8847 std::cerr
<< "start-date not allowed." << std::endl
;
8850 if (!end_date
.empty()) {
8851 std::cerr
<< "end-date not allowed." << std::endl
;
8854 if (!end_marker
.empty()) {
8855 std::cerr
<< "end-marker not allowed." << std::endl
;
8858 if (!start_marker
.empty()) {
8859 if (marker
.empty()) {
8860 marker
= start_marker
;
8862 std::cerr
<< "start-marker and marker not both allowed." << std::endl
;
8867 int i
= (specified_shard_id
? shard_id
: 0);
8869 if (period_id
.empty()) {
8870 // use realm's current period
8872 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
8873 realm_id
, realm_name
, realm
);
8875 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
8878 period_id
= realm
.current_period
;
8879 std::cerr
<< "No --period given, using current period="
8880 << period_id
<< std::endl
;
8882 RGWMetadataLog
*meta_log
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->mdlog
->get_log(period_id
);
8884 formatter
->open_array_section("entries");
8885 for (; i
< g_ceph_context
->_conf
->rgw_md_log_max_shards
; i
++) {
8887 list
<cls_log_entry
> entries
;
8889 meta_log
->init_list_entries(i
, {}, {}, marker
, &handle
);
8892 int ret
= meta_log
->list_entries(dpp(), handle
, 1000, entries
, NULL
, &truncated
);
8894 cerr
<< "ERROR: meta_log->list_entries(): " << cpp_strerror(-ret
) << std::endl
;
8898 for (list
<cls_log_entry
>::iterator iter
= entries
.begin(); iter
!= entries
.end(); ++iter
) {
8899 cls_log_entry
& entry
= *iter
;
8900 static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->dump_log_entry(entry
, formatter
.get());
8902 formatter
->flush(cout
);
8903 } while (truncated
);
8905 meta_log
->complete_list_entries(handle
);
8907 if (specified_shard_id
)
8912 formatter
->close_section();
8913 formatter
->flush(cout
);
8916 if (opt_cmd
== OPT::MDLOG_STATUS
) {
8917 int i
= (specified_shard_id
? shard_id
: 0);
8919 if (period_id
.empty()) {
8920 // use realm's current period
8922 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
8923 realm_id
, realm_name
, realm
);
8925 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
8928 period_id
= realm
.current_period
;
8929 std::cerr
<< "No --period given, using current period="
8930 << period_id
<< std::endl
;
8932 RGWMetadataLog
*meta_log
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->mdlog
->get_log(period_id
);
8934 formatter
->open_array_section("entries");
8936 for (; i
< g_ceph_context
->_conf
->rgw_md_log_max_shards
; i
++) {
8937 RGWMetadataLogInfo info
;
8938 meta_log
->get_info(dpp(), i
, &info
);
8940 ::encode_json("info", info
, formatter
.get());
8942 if (specified_shard_id
)
8947 formatter
->close_section();
8948 formatter
->flush(cout
);
8951 if (opt_cmd
== OPT::MDLOG_AUTOTRIM
) {
8952 // need a full history for purging old mdlog periods
8953 static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->mdlog
->init_oldest_log_period(null_yield
, dpp());
8955 RGWCoroutinesManager
crs(driver
->ctx(), driver
->get_cr_registry());
8956 RGWHTTPManager
http(driver
->ctx(), crs
.get_completion_mgr());
8957 int ret
= http
.start();
8959 cerr
<< "failed to initialize http client with " << cpp_strerror(ret
) << std::endl
;
8963 auto num_shards
= g_conf()->rgw_md_log_max_shards
;
8964 auto mltcr
= create_admin_meta_log_trim_cr(
8965 dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), &http
, num_shards
);
8967 cerr
<< "Cluster misconfigured! Unable to trim." << std::endl
;
8970 ret
= crs
.run(dpp(), mltcr
);
8972 cerr
<< "automated mdlog trim failed with " << cpp_strerror(ret
) << std::endl
;
8977 if (opt_cmd
== OPT::MDLOG_TRIM
) {
8978 if (!start_date
.empty()) {
8979 std::cerr
<< "start-date not allowed." << std::endl
;
8982 if (!end_date
.empty()) {
8983 std::cerr
<< "end-date not allowed." << std::endl
;
8986 if (!start_marker
.empty()) {
8987 std::cerr
<< "start-marker not allowed." << std::endl
;
8990 if (!end_marker
.empty()) {
8991 if (marker
.empty()) {
8992 marker
= end_marker
;
8994 std::cerr
<< "end-marker and marker not both allowed." << std::endl
;
8999 if (!specified_shard_id
) {
9000 cerr
<< "ERROR: shard-id must be specified for trim operation" << std::endl
;
9004 if (marker
.empty()) {
9005 cerr
<< "ERROR: marker must be specified for trim operation" << std::endl
;
9009 if (period_id
.empty()) {
9010 std::cerr
<< "missing --period argument" << std::endl
;
9013 RGWMetadataLog
*meta_log
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->mdlog
->get_log(period_id
);
9015 // trim until -ENODATA
9017 ret
= meta_log
->trim(dpp(), shard_id
, {}, {}, {}, marker
);
9019 if (ret
< 0 && ret
!= -ENODATA
) {
9020 cerr
<< "ERROR: meta_log->trim(): " << cpp_strerror(-ret
) << std::endl
;
9025 if (opt_cmd
== OPT::SYNC_INFO
) {
9026 sync_info(opt_effective_zone_id
, opt_bucket
, zone_formatter
.get());
9029 if (opt_cmd
== OPT::SYNC_STATUS
) {
9030 sync_status(formatter
.get());
9033 if (opt_cmd
== OPT::METADATA_SYNC_STATUS
) {
9034 RGWMetaSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor());
9036 int ret
= sync
.init(dpp());
9038 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
9042 rgw_meta_sync_status sync_status
;
9043 ret
= sync
.read_sync_status(dpp(), &sync_status
);
9045 cerr
<< "ERROR: sync.read_sync_status() returned ret=" << ret
<< std::endl
;
9049 formatter
->open_object_section("summary");
9050 encode_json("sync_status", sync_status
, formatter
.get());
9052 uint64_t full_total
= 0;
9053 uint64_t full_complete
= 0;
9055 for (auto marker_iter
: sync_status
.sync_markers
) {
9056 full_total
+= marker_iter
.second
.total_entries
;
9057 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::FullSync
) {
9058 full_complete
+= marker_iter
.second
.pos
;
9060 full_complete
+= marker_iter
.second
.total_entries
;
9064 formatter
->open_object_section("full_sync");
9065 encode_json("total", full_total
, formatter
.get());
9066 encode_json("complete", full_complete
, formatter
.get());
9067 formatter
->close_section();
9068 formatter
->dump_string("current_time",
9069 to_iso_8601(ceph::real_clock::now(),
9070 iso_8601_format::YMDhms
));
9071 formatter
->close_section();
9073 formatter
->flush(cout
);
9077 if (opt_cmd
== OPT::METADATA_SYNC_INIT
) {
9078 RGWMetaSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor());
9080 int ret
= sync
.init(dpp());
9082 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
9085 ret
= sync
.init_sync_status(dpp());
9087 cerr
<< "ERROR: sync.init_sync_status() returned ret=" << ret
<< std::endl
;
9093 if (opt_cmd
== OPT::METADATA_SYNC_RUN
) {
9094 RGWMetaSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor());
9096 int ret
= sync
.init(dpp());
9098 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
9102 ret
= sync
.run(dpp(), null_yield
);
9104 cerr
<< "ERROR: sync.run() returned ret=" << ret
<< std::endl
;
9109 if (opt_cmd
== OPT::DATA_SYNC_STATUS
) {
9110 if (source_zone
.empty()) {
9111 cerr
<< "ERROR: source zone not specified" << std::endl
;
9114 RGWDataSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor(), source_zone
, nullptr);
9116 int ret
= sync
.init(dpp());
9118 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
9122 rgw_data_sync_status sync_status
;
9123 if (specified_shard_id
) {
9124 set
<string
> pending_buckets
;
9125 set
<string
> recovering_buckets
;
9126 rgw_data_sync_marker sync_marker
;
9127 ret
= sync
.read_shard_status(dpp(), shard_id
, pending_buckets
, recovering_buckets
, &sync_marker
,
9128 max_entries_specified
? max_entries
: 20);
9129 if (ret
< 0 && ret
!= -ENOENT
) {
9130 cerr
<< "ERROR: sync.read_shard_status() returned ret=" << ret
<< std::endl
;
9133 formatter
->open_object_section("summary");
9134 encode_json("shard_id", shard_id
, formatter
.get());
9135 encode_json("marker", sync_marker
, formatter
.get());
9136 encode_json("pending_buckets", pending_buckets
, formatter
.get());
9137 encode_json("recovering_buckets", recovering_buckets
, formatter
.get());
9138 formatter
->dump_string("current_time",
9139 to_iso_8601(ceph::real_clock::now(),
9140 iso_8601_format::YMDhms
));
9141 formatter
->close_section();
9142 formatter
->flush(cout
);
9144 ret
= sync
.read_sync_status(dpp(), &sync_status
);
9145 if (ret
< 0 && ret
!= -ENOENT
) {
9146 cerr
<< "ERROR: sync.read_sync_status() returned ret=" << ret
<< std::endl
;
9150 formatter
->open_object_section("summary");
9151 encode_json("sync_status", sync_status
, formatter
.get());
9153 uint64_t full_total
= 0;
9154 uint64_t full_complete
= 0;
9156 for (auto marker_iter
: sync_status
.sync_markers
) {
9157 full_total
+= marker_iter
.second
.total_entries
;
9158 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::FullSync
) {
9159 full_complete
+= marker_iter
.second
.pos
;
9161 full_complete
+= marker_iter
.second
.total_entries
;
9165 formatter
->open_object_section("full_sync");
9166 encode_json("total", full_total
, formatter
.get());
9167 encode_json("complete", full_complete
, formatter
.get());
9168 formatter
->close_section();
9169 formatter
->dump_string("current_time",
9170 to_iso_8601(ceph::real_clock::now(),
9171 iso_8601_format::YMDhms
));
9172 formatter
->close_section();
9174 formatter
->flush(cout
);
9178 if (opt_cmd
== OPT::DATA_SYNC_INIT
) {
9179 if (source_zone
.empty()) {
9180 cerr
<< "ERROR: source zone not specified" << std::endl
;
9184 RGWDataSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor(), source_zone
, nullptr);
9186 int ret
= sync
.init(dpp());
9188 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
9192 ret
= sync
.init_sync_status(dpp());
9194 cerr
<< "ERROR: sync.init_sync_status() returned ret=" << ret
<< std::endl
;
9199 if (opt_cmd
== OPT::DATA_SYNC_RUN
) {
9200 if (source_zone
.empty()) {
9201 cerr
<< "ERROR: source zone not specified" << std::endl
;
9205 RGWSyncModuleInstanceRef sync_module
;
9206 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->sync_modules
->get_manager()->create_instance(dpp(), g_ceph_context
, static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zone().tier_type
,
9207 static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zone_params().tier_config
, &sync_module
);
9209 ldpp_dout(dpp(), -1) << "ERROR: failed to init sync module instance, ret=" << ret
<< dendl
;
9213 RGWDataSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor(), source_zone
, nullptr, sync_module
);
9215 ret
= sync
.init(dpp());
9217 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
9221 ret
= sync
.run(dpp());
9223 cerr
<< "ERROR: sync.run() returned ret=" << ret
<< std::endl
;
9228 if (opt_cmd
== OPT::BUCKET_SYNC_INIT
) {
9229 if (source_zone
.empty()) {
9230 cerr
<< "ERROR: source zone not specified" << std::endl
;
9233 if (bucket_name
.empty()) {
9234 cerr
<< "ERROR: bucket not specified" << std::endl
;
9237 int ret
= init_bucket_for_sync(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9241 auto opt_sb
= opt_source_bucket
;
9242 if (opt_sb
&& opt_sb
->bucket_id
.empty()) {
9244 std::unique_ptr
<rgw::sal::Bucket
> sbuck
;
9245 int ret
= init_bucket_for_sync(user
.get(), opt_sb
->tenant
, opt_sb
->name
, sbid
, &sbuck
);
9249 opt_sb
= sbuck
->get_key();
9252 auto sync
= RGWBucketPipeSyncStatusManager::construct(
9253 dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), source_zone
, opt_sb
,
9254 bucket
->get_key(), extra_info
? &std::cout
: nullptr);
9257 cerr
<< "ERROR: sync.init() returned error=" << sync
.error() << std::endl
;
9258 return -sync
.error();
9260 ret
= (*sync
)->init_sync_status(dpp());
9262 cerr
<< "ERROR: sync.init_sync_status() returned ret=" << ret
<< std::endl
;
9267 if (opt_cmd
== OPT::BUCKET_SYNC_CHECKPOINT
) {
9268 std::optional
<rgw_zone_id
> opt_source_zone
;
9269 if (!source_zone
.empty()) {
9270 opt_source_zone
= source_zone
;
9272 if (bucket_name
.empty()) {
9273 cerr
<< "ERROR: bucket not specified" << std::endl
;
9276 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9281 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->bucket
->bucket_imports_data(bucket
->get_key(), null_yield
, dpp())) {
9282 std::cout
<< "Sync is disabled for bucket " << bucket_name
<< std::endl
;
9286 RGWBucketSyncPolicyHandlerRef handler
;
9287 ret
= driver
->get_sync_policy_handler(dpp(), std::nullopt
, bucket
->get_key(), &handler
, null_yield
);
9289 std::cerr
<< "ERROR: failed to get policy handler for bucket ("
9290 << bucket
<< "): r=" << ret
<< ": " << cpp_strerror(-ret
) << std::endl
;
9294 auto timeout_at
= ceph::coarse_mono_clock::now() + opt_timeout_sec
;
9295 ret
= rgw_bucket_sync_checkpoint(dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), *handler
, bucket
->get_info(),
9296 opt_source_zone
, opt_source_bucket
,
9297 opt_retry_delay_ms
, timeout_at
);
9299 ldpp_dout(dpp(), -1) << "bucket sync checkpoint failed: " << cpp_strerror(ret
) << dendl
;
9304 if ((opt_cmd
== OPT::BUCKET_SYNC_DISABLE
) || (opt_cmd
== OPT::BUCKET_SYNC_ENABLE
)) {
9305 if (bucket_name
.empty()) {
9306 cerr
<< "ERROR: bucket not specified" << std::endl
;
9309 if (opt_cmd
== OPT::BUCKET_SYNC_DISABLE
) {
9310 bucket_op
.set_sync_bucket(false);
9312 bucket_op
.set_sync_bucket(true);
9314 bucket_op
.set_tenant(tenant
);
9316 ret
= RGWBucketAdminOp::sync_bucket(driver
, bucket_op
, dpp(), &err_msg
);
9318 cerr
<< err_msg
<< std::endl
;
9323 if (opt_cmd
== OPT::BUCKET_SYNC_INFO
) {
9324 if (bucket_name
.empty()) {
9325 cerr
<< "ERROR: bucket not specified" << std::endl
;
9328 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9332 bucket_sync_info(driver
, bucket
->get_info(), std::cout
);
9335 if (opt_cmd
== OPT::BUCKET_SYNC_STATUS
) {
9336 if (bucket_name
.empty()) {
9337 cerr
<< "ERROR: bucket not specified" << std::endl
;
9340 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9344 bucket_sync_status(driver
, bucket
->get_info(), source_zone
, opt_source_bucket
, std::cout
);
9347 if (opt_cmd
== OPT::BUCKET_SYNC_MARKERS
) {
9348 if (source_zone
.empty()) {
9349 cerr
<< "ERROR: source zone not specified" << std::endl
;
9352 if (bucket_name
.empty()) {
9353 cerr
<< "ERROR: bucket not specified" << std::endl
;
9356 int ret
= init_bucket_for_sync(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9360 auto sync
= RGWBucketPipeSyncStatusManager::construct(
9361 dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), source_zone
,
9362 opt_source_bucket
, bucket
->get_key(), nullptr);
9365 cerr
<< "ERROR: sync.init() returned error=" << sync
.error() << std::endl
;
9366 return -sync
.error();
9369 auto sync_status
= (*sync
)->read_sync_status(dpp());
9371 cerr
<< "ERROR: sync.read_sync_status() returned error="
9372 << sync_status
.error() << std::endl
;
9373 return -sync_status
.error();
9376 encode_json("sync_status", *sync_status
, formatter
.get());
9377 formatter
->flush(cout
);
9380 if (opt_cmd
== OPT::BUCKET_SYNC_RUN
) {
9381 if (source_zone
.empty()) {
9382 cerr
<< "ERROR: source zone not specified" << std::endl
;
9385 if (bucket_name
.empty()) {
9386 cerr
<< "ERROR: bucket not specified" << std::endl
;
9389 int ret
= init_bucket_for_sync(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9393 auto sync
= RGWBucketPipeSyncStatusManager::construct(
9394 dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), source_zone
,
9395 opt_source_bucket
, bucket
->get_key(), extra_info
? &std::cout
: nullptr);
9398 cerr
<< "ERROR: sync.init() returned error=" << sync
.error() << std::endl
;
9399 return -sync
.error();
9402 ret
= (*sync
)->run(dpp());
9404 cerr
<< "ERROR: sync.run() returned ret=" << ret
<< std::endl
;
9409 if (opt_cmd
== OPT::BILOG_LIST
) {
9410 if (bucket_name
.empty()) {
9411 cerr
<< "ERROR: bucket not specified" << std::endl
;
9414 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9416 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
9419 formatter
->open_array_section("entries");
9422 if (max_entries
< 0)
9425 const auto& logs
= bucket
->get_info().layout
.logs
;
9426 auto log_layout
= std::reference_wrapper
{logs
.back()};
9428 auto i
= std::find_if(logs
.begin(), logs
.end(), rgw::matches_gen(*gen
));
9429 if (i
== logs
.end()) {
9430 cerr
<< "ERROR: no log layout with gen=" << *gen
<< std::endl
;
9437 list
<rgw_bi_log_entry
> entries
;
9438 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->bilog_rados
->log_list(dpp(), bucket
->get_info(), log_layout
, shard_id
, marker
, max_entries
- count
, entries
, &truncated
);
9440 cerr
<< "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret
) << std::endl
;
9444 count
+= entries
.size();
9446 for (list
<rgw_bi_log_entry
>::iterator iter
= entries
.begin(); iter
!= entries
.end(); ++iter
) {
9447 rgw_bi_log_entry
& entry
= *iter
;
9448 encode_json("entry", entry
, formatter
.get());
9452 formatter
->flush(cout
);
9453 } while (truncated
&& count
< max_entries
);
9455 formatter
->close_section();
9456 formatter
->flush(cout
);
9459 if (opt_cmd
== OPT::SYNC_ERROR_LIST
) {
9460 if (max_entries
< 0) {
9463 if (!start_date
.empty()) {
9464 std::cerr
<< "start-date not allowed." << std::endl
;
9467 if (!end_date
.empty()) {
9468 std::cerr
<< "end-date not allowed." << std::endl
;
9471 if (!end_marker
.empty()) {
9472 std::cerr
<< "end-marker not allowed." << std::endl
;
9475 if (!start_marker
.empty()) {
9476 if (marker
.empty()) {
9477 marker
= start_marker
;
9479 std::cerr
<< "start-marker and marker not both allowed." << std::endl
;
9490 formatter
->open_array_section("entries");
9492 for (; shard_id
< ERROR_LOGGER_SHARDS
; ++shard_id
) {
9493 formatter
->open_object_section("shard");
9494 encode_json("shard_id", shard_id
, formatter
.get());
9495 formatter
->open_array_section("entries");
9498 string oid
= RGWSyncErrorLogger::get_shard_oid(RGW_SYNC_ERROR_LOG_SHARD_PREFIX
, shard_id
);
9501 list
<cls_log_entry
> entries
;
9502 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->timelog
.list(dpp(), oid
, {}, {}, max_entries
- count
, entries
, marker
, &marker
, &truncated
,
9504 if (ret
== -ENOENT
) {
9508 cerr
<< "ERROR: svc.cls->timelog.list(): " << cpp_strerror(-ret
) << std::endl
;
9512 count
+= entries
.size();
9514 for (auto& cls_entry
: entries
) {
9515 rgw_sync_error_info log_entry
;
9517 auto iter
= cls_entry
.data
.cbegin();
9519 decode(log_entry
, iter
);
9520 } catch (buffer::error
& err
) {
9521 cerr
<< "ERROR: failed to decode log entry" << std::endl
;
9524 formatter
->open_object_section("entry");
9525 encode_json("id", cls_entry
.id
, formatter
.get());
9526 encode_json("section", cls_entry
.section
, formatter
.get());
9527 encode_json("name", cls_entry
.name
, formatter
.get());
9528 encode_json("timestamp", cls_entry
.timestamp
, formatter
.get());
9529 encode_json("info", log_entry
, formatter
.get());
9530 formatter
->close_section();
9531 formatter
->flush(cout
);
9533 } while (truncated
&& count
< max_entries
);
9535 formatter
->close_section();
9536 formatter
->close_section();
9538 if (specified_shard_id
) {
9543 formatter
->close_section();
9544 formatter
->flush(cout
);
9547 if (opt_cmd
== OPT::SYNC_ERROR_TRIM
) {
9548 if (!start_date
.empty()) {
9549 std::cerr
<< "start-date not allowed." << std::endl
;
9552 if (!end_date
.empty()) {
9553 std::cerr
<< "end-date not allowed." << std::endl
;
9556 if (!start_marker
.empty()) {
9557 std::cerr
<< "end-date not allowed." << std::endl
;
9560 if (!end_marker
.empty()) {
9561 std::cerr
<< "end-date not allowed." << std::endl
;
9569 for (; shard_id
< ERROR_LOGGER_SHARDS
; ++shard_id
) {
9570 ret
= trim_sync_error_log(shard_id
, marker
, trim_delay_ms
);
9572 cerr
<< "ERROR: sync error trim: " << cpp_strerror(-ret
) << std::endl
;
9575 if (specified_shard_id
) {
9581 if (opt_cmd
== OPT::SYNC_GROUP_CREATE
||
9582 opt_cmd
== OPT::SYNC_GROUP_MODIFY
) {
9583 CHECK_TRUE(require_non_empty_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9584 CHECK_TRUE(require_opt(opt_status
), "ERROR: --status is not specified (options: forbidden, allowed, enabled)", EINVAL
);
9586 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9587 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9591 auto& sync_policy
= sync_policy_ctx
.get_policy();
9593 if (opt_cmd
== OPT::SYNC_GROUP_MODIFY
) {
9594 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9595 if (iter
== sync_policy
.groups
.end()) {
9596 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9601 auto& group
= sync_policy
.groups
[*opt_group_id
];
9602 group
.id
= *opt_group_id
;
9605 if (!group
.set_status(*opt_status
)) {
9606 cerr
<< "ERROR: unrecognized status (options: forbidden, allowed, enabled)" << std::endl
;
9611 ret
= sync_policy_ctx
.write_policy();
9616 show_result(sync_policy
, zone_formatter
.get(), cout
);
9619 if (opt_cmd
== OPT::SYNC_GROUP_GET
) {
9620 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9621 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9625 auto& sync_policy
= sync_policy_ctx
.get_policy();
9627 auto& groups
= sync_policy
.groups
;
9629 if (!opt_group_id
) {
9630 show_result(groups
, zone_formatter
.get(), cout
);
9632 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9633 if (iter
== sync_policy
.groups
.end()) {
9634 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9638 show_result(iter
->second
, zone_formatter
.get(), cout
);
9642 if (opt_cmd
== OPT::SYNC_GROUP_REMOVE
) {
9643 CHECK_TRUE(require_non_empty_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9645 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9646 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9650 auto& sync_policy
= sync_policy_ctx
.get_policy();
9652 sync_policy
.groups
.erase(*opt_group_id
);
9654 ret
= sync_policy_ctx
.write_policy();
9660 Formatter::ObjectSection
os(*zone_formatter
.get(), "result");
9661 encode_json("sync_policy", sync_policy
, zone_formatter
.get());
9664 zone_formatter
->flush(cout
);
9667 if (opt_cmd
== OPT::SYNC_GROUP_FLOW_CREATE
) {
9668 CHECK_TRUE(require_non_empty_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9669 CHECK_TRUE(require_non_empty_opt(opt_flow_id
), "ERROR: --flow-id not specified", EINVAL
);
9670 CHECK_TRUE(require_opt(opt_flow_type
),
9671 "ERROR: --flow-type not specified (options: symmetrical, directional)", EINVAL
);
9672 CHECK_TRUE((symmetrical_flow_opt(*opt_flow_type
) ||
9673 directional_flow_opt(*opt_flow_type
)),
9674 "ERROR: --flow-type invalid (options: symmetrical, directional)", EINVAL
);
9676 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9677 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9681 auto& sync_policy
= sync_policy_ctx
.get_policy();
9683 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9684 if (iter
== sync_policy
.groups
.end()) {
9685 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9689 auto& group
= iter
->second
;
9691 if (symmetrical_flow_opt(*opt_flow_type
)) {
9692 CHECK_TRUE(require_non_empty_opt(opt_zone_ids
), "ERROR: --zones not provided for symmetrical flow, or is empty", EINVAL
);
9694 rgw_sync_symmetric_group
*flow_group
;
9696 group
.data_flow
.find_or_create_symmetrical(*opt_flow_id
, &flow_group
);
9698 for (auto& z
: *opt_zone_ids
) {
9699 flow_group
->zones
.insert(z
);
9701 } else { /* directional */
9702 CHECK_TRUE(require_non_empty_opt(opt_source_zone_id
), "ERROR: --source-zone not provided for directional flow rule, or is empty", EINVAL
);
9703 CHECK_TRUE(require_non_empty_opt(opt_dest_zone_id
), "ERROR: --dest-zone not provided for directional flow rule, or is empty", EINVAL
);
9705 rgw_sync_directional_rule
*flow_rule
;
9707 group
.data_flow
.find_or_create_directional(*opt_source_zone_id
, *opt_dest_zone_id
, &flow_rule
);
9710 ret
= sync_policy_ctx
.write_policy();
9715 show_result(sync_policy
, zone_formatter
.get(), cout
);
9718 if (opt_cmd
== OPT::SYNC_GROUP_FLOW_REMOVE
) {
9719 CHECK_TRUE(require_non_empty_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9720 CHECK_TRUE(require_non_empty_opt(opt_flow_id
), "ERROR: --flow-id not specified", EINVAL
);
9721 CHECK_TRUE(require_opt(opt_flow_type
),
9722 "ERROR: --flow-type not specified (options: symmetrical, directional)", EINVAL
);
9723 CHECK_TRUE((symmetrical_flow_opt(*opt_flow_type
) ||
9724 directional_flow_opt(*opt_flow_type
)),
9725 "ERROR: --flow-type invalid (options: symmetrical, directional)", EINVAL
);
9727 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9728 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9732 auto& sync_policy
= sync_policy_ctx
.get_policy();
9734 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9735 if (iter
== sync_policy
.groups
.end()) {
9736 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9740 auto& group
= iter
->second
;
9742 if (symmetrical_flow_opt(*opt_flow_type
)) {
9743 group
.data_flow
.remove_symmetrical(*opt_flow_id
, opt_zone_ids
);
9744 } else { /* directional */
9745 CHECK_TRUE(require_non_empty_opt(opt_source_zone_id
), "ERROR: --source-zone not provided for directional flow rule, or is empty", EINVAL
);
9746 CHECK_TRUE(require_non_empty_opt(opt_dest_zone_id
), "ERROR: --dest-zone not provided for directional flow rule, or is empty", EINVAL
);
9748 group
.data_flow
.remove_directional(*opt_source_zone_id
, *opt_dest_zone_id
);
9751 ret
= sync_policy_ctx
.write_policy();
9756 show_result(sync_policy
, zone_formatter
.get(), cout
);
9759 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_CREATE
||
9760 opt_cmd
== OPT::SYNC_GROUP_PIPE_MODIFY
) {
9761 CHECK_TRUE(require_non_empty_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9762 CHECK_TRUE(require_non_empty_opt(opt_pipe_id
), "ERROR: --pipe-id not specified", EINVAL
);
9763 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_CREATE
) {
9764 CHECK_TRUE(require_non_empty_opt(opt_source_zone_ids
), "ERROR: --source-zones not provided or is empty; should be list of zones or '*'", EINVAL
);
9765 CHECK_TRUE(require_non_empty_opt(opt_dest_zone_ids
), "ERROR: --dest-zones not provided or is empty; should be list of zones or '*'", EINVAL
);
9768 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9769 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9773 auto& sync_policy
= sync_policy_ctx
.get_policy();
9775 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9776 if (iter
== sync_policy
.groups
.end()) {
9777 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9781 auto& group
= iter
->second
;
9783 rgw_sync_bucket_pipes
*pipe
;
9785 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_CREATE
) {
9786 group
.find_pipe(*opt_pipe_id
, true, &pipe
);
9788 if (!group
.find_pipe(*opt_pipe_id
, false, &pipe
)) {
9789 cerr
<< "ERROR: could not find pipe '" << *opt_pipe_id
<< "'" << std::endl
;
9794 if (opt_source_zone_ids
) {
9795 pipe
->source
.add_zones(*opt_source_zone_ids
);
9797 pipe
->source
.set_bucket(opt_source_tenant
,
9798 opt_source_bucket_name
,
9799 opt_source_bucket_id
);
9800 if (opt_dest_zone_ids
) {
9801 pipe
->dest
.add_zones(*opt_dest_zone_ids
);
9803 pipe
->dest
.set_bucket(opt_dest_tenant
,
9804 opt_dest_bucket_name
,
9805 opt_dest_bucket_id
);
9807 pipe
->params
.source
.filter
.set_prefix(opt_prefix
, !!opt_prefix_rm
);
9808 pipe
->params
.source
.filter
.set_tags(tags_add
, tags_rm
);
9809 if (opt_dest_owner
) {
9810 pipe
->params
.dest
.set_owner(*opt_dest_owner
);
9812 if (opt_storage_class
) {
9813 pipe
->params
.dest
.set_storage_class(*opt_storage_class
);
9816 pipe
->params
.priority
= *opt_priority
;
9819 if (*opt_mode
== "system") {
9820 pipe
->params
.mode
= rgw_sync_pipe_params::MODE_SYSTEM
;
9821 } else if (*opt_mode
== "user") {
9822 pipe
->params
.mode
= rgw_sync_pipe_params::MODE_USER
;
9824 cerr
<< "ERROR: bad mode value: should be one of the following: system, user" << std::endl
;
9829 if (!rgw::sal::User::empty(user
)) {
9830 pipe
->params
.user
= user
->get_id();
9831 } else if (pipe
->params
.user
.empty()) {
9832 auto owner
= sync_policy_ctx
.get_owner();
9834 pipe
->params
.user
= *owner
;
9838 ret
= sync_policy_ctx
.write_policy();
9843 show_result(sync_policy
, zone_formatter
.get(), cout
);
9846 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_REMOVE
) {
9847 CHECK_TRUE(require_non_empty_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9848 CHECK_TRUE(require_non_empty_opt(opt_pipe_id
), "ERROR: --pipe-id not specified", EINVAL
);
9850 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9851 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9855 auto& sync_policy
= sync_policy_ctx
.get_policy();
9857 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9858 if (iter
== sync_policy
.groups
.end()) {
9859 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9863 auto& group
= iter
->second
;
9865 rgw_sync_bucket_pipes
*pipe
;
9867 if (!group
.find_pipe(*opt_pipe_id
, false, &pipe
)) {
9868 cerr
<< "ERROR: could not find pipe '" << *opt_pipe_id
<< "'" << std::endl
;
9872 if (opt_source_zone_ids
) {
9873 pipe
->source
.remove_zones(*opt_source_zone_ids
);
9876 pipe
->source
.remove_bucket(opt_source_tenant
,
9877 opt_source_bucket_name
,
9878 opt_source_bucket_id
);
9879 if (opt_dest_zone_ids
) {
9880 pipe
->dest
.remove_zones(*opt_dest_zone_ids
);
9882 pipe
->dest
.remove_bucket(opt_dest_tenant
,
9883 opt_dest_bucket_name
,
9884 opt_dest_bucket_id
);
9886 if (!(opt_source_zone_ids
||
9887 opt_source_tenant
||
9888 opt_source_bucket
||
9889 opt_source_bucket_id
||
9890 opt_dest_zone_ids
||
9893 opt_dest_bucket_id
)) {
9894 group
.remove_pipe(*opt_pipe_id
);
9897 ret
= sync_policy_ctx
.write_policy();
9902 show_result(sync_policy
, zone_formatter
.get(), cout
);
9905 if (opt_cmd
== OPT::SYNC_POLICY_GET
) {
9906 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9907 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9911 auto& sync_policy
= sync_policy_ctx
.get_policy();
9913 show_result(sync_policy
, zone_formatter
.get(), cout
);
9916 if (opt_cmd
== OPT::BILOG_TRIM
) {
9917 if (bucket_name
.empty()) {
9918 cerr
<< "ERROR: bucket not specified" << std::endl
;
9921 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9923 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
9930 ret
= bilog_trim(dpp(), static_cast<rgw::sal::RadosStore
*>(driver
),
9931 bucket
->get_info(), *gen
,
9932 shard_id
, start_marker
, end_marker
);
9934 cerr
<< "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret
) << std::endl
;
9939 if (opt_cmd
== OPT::BILOG_STATUS
) {
9940 if (bucket_name
.empty()) {
9941 cerr
<< "ERROR: bucket not specified" << std::endl
;
9944 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9946 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
9949 map
<int, string
> markers
;
9950 const auto& logs
= bucket
->get_info().layout
.logs
;
9951 auto log_layout
= std::reference_wrapper
{logs
.back()};
9953 auto i
= std::find_if(logs
.begin(), logs
.end(), rgw::matches_gen(*gen
));
9954 if (i
== logs
.end()) {
9955 cerr
<< "ERROR: no log layout with gen=" << *gen
<< std::endl
;
9961 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->bilog_rados
->get_log_status(dpp(), bucket
->get_info(), log_layout
, shard_id
,
9962 &markers
, null_yield
);
9964 cerr
<< "ERROR: get_bi_log_status(): " << cpp_strerror(-ret
) << std::endl
;
9967 formatter
->open_object_section("entries");
9968 encode_json("markers", markers
, formatter
.get());
9969 formatter
->dump_string("current_time",
9970 to_iso_8601(ceph::real_clock::now(),
9971 iso_8601_format::YMDhms
));
9972 formatter
->close_section();
9973 formatter
->flush(cout
);
9976 if (opt_cmd
== OPT::BILOG_AUTOTRIM
) {
9977 RGWCoroutinesManager
crs(driver
->ctx(), driver
->get_cr_registry());
9978 RGWHTTPManager
http(driver
->ctx(), crs
.get_completion_mgr());
9979 int ret
= http
.start();
9981 cerr
<< "failed to initialize http client with " << cpp_strerror(ret
) << std::endl
;
9985 rgw::BucketTrimConfig config
;
9986 configure_bucket_trim(driver
->ctx(), config
);
9988 rgw::BucketTrimManager
trim(static_cast<rgw::sal::RadosStore
*>(driver
), config
);
9991 cerr
<< "trim manager init failed with " << cpp_strerror(ret
) << std::endl
;
9994 ret
= crs
.run(dpp(), trim
.create_admin_bucket_trim_cr(&http
));
9996 cerr
<< "automated bilog trim failed with " << cpp_strerror(ret
) << std::endl
;
10001 if (opt_cmd
== OPT::DATALOG_LIST
) {
10002 formatter
->open_array_section("entries");
10005 if (max_entries
< 0)
10006 max_entries
= 1000;
10007 if (!start_date
.empty()) {
10008 std::cerr
<< "start-date not allowed." << std::endl
;
10011 if (!end_date
.empty()) {
10012 std::cerr
<< "end-date not allowed." << std::endl
;
10015 if (!end_marker
.empty()) {
10016 std::cerr
<< "end-marker not allowed." << std::endl
;
10019 if (!start_marker
.empty()) {
10020 if (marker
.empty()) {
10021 marker
= start_marker
;
10023 std::cerr
<< "start-marker and marker not both allowed." << std::endl
;
10028 auto datalog_svc
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->datalog_rados
;
10029 RGWDataChangesLog::LogMarker log_marker
;
10032 std::vector
<rgw_data_change_log_entry
> entries
;
10033 if (specified_shard_id
) {
10034 ret
= datalog_svc
->list_entries(dpp(), shard_id
, max_entries
- count
,
10036 &marker
, &truncated
,
10039 ret
= datalog_svc
->list_entries(dpp(), max_entries
- count
, entries
,
10040 log_marker
, &truncated
, null_yield
);
10043 cerr
<< "ERROR: datalog_svc->list_entries(): " << cpp_strerror(-ret
) << std::endl
;
10047 count
+= entries
.size();
10049 for (const auto& entry
: entries
) {
10051 encode_json("entry", entry
.entry
, formatter
.get());
10053 encode_json("entry", entry
, formatter
.get());
10056 formatter
.get()->flush(cout
);
10057 } while (truncated
&& count
< max_entries
);
10059 formatter
->close_section();
10060 formatter
->flush(cout
);
10063 if (opt_cmd
== OPT::DATALOG_STATUS
) {
10064 int i
= (specified_shard_id
? shard_id
: 0);
10066 formatter
->open_array_section("entries");
10067 for (; i
< g_ceph_context
->_conf
->rgw_data_log_num_shards
; i
++) {
10068 list
<cls_log_entry
> entries
;
10070 RGWDataChangesLogInfo info
;
10071 static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->
10072 datalog_rados
->get_info(dpp(), i
, &info
, null_yield
);
10074 ::encode_json("info", info
, formatter
.get());
10076 if (specified_shard_id
)
10080 formatter
->close_section();
10081 formatter
->flush(cout
);
10084 if (opt_cmd
== OPT::DATALOG_AUTOTRIM
) {
10085 RGWCoroutinesManager
crs(driver
->ctx(), driver
->get_cr_registry());
10086 RGWHTTPManager
http(driver
->ctx(), crs
.get_completion_mgr());
10087 int ret
= http
.start();
10089 cerr
<< "failed to initialize http client with " << cpp_strerror(ret
) << std::endl
;
10093 auto num_shards
= g_conf()->rgw_data_log_num_shards
;
10094 std::vector
<std::string
> markers(num_shards
);
10095 ret
= crs
.run(dpp(), create_admin_data_log_trim_cr(dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), &http
, num_shards
, markers
));
10097 cerr
<< "automated datalog trim failed with " << cpp_strerror(ret
) << std::endl
;
10102 if (opt_cmd
== OPT::DATALOG_TRIM
) {
10103 if (!start_date
.empty()) {
10104 std::cerr
<< "start-date not allowed." << std::endl
;
10107 if (!end_date
.empty()) {
10108 std::cerr
<< "end-date not allowed." << std::endl
;
10111 if (!start_marker
.empty()) {
10112 std::cerr
<< "start-marker not allowed." << std::endl
;
10115 if (!end_marker
.empty()) {
10116 if (marker
.empty()) {
10117 marker
= end_marker
;
10119 std::cerr
<< "end-marker and marker not both allowed." << std::endl
;
10124 if (!specified_shard_id
) {
10125 cerr
<< "ERROR: requires a --shard-id" << std::endl
;
10129 if (marker
.empty()) {
10130 cerr
<< "ERROR: requires a --marker" << std::endl
;
10134 auto datalog
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->datalog_rados
;
10135 ret
= datalog
->trim_entries(dpp(), shard_id
, marker
, null_yield
);
10137 if (ret
< 0 && ret
!= -ENODATA
) {
10138 cerr
<< "ERROR: trim_entries(): " << cpp_strerror(-ret
) << std::endl
;
10143 if (opt_cmd
== OPT::DATALOG_TYPE
) {
10144 if (!opt_log_type
) {
10145 std::cerr
<< "log-type not specified." << std::endl
;
10148 auto datalog
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->datalog_rados
;
10149 ret
= datalog
->change_format(dpp(), *opt_log_type
, null_yield
);
10151 cerr
<< "ERROR: change_format(): " << cpp_strerror(-ret
) << std::endl
;
10156 if (opt_cmd
== OPT::DATALOG_PRUNE
) {
10157 auto datalog
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->datalog_rados
;
10158 std::optional
<uint64_t> through
;
10159 ret
= datalog
->trim_generations(dpp(), through
, null_yield
);
10162 cerr
<< "ERROR: trim_generations(): " << cpp_strerror(-ret
) << std::endl
;
10167 std::cout
<< "Pruned " << *through
<< " empty generations." << std::endl
;
10169 std::cout
<< "No empty generations." << std::endl
;
10173 bool quota_op
= (opt_cmd
== OPT::QUOTA_SET
|| opt_cmd
== OPT::QUOTA_ENABLE
|| opt_cmd
== OPT::QUOTA_DISABLE
);
10176 if (bucket_name
.empty() && rgw::sal::User::empty(user
)) {
10177 cerr
<< "ERROR: bucket name or uid is required for quota operation" << std::endl
;
10181 if (!bucket_name
.empty()) {
10182 if (!quota_scope
.empty() && quota_scope
!= "bucket") {
10183 cerr
<< "ERROR: invalid quota scope specification." << std::endl
;
10186 set_bucket_quota(driver
, opt_cmd
, tenant
, bucket_name
,
10187 max_size
, max_objects
, have_max_size
, have_max_objects
);
10188 } else if (!rgw::sal::User::empty(user
)) {
10189 if (quota_scope
== "bucket") {
10190 return set_user_bucket_quota(opt_cmd
, ruser
, user_op
, max_size
, max_objects
, have_max_size
, have_max_objects
);
10191 } else if (quota_scope
== "user") {
10192 return set_user_quota(opt_cmd
, ruser
, user_op
, max_size
, max_objects
, have_max_size
, have_max_objects
);
10194 cerr
<< "ERROR: invalid quota scope specification. Please specify either --quota-scope=bucket, or --quota-scope=user" << std::endl
;
10200 bool ratelimit_op_set
= (opt_cmd
== OPT::RATELIMIT_SET
|| opt_cmd
== OPT::RATELIMIT_ENABLE
|| opt_cmd
== OPT::RATELIMIT_DISABLE
);
10201 bool ratelimit_op_get
= opt_cmd
== OPT::RATELIMIT_GET
;
10202 if (ratelimit_op_set
) {
10203 if (bucket_name
.empty() && rgw::sal::User::empty(user
)) {
10204 cerr
<< "ERROR: bucket name or uid is required for ratelimit operation" << std::endl
;
10208 if (!bucket_name
.empty()) {
10209 if (!ratelimit_scope
.empty() && ratelimit_scope
!= "bucket") {
10210 cerr
<< "ERROR: invalid ratelimit scope specification. (bucket scope is not bucket but bucket has been specified)" << std::endl
;
10213 return set_bucket_ratelimit(driver
, opt_cmd
, tenant
, bucket_name
,
10214 max_read_ops
, max_write_ops
,
10215 max_read_bytes
, max_write_bytes
,
10216 have_max_read_ops
, have_max_write_ops
,
10217 have_max_read_bytes
, have_max_write_bytes
);
10218 } else if (!rgw::sal::User::empty(user
)) {
10219 } if (ratelimit_scope
== "user") {
10220 return set_user_ratelimit(opt_cmd
, user
, max_read_ops
, max_write_ops
,
10221 max_read_bytes
, max_write_bytes
,
10222 have_max_read_ops
, have_max_write_ops
,
10223 have_max_read_bytes
, have_max_write_bytes
);
10225 cerr
<< "ERROR: invalid ratelimit scope specification. Please specify either --ratelimit-scope=bucket, or --ratelimit-scope=user" << std::endl
;
10230 if (ratelimit_op_get
) {
10231 if (bucket_name
.empty() && rgw::sal::User::empty(user
)) {
10232 cerr
<< "ERROR: bucket name or uid is required for ratelimit operation" << std::endl
;
10236 if (!bucket_name
.empty()) {
10237 if (!ratelimit_scope
.empty() && ratelimit_scope
!= "bucket") {
10238 cerr
<< "ERROR: invalid ratelimit scope specification. (bucket scope is not bucket but bucket has been specified)" << std::endl
;
10241 return show_bucket_ratelimit(driver
, tenant
, bucket_name
, formatter
.get());
10242 } else if (!rgw::sal::User::empty(user
)) {
10243 } if (ratelimit_scope
== "user") {
10244 return show_user_ratelimit(user
, formatter
.get());
10246 cerr
<< "ERROR: invalid ratelimit scope specification. Please specify either --ratelimit-scope=bucket, or --ratelimit-scope=user" << std::endl
;
10251 if (opt_cmd
== OPT::MFA_CREATE
) {
10252 rados::cls::otp::otp_info_t config
;
10254 if (rgw::sal::User::empty(user
)) {
10255 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10259 if (totp_serial
.empty()) {
10260 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
10264 if (totp_seed
.empty()) {
10265 cerr
<< "ERROR: TOTP device seed was not provided (via --totp-seed)" << std::endl
;
10270 rados::cls::otp::SeedType seed_type
;
10271 if (totp_seed_type
== "hex") {
10272 seed_type
= rados::cls::otp::OTP_SEED_HEX
;
10273 } else if (totp_seed_type
== "base32") {
10274 seed_type
= rados::cls::otp::OTP_SEED_BASE32
;
10276 cerr
<< "ERROR: invalid seed type: " << totp_seed_type
<< std::endl
;
10280 config
.id
= totp_serial
;
10281 config
.seed
= totp_seed
;
10282 config
.seed_type
= seed_type
;
10284 if (totp_seconds
> 0) {
10285 config
.step_size
= totp_seconds
;
10288 if (totp_window
> 0) {
10289 config
.window
= totp_window
;
10292 real_time mtime
= real_clock::now();
10293 string oid
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.get_mfa_oid(user
->get_id());
10295 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->mutate(RGWSI_MetaBackend_OTP::get_meta_key(user
->get_id()),
10296 mtime
, &objv_tracker
,
10298 MDLOG_STATUS_WRITE
,
10300 return static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.create_mfa(dpp(), user
->get_id(), config
, &objv_tracker
, mtime
, null_yield
);
10303 cerr
<< "MFA creation failed, error: " << cpp_strerror(-ret
) << std::endl
;
10307 RGWUserInfo
& user_info
= user_op
.get_user_info();
10308 user_info
.mfa_ids
.insert(totp_serial
);
10309 user_op
.set_mfa_ids(user_info
.mfa_ids
);
10311 ret
= ruser
.modify(dpp(), user_op
, null_yield
, &err
);
10313 cerr
<< "ERROR: failed storing user info, error: " << err
<< std::endl
;
10318 if (opt_cmd
== OPT::MFA_REMOVE
) {
10319 if (rgw::sal::User::empty(user
)) {
10320 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10324 if (totp_serial
.empty()) {
10325 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
10329 real_time mtime
= real_clock::now();
10331 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->mutate(RGWSI_MetaBackend_OTP::get_meta_key(user
->get_id()),
10332 mtime
, &objv_tracker
,
10334 MDLOG_STATUS_WRITE
,
10336 return static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.remove_mfa(dpp(), user
->get_id(), totp_serial
, &objv_tracker
, mtime
, null_yield
);
10339 cerr
<< "MFA removal failed, error: " << cpp_strerror(-ret
) << std::endl
;
10343 RGWUserInfo
& user_info
= user_op
.get_user_info();
10344 user_info
.mfa_ids
.erase(totp_serial
);
10345 user_op
.set_mfa_ids(user_info
.mfa_ids
);
10347 ret
= ruser
.modify(dpp(), user_op
, null_yield
, &err
);
10349 cerr
<< "ERROR: failed storing user info, error: " << err
<< std::endl
;
10354 if (opt_cmd
== OPT::MFA_GET
) {
10355 if (rgw::sal::User::empty(user
)) {
10356 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10360 if (totp_serial
.empty()) {
10361 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
10365 rados::cls::otp::otp_info_t result
;
10366 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.get_mfa(dpp(), user
->get_id(), totp_serial
, &result
, null_yield
);
10368 if (ret
== -ENOENT
|| ret
== -ENODATA
) {
10369 cerr
<< "MFA serial id not found" << std::endl
;
10371 cerr
<< "MFA retrieval failed, error: " << cpp_strerror(-ret
) << std::endl
;
10375 formatter
->open_object_section("result");
10376 encode_json("entry", result
, formatter
.get());
10377 formatter
->close_section();
10378 formatter
->flush(cout
);
10381 if (opt_cmd
== OPT::MFA_LIST
) {
10382 if (rgw::sal::User::empty(user
)) {
10383 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10387 list
<rados::cls::otp::otp_info_t
> result
;
10388 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.list_mfa(dpp(), user
->get_id(), &result
, null_yield
);
10389 if (ret
< 0 && ret
!= -ENOENT
) {
10390 cerr
<< "MFA listing failed, error: " << cpp_strerror(-ret
) << std::endl
;
10393 formatter
->open_object_section("result");
10394 encode_json("entries", result
, formatter
.get());
10395 formatter
->close_section();
10396 formatter
->flush(cout
);
10399 if (opt_cmd
== OPT::MFA_CHECK
) {
10400 if (rgw::sal::User::empty(user
)) {
10401 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10405 if (totp_serial
.empty()) {
10406 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
10410 if (totp_pin
.empty()) {
10411 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-pin)" << std::endl
;
10415 list
<rados::cls::otp::otp_info_t
> result
;
10416 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.check_mfa(dpp(), user
->get_id(), totp_serial
, totp_pin
.front(), null_yield
);
10418 cerr
<< "MFA check failed, error: " << cpp_strerror(-ret
) << std::endl
;
10422 cout
<< "ok" << std::endl
;
10425 if (opt_cmd
== OPT::MFA_RESYNC
) {
10426 if (rgw::sal::User::empty(user
)) {
10427 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10431 if (totp_serial
.empty()) {
10432 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
10436 if (totp_pin
.size() != 2) {
10437 cerr
<< "ERROR: missing two --totp-pin params (--totp-pin=<first> --totp-pin=<second>)" << std::endl
;
10441 rados::cls::otp::otp_info_t config
;
10442 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.get_mfa(dpp(), user
->get_id(), totp_serial
, &config
, null_yield
);
10444 if (ret
== -ENOENT
|| ret
== -ENODATA
) {
10445 cerr
<< "MFA serial id not found" << std::endl
;
10447 cerr
<< "MFA retrieval failed, error: " << cpp_strerror(-ret
) << std::endl
;
10452 ceph::real_time now
;
10454 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.otp_get_current_time(dpp(), user
->get_id(), &now
, null_yield
);
10456 cerr
<< "ERROR: failed to fetch current time from osd: " << cpp_strerror(-ret
) << std::endl
;
10461 ret
= scan_totp(driver
->ctx(), now
, config
, totp_pin
, &time_ofs
);
10463 if (ret
== -ENOENT
) {
10464 cerr
<< "failed to resync, TOTP values not found in range" << std::endl
;
10466 cerr
<< "ERROR: failed to scan for TOTP values: " << cpp_strerror(-ret
) << std::endl
;
10471 config
.time_ofs
= time_ofs
;
10473 /* now update the backend */
10474 real_time mtime
= real_clock::now();
10476 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->mutate(RGWSI_MetaBackend_OTP::get_meta_key(user
->get_id()),
10477 mtime
, &objv_tracker
,
10479 MDLOG_STATUS_WRITE
,
10481 return static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.create_mfa(dpp(), user
->get_id(), config
, &objv_tracker
, mtime
, null_yield
);
10484 cerr
<< "MFA update failed, error: " << cpp_strerror(-ret
) << std::endl
;
10490 if (opt_cmd
== OPT::RESHARD_STALE_INSTANCES_LIST
) {
10491 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->can_reshard() && !yes_i_really_mean_it
) {
10492 cerr
<< "Resharding disabled in a multisite env, stale instances unlikely from resharding" << std::endl
;
10493 cerr
<< "These instances may not be safe to delete." << std::endl
;
10494 cerr
<< "Use --yes-i-really-mean-it to force displaying these instances." << std::endl
;
10498 ret
= RGWBucketAdminOp::list_stale_instances(driver
, bucket_op
, stream_flusher
, dpp());
10500 cerr
<< "ERROR: listing stale instances" << cpp_strerror(-ret
) << std::endl
;
10504 if (opt_cmd
== OPT::RESHARD_STALE_INSTANCES_DELETE
) {
10505 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->can_reshard()) {
10506 cerr
<< "Resharding disabled in a multisite env. Stale instances are not safe to be deleted." << std::endl
;
10510 ret
= RGWBucketAdminOp::clear_stale_instances(driver
, bucket_op
, stream_flusher
, dpp());
10512 cerr
<< "ERROR: deleting stale instances" << cpp_strerror(-ret
) << std::endl
;
10516 if (opt_cmd
== OPT::PUBSUB_NOTIFICATION_LIST
) {
10517 if (bucket_name
.empty()) {
10518 cerr
<< "ERROR: bucket name was not provided (via --bucket)" << std::endl
;
10522 RGWPubSub
ps(driver
, tenant
);
10524 rgw_pubsub_bucket_topics result
;
10525 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
10527 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
10531 const RGWPubSub::Bucket
b(ps
, bucket
.get());
10532 ret
= b
.get_topics(dpp(), result
, null_yield
);
10533 if (ret
< 0 && ret
!= -ENOENT
) {
10534 cerr
<< "ERROR: could not get topics: " << cpp_strerror(-ret
) << std::endl
;
10537 encode_json("result", result
, formatter
.get());
10538 formatter
->flush(cout
);
10541 if (opt_cmd
== OPT::PUBSUB_TOPIC_LIST
) {
10542 RGWPubSub
ps(driver
, tenant
);
10544 rgw_pubsub_topics result
;
10545 int ret
= ps
.get_topics(dpp(), result
, null_yield
);
10546 if (ret
< 0 && ret
!= -ENOENT
) {
10547 cerr
<< "ERROR: could not get topics: " << cpp_strerror(-ret
) << std::endl
;
10550 encode_json("result", result
, formatter
.get());
10551 formatter
->flush(cout
);
10554 if (opt_cmd
== OPT::PUBSUB_TOPIC_GET
) {
10555 if (topic_name
.empty()) {
10556 cerr
<< "ERROR: topic name was not provided (via --topic)" << std::endl
;
10560 RGWPubSub
ps(driver
, tenant
);
10562 rgw_pubsub_topic topic
;
10563 ret
= ps
.get_topic(dpp(), topic_name
, topic
, null_yield
);
10565 cerr
<< "ERROR: could not get topic: " << cpp_strerror(-ret
) << std::endl
;
10568 encode_json("topic", topic
, formatter
.get());
10569 formatter
->flush(cout
);
10572 if (opt_cmd
== OPT::PUBSUB_NOTIFICATION_GET
) {
10573 if (notification_id
.empty()) {
10574 cerr
<< "ERROR: notification-id was not provided (via --notification-id)" << std::endl
;
10577 if (bucket_name
.empty()) {
10578 cerr
<< "ERROR: bucket name was not provided (via --bucket)" << std::endl
;
10582 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
10584 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
10588 RGWPubSub
ps(driver
, tenant
);
10590 rgw_pubsub_bucket_topics bucket_topics
;
10591 const RGWPubSub::Bucket
b(ps
, bucket
.get());
10592 ret
= b
.get_topics(dpp(), bucket_topics
, null_yield
);
10593 if (ret
< 0 && ret
!= -ENOENT
) {
10594 cerr
<< "ERROR: could not get bucket notifications: " << cpp_strerror(-ret
) << std::endl
;
10598 rgw_pubsub_topic_filter bucket_topic
;
10599 ret
= b
.get_notification_by_id(dpp(), notification_id
, bucket_topic
, null_yield
);
10601 cerr
<< "ERROR: could not get notification: " << cpp_strerror(-ret
) << std::endl
;
10604 encode_json("notification", bucket_topic
, formatter
.get());
10605 formatter
->flush(cout
);
10608 if (opt_cmd
== OPT::PUBSUB_TOPIC_RM
) {
10609 if (topic_name
.empty()) {
10610 cerr
<< "ERROR: topic name was not provided (via --topic)" << std::endl
;
10614 ret
= rgw::notify::remove_persistent_topic(dpp(), static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->get_notif_pool_ctx(), topic_name
, null_yield
);
10616 cerr
<< "ERROR: could not remove persistent topic: " << cpp_strerror(-ret
) << std::endl
;
10620 RGWPubSub
ps(driver
, tenant
);
10622 ret
= ps
.remove_topic(dpp(), topic_name
, null_yield
);
10624 cerr
<< "ERROR: could not remove topic: " << cpp_strerror(-ret
) << std::endl
;
10629 if (opt_cmd
== OPT::PUBSUB_NOTIFICATION_RM
) {
10630 if (bucket_name
.empty()) {
10631 cerr
<< "ERROR: bucket name was not provided (via --bucket)" << std::endl
;
10635 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
10637 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
10641 RGWPubSub
ps(driver
, tenant
);
10643 rgw_pubsub_bucket_topics bucket_topics
;
10644 const RGWPubSub::Bucket
b(ps
, bucket
.get());
10645 ret
= b
.get_topics(dpp(), bucket_topics
, null_yield
);
10646 if (ret
< 0 && ret
!= -ENOENT
) {
10647 cerr
<< "ERROR: could not get bucket notifications: " << cpp_strerror(-ret
) << std::endl
;
10651 rgw_pubsub_topic_filter bucket_topic
;
10652 if(notification_id
.empty()) {
10653 ret
= b
.remove_notifications(dpp(), null_yield
);
10655 ret
= b
.remove_notification_by_id(dpp(), notification_id
, null_yield
);
10659 if (opt_cmd
== OPT::SCRIPT_PUT
) {
10660 if (!str_script_ctx
) {
10661 cerr
<< "ERROR: context was not provided (via --context)" << std::endl
;
10664 if (infile
.empty()) {
10665 cerr
<< "ERROR: infile was not provided (via --infile)" << std::endl
;
10669 auto rc
= read_input(infile
, bl
);
10671 cerr
<< "ERROR: failed to read script: '" << infile
<< "'. error: " << rc
<< std::endl
;
10674 const std::string script
= bl
.to_str();
10675 std::string err_msg
;
10676 if (!rgw::lua::verify(script
, err_msg
)) {
10677 cerr
<< "ERROR: script: '" << infile
<< "' has error: " << std::endl
<< err_msg
<< std::endl
;
10680 const rgw::lua::context script_ctx
= rgw::lua::to_context(*str_script_ctx
);
10681 if (script_ctx
== rgw::lua::context::none
) {
10682 cerr
<< "ERROR: invalid script context: " << *str_script_ctx
<< ". must be one of: " << LUA_CONTEXT_LIST
<< std::endl
;
10685 if (script_ctx
== rgw::lua::context::background
&& !tenant
.empty()) {
10686 cerr
<< "ERROR: cannot specify tenant in background context" << std::endl
;
10689 auto lua_manager
= driver
->get_lua_manager();
10690 rc
= rgw::lua::write_script(dpp(), lua_manager
.get(), tenant
, null_yield
, script_ctx
, script
);
10692 cerr
<< "ERROR: failed to put script. error: " << rc
<< std::endl
;
10697 if (opt_cmd
== OPT::SCRIPT_GET
) {
10698 if (!str_script_ctx
) {
10699 cerr
<< "ERROR: context was not provided (via --context)" << std::endl
;
10702 const rgw::lua::context script_ctx
= rgw::lua::to_context(*str_script_ctx
);
10703 if (script_ctx
== rgw::lua::context::none
) {
10704 cerr
<< "ERROR: invalid script context: " << *str_script_ctx
<< ". must be one of: " << LUA_CONTEXT_LIST
<< std::endl
;
10707 auto lua_manager
= driver
->get_lua_manager();
10708 std::string script
;
10709 const auto rc
= rgw::lua::read_script(dpp(), lua_manager
.get(), tenant
, null_yield
, script_ctx
, script
);
10710 if (rc
== -ENOENT
) {
10711 std::cout
<< "no script exists for context: " << *str_script_ctx
<<
10712 (tenant
.empty() ? "" : (" in tenant: " + tenant
)) << std::endl
;
10713 } else if (rc
< 0) {
10714 cerr
<< "ERROR: failed to read script. error: " << rc
<< std::endl
;
10717 std::cout
<< script
<< std::endl
;
10721 if (opt_cmd
== OPT::SCRIPT_RM
) {
10722 if (!str_script_ctx
) {
10723 cerr
<< "ERROR: context was not provided (via --context)" << std::endl
;
10726 const rgw::lua::context script_ctx
= rgw::lua::to_context(*str_script_ctx
);
10727 if (script_ctx
== rgw::lua::context::none
) {
10728 cerr
<< "ERROR: invalid script context: " << *str_script_ctx
<< ". must be one of: " << LUA_CONTEXT_LIST
<< std::endl
;
10731 auto lua_manager
= driver
->get_lua_manager();
10732 const auto rc
= rgw::lua::delete_script(dpp(), lua_manager
.get(), tenant
, null_yield
, script_ctx
);
10734 cerr
<< "ERROR: failed to remove script. error: " << rc
<< std::endl
;
10739 if (opt_cmd
== OPT::SCRIPT_PACKAGE_ADD
) {
10740 #ifdef WITH_RADOSGW_LUA_PACKAGES
10741 if (!script_package
) {
10742 cerr
<< "ERROR: lua package name was not provided (via --package)" << std::endl
;
10745 const auto rc
= rgw::lua::add_package(dpp(), driver
, null_yield
, *script_package
, bool(allow_compilation
));
10747 cerr
<< "ERROR: failed to add lua package: " << script_package
<< " .error: " << rc
<< std::endl
;
10751 cerr
<< "ERROR: adding lua packages is not permitted" << std::endl
;
10756 if (opt_cmd
== OPT::SCRIPT_PACKAGE_RM
) {
10757 #ifdef WITH_RADOSGW_LUA_PACKAGES
10758 if (!script_package
) {
10759 cerr
<< "ERROR: lua package name was not provided (via --package)" << std::endl
;
10762 const auto rc
= rgw::lua::remove_package(dpp(), driver
, null_yield
, *script_package
);
10763 if (rc
== -ENOENT
) {
10764 cerr
<< "WARNING: package " << script_package
<< " did not exists or already removed" << std::endl
;
10768 cerr
<< "ERROR: failed to remove lua package: " << script_package
<< " .error: " << rc
<< std::endl
;
10772 cerr
<< "ERROR: removing lua packages in not permitted" << std::endl
;
10777 if (opt_cmd
== OPT::SCRIPT_PACKAGE_LIST
) {
10778 #ifdef WITH_RADOSGW_LUA_PACKAGES
10779 rgw::lua::packages_t packages
;
10780 const auto rc
= rgw::lua::list_packages(dpp(), driver
, null_yield
, packages
);
10781 if (rc
== -ENOENT
) {
10782 std::cout
<< "no lua packages in allowlist" << std::endl
;
10783 } else if (rc
< 0) {
10784 cerr
<< "ERROR: failed to read lua packages allowlist. error: " << rc
<< std::endl
;
10787 for (const auto& package
: packages
) {
10788 std::cout
<< package
<< std::endl
;
10792 cerr
<< "ERROR: listing lua packages in not permitted" << std::endl
;