1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
9 #include <boost/optional.hpp>
12 #include <liboath/oath.h>
15 #include <fmt/format.h>
17 #include "auth/Crypto.h"
18 #include "compressor/Compressor.h"
20 #include "common/armor.h"
21 #include "common/ceph_json.h"
22 #include "common/config.h"
23 #include "common/ceph_argparse.h"
24 #include "common/Formatter.h"
25 #include "common/errno.h"
26 #include "common/safe_io.h"
27 #include "common/fault_injector.h"
29 #include "include/util.h"
31 #include "cls/rgw/cls_rgw_types.h"
32 #include "cls/rgw/cls_rgw_client.h"
34 #include "include/utime.h"
35 #include "include/str_list.h"
39 #include "rgw_rados.h"
41 #include "rgw_acl_s3.h"
42 #include "rgw_datalog.h"
45 #include "rgw_formats.h"
46 #include "rgw_usage.h"
47 #include "rgw_orphan.h"
49 #include "rgw_trim_bilog.h"
50 #include "rgw_trim_datalog.h"
51 #include "rgw_trim_mdlog.h"
52 #include "rgw_data_sync.h"
53 #include "rgw_rest_conn.h"
54 #include "rgw_realm_watcher.h"
56 #include "rgw_reshard.h"
57 #include "rgw_http_client_curl.h"
59 #include "rgw_pubsub.h"
60 #include "rgw_bucket_sync.h"
61 #include "rgw_sync_checkpoint.h"
64 #include "rgw_sal_config.h"
66 #include "services/svc_sync_modules.h"
67 #include "services/svc_cls.h"
68 #include "services/svc_bilog_rados.h"
69 #include "services/svc_mdlog.h"
70 #include "services/svc_meta_be_otp.h"
71 #include "services/svc_user.h"
72 #include "services/svc_zone.h"
74 #include "driver/rados/rgw_bucket.h"
75 #include "driver/rados/rgw_sal_rados.h"
77 #define dout_context g_ceph_context
79 #define SECRET_KEY_LEN 40
80 #define PUBLIC_ID_LEN 20
84 static rgw::sal::Driver
* driver
= NULL
;
85 static constexpr auto dout_subsys
= ceph_subsys_rgw
;
88 static const DoutPrefixProvider
* dpp() {
89 struct GlobalPrefix
: public DoutPrefixProvider
{
90 CephContext
*get_cct() const override
{ return dout_context
; }
91 unsigned get_subsys() const override
{ return dout_subsys
; }
92 std::ostream
& gen_prefix(std::ostream
& out
) const override
{ return out
; }
94 static GlobalPrefix global_dpp
;
98 #define CHECK_TRUE(x, msg, err) \
101 cerr << msg << std::endl; \
106 #define CHECK_SUCCESS(x, msg) \
110 cerr << msg << ": " << cpp_strerror(-_x_val) << std::endl; \
115 static inline int posix_errortrans(int r
)
118 case ERR_NO_SUCH_BUCKET
:
128 static const std::string
LUA_CONTEXT_LIST("prerequest, postrequest, background, getdata, putdata");
132 cout
<< "usage: radosgw-admin <cmd> [options...]" << std::endl
;
133 cout
<< "commands:\n";
134 cout
<< " user create create a new user\n" ;
135 cout
<< " user modify modify user\n";
136 cout
<< " user info get user info\n";
137 cout
<< " user rename rename user\n";
138 cout
<< " user rm remove user\n";
139 cout
<< " user suspend suspend a user\n";
140 cout
<< " user enable re-enable user after suspension\n";
141 cout
<< " user check check user info\n";
142 cout
<< " user stats show user stats as accounted by quota subsystem\n";
143 cout
<< " user list list users\n";
144 cout
<< " caps add add user capabilities\n";
145 cout
<< " caps rm remove user capabilities\n";
146 cout
<< " subuser create create a new subuser\n" ;
147 cout
<< " subuser modify modify subuser\n";
148 cout
<< " subuser rm remove subuser\n";
149 cout
<< " key create create access key\n";
150 cout
<< " key rm remove access key\n";
151 cout
<< " bucket list list buckets (specify --allow-unordered for\n";
152 cout
<< " faster, unsorted listing)\n";
153 cout
<< " bucket limit check show bucket sharding stats\n";
154 cout
<< " bucket link link bucket to specified user\n";
155 cout
<< " bucket unlink unlink bucket from specified user\n";
156 cout
<< " bucket stats returns bucket statistics\n";
157 cout
<< " bucket rm remove bucket\n";
158 cout
<< " bucket check check bucket index\n";
159 cout
<< " bucket chown link bucket to specified user and update its object ACLs\n";
160 cout
<< " bucket reshard reshard bucket\n";
161 cout
<< " bucket rewrite rewrite all objects in the specified bucket\n";
162 cout
<< " bucket sync checkpoint poll a bucket's sync status until it catches up to its remote\n";
163 cout
<< " bucket sync disable disable bucket sync\n";
164 cout
<< " bucket sync enable enable bucket sync\n";
165 cout
<< " bucket radoslist list rados objects backing bucket's objects\n";
166 cout
<< " bi get retrieve bucket index object entries\n";
167 cout
<< " bi put store bucket index object entries\n";
168 cout
<< " bi list list raw bucket index entries\n";
169 cout
<< " bi purge purge bucket index entries\n";
170 cout
<< " object rm remove object\n";
171 cout
<< " object put put object\n";
172 cout
<< " object stat stat an object for its metadata\n";
173 cout
<< " object unlink unlink object from bucket index\n";
174 cout
<< " object rewrite rewrite the specified object\n";
175 cout
<< " object reindex reindex the object(s) indicated by --bucket and either --object or --objects-file\n";
176 cout
<< " objects expire run expired objects cleanup\n";
177 cout
<< " objects expire-stale list list stale expired objects (caused by reshard)\n";
178 cout
<< " objects expire-stale rm remove stale expired objects\n";
179 cout
<< " period rm remove a period\n";
180 cout
<< " period get get period info\n";
181 cout
<< " period get-current get current period info\n";
182 cout
<< " period pull pull a period\n";
183 cout
<< " period push push a period\n";
184 cout
<< " period list list all periods\n";
185 cout
<< " period update update the staging period\n";
186 cout
<< " period commit commit the staging period\n";
187 cout
<< " quota set set quota params\n";
188 cout
<< " quota enable enable quota\n";
189 cout
<< " quota disable disable quota\n";
190 cout
<< " ratelimit get get ratelimit params\n";
191 cout
<< " ratelimit set set ratelimit params\n";
192 cout
<< " ratelimit enable enable ratelimit\n";
193 cout
<< " ratelimit disable disable ratelimit\n";
194 cout
<< " global quota get view global quota params\n";
195 cout
<< " global quota set set global quota params\n";
196 cout
<< " global quota enable enable a global quota\n";
197 cout
<< " global quota disable disable a global quota\n";
198 cout
<< " global ratelimit get view global ratelimit params\n";
199 cout
<< " global ratelimit set set global ratelimit params\n";
200 cout
<< " global ratelimit enable enable a ratelimit quota\n";
201 cout
<< " global ratelimit disable disable a ratelimit quota\n";
202 cout
<< " realm create create a new realm\n";
203 cout
<< " realm rm remove a realm\n";
204 cout
<< " realm get show realm info\n";
205 cout
<< " realm get-default get default realm name\n";
206 cout
<< " realm list list realms\n";
207 cout
<< " realm list-periods list all realm periods\n";
208 cout
<< " realm rename rename a realm\n";
209 cout
<< " realm set set realm info (requires infile)\n";
210 cout
<< " realm default set realm as default\n";
211 cout
<< " realm pull pull a realm and its current period\n";
212 cout
<< " zonegroup add add a zone to a zonegroup\n";
213 cout
<< " zonegroup create create a new zone group info\n";
214 cout
<< " zonegroup default set default zone group\n";
215 cout
<< " zonegroup delete delete a zone group info\n";
216 cout
<< " zonegroup get show zone group info\n";
217 cout
<< " zonegroup modify modify an existing zonegroup\n";
218 cout
<< " zonegroup set set zone group info (requires infile)\n";
219 cout
<< " zonegroup rm remove a zone from a zonegroup\n";
220 cout
<< " zonegroup rename rename a zone group\n";
221 cout
<< " zonegroup list list all zone groups set on this cluster\n";
222 cout
<< " zonegroup placement list list zonegroup's placement targets\n";
223 cout
<< " zonegroup placement get get a placement target of a specific zonegroup\n";
224 cout
<< " zonegroup placement add add a placement target id to a zonegroup\n";
225 cout
<< " zonegroup placement modify modify a placement target of a specific zonegroup\n";
226 cout
<< " zonegroup placement rm remove a placement target from a zonegroup\n";
227 cout
<< " zonegroup placement default set a zonegroup's default placement target\n";
228 cout
<< " zone create create a new zone\n";
229 cout
<< " zone rm remove a zone\n";
230 cout
<< " zone get show zone cluster params\n";
231 cout
<< " zone modify modify an existing zone\n";
232 cout
<< " zone set set zone cluster params (requires infile)\n";
233 cout
<< " zone list list all zones set on this cluster\n";
234 cout
<< " zone rename rename a zone\n";
235 cout
<< " zone placement list list zone's placement targets\n";
236 cout
<< " zone placement get get a zone placement target\n";
237 cout
<< " zone placement add add a zone placement target\n";
238 cout
<< " zone placement modify modify a zone placement target\n";
239 cout
<< " zone placement rm remove a zone placement target\n";
240 cout
<< " metadata sync status get metadata sync status\n";
241 cout
<< " metadata sync init init metadata sync\n";
242 cout
<< " metadata sync run run metadata sync\n";
243 cout
<< " data sync status get data sync status of the specified source zone\n";
244 cout
<< " data sync init init data sync for the specified source zone\n";
245 cout
<< " data sync run run data sync for the specified source zone\n";
246 cout
<< " pool add add an existing pool for data placement\n";
247 cout
<< " pool rm remove an existing pool from data placement set\n";
248 cout
<< " pools list list placement active set\n";
249 cout
<< " policy read bucket/object policy\n";
250 cout
<< " log list list log objects\n";
251 cout
<< " log show dump a log from specific object or (bucket + date\n";
252 cout
<< " + bucket-id)\n";
253 cout
<< " (NOTE: required to specify formatting of date\n";
254 cout
<< " to \"YYYY-MM-DD-hh\")\n";
255 cout
<< " log rm remove log object\n";
256 cout
<< " usage show show usage (by user, by bucket, date range)\n";
257 cout
<< " usage trim trim usage (by user, by bucket, date range)\n";
258 cout
<< " usage clear reset all the usage stats for the cluster\n";
259 cout
<< " gc list dump expired garbage collection objects (specify\n";
260 cout
<< " --include-all to list all entries, including unexpired)\n";
261 cout
<< " gc process manually process garbage (specify\n";
262 cout
<< " --include-all to process all entries, including unexpired)\n";
263 cout
<< " lc list list all bucket lifecycle progress\n";
264 cout
<< " lc get get a lifecycle bucket configuration\n";
265 cout
<< " lc process manually process lifecycle\n";
266 cout
<< " lc reshard fix fix LC for a resharded bucket\n";
267 cout
<< " metadata get get metadata info\n";
268 cout
<< " metadata put put metadata info\n";
269 cout
<< " metadata rm remove metadata info\n";
270 cout
<< " metadata list list metadata info\n";
271 cout
<< " mdlog list list metadata log\n";
272 cout
<< " mdlog autotrim auto trim metadata log\n";
273 cout
<< " mdlog trim trim metadata log (use marker)\n";
274 cout
<< " mdlog status read metadata log status\n";
275 cout
<< " bilog list list bucket index log\n";
276 cout
<< " bilog trim trim bucket index log (use start-marker, end-marker)\n";
277 cout
<< " bilog status read bucket index log status\n";
278 cout
<< " bilog autotrim auto trim bucket index log\n";
279 cout
<< " datalog list list data log\n";
280 cout
<< " datalog trim trim data log\n";
281 cout
<< " datalog status read data log status\n";
282 cout
<< " datalog type change datalog type to --log_type={fifo,omap}\n";
283 cout
<< " orphans find deprecated -- init and run search for leaked rados objects (use job-id, pool)\n";
284 cout
<< " orphans finish deprecated -- clean up search for leaked rados objects\n";
285 cout
<< " orphans list-jobs deprecated -- list the current job-ids for orphans search\n";
286 cout
<< " * the three 'orphans' sub-commands are now deprecated; consider using the `rgw-orphan-list` tool\n";
287 cout
<< " role create create a AWS role for use with STS\n";
288 cout
<< " role delete remove a role\n";
289 cout
<< " role get get a role\n";
290 cout
<< " role list list roles with specified path prefix\n";
291 cout
<< " role-trust-policy modify modify the assume role policy of an existing role\n";
292 cout
<< " role-policy put add/update permission policy to role\n";
293 cout
<< " role-policy list list policies attached to a role\n";
294 cout
<< " role-policy get get the specified inline policy document embedded with the given role\n";
295 cout
<< " role-policy delete remove policy attached to a role\n";
296 cout
<< " role update update max_session_duration of a role\n";
297 cout
<< " reshard add schedule a resharding of a bucket\n";
298 cout
<< " reshard list list all bucket resharding or scheduled to be resharded\n";
299 cout
<< " reshard status read bucket resharding status\n";
300 cout
<< " reshard process process of scheduled reshard jobs\n";
301 cout
<< " reshard cancel cancel resharding a bucket\n";
302 cout
<< " reshard stale-instances list list stale-instances from bucket resharding\n";
303 cout
<< " reshard stale-instances delete cleanup stale-instances from bucket resharding\n";
304 cout
<< " sync error list list sync error\n";
305 cout
<< " sync error trim trim sync error\n";
306 cout
<< " mfa create create a new MFA TOTP token\n";
307 cout
<< " mfa list list MFA TOTP tokens\n";
308 cout
<< " mfa get show MFA TOTP token\n";
309 cout
<< " mfa remove delete MFA TOTP token\n";
310 cout
<< " mfa check check MFA TOTP token\n";
311 cout
<< " mfa resync re-sync MFA TOTP token\n";
312 cout
<< " topic list list bucket notifications topics\n";
313 cout
<< " topic get get a bucket notifications topic\n";
314 cout
<< " topic rm remove a bucket notifications topic\n";
315 cout
<< " script put upload a lua script to a context\n";
316 cout
<< " script get get the lua script of a context\n";
317 cout
<< " script rm remove the lua scripts of a context\n";
318 cout
<< " script-package add add a lua package to the scripts allowlist\n";
319 cout
<< " script-package rm remove a lua package from the scripts allowlist\n";
320 cout
<< " script-package list get the lua packages allowlist\n";
321 cout
<< "options:\n";
322 cout
<< " --tenant=<tenant> tenant name\n";
323 cout
<< " --user_ns=<namespace> namespace of user (oidc in case of users authenticated with oidc provider)\n";
324 cout
<< " --uid=<id> user id\n";
325 cout
<< " --new-uid=<id> new user id\n";
326 cout
<< " --subuser=<name> subuser name\n";
327 cout
<< " --access-key=<key> S3 access key\n";
328 cout
<< " --email=<email> user's email address\n";
329 cout
<< " --secret/--secret-key=<key>\n";
330 cout
<< " specify secret key\n";
331 cout
<< " --gen-access-key generate random access key (for S3)\n";
332 cout
<< " --gen-secret generate random secret key\n";
333 cout
<< " --key-type=<type> key type, options are: swift, s3\n";
334 cout
<< " --temp-url-key[-2]=<key> temp url key\n";
335 cout
<< " --access=<access> Set access permissions for sub-user, should be one\n";
336 cout
<< " of read, write, readwrite, full\n";
337 cout
<< " --display-name=<name> user's display name\n";
338 cout
<< " --max-buckets max number of buckets for a user\n";
339 cout
<< " --admin set the admin flag on the user\n";
340 cout
<< " --system set the system flag on the user\n";
341 cout
<< " --op-mask set the op mask on the user\n";
342 cout
<< " --bucket=<bucket> Specify the bucket name. Also used by the quota command.\n";
343 cout
<< " --pool=<pool> Specify the pool name. Also used to scan for leaked rados objects.\n";
344 cout
<< " --object=<object> object name\n";
345 cout
<< " --objects-file=<file> file containing a list of object names to process\n";
346 cout
<< " --object-version=<version> object version\n";
347 cout
<< " --date=<date> date in the format yyyy-mm-dd\n";
348 cout
<< " --start-date=<date> start date in the format yyyy-mm-dd\n";
349 cout
<< " --end-date=<date> end date in the format yyyy-mm-dd\n";
350 cout
<< " --bucket-id=<bucket-id> bucket id\n";
351 cout
<< " --bucket-new-name=<bucket>\n";
352 cout
<< " for bucket link: optional new name\n";
353 cout
<< " --shard-id=<shard-id> optional for: \n";
354 cout
<< " mdlog list\n";
355 cout
<< " data sync status\n";
356 cout
<< " required for: \n";
357 cout
<< " mdlog trim\n";
358 cout
<< " --gen=<gen-id> optional for: \n";
359 cout
<< " bilog list\n";
360 cout
<< " bilog trim\n";
361 cout
<< " bilog status\n";
362 cout
<< " --max-entries=<entries> max entries for listing operations\n";
363 cout
<< " --metadata-key=<key> key to retrieve metadata from with metadata get\n";
364 cout
<< " --remote=<remote> zone or zonegroup id of remote gateway\n";
365 cout
<< " --period=<id> period id\n";
366 cout
<< " --url=<url> url for pushing/pulling period/realm\n";
367 cout
<< " --epoch=<number> period epoch\n";
368 cout
<< " --commit commit the period during 'period update'\n";
369 cout
<< " --staging get staging period info\n";
370 cout
<< " --master set as master\n";
371 cout
<< " --master-zone=<id> master zone id\n";
372 cout
<< " --rgw-realm=<name> realm name\n";
373 cout
<< " --realm-id=<id> realm id\n";
374 cout
<< " --realm-new-name=<name> realm new name\n";
375 cout
<< " --rgw-zonegroup=<name> zonegroup name\n";
376 cout
<< " --zonegroup-id=<id> zonegroup id\n";
377 cout
<< " --zonegroup-new-name=<name>\n";
378 cout
<< " zonegroup new name\n";
379 cout
<< " --rgw-zone=<name> name of zone in which radosgw is running\n";
380 cout
<< " --zone-id=<id> zone id\n";
381 cout
<< " --zone-new-name=<name> zone new name\n";
382 cout
<< " --source-zone specify the source zone (for data sync)\n";
383 cout
<< " --default set entity (realm, zonegroup, zone) as default\n";
384 cout
<< " --read-only set zone as read-only (when adding to zonegroup)\n";
385 cout
<< " --redirect-zone specify zone id to redirect when response is 404 (not found)\n";
386 cout
<< " --placement-id placement id for zonegroup placement commands\n";
387 cout
<< " --storage-class storage class for zonegroup placement commands\n";
388 cout
<< " --tags=<list> list of tags for zonegroup placement add and modify commands\n";
389 cout
<< " --tags-add=<list> list of tags to add for zonegroup placement modify command\n";
390 cout
<< " --tags-rm=<list> list of tags to remove for zonegroup placement modify command\n";
391 cout
<< " --endpoints=<list> zone endpoints\n";
392 cout
<< " --index-pool=<pool> placement target index pool\n";
393 cout
<< " --data-pool=<pool> placement target data pool\n";
394 cout
<< " --data-extra-pool=<pool> placement target data extra (non-ec) pool\n";
395 cout
<< " --placement-index-type=<type>\n";
396 cout
<< " placement target index type (normal, indexless, or #id)\n";
397 cout
<< " --placement-inline-data=<true>\n";
398 cout
<< " set whether the placement target is configured to store a data\n";
399 cout
<< " chunk inline in head objects\n";
400 cout
<< " --compression=<type> placement target compression type (plugin name or empty/none)\n";
401 cout
<< " --tier-type=<type> zone tier type\n";
402 cout
<< " --tier-config=<k>=<v>[,...]\n";
403 cout
<< " set zone tier config keys, values\n";
404 cout
<< " --tier-config-rm=<k>[,...]\n";
405 cout
<< " unset zone tier config keys\n";
406 cout
<< " --sync-from-all[=false] set/reset whether zone syncs from all zonegroup peers\n";
407 cout
<< " --sync-from=[zone-name][,...]\n";
408 cout
<< " set list of zones to sync from\n";
409 cout
<< " --sync-from-rm=[zone-name][,...]\n";
410 cout
<< " remove zones from list of zones to sync from\n";
411 cout
<< " --bucket-index-max-shards override a zone/zonegroup's default bucket index shard count\n";
412 cout
<< " --fix besides checking bucket index, will also fix it\n";
413 cout
<< " --check-objects bucket check: rebuilds bucket index according to\n";
414 cout
<< " actual objects state\n";
415 cout
<< " --format=<format> specify output format for certain operations: xml,\n";
417 cout
<< " --purge-data when specified, user removal will also purge all the\n";
418 cout
<< " user data\n";
419 cout
<< " --purge-keys when specified, subuser removal will also purge all the\n";
420 cout
<< " subuser keys\n";
421 cout
<< " --purge-objects remove a bucket's objects before deleting it\n";
422 cout
<< " (NOTE: required to delete a non-empty bucket)\n";
423 cout
<< " --sync-stats option to 'user stats', update user stats with current\n";
424 cout
<< " stats reported by user's buckets indexes\n";
425 cout
<< " --reset-stats option to 'user stats', reset stats in accordance with user buckets\n";
426 cout
<< " --show-config show configuration\n";
427 cout
<< " --show-log-entries=<flag> enable/disable dump of log entries on log show\n";
428 cout
<< " --show-log-sum=<flag> enable/disable dump of log summation on log show\n";
429 cout
<< " --skip-zero-entries log show only dumps entries that don't have zero value\n";
430 cout
<< " in one of the numeric field\n";
431 cout
<< " --infile=<file> file to read in when setting data\n";
432 cout
<< " --categories=<list> comma separated list of categories, used in usage show\n";
433 cout
<< " --caps=<caps> list of caps (e.g., \"usage=read, write; user=read\")\n";
434 cout
<< " --op-mask=<op-mask> permission of user's operations (e.g., \"read, write, delete, *\")\n";
435 cout
<< " --yes-i-really-mean-it required for certain operations\n";
436 cout
<< " --warnings-only when specified with bucket limit check, list\n";
437 cout
<< " only buckets nearing or over the current max\n";
438 cout
<< " objects per shard value\n";
439 cout
<< " --bypass-gc when specified with bucket deletion, triggers\n";
440 cout
<< " object deletions by not involving GC\n";
441 cout
<< " --inconsistent-index when specified with bucket deletion and bypass-gc set to true,\n";
442 cout
<< " ignores bucket index consistency\n";
443 cout
<< " --min-rewrite-size min object size for bucket rewrite (default 4M)\n";
444 cout
<< " --max-rewrite-size max object size for bucket rewrite (default ULLONG_MAX)\n";
445 cout
<< " --min-rewrite-stripe-size min stripe size for object rewrite (default 0)\n";
446 cout
<< " --trim-delay-ms time interval in msec to limit the frequency of sync error log entries trimming operations,\n";
447 cout
<< " the trimming process will sleep the specified msec for every 1000 entries trimmed\n";
448 cout
<< " --max-concurrent-ios maximum concurrent ios for bucket operations (default: 32)\n";
449 cout
<< " --enable-feature enable a zone/zonegroup feature\n";
450 cout
<< " --disable-feature disable a zone/zonegroup feature\n";
452 cout
<< "<date> := \"YYYY-MM-DD[ hh:mm:ss]\"\n";
453 cout
<< "\nQuota options:\n";
454 cout
<< " --max-objects specify max objects (negative value to disable)\n";
455 cout
<< " --max-size specify max size (in B/K/M/G/T, negative value to disable)\n";
456 cout
<< " --quota-scope scope of quota (bucket, user)\n";
457 cout
<< "\nRate limiting options:\n";
458 cout
<< " --max-read-ops specify max requests per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n";
459 cout
<< " --max-read-bytes specify max bytes per minute for READ ops per RGW (GET and HEAD request methods), 0 means unlimited\n";
460 cout
<< " --max-write-ops specify max requests per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n";
461 cout
<< " --max-write-bytes specify max bytes per minute for WRITE ops per RGW (Not GET or HEAD request methods), 0 means unlimited\n";
462 cout
<< " --ratelimit-scope scope of rate limiting: bucket, user, anonymous\n";
463 cout
<< " anonymous can be configured only with global rate limit\n";
464 cout
<< "\nOrphans search options:\n";
465 cout
<< " --num-shards num of shards to use for keeping the temporary scan info\n";
466 cout
<< " --orphan-stale-secs num of seconds to wait before declaring an object to be an orphan (default: 86400)\n";
467 cout
<< " --job-id set the job id (for orphans find)\n";
468 cout
<< " --detail detailed mode, log and stat head objects as well\n";
469 cout
<< "\nOrphans list-jobs options:\n";
470 cout
<< " --extra-info provide extra info in job list\n";
471 cout
<< "\nRole options:\n";
472 cout
<< " --role-name name of the role to create\n";
473 cout
<< " --path path to the role\n";
474 cout
<< " --assume-role-policy-doc the trust relationship policy document that grants an entity permission to assume the role\n";
475 cout
<< " --policy-name name of the policy document\n";
476 cout
<< " --policy-doc permission policy document\n";
477 cout
<< " --path-prefix path prefix for filtering roles\n";
478 cout
<< "\nMFA options:\n";
479 cout
<< " --totp-serial a string that represents the ID of a TOTP token\n";
480 cout
<< " --totp-seed the secret seed that is used to calculate the TOTP\n";
481 cout
<< " --totp-seconds the time resolution that is being used for TOTP generation\n";
482 cout
<< " --totp-window the number of TOTP tokens that are checked before and after the current token when validating token\n";
483 cout
<< " --totp-pin the valid value of a TOTP token at a certain time\n";
484 cout
<< "\nBucket notifications options:\n";
485 cout
<< " --topic bucket notifications topic name\n";
486 cout
<< "\nScript options:\n";
487 cout
<< " --context context in which the script runs. one of: "+LUA_CONTEXT_LIST
+"\n";
488 cout
<< " --package name of the lua package that should be added/removed to/from the allowlist\n";
489 cout
<< " --allow-compilation package is allowed to compile C code as part of its installation\n";
490 cout
<< "\nradoslist options:\n";
491 cout
<< " --rgw-obj-fs the field separator that will separate the rados\n";
492 cout
<< " object name from the rgw object name;\n";
493 cout
<< " additionally rados objects for incomplete\n";
494 cout
<< " multipart uploads will not be output\n";
496 generic_client_usage();
507 using Aliases
= std::vector
<std::set
<string
> >;
508 using Commands
= std::vector
<Def
>;
512 map
<string
, Node
> next
;
513 set
<string
> expected
; /* separate un-normalized list */
518 map
<string
, string
> alias_map
;
520 string
normalize_alias(const string
& s
) const {
521 auto iter
= alias_map
.find(s
);
522 if (iter
== alias_map
.end()) {
528 void init_alias_map(Aliases
& aliases
) {
529 for (auto& alias_set
: aliases
) {
530 std::optional
<string
> first
;
532 for (auto& alias
: alias_set
) {
536 alias_map
[alias
] = *first
;
542 bool gen_next_expected(Node
*node
, vector
<string
> *expected
, bool ret
) {
543 for (auto& next_cmd
: node
->expected
) {
544 expected
->push_back(next_cmd
);
554 SimpleCmd(std::optional
<Commands
> cmds
,
555 std::optional
<Aliases
> aliases
) {
557 add_aliases(*aliases
);
565 void add_aliases(Aliases
& aliases
) {
566 init_alias_map(aliases
);
569 void add_commands(std::vector
<Def
>& cmds
) {
570 for (auto& cmd
: cmds
) {
571 vector
<string
> words
;
572 get_str_vec(cmd
.cmd
, " ", words
);
574 auto node
= &cmd_root
;
575 for (auto& word
: words
) {
576 auto norm
= normalize_alias(word
);
579 node
->expected
.insert(word
);
581 node
= &node
->next
[norm
];
583 if (norm
== "[*]") { /* optional param at the end */
584 parent
->next
["*"] = *node
; /* can be also looked up by '*' */
585 parent
->opt
= cmd
.opt
;
593 template <class Container
>
594 bool find_command(Container
& args
,
596 vector
<string
> *extra_args
,
598 vector
<string
> *expected
) {
599 auto node
= &cmd_root
;
601 std::optional
<std::any
> found_opt
;
603 for (auto& arg
: args
) {
604 string norm
= normalize_alias(arg
);
605 auto iter
= node
->next
.find(norm
);
606 if (iter
== node
->next
.end()) {
607 iter
= node
->next
.find("*");
608 if (iter
== node
->next
.end()) {
609 *error
= string("ERROR: Unrecognized argument: '") + arg
+ "'";
610 return gen_next_expected(node
, expected
, false);
612 extra_args
->push_back(arg
);
614 found_opt
= node
->opt
;
617 node
= &(iter
->second
);
620 *opt_cmd
= found_opt
.value_or(node
->opt
);
622 if (!opt_cmd
->has_value()) {
623 *error
="ERROR: Unknown command";
624 return gen_next_expected(node
, expected
, false);
632 namespace rgw_admin
{
658 BUCKET_SYNC_CHECKPOINT
,
671 BUCKET_SHARD_OBJECTS
,
690 OBJECTS_EXPIRE_STALE_LIST
,
691 OBJECTS_EXPIRE_STALE_RM
,
724 ZONEGROUP_PLACEMENT_ADD
,
725 ZONEGROUP_PLACEMENT_MODIFY
,
726 ZONEGROUP_PLACEMENT_RM
,
727 ZONEGROUP_PLACEMENT_LIST
,
728 ZONEGROUP_PLACEMENT_GET
,
729 ZONEGROUP_PLACEMENT_DEFAULT
,
739 ZONE_PLACEMENT_MODIFY
,
749 METADATA_SYNC_STATUS
,
763 SYNC_GROUP_FLOW_CREATE
,
764 SYNC_GROUP_FLOW_REMOVE
,
765 SYNC_GROUP_PIPE_CREATE
,
766 SYNC_GROUP_PIPE_MODIFY
,
767 SYNC_GROUP_PIPE_REMOVE
,
803 GLOBAL_QUOTA_DISABLE
,
804 GLOBAL_RATELIMIT_GET
,
805 GLOBAL_RATELIMIT_SET
,
806 GLOBAL_RATELIMIT_ENABLE
,
807 GLOBAL_RATELIMIT_DISABLE
,
813 ROLE_TRUST_POLICY_MODIFY
,
831 RESHARD_STALE_INSTANCES_LIST
,
832 RESHARD_STALE_INSTANCES_DELETE
,
846 using namespace rgw_admin
;
848 static SimpleCmd::Commands all_cmds
= {
849 { "user create", OPT::USER_CREATE
},
850 { "user info", OPT::USER_INFO
},
851 { "user modify", OPT::USER_MODIFY
},
852 { "user rename", OPT::USER_RENAME
},
853 { "user rm", OPT::USER_RM
},
854 { "user suspend", OPT::USER_SUSPEND
},
855 { "user enable", OPT::USER_ENABLE
},
856 { "user check", OPT::USER_CHECK
},
857 { "user stats", OPT::USER_STATS
},
858 { "user list", OPT::USER_LIST
},
859 { "subuser create", OPT::SUBUSER_CREATE
},
860 { "subuser modify", OPT::SUBUSER_MODIFY
},
861 { "subuser rm", OPT::SUBUSER_RM
},
862 { "key create", OPT::KEY_CREATE
},
863 { "key rm", OPT::KEY_RM
},
864 { "buckets list", OPT::BUCKETS_LIST
},
865 { "bucket list", OPT::BUCKETS_LIST
},
866 { "bucket limit check", OPT::BUCKET_LIMIT_CHECK
},
867 { "bucket link", OPT::BUCKET_LINK
},
868 { "bucket unlink", OPT::BUCKET_UNLINK
},
869 { "bucket layout", OPT::BUCKET_LAYOUT
},
870 { "bucket stats", OPT::BUCKET_STATS
},
871 { "bucket check", OPT::BUCKET_CHECK
},
872 { "bucket sync checkpoint", OPT::BUCKET_SYNC_CHECKPOINT
},
873 { "bucket sync info", OPT::BUCKET_SYNC_INFO
},
874 { "bucket sync status", OPT::BUCKET_SYNC_STATUS
},
875 { "bucket sync markers", OPT::BUCKET_SYNC_MARKERS
},
876 { "bucket sync init", OPT::BUCKET_SYNC_INIT
},
877 { "bucket sync run", OPT::BUCKET_SYNC_RUN
},
878 { "bucket sync disable", OPT::BUCKET_SYNC_DISABLE
},
879 { "bucket sync enable", OPT::BUCKET_SYNC_ENABLE
},
880 { "bucket rm", OPT::BUCKET_RM
},
881 { "bucket rewrite", OPT::BUCKET_REWRITE
},
882 { "bucket reshard", OPT::BUCKET_RESHARD
},
883 { "bucket chown", OPT::BUCKET_CHOWN
},
884 { "bucket radoslist", OPT::BUCKET_RADOS_LIST
},
885 { "bucket rados list", OPT::BUCKET_RADOS_LIST
},
886 { "bucket shard objects", OPT::BUCKET_SHARD_OBJECTS
},
887 { "bucket shard object", OPT::BUCKET_SHARD_OBJECTS
},
888 { "bucket object shard", OPT::BUCKET_OBJECT_SHARD
},
889 { "policy", OPT::POLICY
},
890 { "pool add", OPT::POOL_ADD
},
891 { "pool rm", OPT::POOL_RM
},
892 { "pool list", OPT::POOLS_LIST
},
893 { "pools list", OPT::POOLS_LIST
},
894 { "log list", OPT::LOG_LIST
},
895 { "log show", OPT::LOG_SHOW
},
896 { "log rm", OPT::LOG_RM
},
897 { "usage show", OPT::USAGE_SHOW
},
898 { "usage trim", OPT::USAGE_TRIM
},
899 { "usage clear", OPT::USAGE_CLEAR
},
900 { "object put", OPT::OBJECT_PUT
},
901 { "object rm", OPT::OBJECT_RM
},
902 { "object unlink", OPT::OBJECT_UNLINK
},
903 { "object stat", OPT::OBJECT_STAT
},
904 { "object rewrite", OPT::OBJECT_REWRITE
},
905 { "object reindex", OPT::OBJECT_REINDEX
},
906 { "objects expire", OPT::OBJECTS_EXPIRE
},
907 { "objects expire-stale list", OPT::OBJECTS_EXPIRE_STALE_LIST
},
908 { "objects expire-stale rm", OPT::OBJECTS_EXPIRE_STALE_RM
},
909 { "bi get", OPT::BI_GET
},
910 { "bi put", OPT::BI_PUT
},
911 { "bi list", OPT::BI_LIST
},
912 { "bi purge", OPT::BI_PURGE
},
913 { "olh get", OPT::OLH_GET
},
914 { "olh readlog", OPT::OLH_READLOG
},
915 { "quota set", OPT::QUOTA_SET
},
916 { "quota enable", OPT::QUOTA_ENABLE
},
917 { "quota disable", OPT::QUOTA_DISABLE
},
918 { "ratelimit get", OPT::RATELIMIT_GET
},
919 { "ratelimit set", OPT::RATELIMIT_SET
},
920 { "ratelimit enable", OPT::RATELIMIT_ENABLE
},
921 { "ratelimit disable", OPT::RATELIMIT_DISABLE
},
922 { "gc list", OPT::GC_LIST
},
923 { "gc process", OPT::GC_PROCESS
},
924 { "lc list", OPT::LC_LIST
},
925 { "lc get", OPT::LC_GET
},
926 { "lc process", OPT::LC_PROCESS
},
927 { "lc reshard fix", OPT::LC_RESHARD_FIX
},
928 { "orphans find", OPT::ORPHANS_FIND
},
929 { "orphans finish", OPT::ORPHANS_FINISH
},
930 { "orphans list jobs", OPT::ORPHANS_LIST_JOBS
},
931 { "orphans list-jobs", OPT::ORPHANS_LIST_JOBS
},
932 { "zonegroup add", OPT::ZONEGROUP_ADD
},
933 { "zonegroup create", OPT::ZONEGROUP_CREATE
},
934 { "zonegroup default", OPT::ZONEGROUP_DEFAULT
},
935 { "zonegroup delete", OPT::ZONEGROUP_DELETE
},
936 { "zonegroup get", OPT::ZONEGROUP_GET
},
937 { "zonegroup modify", OPT::ZONEGROUP_MODIFY
},
938 { "zonegroup set", OPT::ZONEGROUP_SET
},
939 { "zonegroup list", OPT::ZONEGROUP_LIST
},
940 { "zonegroups list", OPT::ZONEGROUP_LIST
},
941 { "zonegroup remove", OPT::ZONEGROUP_REMOVE
},
942 { "zonegroup remove zone", OPT::ZONEGROUP_REMOVE
},
943 { "zonegroup rename", OPT::ZONEGROUP_RENAME
},
944 { "zonegroup placement add", OPT::ZONEGROUP_PLACEMENT_ADD
},
945 { "zonegroup placement modify", OPT::ZONEGROUP_PLACEMENT_MODIFY
},
946 { "zonegroup placement rm", OPT::ZONEGROUP_PLACEMENT_RM
},
947 { "zonegroup placement list", OPT::ZONEGROUP_PLACEMENT_LIST
},
948 { "zonegroup placement get", OPT::ZONEGROUP_PLACEMENT_GET
},
949 { "zonegroup placement default", OPT::ZONEGROUP_PLACEMENT_DEFAULT
},
950 { "zone create", OPT::ZONE_CREATE
},
951 { "zone delete", OPT::ZONE_DELETE
},
952 { "zone get", OPT::ZONE_GET
},
953 { "zone modify", OPT::ZONE_MODIFY
},
954 { "zone set", OPT::ZONE_SET
},
955 { "zone list", OPT::ZONE_LIST
},
956 { "zones list", OPT::ZONE_LIST
},
957 { "zone rename", OPT::ZONE_RENAME
},
958 { "zone default", OPT::ZONE_DEFAULT
},
959 { "zone placement add", OPT::ZONE_PLACEMENT_ADD
},
960 { "zone placement modify", OPT::ZONE_PLACEMENT_MODIFY
},
961 { "zone placement rm", OPT::ZONE_PLACEMENT_RM
},
962 { "zone placement list", OPT::ZONE_PLACEMENT_LIST
},
963 { "zone placement get", OPT::ZONE_PLACEMENT_GET
},
964 { "caps add", OPT::CAPS_ADD
},
965 { "caps rm", OPT::CAPS_RM
},
966 { "metadata get [*]", OPT::METADATA_GET
},
967 { "metadata put [*]", OPT::METADATA_PUT
},
968 { "metadata rm [*]", OPT::METADATA_RM
},
969 { "metadata list [*]", OPT::METADATA_LIST
},
970 { "metadata sync status", OPT::METADATA_SYNC_STATUS
},
971 { "metadata sync init", OPT::METADATA_SYNC_INIT
},
972 { "metadata sync run", OPT::METADATA_SYNC_RUN
},
973 { "mdlog list", OPT::MDLOG_LIST
},
974 { "mdlog autotrim", OPT::MDLOG_AUTOTRIM
},
975 { "mdlog trim", OPT::MDLOG_TRIM
},
976 { "mdlog fetch", OPT::MDLOG_FETCH
},
977 { "mdlog status", OPT::MDLOG_STATUS
},
978 { "sync error list", OPT::SYNC_ERROR_LIST
},
979 { "sync error trim", OPT::SYNC_ERROR_TRIM
},
980 { "sync policy get", OPT::SYNC_POLICY_GET
},
981 { "sync group create", OPT::SYNC_GROUP_CREATE
},
982 { "sync group modify", OPT::SYNC_GROUP_MODIFY
},
983 { "sync group get", OPT::SYNC_GROUP_GET
},
984 { "sync group remove", OPT::SYNC_GROUP_REMOVE
},
985 { "sync group flow create", OPT::SYNC_GROUP_FLOW_CREATE
},
986 { "sync group flow remove", OPT::SYNC_GROUP_FLOW_REMOVE
},
987 { "sync group pipe create", OPT::SYNC_GROUP_PIPE_CREATE
},
988 { "sync group pipe modify", OPT::SYNC_GROUP_PIPE_MODIFY
},
989 { "sync group pipe remove", OPT::SYNC_GROUP_PIPE_REMOVE
},
990 { "bilog list", OPT::BILOG_LIST
},
991 { "bilog trim", OPT::BILOG_TRIM
},
992 { "bilog status", OPT::BILOG_STATUS
},
993 { "bilog autotrim", OPT::BILOG_AUTOTRIM
},
994 { "data sync status", OPT::DATA_SYNC_STATUS
},
995 { "data sync init", OPT::DATA_SYNC_INIT
},
996 { "data sync run", OPT::DATA_SYNC_RUN
},
997 { "datalog list", OPT::DATALOG_LIST
},
998 { "datalog status", OPT::DATALOG_STATUS
},
999 { "datalog autotrim", OPT::DATALOG_AUTOTRIM
},
1000 { "datalog trim", OPT::DATALOG_TRIM
},
1001 { "datalog type", OPT::DATALOG_TYPE
},
1002 { "datalog prune", OPT::DATALOG_PRUNE
},
1003 { "realm create", OPT::REALM_CREATE
},
1004 { "realm rm", OPT::REALM_DELETE
},
1005 { "realm get", OPT::REALM_GET
},
1006 { "realm get default", OPT::REALM_GET_DEFAULT
},
1007 { "realm get-default", OPT::REALM_GET_DEFAULT
},
1008 { "realm list", OPT::REALM_LIST
},
1009 { "realm list periods", OPT::REALM_LIST_PERIODS
},
1010 { "realm list-periods", OPT::REALM_LIST_PERIODS
},
1011 { "realm rename", OPT::REALM_RENAME
},
1012 { "realm set", OPT::REALM_SET
},
1013 { "realm default", OPT::REALM_DEFAULT
},
1014 { "realm pull", OPT::REALM_PULL
},
1015 { "period delete", OPT::PERIOD_DELETE
},
1016 { "period get", OPT::PERIOD_GET
},
1017 { "period get-current", OPT::PERIOD_GET_CURRENT
},
1018 { "period get current", OPT::PERIOD_GET_CURRENT
},
1019 { "period pull", OPT::PERIOD_PULL
},
1020 { "period push", OPT::PERIOD_PUSH
},
1021 { "period list", OPT::PERIOD_LIST
},
1022 { "period update", OPT::PERIOD_UPDATE
},
1023 { "period commit", OPT::PERIOD_COMMIT
},
1024 { "global quota get", OPT::GLOBAL_QUOTA_GET
},
1025 { "global quota set", OPT::GLOBAL_QUOTA_SET
},
1026 { "global quota enable", OPT::GLOBAL_QUOTA_ENABLE
},
1027 { "global quota disable", OPT::GLOBAL_QUOTA_DISABLE
},
1028 { "global ratelimit get", OPT::GLOBAL_RATELIMIT_GET
},
1029 { "global ratelimit set", OPT::GLOBAL_RATELIMIT_SET
},
1030 { "global ratelimit enable", OPT::GLOBAL_RATELIMIT_ENABLE
},
1031 { "global ratelimit disable", OPT::GLOBAL_RATELIMIT_DISABLE
},
1032 { "sync info", OPT::SYNC_INFO
},
1033 { "sync status", OPT::SYNC_STATUS
},
1034 { "role create", OPT::ROLE_CREATE
},
1035 { "role delete", OPT::ROLE_DELETE
},
1036 { "role get", OPT::ROLE_GET
},
1037 { "role-trust-policy modify", OPT::ROLE_TRUST_POLICY_MODIFY
},
1038 { "role list", OPT::ROLE_LIST
},
1039 { "role policy put", OPT::ROLE_POLICY_PUT
},
1040 { "role-policy put", OPT::ROLE_POLICY_PUT
},
1041 { "role policy list", OPT::ROLE_POLICY_LIST
},
1042 { "role-policy list", OPT::ROLE_POLICY_LIST
},
1043 { "role policy get", OPT::ROLE_POLICY_GET
},
1044 { "role-policy get", OPT::ROLE_POLICY_GET
},
1045 { "role policy delete", OPT::ROLE_POLICY_DELETE
},
1046 { "role-policy delete", OPT::ROLE_POLICY_DELETE
},
1047 { "role update", OPT::ROLE_UPDATE
},
1048 { "reshard bucket", OPT::BUCKET_RESHARD
},
1049 { "reshard add", OPT::RESHARD_ADD
},
1050 { "reshard list", OPT::RESHARD_LIST
},
1051 { "reshard status", OPT::RESHARD_STATUS
},
1052 { "reshard process", OPT::RESHARD_PROCESS
},
1053 { "reshard cancel", OPT::RESHARD_CANCEL
},
1054 { "mfa create", OPT::MFA_CREATE
},
1055 { "mfa remove", OPT::MFA_REMOVE
},
1056 { "mfa get", OPT::MFA_GET
},
1057 { "mfa list", OPT::MFA_LIST
},
1058 { "mfa check", OPT::MFA_CHECK
},
1059 { "mfa resync", OPT::MFA_RESYNC
},
1060 { "reshard stale-instances list", OPT::RESHARD_STALE_INSTANCES_LIST
},
1061 { "reshard stale list", OPT::RESHARD_STALE_INSTANCES_LIST
},
1062 { "reshard stale-instances delete", OPT::RESHARD_STALE_INSTANCES_DELETE
},
1063 { "reshard stale delete", OPT::RESHARD_STALE_INSTANCES_DELETE
},
1064 { "topic list", OPT::PUBSUB_TOPICS_LIST
},
1065 { "topic get", OPT::PUBSUB_TOPIC_GET
},
1066 { "topic rm", OPT::PUBSUB_TOPIC_RM
},
1067 { "script put", OPT::SCRIPT_PUT
},
1068 { "script get", OPT::SCRIPT_GET
},
1069 { "script rm", OPT::SCRIPT_RM
},
1070 { "script-package add", OPT::SCRIPT_PACKAGE_ADD
},
1071 { "script-package rm", OPT::SCRIPT_PACKAGE_RM
},
1072 { "script-package list", OPT::SCRIPT_PACKAGE_LIST
},
1075 static SimpleCmd::Aliases cmd_aliases
= {
1076 { "delete", "del" },
1083 BIIndexType
get_bi_index_type(const string
& type_str
) {
1084 if (type_str
== "plain")
1085 return BIIndexType::Plain
;
1086 if (type_str
== "instance")
1087 return BIIndexType::Instance
;
1088 if (type_str
== "olh")
1089 return BIIndexType::OLH
;
1091 return BIIndexType::Invalid
;
1094 log_type
get_log_type(const string
& type_str
) {
1095 if (strcasecmp(type_str
.c_str(), "fifo") == 0)
1096 return log_type::fifo
;
1097 if (strcasecmp(type_str
.c_str(), "omap") == 0)
1098 return log_type::omap
;
1100 return static_cast<log_type
>(0xff);
1103 void dump_bi_entry(bufferlist
& bl
, BIIndexType index_type
, Formatter
*formatter
)
1105 auto iter
= bl
.cbegin();
1106 switch (index_type
) {
1107 case BIIndexType::Plain
:
1108 case BIIndexType::Instance
:
1110 rgw_bucket_dir_entry entry
;
1111 decode(entry
, iter
);
1112 encode_json("entry", entry
, formatter
);
1115 case BIIndexType::OLH
:
1117 rgw_bucket_olh_entry entry
;
1118 decode(entry
, iter
);
1119 encode_json("entry", entry
, formatter
);
1128 static void show_user_info(RGWUserInfo
& info
, Formatter
*formatter
)
1130 encode_json("user_info", info
, formatter
);
1131 formatter
->flush(cout
);
1135 static void show_perm_policy(string perm_policy
, Formatter
* formatter
)
1137 formatter
->open_object_section("role");
1138 formatter
->dump_string("Permission policy", perm_policy
);
1139 formatter
->close_section();
1140 formatter
->flush(cout
);
1143 static void show_policy_names(std::vector
<string
> policy_names
, Formatter
* formatter
)
1145 formatter
->open_array_section("PolicyNames");
1146 for (const auto& it
: policy_names
) {
1147 formatter
->dump_string("policyname", it
);
1149 formatter
->close_section();
1150 formatter
->flush(cout
);
1153 static void show_role_info(rgw::sal::RGWRole
* role
, Formatter
* formatter
)
1155 formatter
->open_object_section("role");
1156 role
->dump(formatter
);
1157 formatter
->close_section();
1158 formatter
->flush(cout
);
1161 static void show_roles_info(vector
<std::unique_ptr
<rgw::sal::RGWRole
>>& roles
, Formatter
* formatter
)
1163 formatter
->open_array_section("Roles");
1164 for (const auto& it
: roles
) {
1165 formatter
->open_object_section("role");
1166 it
->dump(formatter
);
1167 formatter
->close_section();
1169 formatter
->close_section();
1170 formatter
->flush(cout
);
1173 static void show_reshard_status(
1174 const list
<cls_rgw_bucket_instance_entry
>& status
, Formatter
*formatter
)
1176 formatter
->open_array_section("status");
1177 for (const auto& entry
: status
) {
1178 formatter
->open_object_section("entry");
1179 formatter
->dump_string("reshard_status", to_string(entry
.reshard_status
));
1180 formatter
->close_section();
1182 formatter
->close_section();
1183 formatter
->flush(cout
);
1186 class StoreDestructor
{
1187 rgw::sal::Driver
* driver
;
1189 explicit StoreDestructor(rgw::sal::Driver
* _s
) : driver(_s
) {}
1190 ~StoreDestructor() {
1191 DriverManager::close_storage(driver
);
1192 rgw_http_client_cleanup();
1196 static int init_bucket(rgw::sal::User
* user
, const rgw_bucket
& b
,
1197 std::unique_ptr
<rgw::sal::Bucket
>* bucket
)
1199 return driver
->get_bucket(dpp(), user
, b
, bucket
, null_yield
);
1202 static int init_bucket(rgw::sal::User
* user
,
1203 const string
& tenant_name
,
1204 const string
& bucket_name
,
1205 const string
& bucket_id
,
1206 std::unique_ptr
<rgw::sal::Bucket
>* bucket
)
1208 rgw_bucket b
{tenant_name
, bucket_name
, bucket_id
};
1209 return init_bucket(user
, b
, bucket
);
1212 static int read_input(const string
& infile
, bufferlist
& bl
)
1215 if (infile
.size()) {
1216 fd
= open(infile
.c_str(), O_RDONLY
);
1219 cerr
<< "error reading input file " << infile
<< std::endl
;
1224 #define READ_CHUNK 8196
1229 char buf
[READ_CHUNK
];
1231 r
= safe_read(fd
, buf
, READ_CHUNK
);
1234 cerr
<< "error while reading input" << std::endl
;
1242 if (infile
.size()) {
1249 static int read_decode_json(const string
& infile
, T
& t
)
1252 int ret
= read_input(infile
, bl
);
1254 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
1258 if (!p
.parse(bl
.c_str(), bl
.length())) {
1259 cout
<< "failed to parse JSON" << std::endl
;
1264 decode_json_obj(t
, &p
);
1265 } catch (const JSONDecoder::err
& e
) {
1266 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
1272 template <class T
, class K
>
1273 static int read_decode_json(const string
& infile
, T
& t
, K
*k
)
1276 int ret
= read_input(infile
, bl
);
1278 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
1282 if (!p
.parse(bl
.c_str(), bl
.length())) {
1283 cout
<< "failed to parse JSON" << std::endl
;
1288 t
.decode_json(&p
, k
);
1289 } catch (const JSONDecoder::err
& e
) {
1290 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
1297 static bool decode_dump(const char *field_name
, bufferlist
& bl
, Formatter
*f
)
1301 auto iter
= bl
.cbegin();
1305 } catch (buffer::error
& err
) {
1309 encode_json(field_name
, t
, f
);
1314 static bool dump_string(const char *field_name
, bufferlist
& bl
, Formatter
*f
)
1316 string val
= bl
.to_str();
1317 f
->dump_string(field_name
, val
.c_str() /* hide encoded null termination chars */);
1322 bool set_ratelimit_info(RGWRateLimitInfo
& ratelimit
, OPT opt_cmd
, int64_t max_read_ops
, int64_t max_write_ops
,
1323 int64_t max_read_bytes
, int64_t max_write_bytes
,
1324 bool have_max_read_ops
, bool have_max_write_ops
,
1325 bool have_max_read_bytes
, bool have_max_write_bytes
)
1327 bool ratelimit_configured
= true;
1329 case OPT::RATELIMIT_ENABLE
:
1330 case OPT::GLOBAL_RATELIMIT_ENABLE
:
1331 ratelimit
.enabled
= true;
1334 case OPT::RATELIMIT_SET
:
1335 case OPT::GLOBAL_RATELIMIT_SET
:
1336 ratelimit_configured
= false;
1337 if (have_max_read_ops
) {
1338 if (max_read_ops
>= 0) {
1339 ratelimit
.max_read_ops
= max_read_ops
;
1340 ratelimit_configured
= true;
1343 if (have_max_write_ops
) {
1344 if (max_write_ops
>= 0) {
1345 ratelimit
.max_write_ops
= max_write_ops
;
1346 ratelimit_configured
= true;
1349 if (have_max_read_bytes
) {
1350 if (max_read_bytes
>= 0) {
1351 ratelimit
.max_read_bytes
= max_read_bytes
;
1352 ratelimit_configured
= true;
1355 if (have_max_write_bytes
) {
1356 if (max_write_bytes
>= 0) {
1357 ratelimit
.max_write_bytes
= max_write_bytes
;
1358 ratelimit_configured
= true;
1362 case OPT::RATELIMIT_DISABLE
:
1363 case OPT::GLOBAL_RATELIMIT_DISABLE
:
1364 ratelimit
.enabled
= false;
1369 return ratelimit_configured
;
1372 void set_quota_info(RGWQuotaInfo
& quota
, OPT opt_cmd
, int64_t max_size
, int64_t max_objects
,
1373 bool have_max_size
, bool have_max_objects
)
1376 case OPT::QUOTA_ENABLE
:
1377 case OPT::GLOBAL_QUOTA_ENABLE
:
1378 quota
.enabled
= true;
1380 // falling through on purpose
1382 case OPT::QUOTA_SET
:
1383 case OPT::GLOBAL_QUOTA_SET
:
1384 if (have_max_objects
) {
1385 if (max_objects
< 0) {
1386 quota
.max_objects
= -1;
1388 quota
.max_objects
= max_objects
;
1391 if (have_max_size
) {
1393 quota
.max_size
= -1;
1395 quota
.max_size
= rgw_rounded_kb(max_size
) * 1024;
1399 case OPT::QUOTA_DISABLE
:
1400 case OPT::GLOBAL_QUOTA_DISABLE
:
1401 quota
.enabled
= false;
1408 int set_bucket_quota(rgw::sal::Driver
* driver
, OPT opt_cmd
,
1409 const string
& tenant_name
, const string
& bucket_name
,
1410 int64_t max_size
, int64_t max_objects
,
1411 bool have_max_size
, bool have_max_objects
)
1413 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1414 int r
= driver
->get_bucket(dpp(), nullptr, tenant_name
, bucket_name
, &bucket
, null_yield
);
1416 cerr
<< "could not get bucket info for bucket=" << bucket_name
<< ": " << cpp_strerror(-r
) << std::endl
;
1420 set_quota_info(bucket
->get_info().quota
, opt_cmd
, max_size
, max_objects
, have_max_size
, have_max_objects
);
1422 r
= bucket
->put_info(dpp(), false, real_time());
1424 cerr
<< "ERROR: failed writing bucket instance info: " << cpp_strerror(-r
) << std::endl
;
1430 int set_bucket_ratelimit(rgw::sal::Driver
* driver
, OPT opt_cmd
,
1431 const string
& tenant_name
, const string
& bucket_name
,
1432 int64_t max_read_ops
, int64_t max_write_ops
,
1433 int64_t max_read_bytes
, int64_t max_write_bytes
,
1434 bool have_max_read_ops
, bool have_max_write_ops
,
1435 bool have_max_read_bytes
, bool have_max_write_bytes
)
1437 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1438 int r
= driver
->get_bucket(dpp(), nullptr, tenant_name
, bucket_name
, &bucket
, null_yield
);
1440 cerr
<< "could not get bucket info for bucket=" << bucket_name
<< ": " << cpp_strerror(-r
) << std::endl
;
1443 RGWRateLimitInfo ratelimit_info
;
1444 auto iter
= bucket
->get_attrs().find(RGW_ATTR_RATELIMIT
);
1445 if(iter
!= bucket
->get_attrs().end()) {
1447 bufferlist
& bl
= iter
->second
;
1448 auto biter
= bl
.cbegin();
1449 decode(ratelimit_info
, biter
);
1450 } catch (buffer::error
& err
) {
1451 ldpp_dout(dpp(), 0) << "ERROR: failed to decode rate limit" << dendl
;
1455 bool ratelimit_configured
= set_ratelimit_info(ratelimit_info
, opt_cmd
, max_read_ops
, max_write_ops
,
1456 max_read_bytes
, max_write_bytes
,
1457 have_max_read_ops
, have_max_write_ops
,
1458 have_max_read_bytes
, have_max_write_bytes
);
1459 if (!ratelimit_configured
) {
1460 ldpp_dout(dpp(), 0) << "ERROR: no rate limit values have been specified" << dendl
;
1464 ratelimit_info
.encode(bl
);
1465 rgw::sal::Attrs attr
;
1466 attr
[RGW_ATTR_RATELIMIT
] = bl
;
1467 r
= bucket
->merge_and_store_attrs(dpp(), attr
, null_yield
);
1469 cerr
<< "ERROR: failed writing bucket instance info: " << cpp_strerror(-r
) << std::endl
;
1475 int set_user_ratelimit(OPT opt_cmd
, std::unique_ptr
<rgw::sal::User
>& user
,
1476 int64_t max_read_ops
, int64_t max_write_ops
,
1477 int64_t max_read_bytes
, int64_t max_write_bytes
,
1478 bool have_max_read_ops
, bool have_max_write_ops
,
1479 bool have_max_read_bytes
, bool have_max_write_bytes
)
1481 RGWRateLimitInfo ratelimit_info
;
1482 user
->load_user(dpp(), null_yield
);
1483 auto iter
= user
->get_attrs().find(RGW_ATTR_RATELIMIT
);
1484 if(iter
!= user
->get_attrs().end()) {
1486 bufferlist
& bl
= iter
->second
;
1487 auto biter
= bl
.cbegin();
1488 decode(ratelimit_info
, biter
);
1489 } catch (buffer::error
& err
) {
1490 ldpp_dout(dpp(), 0) << "ERROR: failed to decode rate limit" << dendl
;
1494 bool ratelimit_configured
= set_ratelimit_info(ratelimit_info
, opt_cmd
, max_read_ops
, max_write_ops
,
1495 max_read_bytes
, max_write_bytes
,
1496 have_max_read_ops
, have_max_write_ops
,
1497 have_max_read_bytes
, have_max_write_bytes
);
1498 if (!ratelimit_configured
) {
1499 ldpp_dout(dpp(), 0) << "ERROR: no rate limit values have been specified" << dendl
;
1503 ratelimit_info
.encode(bl
);
1504 rgw::sal::Attrs attr
;
1505 attr
[RGW_ATTR_RATELIMIT
] = bl
;
1506 int r
= user
->merge_and_store_attrs(dpp(), attr
, null_yield
);
1508 cerr
<< "ERROR: failed writing user instance info: " << cpp_strerror(-r
) << std::endl
;
1514 int show_user_ratelimit(std::unique_ptr
<rgw::sal::User
>& user
, Formatter
*formatter
)
1516 RGWRateLimitInfo ratelimit_info
;
1517 user
->load_user(dpp(), null_yield
);
1518 auto iter
= user
->get_attrs().find(RGW_ATTR_RATELIMIT
);
1519 if(iter
!= user
->get_attrs().end()) {
1521 bufferlist
& bl
= iter
->second
;
1522 auto biter
= bl
.cbegin();
1523 decode(ratelimit_info
, biter
);
1524 } catch (buffer::error
& err
) {
1525 ldpp_dout(dpp(), 0) << "ERROR: failed to decode rate limit" << dendl
;
1529 formatter
->open_object_section("user_ratelimit");
1530 encode_json("user_ratelimit", ratelimit_info
, formatter
);
1531 formatter
->close_section();
1532 formatter
->flush(cout
);
1537 int show_bucket_ratelimit(rgw::sal::Driver
* driver
, const string
& tenant_name
,
1538 const string
& bucket_name
, Formatter
*formatter
)
1540 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1541 int r
= driver
->get_bucket(dpp(), nullptr, tenant_name
, bucket_name
, &bucket
, null_yield
);
1543 cerr
<< "could not get bucket info for bucket=" << bucket_name
<< ": " << cpp_strerror(-r
) << std::endl
;
1546 RGWRateLimitInfo ratelimit_info
;
1547 auto iter
= bucket
->get_attrs().find(RGW_ATTR_RATELIMIT
);
1548 if (iter
!= bucket
->get_attrs().end()) {
1550 bufferlist
& bl
= iter
->second
;
1551 auto biter
= bl
.cbegin();
1552 decode(ratelimit_info
, biter
);
1553 } catch (buffer::error
& err
) {
1554 ldpp_dout(dpp(), 0) << "ERROR: failed to decode rate limit" << dendl
;
1558 formatter
->open_object_section("bucket_ratelimit");
1559 encode_json("bucket_ratelimit", ratelimit_info
, formatter
);
1560 formatter
->close_section();
1561 formatter
->flush(cout
);
1565 int set_user_bucket_quota(OPT opt_cmd
, RGWUser
& user
, RGWUserAdminOpState
& op_state
, int64_t max_size
, int64_t max_objects
,
1566 bool have_max_size
, bool have_max_objects
)
1568 RGWUserInfo
& user_info
= op_state
.get_user_info();
1570 set_quota_info(user_info
.quota
.bucket_quota
, opt_cmd
, max_size
, max_objects
, have_max_size
, have_max_objects
);
1572 op_state
.set_bucket_quota(user_info
.quota
.bucket_quota
);
1575 int r
= user
.modify(dpp(), op_state
, null_yield
, &err
);
1577 cerr
<< "ERROR: failed updating user info: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
1583 int set_user_quota(OPT opt_cmd
, RGWUser
& user
, RGWUserAdminOpState
& op_state
, int64_t max_size
, int64_t max_objects
,
1584 bool have_max_size
, bool have_max_objects
)
1586 RGWUserInfo
& user_info
= op_state
.get_user_info();
1588 set_quota_info(user_info
.quota
.user_quota
, opt_cmd
, max_size
, max_objects
, have_max_size
, have_max_objects
);
1590 op_state
.set_user_quota(user_info
.quota
.user_quota
);
1593 int r
= user
.modify(dpp(), op_state
, null_yield
, &err
);
1595 cerr
<< "ERROR: failed updating user info: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
1601 int check_min_obj_stripe_size(rgw::sal::Driver
* driver
, rgw::sal::Object
* obj
, uint64_t min_stripe_size
, bool *need_rewrite
)
1603 int ret
= obj
->get_obj_attrs(null_yield
, dpp());
1605 ldpp_dout(dpp(), -1) << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret
) << dendl
;
1609 map
<string
, bufferlist
>::iterator iter
;
1610 iter
= obj
->get_attrs().find(RGW_ATTR_MANIFEST
);
1611 if (iter
== obj
->get_attrs().end()) {
1612 *need_rewrite
= (obj
->get_obj_size() >= min_stripe_size
);
1616 RGWObjManifest manifest
;
1619 bufferlist
& bl
= iter
->second
;
1620 auto biter
= bl
.cbegin();
1621 decode(manifest
, biter
);
1622 } catch (buffer::error
& err
) {
1623 ldpp_dout(dpp(), 0) << "ERROR: failed to decode manifest" << dendl
;
1627 map
<uint64_t, RGWObjManifestPart
>& objs
= manifest
.get_explicit_objs();
1628 map
<uint64_t, RGWObjManifestPart
>::iterator oiter
;
1629 for (oiter
= objs
.begin(); oiter
!= objs
.end(); ++oiter
) {
1630 RGWObjManifestPart
& part
= oiter
->second
;
1632 if (part
.size
>= min_stripe_size
) {
1633 *need_rewrite
= true;
1637 *need_rewrite
= false;
1643 int check_obj_locator_underscore(rgw::sal::Object
* obj
, bool fix
, bool remove_bad
, Formatter
*f
) {
1644 f
->open_object_section("object");
1645 f
->open_object_section("key");
1646 f
->dump_string("type", "head");
1647 f
->dump_string("name", obj
->get_name());
1648 f
->dump_string("instance", obj
->get_instance());
1654 get_obj_bucket_and_oid_loc(obj
->get_obj(), oid
, locator
);
1656 f
->dump_string("oid", oid
);
1657 f
->dump_string("locator", locator
);
1659 std::unique_ptr
<rgw::sal::Object::ReadOp
> read_op
= obj
->get_read_op();
1661 int ret
= read_op
->prepare(null_yield
, dpp());
1662 bool needs_fixing
= (ret
== -ENOENT
);
1664 f
->dump_bool("needs_fixing", needs_fixing
);
1666 string status
= (needs_fixing
? "needs_fixing" : "ok");
1668 if ((needs_fixing
|| remove_bad
) && fix
) {
1669 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->fix_head_obj_locator(dpp(), obj
->get_bucket()->get_info(), needs_fixing
, remove_bad
, obj
->get_key());
1671 cerr
<< "ERROR: fix_head_object_locator() returned ret=" << ret
<< std::endl
;
1678 f
->dump_string("status", status
);
1685 int check_obj_tail_locator_underscore(RGWBucketInfo
& bucket_info
, rgw_obj_key
& key
, bool fix
, Formatter
*f
) {
1686 f
->open_object_section("object");
1687 f
->open_object_section("key");
1688 f
->dump_string("type", "tail");
1689 f
->dump_string("name", key
.name
);
1690 f
->dump_string("instance", key
.instance
);
1696 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->fix_tail_obj_locator(dpp(), bucket_info
, key
, fix
, &needs_fixing
, null_yield
);
1698 cerr
<< "ERROR: fix_tail_object_locator_underscore() returned ret=" << ret
<< std::endl
;
1701 status
= (needs_fixing
&& !fix
? "needs_fixing" : "ok");
1704 f
->dump_bool("needs_fixing", needs_fixing
);
1705 f
->dump_string("status", status
);
1712 int do_check_object_locator(const string
& tenant_name
, const string
& bucket_name
,
1713 bool fix
, bool remove_bad
, Formatter
*f
)
1715 if (remove_bad
&& !fix
) {
1716 cerr
<< "ERROR: can't have remove_bad specified without fix" << std::endl
;
1720 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
1723 f
->open_object_section("bucket");
1724 f
->dump_string("bucket", bucket_name
);
1725 int ret
= init_bucket(nullptr, tenant_name
, bucket_name
, bucket_id
, &bucket
);
1727 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
1732 int max_entries
= 1000;
1737 vector
<rgw_bucket_dir_entry
> result
;
1740 rgw::sal::Bucket::ListParams params
;
1741 rgw::sal::Bucket::ListResults results
;
1743 params
.prefix
= prefix
;
1744 params
.delim
= delim
;
1745 params
.marker
= rgw_obj_key(marker
);
1747 params
.enforce_ns
= true;
1748 params
.list_versions
= true;
1750 f
->open_array_section("check_objects");
1752 ret
= bucket
->list(dpp(), params
, max_entries
- count
, results
, null_yield
);
1754 cerr
<< "ERROR: driver->list_objects(): " << cpp_strerror(-ret
) << std::endl
;
1758 count
+= results
.objs
.size();
1760 for (vector
<rgw_bucket_dir_entry
>::iterator iter
= results
.objs
.begin(); iter
!= results
.objs
.end(); ++iter
) {
1761 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(iter
->key
);
1763 if (obj
->get_name()[0] == '_') {
1764 ret
= check_obj_locator_underscore(obj
.get(), fix
, remove_bad
, f
);
1767 ret
= check_obj_tail_locator_underscore(bucket
->get_info(), obj
->get_key(), fix
, f
);
1769 cerr
<< "ERROR: check_obj_tail_locator_underscore(): " << cpp_strerror(-ret
) << std::endl
;
1776 } while (results
.is_truncated
&& count
< max_entries
);
1785 /// search for a matching zone/zonegroup id and return a connection if found
1786 static boost::optional
<RGWRESTConn
> get_remote_conn(rgw::sal::RadosStore
* driver
,
1787 const RGWZoneGroup
& zonegroup
,
1788 const std::string
& remote
)
1790 boost::optional
<RGWRESTConn
> conn
;
1791 if (remote
== zonegroup
.get_id()) {
1792 conn
.emplace(driver
->ctx(), driver
, remote
, zonegroup
.endpoints
, zonegroup
.api_name
);
1794 for (const auto& z
: zonegroup
.zones
) {
1795 const auto& zone
= z
.second
;
1796 if (remote
== zone
.id
) {
1797 conn
.emplace(driver
->ctx(), driver
, remote
, zone
.endpoints
, zonegroup
.api_name
);
1805 /// search each zonegroup for a connection
1806 static boost::optional
<RGWRESTConn
> get_remote_conn(rgw::sal::RadosStore
* driver
,
1807 const RGWPeriodMap
& period_map
,
1808 const std::string
& remote
)
1810 boost::optional
<RGWRESTConn
> conn
;
1811 for (const auto& zg
: period_map
.zonegroups
) {
1812 conn
= get_remote_conn(driver
, zg
.second
, remote
);
1820 // we expect a very small response
1821 static constexpr size_t MAX_REST_RESPONSE
= 128 * 1024;
1823 static int send_to_remote_gateway(RGWRESTConn
* conn
, req_info
& info
,
1824 bufferlist
& in_data
, JSONParser
& parser
)
1830 ceph::bufferlist response
;
1832 int ret
= conn
->forward(dpp(), user
, info
, nullptr, MAX_REST_RESPONSE
, &in_data
, &response
, null_yield
);
1834 int parse_ret
= parser
.parse(response
.c_str(), response
.length());
1835 if (parse_ret
< 0) {
1836 cerr
<< "failed to parse response" << std::endl
;
1842 static int send_to_url(const string
& url
,
1843 std::optional
<string
> opt_region
,
1844 const string
& access
,
1845 const string
& secret
, req_info
& info
,
1846 bufferlist
& in_data
, JSONParser
& parser
)
1848 if (access
.empty() || secret
.empty()) {
1849 cerr
<< "An --access-key and --secret must be provided with --url." << std::endl
;
1857 RGWRESTSimpleRequest
req(g_ceph_context
, info
.method
, url
, NULL
, ¶ms
, opt_region
);
1859 bufferlist response
;
1860 int ret
= req
.forward_request(dpp(), key
, info
, MAX_REST_RESPONSE
, &in_data
, &response
, null_yield
);
1862 int parse_ret
= parser
.parse(response
.c_str(), response
.length());
1863 if (parse_ret
< 0) {
1864 cout
<< "failed to parse response" << std::endl
;
1870 static int send_to_remote_or_url(RGWRESTConn
*conn
, const string
& url
,
1871 std::optional
<string
> opt_region
,
1872 const string
& access
, const string
& secret
,
1873 req_info
& info
, bufferlist
& in_data
,
1877 return send_to_remote_gateway(conn
, info
, in_data
, parser
);
1879 return send_to_url(url
, opt_region
, access
, secret
, info
, in_data
, parser
);
1882 static int commit_period(rgw::sal::ConfigStore
* cfgstore
,
1883 RGWRealm
& realm
, rgw::sal::RealmWriter
& realm_writer
,
1884 RGWPeriod
& period
, string remote
, const string
& url
,
1885 std::optional
<string
> opt_region
,
1886 const string
& access
, const string
& secret
,
1889 auto& master_zone
= period
.get_master_zone().id
;
1890 if (master_zone
.empty()) {
1891 cerr
<< "cannot commit period: period does not have a master zone of a master zonegroup" << std::endl
;
1894 // are we the period's master zone?
1895 if (driver
->get_zone()->get_id() == master_zone
) {
1896 // read the current period
1897 RGWPeriod current_period
;
1898 int ret
= cfgstore
->read_period(dpp(), null_yield
, realm
.current_period
,
1899 std::nullopt
, current_period
);
1901 cerr
<< "failed to load current period: " << cpp_strerror(ret
) << std::endl
;
1904 // the master zone can commit locally
1905 ret
= rgw::commit_period(dpp(), null_yield
, cfgstore
, driver
,
1906 realm
, realm_writer
, current_period
,
1907 period
, cerr
, force
);
1909 cerr
<< "failed to commit period: " << cpp_strerror(-ret
) << std::endl
;
1914 if (remote
.empty() && url
.empty()) {
1915 // use the new master zone's connection
1916 remote
= master_zone
;
1917 cerr
<< "Sending period to new master zone " << remote
<< std::endl
;
1919 boost::optional
<RGWRESTConn
> conn
;
1920 RGWRESTConn
*remote_conn
= nullptr;
1921 if (!remote
.empty()) {
1922 conn
= get_remote_conn(static_cast<rgw::sal::RadosStore
*>(driver
), period
.get_map(), remote
);
1924 cerr
<< "failed to find a zone or zonegroup for remote "
1925 << remote
<< std::endl
;
1928 remote_conn
= &*conn
;
1931 // push period to the master with an empty period id
1932 period
.set_id(string());
1935 req_info
info(g_ceph_context
, &env
);
1936 info
.method
= "POST";
1937 info
.request_uri
= "/admin/realm/period";
1939 // json format into a bufferlist
1940 JSONFormatter
jf(false);
1941 encode_json("period", period
, &jf
);
1946 int ret
= send_to_remote_or_url(remote_conn
, url
, opt_region
, access
, secret
, info
, bl
, p
);
1948 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
1950 // did we parse an error message?
1951 auto message
= p
.find_obj("Message");
1953 cerr
<< "Reason: " << message
->get_data() << std::endl
;
1958 // decode the response and driver it back
1960 decode_json_obj(period
, &p
);
1961 } catch (const JSONDecoder::err
& e
) {
1962 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
1965 if (period
.get_id().empty()) {
1966 cerr
<< "Period commit got back an empty period id" << std::endl
;
1969 // the master zone gave us back the period that it committed, so it's
1970 // safe to save it as our latest epoch
1971 constexpr bool exclusive
= false;
1972 ret
= cfgstore
->create_period(dpp(), null_yield
, exclusive
, period
);
1974 cerr
<< "Error storing committed period " << period
.get_id() << ": "
1975 << cpp_strerror(ret
) << std::endl
;
1978 ret
= rgw::reflect_period(dpp(), null_yield
, cfgstore
, period
);
1980 cerr
<< "Error updating local objects: " << cpp_strerror(ret
) << std::endl
;
1983 (void) cfgstore
->realm_notify_new_period(dpp(), null_yield
, period
);
1987 static int update_period(rgw::sal::ConfigStore
* cfgstore
,
1988 const string
& realm_id
, const string
& realm_name
,
1989 const string
& period_epoch
, bool commit
,
1990 const string
& remote
, const string
& url
,
1991 std::optional
<string
> opt_region
,
1992 const string
& access
, const string
& secret
,
1993 Formatter
*formatter
, bool force
)
1996 std::unique_ptr
<rgw::sal::RealmWriter
> realm_writer
;
1997 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
,
1998 realm_id
, realm_name
,
1999 realm
, &realm_writer
);
2001 cerr
<< "failed to load realm " << cpp_strerror(-ret
) << std::endl
;
2004 std::optional
<epoch_t
> epoch
;
2005 if (!period_epoch
.empty()) {
2006 epoch
= atoi(period_epoch
.c_str());
2009 ret
= cfgstore
->read_period(dpp(), null_yield
, realm
.current_period
,
2012 cerr
<< "failed to load current period: " << cpp_strerror(-ret
) << std::endl
;
2015 // convert to the realm's staging period
2016 rgw::fork_period(dpp(), period
);
2017 // update the staging period with all of the realm's zonegroups
2018 ret
= rgw::update_period(dpp(), null_yield
, cfgstore
, period
);
2023 constexpr bool exclusive
= false;
2024 ret
= cfgstore
->create_period(dpp(), null_yield
, exclusive
, period
);
2026 cerr
<< "failed to driver period: " << cpp_strerror(-ret
) << std::endl
;
2030 ret
= commit_period(cfgstore
, realm
, *realm_writer
, period
, remote
, url
,
2031 opt_region
, access
, secret
, force
);
2033 cerr
<< "failed to commit period: " << cpp_strerror(-ret
) << std::endl
;
2037 encode_json("period", period
, formatter
);
2038 formatter
->flush(cout
);
2042 static int init_bucket_for_sync(rgw::sal::User
* user
,
2043 const string
& tenant
, const string
& bucket_name
,
2044 const string
& bucket_id
,
2045 std::unique_ptr
<rgw::sal::Bucket
>* bucket
)
2047 int ret
= init_bucket(user
, tenant
, bucket_name
, bucket_id
, bucket
);
2049 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
2056 static int do_period_pull(rgw::sal::ConfigStore
* cfgstore
,
2057 RGWRESTConn
*remote_conn
, const string
& url
,
2058 std::optional
<string
> opt_region
,
2059 const string
& access_key
, const string
& secret_key
,
2060 const string
& realm_id
, const string
& realm_name
,
2061 const string
& period_id
, const string
& period_epoch
,
2065 req_info
info(g_ceph_context
, &env
);
2066 info
.method
= "GET";
2067 info
.request_uri
= "/admin/realm/period";
2069 map
<string
, string
> ¶ms
= info
.args
.get_params();
2070 if (!realm_id
.empty())
2071 params
["realm_id"] = realm_id
;
2072 if (!realm_name
.empty())
2073 params
["realm_name"] = realm_name
;
2074 if (!period_id
.empty())
2075 params
["period_id"] = period_id
;
2076 if (!period_epoch
.empty())
2077 params
["epoch"] = period_epoch
;
2081 int ret
= send_to_remote_or_url(remote_conn
, url
, opt_region
, access_key
, secret_key
,
2084 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
2088 decode_json_obj(*period
, &p
);
2089 } catch (const JSONDecoder::err
& e
) {
2090 cout
<< "failed to decode JSON input: " << e
.what() << std::endl
;
2093 constexpr bool exclusive
= false;
2094 ret
= cfgstore
->create_period(dpp(), null_yield
, exclusive
, *period
);
2096 cerr
<< "Error storing period " << period
->get_id() << ": " << cpp_strerror(ret
) << std::endl
;
2101 void flush_ss(stringstream
& ss
, list
<string
>& l
)
2103 if (!ss
.str().empty()) {
2104 l
.push_back(ss
.str());
2109 stringstream
& push_ss(stringstream
& ss
, list
<string
>& l
, int tab
= 0)
2113 ss
<< setw(tab
) << "" << setw(1);
2118 static void get_md_sync_status(list
<string
>& status
)
2120 RGWMetaSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor());
2122 int ret
= sync
.init(dpp());
2124 status
.push_back(string("failed to retrieve sync info: sync.init() failed: ") + cpp_strerror(-ret
));
2128 rgw_meta_sync_status sync_status
;
2129 ret
= sync
.read_sync_status(dpp(), &sync_status
);
2131 status
.push_back(string("failed to read sync status: ") + cpp_strerror(-ret
));
2136 switch (sync_status
.sync_info
.state
) {
2137 case rgw_meta_sync_info::StateInit
:
2138 status_str
= "init";
2140 case rgw_meta_sync_info::StateBuildingFullSyncMaps
:
2141 status_str
= "preparing for full sync";
2143 case rgw_meta_sync_info::StateSync
:
2144 status_str
= "syncing";
2147 status_str
= "unknown";
2150 status
.push_back(status_str
);
2152 uint64_t full_total
= 0;
2153 uint64_t full_complete
= 0;
2157 int total_shards
= 0;
2158 set
<int> shards_behind_set
;
2160 for (auto marker_iter
: sync_status
.sync_markers
) {
2161 full_total
+= marker_iter
.second
.total_entries
;
2163 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::FullSync
) {
2165 full_complete
+= marker_iter
.second
.pos
;
2166 int shard_id
= marker_iter
.first
;
2167 shards_behind_set
.insert(shard_id
);
2169 full_complete
+= marker_iter
.second
.total_entries
;
2171 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::IncrementalSync
) {
2177 push_ss(ss
, status
) << "full sync: " << num_full
<< "/" << total_shards
<< " shards";
2180 push_ss(ss
, status
) << "full sync: " << full_total
- full_complete
<< " entries to sync";
2183 push_ss(ss
, status
) << "incremental sync: " << num_inc
<< "/" << total_shards
<< " shards";
2185 map
<int, RGWMetadataLogInfo
> master_shards_info
;
2186 string master_period
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_current_period_id();
2188 ret
= sync
.read_master_log_shards_info(dpp(), master_period
, &master_shards_info
);
2190 status
.push_back(string("failed to fetch master sync status: ") + cpp_strerror(-ret
));
2194 map
<int, string
> shards_behind
;
2195 if (sync_status
.sync_info
.period
!= master_period
) {
2196 status
.push_back(string("master is on a different period: master_period=" +
2197 master_period
+ " local_period=" + sync_status
.sync_info
.period
));
2199 for (auto local_iter
: sync_status
.sync_markers
) {
2200 int shard_id
= local_iter
.first
;
2201 auto iter
= master_shards_info
.find(shard_id
);
2203 if (iter
== master_shards_info
.end()) {
2205 derr
<< "ERROR: could not find remote sync shard status for shard_id=" << shard_id
<< dendl
;
2208 auto master_marker
= iter
->second
.marker
;
2209 if (local_iter
.second
.state
== rgw_meta_sync_marker::SyncState::IncrementalSync
&&
2210 master_marker
> local_iter
.second
.marker
) {
2211 shards_behind
[shard_id
] = local_iter
.second
.marker
;
2212 shards_behind_set
.insert(shard_id
);
2217 // fetch remote log entries to determine the oldest change
2218 std::optional
<std::pair
<int, ceph::real_time
>> oldest
;
2219 if (!shards_behind
.empty()) {
2220 map
<int, rgw_mdlog_shard_data
> master_pos
;
2221 ret
= sync
.read_master_log_shards_next(dpp(), sync_status
.sync_info
.period
, shards_behind
, &master_pos
);
2223 derr
<< "ERROR: failed to fetch master next positions (" << cpp_strerror(-ret
) << ")" << dendl
;
2225 for (auto iter
: master_pos
) {
2226 rgw_mdlog_shard_data
& shard_data
= iter
.second
;
2228 if (shard_data
.entries
.empty()) {
2229 // there aren't any entries in this shard, so we're not really behind
2230 shards_behind
.erase(iter
.first
);
2231 shards_behind_set
.erase(iter
.first
);
2233 rgw_mdlog_entry
& entry
= shard_data
.entries
.front();
2235 oldest
.emplace(iter
.first
, entry
.timestamp
);
2236 } else if (!ceph::real_clock::is_zero(entry
.timestamp
) && entry
.timestamp
< oldest
->second
) {
2237 oldest
.emplace(iter
.first
, entry
.timestamp
);
2244 int total_behind
= shards_behind
.size() + (sync_status
.sync_info
.num_shards
- num_inc
);
2245 if (total_behind
== 0) {
2246 push_ss(ss
, status
) << "metadata is caught up with master";
2248 push_ss(ss
, status
) << "metadata is behind on " << total_behind
<< " shards";
2249 push_ss(ss
, status
) << "behind shards: " << "[" << shards_behind_set
<< "]";
2251 push_ss(ss
, status
) << "oldest incremental change not applied: "
2252 << oldest
->second
<< " [" << oldest
->first
<< ']';
2256 flush_ss(ss
, status
);
2259 static void get_data_sync_status(const rgw_zone_id
& source_zone
, list
<string
>& status
, int tab
)
2265 if (!(sz
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->find_zone(source_zone
))) {
2266 push_ss(ss
, status
, tab
) << string("zone not found");
2267 flush_ss(ss
, status
);
2271 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->zone_syncs_from(static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zone(), *sz
)) {
2272 push_ss(ss
, status
, tab
) << string("not syncing from zone");
2273 flush_ss(ss
, status
);
2276 RGWDataSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor(), source_zone
, nullptr);
2278 int ret
= sync
.init(dpp());
2280 push_ss(ss
, status
, tab
) << string("failed to retrieve sync info: ") + cpp_strerror(-ret
);
2281 flush_ss(ss
, status
);
2285 rgw_data_sync_status sync_status
;
2286 ret
= sync
.read_sync_status(dpp(), &sync_status
);
2287 if (ret
< 0 && ret
!= -ENOENT
) {
2288 push_ss(ss
, status
, tab
) << string("failed read sync status: ") + cpp_strerror(-ret
);
2292 set
<int> recovering_shards
;
2293 ret
= sync
.read_recovering_shards(dpp(), sync_status
.sync_info
.num_shards
, recovering_shards
);
2294 if (ret
< 0 && ret
!= ENOENT
) {
2295 push_ss(ss
, status
, tab
) << string("failed read recovering shards: ") + cpp_strerror(-ret
);
2300 switch (sync_status
.sync_info
.state
) {
2301 case rgw_data_sync_info::StateInit
:
2302 status_str
= "init";
2304 case rgw_data_sync_info::StateBuildingFullSyncMaps
:
2305 status_str
= "preparing for full sync";
2307 case rgw_data_sync_info::StateSync
:
2308 status_str
= "syncing";
2311 status_str
= "unknown";
2314 push_ss(ss
, status
, tab
) << status_str
;
2316 uint64_t full_total
= 0;
2317 uint64_t full_complete
= 0;
2321 int total_shards
= 0;
2322 set
<int> shards_behind_set
;
2324 for (auto marker_iter
: sync_status
.sync_markers
) {
2325 full_total
+= marker_iter
.second
.total_entries
;
2327 if (marker_iter
.second
.state
== rgw_data_sync_marker::SyncState::FullSync
) {
2329 full_complete
+= marker_iter
.second
.pos
;
2330 int shard_id
= marker_iter
.first
;
2331 shards_behind_set
.insert(shard_id
);
2333 full_complete
+= marker_iter
.second
.total_entries
;
2335 if (marker_iter
.second
.state
== rgw_data_sync_marker::SyncState::IncrementalSync
) {
2340 push_ss(ss
, status
, tab
) << "full sync: " << num_full
<< "/" << total_shards
<< " shards";
2343 push_ss(ss
, status
, tab
) << "full sync: " << full_total
- full_complete
<< " buckets to sync";
2346 push_ss(ss
, status
, tab
) << "incremental sync: " << num_inc
<< "/" << total_shards
<< " shards";
2348 map
<int, RGWDataChangesLogInfo
> source_shards_info
;
2350 ret
= sync
.read_source_log_shards_info(dpp(), &source_shards_info
);
2352 push_ss(ss
, status
, tab
) << string("failed to fetch source sync status: ") + cpp_strerror(-ret
);
2356 map
<int, string
> shards_behind
;
2358 for (auto local_iter
: sync_status
.sync_markers
) {
2359 int shard_id
= local_iter
.first
;
2360 auto iter
= source_shards_info
.find(shard_id
);
2362 if (iter
== source_shards_info
.end()) {
2364 derr
<< "ERROR: could not find remote sync shard status for shard_id=" << shard_id
<< dendl
;
2367 auto master_marker
= iter
->second
.marker
;
2368 if (local_iter
.second
.state
== rgw_data_sync_marker::SyncState::IncrementalSync
&&
2369 master_marker
> local_iter
.second
.marker
) {
2370 shards_behind
[shard_id
] = local_iter
.second
.marker
;
2371 shards_behind_set
.insert(shard_id
);
2375 std::optional
<std::pair
<int, ceph::real_time
>> oldest
;
2376 if (!shards_behind
.empty()) {
2377 map
<int, rgw_datalog_shard_data
> master_pos
;
2378 ret
= sync
.read_source_log_shards_next(dpp(), shards_behind
, &master_pos
);
2381 derr
<< "ERROR: failed to fetch next positions (" << cpp_strerror(-ret
) << ")" << dendl
;
2383 for (auto iter
: master_pos
) {
2384 rgw_datalog_shard_data
& shard_data
= iter
.second
;
2385 if (shard_data
.entries
.empty()) {
2386 // there aren't any entries in this shard, so we're not really behind
2387 shards_behind
.erase(iter
.first
);
2388 shards_behind_set
.erase(iter
.first
);
2390 rgw_datalog_entry
& entry
= shard_data
.entries
.front();
2392 oldest
.emplace(iter
.first
, entry
.timestamp
);
2393 } else if (!ceph::real_clock::is_zero(entry
.timestamp
) && entry
.timestamp
< oldest
->second
) {
2394 oldest
.emplace(iter
.first
, entry
.timestamp
);
2401 int total_behind
= shards_behind
.size() + (sync_status
.sync_info
.num_shards
- num_inc
);
2402 int total_recovering
= recovering_shards
.size();
2404 if (total_behind
== 0 && total_recovering
== 0) {
2405 push_ss(ss
, status
, tab
) << "data is caught up with source";
2406 } else if (total_behind
> 0) {
2407 push_ss(ss
, status
, tab
) << "data is behind on " << total_behind
<< " shards";
2408 push_ss(ss
, status
, tab
) << "behind shards: " << "[" << shards_behind_set
<< "]" ;
2410 push_ss(ss
, status
, tab
) << "oldest incremental change not applied: "
2411 << oldest
->second
<< " [" << oldest
->first
<< ']';
2415 if (total_recovering
> 0) {
2416 push_ss(ss
, status
, tab
) << total_recovering
<< " shards are recovering";
2417 push_ss(ss
, status
, tab
) << "recovering shards: " << "[" << recovering_shards
<< "]";
2420 flush_ss(ss
, status
);
2423 static void tab_dump(const string
& header
, int width
, const list
<string
>& entries
)
2427 for (auto e
: entries
) {
2428 cout
<< std::setw(width
) << s
<< std::setw(1) << " " << e
<< std::endl
;
2433 // return features that are supported but not enabled
2434 static auto get_disabled_features(const rgw::zone_features::set
& enabled
) {
2435 auto features
= rgw::zone_features::set
{rgw::zone_features::supported
.begin(),
2436 rgw::zone_features::supported
.end()};
2437 for (const auto& feature
: enabled
) {
2438 features
.erase(feature
);
2444 static void sync_status(Formatter
*formatter
)
2446 const rgw::sal::ZoneGroup
& zonegroup
= driver
->get_zone()->get_zonegroup();
2447 rgw::sal::Zone
* zone
= driver
->get_zone();
2451 cout
<< std::setw(width
) << "realm" << std::setw(1) << " " << zone
->get_realm_id() << " (" << zone
->get_realm_name() << ")" << std::endl
;
2452 cout
<< std::setw(width
) << "zonegroup" << std::setw(1) << " " << zonegroup
.get_id() << " (" << zonegroup
.get_name() << ")" << std::endl
;
2453 cout
<< std::setw(width
) << "zone" << std::setw(1) << " " << zone
->get_id() << " (" << zone
->get_name() << ")" << std::endl
;
2454 cout
<< std::setw(width
) << "current time" << std::setw(1) << " "
2455 << to_iso_8601(ceph::real_clock::now(), iso_8601_format::YMDhms
) << std::endl
;
2458 static_cast<const rgw::sal::RadosZoneGroup
&>(zonegroup
).get_group();
2460 cout
<< std::setw(width
) << "zonegroup features enabled: " << rzg
.enabled_features
<< std::endl
;
2461 if (auto d
= get_disabled_features(rzg
.enabled_features
); !d
.empty()) {
2462 cout
<< std::setw(width
) << " disabled: " << d
<< std::endl
;
2465 list
<string
> md_status
;
2467 if (driver
->is_meta_master()) {
2468 md_status
.push_back("no sync (zone is master)");
2470 get_md_sync_status(md_status
);
2473 tab_dump("metadata sync", width
, md_status
);
2475 list
<string
> data_status
;
2477 auto& zone_conn_map
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zone_conn_map();
2479 for (auto iter
: zone_conn_map
) {
2480 const rgw_zone_id
& source_id
= iter
.first
;
2481 string source_str
= "source: ";
2482 string s
= source_str
+ source_id
.id
;
2483 std::unique_ptr
<rgw::sal::Zone
> sz
;
2484 if (driver
->get_zone()->get_zonegroup().get_zone_by_id(source_id
.id
, &sz
) == 0) {
2485 s
+= string(" (") + sz
->get_name() + ")";
2487 data_status
.push_back(s
);
2488 get_data_sync_status(source_id
, data_status
, source_str
.size());
2491 tab_dump("data sync", width
, data_status
);
2495 int w
; // indent width
2496 std::string_view header
;
2497 indented(int w
, std::string_view header
= "") : w(w
), header(header
) {}
2499 std::ostream
& operator<<(std::ostream
& out
, const indented
& h
) {
2500 return out
<< std::setw(h
.w
) << h
.header
<< std::setw(1) << ' ';
2503 static int bucket_source_sync_status(const DoutPrefixProvider
*dpp
, rgw::sal::RadosStore
* driver
, const RGWZone
& zone
,
2504 const RGWZone
& source
, RGWRESTConn
*conn
,
2505 const RGWBucketInfo
& bucket_info
,
2506 rgw_sync_bucket_pipe pipe
,
2507 int width
, std::ostream
& out
)
2509 out
<< indented
{width
, "source zone"} << source
.id
<< " (" << source
.name
<< ")" << std::endl
;
2511 // syncing from this zone?
2512 if (!driver
->svc()->zone
->zone_syncs_from(zone
, source
)) {
2513 out
<< indented
{width
} << "does not sync from zone\n";
2517 if (!pipe
.source
.bucket
) {
2518 ldpp_dout(dpp
, -1) << __func__
<< "(): missing source bucket" << dendl
;
2522 std::unique_ptr
<rgw::sal::Bucket
> source_bucket
;
2523 int r
= init_bucket(nullptr, *pipe
.source
.bucket
, &source_bucket
);
2525 ldpp_dout(dpp
, -1) << "failed to read source bucket info: " << cpp_strerror(r
) << dendl
;
2529 out
<< indented
{width
, "source bucket"} << source_bucket
->get_key() << std::endl
;
2530 pipe
.source
.bucket
= source_bucket
->get_key();
2532 pipe
.dest
.bucket
= bucket_info
.bucket
;
2535 std::vector
<rgw_bucket_shard_sync_info
> shard_status
;
2537 // check for full sync status
2538 rgw_bucket_sync_status full_status
;
2539 r
= rgw_read_bucket_full_sync_status(dpp
, driver
, pipe
, &full_status
, null_yield
);
2541 if (full_status
.state
== BucketSyncState::Init
) {
2542 out
<< indented
{width
} << "init: bucket sync has not started\n";
2545 if (full_status
.state
== BucketSyncState::Stopped
) {
2546 out
<< indented
{width
} << "stopped: bucket sync is disabled\n";
2549 if (full_status
.state
== BucketSyncState::Full
) {
2550 out
<< indented
{width
} << "full sync: " << full_status
.full
.count
<< " objects completed\n";
2553 gen
= full_status
.incremental_gen
;
2554 shard_status
.resize(full_status
.shards_done_with_gen
.size());
2555 } else if (r
== -ENOENT
) {
2556 // no full status, but there may be per-shard status from before upgrade
2557 const auto& logs
= source_bucket
->get_info().layout
.logs
;
2559 out
<< indented
{width
} << "init: bucket sync has not started\n";
2562 const auto& log
= logs
.front();
2564 // this isn't the backward-compatible case, so we just haven't started yet
2565 out
<< indented
{width
} << "init: bucket sync has not started\n";
2568 if (log
.layout
.type
!= rgw::BucketLogType::InIndex
) {
2569 ldpp_dout(dpp
, -1) << "unrecognized log layout type " << log
.layout
.type
<< dendl
;
2572 // use shard count from our log gen=0
2573 shard_status
.resize(rgw::num_shards(log
.layout
.in_index
));
2575 lderr(driver
->ctx()) << "failed to read bucket full sync status: " << cpp_strerror(r
) << dendl
;
2579 r
= rgw_read_bucket_inc_sync_status(dpp
, driver
, pipe
, gen
, &shard_status
);
2581 lderr(driver
->ctx()) << "failed to read bucket incremental sync status: " << cpp_strerror(r
) << dendl
;
2585 const int total_shards
= shard_status
.size();
2587 out
<< indented
{width
} << "incremental sync on " << total_shards
<< " shards\n";
2589 rgw_bucket_index_marker_info remote_info
;
2590 BucketIndexShardsManager remote_markers
;
2591 r
= rgw_read_remote_bilog_info(dpp
, conn
, source_bucket
->get_key(),
2592 remote_info
, remote_markers
, null_yield
);
2594 ldpp_dout(dpp
, -1) << "failed to read remote log: " << cpp_strerror(r
) << dendl
;
2598 std::set
<int> shards_behind
;
2599 for (const auto& r
: remote_markers
.get()) {
2600 auto shard_id
= r
.first
;
2601 if (r
.second
.empty()) {
2602 continue; // empty bucket index shard
2604 if (shard_id
>= total_shards
) {
2605 // unexpected shard id. we don't have status for it, so we're behind
2606 shards_behind
.insert(shard_id
);
2609 auto& m
= shard_status
[shard_id
];
2610 const auto pos
= BucketIndexShardsManager::get_shard_marker(m
.inc_marker
.position
);
2611 if (pos
< r
.second
) {
2612 shards_behind
.insert(shard_id
);
2615 if (!shards_behind
.empty()) {
2616 out
<< indented
{width
} << "bucket is behind on " << shards_behind
.size() << " shards\n";
2617 out
<< indented
{width
} << "behind shards: [" << shards_behind
<< "]\n" ;
2619 out
<< indented
{width
} << "bucket is caught up with source\n";
2624 void encode_json(const char *name
, const RGWBucketSyncFlowManager::pipe_set
& pset
, Formatter
*f
)
2626 Formatter::ObjectSection
top_section(*f
, name
);
2627 Formatter::ArraySection
as(*f
, "entries");
2629 for (auto& pipe_handler
: pset
) {
2630 Formatter::ObjectSection
hs(*f
, "handler");
2631 encode_json("source", pipe_handler
.source
, f
);
2632 encode_json("dest", pipe_handler
.dest
, f
);
2636 static std::vector
<string
> convert_bucket_set_to_str_vec(const std::set
<rgw_bucket
>& bs
)
2638 std::vector
<string
> result
;
2639 result
.reserve(bs
.size());
2640 for (auto& b
: bs
) {
2641 result
.push_back(b
.get_key());
2646 static void get_hint_entities(const std::set
<rgw_zone_id
>& zones
, const std::set
<rgw_bucket
>& buckets
,
2647 std::set
<rgw_sync_bucket_entity
> *hint_entities
)
2649 for (auto& zone_id
: zones
) {
2650 for (auto& b
: buckets
) {
2651 std::unique_ptr
<rgw::sal::Bucket
> hint_bucket
;
2652 int ret
= init_bucket(nullptr, b
, &hint_bucket
);
2654 ldpp_dout(dpp(), 20) << "could not init bucket info for hint bucket=" << b
<< " ... skipping" << dendl
;
2658 hint_entities
->insert(rgw_sync_bucket_entity(zone_id
, hint_bucket
->get_key()));
2663 static rgw_zone_id
resolve_zone_id(const string
& s
)
2665 std::unique_ptr
<rgw::sal::Zone
> zone
;
2666 int ret
= driver
->get_zone()->get_zonegroup().get_zone_by_id(s
, &zone
);
2668 ret
= driver
->get_zone()->get_zonegroup().get_zone_by_name(s
, &zone
);
2670 return rgw_zone_id(s
);
2672 return rgw_zone_id(zone
->get_id());
2675 rgw_zone_id
validate_zone_id(const rgw_zone_id
& zone_id
)
2677 return resolve_zone_id(zone_id
.id
);
2680 static int sync_info(std::optional
<rgw_zone_id
> opt_target_zone
, std::optional
<rgw_bucket
> opt_bucket
, Formatter
*formatter
)
2682 rgw_zone_id zone_id
= opt_target_zone
.value_or(driver
->get_zone()->get_id());
2684 auto zone_policy_handler
= driver
->get_zone()->get_sync_policy_handler();
2686 RGWBucketSyncPolicyHandlerRef bucket_handler
;
2688 std::optional
<rgw_bucket
> eff_bucket
= opt_bucket
;
2690 auto handler
= zone_policy_handler
;
2693 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
2695 int ret
= init_bucket(nullptr, *eff_bucket
, &bucket
);
2696 if (ret
< 0 && ret
!= -ENOENT
) {
2697 cerr
<< "ERROR: init_bucket failed: " << cpp_strerror(-ret
) << std::endl
;
2702 rgw::sal::Attrs attrs
= bucket
->get_attrs();
2703 bucket_handler
.reset(handler
->alloc_child(bucket
->get_info(), std::move(attrs
)));
2705 cerr
<< "WARNING: bucket not found, simulating result" << std::endl
;
2706 bucket_handler
.reset(handler
->alloc_child(*eff_bucket
, nullopt
));
2709 ret
= bucket_handler
->init(dpp(), null_yield
);
2711 cerr
<< "ERROR: failed to init bucket sync policy handler: " << cpp_strerror(-ret
) << " (ret=" << ret
<< ")" << std::endl
;
2715 handler
= bucket_handler
;
2718 std::set
<rgw_sync_bucket_pipe
> sources
;
2719 std::set
<rgw_sync_bucket_pipe
> dests
;
2721 handler
->get_pipes(&sources
, &dests
, std::nullopt
);
2723 auto source_hints_vec
= convert_bucket_set_to_str_vec(handler
->get_source_hints());
2724 auto target_hints_vec
= convert_bucket_set_to_str_vec(handler
->get_target_hints());
2726 std::set
<rgw_sync_bucket_pipe
> resolved_sources
;
2727 std::set
<rgw_sync_bucket_pipe
> resolved_dests
;
2729 rgw_sync_bucket_entity
self_entity(zone_id
, opt_bucket
);
2731 set
<rgw_zone_id
> source_zones
;
2732 set
<rgw_zone_id
> target_zones
;
2734 zone_policy_handler
->reflect(dpp(), nullptr, nullptr,
2738 false); /* relaxed: also get all zones that we allow to sync to/from */
2740 std::set
<rgw_sync_bucket_entity
> hint_entities
;
2742 get_hint_entities(source_zones
, handler
->get_source_hints(), &hint_entities
);
2743 get_hint_entities(target_zones
, handler
->get_target_hints(), &hint_entities
);
2745 for (auto& hint_entity
: hint_entities
) {
2746 if (!hint_entity
.zone
||
2747 !hint_entity
.bucket
) {
2748 continue; /* shouldn't really happen */
2751 auto zid
= validate_zone_id(*hint_entity
.zone
);
2752 auto& hint_bucket
= *hint_entity
.bucket
;
2754 RGWBucketSyncPolicyHandlerRef hint_bucket_handler
;
2755 int r
= driver
->get_sync_policy_handler(dpp(), zid
, hint_bucket
, &hint_bucket_handler
, null_yield
);
2757 ldpp_dout(dpp(), 20) << "could not get bucket sync policy handler for hint bucket=" << hint_bucket
<< " ... skipping" << dendl
;
2761 hint_bucket_handler
->get_pipes(&resolved_dests
,
2763 self_entity
); /* flipping resolved dests and sources as these are
2764 relative to the remote entity */
2768 Formatter::ObjectSection
os(*formatter
, "result");
2769 encode_json("sources", sources
, formatter
);
2770 encode_json("dests", dests
, formatter
);
2772 Formatter::ObjectSection
hints_section(*formatter
, "hints");
2773 encode_json("sources", source_hints_vec
, formatter
);
2774 encode_json("dests", target_hints_vec
, formatter
);
2777 Formatter::ObjectSection
resolved_hints_section(*formatter
, "resolved-hints-1");
2778 encode_json("sources", resolved_sources
, formatter
);
2779 encode_json("dests", resolved_dests
, formatter
);
2782 Formatter::ObjectSection
resolved_hints_section(*formatter
, "resolved-hints");
2783 encode_json("sources", handler
->get_resolved_source_hints(), formatter
);
2784 encode_json("dests", handler
->get_resolved_dest_hints(), formatter
);
2788 formatter
->flush(cout
);
2793 static int bucket_sync_info(rgw::sal::Driver
* driver
, const RGWBucketInfo
& info
,
2796 const rgw::sal::ZoneGroup
& zonegroup
= driver
->get_zone()->get_zonegroup();
2797 rgw::sal::Zone
* zone
= driver
->get_zone();
2798 constexpr int width
= 15;
2800 out
<< indented
{width
, "realm"} << zone
->get_realm_id() << " (" << zone
->get_realm_name() << ")\n";
2801 out
<< indented
{width
, "zonegroup"} << zonegroup
.get_id() << " (" << zonegroup
.get_name() << ")\n";
2802 out
<< indented
{width
, "zone"} << zone
->get_id() << " (" << zone
->get_name() << ")\n";
2803 out
<< indented
{width
, "bucket"} << info
.bucket
<< "\n\n";
2805 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->bucket
->bucket_imports_data(info
.bucket
, null_yield
, dpp())) {
2806 out
<< "Sync is disabled for bucket " << info
.bucket
.name
<< '\n';
2810 RGWBucketSyncPolicyHandlerRef handler
;
2812 int r
= driver
->get_sync_policy_handler(dpp(), std::nullopt
, info
.bucket
, &handler
, null_yield
);
2814 ldpp_dout(dpp(), -1) << "ERROR: failed to get policy handler for bucket (" << info
.bucket
<< "): r=" << r
<< ": " << cpp_strerror(-r
) << dendl
;
2818 auto& sources
= handler
->get_sources();
2820 for (auto& m
: sources
) {
2821 auto& zone
= m
.first
;
2822 out
<< indented
{width
, "source zone"} << zone
<< std::endl
;
2823 for (auto& pipe_handler
: m
.second
) {
2824 out
<< indented
{width
, "bucket"} << *pipe_handler
.source
.bucket
<< std::endl
;
2831 static int bucket_sync_status(rgw::sal::Driver
* driver
, const RGWBucketInfo
& info
,
2832 const rgw_zone_id
& source_zone_id
,
2833 std::optional
<rgw_bucket
>& opt_source_bucket
,
2836 const rgw::sal::ZoneGroup
& zonegroup
= driver
->get_zone()->get_zonegroup();
2837 rgw::sal::Zone
* zone
= driver
->get_zone();
2838 constexpr int width
= 15;
2840 out
<< indented
{width
, "realm"} << zone
->get_realm_id() << " (" << zone
->get_realm_name() << ")\n";
2841 out
<< indented
{width
, "zonegroup"} << zonegroup
.get_id() << " (" << zonegroup
.get_name() << ")\n";
2842 out
<< indented
{width
, "zone"} << zone
->get_id() << " (" << zone
->get_name() << ")\n";
2843 out
<< indented
{width
, "bucket"} << info
.bucket
<< "\n";
2844 out
<< indented
{width
, "current time"}
2845 << to_iso_8601(ceph::real_clock::now(), iso_8601_format::YMDhms
) << "\n\n";
2848 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->bucket
->bucket_imports_data(info
.bucket
, null_yield
, dpp())) {
2849 out
<< "Sync is disabled for bucket " << info
.bucket
.name
<< " or bucket has no sync sources" << std::endl
;
2853 RGWBucketSyncPolicyHandlerRef handler
;
2855 int r
= driver
->get_sync_policy_handler(dpp(), std::nullopt
, info
.bucket
, &handler
, null_yield
);
2857 ldpp_dout(dpp(), -1) << "ERROR: failed to get policy handler for bucket (" << info
.bucket
<< "): r=" << r
<< ": " << cpp_strerror(-r
) << dendl
;
2861 auto sources
= handler
->get_all_sources();
2863 auto& zone_conn_map
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zone_conn_map();
2864 set
<rgw_zone_id
> zone_ids
;
2866 if (!source_zone_id
.empty()) {
2867 std::unique_ptr
<rgw::sal::Zone
> zone
;
2868 int ret
= driver
->get_zone()->get_zonegroup().get_zone_by_id(source_zone_id
.id
, &zone
);
2870 ldpp_dout(dpp(), -1) << "Source zone not found in zonegroup "
2871 << zonegroup
.get_name() << dendl
;
2874 auto c
= zone_conn_map
.find(source_zone_id
);
2875 if (c
== zone_conn_map
.end()) {
2876 ldpp_dout(dpp(), -1) << "No connection to zone " << zone
->get_name() << dendl
;
2879 zone_ids
.insert(source_zone_id
);
2881 std::list
<std::string
> ids
;
2882 int ret
= driver
->get_zone()->get_zonegroup().list_zones(ids
);
2884 for (const auto& entry
: ids
) {
2885 zone_ids
.insert(entry
);
2890 for (auto& zone_id
: zone_ids
) {
2891 auto z
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zonegroup().zones
.find(zone_id
.id
);
2892 if (z
== static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zonegroup().zones
.end()) { /* should't happen */
2895 auto c
= zone_conn_map
.find(zone_id
.id
);
2896 if (c
== zone_conn_map
.end()) { /* should't happen */
2900 for (auto& entry
: sources
) {
2901 auto& pipe
= entry
.second
;
2902 if (opt_source_bucket
&&
2903 pipe
.source
.bucket
!= opt_source_bucket
) {
2906 if (pipe
.source
.zone
.value_or(rgw_zone_id()) == z
->second
.id
) {
2907 bucket_source_sync_status(dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zone(), z
->second
,
2918 static void parse_tier_config_param(const string
& s
, map
<string
, string
, ltstr_nocase
>& out
)
2926 confs
.push_back(cur_conf
);
2933 } else if (c
== '}') {
2938 if (!cur_conf
.empty()) {
2939 confs
.push_back(cur_conf
);
2942 for (auto c
: confs
) {
2943 ssize_t pos
= c
.find("=");
2947 out
[c
.substr(0, pos
)] = c
.substr(pos
+ 1);
2952 static int check_pool_support_omap(const rgw_pool
& pool
)
2954 librados::IoCtx io_ctx
;
2955 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->get_rados_handle()->ioctx_create(pool
.to_str().c_str(), io_ctx
);
2957 // the pool may not exist at this moment, we have no way to check if it supports omap.
2961 ret
= io_ctx
.omap_clear("__omap_test_not_exist_oid__");
2962 if (ret
== -EOPNOTSUPP
) {
2970 int check_reshard_bucket_params(rgw::sal::Driver
* driver
,
2971 const string
& bucket_name
,
2972 const string
& tenant
,
2973 const string
& bucket_id
,
2974 bool num_shards_specified
,
2976 int yes_i_really_mean_it
,
2977 std::unique_ptr
<rgw::sal::Bucket
>* bucket
)
2979 if (bucket_name
.empty()) {
2980 cerr
<< "ERROR: bucket not specified" << std::endl
;
2984 if (!num_shards_specified
) {
2985 cerr
<< "ERROR: --num-shards not specified" << std::endl
;
2989 if (num_shards
> (int)static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->get_max_bucket_shards()) {
2990 cerr
<< "ERROR: num_shards too high, max value: " << static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->get_max_bucket_shards() << std::endl
;
2994 if (num_shards
< 0) {
2995 cerr
<< "ERROR: num_shards must be non-negative integer" << std::endl
;
2999 int ret
= init_bucket(nullptr, tenant
, bucket_name
, bucket_id
, bucket
);
3001 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
3005 int num_source_shards
= rgw::current_num_shards((*bucket
)->get_info().layout
);
3007 if (num_shards
<= num_source_shards
&& !yes_i_really_mean_it
) {
3008 cerr
<< "num shards is less or equal to current shards count" << std::endl
3009 << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
3015 static int scan_totp(CephContext
*cct
, ceph::real_time
& now
, rados::cls::otp::otp_info_t
& totp
, vector
<string
>& pins
,
3018 #define MAX_TOTP_SKEW_HOURS (24 * 7)
3019 time_t start_time
= ceph::real_clock::to_time_t(now
);
3020 time_t time_ofs
= 0, time_ofs_abs
= 0;
3021 time_t step_size
= totp
.step_size
;
3022 if (step_size
== 0) {
3023 step_size
= OATH_TOTP_DEFAULT_TIME_STEP_SIZE
;
3028 uint32_t max_skew
= MAX_TOTP_SKEW_HOURS
* 3600;
3030 while (time_ofs_abs
< max_skew
) {
3031 int rc
= oath_totp_validate2(totp
.seed_bin
.c_str(), totp
.seed_bin
.length(),
3038 if (rc
!= OATH_INVALID_OTP
) {
3039 rc
= oath_totp_validate2(totp
.seed_bin
.c_str(), totp
.seed_bin
.length(),
3042 time_ofs
- step_size
, /* smaller time_ofs moves time forward */
3046 if (rc
!= OATH_INVALID_OTP
) {
3047 *pofs
= time_ofs
- step_size
+ step_size
* totp
.window
/ 2;
3048 ldpp_dout(dpp(), 20) << "found at time=" << start_time
- time_ofs
<< " time_ofs=" << time_ofs
<< dendl
;
3053 time_ofs_abs
= (++count
) * step_size
;
3054 time_ofs
= sign
* time_ofs_abs
;
3060 static int trim_sync_error_log(int shard_id
, const string
& marker
, int delay_ms
)
3062 auto oid
= RGWSyncErrorLogger::get_shard_oid(RGW_SYNC_ERROR_LOG_SHARD_PREFIX
,
3064 // call cls_log_trim() until it returns -ENODATA
3066 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->timelog
.trim(dpp(), oid
, {}, {}, {}, marker
, nullptr,
3068 if (ret
== -ENODATA
) {
3075 std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms
));
3081 static bool symmetrical_flow_opt(const string
& opt
)
3083 return (opt
== "symmetrical" || opt
== "symmetric");
3086 static bool directional_flow_opt(const string
& opt
)
3088 return (opt
== "directional" || opt
== "direction");
3092 static bool require_opt(std::optional
<T
> opt
, bool extra_check
= true)
3094 if (!opt
|| !extra_check
) {
3101 static bool require_non_empty_opt(std::optional
<T
> opt
, bool extra_check
= true)
3103 if (!opt
|| opt
->empty() || !extra_check
) {
3110 static void show_result(T
& obj
,
3111 Formatter
*formatter
,
3114 encode_json("obj", obj
, formatter
);
3116 formatter
->flush(cout
);
3119 void init_optional_bucket(std::optional
<rgw_bucket
>& opt_bucket
,
3120 std::optional
<string
>& opt_tenant
,
3121 std::optional
<string
>& opt_bucket_name
,
3122 std::optional
<string
>& opt_bucket_id
)
3124 if (opt_tenant
|| opt_bucket_name
|| opt_bucket_id
) {
3125 opt_bucket
.emplace();
3127 opt_bucket
->tenant
= *opt_tenant
;
3129 if (opt_bucket_name
) {
3130 opt_bucket
->name
= *opt_bucket_name
;
3132 if (opt_bucket_id
) {
3133 opt_bucket
->bucket_id
= *opt_bucket_id
;
3138 class SyncPolicyContext
3140 rgw::sal::ConfigStore
* cfgstore
;
3141 RGWZoneGroup zonegroup
;
3142 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> zonegroup_writer
;
3144 std::optional
<rgw_bucket
> b
;
3145 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
3147 rgw_sync_policy_info
*policy
{nullptr};
3149 std::optional
<rgw_user
> owner
;
3152 SyncPolicyContext(rgw::sal::ConfigStore
* cfgstore
,
3153 std::optional
<rgw_bucket
> _bucket
)
3154 : cfgstore(cfgstore
), b(std::move(_bucket
)) {}
3156 int init(const string
& zonegroup_id
, const string
& zonegroup_name
) {
3157 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
,
3158 zonegroup_id
, zonegroup_name
,
3159 zonegroup
, &zonegroup_writer
);
3161 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
3166 policy
= &zonegroup
.sync_policy
;
3170 ret
= init_bucket(nullptr, *b
, &bucket
);
3172 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
3176 owner
= bucket
->get_info().owner
;
3178 if (!bucket
->get_info().sync_policy
) {
3179 rgw_sync_policy_info new_policy
;
3180 bucket
->get_info().set_sync_policy(std::move(new_policy
));
3183 policy
= &(*bucket
->get_info().sync_policy
);
3188 int write_policy() {
3190 int ret
= zonegroup_writer
->write(dpp(), null_yield
, zonegroup
);
3192 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
3198 int ret
= bucket
->put_info(dpp(), false, real_time());
3200 cerr
<< "failed to driver bucket info: " << cpp_strerror(-ret
) << std::endl
;
3207 rgw_sync_policy_info
& get_policy() {
3211 std::optional
<rgw_user
>& get_owner() {
3216 void resolve_zone_id_opt(std::optional
<string
>& zone_name
, std::optional
<rgw_zone_id
>& zone_id
)
3218 if (!zone_name
|| zone_id
) {
3222 std::unique_ptr
<rgw::sal::Zone
> zone
;
3223 int ret
= driver
->get_zone()->get_zonegroup().get_zone_by_name(*zone_name
, &zone
);
3225 cerr
<< "WARNING: cannot find source zone id for name=" << *zone_name
<< std::endl
;
3226 zone_id
= rgw_zone_id(*zone_name
);
3228 zone_id
->id
= zone
->get_id();
3231 void resolve_zone_ids_opt(std::optional
<vector
<string
> >& names
, std::optional
<vector
<rgw_zone_id
> >& ids
)
3233 if (!names
|| ids
) {
3237 for (auto& name
: *names
) {
3239 std::unique_ptr
<rgw::sal::Zone
> zone
;
3240 int ret
= driver
->get_zone()->get_zonegroup().get_zone_by_name(name
, &zone
);
3242 cerr
<< "WARNING: cannot find source zone id for name=" << name
<< std::endl
;
3243 zid
= rgw_zone_id(name
);
3245 zid
.id
= zone
->get_id();
3247 ids
->push_back(zid
);
3251 static vector
<rgw_zone_id
> zone_ids_from_str(const string
& val
)
3253 vector
<rgw_zone_id
> result
;
3255 get_str_vec(val
, v
);
3257 result
.push_back(rgw_zone_id(z
));
3262 class JSONFormatter_PrettyZone
: public JSONFormatter
{
3263 class Handler
: public JSONEncodeFilter::Handler
<rgw_zone_id
> {
3264 void encode_json(const char *name
, const void *pval
, ceph::Formatter
*f
) const override
{
3265 auto zone_id
= *(static_cast<const rgw_zone_id
*>(pval
));
3267 std::unique_ptr
<rgw::sal::Zone
> zone
;
3268 if (driver
->get_zone()->get_zonegroup().get_zone_by_id(zone_id
.id
, &zone
) == 0) {
3269 zone_name
= zone
->get_name();
3271 cerr
<< "WARNING: cannot find zone name for id=" << zone_id
<< std::endl
;
3272 zone_name
= zone_id
.id
;
3275 ::encode_json(name
, zone_name
, f
);
3277 } zone_id_type_handler
;
3279 JSONEncodeFilter encode_filter
;
3281 JSONFormatter_PrettyZone(bool pretty_format
) : JSONFormatter(pretty_format
) {
3282 encode_filter
.register_type(&zone_id_type_handler
);
3285 void *get_external_feature_handler(const std::string
& feature
) override
{
3286 if (feature
!= "JSONEncodeFilter") {
3289 return &encode_filter
;
3293 void init_realm_param(CephContext
*cct
, string
& var
, std::optional
<string
>& opt_var
, const string
& conf_name
)
3295 var
= cct
->_conf
.get_val
<string
>(conf_name
);
3301 int main(int argc
, const char **argv
)
3303 auto args
= argv_to_vec(argc
, argv
);
3305 cerr
<< argv
[0] << ": -h or --help for usage" << std::endl
;
3308 if (ceph_argparse_need_usage(args
)) {
3313 auto cct
= rgw_global_init(NULL
, args
, CEPH_ENTITY_TYPE_CLIENT
,
3314 CODE_ENVIRONMENT_UTILITY
, 0);
3316 // for region -> zonegroup conversion (must happen before common_init_finish())
3317 if (!g_conf()->rgw_region
.empty() && g_conf()->rgw_zonegroup
.empty()) {
3318 g_conf().set_val_or_die("rgw_zonegroup", g_conf()->rgw_region
.c_str());
3321 rgw_user user_id_arg
;
3322 std::unique_ptr
<rgw::sal::User
> user
;
3325 rgw_user new_user_id
;
3326 std::string access_key
, secret_key
, user_email
, display_name
;
3327 std::string bucket_name
, pool_name
, object
;
3329 std::string date
, subuser
, access
, format
;
3330 std::string start_date
, end_date
;
3331 std::string key_type_str
;
3332 std::string period_id
, period_epoch
, remote
, url
;
3333 std::optional
<string
> opt_region
;
3334 std::string master_zone
;
3335 std::string realm_name
, realm_id
, realm_new_name
;
3336 std::optional
<string
> opt_realm_name
, opt_realm_id
;
3337 std::string zone_name
, zone_id
, zone_new_name
;
3338 std::optional
<string
> opt_zone_name
, opt_zone_id
;
3339 std::string zonegroup_name
, zonegroup_id
, zonegroup_new_name
;
3340 std::optional
<string
> opt_zonegroup_name
, opt_zonegroup_id
;
3341 std::string api_name
;
3342 std::string role_name
, path
, assume_role_doc
, policy_name
, perm_policy_doc
, path_prefix
, max_session_duration
;
3343 std::string redirect_zone
;
3344 bool redirect_zone_set
= false;
3345 list
<string
> endpoints
;
3347 int sync_from_all_specified
= false;
3348 bool sync_from_all
= false;
3349 list
<string
> sync_from
;
3350 list
<string
> sync_from_rm
;
3352 int set_default
= 0;
3353 bool is_master
= false;
3354 bool is_master_set
= false;
3356 bool read_only
= false;
3357 int is_read_only_set
= false;
3359 int staging
= false;
3360 int key_type
= KEY_TYPE_UNDEFINED
;
3361 std::unique_ptr
<rgw::sal::Bucket
> bucket
;
3362 uint32_t perm_mask
= 0;
3364 OPT opt_cmd
= OPT::NO_CMD
;
3365 int gen_access_key
= 0;
3366 int gen_secret_key
= 0;
3367 bool set_perm
= false;
3368 bool set_temp_url_key
= false;
3369 map
<int, string
> temp_url_keys
;
3371 string new_bucket_name
;
3372 std::unique_ptr
<Formatter
> formatter
;
3373 std::unique_ptr
<Formatter
> zone_formatter
;
3374 int purge_data
= false;
3375 int pretty_format
= false;
3376 int show_log_entries
= true;
3377 int show_log_sum
= true;
3378 int skip_zero_entries
= false; // log show
3379 int purge_keys
= false;
3380 int yes_i_really_mean_it
= false;
3381 int delete_child_objects
= false;
3383 int remove_bad
= false;
3384 int check_head_obj_locator
= false;
3385 int max_buckets
= -1;
3386 bool max_buckets_specified
= false;
3387 map
<string
, bool> categories
;
3389 int check_objects
= false;
3390 RGWBucketAdminOpState bucket_op
;
3392 string metadata_key
;
3393 RGWObjVersionTracker objv_tracker
;
3395 string start_marker
;
3397 int max_entries
= -1;
3398 bool max_entries_specified
= false;
3400 bool admin_specified
= false;
3402 bool system_specified
= false;
3404 bool specified_shard_id
= false;
3409 string ratelimit_scope
;
3410 std::string objects_file
;
3411 string object_version
;
3412 string placement_id
;
3413 std::optional
<string
> opt_storage_class
;
3415 list
<string
> tags_add
;
3416 list
<string
> tags_rm
;
3417 int placement_inline_data
= true;
3418 bool placement_inline_data_specified
= false;
3420 int64_t max_objects
= -1;
3421 int64_t max_size
= -1;
3422 int64_t max_read_ops
= 0;
3423 int64_t max_write_ops
= 0;
3424 int64_t max_read_bytes
= 0;
3425 int64_t max_write_bytes
= 0;
3426 bool have_max_objects
= false;
3427 bool have_max_size
= false;
3428 bool have_max_write_ops
= false;
3429 bool have_max_read_ops
= false;
3430 bool have_max_write_bytes
= false;
3431 bool have_max_read_bytes
= false;
3432 int include_all
= false;
3433 int allow_unordered
= false;
3435 int sync_stats
= false;
3436 int reset_stats
= false;
3437 int bypass_gc
= false;
3438 int warnings_only
= false;
3439 int inconsistent_index
= false;
3441 int verbose
= false;
3443 int extra_info
= false;
3445 uint64_t min_rewrite_size
= 4 * 1024 * 1024;
3446 uint64_t max_rewrite_size
= ULLONG_MAX
;
3447 uint64_t min_rewrite_stripe_size
= 0;
3449 BIIndexType bi_index_type
= BIIndexType::Plain
;
3450 std::optional
<log_type
> opt_log_type
;
3454 bool num_shards_specified
= false;
3455 std::optional
<int> bucket_index_max_shards
;
3457 int max_concurrent_ios
= 32;
3458 uint64_t orphan_stale_secs
= (24 * 3600);
3462 std::ostringstream errs
;
3465 string source_zone_name
;
3466 rgw_zone_id source_zone
; /* zone id */
3469 bool tier_type_specified
= false;
3471 map
<string
, string
, ltstr_nocase
> tier_config_add
;
3472 map
<string
, string
, ltstr_nocase
> tier_config_rm
;
3474 boost::optional
<string
> index_pool
;
3475 boost::optional
<string
> data_pool
;
3476 boost::optional
<string
> data_extra_pool
;
3477 rgw::BucketIndexType placement_index_type
= rgw::BucketIndexType::Normal
;
3478 bool index_type_specified
= false;
3480 boost::optional
<std::string
> compression_type
;
3484 string totp_seed_type
= "hex";
3485 vector
<string
> totp_pin
;
3486 int totp_seconds
= 0;
3487 int totp_window
= 0;
3488 int trim_delay_ms
= 0;
3494 std::optional
<uint64_t> gen
;
3495 std::optional
<std::string
> str_script_ctx
;
3496 std::optional
<std::string
> script_package
;
3497 int allow_compilation
= false;
3499 std::optional
<string
> opt_group_id
;
3500 std::optional
<string
> opt_status
;
3501 std::optional
<string
> opt_flow_type
;
3502 std::optional
<vector
<string
> > opt_zone_names
;
3503 std::optional
<vector
<rgw_zone_id
> > opt_zone_ids
;
3504 std::optional
<string
> opt_flow_id
;
3505 std::optional
<string
> opt_source_zone_name
;
3506 std::optional
<rgw_zone_id
> opt_source_zone_id
;
3507 std::optional
<string
> opt_dest_zone_name
;
3508 std::optional
<rgw_zone_id
> opt_dest_zone_id
;
3509 std::optional
<vector
<string
> > opt_source_zone_names
;
3510 std::optional
<vector
<rgw_zone_id
> > opt_source_zone_ids
;
3511 std::optional
<vector
<string
> > opt_dest_zone_names
;
3512 std::optional
<vector
<rgw_zone_id
> > opt_dest_zone_ids
;
3513 std::optional
<string
> opt_pipe_id
;
3514 std::optional
<rgw_bucket
> opt_bucket
;
3515 std::optional
<string
> opt_tenant
;
3516 std::optional
<string
> opt_bucket_name
;
3517 std::optional
<string
> opt_bucket_id
;
3518 std::optional
<rgw_bucket
> opt_source_bucket
;
3519 std::optional
<string
> opt_source_tenant
;
3520 std::optional
<string
> opt_source_bucket_name
;
3521 std::optional
<string
> opt_source_bucket_id
;
3522 std::optional
<rgw_bucket
> opt_dest_bucket
;
3523 std::optional
<string
> opt_dest_tenant
;
3524 std::optional
<string
> opt_dest_bucket_name
;
3525 std::optional
<string
> opt_dest_bucket_id
;
3526 std::optional
<string
> opt_effective_zone_name
;
3527 std::optional
<rgw_zone_id
> opt_effective_zone_id
;
3529 std::optional
<string
> opt_prefix
;
3530 std::optional
<string
> opt_prefix_rm
;
3532 std::optional
<int> opt_priority
;
3533 std::optional
<string
> opt_mode
;
3534 std::optional
<rgw_user
> opt_dest_owner
;
3535 ceph::timespan opt_retry_delay_ms
= std::chrono::milliseconds(2000);
3536 ceph::timespan opt_timeout_sec
= std::chrono::seconds(60);
3538 std::optional
<std::string
> inject_error_at
;
3539 std::optional
<int> inject_error_code
;
3540 std::optional
<std::string
> inject_abort_at
;
3542 rgw::zone_features::set enable_features
;
3543 rgw::zone_features::set disable_features
;
3545 SimpleCmd
cmd(all_cmds
, cmd_aliases
);
3546 bool raw_storage_op
= false;
3548 std::optional
<std::string
> rgw_obj_fs
; // radoslist field separator
3550 init_realm_param(cct
.get(), realm_id
, opt_realm_id
, "rgw_realm_id");
3551 init_realm_param(cct
.get(), zonegroup_id
, opt_zonegroup_id
, "rgw_zonegroup_id");
3552 init_realm_param(cct
.get(), zone_id
, opt_zone_id
, "rgw_zone_id");
3554 for (std::vector
<const char*>::iterator i
= args
.begin(); i
!= args
.end(); ) {
3555 if (ceph_argparse_double_dash(args
, i
)) {
3557 } else if (ceph_argparse_witharg(args
, i
, &val
, "-i", "--uid", (char*)NULL
)) {
3558 user_id_arg
.from_str(val
);
3559 if (user_id_arg
.empty()) {
3560 cerr
<< "no value for uid" << std::endl
;
3563 } else if (ceph_argparse_witharg(args
, i
, &val
, "--new-uid", (char*)NULL
)) {
3564 new_user_id
.from_str(val
);
3565 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tenant", (char*)NULL
)) {
3568 } else if (ceph_argparse_witharg(args
, i
, &val
, "--user_ns", (char*)NULL
)) {
3570 } else if (ceph_argparse_witharg(args
, i
, &val
, "--access-key", (char*)NULL
)) {
3572 } else if (ceph_argparse_witharg(args
, i
, &val
, "--subuser", (char*)NULL
)) {
3574 } else if (ceph_argparse_witharg(args
, i
, &val
, "--secret", "--secret-key", (char*)NULL
)) {
3576 } else if (ceph_argparse_witharg(args
, i
, &val
, "-e", "--email", (char*)NULL
)) {
3578 } else if (ceph_argparse_witharg(args
, i
, &val
, "-n", "--display-name", (char*)NULL
)) {
3580 } else if (ceph_argparse_witharg(args
, i
, &val
, "-b", "--bucket", (char*)NULL
)) {
3582 opt_bucket_name
= val
;
3583 } else if (ceph_argparse_witharg(args
, i
, &val
, "-p", "--pool", (char*)NULL
)) {
3585 pool
= rgw_pool(pool_name
);
3586 } else if (ceph_argparse_witharg(args
, i
, &val
, "-o", "--object", (char*)NULL
)) {
3588 } else if (ceph_argparse_witharg(args
, i
, &val
, "--objects-file", (char*)NULL
)) {
3590 } else if (ceph_argparse_witharg(args
, i
, &val
, "--object-version", (char*)NULL
)) {
3591 object_version
= val
;
3592 } else if (ceph_argparse_witharg(args
, i
, &val
, "--client-id", (char*)NULL
)) {
3594 } else if (ceph_argparse_witharg(args
, i
, &val
, "--op-id", (char*)NULL
)) {
3596 } else if (ceph_argparse_witharg(args
, i
, &val
, "--op-mask", (char*)NULL
)) {
3598 } else if (ceph_argparse_witharg(args
, i
, &val
, "--key-type", (char*)NULL
)) {
3600 if (key_type_str
.compare("swift") == 0) {
3601 key_type
= KEY_TYPE_SWIFT
;
3602 } else if (key_type_str
.compare("s3") == 0) {
3603 key_type
= KEY_TYPE_S3
;
3605 cerr
<< "bad key type: " << key_type_str
<< std::endl
;
3608 } else if (ceph_argparse_witharg(args
, i
, &val
, "--job-id", (char*)NULL
)) {
3610 } else if (ceph_argparse_binary_flag(args
, i
, &gen_access_key
, NULL
, "--gen-access-key", (char*)NULL
)) {
3612 } else if (ceph_argparse_binary_flag(args
, i
, &gen_secret_key
, NULL
, "--gen-secret", (char*)NULL
)) {
3614 } else if (ceph_argparse_binary_flag(args
, i
, &show_log_entries
, NULL
, "--show-log-entries", (char*)NULL
)) {
3616 } else if (ceph_argparse_binary_flag(args
, i
, &show_log_sum
, NULL
, "--show-log-sum", (char*)NULL
)) {
3618 } else if (ceph_argparse_binary_flag(args
, i
, &skip_zero_entries
, NULL
, "--skip-zero-entries", (char*)NULL
)) {
3620 } else if (ceph_argparse_binary_flag(args
, i
, &admin
, NULL
, "--admin", (char*)NULL
)) {
3621 admin_specified
= true;
3622 } else if (ceph_argparse_binary_flag(args
, i
, &system
, NULL
, "--system", (char*)NULL
)) {
3623 system_specified
= true;
3624 } else if (ceph_argparse_binary_flag(args
, i
, &verbose
, NULL
, "--verbose", (char*)NULL
)) {
3626 } else if (ceph_argparse_binary_flag(args
, i
, &staging
, NULL
, "--staging", (char*)NULL
)) {
3628 } else if (ceph_argparse_binary_flag(args
, i
, &commit
, NULL
, "--commit", (char*)NULL
)) {
3630 } else if (ceph_argparse_witharg(args
, i
, &val
, "--min-rewrite-size", (char*)NULL
)) {
3631 min_rewrite_size
= (uint64_t)atoll(val
.c_str());
3632 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-rewrite-size", (char*)NULL
)) {
3633 max_rewrite_size
= (uint64_t)atoll(val
.c_str());
3634 } else if (ceph_argparse_witharg(args
, i
, &val
, "--min-rewrite-stripe-size", (char*)NULL
)) {
3635 min_rewrite_stripe_size
= (uint64_t)atoll(val
.c_str());
3636 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-buckets", (char*)NULL
)) {
3637 max_buckets
= (int)strict_strtol(val
.c_str(), 10, &err
);
3639 cerr
<< "ERROR: failed to parse max buckets: " << err
<< std::endl
;
3642 max_buckets_specified
= true;
3643 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-entries", (char*)NULL
)) {
3644 max_entries
= (int)strict_strtol(val
.c_str(), 10, &err
);
3645 max_entries_specified
= true;
3647 cerr
<< "ERROR: failed to parse max entries: " << err
<< std::endl
;
3650 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-size", (char*)NULL
)) {
3651 max_size
= strict_iec_cast
<long long>(val
, &err
);
3653 cerr
<< "ERROR: failed to parse max size: " << err
<< std::endl
;
3656 have_max_size
= true;
3657 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-objects", (char*)NULL
)) {
3658 max_objects
= (int64_t)strict_strtoll(val
.c_str(), 10, &err
);
3660 cerr
<< "ERROR: failed to parse max objects: " << err
<< std::endl
;
3663 have_max_objects
= true;
3664 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-read-ops", (char*)NULL
)) {
3665 max_read_ops
= (int64_t)strict_strtoll(val
.c_str(), 10, &err
);
3667 cerr
<< "ERROR: failed to parse max read requests: " << err
<< std::endl
;
3670 have_max_read_ops
= true;
3671 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-write-ops", (char*)NULL
)) {
3672 max_write_ops
= (int64_t)strict_strtoll(val
.c_str(), 10, &err
);
3674 cerr
<< "ERROR: failed to parse max write requests: " << err
<< std::endl
;
3677 have_max_write_ops
= true;
3678 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-read-bytes", (char*)NULL
)) {
3679 max_read_bytes
= (int64_t)strict_strtoll(val
.c_str(), 10, &err
);
3681 cerr
<< "ERROR: failed to parse max read bytes: " << err
<< std::endl
;
3684 have_max_read_bytes
= true;
3685 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-write-bytes", (char*)NULL
)) {
3686 max_write_bytes
= (int64_t)strict_strtoll(val
.c_str(), 10, &err
);
3688 cerr
<< "ERROR: failed to parse max write bytes: " << err
<< std::endl
;
3691 have_max_write_bytes
= true;
3692 } else if (ceph_argparse_witharg(args
, i
, &val
, "--date", "--time", (char*)NULL
)) {
3694 if (end_date
.empty())
3696 } else if (ceph_argparse_witharg(args
, i
, &val
, "--start-date", "--start-time", (char*)NULL
)) {
3698 } else if (ceph_argparse_witharg(args
, i
, &val
, "--end-date", "--end-time", (char*)NULL
)) {
3700 } else if (ceph_argparse_witharg(args
, i
, &val
, "--num-shards", (char*)NULL
)) {
3701 num_shards
= (int)strict_strtol(val
.c_str(), 10, &err
);
3703 cerr
<< "ERROR: failed to parse num shards: " << err
<< std::endl
;
3706 num_shards_specified
= true;
3707 } else if (ceph_argparse_witharg(args
, i
, &val
, "--bucket-index-max-shards", (char*)NULL
)) {
3708 bucket_index_max_shards
= (int)strict_strtol(val
.c_str(), 10, &err
);
3710 cerr
<< "ERROR: failed to parse bucket-index-max-shards: " << err
<< std::endl
;
3713 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-concurrent-ios", (char*)NULL
)) {
3714 max_concurrent_ios
= (int)strict_strtol(val
.c_str(), 10, &err
);
3716 cerr
<< "ERROR: failed to parse max concurrent ios: " << err
<< std::endl
;
3719 } else if (ceph_argparse_witharg(args
, i
, &val
, "--orphan-stale-secs", (char*)NULL
)) {
3720 orphan_stale_secs
= (uint64_t)strict_strtoll(val
.c_str(), 10, &err
);
3722 cerr
<< "ERROR: failed to parse orphan stale secs: " << err
<< std::endl
;
3725 } else if (ceph_argparse_witharg(args
, i
, &val
, "--shard-id", (char*)NULL
)) {
3726 shard_id
= (int)strict_strtol(val
.c_str(), 10, &err
);
3728 cerr
<< "ERROR: failed to parse shard id: " << err
<< std::endl
;
3731 specified_shard_id
= true;
3732 } else if (ceph_argparse_witharg(args
, i
, &val
, "--gen", (char*)NULL
)) {
3733 gen
= strict_strtoll(val
.c_str(), 10, &err
);
3735 cerr
<< "ERROR: failed to parse gen id: " << err
<< std::endl
;
3738 } else if (ceph_argparse_witharg(args
, i
, &val
, "--access", (char*)NULL
)) {
3740 perm_mask
= rgw_str_to_perm(access
.c_str());
3742 } else if (ceph_argparse_witharg(args
, i
, &val
, "--temp-url-key", (char*)NULL
)) {
3743 temp_url_keys
[0] = val
;
3744 set_temp_url_key
= true;
3745 } else if (ceph_argparse_witharg(args
, i
, &val
, "--temp-url-key2", "--temp-url-key-2", (char*)NULL
)) {
3746 temp_url_keys
[1] = val
;
3747 set_temp_url_key
= true;
3748 } else if (ceph_argparse_witharg(args
, i
, &val
, "--bucket-id", (char*)NULL
)) {
3750 opt_bucket_id
= val
;
3751 if (bucket_id
.empty()) {
3752 cerr
<< "no value for bucket-id" << std::endl
;
3755 } else if (ceph_argparse_witharg(args
, i
, &val
, "--bucket-new-name", (char*)NULL
)) {
3756 new_bucket_name
= val
;
3757 } else if (ceph_argparse_witharg(args
, i
, &val
, "--format", (char*)NULL
)) {
3759 } else if (ceph_argparse_witharg(args
, i
, &val
, "--categories", (char*)NULL
)) {
3760 string cat_str
= val
;
3761 list
<string
> cat_list
;
3762 list
<string
>::iterator iter
;
3763 get_str_list(cat_str
, cat_list
);
3764 for (iter
= cat_list
.begin(); iter
!= cat_list
.end(); ++iter
) {
3765 categories
[*iter
] = true;
3767 } else if (ceph_argparse_binary_flag(args
, i
, &delete_child_objects
, NULL
, "--purge-objects", (char*)NULL
)) {
3769 } else if (ceph_argparse_binary_flag(args
, i
, &pretty_format
, NULL
, "--pretty-format", (char*)NULL
)) {
3771 } else if (ceph_argparse_binary_flag(args
, i
, &purge_data
, NULL
, "--purge-data", (char*)NULL
)) {
3772 delete_child_objects
= purge_data
;
3773 } else if (ceph_argparse_binary_flag(args
, i
, &purge_keys
, NULL
, "--purge-keys", (char*)NULL
)) {
3775 } else if (ceph_argparse_binary_flag(args
, i
, &yes_i_really_mean_it
, NULL
, "--yes-i-really-mean-it", (char*)NULL
)) {
3777 } else if (ceph_argparse_binary_flag(args
, i
, &fix
, NULL
, "--fix", (char*)NULL
)) {
3779 } else if (ceph_argparse_binary_flag(args
, i
, &remove_bad
, NULL
, "--remove-bad", (char*)NULL
)) {
3781 } else if (ceph_argparse_binary_flag(args
, i
, &check_head_obj_locator
, NULL
, "--check-head-obj-locator", (char*)NULL
)) {
3783 } else if (ceph_argparse_binary_flag(args
, i
, &check_objects
, NULL
, "--check-objects", (char*)NULL
)) {
3785 } else if (ceph_argparse_binary_flag(args
, i
, &sync_stats
, NULL
, "--sync-stats", (char*)NULL
)) {
3787 } else if (ceph_argparse_binary_flag(args
, i
, &reset_stats
, NULL
, "--reset-stats", (char*)NULL
)) {
3789 } else if (ceph_argparse_binary_flag(args
, i
, &include_all
, NULL
, "--include-all", (char*)NULL
)) {
3791 } else if (ceph_argparse_binary_flag(args
, i
, &allow_unordered
, NULL
, "--allow-unordered", (char*)NULL
)) {
3793 } else if (ceph_argparse_binary_flag(args
, i
, &extra_info
, NULL
, "--extra-info", (char*)NULL
)) {
3795 } else if (ceph_argparse_binary_flag(args
, i
, &bypass_gc
, NULL
, "--bypass-gc", (char*)NULL
)) {
3797 } else if (ceph_argparse_binary_flag(args
, i
, &warnings_only
, NULL
, "--warnings-only", (char*)NULL
)) {
3799 } else if (ceph_argparse_binary_flag(args
, i
, &inconsistent_index
, NULL
, "--inconsistent-index", (char*)NULL
)) {
3801 } else if (ceph_argparse_binary_flag(args
, i
, &placement_inline_data
, NULL
, "--placement-inline-data", (char*)NULL
)) {
3802 placement_inline_data_specified
= true;
3804 } else if (ceph_argparse_witharg(args
, i
, &val
, "--caps", (char*)NULL
)) {
3806 } else if (ceph_argparse_witharg(args
, i
, &val
, "--infile", (char*)NULL
)) {
3808 } else if (ceph_argparse_witharg(args
, i
, &val
, "--metadata-key", (char*)NULL
)) {
3810 } else if (ceph_argparse_witharg(args
, i
, &val
, "--marker", (char*)NULL
)) {
3812 } else if (ceph_argparse_witharg(args
, i
, &val
, "--start-marker", (char*)NULL
)) {
3814 } else if (ceph_argparse_witharg(args
, i
, &val
, "--end-marker", (char*)NULL
)) {
3816 } else if (ceph_argparse_witharg(args
, i
, &val
, "--quota-scope", (char*)NULL
)) {
3818 } else if (ceph_argparse_witharg(args
, i
, &val
, "--ratelimit-scope", (char*)NULL
)) {
3819 ratelimit_scope
= val
;
3820 } else if (ceph_argparse_witharg(args
, i
, &val
, "--index-type", (char*)NULL
)) {
3821 string index_type_str
= val
;
3822 bi_index_type
= get_bi_index_type(index_type_str
);
3823 if (bi_index_type
== BIIndexType::Invalid
) {
3824 cerr
<< "ERROR: invalid bucket index entry type" << std::endl
;
3827 } else if (ceph_argparse_witharg(args
, i
, &val
, "--log-type", (char*)NULL
)) {
3828 string log_type_str
= val
;
3829 auto l
= get_log_type(log_type_str
);
3830 if (l
== static_cast<log_type
>(0xff)) {
3831 cerr
<< "ERROR: invalid log type" << std::endl
;
3835 } else if (ceph_argparse_binary_flag(args
, i
, &is_master_int
, NULL
, "--master", (char*)NULL
)) {
3836 is_master
= (bool)is_master_int
;
3837 is_master_set
= true;
3838 } else if (ceph_argparse_binary_flag(args
, i
, &set_default
, NULL
, "--default", (char*)NULL
)) {
3840 } else if (ceph_argparse_witharg(args
, i
, &val
, "--redirect-zone", (char*)NULL
)) {
3841 redirect_zone
= val
;
3842 redirect_zone_set
= true;
3843 } else if (ceph_argparse_binary_flag(args
, i
, &read_only_int
, NULL
, "--read-only", (char*)NULL
)) {
3844 read_only
= (bool)read_only_int
;
3845 is_read_only_set
= true;
3846 } else if (ceph_argparse_witharg(args
, i
, &val
, "--master-zone", (char*)NULL
)) {
3848 } else if (ceph_argparse_witharg(args
, i
, &val
, "--period", (char*)NULL
)) {
3850 } else if (ceph_argparse_witharg(args
, i
, &val
, "--epoch", (char*)NULL
)) {
3852 } else if (ceph_argparse_witharg(args
, i
, &val
, "--remote", (char*)NULL
)) {
3854 } else if (ceph_argparse_witharg(args
, i
, &val
, "--url", (char*)NULL
)) {
3856 } else if (ceph_argparse_witharg(args
, i
, &val
, "--region", (char*)NULL
)) {
3858 } else if (ceph_argparse_witharg(args
, i
, &val
, "--realm-id", (char*)NULL
)) {
3861 g_conf().set_val("rgw_realm_id", val
);
3862 } else if (ceph_argparse_witharg(args
, i
, &val
, "--realm-new-name", (char*)NULL
)) {
3863 realm_new_name
= val
;
3864 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zonegroup-id", (char*)NULL
)) {
3866 opt_zonegroup_id
= val
;
3867 g_conf().set_val("rgw_zonegroup_id", val
);
3868 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zonegroup-new-name", (char*)NULL
)) {
3869 zonegroup_new_name
= val
;
3870 } else if (ceph_argparse_witharg(args
, i
, &val
, "--placement-id", (char*)NULL
)) {
3872 } else if (ceph_argparse_witharg(args
, i
, &val
, "--storage-class", (char*)NULL
)) {
3873 opt_storage_class
= val
;
3874 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tags", (char*)NULL
)) {
3875 get_str_list(val
, ",", tags
);
3876 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tags-add", (char*)NULL
)) {
3877 get_str_list(val
, ",", tags_add
);
3878 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tags-rm", (char*)NULL
)) {
3879 get_str_list(val
, ",", tags_rm
);
3880 } else if (ceph_argparse_witharg(args
, i
, &val
, "--api-name", (char*)NULL
)) {
3882 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zone-id", (char*)NULL
)) {
3885 g_conf().set_val("rgw_zone_id", val
);
3886 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zone-new-name", (char*)NULL
)) {
3887 zone_new_name
= val
;
3888 } else if (ceph_argparse_witharg(args
, i
, &val
, "--endpoints", (char*)NULL
)) {
3889 get_str_list(val
, endpoints
);
3890 } else if (ceph_argparse_witharg(args
, i
, &val
, "--sync-from", (char*)NULL
)) {
3891 get_str_list(val
, sync_from
);
3892 } else if (ceph_argparse_witharg(args
, i
, &val
, "--sync-from-rm", (char*)NULL
)) {
3893 get_str_list(val
, sync_from_rm
);
3894 } else if (ceph_argparse_binary_flag(args
, i
, &tmp_int
, NULL
, "--sync-from-all", (char*)NULL
)) {
3895 sync_from_all
= (bool)tmp_int
;
3896 sync_from_all_specified
= true;
3897 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zone", (char*)NULL
)) {
3898 source_zone_name
= val
;
3899 opt_source_zone_name
= val
;
3900 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zone-id", (char*)NULL
)) {
3901 opt_source_zone_id
= val
;
3902 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zone", (char*)NULL
)) {
3903 opt_dest_zone_name
= val
;
3904 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zone-id", (char*)NULL
)) {
3905 opt_dest_zone_id
= val
;
3906 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tier-type", (char*)NULL
)) {
3908 tier_type_specified
= true;
3909 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tier-config", (char*)NULL
)) {
3910 parse_tier_config_param(val
, tier_config_add
);
3911 } else if (ceph_argparse_witharg(args
, i
, &val
, "--tier-config-rm", (char*)NULL
)) {
3912 parse_tier_config_param(val
, tier_config_rm
);
3913 } else if (ceph_argparse_witharg(args
, i
, &val
, "--index-pool", (char*)NULL
)) {
3915 } else if (ceph_argparse_witharg(args
, i
, &val
, "--data-pool", (char*)NULL
)) {
3917 } else if (ceph_argparse_witharg(args
, i
, &val
, "--data-extra-pool", (char*)NULL
)) {
3918 data_extra_pool
= val
;
3919 } else if (ceph_argparse_witharg(args
, i
, &val
, "--placement-index-type", (char*)NULL
)) {
3920 if (val
== "normal") {
3921 placement_index_type
= rgw::BucketIndexType::Normal
;
3922 } else if (val
== "indexless") {
3923 placement_index_type
= rgw::BucketIndexType::Indexless
;
3925 placement_index_type
= (rgw::BucketIndexType
)strict_strtol(val
.c_str(), 10, &err
);
3927 cerr
<< "ERROR: failed to parse index type index: " << err
<< std::endl
;
3931 index_type_specified
= true;
3932 } else if (ceph_argparse_witharg(args
, i
, &val
, "--compression", (char*)NULL
)) {
3933 compression_type
= val
;
3934 } else if (ceph_argparse_witharg(args
, i
, &val
, "--role-name", (char*)NULL
)) {
3936 } else if (ceph_argparse_witharg(args
, i
, &val
, "--path", (char*)NULL
)) {
3938 } else if (ceph_argparse_witharg(args
, i
, &val
, "--assume-role-policy-doc", (char*)NULL
)) {
3939 assume_role_doc
= val
;
3940 } else if (ceph_argparse_witharg(args
, i
, &val
, "--policy-name", (char*)NULL
)) {
3942 } else if (ceph_argparse_witharg(args
, i
, &val
, "--policy-doc", (char*)NULL
)) {
3943 perm_policy_doc
= val
;
3944 } else if (ceph_argparse_witharg(args
, i
, &val
, "--path-prefix", (char*)NULL
)) {
3946 } else if (ceph_argparse_witharg(args
, i
, &val
, "--max-session-duration", (char*)NULL
)) {
3947 max_session_duration
= val
;
3948 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-serial", (char*)NULL
)) {
3950 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-pin", (char*)NULL
)) {
3951 totp_pin
.push_back(val
);
3952 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-seed", (char*)NULL
)) {
3954 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-seed-type", (char*)NULL
)) {
3955 totp_seed_type
= val
;
3956 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-seconds", (char*)NULL
)) {
3957 totp_seconds
= atoi(val
.c_str());
3958 } else if (ceph_argparse_witharg(args
, i
, &val
, "--totp-window", (char*)NULL
)) {
3959 totp_window
= atoi(val
.c_str());
3960 } else if (ceph_argparse_witharg(args
, i
, &val
, "--trim-delay-ms", (char*)NULL
)) {
3961 trim_delay_ms
= atoi(val
.c_str());
3962 } else if (ceph_argparse_witharg(args
, i
, &val
, "--topic", (char*)NULL
)) {
3964 } else if (ceph_argparse_witharg(args
, i
, &val
, "--subscription", (char*)NULL
)) {
3966 } else if (ceph_argparse_witharg(args
, i
, &val
, "--event-id", (char*)NULL
)) {
3968 } else if (ceph_argparse_witharg(args
, i
, &val
, "--group-id", (char*)NULL
)) {
3970 } else if (ceph_argparse_witharg(args
, i
, &val
, "--status", (char*)NULL
)) {
3972 } else if (ceph_argparse_witharg(args
, i
, &val
, "--flow-type", (char*)NULL
)) {
3973 opt_flow_type
= val
;
3974 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zones", "--zone-names", (char*)NULL
)) {
3976 get_str_vec(val
, v
);
3977 opt_zone_names
= std::move(v
);
3978 } else if (ceph_argparse_witharg(args
, i
, &val
, "--zone-ids", (char*)NULL
)) {
3979 opt_zone_ids
= zone_ids_from_str(val
);
3980 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zones", "--source-zone-names", (char*)NULL
)) {
3982 get_str_vec(val
, v
);
3983 opt_source_zone_names
= std::move(v
);
3984 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-zone-ids", (char*)NULL
)) {
3985 opt_source_zone_ids
= zone_ids_from_str(val
);
3986 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zones", "--dest-zone-names", (char*)NULL
)) {
3988 get_str_vec(val
, v
);
3989 opt_dest_zone_names
= std::move(v
);
3990 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-zone-ids", (char*)NULL
)) {
3991 opt_dest_zone_ids
= zone_ids_from_str(val
);
3992 } else if (ceph_argparse_witharg(args
, i
, &val
, "--flow-id", (char*)NULL
)) {
3994 } else if (ceph_argparse_witharg(args
, i
, &val
, "--pipe-id", (char*)NULL
)) {
3996 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-tenant", (char*)NULL
)) {
3997 opt_source_tenant
= val
;
3998 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-bucket", (char*)NULL
)) {
3999 opt_source_bucket_name
= val
;
4000 } else if (ceph_argparse_witharg(args
, i
, &val
, "--source-bucket-id", (char*)NULL
)) {
4001 opt_source_bucket_id
= val
;
4002 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-tenant", (char*)NULL
)) {
4003 opt_dest_tenant
= val
;
4004 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-bucket", (char*)NULL
)) {
4005 opt_dest_bucket_name
= val
;
4006 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-bucket-id", (char*)NULL
)) {
4007 opt_dest_bucket_id
= val
;
4008 } else if (ceph_argparse_witharg(args
, i
, &val
, "--effective-zone-name", "--effective-zone", (char*)NULL
)) {
4009 opt_effective_zone_name
= val
;
4010 } else if (ceph_argparse_witharg(args
, i
, &val
, "--effective-zone-id", (char*)NULL
)) {
4011 opt_effective_zone_id
= rgw_zone_id(val
);
4012 } else if (ceph_argparse_witharg(args
, i
, &val
, "--prefix", (char*)NULL
)) {
4014 } else if (ceph_argparse_witharg(args
, i
, &val
, "--prefix-rm", (char*)NULL
)) {
4015 opt_prefix_rm
= val
;
4016 } else if (ceph_argparse_witharg(args
, i
, &val
, "--priority", (char*)NULL
)) {
4017 opt_priority
= atoi(val
.c_str());
4018 } else if (ceph_argparse_witharg(args
, i
, &val
, "--mode", (char*)NULL
)) {
4020 } else if (ceph_argparse_witharg(args
, i
, &val
, "--dest-owner", (char*)NULL
)) {
4021 opt_dest_owner
.emplace(val
);
4022 opt_dest_owner
= val
;
4023 } else if (ceph_argparse_witharg(args
, i
, &val
, "--retry-delay-ms", (char*)NULL
)) {
4024 opt_retry_delay_ms
= std::chrono::milliseconds(atoi(val
.c_str()));
4025 } else if (ceph_argparse_witharg(args
, i
, &val
, "--timeout-sec", (char*)NULL
)) {
4026 opt_timeout_sec
= std::chrono::seconds(atoi(val
.c_str()));
4027 } else if (ceph_argparse_witharg(args
, i
, &val
, "--inject-error-at", (char*)NULL
)) {
4028 inject_error_at
= val
;
4029 } else if (ceph_argparse_witharg(args
, i
, &val
, "--inject-error-code", (char*)NULL
)) {
4030 inject_error_code
= atoi(val
.c_str());
4031 } else if (ceph_argparse_witharg(args
, i
, &val
, "--inject-abort-at", (char*)NULL
)) {
4032 inject_abort_at
= val
;
4033 } else if (ceph_argparse_binary_flag(args
, i
, &detail
, NULL
, "--detail", (char*)NULL
)) {
4035 } else if (ceph_argparse_witharg(args
, i
, &val
, "--context", (char*)NULL
)) {
4036 str_script_ctx
= val
;
4037 } else if (ceph_argparse_witharg(args
, i
, &val
, "--package", (char*)NULL
)) {
4038 script_package
= val
;
4039 } else if (ceph_argparse_binary_flag(args
, i
, &allow_compilation
, NULL
, "--allow-compilation", (char*)NULL
)) {
4041 } else if (ceph_argparse_witharg(args
, i
, &val
, "--rgw-obj-fs", (char*)NULL
)) {
4043 } else if (ceph_argparse_witharg(args
, i
, &val
, "--enable-feature", (char*)NULL
)) {
4044 if (!rgw::zone_features::supports(val
)) {
4045 std::cerr
<< "ERROR: Cannot enable unrecognized zone feature \"" << val
<< "\"" << std::endl
;
4048 enable_features
.insert(val
);
4049 } else if (ceph_argparse_witharg(args
, i
, &val
, "--disable-feature", (char*)NULL
)) {
4050 disable_features
.insert(val
);
4051 } else if (strncmp(*i
, "-", 1) == 0) {
4052 cerr
<< "ERROR: invalid flag " << *i
<< std::endl
;
4059 /* common_init_finish needs to be called after g_conf().set_val() */
4060 common_init_finish(g_ceph_context
);
4062 std::unique_ptr
<rgw::sal::ConfigStore
> cfgstore
;
4069 std::vector
<string
> extra_args
;
4070 std::vector
<string
> expected
;
4074 if (!cmd
.find_command(args
, &_opt_cmd
, &extra_args
, &err
, &expected
)) {
4075 if (!expected
.empty()) {
4076 cerr
<< err
<< std::endl
;
4077 cerr
<< "Expected one of the following:" << std::endl
;
4078 for (auto& exp
: expected
) {
4079 if (exp
== "*" || exp
== "[*]") {
4082 cerr
<< " " << exp
<< std::endl
;
4085 cerr
<< "Command not found:";
4086 for (auto& arg
: args
) {
4094 opt_cmd
= std::any_cast
<OPT
>(_opt_cmd
);
4096 /* some commands may have an optional extra param */
4097 if (!extra_args
.empty()) {
4099 case OPT::METADATA_GET
:
4100 case OPT::METADATA_PUT
:
4101 case OPT::METADATA_RM
:
4102 case OPT::METADATA_LIST
:
4103 metadata_key
= extra_args
[0];
4110 // not a raw op if 'period update' needs to commit to master
4111 bool raw_period_update
= opt_cmd
== OPT::PERIOD_UPDATE
&& !commit
;
4112 // not a raw op if 'period pull' needs to read zone/period configuration
4113 bool raw_period_pull
= opt_cmd
== OPT::PERIOD_PULL
&& !url
.empty();
4115 std::set
<OPT
> raw_storage_ops_list
= {OPT::ZONEGROUP_ADD
, OPT::ZONEGROUP_CREATE
,
4116 OPT::ZONEGROUP_DELETE
,
4117 OPT::ZONEGROUP_GET
, OPT::ZONEGROUP_LIST
,
4118 OPT::ZONEGROUP_SET
, OPT::ZONEGROUP_DEFAULT
,
4119 OPT::ZONEGROUP_RENAME
, OPT::ZONEGROUP_MODIFY
,
4120 OPT::ZONEGROUP_REMOVE
,
4121 OPT::ZONEGROUP_PLACEMENT_ADD
, OPT::ZONEGROUP_PLACEMENT_RM
,
4122 OPT::ZONEGROUP_PLACEMENT_MODIFY
, OPT::ZONEGROUP_PLACEMENT_LIST
,
4123 OPT::ZONEGROUP_PLACEMENT_GET
,
4124 OPT::ZONEGROUP_PLACEMENT_DEFAULT
,
4125 OPT::ZONE_CREATE
, OPT::ZONE_DELETE
,
4126 OPT::ZONE_GET
, OPT::ZONE_SET
, OPT::ZONE_RENAME
,
4127 OPT::ZONE_LIST
, OPT::ZONE_MODIFY
, OPT::ZONE_DEFAULT
,
4128 OPT::ZONE_PLACEMENT_ADD
, OPT::ZONE_PLACEMENT_RM
,
4129 OPT::ZONE_PLACEMENT_MODIFY
, OPT::ZONE_PLACEMENT_LIST
,
4130 OPT::ZONE_PLACEMENT_GET
,
4132 OPT::PERIOD_DELETE
, OPT::PERIOD_GET
,
4133 OPT::PERIOD_GET_CURRENT
, OPT::PERIOD_LIST
,
4134 OPT::GLOBAL_QUOTA_GET
, OPT::GLOBAL_QUOTA_SET
,
4135 OPT::GLOBAL_QUOTA_ENABLE
, OPT::GLOBAL_QUOTA_DISABLE
,
4136 OPT::GLOBAL_RATELIMIT_GET
, OPT::GLOBAL_RATELIMIT_SET
,
4137 OPT::GLOBAL_RATELIMIT_ENABLE
, OPT::GLOBAL_RATELIMIT_DISABLE
,
4138 OPT::REALM_DELETE
, OPT::REALM_GET
, OPT::REALM_LIST
,
4139 OPT::REALM_LIST_PERIODS
,
4140 OPT::REALM_GET_DEFAULT
,
4141 OPT::REALM_RENAME
, OPT::REALM_SET
,
4142 OPT::REALM_DEFAULT
, OPT::REALM_PULL
};
4144 std::set
<OPT
> readonly_ops_list
= {
4148 OPT::BUCKET_LIMIT_CHECK
,
4151 OPT::BUCKET_SYNC_CHECKPOINT
,
4152 OPT::BUCKET_SYNC_INFO
,
4153 OPT::BUCKET_SYNC_STATUS
,
4154 OPT::BUCKET_SYNC_MARKERS
,
4155 OPT::BUCKET_SHARD_OBJECTS
,
4156 OPT::BUCKET_OBJECT_SHARD
,
4167 OPT::ORPHANS_LIST_JOBS
,
4169 OPT::ZONEGROUP_LIST
,
4170 OPT::ZONEGROUP_PLACEMENT_LIST
,
4171 OPT::ZONEGROUP_PLACEMENT_GET
,
4174 OPT::ZONE_PLACEMENT_LIST
,
4175 OPT::ZONE_PLACEMENT_GET
,
4178 OPT::METADATA_SYNC_STATUS
,
4181 OPT::SYNC_ERROR_LIST
,
4182 OPT::SYNC_GROUP_GET
,
4183 OPT::SYNC_POLICY_GET
,
4186 OPT::DATA_SYNC_STATUS
,
4188 OPT::DATALOG_STATUS
,
4190 OPT::REALM_GET_DEFAULT
,
4192 OPT::REALM_LIST_PERIODS
,
4194 OPT::PERIOD_GET_CURRENT
,
4196 OPT::GLOBAL_QUOTA_GET
,
4197 OPT::GLOBAL_RATELIMIT_GET
,
4202 OPT::ROLE_POLICY_LIST
,
4203 OPT::ROLE_POLICY_GET
,
4205 OPT::RESHARD_STATUS
,
4206 OPT::PUBSUB_TOPICS_LIST
,
4207 OPT::PUBSUB_TOPIC_GET
,
4211 std::set
<OPT
> gc_ops_list
= {
4215 OPT::BUCKET_RM
, // --purge-objects
4216 OPT::USER_RM
, // --purge-data
4217 OPT::OBJECTS_EXPIRE
,
4218 OPT::OBJECTS_EXPIRE_STALE_RM
,
4220 OPT::BUCKET_SYNC_RUN
,
4222 OPT::BUCKET_REWRITE
,
4226 raw_storage_op
= (raw_storage_ops_list
.find(opt_cmd
) != raw_storage_ops_list
.end() ||
4227 raw_period_update
|| raw_period_pull
);
4228 bool need_cache
= readonly_ops_list
.find(opt_cmd
) == readonly_ops_list
.end();
4229 bool need_gc
= (gc_ops_list
.find(opt_cmd
) != gc_ops_list
.end()) && !bypass_gc
;
4231 DriverManager::Config cfg
= DriverManager::get_config(true, g_ceph_context
);
4233 auto config_store_type
= g_conf().get_val
<std::string
>("rgw_config_store");
4234 cfgstore
= DriverManager::create_config_store(dpp(), config_store_type
);
4236 cerr
<< "couldn't init config storage provider" << std::endl
;
4240 if (raw_storage_op
) {
4241 driver
= DriverManager::get_raw_storage(dpp(),
4245 driver
= DriverManager::get_storage(dpp(),
4253 need_cache
&& g_conf()->rgw_cache_enabled
,
4257 cerr
<< "couldn't init storage provider" << std::endl
;
4261 /* Needs to be after the driver is initialized. Note, user could be empty here. */
4262 user
= driver
->get_user(user_id_arg
);
4264 init_optional_bucket(opt_bucket
, opt_tenant
,
4265 opt_bucket_name
, opt_bucket_id
);
4266 init_optional_bucket(opt_source_bucket
, opt_source_tenant
,
4267 opt_source_bucket_name
, opt_source_bucket_id
);
4268 init_optional_bucket(opt_dest_bucket
, opt_dest_tenant
,
4269 opt_dest_bucket_name
, opt_dest_bucket_id
);
4271 if (tenant
.empty()) {
4272 tenant
= user
->get_tenant();
4274 if (rgw::sal::User::empty(user
) && opt_cmd
!= OPT::ROLE_CREATE
4275 && opt_cmd
!= OPT::ROLE_DELETE
4276 && opt_cmd
!= OPT::ROLE_GET
4277 && opt_cmd
!= OPT::ROLE_TRUST_POLICY_MODIFY
4278 && opt_cmd
!= OPT::ROLE_LIST
4279 && opt_cmd
!= OPT::ROLE_POLICY_PUT
4280 && opt_cmd
!= OPT::ROLE_POLICY_LIST
4281 && opt_cmd
!= OPT::ROLE_POLICY_GET
4282 && opt_cmd
!= OPT::ROLE_POLICY_DELETE
4283 && opt_cmd
!= OPT::ROLE_UPDATE
4284 && opt_cmd
!= OPT::RESHARD_ADD
4285 && opt_cmd
!= OPT::RESHARD_CANCEL
4286 && opt_cmd
!= OPT::RESHARD_STATUS
4287 && opt_cmd
!= OPT::PUBSUB_TOPICS_LIST
4288 && opt_cmd
!= OPT::PUBSUB_TOPIC_GET
4289 && opt_cmd
!= OPT::PUBSUB_TOPIC_RM
) {
4290 cerr
<< "ERROR: --tenant is set, but there's no user ID" << std::endl
;
4293 user
->set_tenant(tenant
);
4295 if (user_ns
.empty()) {
4296 user_ns
= user
->get_id().ns
;
4298 user
->set_ns(user_ns
);
4301 if (!new_user_id
.empty() && !tenant
.empty()) {
4302 new_user_id
.tenant
= tenant
;
4305 /* check key parameter conflict */
4306 if ((!access_key
.empty()) && gen_access_key
) {
4307 cerr
<< "ERROR: key parameter conflict, --access-key & --gen-access-key" << std::endl
;
4310 if ((!secret_key
.empty()) && gen_secret_key
) {
4311 cerr
<< "ERROR: key parameter conflict, --secret & --gen-secret" << std::endl
;
4316 // default to pretty json
4317 if (format
.empty()) {
4319 pretty_format
= true;
4322 if (format
== "xml")
4323 formatter
= make_unique
<XMLFormatter
>(new XMLFormatter(pretty_format
));
4324 else if (format
== "json")
4325 formatter
= make_unique
<JSONFormatter
>(new JSONFormatter(pretty_format
));
4327 cerr
<< "unrecognized format: " << format
<< std::endl
;
4331 zone_formatter
= std::make_unique
<JSONFormatter_PrettyZone
>(pretty_format
);
4333 realm_name
= g_conf()->rgw_realm
;
4334 zone_name
= g_conf()->rgw_zone
;
4335 zonegroup_name
= g_conf()->rgw_zonegroup
;
4337 if (!realm_name
.empty()) {
4338 opt_realm_name
= realm_name
;
4341 if (!zone_name
.empty()) {
4342 opt_zone_name
= zone_name
;
4345 if (!zonegroup_name
.empty()) {
4346 opt_zonegroup_name
= zonegroup_name
;
4349 RGWStreamFlusher
stream_flusher(formatter
.get(), cout
);
4351 RGWUserAdminOpState
user_op(driver
);
4352 if (!user_email
.empty()) {
4353 user_op
.user_email_specified
=true;
4356 if (!source_zone_name
.empty()) {
4357 std::unique_ptr
<rgw::sal::Zone
> zone
;
4358 if (driver
->get_zone()->get_zonegroup().get_zone_by_name(source_zone_name
, &zone
) < 0) {
4359 cerr
<< "WARNING: cannot find source zone id for name=" << source_zone_name
<< std::endl
;
4360 source_zone
= source_zone_name
;
4362 source_zone
.id
= zone
->get_id();
4366 rgw_http_client_init(g_ceph_context
);
4368 struct rgw_curl_setup
{
4370 rgw::curl::setup_curl(boost::none
);
4373 rgw::curl::cleanup_curl();
4379 StoreDestructor
store_destructor(driver
);
4381 if (raw_storage_op
) {
4383 case OPT::PERIOD_DELETE
:
4385 if (period_id
.empty()) {
4386 cerr
<< "missing period id" << std::endl
;
4389 int ret
= cfgstore
->delete_period(dpp(), null_yield
, period_id
);
4391 cerr
<< "ERROR: couldn't delete period: " << cpp_strerror(-ret
) << std::endl
;
4397 case OPT::PERIOD_GET
:
4399 std::optional
<epoch_t
> epoch
;
4400 if (!period_epoch
.empty()) {
4401 epoch
= atoi(period_epoch
.c_str());
4405 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4406 realm_id
, realm_name
, realm
);
4408 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4411 realm_id
= realm
.get_id();
4412 realm_name
= realm
.get_name();
4413 period_id
= RGWPeriod::get_staging_id(realm_id
);
4416 if (period_id
.empty()) {
4417 // use realm's current period
4419 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4420 realm_id
, realm_name
, realm
);
4422 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4425 period_id
= realm
.current_period
;
4429 int ret
= cfgstore
->read_period(dpp(), null_yield
, period_id
,
4432 cerr
<< "failed to load period: " << cpp_strerror(-ret
) << std::endl
;
4435 encode_json("period", period
, formatter
.get());
4436 formatter
->flush(cout
);
4439 case OPT::PERIOD_GET_CURRENT
:
4442 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4443 realm_id
, realm_name
, realm
);
4445 std::cerr
<< "failed to load realm: " << cpp_strerror(ret
) << std::endl
;
4449 formatter
->open_object_section("period_get_current");
4450 encode_json("current_period", realm
.current_period
, formatter
.get());
4451 formatter
->close_section();
4452 formatter
->flush(cout
);
4455 case OPT::PERIOD_LIST
:
4457 Formatter::ObjectSection periods_list
{*formatter
, "periods_list"};
4458 Formatter::ArraySection periods
{*formatter
, "periods"};
4459 rgw::sal::ListResult
<std::string
> listing
;
4460 std::array
<std::string
, 1000> period_ids
; // list in pages of 1000
4462 int ret
= cfgstore
->list_period_ids(dpp(), null_yield
, listing
.next
,
4463 period_ids
, listing
);
4465 std::cerr
<< "failed to list periods: " << cpp_strerror(-ret
) << std::endl
;
4468 for (const auto& id
: listing
.entries
) {
4469 encode_json("id", id
, formatter
.get());
4471 } while (!listing
.next
.empty());
4472 } // close sections periods and periods_list
4473 formatter
->flush(cout
);
4475 case OPT::PERIOD_UPDATE
:
4477 int ret
= update_period(cfgstore
.get(), realm_id
, realm_name
,
4478 period_epoch
, commit
, remote
, url
,
4479 opt_region
, access_key
, secret_key
,
4480 formatter
.get(), yes_i_really_mean_it
);
4486 case OPT::PERIOD_PULL
:
4488 boost::optional
<RGWRESTConn
> conn
;
4489 RGWRESTConn
*remote_conn
= nullptr;
4491 // load current period for endpoints
4493 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4494 realm_id
, realm_name
, realm
);
4496 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4499 period_id
= realm
.current_period
;
4501 RGWPeriod current_period
;
4502 ret
= cfgstore
->read_period(dpp(), null_yield
, period_id
,
4503 std::nullopt
, current_period
);
4505 cerr
<< "failed to load current period: " << cpp_strerror(-ret
) << std::endl
;
4508 if (remote
.empty()) {
4509 // use realm master zone as remote
4510 remote
= current_period
.get_master_zone().id
;
4512 conn
= get_remote_conn(static_cast<rgw::sal::RadosStore
*>(driver
), current_period
.get_map(), remote
);
4514 cerr
<< "failed to find a zone or zonegroup for remote "
4515 << remote
<< std::endl
;
4518 remote_conn
= &*conn
;
4522 int ret
= do_period_pull(cfgstore
.get(), remote_conn
, url
,
4523 opt_region
, access_key
, secret_key
,
4524 realm_id
, realm_name
, period_id
, period_epoch
,
4527 cerr
<< "period pull failed: " << cpp_strerror(-ret
) << std::endl
;
4531 encode_json("period", period
, formatter
.get());
4532 formatter
->flush(cout
);
4535 case OPT::GLOBAL_RATELIMIT_GET
:
4536 case OPT::GLOBAL_RATELIMIT_SET
:
4537 case OPT::GLOBAL_RATELIMIT_ENABLE
:
4538 case OPT::GLOBAL_RATELIMIT_DISABLE
:
4540 if (realm_id
.empty()) {
4541 if (!realm_name
.empty()) {
4542 // look up realm_id for the given realm_name
4543 int ret
= cfgstore
->read_realm_id(dpp(), null_yield
,
4544 realm_name
, realm_id
);
4546 cerr
<< "ERROR: failed to read realm for " << realm_name
4547 << ": " << cpp_strerror(-ret
) << std::endl
;
4551 // use default realm_id when none is given
4552 int ret
= cfgstore
->read_default_realm_id(dpp(), null_yield
,
4554 if (ret
< 0 && ret
!= -ENOENT
) { // on ENOENT, use empty realm_id
4555 cerr
<< "ERROR: failed to read default realm: "
4556 << cpp_strerror(-ret
) << std::endl
;
4562 RGWPeriodConfig period_config
;
4563 int ret
= cfgstore
->read_period_config(dpp(), null_yield
, realm_id
,
4565 if (ret
< 0 && ret
!= -ENOENT
) {
4566 cerr
<< "ERROR: failed to read period config: "
4567 << cpp_strerror(-ret
) << std::endl
;
4570 bool ratelimit_configured
= true;
4571 formatter
->open_object_section("period_config");
4572 if (ratelimit_scope
== "bucket") {
4573 ratelimit_configured
= set_ratelimit_info(period_config
.bucket_ratelimit
, opt_cmd
,
4574 max_read_ops
, max_write_ops
,
4575 max_read_bytes
, max_write_bytes
,
4576 have_max_read_ops
, have_max_write_ops
,
4577 have_max_read_bytes
, have_max_write_bytes
);
4578 encode_json("bucket_ratelimit", period_config
.bucket_ratelimit
, formatter
.get());
4579 } else if (ratelimit_scope
== "user") {
4580 ratelimit_configured
= set_ratelimit_info(period_config
.user_ratelimit
, opt_cmd
,
4581 max_read_ops
, max_write_ops
,
4582 max_read_bytes
, max_write_bytes
,
4583 have_max_read_ops
, have_max_write_ops
,
4584 have_max_read_bytes
, have_max_write_bytes
);
4585 encode_json("user_ratelimit", period_config
.user_ratelimit
, formatter
.get());
4586 } else if (ratelimit_scope
== "anonymous") {
4587 ratelimit_configured
= set_ratelimit_info(period_config
.anon_ratelimit
, opt_cmd
,
4588 max_read_ops
, max_write_ops
,
4589 max_read_bytes
, max_write_bytes
,
4590 have_max_read_ops
, have_max_write_ops
,
4591 have_max_read_bytes
, have_max_write_bytes
);
4592 encode_json("anonymous_ratelimit", period_config
.anon_ratelimit
, formatter
.get());
4593 } else if (ratelimit_scope
.empty() && opt_cmd
== OPT::GLOBAL_RATELIMIT_GET
) {
4594 // if no scope is given for GET, print both
4595 encode_json("bucket_ratelimit", period_config
.bucket_ratelimit
, formatter
.get());
4596 encode_json("user_ratelimit", period_config
.user_ratelimit
, formatter
.get());
4597 encode_json("anonymous_ratelimit", period_config
.anon_ratelimit
, formatter
.get());
4599 cerr
<< "ERROR: invalid rate limit scope specification. Please specify "
4600 "either --ratelimit-scope=bucket, or --ratelimit-scope=user or --ratelimit-scope=anonymous" << std::endl
;
4603 if (!ratelimit_configured
) {
4604 cerr
<< "ERROR: no rate limit values have been specified" << std::endl
;
4608 formatter
->close_section();
4610 if (opt_cmd
!= OPT::GLOBAL_RATELIMIT_GET
) {
4611 // write the modified period config
4612 constexpr bool exclusive
= false;
4613 ret
= cfgstore
->write_period_config(dpp(), null_yield
, exclusive
,
4614 realm_id
, period_config
);
4616 cerr
<< "ERROR: failed to write period config: "
4617 << cpp_strerror(-ret
) << std::endl
;
4620 if (!realm_id
.empty()) {
4621 cout
<< "Global ratelimit changes saved. Use 'period update' to apply "
4622 "them to the staging period, and 'period commit' to commit the "
4623 "new period." << std::endl
;
4625 cout
<< "Global ratelimit changes saved. They will take effect as "
4626 "the gateways are restarted." << std::endl
;
4630 formatter
->flush(cout
);
4633 case OPT::GLOBAL_QUOTA_GET
:
4634 case OPT::GLOBAL_QUOTA_SET
:
4635 case OPT::GLOBAL_QUOTA_ENABLE
:
4636 case OPT::GLOBAL_QUOTA_DISABLE
:
4638 if (realm_id
.empty()) {
4639 if (!realm_name
.empty()) {
4640 // look up realm_id for the given realm_name
4641 int ret
= cfgstore
->read_realm_id(dpp(), null_yield
,
4642 realm_name
, realm_id
);
4644 cerr
<< "ERROR: failed to read realm for " << realm_name
4645 << ": " << cpp_strerror(-ret
) << std::endl
;
4649 // use default realm_id when none is given
4650 int ret
= cfgstore
->read_default_realm_id(dpp(), null_yield
,
4652 if (ret
< 0 && ret
!= -ENOENT
) { // on ENOENT, use empty realm_id
4653 cerr
<< "ERROR: failed to read default realm: "
4654 << cpp_strerror(-ret
) << std::endl
;
4660 RGWPeriodConfig period_config
;
4661 int ret
= cfgstore
->read_period_config(dpp(), null_yield
, realm_id
,
4663 if (ret
< 0 && ret
!= -ENOENT
) {
4664 cerr
<< "ERROR: failed to read period config: "
4665 << cpp_strerror(-ret
) << std::endl
;
4669 formatter
->open_object_section("period_config");
4670 if (quota_scope
== "bucket") {
4671 set_quota_info(period_config
.quota
.bucket_quota
, opt_cmd
,
4672 max_size
, max_objects
,
4673 have_max_size
, have_max_objects
);
4674 encode_json("bucket quota", period_config
.quota
.bucket_quota
, formatter
.get());
4675 } else if (quota_scope
== "user") {
4676 set_quota_info(period_config
.quota
.user_quota
, opt_cmd
,
4677 max_size
, max_objects
,
4678 have_max_size
, have_max_objects
);
4679 encode_json("user quota", period_config
.quota
.user_quota
, formatter
.get());
4680 } else if (quota_scope
.empty() && opt_cmd
== OPT::GLOBAL_QUOTA_GET
) {
4681 // if no scope is given for GET, print both
4682 encode_json("bucket quota", period_config
.quota
.bucket_quota
, formatter
.get());
4683 encode_json("user quota", period_config
.quota
.user_quota
, formatter
.get());
4685 cerr
<< "ERROR: invalid quota scope specification. Please specify "
4686 "either --quota-scope=bucket, or --quota-scope=user" << std::endl
;
4689 formatter
->close_section();
4691 if (opt_cmd
!= OPT::GLOBAL_QUOTA_GET
) {
4692 // write the modified period config
4693 constexpr bool exclusive
= false;
4694 ret
= cfgstore
->write_period_config(dpp(), null_yield
, exclusive
,
4695 realm_id
, period_config
);
4697 cerr
<< "ERROR: failed to write period config: "
4698 << cpp_strerror(-ret
) << std::endl
;
4701 if (!realm_id
.empty()) {
4702 cout
<< "Global quota changes saved. Use 'period update' to apply "
4703 "them to the staging period, and 'period commit' to commit the "
4704 "new period." << std::endl
;
4706 cout
<< "Global quota changes saved. They will take effect as "
4707 "the gateways are restarted." << std::endl
;
4711 formatter
->flush(cout
);
4714 case OPT::REALM_CREATE
:
4716 if (realm_name
.empty()) {
4717 cerr
<< "missing realm name" << std::endl
;
4722 realm
.name
= realm_name
;
4724 constexpr bool exclusive
= true;
4725 int ret
= rgw::create_realm(dpp(), null_yield
, cfgstore
.get(),
4728 cerr
<< "ERROR: couldn't create realm " << realm_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
4733 ret
= rgw::set_default_realm(dpp(), null_yield
, cfgstore
.get(), realm
);
4735 cerr
<< "failed to set realm " << realm_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
4739 encode_json("realm", realm
, formatter
.get());
4740 formatter
->flush(cout
);
4743 case OPT::REALM_DELETE
:
4745 if (realm_id
.empty() && realm_name
.empty()) {
4746 cerr
<< "missing realm name or id" << std::endl
;
4750 std::unique_ptr
<rgw::sal::RealmWriter
> writer
;
4751 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4752 realm_id
, realm_name
, realm
, &writer
);
4754 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4757 ret
= writer
->remove(dpp(), null_yield
);
4759 cerr
<< "failed to remove realm: " << cpp_strerror(-ret
) << std::endl
;
4765 case OPT::REALM_GET
:
4768 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4769 realm_id
, realm_name
, realm
);
4771 if (ret
== -ENOENT
&& realm_name
.empty() && realm_id
.empty()) {
4772 cerr
<< "missing realm name or id, or default realm not found" << std::endl
;
4774 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4778 encode_json("realm", realm
, formatter
.get());
4779 formatter
->flush(cout
);
4782 case OPT::REALM_GET_DEFAULT
:
4785 int ret
= cfgstore
->read_default_realm_id(dpp(), null_yield
, default_id
);
4786 if (ret
== -ENOENT
) {
4787 cout
<< "No default realm is set" << std::endl
;
4789 } else if (ret
< 0) {
4790 cerr
<< "Error reading default realm: " << cpp_strerror(-ret
) << std::endl
;
4793 cout
<< "default realm: " << default_id
<< std::endl
;
4796 case OPT::REALM_LIST
:
4798 std::string default_id
;
4799 int ret
= cfgstore
->read_default_realm_id(dpp(), null_yield
,
4801 if (ret
< 0 && ret
!= -ENOENT
) {
4802 cerr
<< "could not determine default realm: " << cpp_strerror(-ret
) << std::endl
;
4805 Formatter::ObjectSection realms_list
{*formatter
, "realms_list"};
4806 encode_json("default_info", default_id
, formatter
.get());
4808 Formatter::ArraySection realms
{*formatter
, "realms"};
4809 rgw::sal::ListResult
<std::string
> listing
;
4810 std::array
<std::string
, 1000> names
; // list in pages of 1000
4812 ret
= cfgstore
->list_realm_names(dpp(), null_yield
, listing
.next
,
4815 std::cerr
<< "failed to list realms: " << cpp_strerror(-ret
) << std::endl
;
4818 for (const auto& name
: listing
.entries
) {
4819 encode_json("name", name
, formatter
.get());
4821 } while (!listing
.next
.empty());
4822 } // close sections realms and realms_list
4823 formatter
->flush(cout
);
4825 case OPT::REALM_LIST_PERIODS
:
4827 // use realm's current period
4829 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4830 realm_id
, realm_name
, realm
);
4832 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4835 period_id
= realm
.current_period
;
4837 Formatter::ObjectSection periods_list
{*formatter
, "realm_periods_list"};
4838 encode_json("current_period", period_id
, formatter
.get());
4840 Formatter::ArraySection periods
{*formatter
, "periods"};
4842 while (!period_id
.empty()) {
4844 ret
= cfgstore
->read_period(dpp(), null_yield
, period_id
,
4845 std::nullopt
, period
);
4847 cerr
<< "failed to load period id " << period_id
4848 << ": " << cpp_strerror(-ret
) << std::endl
;
4851 encode_json("id", period_id
, formatter
.get());
4852 period_id
= period
.predecessor_uuid
;
4854 } // close sections periods and realm_periods_list
4855 formatter
->flush(cout
);
4858 case OPT::REALM_RENAME
:
4860 if (realm_new_name
.empty()) {
4861 cerr
<< "missing realm new name" << std::endl
;
4864 if (realm_name
.empty() && realm_id
.empty()) {
4865 cerr
<< "missing realm name or id" << std::endl
;
4870 std::unique_ptr
<rgw::sal::RealmWriter
> writer
;
4871 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4872 realm_id
, realm_name
, realm
, &writer
);
4874 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4877 ret
= writer
->rename(dpp(), null_yield
, realm
, realm_new_name
);
4879 cerr
<< "rename failed: " << cpp_strerror(-ret
) << std::endl
;
4882 cout
<< "Realm name updated. Note that this change only applies to "
4883 "the current cluster, so this command must be run separately "
4884 "on each of the realm's other clusters." << std::endl
;
4887 case OPT::REALM_SET
:
4889 if (realm_id
.empty() && realm_name
.empty()) {
4890 cerr
<< "no realm name or id provided" << std::endl
;
4893 bool new_realm
= false;
4895 std::unique_ptr
<rgw::sal::RealmWriter
> writer
;
4896 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4897 realm_id
, realm_name
, realm
, &writer
);
4898 if (ret
< 0 && ret
!= -ENOENT
) {
4899 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
4901 } else if (ret
== -ENOENT
) {
4904 ret
= read_decode_json(infile
, realm
);
4908 if (!realm_name
.empty() && realm
.get_name() != realm_name
) {
4909 cerr
<< "mismatch between --rgw-realm " << realm_name
<< " and json input file name " <<
4910 realm
.get_name() << std::endl
;
4915 cout
<< "clearing period and epoch for new realm" << std::endl
;
4916 realm
.clear_current_period_and_epoch();
4917 constexpr bool exclusive
= true;
4918 ret
= rgw::create_realm(dpp(), null_yield
, cfgstore
.get(),
4921 cerr
<< "ERROR: couldn't create new realm: " << cpp_strerror(-ret
) << std::endl
;
4925 ret
= writer
->write(dpp(), null_yield
, realm
);
4927 cerr
<< "ERROR: couldn't driver realm info: " << cpp_strerror(-ret
) << std::endl
;
4933 ret
= rgw::set_default_realm(dpp(), null_yield
, cfgstore
.get(), realm
);
4935 cerr
<< "failed to set realm " << realm_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
4938 encode_json("realm", realm
, formatter
.get());
4939 formatter
->flush(cout
);
4943 case OPT::REALM_DEFAULT
:
4946 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
4947 realm_id
, realm_name
, realm
);
4949 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
4952 ret
= rgw::set_default_realm(dpp(), null_yield
, cfgstore
.get(), realm
);
4954 cerr
<< "failed to set realm as default: " << cpp_strerror(-ret
) << std::endl
;
4959 case OPT::REALM_PULL
:
4962 cerr
<< "A --url must be provided." << std::endl
;
4966 req_info
info(g_ceph_context
, &env
);
4967 info
.method
= "GET";
4968 info
.request_uri
= "/admin/realm";
4970 map
<string
, string
> ¶ms
= info
.args
.get_params();
4971 if (!realm_id
.empty())
4972 params
["id"] = realm_id
;
4973 if (!realm_name
.empty())
4974 params
["name"] = realm_name
;
4978 int ret
= send_to_url(url
, opt_region
, access_key
, secret_key
, info
, bl
, p
);
4980 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
4981 if (ret
== -EACCES
) {
4982 cerr
<< "If the realm has been changed on the master zone, the "
4983 "master zone's gateway may need to be restarted to recognize "
4984 "this user." << std::endl
;
4990 decode_json_obj(realm
, &p
);
4991 } catch (const JSONDecoder::err
& e
) {
4992 cerr
<< "failed to decode JSON response: " << e
.what() << std::endl
;
4996 auto& current_period
= realm
.get_current_period();
4997 if (!current_period
.empty()) {
4998 // pull the latest epoch of the realm's current period
4999 ret
= do_period_pull(cfgstore
.get(), nullptr, url
, opt_region
,
5000 access_key
, secret_key
,
5001 realm_id
, realm_name
, current_period
, "",
5004 cerr
<< "could not fetch period " << current_period
<< std::endl
;
5008 constexpr bool exclusive
= false;
5009 ret
= rgw::create_realm(dpp(), null_yield
, cfgstore
.get(),
5012 cerr
<< "Error storing realm " << realm
.get_id() << ": "
5013 << cpp_strerror(ret
) << std::endl
;
5018 ret
= rgw::set_default_realm(dpp(), null_yield
, cfgstore
.get(), realm
);
5020 cerr
<< "failed to set realm " << realm_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5024 encode_json("realm", realm
, formatter
.get());
5025 formatter
->flush(cout
);
5029 case OPT::ZONEGROUP_ADD
:
5031 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
5032 cerr
<< "no zonegroup name or id provided" << std::endl
;
5036 // load the zonegroup and zone params
5037 RGWZoneGroup zonegroup
;
5038 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> zonegroup_writer
;
5039 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5040 zonegroup_id
, zonegroup_name
,
5041 zonegroup
, &zonegroup_writer
);
5043 cerr
<< "failed to load zonegroup " << zonegroup_name
<< " id "
5044 << zonegroup_id
<< ": " << cpp_strerror(-ret
) << std::endl
;
5048 RGWZoneParams zone_params
;
5049 std::unique_ptr
<rgw::sal::ZoneWriter
> zone_writer
;
5050 ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5051 zone_id
, zone_name
, zone_params
, &zone_writer
);
5053 cerr
<< "unable to load zone: " << cpp_strerror(-ret
) << std::endl
;
5057 // update zone_params if necessary
5058 bool need_zone_update
= false;
5060 if (zone_params
.realm_id
!= zonegroup
.realm_id
) {
5061 if (!zone_params
.realm_id
.empty()) {
5062 cerr
<< "WARNING: overwriting zone realm_id=" << zone_params
.realm_id
5063 << " to match zonegroup realm_id=" << zonegroup
.realm_id
<< std::endl
;
5065 zone_params
.realm_id
= zonegroup
.realm_id
;
5066 need_zone_update
= true;
5069 for (auto a
: tier_config_add
) {
5070 ret
= zone_params
.tier_config
.set(a
.first
, a
.second
);
5072 cerr
<< "ERROR: failed to set configurable: " << a
<< std::endl
;
5075 need_zone_update
= true;
5078 if (need_zone_update
) {
5079 ret
= zone_writer
->write(dpp(), null_yield
, zone_params
);
5081 cerr
<< "failed to save zone info: " << cpp_strerror(-ret
) << std::endl
;
5086 const bool *pis_master
= (is_master_set
? &is_master
: nullptr);
5087 const bool *pread_only
= (is_read_only_set
? &read_only
: nullptr);
5088 const bool *psync_from_all
= (sync_from_all_specified
? &sync_from_all
: nullptr);
5089 const string
*predirect_zone
= (redirect_zone_set
? &redirect_zone
: nullptr);
5091 // validate --tier-type if specified
5092 const string
*ptier_type
= (tier_type_specified
? &tier_type
: nullptr);
5094 auto sync_mgr
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->sync_modules
->get_manager();
5095 if (!sync_mgr
->get_module(*ptier_type
, nullptr)) {
5096 ldpp_dout(dpp(), -1) << "ERROR: could not find sync module: "
5097 << *ptier_type
<< ", valid sync modules: "
5098 << sync_mgr
->get_registered_module_names() << dendl
;
5103 if (enable_features
.empty()) { // enable all features by default
5104 enable_features
.insert(rgw::zone_features::supported
.begin(),
5105 rgw::zone_features::supported
.end());
5108 // add/update the public zone information stored in the zonegroup
5109 ret
= rgw::add_zone_to_group(dpp(), zonegroup
, zone_params
,
5110 pis_master
, pread_only
, endpoints
,
5111 ptier_type
, psync_from_all
,
5112 sync_from
, sync_from_rm
,
5113 predirect_zone
, bucket_index_max_shards
,
5114 enable_features
, disable_features
);
5119 // write the updated zonegroup
5120 ret
= zonegroup_writer
->write(dpp(), null_yield
, zonegroup
);
5122 cerr
<< "failed to write updated zonegroup " << zonegroup
.get_name()
5123 << ": " << cpp_strerror(-ret
) << std::endl
;
5127 encode_json("zonegroup", zonegroup
, formatter
.get());
5128 formatter
->flush(cout
);
5131 case OPT::ZONEGROUP_CREATE
:
5133 if (zonegroup_name
.empty()) {
5134 cerr
<< "Missing zonegroup name" << std::endl
;
5138 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
5139 realm_id
, realm_name
, realm
);
5141 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
5145 RGWZoneGroup zonegroup
;
5146 zonegroup
.name
= zonegroup_name
;
5147 zonegroup
.is_master
= is_master
;
5148 zonegroup
.realm_id
= realm
.get_id();
5149 zonegroup
.endpoints
= endpoints
;
5150 zonegroup
.api_name
= (api_name
.empty() ? zonegroup_name
: api_name
);
5152 zonegroup
.enabled_features
= enable_features
;
5153 if (zonegroup
.enabled_features
.empty()) { // enable all features by default
5154 zonegroup
.enabled_features
.insert(rgw::zone_features::supported
.begin(),
5155 rgw::zone_features::supported
.end());
5157 for (const auto& feature
: disable_features
) {
5158 auto i
= zonegroup
.enabled_features
.find(feature
);
5159 if (i
== zonegroup
.enabled_features
.end()) {
5160 ldout(cct
, 1) << "WARNING: zone feature \"" << feature
5161 << "\" was not enabled in zonegroup " << zonegroup_name
<< dendl
;
5164 zonegroup
.enabled_features
.erase(i
);
5167 constexpr bool exclusive
= true;
5168 ret
= rgw::create_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5169 exclusive
, zonegroup
);
5171 cerr
<< "failed to create zonegroup " << zonegroup_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
5176 ret
= rgw::set_default_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5179 cerr
<< "failed to set zonegroup " << zonegroup_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5183 encode_json("zonegroup", zonegroup
, formatter
.get());
5184 formatter
->flush(cout
);
5187 case OPT::ZONEGROUP_DEFAULT
:
5189 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
5190 cerr
<< "no zonegroup name or id provided" << std::endl
;
5194 RGWZoneGroup zonegroup
;
5195 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5196 zonegroup_id
, zonegroup_name
,
5199 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5203 ret
= rgw::set_default_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5206 cerr
<< "failed to set zonegroup as default: " << cpp_strerror(-ret
) << std::endl
;
5211 case OPT::ZONEGROUP_DELETE
:
5213 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
5214 cerr
<< "no zonegroup name or id provided" << std::endl
;
5217 RGWZoneGroup zonegroup
;
5218 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> writer
;
5219 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5220 zonegroup_id
, zonegroup_name
,
5221 zonegroup
, &writer
);
5223 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5226 ret
= writer
->remove(dpp(), null_yield
);
5228 cerr
<< "ERROR: couldn't delete zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5233 case OPT::ZONEGROUP_GET
:
5235 RGWZoneGroup zonegroup
;
5236 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5237 zonegroup_id
, zonegroup_name
, zonegroup
);
5239 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5243 encode_json("zonegroup", zonegroup
, formatter
.get());
5244 formatter
->flush(cout
);
5247 case OPT::ZONEGROUP_LIST
:
5249 RGWZoneGroup default_zonegroup
;
5250 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5251 {}, {}, default_zonegroup
);
5252 if (ret
< 0 && ret
!= -ENOENT
) {
5253 cerr
<< "could not determine default zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5256 Formatter::ObjectSection zonegroups_list
{*formatter
, "zonegroups_list"};
5257 encode_json("default_info", default_zonegroup
.id
, formatter
.get());
5259 Formatter::ArraySection zonegroups
{*formatter
, "zonegroups"};
5260 rgw::sal::ListResult
<std::string
> listing
;
5261 std::array
<std::string
, 1000> names
; // list in pages of 1000
5263 ret
= cfgstore
->list_zonegroup_names(dpp(), null_yield
, listing
.next
,
5266 std::cerr
<< "failed to list zonegroups: " << cpp_strerror(-ret
) << std::endl
;
5269 for (const auto& name
: listing
.entries
) {
5270 encode_json("name", name
, formatter
.get());
5272 } while (!listing
.next
.empty());
5273 } // close sections zonegroups and zonegroups_list
5274 formatter
->flush(cout
);
5276 case OPT::ZONEGROUP_MODIFY
:
5278 RGWZoneGroup zonegroup
;
5279 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> writer
;
5280 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5281 zonegroup_id
, zonegroup_name
,
5282 zonegroup
, &writer
);
5284 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5288 bool need_update
= false;
5290 if (!master_zone
.empty()) {
5291 zonegroup
.master_zone
= master_zone
;
5295 if (is_master_set
) {
5296 zonegroup
.is_master
= is_master
;
5300 if (!endpoints
.empty()) {
5301 zonegroup
.endpoints
= endpoints
;
5305 if (!api_name
.empty()) {
5306 zonegroup
.api_name
= api_name
;
5310 if (!realm_id
.empty()) {
5311 zonegroup
.realm_id
= realm_id
;
5313 } else if (!realm_name
.empty()) {
5314 // get realm id from name
5315 ret
= cfgstore
->read_realm_id(dpp(), null_yield
, realm_name
,
5316 zonegroup
.realm_id
);
5318 cerr
<< "failed to find realm by name " << realm_name
<< std::endl
;
5324 if (bucket_index_max_shards
) {
5325 for (auto& [name
, zone
] : zonegroup
.zones
) {
5326 zone
.bucket_index_max_shards
= *bucket_index_max_shards
;
5331 for (const auto& feature
: enable_features
) {
5332 zonegroup
.enabled_features
.insert(feature
);
5335 for (const auto& feature
: disable_features
) {
5336 auto i
= zonegroup
.enabled_features
.find(feature
);
5337 if (i
== zonegroup
.enabled_features
.end()) {
5338 ldout(cct
, 1) << "WARNING: zone feature \"" << feature
5339 << "\" was not enabled in zonegroup "
5340 << zonegroup
.get_name() << dendl
;
5343 zonegroup
.enabled_features
.erase(i
);
5348 ret
= writer
->write(dpp(), null_yield
, zonegroup
);
5350 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5356 ret
= rgw::set_default_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5359 cerr
<< "failed to set zonegroup " << zonegroup_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5363 encode_json("zonegroup", zonegroup
, formatter
.get());
5364 formatter
->flush(cout
);
5367 case OPT::ZONEGROUP_SET
:
5370 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
5371 realm_id
, realm_name
, realm
);
5372 bool default_realm_not_exist
= (ret
== -ENOENT
&& realm_id
.empty() && realm_name
.empty());
5374 if (ret
< 0 && !default_realm_not_exist
) {
5375 cerr
<< "failed to init realm: " << cpp_strerror(-ret
) << std::endl
;
5379 RGWZoneGroup zonegroup
;
5380 ret
= read_decode_json(infile
, zonegroup
);
5384 if (zonegroup
.realm_id
.empty() && !default_realm_not_exist
) {
5385 zonegroup
.realm_id
= realm
.get_id();
5387 // validate zonegroup features
5388 for (const auto& feature
: zonegroup
.enabled_features
) {
5389 if (!rgw::zone_features::supports(feature
)) {
5390 std::cerr
<< "ERROR: Unrecognized zonegroup feature \""
5391 << feature
<< "\"" << std::endl
;
5395 for (const auto& [name
, zone
] : zonegroup
.zones
) {
5396 // validate zone features
5397 for (const auto& feature
: zone
.supported_features
) {
5398 if (!rgw::zone_features::supports(feature
)) {
5399 std::cerr
<< "ERROR: Unrecognized zone feature \""
5400 << feature
<< "\" in zone " << zone
.name
<< std::endl
;
5404 // zone must support everything zonegroup does
5405 for (const auto& feature
: zonegroup
.enabled_features
) {
5406 if (!zone
.supports(feature
)) {
5407 std::cerr
<< "ERROR: Zone " << name
<< " does not support feature \""
5408 << feature
<< "\" required by zonegroup" << std::endl
;
5414 // create/overwrite the zonegroup info
5415 constexpr bool exclusive
= false;
5416 ret
= rgw::create_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5417 exclusive
, zonegroup
);
5419 cerr
<< "ERROR: couldn't create zonegroup info: " << cpp_strerror(-ret
) << std::endl
;
5424 ret
= rgw::set_default_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5427 cerr
<< "failed to set zonegroup " << zonegroup_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5431 encode_json("zonegroup", zonegroup
, formatter
.get());
5432 formatter
->flush(cout
);
5435 case OPT::ZONEGROUP_REMOVE
:
5437 RGWZoneGroup zonegroup
;
5438 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> writer
;
5439 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5440 zonegroup_id
, zonegroup_name
,
5441 zonegroup
, &writer
);
5443 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5447 if (zone_id
.empty()) {
5448 if (zone_name
.empty()) {
5449 cerr
<< "no --zone-id or --rgw-zone name provided" << std::endl
;
5452 // look up zone id by name
5453 for (auto& z
: zonegroup
.zones
) {
5454 if (zone_name
== z
.second
.name
) {
5455 zone_id
= z
.second
.id
;
5459 if (zone_id
.empty()) {
5460 cerr
<< "zone name " << zone_name
<< " not found in zonegroup "
5461 << zonegroup
.get_name() << std::endl
;
5466 ret
= rgw::remove_zone_from_group(dpp(), zonegroup
, zone_id
);
5468 cerr
<< "failed to remove zone: " << cpp_strerror(-ret
) << std::endl
;
5472 ret
= writer
->write(dpp(), null_yield
, zonegroup
);
5474 cerr
<< "failed to write zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5478 encode_json("zonegroup", zonegroup
, formatter
.get());
5479 formatter
->flush(cout
);
5482 case OPT::ZONEGROUP_RENAME
:
5484 if (zonegroup_new_name
.empty()) {
5485 cerr
<< " missing zonegroup new name" << std::endl
;
5488 if (zonegroup_id
.empty() && zonegroup_name
.empty()) {
5489 cerr
<< "no zonegroup name or id provided" << std::endl
;
5492 RGWZoneGroup zonegroup
;
5493 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> writer
;
5494 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5495 zonegroup_id
, zonegroup_name
,
5496 zonegroup
, &writer
);
5498 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5501 ret
= writer
->rename(dpp(), null_yield
, zonegroup
, zonegroup_new_name
);
5503 cerr
<< "failed to rename zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5508 case OPT::ZONEGROUP_PLACEMENT_LIST
:
5510 RGWZoneGroup zonegroup
;
5511 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5512 zonegroup_id
, zonegroup_name
, zonegroup
);
5514 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5518 encode_json("placement_targets", zonegroup
.placement_targets
, formatter
.get());
5519 formatter
->flush(cout
);
5522 case OPT::ZONEGROUP_PLACEMENT_GET
:
5524 if (placement_id
.empty()) {
5525 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
5529 RGWZoneGroup zonegroup
;
5530 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5531 zonegroup_id
, zonegroup_name
, zonegroup
);
5533 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5537 auto p
= zonegroup
.placement_targets
.find(placement_id
);
5538 if (p
== zonegroup
.placement_targets
.end()) {
5539 cerr
<< "failed to find a zonegroup placement target named '" << placement_id
<< "'" << std::endl
;
5542 encode_json("placement_targets", p
->second
, formatter
.get());
5543 formatter
->flush(cout
);
5546 case OPT::ZONEGROUP_PLACEMENT_ADD
:
5547 case OPT::ZONEGROUP_PLACEMENT_MODIFY
:
5548 case OPT::ZONEGROUP_PLACEMENT_RM
:
5549 case OPT::ZONEGROUP_PLACEMENT_DEFAULT
:
5551 if (placement_id
.empty()) {
5552 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
5556 rgw_placement_rule rule
;
5557 rule
.from_str(placement_id
);
5559 if (!rule
.storage_class
.empty() && opt_storage_class
&&
5560 rule
.storage_class
!= *opt_storage_class
) {
5561 cerr
<< "ERROR: provided contradicting storage class configuration" << std::endl
;
5563 } else if (rule
.storage_class
.empty()) {
5564 rule
.storage_class
= opt_storage_class
.value_or(string());
5567 RGWZoneGroup zonegroup
;
5568 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> writer
;
5569 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5570 zonegroup_id
, zonegroup_name
,
5571 zonegroup
, &writer
);
5573 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5577 if (opt_cmd
== OPT::ZONEGROUP_PLACEMENT_ADD
||
5578 opt_cmd
== OPT::ZONEGROUP_PLACEMENT_MODIFY
) {
5579 RGWZoneGroupPlacementTarget
& target
= zonegroup
.placement_targets
[placement_id
];
5580 if (!tags
.empty()) {
5581 target
.tags
.clear();
5582 for (auto& t
: tags
) {
5583 target
.tags
.insert(t
);
5587 target
.name
= placement_id
;
5588 for (auto& t
: tags_rm
) {
5589 target
.tags
.erase(t
);
5591 for (auto& t
: tags_add
) {
5592 target
.tags
.insert(t
);
5594 target
.storage_classes
.insert(rule
.get_storage_class());
5597 bool tier_class
= false;
5598 std::string storage_class
= rule
.get_storage_class();
5599 RGWZoneGroupPlacementTier t
{storage_class
};
5600 RGWZoneGroupPlacementTier
*pt
= &t
;
5602 auto ptiter
= target
.tier_targets
.find(storage_class
);
5603 if (ptiter
!= target
.tier_targets
.end()) {
5604 pt
= &ptiter
->second
;
5606 } else if (tier_type_specified
) {
5607 if (tier_type
== "cloud-s3") {
5608 /* we support only cloud-s3 tier-type for now.
5609 * Once set cant be reset. */
5611 pt
->tier_type
= tier_type
;
5612 pt
->storage_class
= storage_class
;
5614 cerr
<< "ERROR: Invalid tier-type specified" << std::endl
;
5620 if (tier_config_add
.size() > 0) {
5621 JSONFormattable tconfig
;
5622 for (auto add
: tier_config_add
) {
5623 int r
= tconfig
.set(add
.first
, add
.second
);
5625 cerr
<< "ERROR: failed to set configurable: " << add
<< std::endl
;
5629 int r
= pt
->update_params(tconfig
);
5631 cerr
<< "ERROR: failed to update tier_config options"<< std::endl
;
5634 if (tier_config_rm
.size() > 0) {
5635 JSONFormattable tconfig
;
5636 for (auto add
: tier_config_rm
) {
5637 int r
= tconfig
.set(add
.first
, add
.second
);
5639 cerr
<< "ERROR: failed to set configurable: " << add
<< std::endl
;
5643 int r
= pt
->clear_params(tconfig
);
5645 cerr
<< "ERROR: failed to update tier_config options"<< std::endl
;
5649 target
.tier_targets
.emplace(std::make_pair(storage_class
, *pt
));
5652 if (zonegroup
.default_placement
.empty()) {
5653 zonegroup
.default_placement
.init(rule
.name
, RGW_STORAGE_CLASS_STANDARD
);
5655 } else if (opt_cmd
== OPT::ZONEGROUP_PLACEMENT_RM
) {
5656 if (!opt_storage_class
|| opt_storage_class
->empty()) {
5657 zonegroup
.placement_targets
.erase(placement_id
);
5658 if (zonegroup
.default_placement
.name
== placement_id
) {
5659 // clear default placement
5660 zonegroup
.default_placement
.clear();
5663 auto iter
= zonegroup
.placement_targets
.find(placement_id
);
5664 if (iter
!= zonegroup
.placement_targets
.end()) {
5665 RGWZoneGroupPlacementTarget
& info
= zonegroup
.placement_targets
[placement_id
];
5666 info
.storage_classes
.erase(*opt_storage_class
);
5668 if (zonegroup
.default_placement
== rule
) {
5669 // clear default storage class
5670 zonegroup
.default_placement
.storage_class
.clear();
5673 auto ptiter
= info
.tier_targets
.find(*opt_storage_class
);
5674 if (ptiter
!= info
.tier_targets
.end()) {
5675 info
.tier_targets
.erase(ptiter
);
5679 } else if (opt_cmd
== OPT::ZONEGROUP_PLACEMENT_DEFAULT
) {
5680 if (!zonegroup
.placement_targets
.count(placement_id
)) {
5681 cerr
<< "failed to find a zonegroup placement target named '"
5682 << placement_id
<< "'" << std::endl
;
5685 zonegroup
.default_placement
= rule
;
5688 ret
= writer
->write(dpp(), null_yield
, zonegroup
);
5690 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
5694 encode_json("placement_targets", zonegroup
.placement_targets
, formatter
.get());
5695 formatter
->flush(cout
);
5698 case OPT::ZONE_CREATE
:
5700 if (zone_name
.empty()) {
5701 cerr
<< "zone name not provided" << std::endl
;
5705 RGWZoneGroup zonegroup
;
5706 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> zonegroup_writer
;
5707 /* if the user didn't provide zonegroup info , create stand alone zone */
5708 if (!zonegroup_id
.empty() || !zonegroup_name
.empty()) {
5709 int ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
5710 zonegroup_id
, zonegroup_name
,
5711 zonegroup
, &zonegroup_writer
);
5713 cerr
<< "failed to load zonegroup " << zonegroup_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
5716 if (realm_id
.empty() && realm_name
.empty()) {
5717 realm_id
= zonegroup
.realm_id
;
5721 // create the local zone params
5722 RGWZoneParams zone_params
;
5723 zone_params
.id
= zone_id
;
5724 zone_params
.name
= zone_name
;
5726 zone_params
.system_key
.id
= access_key
;
5727 zone_params
.system_key
.key
= secret_key
;
5728 zone_params
.realm_id
= realm_id
;
5729 for (const auto& a
: tier_config_add
) {
5730 int r
= zone_params
.tier_config
.set(a
.first
, a
.second
);
5732 cerr
<< "ERROR: failed to set configurable: " << a
<< std::endl
;
5737 if (zone_params
.realm_id
.empty()) {
5739 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
5740 realm_id
, realm_name
, realm
);
5741 if (ret
< 0 && ret
!= -ENOENT
) {
5742 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
5745 zone_params
.realm_id
= realm
.id
;
5746 cerr
<< "NOTICE: set zone's realm_id=" << realm
.id
<< std::endl
;
5749 constexpr bool exclusive
= true;
5750 int ret
= rgw::create_zone(dpp(), null_yield
, cfgstore
.get(),
5751 exclusive
, zone_params
);
5753 cerr
<< "failed to create zone " << zone_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
5757 if (zonegroup_writer
) {
5758 const bool *pis_master
= (is_master_set
? &is_master
: nullptr);
5759 const bool *pread_only
= (is_read_only_set
? &read_only
: nullptr);
5760 const bool *psync_from_all
= (sync_from_all_specified
? &sync_from_all
: nullptr);
5761 const string
*predirect_zone
= (redirect_zone_set
? &redirect_zone
: nullptr);
5763 // validate --tier-type if specified
5764 const string
*ptier_type
= (tier_type_specified
? &tier_type
: nullptr);
5766 auto sync_mgr
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->sync_modules
->get_manager();
5767 if (!sync_mgr
->get_module(*ptier_type
, nullptr)) {
5768 ldpp_dout(dpp(), -1) << "ERROR: could not find sync module: "
5769 << *ptier_type
<< ", valid sync modules: "
5770 << sync_mgr
->get_registered_module_names() << dendl
;
5775 if (enable_features
.empty()) { // enable all features by default
5776 enable_features
.insert(rgw::zone_features::supported
.begin(),
5777 rgw::zone_features::supported
.end());
5780 // add/update the public zone information stored in the zonegroup
5781 ret
= rgw::add_zone_to_group(dpp(), zonegroup
, zone_params
,
5782 pis_master
, pread_only
, endpoints
,
5783 ptier_type
, psync_from_all
,
5784 sync_from
, sync_from_rm
,
5785 predirect_zone
, bucket_index_max_shards
,
5786 enable_features
, disable_features
);
5791 // write the updated zonegroup
5792 ret
= zonegroup_writer
->write(dpp(), null_yield
, zonegroup
);
5794 cerr
<< "failed to add zone " << zone_name
<< " to zonegroup " << zonegroup
.get_name()
5795 << ": " << cpp_strerror(-ret
) << std::endl
;
5801 ret
= rgw::set_default_zone(dpp(), null_yield
, cfgstore
.get(),
5804 cerr
<< "failed to set zone " << zone_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5808 encode_json("zone", zone_params
, formatter
.get());
5809 formatter
->flush(cout
);
5812 case OPT::ZONE_DEFAULT
:
5814 if (zone_id
.empty() && zone_name
.empty()) {
5815 cerr
<< "no zone name or id provided" << std::endl
;
5818 RGWZoneParams zone_params
;
5819 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5820 zone_id
, zone_name
, zone_params
);
5822 cerr
<< "unable to load zone: " << cpp_strerror(-ret
) << std::endl
;
5826 ret
= rgw::set_default_zone(dpp(), null_yield
, cfgstore
.get(),
5829 cerr
<< "failed to set zone as default: " << cpp_strerror(-ret
) << std::endl
;
5834 case OPT::ZONE_DELETE
:
5836 if (zone_id
.empty() && zone_name
.empty()) {
5837 cerr
<< "no zone name or id provided" << std::endl
;
5840 RGWZoneParams zone_params
;
5841 std::unique_ptr
<rgw::sal::ZoneWriter
> writer
;
5842 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5843 zone_id
, zone_name
, zone_params
, &writer
);
5845 cerr
<< "failed to load zone: " << cpp_strerror(-ret
) << std::endl
;
5849 ret
= rgw::delete_zone(dpp(), null_yield
, cfgstore
.get(),
5850 zone_params
, *writer
);
5852 cerr
<< "failed to delete zone " << zone_params
.get_name()
5853 << ": " << cpp_strerror(-ret
) << std::endl
;
5860 RGWZoneParams zone_params
;
5861 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5862 zone_id
, zone_name
, zone_params
);
5864 cerr
<< "failed to load zone: " << cpp_strerror(-ret
) << std::endl
;
5867 encode_json("zone", zone_params
, formatter
.get());
5868 formatter
->flush(cout
);
5874 std::unique_ptr
<rgw::sal::ZoneWriter
> writer
;
5875 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5876 zone_id
, zone_name
, zone
, &writer
);
5877 if (ret
< 0 && ret
!= -ENOENT
) {
5878 cerr
<< "failed to load zone: " << cpp_strerror(ret
) << std::endl
;
5882 string orig_id
= zone
.get_id();
5884 ret
= read_decode_json(infile
, zone
);
5889 if (zone
.realm_id
.empty()) {
5891 ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
5892 realm_id
, realm_name
, realm
);
5893 if (ret
< 0 && ret
!= -ENOENT
) {
5894 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
5897 zone
.realm_id
= realm
.get_id();
5898 cerr
<< "NOTICE: set zone's realm_id=" << zone
.realm_id
<< std::endl
;
5901 if (!zone_name
.empty() && !zone
.get_name().empty() && zone
.get_name() != zone_name
) {
5902 cerr
<< "Error: zone name " << zone_name
<< " is different than the zone name " << zone
.get_name() << " in the provided json " << std::endl
;
5906 if (zone
.get_name().empty()) {
5907 zone
.set_name(zone_name
);
5908 if (zone
.get_name().empty()) {
5909 cerr
<< "no zone name specified" << std::endl
;
5914 zone_name
= zone
.get_name();
5916 if (zone
.get_id().empty()) {
5917 zone
.set_id(orig_id
);
5920 constexpr bool exclusive
= false;
5921 ret
= rgw::create_zone(dpp(), null_yield
, cfgstore
.get(),
5924 cerr
<< "ERROR: couldn't create zone: " << cpp_strerror(-ret
) << std::endl
;
5929 ret
= rgw::set_default_zone(dpp(), null_yield
, cfgstore
.get(), zone
);
5931 cerr
<< "failed to set zone " << zone_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
5935 encode_json("zone", zone
, formatter
.get());
5936 formatter
->flush(cout
);
5939 case OPT::ZONE_LIST
:
5941 RGWZoneParams default_zone_params
;
5942 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5943 {}, {}, default_zone_params
);
5944 if (ret
< 0 && ret
!= -ENOENT
) {
5945 cerr
<< "could not determine default zone: " << cpp_strerror(-ret
) << std::endl
;
5948 Formatter::ObjectSection zones_list
{*formatter
, "zones_list"};
5949 encode_json("default_info", default_zone_params
.id
, formatter
.get());
5951 Formatter::ArraySection zones
{*formatter
, "zones"};
5952 rgw::sal::ListResult
<std::string
> listing
;
5953 std::array
<std::string
, 1000> names
; // list in pages of 1000
5955 ret
= cfgstore
->list_zone_names(dpp(), null_yield
, listing
.next
,
5958 std::cerr
<< "failed to list zones: " << cpp_strerror(-ret
) << std::endl
;
5961 for (const auto& name
: listing
.entries
) {
5962 encode_json("name", name
, formatter
.get());
5964 } while (!listing
.next
.empty());
5965 } // close sections zones and zones_list
5966 formatter
->flush(cout
);
5968 case OPT::ZONE_MODIFY
:
5970 RGWZoneParams zone_params
;
5971 std::unique_ptr
<rgw::sal::ZoneWriter
> zone_writer
;
5972 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
5973 zone_id
, zone_name
, zone_params
, &zone_writer
);
5975 cerr
<< "failed to load zone: " << cpp_strerror(-ret
) << std::endl
;
5979 bool need_zone_update
= false;
5980 if (!access_key
.empty()) {
5981 zone_params
.system_key
.id
= access_key
;
5982 need_zone_update
= true;
5985 if (!secret_key
.empty()) {
5986 zone_params
.system_key
.key
= secret_key
;
5987 need_zone_update
= true;
5990 if (!realm_id
.empty()) {
5991 zone_params
.realm_id
= realm_id
;
5992 need_zone_update
= true;
5993 } else if (!realm_name
.empty()) {
5994 // get realm id from name
5995 ret
= cfgstore
->read_realm_id(dpp(), null_yield
,
5996 realm_name
, zone_params
.realm_id
);
5998 cerr
<< "failed to find realm by name " << realm_name
<< std::endl
;
6001 need_zone_update
= true;
6004 for (const auto& add
: tier_config_add
) {
6005 ret
= zone_params
.tier_config
.set(add
.first
, add
.second
);
6007 cerr
<< "ERROR: failed to set configurable: " << add
<< std::endl
;
6010 need_zone_update
= true;
6013 for (const auto& rm
: tier_config_rm
) {
6014 if (!rm
.first
.empty()) { /* otherwise will remove the entire config */
6015 zone_params
.tier_config
.erase(rm
.first
);
6016 need_zone_update
= true;
6020 if (need_zone_update
) {
6021 ret
= zone_writer
->write(dpp(), null_yield
, zone_params
);
6023 cerr
<< "failed to save zone info: " << cpp_strerror(-ret
) << std::endl
;
6028 RGWZoneGroup zonegroup
;
6029 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> zonegroup_writer
;
6030 ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
6031 zonegroup_id
, zonegroup_name
,
6032 zonegroup
, &zonegroup_writer
);
6034 cerr
<< "failed to load zonegroup: " << cpp_strerror(-ret
) << std::endl
;
6038 const bool *pis_master
= (is_master_set
? &is_master
: nullptr);
6039 const bool *pread_only
= (is_read_only_set
? &read_only
: nullptr);
6040 const bool *psync_from_all
= (sync_from_all_specified
? &sync_from_all
: nullptr);
6041 const string
*predirect_zone
= (redirect_zone_set
? &redirect_zone
: nullptr);
6043 // validate --tier-type if specified
6044 const string
*ptier_type
= (tier_type_specified
? &tier_type
: nullptr);
6046 auto sync_mgr
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->sync_modules
->get_manager();
6047 if (!sync_mgr
->get_module(*ptier_type
, nullptr)) {
6048 ldpp_dout(dpp(), -1) << "ERROR: could not find sync module: "
6049 << *ptier_type
<< ", valid sync modules: "
6050 << sync_mgr
->get_registered_module_names() << dendl
;
6055 if (enable_features
.empty()) { // enable all features by default
6056 enable_features
.insert(rgw::zone_features::supported
.begin(),
6057 rgw::zone_features::supported
.end());
6060 // add/update the public zone information stored in the zonegroup
6061 ret
= rgw::add_zone_to_group(dpp(), zonegroup
, zone_params
,
6062 pis_master
, pread_only
, endpoints
,
6063 ptier_type
, psync_from_all
,
6064 sync_from
, sync_from_rm
,
6065 predirect_zone
, bucket_index_max_shards
,
6066 enable_features
, disable_features
);
6071 // write the updated zonegroup
6072 ret
= zonegroup_writer
->write(dpp(), null_yield
, zonegroup
);
6074 cerr
<< "failed to update zonegroup: " << cpp_strerror(-ret
) << std::endl
;
6079 ret
= rgw::set_default_zone(dpp(), null_yield
, cfgstore
.get(),
6082 cerr
<< "failed to set zone " << zone_name
<< " as default: " << cpp_strerror(-ret
) << std::endl
;
6086 encode_json("zone", zone_params
, formatter
.get());
6087 formatter
->flush(cout
);
6090 case OPT::ZONE_RENAME
:
6092 if (zone_new_name
.empty()) {
6093 cerr
<< " missing zone new name" << std::endl
;
6096 if (zone_id
.empty() && zone_name
.empty()) {
6097 cerr
<< "no zone name or id provided" << std::endl
;
6101 RGWZoneParams zone_params
;
6102 std::unique_ptr
<rgw::sal::ZoneWriter
> zone_writer
;
6103 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
6104 zone_id
, zone_name
, zone_params
, &zone_writer
);
6106 cerr
<< "failed to load zone: " << cpp_strerror(-ret
) << std::endl
;
6110 ret
= zone_writer
->rename(dpp(), null_yield
, zone_params
, zone_new_name
);
6112 cerr
<< "failed to rename zone " << zone_name
<< " to " << zone_new_name
<< ": " << cpp_strerror(-ret
)
6117 RGWZoneGroup zonegroup
;
6118 std::unique_ptr
<rgw::sal::ZoneGroupWriter
> zonegroup_writer
;
6119 ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
6120 zonegroup_id
, zonegroup_name
,
6121 zonegroup
, &zonegroup_writer
);
6123 cerr
<< "WARNING: failed to load zonegroup " << zonegroup_name
<< std::endl
;
6124 return EXIT_SUCCESS
;
6127 auto z
= zonegroup
.zones
.find(zone_params
.id
);
6128 if (z
== zonegroup
.zones
.end()) {
6129 return EXIT_SUCCESS
;
6131 z
->second
.name
= zone_params
.name
;
6133 ret
= zonegroup_writer
->write(dpp(), null_yield
, zonegroup
);
6135 cerr
<< "Error in zonegroup rename for " << zone_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
6140 case OPT::ZONE_PLACEMENT_ADD
:
6141 case OPT::ZONE_PLACEMENT_MODIFY
:
6142 case OPT::ZONE_PLACEMENT_RM
:
6144 if (placement_id
.empty()) {
6145 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
6148 // validate compression type
6149 if (compression_type
&& *compression_type
!= "random"
6150 && !Compressor::get_comp_alg_type(*compression_type
)) {
6151 std::cerr
<< "Unrecognized compression type" << std::endl
;
6156 std::unique_ptr
<rgw::sal::ZoneWriter
> writer
;
6157 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
6158 zone_id
, zone_name
, zone
, &writer
);
6160 cerr
<< "failed to init zone: " << cpp_strerror(-ret
) << std::endl
;
6164 if (opt_cmd
== OPT::ZONE_PLACEMENT_ADD
||
6165 opt_cmd
== OPT::ZONE_PLACEMENT_MODIFY
) {
6166 RGWZoneGroup zonegroup
;
6167 ret
= rgw::read_zonegroup(dpp(), null_yield
, cfgstore
.get(),
6168 zonegroup_id
, zonegroup_name
, zonegroup
);
6170 cerr
<< "failed to init zonegroup: " << cpp_strerror(-ret
) << std::endl
;
6174 auto ptiter
= zonegroup
.placement_targets
.find(placement_id
);
6175 if (ptiter
== zonegroup
.placement_targets
.end()) {
6176 cerr
<< "ERROR: placement id '" << placement_id
<< "' is not configured in zonegroup placement targets" << std::endl
;
6180 string storage_class
= rgw_placement_rule::get_canonical_storage_class(opt_storage_class
.value_or(string()));
6181 if (ptiter
->second
.storage_classes
.find(storage_class
) == ptiter
->second
.storage_classes
.end()) {
6182 cerr
<< "ERROR: storage class '" << storage_class
<< "' is not defined in zonegroup '" << placement_id
<< "' placement target" << std::endl
;
6185 if (ptiter
->second
.tier_targets
.find(storage_class
) != ptiter
->second
.tier_targets
.end()) {
6186 cerr
<< "ERROR: storage class '" << storage_class
<< "' is of tier type in zonegroup '" << placement_id
<< "' placement target" << std::endl
;
6190 RGWZonePlacementInfo
& info
= zone
.placement_pools
[placement_id
];
6192 string opt_index_pool
= index_pool
.value_or(string());
6193 string opt_data_pool
= data_pool
.value_or(string());
6195 if (!opt_index_pool
.empty()) {
6196 info
.index_pool
= opt_index_pool
;
6199 if (info
.index_pool
.empty()) {
6200 cerr
<< "ERROR: index pool not configured, need to specify --index-pool" << std::endl
;
6204 if (opt_data_pool
.empty()) {
6205 const RGWZoneStorageClass
*porig_sc
{nullptr};
6206 if (info
.storage_classes
.find(storage_class
, &porig_sc
)) {
6207 if (porig_sc
->data_pool
) {
6208 opt_data_pool
= porig_sc
->data_pool
->to_str();
6211 if (opt_data_pool
.empty()) {
6212 cerr
<< "ERROR: data pool not configured, need to specify --data-pool" << std::endl
;
6217 rgw_pool dp
= opt_data_pool
;
6218 info
.storage_classes
.set_storage_class(storage_class
, &dp
, compression_type
.get_ptr());
6220 if (data_extra_pool
) {
6221 info
.data_extra_pool
= *data_extra_pool
;
6223 if (index_type_specified
) {
6224 info
.index_type
= placement_index_type
;
6226 if (placement_inline_data_specified
) {
6227 info
.inline_data
= placement_inline_data
;
6230 ret
= check_pool_support_omap(info
.get_data_extra_pool());
6232 cerr
<< "ERROR: the data extra (non-ec) pool '" << info
.get_data_extra_pool()
6233 << "' does not support omap" << std::endl
;
6236 } else if (opt_cmd
== OPT::ZONE_PLACEMENT_RM
) {
6237 if (!opt_storage_class
||
6238 opt_storage_class
->empty()) {
6239 zone
.placement_pools
.erase(placement_id
);
6241 auto iter
= zone
.placement_pools
.find(placement_id
);
6242 if (iter
!= zone
.placement_pools
.end()) {
6243 RGWZonePlacementInfo
& info
= zone
.placement_pools
[placement_id
];
6244 info
.storage_classes
.remove_storage_class(*opt_storage_class
);
6249 ret
= writer
->write(dpp(), null_yield
, zone
);
6251 cerr
<< "failed to save zone info: " << cpp_strerror(-ret
) << std::endl
;
6255 encode_json("zone", zone
, formatter
.get());
6256 formatter
->flush(cout
);
6259 case OPT::ZONE_PLACEMENT_LIST
:
6262 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
6263 zone_id
, zone_name
, zone
);
6265 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
6268 encode_json("placement_pools", zone
.placement_pools
, formatter
.get());
6269 formatter
->flush(cout
);
6272 case OPT::ZONE_PLACEMENT_GET
:
6274 if (placement_id
.empty()) {
6275 cerr
<< "ERROR: --placement-id not specified" << std::endl
;
6280 int ret
= rgw::read_zone(dpp(), null_yield
, cfgstore
.get(),
6281 zone_id
, zone_name
, zone
);
6283 cerr
<< "unable to initialize zone: " << cpp_strerror(-ret
) << std::endl
;
6286 auto p
= zone
.placement_pools
.find(placement_id
);
6287 if (p
== zone
.placement_pools
.end()) {
6288 cerr
<< "ERROR: zone placement target '" << placement_id
<< "' not found" << std::endl
;
6291 encode_json("placement_pools", p
->second
, formatter
.get());
6292 formatter
->flush(cout
);
6300 resolve_zone_id_opt(opt_effective_zone_name
, opt_effective_zone_id
);
6301 resolve_zone_id_opt(opt_source_zone_name
, opt_source_zone_id
);
6302 resolve_zone_id_opt(opt_dest_zone_name
, opt_dest_zone_id
);
6303 resolve_zone_ids_opt(opt_zone_names
, opt_zone_ids
);
6304 resolve_zone_ids_opt(opt_source_zone_names
, opt_source_zone_ids
);
6305 resolve_zone_ids_opt(opt_dest_zone_names
, opt_dest_zone_ids
);
6307 bool non_master_cmd
= (!driver
->is_meta_master() && !yes_i_really_mean_it
);
6308 std::set
<OPT
> non_master_ops_list
= {OPT::USER_CREATE
, OPT::USER_RM
,
6309 OPT::USER_MODIFY
, OPT::USER_ENABLE
,
6310 OPT::USER_SUSPEND
, OPT::SUBUSER_CREATE
,
6311 OPT::SUBUSER_MODIFY
, OPT::SUBUSER_RM
,
6312 OPT::BUCKET_LINK
, OPT::BUCKET_UNLINK
,
6314 OPT::BUCKET_CHOWN
, OPT::METADATA_PUT
,
6315 OPT::METADATA_RM
, OPT::MFA_CREATE
,
6316 OPT::MFA_REMOVE
, OPT::MFA_RESYNC
,
6317 OPT::CAPS_ADD
, OPT::CAPS_RM
,
6318 OPT::ROLE_CREATE
, OPT::ROLE_DELETE
,
6319 OPT::ROLE_POLICY_PUT
, OPT::ROLE_POLICY_DELETE
};
6321 bool print_warning_message
= (non_master_ops_list
.find(opt_cmd
) != non_master_ops_list
.end() &&
6324 if (print_warning_message
) {
6325 cerr
<< "Please run the command on master zone. Performing this operation on non-master zone leads to inconsistent metadata between zones" << std::endl
;
6326 cerr
<< "Are you sure you want to go ahead? (requires --yes-i-really-mean-it)" << std::endl
;
6330 if (!rgw::sal::User::empty(user
)) {
6331 user_op
.set_user_id(user
->get_id());
6332 bucket_op
.set_user_id(user
->get_id());
6335 if (!display_name
.empty())
6336 user_op
.set_display_name(display_name
);
6338 if (!user_email
.empty())
6339 user_op
.set_user_email(user_email
);
6341 if (!rgw::sal::User::empty(user
)) {
6342 user_op
.set_new_user_id(new_user_id
);
6345 if (!access_key
.empty())
6346 user_op
.set_access_key(access_key
);
6348 if (!secret_key
.empty())
6349 user_op
.set_secret_key(secret_key
);
6351 if (!subuser
.empty())
6352 user_op
.set_subuser(subuser
);
6355 user_op
.set_caps(caps
);
6357 user_op
.set_purge_data(purge_data
);
6360 user_op
.set_purge_keys();
6363 user_op
.set_generate_key();
6366 user_op
.set_gen_secret(); // assume that a key pair should be created
6368 if (max_buckets_specified
)
6369 user_op
.set_max_buckets(max_buckets
);
6371 if (admin_specified
)
6372 user_op
.set_admin(admin
);
6374 if (system_specified
)
6375 user_op
.set_system(system
);
6378 user_op
.set_perm(perm_mask
);
6380 if (set_temp_url_key
) {
6381 map
<int, string
>::iterator iter
= temp_url_keys
.begin();
6382 for (; iter
!= temp_url_keys
.end(); ++iter
) {
6383 user_op
.set_temp_url_key(iter
->second
, iter
->first
);
6387 if (!op_mask_str
.empty()) {
6389 int ret
= rgw_parse_op_type_list(op_mask_str
, &op_mask
);
6391 cerr
<< "failed to parse op_mask: " << cpp_strerror(-ret
) << std::endl
;
6395 user_op
.set_op_mask(op_mask
);
6398 if (key_type
!= KEY_TYPE_UNDEFINED
)
6399 user_op
.set_key_type(key_type
);
6401 // set suspension operation parameters
6402 if (opt_cmd
== OPT::USER_ENABLE
)
6403 user_op
.set_suspension(false);
6404 else if (opt_cmd
== OPT::USER_SUSPEND
)
6405 user_op
.set_suspension(true);
6407 if (!placement_id
.empty()) {
6408 rgw_placement_rule target_rule
;
6409 target_rule
.name
= placement_id
;
6410 target_rule
.storage_class
= opt_storage_class
.value_or("");
6411 if (!driver
->valid_placement(target_rule
)) {
6412 cerr
<< "NOTICE: invalid dest placement: " << target_rule
.to_str() << std::endl
;
6415 user_op
.set_default_placement(target_rule
);
6418 if (!tags
.empty()) {
6419 user_op
.set_placement_tags(tags
);
6422 // RGWUser to use for user operations
6425 if (!(rgw::sal::User::empty(user
) && access_key
.empty()) || !subuser
.empty()) {
6426 ret
= ruser
.init(dpp(), driver
, user_op
, null_yield
);
6428 cerr
<< "user.init failed: " << cpp_strerror(-ret
) << std::endl
;
6433 /* populate bucket operation */
6434 bucket_op
.set_bucket_name(bucket_name
);
6435 bucket_op
.set_object(object
);
6436 bucket_op
.set_check_objects(check_objects
);
6437 bucket_op
.set_delete_children(delete_child_objects
);
6438 bucket_op
.set_fix_index(fix
);
6439 bucket_op
.set_max_aio(max_concurrent_ios
);
6441 // required to gather errors from operations
6442 std::string err_msg
;
6444 bool output_user_info
= true;
6447 case OPT::USER_INFO
:
6448 if (rgw::sal::User::empty(user
) && access_key
.empty()) {
6449 cerr
<< "ERROR: --uid or --access-key required" << std::endl
;
6453 case OPT::USER_CREATE
:
6454 if (!user_op
.has_existing_user()) {
6455 user_op
.set_generate_key(); // generate a new key by default
6457 ret
= ruser
.add(dpp(), user_op
, null_yield
, &err_msg
);
6459 cerr
<< "could not create user: " << err_msg
<< std::endl
;
6460 if (ret
== -ERR_INVALID_TENANT_NAME
)
6465 if (!subuser
.empty()) {
6466 ret
= ruser
.subusers
.add(dpp(),user_op
, null_yield
, &err_msg
);
6468 cerr
<< "could not create subuser: " << err_msg
<< std::endl
;
6474 ret
= ruser
.remove(dpp(), user_op
, null_yield
, &err_msg
);
6476 cerr
<< "could not remove user: " << err_msg
<< std::endl
;
6480 output_user_info
= false;
6482 case OPT::USER_RENAME
:
6483 if (yes_i_really_mean_it
) {
6484 user_op
.set_overwrite_new_user(true);
6486 ret
= ruser
.rename(user_op
, null_yield
, dpp(), &err_msg
);
6488 if (ret
== -EEXIST
) {
6489 err_msg
+= ". to overwrite this user, add --yes-i-really-mean-it";
6491 cerr
<< "could not rename user: " << err_msg
<< std::endl
;
6496 case OPT::USER_ENABLE
:
6497 case OPT::USER_SUSPEND
:
6498 case OPT::USER_MODIFY
:
6499 ret
= ruser
.modify(dpp(), user_op
, null_yield
, &err_msg
);
6501 cerr
<< "could not modify user: " << err_msg
<< std::endl
;
6506 case OPT::SUBUSER_CREATE
:
6507 ret
= ruser
.subusers
.add(dpp(), user_op
, null_yield
, &err_msg
);
6509 cerr
<< "could not create subuser: " << err_msg
<< std::endl
;
6514 case OPT::SUBUSER_MODIFY
:
6515 ret
= ruser
.subusers
.modify(dpp(), user_op
, null_yield
, &err_msg
);
6517 cerr
<< "could not modify subuser: " << err_msg
<< std::endl
;
6522 case OPT::SUBUSER_RM
:
6523 ret
= ruser
.subusers
.remove(dpp(), user_op
, null_yield
, &err_msg
);
6525 cerr
<< "could not remove subuser: " << err_msg
<< std::endl
;
6531 ret
= ruser
.caps
.add(dpp(), user_op
, null_yield
, &err_msg
);
6533 cerr
<< "could not add caps: " << err_msg
<< std::endl
;
6539 ret
= ruser
.caps
.remove(dpp(), user_op
, null_yield
, &err_msg
);
6541 cerr
<< "could not remove caps: " << err_msg
<< std::endl
;
6546 case OPT::KEY_CREATE
:
6547 ret
= ruser
.keys
.add(dpp(), user_op
, null_yield
, &err_msg
);
6549 cerr
<< "could not create key: " << err_msg
<< std::endl
;
6555 ret
= ruser
.keys
.remove(dpp(), user_op
, null_yield
, &err_msg
);
6557 cerr
<< "could not remove key: " << err_msg
<< std::endl
;
6561 case OPT::PERIOD_PUSH
:
6564 req_info
info(g_ceph_context
, &env
);
6565 info
.method
= "POST";
6566 info
.request_uri
= "/admin/realm/period";
6568 map
<string
, string
> ¶ms
= info
.args
.get_params();
6569 if (!realm_id
.empty())
6570 params
["realm_id"] = realm_id
;
6571 if (!realm_name
.empty())
6572 params
["realm_name"] = realm_name
;
6573 if (!period_id
.empty())
6574 params
["period_id"] = period_id
;
6575 if (!period_epoch
.empty())
6576 params
["epoch"] = period_epoch
;
6580 int ret
= cfgstore
->read_period(dpp(), null_yield
, period_id
,
6581 std::nullopt
, period
);
6583 cerr
<< "failed to load period: " << cpp_strerror(-ret
) << std::endl
;
6586 // json format into a bufferlist
6587 JSONFormatter
jf(false);
6588 encode_json("period", period
, &jf
);
6593 ret
= send_to_remote_or_url(nullptr, url
, opt_region
,
6594 access_key
, secret_key
,
6597 cerr
<< "request failed: " << cpp_strerror(-ret
) << std::endl
;
6602 case OPT::PERIOD_UPDATE
:
6604 int ret
= update_period(cfgstore
.get(), realm_id
, realm_name
,
6605 period_epoch
, commit
, remote
, url
,
6606 opt_region
, access_key
, secret_key
,
6607 formatter
.get(), yes_i_really_mean_it
);
6613 case OPT::PERIOD_COMMIT
:
6615 // read realm and staging period
6617 std::unique_ptr
<rgw::sal::RealmWriter
> realm_writer
;
6618 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
6619 realm_id
, realm_name
,
6620 realm
, &realm_writer
);
6622 cerr
<< "Error initializing realm: " << cpp_strerror(-ret
) << std::endl
;
6625 period_id
= rgw::get_staging_period_id(realm
.id
);
6629 ret
= cfgstore
->read_period(dpp(), null_yield
, period_id
, epoch
, period
);
6631 cerr
<< "failed to load period: " << cpp_strerror(-ret
) << std::endl
;
6634 ret
= commit_period(cfgstore
.get(), realm
, *realm_writer
, period
,
6635 remote
, url
, opt_region
, access_key
, secret_key
,
6636 yes_i_really_mean_it
);
6638 cerr
<< "failed to commit period: " << cpp_strerror(-ret
) << std::endl
;
6642 encode_json("period", period
, formatter
.get());
6643 formatter
->flush(cout
);
6646 case OPT::ROLE_CREATE
:
6648 if (role_name
.empty()) {
6649 cerr
<< "ERROR: role name is empty" << std::endl
;
6653 if (assume_role_doc
.empty()) {
6654 cerr
<< "ERROR: assume role policy document is empty" << std::endl
;
6657 bufferlist bl
= bufferlist::static_from_string(assume_role_doc
);
6659 const rgw::IAM::Policy
p(
6660 g_ceph_context
, tenant
, bl
,
6661 g_ceph_context
->_conf
.get_val
<bool>(
6662 "rgw_policy_reject_invalid_principals"));
6663 } catch (rgw::IAM::PolicyParseException
& e
) {
6664 cerr
<< "failed to parse policy: " << e
.what() << std::endl
;
6667 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
, path
, assume_role_doc
);
6668 ret
= role
->create(dpp(), true, "", null_yield
);
6672 show_role_info(role
.get(), formatter
.get());
6675 case OPT::ROLE_DELETE
:
6677 if (role_name
.empty()) {
6678 cerr
<< "ERROR: empty role name" << std::endl
;
6681 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6682 ret
= role
->delete_obj(dpp(), null_yield
);
6686 cout
<< "role: " << role_name
<< " successfully deleted" << std::endl
;
6691 if (role_name
.empty()) {
6692 cerr
<< "ERROR: empty role name" << std::endl
;
6695 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6696 ret
= role
->get(dpp(), null_yield
);
6700 show_role_info(role
.get(), formatter
.get());
6703 case OPT::ROLE_TRUST_POLICY_MODIFY
:
6705 if (role_name
.empty()) {
6706 cerr
<< "ERROR: role name is empty" << std::endl
;
6710 if (assume_role_doc
.empty()) {
6711 cerr
<< "ERROR: assume role policy document is empty" << std::endl
;
6715 bufferlist bl
= bufferlist::static_from_string(assume_role_doc
);
6717 const rgw::IAM::Policy
p(g_ceph_context
, tenant
, bl
,
6718 g_ceph_context
->_conf
.get_val
<bool>(
6719 "rgw_policy_reject_invalid_principals"));
6720 } catch (rgw::IAM::PolicyParseException
& e
) {
6721 cerr
<< "failed to parse policy: " << e
.what() << std::endl
;
6725 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6726 ret
= role
->get(dpp(), null_yield
);
6730 role
->update_trust_policy(assume_role_doc
);
6731 ret
= role
->update(dpp(), null_yield
);
6735 cout
<< "Assume role policy document updated successfully for role: " << role_name
<< std::endl
;
6738 case OPT::ROLE_LIST
:
6740 vector
<std::unique_ptr
<rgw::sal::RGWRole
>> result
;
6741 ret
= driver
->get_roles(dpp(), null_yield
, path_prefix
, tenant
, result
);
6745 show_roles_info(result
, formatter
.get());
6748 case OPT::ROLE_POLICY_PUT
:
6750 if (role_name
.empty()) {
6751 cerr
<< "role name is empty" << std::endl
;
6755 if (policy_name
.empty()) {
6756 cerr
<< "policy name is empty" << std::endl
;
6760 if (perm_policy_doc
.empty() && infile
.empty()) {
6761 cerr
<< "permission policy document is empty" << std::endl
;
6766 if (!infile
.empty()) {
6767 int ret
= read_input(infile
, bl
);
6769 cerr
<< "ERROR: failed to read input policy document: " << cpp_strerror(-ret
) << std::endl
;
6772 perm_policy_doc
= bl
.to_str();
6774 bl
= bufferlist::static_from_string(perm_policy_doc
);
6777 const rgw::IAM::Policy
p(g_ceph_context
, tenant
, bl
,
6778 g_ceph_context
->_conf
.get_val
<bool>(
6779 "rgw_policy_reject_invalid_principals"));
6780 } catch (rgw::IAM::PolicyParseException
& e
) {
6781 cerr
<< "failed to parse perm policy: " << e
.what() << std::endl
;
6785 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6786 ret
= role
->get(dpp(), null_yield
);
6790 role
->set_perm_policy(policy_name
, perm_policy_doc
);
6791 ret
= role
->update(dpp(), null_yield
);
6795 cout
<< "Permission policy attached successfully" << std::endl
;
6798 case OPT::ROLE_POLICY_LIST
:
6800 if (role_name
.empty()) {
6801 cerr
<< "ERROR: Role name is empty" << std::endl
;
6804 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6805 ret
= role
->get(dpp(), null_yield
);
6809 std::vector
<string
> policy_names
= role
->get_role_policy_names();
6810 show_policy_names(policy_names
, formatter
.get());
6813 case OPT::ROLE_POLICY_GET
:
6815 if (role_name
.empty()) {
6816 cerr
<< "ERROR: role name is empty" << std::endl
;
6820 if (policy_name
.empty()) {
6821 cerr
<< "ERROR: policy name is empty" << std::endl
;
6824 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6825 int ret
= role
->get(dpp(), null_yield
);
6830 ret
= role
->get_role_policy(dpp(), policy_name
, perm_policy
);
6834 show_perm_policy(perm_policy
, formatter
.get());
6837 case OPT::ROLE_POLICY_DELETE
:
6839 if (role_name
.empty()) {
6840 cerr
<< "ERROR: role name is empty" << std::endl
;
6844 if (policy_name
.empty()) {
6845 cerr
<< "ERROR: policy name is empty" << std::endl
;
6848 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6849 ret
= role
->get(dpp(), null_yield
);
6853 ret
= role
->delete_policy(dpp(), policy_name
);
6857 ret
= role
->update(dpp(), null_yield
);
6861 cout
<< "Policy: " << policy_name
<< " successfully deleted for role: "
6862 << role_name
<< std::endl
;
6865 case OPT::ROLE_UPDATE
:
6867 if (role_name
.empty()) {
6868 cerr
<< "ERROR: role name is empty" << std::endl
;
6872 std::unique_ptr
<rgw::sal::RGWRole
> role
= driver
->get_role(role_name
, tenant
);
6873 ret
= role
->get(dpp(), null_yield
);
6877 if (!role
->validate_max_session_duration(dpp())) {
6881 role
->update_max_session_duration(max_session_duration
);
6882 ret
= role
->update(dpp(), null_yield
);
6886 cout
<< "Max session duration updated successfully for role: " << role_name
<< std::endl
;
6890 output_user_info
= false;
6893 // output the result of a user operation
6894 if (output_user_info
) {
6895 ret
= ruser
.info(info
, &err_msg
);
6897 cerr
<< "could not fetch user info: " << err_msg
<< std::endl
;
6900 show_user_info(info
, formatter
.get());
6903 if (opt_cmd
== OPT::POLICY
) {
6904 if (format
== "xml") {
6905 int ret
= RGWBucketAdminOp::dump_s3_policy(driver
, bucket_op
, cout
, dpp());
6907 cerr
<< "ERROR: failed to get policy: " << cpp_strerror(-ret
) << std::endl
;
6911 int ret
= RGWBucketAdminOp::get_policy(driver
, bucket_op
, stream_flusher
, dpp());
6913 cerr
<< "ERROR: failed to get policy: " << cpp_strerror(-ret
) << std::endl
;
6919 if (opt_cmd
== OPT::BUCKET_LIMIT_CHECK
) {
6921 std::list
<std::string
> user_ids
;
6922 metadata_key
= "user";
6927 if (!rgw::sal::User::empty(user
)) {
6928 user_ids
.push_back(user
->get_id().id
);
6930 RGWBucketAdminOp::limit_check(driver
, bucket_op
, user_ids
, stream_flusher
,
6931 null_yield
, dpp(), warnings_only
);
6933 /* list users in groups of max-keys, then perform user-bucket
6934 * limit-check on each group */
6935 ret
= driver
->meta_list_keys_init(dpp(), metadata_key
, string(), &handle
);
6937 cerr
<< "ERROR: buckets limit check can't get user metadata_key: "
6938 << cpp_strerror(-ret
) << std::endl
;
6943 ret
= driver
->meta_list_keys_next(dpp(), handle
, max
, user_ids
,
6945 if (ret
< 0 && ret
!= -ENOENT
) {
6946 cerr
<< "ERROR: buckets limit check lists_keys_next(): "
6947 << cpp_strerror(-ret
) << std::endl
;
6950 /* ok, do the limit checks for this group */
6952 RGWBucketAdminOp::limit_check(driver
, bucket_op
, user_ids
, stream_flusher
,
6953 null_yield
, dpp(), warnings_only
);
6958 } while (truncated
);
6959 driver
->meta_list_keys_complete(handle
);
6962 } /* OPT::BUCKET_LIMIT_CHECK */
6964 if (opt_cmd
== OPT::BUCKETS_LIST
) {
6965 if (bucket_name
.empty()) {
6966 if (!rgw::sal::User::empty(user
)) {
6967 if (!user_op
.has_existing_user()) {
6968 cerr
<< "ERROR: could not find user: " << user
<< std::endl
;
6972 RGWBucketAdminOp::info(driver
, bucket_op
, stream_flusher
, null_yield
, dpp());
6974 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
6976 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
6979 formatter
->open_array_section("entries");
6983 static constexpr int MAX_PAGINATE_SIZE
= 10000;
6984 static constexpr int DEFAULT_MAX_ENTRIES
= 1000;
6986 if (max_entries
< 0) {
6987 max_entries
= DEFAULT_MAX_ENTRIES
;
6989 const int paginate_size
= std::min(max_entries
, MAX_PAGINATE_SIZE
);
6995 rgw::sal::Bucket::ListParams params
;
6996 rgw::sal::Bucket::ListResults results
;
6998 params
.prefix
= prefix
;
6999 params
.delim
= delim
;
7000 params
.marker
= rgw_obj_key(marker
);
7002 params
.enforce_ns
= false;
7003 params
.list_versions
= true;
7004 params
.allow_unordered
= bool(allow_unordered
);
7007 const int remaining
= max_entries
- count
;
7008 ret
= bucket
->list(dpp(), params
, std::min(remaining
, paginate_size
), results
,
7011 cerr
<< "ERROR: driver->list_objects(): " << cpp_strerror(-ret
) << std::endl
;
7014 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<<
7015 ": list() returned without error; results.objs.sizie()=" <<
7016 results
.objs
.size() << "results.is_truncated=" << results
.is_truncated
<< ", marker=" <<
7017 params
.marker
<< dendl
;
7019 count
+= results
.objs
.size();
7021 for (const auto& entry
: results
.objs
) {
7022 encode_json("entry", entry
, formatter
.get());
7024 formatter
->flush(cout
);
7025 } while (results
.is_truncated
&& count
< max_entries
);
7026 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<< ": done" << dendl
;
7028 formatter
->close_section();
7029 formatter
->flush(cout
);
7030 } /* have bucket_name */
7031 } /* OPT::BUCKETS_LIST */
7033 if (opt_cmd
== OPT::BUCKET_RADOS_LIST
) {
7034 RGWRadosList
lister(static_cast<rgw::sal::RadosStore
*>(driver
),
7035 max_concurrent_ios
, orphan_stale_secs
, tenant
);
7037 lister
.set_field_separator(*rgw_obj_fs
);
7040 if (bucket_name
.empty()) {
7041 // yes_i_really_mean_it means continue with listing even if
7042 // there are indexless buckets
7043 ret
= lister
.run(dpp(), yes_i_really_mean_it
);
7045 ret
= lister
.run(dpp(), bucket_name
);
7050 "ERROR: bucket radoslist failed to finish before " <<
7051 "encountering error: " << cpp_strerror(-ret
) << std::endl
;
7052 std::cerr
<< "************************************"
7053 "************************************" << std::endl
;
7054 std::cerr
<< "WARNING: THE RESULTS ARE NOT RELIABLE AND SHOULD NOT " <<
7055 "BE USED IN DELETING ORPHANS" << std::endl
;
7056 std::cerr
<< "************************************"
7057 "************************************" << std::endl
;
7062 if (opt_cmd
== OPT::BUCKET_LAYOUT
) {
7063 if (bucket_name
.empty()) {
7064 cerr
<< "ERROR: bucket not specified" << std::endl
;
7067 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7071 const auto& bucket_info
= bucket
->get_info();
7072 formatter
->open_object_section("layout");
7073 encode_json("layout", bucket_info
.layout
, formatter
.get());
7074 formatter
->close_section();
7075 formatter
->flush(cout
);
7078 if (opt_cmd
== OPT::BUCKET_STATS
) {
7079 if (bucket_name
.empty() && !bucket_id
.empty()) {
7081 if (!rgw_find_bucket_by_id(dpp(), driver
->ctx(), driver
, marker
, bucket_id
, &bucket
)) {
7082 cerr
<< "failure: no such bucket id" << std::endl
;
7085 bucket_op
.set_tenant(bucket
.tenant
);
7086 bucket_op
.set_bucket_name(bucket
.name
);
7088 bucket_op
.set_fetch_stats(true);
7090 int r
= RGWBucketAdminOp::info(driver
, bucket_op
, stream_flusher
, null_yield
, dpp());
7092 cerr
<< "failure: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
7093 return posix_errortrans(-r
);
7097 if (opt_cmd
== OPT::BUCKET_LINK
) {
7098 bucket_op
.set_bucket_id(bucket_id
);
7099 bucket_op
.set_new_bucket_name(new_bucket_name
);
7101 int r
= RGWBucketAdminOp::link(driver
, bucket_op
, dpp(), &err
);
7103 cerr
<< "failure: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
7108 if (opt_cmd
== OPT::BUCKET_UNLINK
) {
7109 int r
= RGWBucketAdminOp::unlink(driver
, bucket_op
, dpp());
7111 cerr
<< "failure: " << cpp_strerror(-r
) << std::endl
;
7116 if (opt_cmd
== OPT::BUCKET_SHARD_OBJECTS
) {
7117 const auto prefix
= opt_prefix
? *opt_prefix
: "obj"s
;
7118 if (!num_shards_specified
) {
7119 cerr
<< "ERROR: num-shards must be specified."
7124 if (specified_shard_id
) {
7125 if (shard_id
>= num_shards
) {
7126 cerr
<< "ERROR: shard-id must be less than num-shards."
7134 obj
= fmt::format("{}{:0>20}", prefix
, ctr
);
7135 shard
= RGWSI_BucketIndex_RADOS::bucket_shard_index(obj
, num_shards
);
7137 } while (shard
!= shard_id
);
7139 formatter
->open_object_section("shard_obj");
7140 encode_json("obj", obj
, formatter
.get());
7141 formatter
->close_section();
7142 formatter
->flush(cout
);
7144 std::vector
<std::string
> objs(num_shards
);
7145 for (uint64_t ctr
= 0, shardsleft
= num_shards
; shardsleft
> 0; ++ctr
) {
7146 auto key
= fmt::format("{}{:0>20}", prefix
, ctr
);
7147 auto shard
= RGWSI_BucketIndex_RADOS::bucket_shard_index(key
, num_shards
);
7148 if (objs
[shard
].empty()) {
7149 objs
[shard
] = std::move(key
);
7154 formatter
->open_object_section("shard_objs");
7155 encode_json("objs", objs
, formatter
.get());
7156 formatter
->close_section();
7157 formatter
->flush(cout
);
7161 if (opt_cmd
== OPT::BUCKET_OBJECT_SHARD
) {
7162 if (!num_shards_specified
|| object
.empty()) {
7163 cerr
<< "ERROR: num-shards and object must be specified."
7167 auto shard
= RGWSI_BucketIndex_RADOS::bucket_shard_index(object
, num_shards
);
7168 formatter
->open_object_section("obj_shard");
7169 encode_json("shard", shard
, formatter
.get());
7170 formatter
->close_section();
7171 formatter
->flush(cout
);
7174 if (opt_cmd
== OPT::BUCKET_CHOWN
) {
7175 if (bucket_name
.empty()) {
7176 cerr
<< "ERROR: bucket name not specified" << std::endl
;
7180 bucket_op
.set_bucket_name(bucket_name
);
7181 bucket_op
.set_new_bucket_name(new_bucket_name
);
7184 int r
= RGWBucketAdminOp::chown(driver
, bucket_op
, marker
, dpp(), &err
);
7186 cerr
<< "failure: " << cpp_strerror(-r
) << ": " << err
<< std::endl
;
7191 if (opt_cmd
== OPT::LOG_LIST
) {
7193 if (date
.size() && date
.size() != 10) {
7194 cerr
<< "bad date format for '" << date
<< "', expect YYYY-MM-DD" << std::endl
;
7199 formatter
->open_array_section("logs");
7201 int r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_list_init(dpp(), date
, &h
);
7206 cerr
<< "log list: error " << r
<< std::endl
;
7211 int r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_list_next(h
, &name
);
7215 cerr
<< "log list: error " << r
<< std::endl
;
7218 formatter
->dump_string("object", name
);
7221 formatter
->close_section();
7222 formatter
->flush(cout
);
7226 if (opt_cmd
== OPT::LOG_SHOW
|| opt_cmd
== OPT::LOG_RM
) {
7227 if (object
.empty() && (date
.empty() || bucket_name
.empty() || bucket_id
.empty())) {
7228 cerr
<< "specify an object or a date, bucket and bucket-id" << std::endl
;
7233 if (!object
.empty()) {
7243 if (opt_cmd
== OPT::LOG_SHOW
) {
7246 int r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_show_init(dpp(), oid
, &h
);
7248 cerr
<< "error opening log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
7253 formatter
->open_object_section("log");
7255 struct rgw_log_entry entry
;
7257 // peek at first entry to get bucket metadata
7258 r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_show_next(dpp(), h
, &entry
);
7260 cerr
<< "error reading log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
7263 formatter
->dump_string("bucket_id", entry
.bucket_id
);
7264 formatter
->dump_string("bucket_owner", entry
.bucket_owner
.to_str());
7265 formatter
->dump_string("bucket", entry
.bucket
);
7267 uint64_t agg_time
= 0;
7268 uint64_t agg_bytes_sent
= 0;
7269 uint64_t agg_bytes_received
= 0;
7270 uint64_t total_entries
= 0;
7272 if (show_log_entries
)
7273 formatter
->open_array_section("log_entries");
7276 using namespace std::chrono
;
7277 uint64_t total_time
= duration_cast
<milliseconds
>(entry
.total_time
).count();
7279 agg_time
+= total_time
;
7280 agg_bytes_sent
+= entry
.bytes_sent
;
7281 agg_bytes_received
+= entry
.bytes_received
;
7284 if (skip_zero_entries
&& entry
.bytes_sent
== 0 &&
7285 entry
.bytes_received
== 0)
7288 if (show_log_entries
) {
7290 rgw_format_ops_log_entry(entry
, formatter
.get());
7291 formatter
->flush(cout
);
7294 r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_show_next(dpp(), h
, &entry
);
7298 cerr
<< "error reading log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
7301 if (show_log_entries
)
7302 formatter
->close_section();
7305 formatter
->open_object_section("log_sum");
7306 formatter
->dump_int("bytes_sent", agg_bytes_sent
);
7307 formatter
->dump_int("bytes_received", agg_bytes_received
);
7308 formatter
->dump_int("total_time", agg_time
);
7309 formatter
->dump_int("total_entries", total_entries
);
7310 formatter
->close_section();
7312 formatter
->close_section();
7313 formatter
->flush(cout
);
7316 if (opt_cmd
== OPT::LOG_RM
) {
7317 int r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->log_remove(dpp(), oid
);
7319 cerr
<< "error removing log " << oid
<< ": " << cpp_strerror(-r
) << std::endl
;
7325 if (opt_cmd
== OPT::POOL_ADD
) {
7326 if (pool_name
.empty()) {
7327 cerr
<< "need to specify pool to add!" << std::endl
;
7331 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->add_bucket_placement(dpp(), pool
, null_yield
);
7333 cerr
<< "failed to add bucket placement: " << cpp_strerror(-ret
) << std::endl
;
7336 if (opt_cmd
== OPT::POOL_RM
) {
7337 if (pool_name
.empty()) {
7338 cerr
<< "need to specify pool to remove!" << std::endl
;
7342 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->remove_bucket_placement(dpp(), pool
, null_yield
);
7344 cerr
<< "failed to remove bucket placement: " << cpp_strerror(-ret
) << std::endl
;
7347 if (opt_cmd
== OPT::POOLS_LIST
) {
7348 set
<rgw_pool
> pools
;
7349 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->list_placement_set(dpp(), pools
, null_yield
);
7351 cerr
<< "could not list placement set: " << cpp_strerror(-ret
) << std::endl
;
7355 formatter
->open_array_section("pools");
7356 for (auto siter
= pools
.begin(); siter
!= pools
.end(); ++siter
) {
7357 formatter
->open_object_section("pool");
7358 formatter
->dump_string("name", siter
->to_str());
7359 formatter
->close_section();
7361 formatter
->close_section();
7362 formatter
->flush(cout
);
7366 if (opt_cmd
== OPT::USAGE_SHOW
) {
7367 uint64_t start_epoch
= 0;
7368 uint64_t end_epoch
= (uint64_t)-1;
7372 if (!start_date
.empty()) {
7373 ret
= utime_t::parse_date(start_date
, &start_epoch
, NULL
);
7375 cerr
<< "ERROR: failed to parse start date" << std::endl
;
7379 if (!end_date
.empty()) {
7380 ret
= utime_t::parse_date(end_date
, &end_epoch
, NULL
);
7382 cerr
<< "ERROR: failed to parse end date" << std::endl
;
7388 if (!bucket_name
.empty()) {
7389 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7391 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7395 ret
= RGWUsage::show(dpp(), driver
, user
.get(), bucket
.get(), start_epoch
,
7396 end_epoch
, show_log_entries
, show_log_sum
, &categories
,
7399 cerr
<< "ERROR: failed to show usage" << std::endl
;
7404 if (opt_cmd
== OPT::USAGE_TRIM
) {
7405 if (rgw::sal::User::empty(user
) && bucket_name
.empty() &&
7406 start_date
.empty() && end_date
.empty() && !yes_i_really_mean_it
) {
7407 cerr
<< "usage trim without user/date/bucket specified will remove *all* users data" << std::endl
;
7408 cerr
<< "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
7412 uint64_t start_epoch
= 0;
7413 uint64_t end_epoch
= (uint64_t)-1;
7416 if (!start_date
.empty()) {
7417 ret
= utime_t::parse_date(start_date
, &start_epoch
, NULL
);
7419 cerr
<< "ERROR: failed to parse start date" << std::endl
;
7424 if (!end_date
.empty()) {
7425 ret
= utime_t::parse_date(end_date
, &end_epoch
, NULL
);
7427 cerr
<< "ERROR: failed to parse end date" << std::endl
;
7432 if (!bucket_name
.empty()) {
7433 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7435 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7439 ret
= RGWUsage::trim(dpp(), driver
, user
.get(), bucket
.get(), start_epoch
, end_epoch
);
7441 cerr
<< "ERROR: read_usage() returned ret=" << ret
<< std::endl
;
7446 if (opt_cmd
== OPT::USAGE_CLEAR
) {
7447 if (!yes_i_really_mean_it
) {
7448 cerr
<< "usage clear would remove *all* users usage data for all time" << std::endl
;
7449 cerr
<< "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
7453 ret
= RGWUsage::clear(dpp(), driver
);
7460 if (opt_cmd
== OPT::OLH_GET
|| opt_cmd
== OPT::OLH_READLOG
) {
7461 if (bucket_name
.empty()) {
7462 cerr
<< "ERROR: bucket not specified" << std::endl
;
7465 if (object
.empty()) {
7466 cerr
<< "ERROR: object not specified" << std::endl
;
7471 if (opt_cmd
== OPT::OLH_GET
) {
7472 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7474 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7478 rgw_obj
obj(bucket
->get_key(), object
);
7479 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->get_olh(dpp(), bucket
->get_info(), obj
, &olh
);
7481 cerr
<< "ERROR: failed reading olh: " << cpp_strerror(-ret
) << std::endl
;
7484 encode_json("olh", olh
, formatter
.get());
7485 formatter
->flush(cout
);
7488 if (opt_cmd
== OPT::OLH_READLOG
) {
7489 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7491 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7494 map
<uint64_t, vector
<rgw_bucket_olh_log_entry
> > log
;
7497 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(object
);
7501 ret
= obj
->get_obj_state(dpp(), &state
, null_yield
);
7506 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->bucket_index_read_olh_log(dpp(), bucket
->get_info(), *state
, obj
->get_obj(), 0, &log
, &is_truncated
);
7508 cerr
<< "ERROR: failed reading olh: " << cpp_strerror(-ret
) << std::endl
;
7511 formatter
->open_object_section("result");
7512 encode_json("is_truncated", is_truncated
, formatter
.get());
7513 encode_json("log", log
, formatter
.get());
7514 formatter
->close_section();
7515 formatter
->flush(cout
);
7518 if (opt_cmd
== OPT::BI_GET
) {
7519 if (bucket_name
.empty()) {
7520 cerr
<< "ERROR: bucket name not specified" << std::endl
;
7523 if (object
.empty()) {
7524 cerr
<< "ERROR: object not specified" << std::endl
;
7527 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7529 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7532 rgw_obj
obj(bucket
->get_key(), object
);
7533 if (!object_version
.empty()) {
7534 obj
.key
.set_instance(object_version
);
7537 rgw_cls_bi_entry entry
;
7538 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->bi_get(dpp(), bucket
->get_info(), obj
, bi_index_type
, &entry
);
7540 cerr
<< "ERROR: bi_get(): " << cpp_strerror(-ret
) << std::endl
;
7544 encode_json("entry", entry
, formatter
.get());
7545 formatter
->flush(cout
);
7548 if (opt_cmd
== OPT::BI_PUT
) {
7549 if (bucket_name
.empty()) {
7550 cerr
<< "ERROR: bucket name not specified" << std::endl
;
7553 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7555 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7559 rgw_cls_bi_entry entry
;
7560 cls_rgw_obj_key key
;
7561 ret
= read_decode_json(infile
, entry
, &key
);
7566 rgw_obj
obj(bucket
->get_key(), key
);
7568 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->bi_put(dpp(), bucket
->get_key(), obj
, entry
);
7570 cerr
<< "ERROR: bi_put(): " << cpp_strerror(-ret
) << std::endl
;
7575 if (opt_cmd
== OPT::BI_LIST
) {
7576 if (bucket_name
.empty()) {
7577 cerr
<< "ERROR: bucket name not specified" << std::endl
;
7581 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7583 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7587 std::list
<rgw_cls_bi_entry
> entries
;
7589 const auto& index
= bucket
->get_info().layout
.current_index
;
7590 const int max_shards
= rgw::num_shards(index
);
7591 if (max_entries
< 0) {
7595 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<< ": max_entries=" << max_entries
<<
7596 ", index=" << index
<< ", max_shards=" << max_shards
<< dendl
;
7598 formatter
->open_array_section("entries");
7600 int i
= (specified_shard_id
? shard_id
: 0);
7601 for (; i
< max_shards
; i
++) {
7602 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<< ": starting shard=" << i
<< dendl
;
7604 RGWRados::BucketShard
bs(static_cast<rgw::sal::RadosStore
*>(driver
)->getRados());
7605 int ret
= bs
.init(dpp(), bucket
->get_info(), index
, i
);
7609 cerr
<< "ERROR: bs.init(bucket=" << bucket
<< ", shard=" << i
<< "): " << cpp_strerror(-ret
) << std::endl
;
7615 // if object is specified, we use that as a filter to only retrieve some some entries
7616 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->bi_list(bs
, object
, marker
, max_entries
, &entries
, &is_truncated
);
7618 cerr
<< "ERROR: bi_list(): " << cpp_strerror(-ret
) << std::endl
;
7621 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<<
7622 ": bi_list() returned without error; entries.size()=" <<
7623 entries
.size() << ", is_truncated=" << is_truncated
<<
7624 ", marker=" << marker
<< dendl
;
7626 for (const auto& entry
: entries
) {
7627 encode_json("entry", entry
, formatter
.get());
7630 formatter
->flush(cout
);
7631 } while (is_truncated
);
7633 formatter
->flush(cout
);
7635 if (specified_shard_id
) {
7639 ldpp_dout(dpp(), 20) << "INFO: " << __func__
<< ": done" << dendl
;
7641 formatter
->close_section();
7642 formatter
->flush(cout
);
7645 if (opt_cmd
== OPT::BI_PURGE
) {
7646 if (bucket_name
.empty()) {
7647 cerr
<< "ERROR: bucket name not specified" << std::endl
;
7650 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7652 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7656 std::unique_ptr
<rgw::sal::Bucket
> cur_bucket
;
7657 ret
= init_bucket(user
.get(), tenant
, bucket_name
, string(), &cur_bucket
);
7658 if (ret
== -ENOENT
) {
7659 // no bucket entrypoint
7660 } else if (ret
< 0) {
7661 cerr
<< "ERROR: could not init current bucket info for bucket_name=" << bucket_name
<< ": " << cpp_strerror(-ret
) << std::endl
;
7663 } else if (cur_bucket
->get_bucket_id() == bucket
->get_bucket_id() &&
7664 !yes_i_really_mean_it
) {
7665 cerr
<< "specified bucket instance points to a current bucket instance" << std::endl
;
7666 cerr
<< "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
7670 const auto& index
= bucket
->get_info().layout
.current_index
;
7671 if (index
.layout
.type
== rgw::BucketIndexType::Indexless
) {
7672 cerr
<< "ERROR: indexless bucket has no index to purge" << std::endl
;
7676 const int max_shards
= rgw::num_shards(index
);
7677 for (int i
= 0; i
< max_shards
; i
++) {
7678 RGWRados::BucketShard
bs(static_cast<rgw::sal::RadosStore
*>(driver
)->getRados());
7679 int ret
= bs
.init(dpp(), bucket
->get_info(), index
, i
);
7681 cerr
<< "ERROR: bs.init(bucket=" << bucket
<< ", shard=" << i
<< "): " << cpp_strerror(-ret
) << std::endl
;
7685 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->bi_remove(dpp(), bs
);
7687 cerr
<< "ERROR: failed to remove bucket index object: " << cpp_strerror(-ret
) << std::endl
;
7693 if (opt_cmd
== OPT::OBJECT_PUT
) {
7694 if (bucket_name
.empty()) {
7695 cerr
<< "ERROR: bucket not specified" << std::endl
;
7698 if (object
.empty()) {
7699 cerr
<< "ERROR: object not specified" << std::endl
;
7703 RGWDataAccess
data_access(driver
);
7704 rgw_obj_key
key(object
, object_version
);
7706 RGWDataAccess::BucketRef b
;
7707 RGWDataAccess::ObjectRef obj
;
7709 int ret
= data_access
.get_bucket(dpp(), tenant
, bucket_name
, bucket_id
, &b
, null_yield
);
7711 cerr
<< "ERROR: failed to init bucket: " << cpp_strerror(-ret
) << std::endl
;
7715 ret
= b
->get_object(key
, &obj
);
7717 cerr
<< "ERROR: failed to get object: " << cpp_strerror(-ret
) << std::endl
;
7722 ret
= read_input(infile
, bl
);
7724 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
7727 map
<string
, bufferlist
> attrs
;
7728 ret
= obj
->put(bl
, attrs
, dpp(), null_yield
);
7730 cerr
<< "ERROR: put object returned error: " << cpp_strerror(-ret
) << std::endl
;
7734 if (opt_cmd
== OPT::OBJECT_RM
) {
7735 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7737 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7740 rgw_obj_key
key(object
, object_version
);
7741 ret
= rgw_remove_object(dpp(), driver
, bucket
.get(), key
);
7744 cerr
<< "ERROR: object remove returned: " << cpp_strerror(-ret
) << std::endl
;
7749 if (opt_cmd
== OPT::OBJECT_REWRITE
) {
7750 if (bucket_name
.empty()) {
7751 cerr
<< "ERROR: bucket not specified" << std::endl
;
7754 if (object
.empty()) {
7755 cerr
<< "ERROR: object not specified" << std::endl
;
7759 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7761 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7765 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(object
);
7766 obj
->set_instance(object_version
);
7767 bool need_rewrite
= true;
7768 if (min_rewrite_stripe_size
> 0) {
7769 ret
= check_min_obj_stripe_size(driver
, obj
.get(), min_rewrite_stripe_size
, &need_rewrite
);
7771 ldpp_dout(dpp(), 0) << "WARNING: check_min_obj_stripe_size failed, r=" << ret
<< dendl
;
7775 RGWRados
* store
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados();
7776 ret
= store
->rewrite_obj(bucket
->get_info(), obj
->get_obj(), dpp(), null_yield
);
7778 cerr
<< "ERROR: object rewrite returned: " << cpp_strerror(-ret
) << std::endl
;
7782 ldpp_dout(dpp(), 20) << "skipped object" << dendl
;
7784 } // OPT::OBJECT_REWRITE
7786 if (opt_cmd
== OPT::OBJECT_REINDEX
) {
7787 if (bucket_name
.empty()) {
7788 cerr
<< "ERROR: --bucket not specified." << std::endl
;
7791 if (object
.empty() && objects_file
.empty()) {
7792 cerr
<< "ERROR: neither --object nor --objects-file specified." << std::endl
;
7794 } else if (!object
.empty() && !objects_file
.empty()) {
7795 cerr
<< "ERROR: both --object and --objects-file specified and only one is allowed." << std::endl
;
7797 } else if (!objects_file
.empty() && !object_version
.empty()) {
7798 cerr
<< "ERROR: cannot specify --object_version when --objects-file specified." << std::endl
;
7802 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7804 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) <<
7809 rgw::sal::RadosStore
* rados_store
= dynamic_cast<rgw::sal::RadosStore
*>(driver
);
7812 "ERROR: this command can only work when the cluster has a RADOS backing store." <<
7816 RGWRados
* store
= rados_store
->getRados();
7818 auto process
= [&](const std::string
& p_object
, const std::string
& p_object_version
) -> int {
7819 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(p_object
);
7820 obj
->set_instance(p_object_version
);
7821 ret
= store
->reindex_obj(bucket
->get_info(), obj
->get_obj(), dpp(), null_yield
);
7828 if (!object
.empty()) {
7829 ret
= process(object
, object_version
);
7835 file
.open(objects_file
);
7836 if (!file
.is_open()) {
7837 std::cerr
<< "ERROR: unable to open objects-file \"" <<
7838 objects_file
<< "\"." << std::endl
;
7842 std::string obj_name
;
7843 const std::string empty_version
;
7844 while (std::getline(file
, obj_name
)) {
7845 ret
= process(obj_name
, empty_version
);
7847 std::cerr
<< "ERROR: while processing \"" << obj_name
<<
7848 "\", received " << cpp_strerror(-ret
) << "." << std::endl
;
7849 if (!yes_i_really_mean_it
) {
7851 "NOTE: with *caution* you can use --yes-i-really-mean-it to push through errors and continue processing." <<
7858 } // OPT::OBJECT_REINDEX
7860 if (opt_cmd
== OPT::OBJECTS_EXPIRE
) {
7861 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->process_expire_objects(dpp())) {
7862 cerr
<< "ERROR: process_expire_objects() processing returned error." << std::endl
;
7867 if (opt_cmd
== OPT::OBJECTS_EXPIRE_STALE_LIST
) {
7868 ret
= RGWBucketAdminOp::fix_obj_expiry(driver
, bucket_op
, stream_flusher
, dpp(), true);
7870 cerr
<< "ERROR: listing returned " << cpp_strerror(-ret
) << std::endl
;
7875 if (opt_cmd
== OPT::OBJECTS_EXPIRE_STALE_RM
) {
7876 ret
= RGWBucketAdminOp::fix_obj_expiry(driver
, bucket_op
, stream_flusher
, dpp(), false);
7878 cerr
<< "ERROR: removing returned " << cpp_strerror(-ret
) << std::endl
;
7883 if (opt_cmd
== OPT::BUCKET_REWRITE
) {
7884 if (bucket_name
.empty()) {
7885 cerr
<< "ERROR: bucket not specified" << std::endl
;
7889 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
7891 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
7895 uint64_t start_epoch
= 0;
7896 uint64_t end_epoch
= 0;
7898 if (!end_date
.empty()) {
7899 int ret
= utime_t::parse_date(end_date
, &end_epoch
, NULL
);
7901 cerr
<< "ERROR: failed to parse end date" << std::endl
;
7905 if (!start_date
.empty()) {
7906 int ret
= utime_t::parse_date(start_date
, &start_epoch
, NULL
);
7908 cerr
<< "ERROR: failed to parse start date" << std::endl
;
7913 bool is_truncated
= true;
7914 bool cls_filtered
= true;
7916 rgw_obj_index_key marker
;
7917 string empty_prefix
;
7918 string empty_delimiter
;
7920 formatter
->open_object_section("result");
7921 formatter
->dump_string("bucket", bucket_name
);
7922 formatter
->open_array_section("objects");
7924 constexpr uint32_t NUM_ENTRIES
= 1000;
7925 uint16_t expansion_factor
= 1;
7926 while (is_truncated
) {
7927 RGWRados::ent_map_t result
;
7928 result
.reserve(NUM_ENTRIES
);
7930 const auto& current_index
= bucket
->get_info().layout
.current_index
;
7931 int r
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->cls_bucket_list_ordered(
7932 dpp(), bucket
->get_info(), current_index
, RGW_NO_SHARD
,
7933 marker
, empty_prefix
, empty_delimiter
,
7934 NUM_ENTRIES
, true, expansion_factor
,
7935 result
, &is_truncated
, &cls_filtered
, &marker
,
7937 rgw_bucket_object_check_filter
);
7938 if (r
< 0 && r
!= -ENOENT
) {
7939 cerr
<< "ERROR: failed operation r=" << r
<< std::endl
;
7940 } else if (r
== -ENOENT
) {
7944 if (result
.size() < NUM_ENTRIES
/ 8) {
7946 } else if (result
.size() > NUM_ENTRIES
* 7 / 8 &&
7947 expansion_factor
> 1) {
7951 for (auto iter
= result
.begin(); iter
!= result
.end(); ++iter
) {
7952 rgw_obj_key key
= iter
->second
.key
;
7953 rgw_bucket_dir_entry
& entry
= iter
->second
;
7955 formatter
->open_object_section("object");
7956 formatter
->dump_string("name", key
.name
);
7957 formatter
->dump_string("instance", key
.instance
);
7958 formatter
->dump_int("size", entry
.meta
.size
);
7959 utime_t
ut(entry
.meta
.mtime
);
7960 ut
.gmtime(formatter
->dump_stream("mtime"));
7962 if ((entry
.meta
.size
< min_rewrite_size
) ||
7963 (entry
.meta
.size
> max_rewrite_size
) ||
7964 (start_epoch
> 0 && start_epoch
> (uint64_t)ut
.sec()) ||
7965 (end_epoch
> 0 && end_epoch
< (uint64_t)ut
.sec())) {
7966 formatter
->dump_string("status", "Skipped");
7968 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(key
);
7970 bool need_rewrite
= true;
7971 if (min_rewrite_stripe_size
> 0) {
7972 r
= check_min_obj_stripe_size(driver
, obj
.get(), min_rewrite_stripe_size
, &need_rewrite
);
7974 ldpp_dout(dpp(), 0) << "WARNING: check_min_obj_stripe_size failed, r=" << r
<< dendl
;
7977 if (!need_rewrite
) {
7978 formatter
->dump_string("status", "Skipped");
7980 RGWRados
* store
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados();
7981 r
= store
->rewrite_obj(bucket
->get_info(), obj
->get_obj(), dpp(), null_yield
);
7983 formatter
->dump_string("status", "Success");
7985 formatter
->dump_string("status", cpp_strerror(-r
));
7989 formatter
->dump_int("flags", entry
.flags
);
7991 formatter
->close_section();
7992 formatter
->flush(cout
);
7995 formatter
->close_section();
7996 formatter
->close_section();
7997 formatter
->flush(cout
);
8000 if (opt_cmd
== OPT::BUCKET_RESHARD
) {
8001 int ret
= check_reshard_bucket_params(driver
,
8005 num_shards_specified
,
8007 yes_i_really_mean_it
,
8013 auto zone_svc
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
;
8014 if (!zone_svc
->can_reshard()) {
8015 const auto& zonegroup
= zone_svc
->get_zonegroup();
8016 std::cerr
<< "The zonegroup '" << zonegroup
.get_name() << "' does not "
8017 "have the resharding feature enabled." << std::endl
;
8020 if (!RGWBucketReshard::can_reshard(bucket
->get_info(), zone_svc
) &&
8021 !yes_i_really_mean_it
) {
8022 std::cerr
<< "Bucket '" << bucket
->get_name() << "' already has too many "
8023 "log generations (" << bucket
->get_info().layout
.logs
.size() << ") "
8024 "from previous reshards that peer zones haven't finished syncing. "
8025 "Resharding is not recommended until the old generations sync, but "
8026 "you can force a reshard with --yes-i-really-mean-it." << std::endl
;
8030 RGWBucketReshard
br(static_cast<rgw::sal::RadosStore
*>(driver
),
8031 bucket
->get_info(), bucket
->get_attrs(),
8032 nullptr /* no callback */);
8034 #define DEFAULT_RESHARD_MAX_ENTRIES 1000
8035 if (max_entries
< 1) {
8036 max_entries
= DEFAULT_RESHARD_MAX_ENTRIES
;
8039 ReshardFaultInjector fault
;
8040 if (inject_error_at
) {
8041 const int code
= -inject_error_code
.value_or(EIO
);
8042 fault
.inject(*inject_error_at
, InjectError
{code
, dpp()});
8043 } else if (inject_abort_at
) {
8044 fault
.inject(*inject_abort_at
, InjectAbort
{});
8046 ret
= br
.execute(num_shards
, fault
, max_entries
, dpp(),
8047 verbose
, &cout
, formatter
.get());
8051 if (opt_cmd
== OPT::RESHARD_ADD
) {
8052 int ret
= check_reshard_bucket_params(driver
,
8056 num_shards_specified
,
8058 yes_i_really_mean_it
,
8064 int num_source_shards
= rgw::current_num_shards(bucket
->get_info().layout
);
8066 RGWReshard
reshard(static_cast<rgw::sal::RadosStore
*>(driver
), dpp());
8067 cls_rgw_reshard_entry entry
;
8068 entry
.time
= real_clock::now();
8069 entry
.tenant
= tenant
;
8070 entry
.bucket_name
= bucket_name
;
8071 entry
.bucket_id
= bucket
->get_info().bucket
.bucket_id
;
8072 entry
.old_num_shards
= num_source_shards
;
8073 entry
.new_num_shards
= num_shards
;
8075 return reshard
.add(dpp(), entry
);
8078 if (opt_cmd
== OPT::RESHARD_LIST
) {
8081 if (max_entries
< 0) {
8086 driver
->ctx()->_conf
.get_val
<uint64_t>("rgw_reshard_num_logs");
8088 RGWReshard
reshard(static_cast<rgw::sal::RadosStore
*>(driver
), dpp());
8090 formatter
->open_array_section("reshard");
8091 for (int i
= 0; i
< num_logshards
; i
++) {
8092 bool is_truncated
= true;
8095 std::list
<cls_rgw_reshard_entry
> entries
;
8096 ret
= reshard
.list(dpp(), i
, marker
, max_entries
- count
, entries
, &is_truncated
);
8098 cerr
<< "Error listing resharding buckets: " << cpp_strerror(-ret
) << std::endl
;
8101 for (const auto& entry
: entries
) {
8102 encode_json("entry", entry
, formatter
.get());
8105 entries
.crbegin()->get_key(&marker
); // last entry's key becomes marker
8107 count
+= entries
.size();
8108 formatter
->flush(cout
);
8109 } while (is_truncated
&& count
< max_entries
);
8111 if (count
>= max_entries
) {
8116 formatter
->close_section();
8117 formatter
->flush(cout
);
8122 if (opt_cmd
== OPT::RESHARD_STATUS
) {
8123 if (bucket_name
.empty()) {
8124 cerr
<< "ERROR: bucket not specified" << std::endl
;
8128 ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8130 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8134 RGWBucketReshard
br(static_cast<rgw::sal::RadosStore
*>(driver
),
8135 bucket
->get_info(), bucket
->get_attrs(),
8136 nullptr /* no callback */);
8137 list
<cls_rgw_bucket_instance_entry
> status
;
8138 int r
= br
.get_status(dpp(), &status
);
8140 cerr
<< "ERROR: could not get resharding status for bucket " <<
8141 bucket_name
<< std::endl
;
8145 show_reshard_status(status
, formatter
.get());
8148 if (opt_cmd
== OPT::RESHARD_PROCESS
) {
8149 RGWReshard
reshard(static_cast<rgw::sal::RadosStore
*>(driver
), true, &cout
);
8151 int ret
= reshard
.process_all_logshards(dpp());
8153 cerr
<< "ERROR: failed to process reshard logs, error=" << cpp_strerror(-ret
) << std::endl
;
8158 if (opt_cmd
== OPT::RESHARD_CANCEL
) {
8159 if (bucket_name
.empty()) {
8160 cerr
<< "ERROR: bucket not specified" << std::endl
;
8164 bool bucket_initable
= true;
8165 ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8167 if (yes_i_really_mean_it
) {
8168 bucket_initable
= false;
8170 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) <<
8171 "; if you want to cancel the reshard request nonetheless, please "
8172 "use the --yes-i-really-mean-it option" << std::endl
;
8177 bool resharding_underway
= true;
8179 if (bucket_initable
) {
8180 // we did not encounter an error, so let's work with the bucket
8181 RGWBucketReshard
br(static_cast<rgw::sal::RadosStore
*>(driver
),
8182 bucket
->get_info(), bucket
->get_attrs(),
8183 nullptr /* no callback */);
8184 int ret
= br
.cancel(dpp());
8186 if (ret
== -EBUSY
) {
8187 cerr
<< "There is ongoing resharding, please retry after " <<
8188 driver
->ctx()->_conf
.get_val
<uint64_t>("rgw_reshard_bucket_lock_duration") <<
8189 " seconds." << std::endl
;
8191 } else if (ret
== -EINVAL
) {
8192 resharding_underway
= false;
8193 // we can continue and try to unschedule
8195 cerr
<< "Error cancelling bucket \"" << bucket_name
<<
8196 "\" resharding: " << cpp_strerror(-ret
) << std::endl
;
8202 RGWReshard
reshard(static_cast<rgw::sal::RadosStore
*>(driver
), dpp());
8204 cls_rgw_reshard_entry entry
;
8205 entry
.tenant
= tenant
;
8206 entry
.bucket_name
= bucket_name
;
8208 ret
= reshard
.remove(dpp(), entry
);
8209 if (ret
== -ENOENT
) {
8210 if (!resharding_underway
) {
8211 cerr
<< "Error, bucket \"" << bucket_name
<<
8212 "\" is neither undergoing resharding nor scheduled to undergo "
8213 "resharding." << std::endl
;
8216 // we cancelled underway resharding above, so we're good
8219 } else if (ret
< 0) {
8220 cerr
<< "Error in updating reshard log with bucket \"" <<
8221 bucket_name
<< "\": " << cpp_strerror(-ret
) << std::endl
;
8224 } // OPT_RESHARD_CANCEL
8226 if (opt_cmd
== OPT::OBJECT_UNLINK
) {
8227 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8229 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8232 list
<rgw_obj_index_key
> oid_list
;
8233 rgw_obj_key
key(object
, object_version
);
8234 rgw_obj_index_key index_key
;
8235 key
.get_index_key(&index_key
);
8236 oid_list
.push_back(index_key
);
8238 // note: under rados this removes directly from rados index objects
8239 ret
= bucket
->remove_objs_from_index(dpp(), oid_list
);
8241 cerr
<< "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret
) << std::endl
;
8246 if (opt_cmd
== OPT::OBJECT_STAT
) {
8247 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8249 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8252 std::unique_ptr
<rgw::sal::Object
> obj
= bucket
->get_object(object
);
8253 obj
->set_instance(object_version
);
8255 ret
= obj
->get_obj_attrs(null_yield
, dpp());
8257 cerr
<< "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret
) << std::endl
;
8260 formatter
->open_object_section("object_metadata");
8261 formatter
->dump_string("name", object
);
8262 formatter
->dump_unsigned("size", obj
->get_obj_size());
8264 map
<string
, bufferlist
>::iterator iter
;
8265 map
<string
, bufferlist
> other_attrs
;
8266 for (iter
= obj
->get_attrs().begin(); iter
!= obj
->get_attrs().end(); ++iter
) {
8267 bufferlist
& bl
= iter
->second
;
8268 bool handled
= false;
8269 if (iter
->first
== RGW_ATTR_MANIFEST
) {
8270 handled
= decode_dump
<RGWObjManifest
>("manifest", bl
, formatter
.get());
8271 } else if (iter
->first
== RGW_ATTR_ACL
) {
8272 handled
= decode_dump
<RGWAccessControlPolicy
>("policy", bl
, formatter
.get());
8273 } else if (iter
->first
== RGW_ATTR_ID_TAG
) {
8274 handled
= dump_string("tag", bl
, formatter
.get());
8275 } else if (iter
->first
== RGW_ATTR_ETAG
) {
8276 handled
= dump_string("etag", bl
, formatter
.get());
8277 } else if (iter
->first
== RGW_ATTR_COMPRESSION
) {
8278 handled
= decode_dump
<RGWCompressionInfo
>("compression", bl
, formatter
.get());
8279 } else if (iter
->first
== RGW_ATTR_DELETE_AT
) {
8280 handled
= decode_dump
<utime_t
>("delete_at", bl
, formatter
.get());
8284 other_attrs
[iter
->first
] = bl
;
8287 formatter
->open_object_section("attrs");
8288 for (iter
= other_attrs
.begin(); iter
!= other_attrs
.end(); ++iter
) {
8289 dump_string(iter
->first
.c_str(), iter
->second
, formatter
.get());
8291 formatter
->close_section();
8292 formatter
->close_section();
8293 formatter
->flush(cout
);
8296 if (opt_cmd
== OPT::BUCKET_CHECK
) {
8297 if (check_head_obj_locator
) {
8298 if (bucket_name
.empty()) {
8299 cerr
<< "ERROR: need to specify bucket name" << std::endl
;
8302 do_check_object_locator(tenant
, bucket_name
, fix
, remove_bad
, formatter
.get());
8304 RGWBucketAdminOp::check_index(driver
, bucket_op
, stream_flusher
, null_yield
, dpp());
8308 if (opt_cmd
== OPT::BUCKET_RM
) {
8309 if (!inconsistent_index
) {
8310 RGWBucketAdminOp::remove_bucket(driver
, bucket_op
, null_yield
, dpp(), bypass_gc
, true);
8312 if (!yes_i_really_mean_it
) {
8313 cerr
<< "using --inconsistent_index can corrupt the bucket index " << std::endl
8314 << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl
;
8317 RGWBucketAdminOp::remove_bucket(driver
, bucket_op
, null_yield
, dpp(), bypass_gc
, false);
8321 if (opt_cmd
== OPT::GC_LIST
) {
8324 bool processing_queue
= false;
8325 formatter
->open_array_section("entries");
8328 list
<cls_rgw_gc_obj_info
> result
;
8329 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->list_gc_objs(&index
, marker
, 1000, !include_all
, result
, &truncated
, processing_queue
);
8331 cerr
<< "ERROR: failed to list objs: " << cpp_strerror(-ret
) << std::endl
;
8336 list
<cls_rgw_gc_obj_info
>::iterator iter
;
8337 for (iter
= result
.begin(); iter
!= result
.end(); ++iter
) {
8338 cls_rgw_gc_obj_info
& info
= *iter
;
8339 formatter
->open_object_section("chain_info");
8340 formatter
->dump_string("tag", info
.tag
);
8341 formatter
->dump_stream("time") << info
.time
;
8342 formatter
->open_array_section("objs");
8343 list
<cls_rgw_obj
>::iterator liter
;
8344 cls_rgw_obj_chain
& chain
= info
.chain
;
8345 for (liter
= chain
.objs
.begin(); liter
!= chain
.objs
.end(); ++liter
) {
8346 cls_rgw_obj
& obj
= *liter
;
8347 encode_json("obj", obj
, formatter
.get());
8349 formatter
->close_section(); // objs
8350 formatter
->close_section(); // obj_chain
8351 formatter
->flush(cout
);
8353 } while (truncated
);
8354 formatter
->close_section();
8355 formatter
->flush(cout
);
8358 if (opt_cmd
== OPT::GC_PROCESS
) {
8359 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->process_gc(!include_all
);
8361 cerr
<< "ERROR: gc processing returned error: " << cpp_strerror(-ret
) << std::endl
;
8366 if (opt_cmd
== OPT::LC_LIST
) {
8367 formatter
->open_array_section("lifecycle_list");
8368 vector
<std::unique_ptr
<rgw::sal::Lifecycle::LCEntry
>> bucket_lc_map
;
8371 #define MAX_LC_LIST_ENTRIES 100
8372 if (max_entries
< 0) {
8373 max_entries
= MAX_LC_LIST_ENTRIES
;
8376 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->list_lc_progress(marker
, max_entries
,
8377 bucket_lc_map
, index
);
8379 cerr
<< "ERROR: failed to list objs: " << cpp_strerror(-ret
)
8383 for (const auto& entry
: bucket_lc_map
) {
8384 formatter
->open_object_section("bucket_lc_info");
8385 formatter
->dump_string("bucket", entry
->get_bucket());
8386 formatter
->dump_string("shard", entry
->get_oid());
8388 time_t t
{time_t(entry
->get_start_time())};
8390 exp_buf
, sizeof(exp_buf
),
8391 "%a, %d %b %Y %T %Z", std::gmtime(&t
))) {
8392 formatter
->dump_string("started", exp_buf
);
8394 string lc_status
= LC_STATUS
[entry
->get_status()];
8395 formatter
->dump_string("status", lc_status
);
8396 formatter
->close_section(); // objs
8397 formatter
->flush(cout
);
8399 } while (!bucket_lc_map
.empty());
8401 formatter
->close_section(); //lifecycle list
8402 formatter
->flush(cout
);
8406 if (opt_cmd
== OPT::LC_GET
) {
8407 if (bucket_name
.empty()) {
8408 cerr
<< "ERROR: bucket not specified" << std::endl
;
8412 RGWLifecycleConfiguration config
;
8413 ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8415 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8419 auto aiter
= bucket
->get_attrs().find(RGW_ATTR_LC
);
8420 if (aiter
== bucket
->get_attrs().end()) {
8424 bufferlist::const_iterator iter
{&aiter
->second
};
8426 config
.decode(iter
);
8427 } catch (const buffer::error
& e
) {
8428 cerr
<< "ERROR: decode life cycle config failed" << std::endl
;
8432 encode_json("result", config
, formatter
.get());
8433 formatter
->flush(cout
);
8436 if (opt_cmd
== OPT::LC_PROCESS
) {
8437 if ((! bucket_name
.empty()) ||
8438 (! bucket_id
.empty())) {
8439 int ret
= init_bucket(nullptr, tenant
, bucket_name
, bucket_id
, &bucket
);
8441 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
)
8448 static_cast<rgw::sal::RadosStore
*>(driver
)->getRados()->process_lc(bucket
);
8450 cerr
<< "ERROR: lc processing returned error: " << cpp_strerror(-ret
) << std::endl
;
8455 if (opt_cmd
== OPT::LC_RESHARD_FIX
) {
8456 ret
= RGWBucketAdminOp::fix_lc_shards(driver
, bucket_op
, stream_flusher
, dpp());
8458 cerr
<< "ERROR: fixing lc shards: " << cpp_strerror(-ret
) << std::endl
;
8463 if (opt_cmd
== OPT::ORPHANS_FIND
) {
8464 if (!yes_i_really_mean_it
) {
8465 cerr
<< "this command is now deprecated; please consider using the rgw-orphan-list tool; "
8466 << "accidental removal of active objects cannot be reversed; "
8467 << "do you really mean it? (requires --yes-i-really-mean-it)"
8471 cerr
<< "IMPORTANT: this command is now deprecated; please consider using the rgw-orphan-list tool"
8475 RGWOrphanSearch
search(static_cast<rgw::sal::RadosStore
*>(driver
), max_concurrent_ios
, orphan_stale_secs
);
8477 if (job_id
.empty()) {
8478 cerr
<< "ERROR: --job-id not specified" << std::endl
;
8481 if (pool_name
.empty()) {
8482 cerr
<< "ERROR: --pool not specified" << std::endl
;
8486 RGWOrphanSearchInfo info
;
8489 info
.job_name
= job_id
;
8490 info
.num_shards
= num_shards
;
8492 int ret
= search
.init(dpp(), job_id
, &info
, detail
);
8494 cerr
<< "could not init search, ret=" << ret
<< std::endl
;
8497 ret
= search
.run(dpp());
8503 if (opt_cmd
== OPT::ORPHANS_FINISH
) {
8504 if (!yes_i_really_mean_it
) {
8505 cerr
<< "this command is now deprecated; please consider using the rgw-orphan-list tool; "
8506 << "accidental removal of active objects cannot be reversed; "
8507 << "do you really mean it? (requires --yes-i-really-mean-it)"
8511 cerr
<< "IMPORTANT: this command is now deprecated; please consider using the rgw-orphan-list tool"
8515 RGWOrphanSearch
search(static_cast<rgw::sal::RadosStore
*>(driver
), max_concurrent_ios
, orphan_stale_secs
);
8517 if (job_id
.empty()) {
8518 cerr
<< "ERROR: --job-id not specified" << std::endl
;
8521 int ret
= search
.init(dpp(), job_id
, NULL
);
8523 if (ret
== -ENOENT
) {
8524 cerr
<< "job not found" << std::endl
;
8528 ret
= search
.finish();
8534 if (opt_cmd
== OPT::ORPHANS_LIST_JOBS
){
8535 if (!yes_i_really_mean_it
) {
8536 cerr
<< "this command is now deprecated; please consider using the rgw-orphan-list tool; "
8537 << "do you really mean it? (requires --yes-i-really-mean-it)"
8541 cerr
<< "IMPORTANT: this command is now deprecated; please consider using the rgw-orphan-list tool"
8545 RGWOrphanStore
orphan_store(static_cast<rgw::sal::RadosStore
*>(driver
));
8546 int ret
= orphan_store
.init(dpp());
8548 cerr
<< "connection to cluster failed!" << std::endl
;
8552 map
<string
,RGWOrphanSearchState
> m
;
8553 ret
= orphan_store
.list_jobs(m
);
8555 cerr
<< "job list failed" << std::endl
;
8558 formatter
->open_array_section("entries");
8559 for (const auto &it
: m
){
8561 formatter
->dump_string("job-id",it
.first
);
8563 encode_json("orphan_search_state", it
.second
, formatter
.get());
8566 formatter
->close_section();
8567 formatter
->flush(cout
);
8570 if (opt_cmd
== OPT::USER_CHECK
) {
8571 check_bad_user_bucket_mapping(driver
, *user
.get(), fix
, null_yield
, dpp());
8574 if (opt_cmd
== OPT::USER_STATS
) {
8575 if (rgw::sal::User::empty(user
)) {
8576 cerr
<< "ERROR: uid not specified" << std::endl
;
8580 if (!bucket_name
.empty()) {
8581 cerr
<< "ERROR: --reset-stats does not work on buckets and "
8582 "bucket specified" << std::endl
;
8586 cerr
<< "ERROR: sync-stats includes the reset-stats functionality, "
8587 "so at most one of the two should be specified" << std::endl
;
8590 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->user
->reset_bucket_stats(dpp(), user
->get_id(), null_yield
);
8592 cerr
<< "ERROR: could not reset user stats: " << cpp_strerror(-ret
) <<
8599 if (!bucket_name
.empty()) {
8600 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
8602 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
8605 ret
= bucket
->sync_user_stats(dpp(), null_yield
);
8607 cerr
<< "ERROR: could not sync bucket stats: " <<
8608 cpp_strerror(-ret
) << std::endl
;
8612 int ret
= rgw_user_sync_all_stats(dpp(), driver
, user
.get(), null_yield
);
8614 cerr
<< "ERROR: could not sync user stats: " <<
8615 cpp_strerror(-ret
) << std::endl
;
8621 constexpr bool omit_utilized_stats
= false;
8622 RGWStorageStats
stats(omit_utilized_stats
);
8623 ceph::real_time last_stats_sync
;
8624 ceph::real_time last_stats_update
;
8625 int ret
= user
->read_stats(dpp(), null_yield
, &stats
, &last_stats_sync
, &last_stats_update
);
8627 if (ret
== -ENOENT
) { /* in case of ENOENT */
8628 cerr
<< "User has not been initialized or user does not exist" << std::endl
;
8630 cerr
<< "ERROR: can't read user: " << cpp_strerror(ret
) << std::endl
;
8637 Formatter::ObjectSection
os(*formatter
, "result");
8638 encode_json("stats", stats
, formatter
.get());
8639 utime_t
last_sync_ut(last_stats_sync
);
8640 encode_json("last_stats_sync", last_sync_ut
, formatter
.get());
8641 utime_t
last_update_ut(last_stats_update
);
8642 encode_json("last_stats_update", last_update_ut
, formatter
.get());
8644 formatter
->flush(cout
);
8647 if (opt_cmd
== OPT::METADATA_GET
) {
8648 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->get(metadata_key
, formatter
.get(), null_yield
, dpp());
8650 cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
8654 formatter
->flush(cout
);
8657 if (opt_cmd
== OPT::METADATA_PUT
) {
8659 int ret
= read_input(infile
, bl
);
8661 cerr
<< "ERROR: failed to read input: " << cpp_strerror(-ret
) << std::endl
;
8664 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->put(metadata_key
, bl
, null_yield
, dpp(), RGWMDLogSyncType::APPLY_ALWAYS
, false);
8666 cerr
<< "ERROR: can't put key: " << cpp_strerror(-ret
) << std::endl
;
8671 if (opt_cmd
== OPT::METADATA_RM
) {
8672 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->remove(metadata_key
, null_yield
, dpp());
8674 cerr
<< "ERROR: can't remove key: " << cpp_strerror(-ret
) << std::endl
;
8679 if (opt_cmd
== OPT::METADATA_LIST
|| opt_cmd
== OPT::USER_LIST
) {
8680 if (opt_cmd
== OPT::USER_LIST
) {
8681 metadata_key
= "user";
8685 int ret
= driver
->meta_list_keys_init(dpp(), metadata_key
, marker
, &handle
);
8687 cerr
<< "ERROR: can't get key: " << cpp_strerror(-ret
) << std::endl
;
8694 if (max_entries_specified
) {
8695 formatter
->open_object_section("result");
8697 formatter
->open_array_section("keys");
8702 left
= (max_entries_specified
? max_entries
- count
: max
);
8703 ret
= driver
->meta_list_keys_next(dpp(), handle
, left
, keys
, &truncated
);
8704 if (ret
< 0 && ret
!= -ENOENT
) {
8705 cerr
<< "ERROR: lists_keys_next(): " << cpp_strerror(-ret
) << std::endl
;
8707 } if (ret
!= -ENOENT
) {
8708 for (list
<string
>::iterator iter
= keys
.begin(); iter
!= keys
.end(); ++iter
) {
8709 formatter
->dump_string("key", *iter
);
8712 formatter
->flush(cout
);
8714 } while (truncated
&& left
> 0);
8716 formatter
->close_section();
8718 if (max_entries_specified
) {
8719 encode_json("truncated", truncated
, formatter
.get());
8720 encode_json("count", count
, formatter
.get());
8722 encode_json("marker", driver
->meta_get_marker(handle
), formatter
.get());
8724 formatter
->close_section();
8726 formatter
->flush(cout
);
8728 driver
->meta_list_keys_complete(handle
);
8731 if (opt_cmd
== OPT::MDLOG_LIST
) {
8732 if (!start_date
.empty()) {
8733 std::cerr
<< "start-date not allowed." << std::endl
;
8736 if (!end_date
.empty()) {
8737 std::cerr
<< "end-date not allowed." << std::endl
;
8740 if (!end_marker
.empty()) {
8741 std::cerr
<< "end-marker not allowed." << std::endl
;
8744 if (!start_marker
.empty()) {
8745 if (marker
.empty()) {
8746 marker
= start_marker
;
8748 std::cerr
<< "start-marker and marker not both allowed." << std::endl
;
8753 int i
= (specified_shard_id
? shard_id
: 0);
8755 if (period_id
.empty()) {
8756 // use realm's current period
8758 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
8759 realm_id
, realm_name
, realm
);
8761 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
8764 period_id
= realm
.current_period
;
8765 std::cerr
<< "No --period given, using current period="
8766 << period_id
<< std::endl
;
8768 RGWMetadataLog
*meta_log
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->mdlog
->get_log(period_id
);
8770 formatter
->open_array_section("entries");
8771 for (; i
< g_ceph_context
->_conf
->rgw_md_log_max_shards
; i
++) {
8773 list
<cls_log_entry
> entries
;
8775 meta_log
->init_list_entries(i
, {}, {}, marker
, &handle
);
8778 int ret
= meta_log
->list_entries(dpp(), handle
, 1000, entries
, NULL
, &truncated
);
8780 cerr
<< "ERROR: meta_log->list_entries(): " << cpp_strerror(-ret
) << std::endl
;
8784 for (list
<cls_log_entry
>::iterator iter
= entries
.begin(); iter
!= entries
.end(); ++iter
) {
8785 cls_log_entry
& entry
= *iter
;
8786 static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->dump_log_entry(entry
, formatter
.get());
8788 formatter
->flush(cout
);
8789 } while (truncated
);
8791 meta_log
->complete_list_entries(handle
);
8793 if (specified_shard_id
)
8798 formatter
->close_section();
8799 formatter
->flush(cout
);
8802 if (opt_cmd
== OPT::MDLOG_STATUS
) {
8803 int i
= (specified_shard_id
? shard_id
: 0);
8805 if (period_id
.empty()) {
8806 // use realm's current period
8808 int ret
= rgw::read_realm(dpp(), null_yield
, cfgstore
.get(),
8809 realm_id
, realm_name
, realm
);
8811 cerr
<< "failed to load realm: " << cpp_strerror(-ret
) << std::endl
;
8814 period_id
= realm
.current_period
;
8815 std::cerr
<< "No --period given, using current period="
8816 << period_id
<< std::endl
;
8818 RGWMetadataLog
*meta_log
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->mdlog
->get_log(period_id
);
8820 formatter
->open_array_section("entries");
8822 for (; i
< g_ceph_context
->_conf
->rgw_md_log_max_shards
; i
++) {
8823 RGWMetadataLogInfo info
;
8824 meta_log
->get_info(dpp(), i
, &info
);
8826 ::encode_json("info", info
, formatter
.get());
8828 if (specified_shard_id
)
8833 formatter
->close_section();
8834 formatter
->flush(cout
);
8837 if (opt_cmd
== OPT::MDLOG_AUTOTRIM
) {
8838 // need a full history for purging old mdlog periods
8839 static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->mdlog
->init_oldest_log_period(null_yield
, dpp());
8841 RGWCoroutinesManager
crs(driver
->ctx(), driver
->get_cr_registry());
8842 RGWHTTPManager
http(driver
->ctx(), crs
.get_completion_mgr());
8843 int ret
= http
.start();
8845 cerr
<< "failed to initialize http client with " << cpp_strerror(ret
) << std::endl
;
8849 auto num_shards
= g_conf()->rgw_md_log_max_shards
;
8850 auto mltcr
= create_admin_meta_log_trim_cr(
8851 dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), &http
, num_shards
);
8853 cerr
<< "Cluster misconfigured! Unable to trim." << std::endl
;
8856 ret
= crs
.run(dpp(), mltcr
);
8858 cerr
<< "automated mdlog trim failed with " << cpp_strerror(ret
) << std::endl
;
8863 if (opt_cmd
== OPT::MDLOG_TRIM
) {
8864 if (!start_date
.empty()) {
8865 std::cerr
<< "start-date not allowed." << std::endl
;
8868 if (!end_date
.empty()) {
8869 std::cerr
<< "end-date not allowed." << std::endl
;
8872 if (!start_marker
.empty()) {
8873 std::cerr
<< "start-marker not allowed." << std::endl
;
8876 if (!end_marker
.empty()) {
8877 if (marker
.empty()) {
8878 marker
= end_marker
;
8880 std::cerr
<< "end-marker and marker not both allowed." << std::endl
;
8885 if (!specified_shard_id
) {
8886 cerr
<< "ERROR: shard-id must be specified for trim operation" << std::endl
;
8890 if (marker
.empty()) {
8891 cerr
<< "ERROR: marker must be specified for trim operation" << std::endl
;
8895 if (period_id
.empty()) {
8896 std::cerr
<< "missing --period argument" << std::endl
;
8899 RGWMetadataLog
*meta_log
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->mdlog
->get_log(period_id
);
8901 // trim until -ENODATA
8903 ret
= meta_log
->trim(dpp(), shard_id
, {}, {}, {}, marker
);
8905 if (ret
< 0 && ret
!= -ENODATA
) {
8906 cerr
<< "ERROR: meta_log->trim(): " << cpp_strerror(-ret
) << std::endl
;
8911 if (opt_cmd
== OPT::SYNC_INFO
) {
8912 sync_info(opt_effective_zone_id
, opt_bucket
, zone_formatter
.get());
8915 if (opt_cmd
== OPT::SYNC_STATUS
) {
8916 sync_status(formatter
.get());
8919 if (opt_cmd
== OPT::METADATA_SYNC_STATUS
) {
8920 RGWMetaSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor());
8922 int ret
= sync
.init(dpp());
8924 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
8928 rgw_meta_sync_status sync_status
;
8929 ret
= sync
.read_sync_status(dpp(), &sync_status
);
8931 cerr
<< "ERROR: sync.read_sync_status() returned ret=" << ret
<< std::endl
;
8935 formatter
->open_object_section("summary");
8936 encode_json("sync_status", sync_status
, formatter
.get());
8938 uint64_t full_total
= 0;
8939 uint64_t full_complete
= 0;
8941 for (auto marker_iter
: sync_status
.sync_markers
) {
8942 full_total
+= marker_iter
.second
.total_entries
;
8943 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::FullSync
) {
8944 full_complete
+= marker_iter
.second
.pos
;
8946 full_complete
+= marker_iter
.second
.total_entries
;
8950 formatter
->open_object_section("full_sync");
8951 encode_json("total", full_total
, formatter
.get());
8952 encode_json("complete", full_complete
, formatter
.get());
8953 formatter
->close_section();
8954 formatter
->dump_string("current_time",
8955 to_iso_8601(ceph::real_clock::now(),
8956 iso_8601_format::YMDhms
));
8957 formatter
->close_section();
8959 formatter
->flush(cout
);
8963 if (opt_cmd
== OPT::METADATA_SYNC_INIT
) {
8964 RGWMetaSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor());
8966 int ret
= sync
.init(dpp());
8968 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
8971 ret
= sync
.init_sync_status(dpp());
8973 cerr
<< "ERROR: sync.init_sync_status() returned ret=" << ret
<< std::endl
;
8979 if (opt_cmd
== OPT::METADATA_SYNC_RUN
) {
8980 RGWMetaSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor());
8982 int ret
= sync
.init(dpp());
8984 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
8988 ret
= sync
.run(dpp(), null_yield
);
8990 cerr
<< "ERROR: sync.run() returned ret=" << ret
<< std::endl
;
8995 if (opt_cmd
== OPT::DATA_SYNC_STATUS
) {
8996 if (source_zone
.empty()) {
8997 cerr
<< "ERROR: source zone not specified" << std::endl
;
9000 RGWDataSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor(), source_zone
, nullptr);
9002 int ret
= sync
.init(dpp());
9004 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
9008 rgw_data_sync_status sync_status
;
9009 if (specified_shard_id
) {
9010 set
<string
> pending_buckets
;
9011 set
<string
> recovering_buckets
;
9012 rgw_data_sync_marker sync_marker
;
9013 ret
= sync
.read_shard_status(dpp(), shard_id
, pending_buckets
, recovering_buckets
, &sync_marker
,
9014 max_entries_specified
? max_entries
: 20);
9015 if (ret
< 0 && ret
!= -ENOENT
) {
9016 cerr
<< "ERROR: sync.read_shard_status() returned ret=" << ret
<< std::endl
;
9019 formatter
->open_object_section("summary");
9020 encode_json("shard_id", shard_id
, formatter
.get());
9021 encode_json("marker", sync_marker
, formatter
.get());
9022 encode_json("pending_buckets", pending_buckets
, formatter
.get());
9023 encode_json("recovering_buckets", recovering_buckets
, formatter
.get());
9024 formatter
->dump_string("current_time",
9025 to_iso_8601(ceph::real_clock::now(),
9026 iso_8601_format::YMDhms
));
9027 formatter
->close_section();
9028 formatter
->flush(cout
);
9030 ret
= sync
.read_sync_status(dpp(), &sync_status
);
9031 if (ret
< 0 && ret
!= -ENOENT
) {
9032 cerr
<< "ERROR: sync.read_sync_status() returned ret=" << ret
<< std::endl
;
9036 formatter
->open_object_section("summary");
9037 encode_json("sync_status", sync_status
, formatter
.get());
9039 uint64_t full_total
= 0;
9040 uint64_t full_complete
= 0;
9042 for (auto marker_iter
: sync_status
.sync_markers
) {
9043 full_total
+= marker_iter
.second
.total_entries
;
9044 if (marker_iter
.second
.state
== rgw_meta_sync_marker::SyncState::FullSync
) {
9045 full_complete
+= marker_iter
.second
.pos
;
9047 full_complete
+= marker_iter
.second
.total_entries
;
9051 formatter
->open_object_section("full_sync");
9052 encode_json("total", full_total
, formatter
.get());
9053 encode_json("complete", full_complete
, formatter
.get());
9054 formatter
->close_section();
9055 formatter
->dump_string("current_time",
9056 to_iso_8601(ceph::real_clock::now(),
9057 iso_8601_format::YMDhms
));
9058 formatter
->close_section();
9060 formatter
->flush(cout
);
9064 if (opt_cmd
== OPT::DATA_SYNC_INIT
) {
9065 if (source_zone
.empty()) {
9066 cerr
<< "ERROR: source zone not specified" << std::endl
;
9070 RGWDataSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor(), source_zone
, nullptr);
9072 int ret
= sync
.init(dpp());
9074 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
9078 ret
= sync
.init_sync_status(dpp());
9080 cerr
<< "ERROR: sync.init_sync_status() returned ret=" << ret
<< std::endl
;
9085 if (opt_cmd
== OPT::DATA_SYNC_RUN
) {
9086 if (source_zone
.empty()) {
9087 cerr
<< "ERROR: source zone not specified" << std::endl
;
9091 RGWSyncModuleInstanceRef sync_module
;
9092 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->sync_modules
->get_manager()->create_instance(dpp(), g_ceph_context
, static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zone().tier_type
,
9093 static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->get_zone_params().tier_config
, &sync_module
);
9095 ldpp_dout(dpp(), -1) << "ERROR: failed to init sync module instance, ret=" << ret
<< dendl
;
9099 RGWDataSyncStatusManager
sync(static_cast<rgw::sal::RadosStore
*>(driver
), static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->rados
->get_async_processor(), source_zone
, nullptr, sync_module
);
9101 ret
= sync
.init(dpp());
9103 cerr
<< "ERROR: sync.init() returned ret=" << ret
<< std::endl
;
9107 ret
= sync
.run(dpp());
9109 cerr
<< "ERROR: sync.run() returned ret=" << ret
<< std::endl
;
9114 if (opt_cmd
== OPT::BUCKET_SYNC_INIT
) {
9115 if (source_zone
.empty()) {
9116 cerr
<< "ERROR: source zone not specified" << std::endl
;
9119 if (bucket_name
.empty()) {
9120 cerr
<< "ERROR: bucket not specified" << std::endl
;
9123 int ret
= init_bucket_for_sync(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9127 auto opt_sb
= opt_source_bucket
;
9128 if (opt_sb
&& opt_sb
->bucket_id
.empty()) {
9130 std::unique_ptr
<rgw::sal::Bucket
> sbuck
;
9131 int ret
= init_bucket_for_sync(user
.get(), opt_sb
->tenant
, opt_sb
->name
, sbid
, &sbuck
);
9135 opt_sb
= sbuck
->get_key();
9138 auto sync
= RGWBucketPipeSyncStatusManager::construct(
9139 dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), source_zone
, opt_sb
,
9140 bucket
->get_key(), extra_info
? &std::cout
: nullptr);
9143 cerr
<< "ERROR: sync.init() returned error=" << sync
.error() << std::endl
;
9144 return -sync
.error();
9146 ret
= (*sync
)->init_sync_status(dpp());
9148 cerr
<< "ERROR: sync.init_sync_status() returned ret=" << ret
<< std::endl
;
9153 if (opt_cmd
== OPT::BUCKET_SYNC_CHECKPOINT
) {
9154 std::optional
<rgw_zone_id
> opt_source_zone
;
9155 if (!source_zone
.empty()) {
9156 opt_source_zone
= source_zone
;
9158 if (bucket_name
.empty()) {
9159 cerr
<< "ERROR: bucket not specified" << std::endl
;
9162 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9167 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->bucket
->bucket_imports_data(bucket
->get_key(), null_yield
, dpp())) {
9168 std::cout
<< "Sync is disabled for bucket " << bucket_name
<< std::endl
;
9172 RGWBucketSyncPolicyHandlerRef handler
;
9173 ret
= driver
->get_sync_policy_handler(dpp(), std::nullopt
, bucket
->get_key(), &handler
, null_yield
);
9175 std::cerr
<< "ERROR: failed to get policy handler for bucket ("
9176 << bucket
<< "): r=" << ret
<< ": " << cpp_strerror(-ret
) << std::endl
;
9180 auto timeout_at
= ceph::coarse_mono_clock::now() + opt_timeout_sec
;
9181 ret
= rgw_bucket_sync_checkpoint(dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), *handler
, bucket
->get_info(),
9182 opt_source_zone
, opt_source_bucket
,
9183 opt_retry_delay_ms
, timeout_at
);
9185 ldpp_dout(dpp(), -1) << "bucket sync checkpoint failed: " << cpp_strerror(ret
) << dendl
;
9190 if ((opt_cmd
== OPT::BUCKET_SYNC_DISABLE
) || (opt_cmd
== OPT::BUCKET_SYNC_ENABLE
)) {
9191 if (bucket_name
.empty()) {
9192 cerr
<< "ERROR: bucket not specified" << std::endl
;
9195 if (opt_cmd
== OPT::BUCKET_SYNC_DISABLE
) {
9196 bucket_op
.set_sync_bucket(false);
9198 bucket_op
.set_sync_bucket(true);
9200 bucket_op
.set_tenant(tenant
);
9202 ret
= RGWBucketAdminOp::sync_bucket(driver
, bucket_op
, dpp(), &err_msg
);
9204 cerr
<< err_msg
<< std::endl
;
9209 if (opt_cmd
== OPT::BUCKET_SYNC_INFO
) {
9210 if (bucket_name
.empty()) {
9211 cerr
<< "ERROR: bucket not specified" << std::endl
;
9214 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9218 bucket_sync_info(driver
, bucket
->get_info(), std::cout
);
9221 if (opt_cmd
== OPT::BUCKET_SYNC_STATUS
) {
9222 if (bucket_name
.empty()) {
9223 cerr
<< "ERROR: bucket not specified" << std::endl
;
9226 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9230 bucket_sync_status(driver
, bucket
->get_info(), source_zone
, opt_source_bucket
, std::cout
);
9233 if (opt_cmd
== OPT::BUCKET_SYNC_MARKERS
) {
9234 if (source_zone
.empty()) {
9235 cerr
<< "ERROR: source zone not specified" << std::endl
;
9238 if (bucket_name
.empty()) {
9239 cerr
<< "ERROR: bucket not specified" << std::endl
;
9242 int ret
= init_bucket_for_sync(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9246 auto sync
= RGWBucketPipeSyncStatusManager::construct(
9247 dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), source_zone
,
9248 opt_source_bucket
, bucket
->get_key(), nullptr);
9251 cerr
<< "ERROR: sync.init() returned error=" << sync
.error() << std::endl
;
9252 return -sync
.error();
9255 auto sync_status
= (*sync
)->read_sync_status(dpp());
9257 cerr
<< "ERROR: sync.read_sync_status() returned error="
9258 << sync_status
.error() << std::endl
;
9259 return -sync_status
.error();
9262 encode_json("sync_status", *sync_status
, formatter
.get());
9263 formatter
->flush(cout
);
9266 if (opt_cmd
== OPT::BUCKET_SYNC_RUN
) {
9267 if (source_zone
.empty()) {
9268 cerr
<< "ERROR: source zone not specified" << std::endl
;
9271 if (bucket_name
.empty()) {
9272 cerr
<< "ERROR: bucket not specified" << std::endl
;
9275 int ret
= init_bucket_for_sync(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9279 auto sync
= RGWBucketPipeSyncStatusManager::construct(
9280 dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), source_zone
,
9281 opt_source_bucket
, bucket
->get_key(), extra_info
? &std::cout
: nullptr);
9284 cerr
<< "ERROR: sync.init() returned error=" << sync
.error() << std::endl
;
9285 return -sync
.error();
9288 ret
= (*sync
)->run(dpp());
9290 cerr
<< "ERROR: sync.run() returned ret=" << ret
<< std::endl
;
9295 if (opt_cmd
== OPT::BILOG_LIST
) {
9296 if (bucket_name
.empty()) {
9297 cerr
<< "ERROR: bucket not specified" << std::endl
;
9300 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9302 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
9305 formatter
->open_array_section("entries");
9308 if (max_entries
< 0)
9311 const auto& logs
= bucket
->get_info().layout
.logs
;
9312 auto log_layout
= std::reference_wrapper
{logs
.back()};
9314 auto i
= std::find_if(logs
.begin(), logs
.end(), rgw::matches_gen(*gen
));
9315 if (i
== logs
.end()) {
9316 cerr
<< "ERROR: no log layout with gen=" << *gen
<< std::endl
;
9323 list
<rgw_bi_log_entry
> entries
;
9324 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->bilog_rados
->log_list(dpp(), bucket
->get_info(), log_layout
, shard_id
, marker
, max_entries
- count
, entries
, &truncated
);
9326 cerr
<< "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret
) << std::endl
;
9330 count
+= entries
.size();
9332 for (list
<rgw_bi_log_entry
>::iterator iter
= entries
.begin(); iter
!= entries
.end(); ++iter
) {
9333 rgw_bi_log_entry
& entry
= *iter
;
9334 encode_json("entry", entry
, formatter
.get());
9338 formatter
->flush(cout
);
9339 } while (truncated
&& count
< max_entries
);
9341 formatter
->close_section();
9342 formatter
->flush(cout
);
9345 if (opt_cmd
== OPT::SYNC_ERROR_LIST
) {
9346 if (max_entries
< 0) {
9349 if (!start_date
.empty()) {
9350 std::cerr
<< "start-date not allowed." << std::endl
;
9353 if (!end_date
.empty()) {
9354 std::cerr
<< "end-date not allowed." << std::endl
;
9357 if (!end_marker
.empty()) {
9358 std::cerr
<< "end-marker not allowed." << std::endl
;
9361 if (!start_marker
.empty()) {
9362 if (marker
.empty()) {
9363 marker
= start_marker
;
9365 std::cerr
<< "start-marker and marker not both allowed." << std::endl
;
9376 formatter
->open_array_section("entries");
9378 for (; shard_id
< ERROR_LOGGER_SHARDS
; ++shard_id
) {
9379 formatter
->open_object_section("shard");
9380 encode_json("shard_id", shard_id
, formatter
.get());
9381 formatter
->open_array_section("entries");
9384 string oid
= RGWSyncErrorLogger::get_shard_oid(RGW_SYNC_ERROR_LOG_SHARD_PREFIX
, shard_id
);
9387 list
<cls_log_entry
> entries
;
9388 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->timelog
.list(dpp(), oid
, {}, {}, max_entries
- count
, entries
, marker
, &marker
, &truncated
,
9390 if (ret
== -ENOENT
) {
9394 cerr
<< "ERROR: svc.cls->timelog.list(): " << cpp_strerror(-ret
) << std::endl
;
9398 count
+= entries
.size();
9400 for (auto& cls_entry
: entries
) {
9401 rgw_sync_error_info log_entry
;
9403 auto iter
= cls_entry
.data
.cbegin();
9405 decode(log_entry
, iter
);
9406 } catch (buffer::error
& err
) {
9407 cerr
<< "ERROR: failed to decode log entry" << std::endl
;
9410 formatter
->open_object_section("entry");
9411 encode_json("id", cls_entry
.id
, formatter
.get());
9412 encode_json("section", cls_entry
.section
, formatter
.get());
9413 encode_json("name", cls_entry
.name
, formatter
.get());
9414 encode_json("timestamp", cls_entry
.timestamp
, formatter
.get());
9415 encode_json("info", log_entry
, formatter
.get());
9416 formatter
->close_section();
9417 formatter
->flush(cout
);
9419 } while (truncated
&& count
< max_entries
);
9421 formatter
->close_section();
9422 formatter
->close_section();
9424 if (specified_shard_id
) {
9429 formatter
->close_section();
9430 formatter
->flush(cout
);
9433 if (opt_cmd
== OPT::SYNC_ERROR_TRIM
) {
9434 if (!start_date
.empty()) {
9435 std::cerr
<< "start-date not allowed." << std::endl
;
9438 if (!end_date
.empty()) {
9439 std::cerr
<< "end-date not allowed." << std::endl
;
9442 if (!start_marker
.empty()) {
9443 std::cerr
<< "end-date not allowed." << std::endl
;
9446 if (!end_marker
.empty()) {
9447 std::cerr
<< "end-date not allowed." << std::endl
;
9455 for (; shard_id
< ERROR_LOGGER_SHARDS
; ++shard_id
) {
9456 ret
= trim_sync_error_log(shard_id
, marker
, trim_delay_ms
);
9458 cerr
<< "ERROR: sync error trim: " << cpp_strerror(-ret
) << std::endl
;
9461 if (specified_shard_id
) {
9467 if (opt_cmd
== OPT::SYNC_GROUP_CREATE
||
9468 opt_cmd
== OPT::SYNC_GROUP_MODIFY
) {
9469 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9470 CHECK_TRUE(require_opt(opt_status
), "ERROR: --status is not specified (options: forbidden, allowed, enabled)", EINVAL
);
9472 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9473 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9477 auto& sync_policy
= sync_policy_ctx
.get_policy();
9479 if (opt_cmd
== OPT::SYNC_GROUP_MODIFY
) {
9480 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9481 if (iter
== sync_policy
.groups
.end()) {
9482 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9487 auto& group
= sync_policy
.groups
[*opt_group_id
];
9488 group
.id
= *opt_group_id
;
9491 if (!group
.set_status(*opt_status
)) {
9492 cerr
<< "ERROR: unrecognized status (options: forbidden, allowed, enabled)" << std::endl
;
9497 ret
= sync_policy_ctx
.write_policy();
9502 show_result(sync_policy
, zone_formatter
.get(), cout
);
9505 if (opt_cmd
== OPT::SYNC_GROUP_GET
) {
9506 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9507 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9511 auto& sync_policy
= sync_policy_ctx
.get_policy();
9513 auto& groups
= sync_policy
.groups
;
9515 if (!opt_group_id
) {
9516 show_result(groups
, zone_formatter
.get(), cout
);
9518 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9519 if (iter
== sync_policy
.groups
.end()) {
9520 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9524 show_result(iter
->second
, zone_formatter
.get(), cout
);
9528 if (opt_cmd
== OPT::SYNC_GROUP_REMOVE
) {
9529 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9531 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9532 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9536 auto& sync_policy
= sync_policy_ctx
.get_policy();
9538 sync_policy
.groups
.erase(*opt_group_id
);
9540 ret
= sync_policy_ctx
.write_policy();
9546 Formatter::ObjectSection
os(*zone_formatter
.get(), "result");
9547 encode_json("sync_policy", sync_policy
, zone_formatter
.get());
9550 zone_formatter
->flush(cout
);
9553 if (opt_cmd
== OPT::SYNC_GROUP_FLOW_CREATE
) {
9554 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9555 CHECK_TRUE(require_opt(opt_flow_id
), "ERROR: --flow-id not specified", EINVAL
);
9556 CHECK_TRUE(require_opt(opt_flow_type
),
9557 "ERROR: --flow-type not specified (options: symmetrical, directional)", EINVAL
);
9558 CHECK_TRUE((symmetrical_flow_opt(*opt_flow_type
) ||
9559 directional_flow_opt(*opt_flow_type
)),
9560 "ERROR: --flow-type invalid (options: symmetrical, directional)", EINVAL
);
9562 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9563 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9567 auto& sync_policy
= sync_policy_ctx
.get_policy();
9569 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9570 if (iter
== sync_policy
.groups
.end()) {
9571 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9575 auto& group
= iter
->second
;
9577 if (symmetrical_flow_opt(*opt_flow_type
)) {
9578 CHECK_TRUE(require_non_empty_opt(opt_zone_ids
), "ERROR: --zones not provided for symmetrical flow, or is empty", EINVAL
);
9580 rgw_sync_symmetric_group
*flow_group
;
9582 group
.data_flow
.find_or_create_symmetrical(*opt_flow_id
, &flow_group
);
9584 for (auto& z
: *opt_zone_ids
) {
9585 flow_group
->zones
.insert(z
);
9587 } else { /* directional */
9588 CHECK_TRUE(require_non_empty_opt(opt_source_zone_id
), "ERROR: --source-zone not provided for directional flow rule, or is empty", EINVAL
);
9589 CHECK_TRUE(require_non_empty_opt(opt_dest_zone_id
), "ERROR: --dest-zone not provided for directional flow rule, or is empty", EINVAL
);
9591 rgw_sync_directional_rule
*flow_rule
;
9593 group
.data_flow
.find_or_create_directional(*opt_source_zone_id
, *opt_dest_zone_id
, &flow_rule
);
9596 ret
= sync_policy_ctx
.write_policy();
9601 show_result(sync_policy
, zone_formatter
.get(), cout
);
9604 if (opt_cmd
== OPT::SYNC_GROUP_FLOW_REMOVE
) {
9605 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9606 CHECK_TRUE(require_opt(opt_flow_id
), "ERROR: --flow-id not specified", EINVAL
);
9607 CHECK_TRUE(require_opt(opt_flow_type
),
9608 "ERROR: --flow-type not specified (options: symmetrical, directional)", EINVAL
);
9609 CHECK_TRUE((symmetrical_flow_opt(*opt_flow_type
) ||
9610 directional_flow_opt(*opt_flow_type
)),
9611 "ERROR: --flow-type invalid (options: symmetrical, directional)", EINVAL
);
9613 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9614 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9618 auto& sync_policy
= sync_policy_ctx
.get_policy();
9620 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9621 if (iter
== sync_policy
.groups
.end()) {
9622 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9626 auto& group
= iter
->second
;
9628 if (symmetrical_flow_opt(*opt_flow_type
)) {
9629 group
.data_flow
.remove_symmetrical(*opt_flow_id
, opt_zone_ids
);
9630 } else { /* directional */
9631 CHECK_TRUE(require_non_empty_opt(opt_source_zone_id
), "ERROR: --source-zone not provided for directional flow rule, or is empty", EINVAL
);
9632 CHECK_TRUE(require_non_empty_opt(opt_dest_zone_id
), "ERROR: --dest-zone not provided for directional flow rule, or is empty", EINVAL
);
9634 group
.data_flow
.remove_directional(*opt_source_zone_id
, *opt_dest_zone_id
);
9637 ret
= sync_policy_ctx
.write_policy();
9642 show_result(sync_policy
, zone_formatter
.get(), cout
);
9645 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_CREATE
||
9646 opt_cmd
== OPT::SYNC_GROUP_PIPE_MODIFY
) {
9647 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9648 CHECK_TRUE(require_opt(opt_pipe_id
), "ERROR: --pipe-id not specified", EINVAL
);
9649 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_CREATE
) {
9650 CHECK_TRUE(require_non_empty_opt(opt_source_zone_ids
), "ERROR: --source-zones not provided or is empty; should be list of zones or '*'", EINVAL
);
9651 CHECK_TRUE(require_non_empty_opt(opt_dest_zone_ids
), "ERROR: --dest-zones not provided or is empty; should be list of zones or '*'", EINVAL
);
9654 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9655 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9659 auto& sync_policy
= sync_policy_ctx
.get_policy();
9661 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9662 if (iter
== sync_policy
.groups
.end()) {
9663 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9667 auto& group
= iter
->second
;
9669 rgw_sync_bucket_pipes
*pipe
;
9671 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_CREATE
) {
9672 group
.find_pipe(*opt_pipe_id
, true, &pipe
);
9674 if (!group
.find_pipe(*opt_pipe_id
, false, &pipe
)) {
9675 cerr
<< "ERROR: could not find pipe '" << *opt_pipe_id
<< "'" << std::endl
;
9680 if (opt_source_zone_ids
) {
9681 pipe
->source
.add_zones(*opt_source_zone_ids
);
9683 pipe
->source
.set_bucket(opt_source_tenant
,
9684 opt_source_bucket_name
,
9685 opt_source_bucket_id
);
9686 if (opt_dest_zone_ids
) {
9687 pipe
->dest
.add_zones(*opt_dest_zone_ids
);
9689 pipe
->dest
.set_bucket(opt_dest_tenant
,
9690 opt_dest_bucket_name
,
9691 opt_dest_bucket_id
);
9693 pipe
->params
.source
.filter
.set_prefix(opt_prefix
, !!opt_prefix_rm
);
9694 pipe
->params
.source
.filter
.set_tags(tags_add
, tags_rm
);
9695 if (opt_dest_owner
) {
9696 pipe
->params
.dest
.set_owner(*opt_dest_owner
);
9698 if (opt_storage_class
) {
9699 pipe
->params
.dest
.set_storage_class(*opt_storage_class
);
9702 pipe
->params
.priority
= *opt_priority
;
9705 if (*opt_mode
== "system") {
9706 pipe
->params
.mode
= rgw_sync_pipe_params::MODE_SYSTEM
;
9707 } else if (*opt_mode
== "user") {
9708 pipe
->params
.mode
= rgw_sync_pipe_params::MODE_USER
;
9710 cerr
<< "ERROR: bad mode value: should be one of the following: system, user" << std::endl
;
9715 if (!rgw::sal::User::empty(user
)) {
9716 pipe
->params
.user
= user
->get_id();
9717 } else if (pipe
->params
.user
.empty()) {
9718 auto owner
= sync_policy_ctx
.get_owner();
9720 pipe
->params
.user
= *owner
;
9724 ret
= sync_policy_ctx
.write_policy();
9729 show_result(sync_policy
, zone_formatter
.get(), cout
);
9732 if (opt_cmd
== OPT::SYNC_GROUP_PIPE_REMOVE
) {
9733 CHECK_TRUE(require_opt(opt_group_id
), "ERROR: --group-id not specified", EINVAL
);
9734 CHECK_TRUE(require_opt(opt_pipe_id
), "ERROR: --pipe-id not specified", EINVAL
);
9736 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9737 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9741 auto& sync_policy
= sync_policy_ctx
.get_policy();
9743 auto iter
= sync_policy
.groups
.find(*opt_group_id
);
9744 if (iter
== sync_policy
.groups
.end()) {
9745 cerr
<< "ERROR: could not find group '" << *opt_group_id
<< "'" << std::endl
;
9749 auto& group
= iter
->second
;
9751 rgw_sync_bucket_pipes
*pipe
;
9753 if (!group
.find_pipe(*opt_pipe_id
, false, &pipe
)) {
9754 cerr
<< "ERROR: could not find pipe '" << *opt_pipe_id
<< "'" << std::endl
;
9758 if (opt_source_zone_ids
) {
9759 pipe
->source
.remove_zones(*opt_source_zone_ids
);
9762 pipe
->source
.remove_bucket(opt_source_tenant
,
9763 opt_source_bucket_name
,
9764 opt_source_bucket_id
);
9765 if (opt_dest_zone_ids
) {
9766 pipe
->dest
.remove_zones(*opt_dest_zone_ids
);
9768 pipe
->dest
.remove_bucket(opt_dest_tenant
,
9769 opt_dest_bucket_name
,
9770 opt_dest_bucket_id
);
9772 if (!(opt_source_zone_ids
||
9773 opt_source_tenant
||
9774 opt_source_bucket
||
9775 opt_source_bucket_id
||
9776 opt_dest_zone_ids
||
9779 opt_dest_bucket_id
)) {
9780 group
.remove_pipe(*opt_pipe_id
);
9783 ret
= sync_policy_ctx
.write_policy();
9788 show_result(sync_policy
, zone_formatter
.get(), cout
);
9791 if (opt_cmd
== OPT::SYNC_POLICY_GET
) {
9792 SyncPolicyContext
sync_policy_ctx(cfgstore
.get(), opt_bucket
);
9793 ret
= sync_policy_ctx
.init(zonegroup_id
, zonegroup_name
);
9797 auto& sync_policy
= sync_policy_ctx
.get_policy();
9799 show_result(sync_policy
, zone_formatter
.get(), cout
);
9802 if (opt_cmd
== OPT::BILOG_TRIM
) {
9803 if (bucket_name
.empty()) {
9804 cerr
<< "ERROR: bucket not specified" << std::endl
;
9807 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9809 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
9816 ret
= bilog_trim(dpp(), static_cast<rgw::sal::RadosStore
*>(driver
),
9817 bucket
->get_info(), *gen
,
9818 shard_id
, start_marker
, end_marker
);
9820 cerr
<< "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret
) << std::endl
;
9825 if (opt_cmd
== OPT::BILOG_STATUS
) {
9826 if (bucket_name
.empty()) {
9827 cerr
<< "ERROR: bucket not specified" << std::endl
;
9830 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
9832 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
9835 map
<int, string
> markers
;
9836 const auto& logs
= bucket
->get_info().layout
.logs
;
9837 auto log_layout
= std::reference_wrapper
{logs
.back()};
9839 auto i
= std::find_if(logs
.begin(), logs
.end(), rgw::matches_gen(*gen
));
9840 if (i
== logs
.end()) {
9841 cerr
<< "ERROR: no log layout with gen=" << *gen
<< std::endl
;
9847 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->bilog_rados
->get_log_status(dpp(), bucket
->get_info(), log_layout
, shard_id
,
9848 &markers
, null_yield
);
9850 cerr
<< "ERROR: get_bi_log_status(): " << cpp_strerror(-ret
) << std::endl
;
9853 formatter
->open_object_section("entries");
9854 encode_json("markers", markers
, formatter
.get());
9855 formatter
->dump_string("current_time",
9856 to_iso_8601(ceph::real_clock::now(),
9857 iso_8601_format::YMDhms
));
9858 formatter
->close_section();
9859 formatter
->flush(cout
);
9862 if (opt_cmd
== OPT::BILOG_AUTOTRIM
) {
9863 RGWCoroutinesManager
crs(driver
->ctx(), driver
->get_cr_registry());
9864 RGWHTTPManager
http(driver
->ctx(), crs
.get_completion_mgr());
9865 int ret
= http
.start();
9867 cerr
<< "failed to initialize http client with " << cpp_strerror(ret
) << std::endl
;
9871 rgw::BucketTrimConfig config
;
9872 configure_bucket_trim(driver
->ctx(), config
);
9874 rgw::BucketTrimManager
trim(static_cast<rgw::sal::RadosStore
*>(driver
), config
);
9877 cerr
<< "trim manager init failed with " << cpp_strerror(ret
) << std::endl
;
9880 ret
= crs
.run(dpp(), trim
.create_admin_bucket_trim_cr(&http
));
9882 cerr
<< "automated bilog trim failed with " << cpp_strerror(ret
) << std::endl
;
9887 if (opt_cmd
== OPT::DATALOG_LIST
) {
9888 formatter
->open_array_section("entries");
9891 if (max_entries
< 0)
9893 if (!start_date
.empty()) {
9894 std::cerr
<< "start-date not allowed." << std::endl
;
9897 if (!end_date
.empty()) {
9898 std::cerr
<< "end-date not allowed." << std::endl
;
9901 if (!end_marker
.empty()) {
9902 std::cerr
<< "end-marker not allowed." << std::endl
;
9905 if (!start_marker
.empty()) {
9906 if (marker
.empty()) {
9907 marker
= start_marker
;
9909 std::cerr
<< "start-marker and marker not both allowed." << std::endl
;
9914 auto datalog_svc
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->datalog_rados
;
9915 RGWDataChangesLog::LogMarker log_marker
;
9918 std::vector
<rgw_data_change_log_entry
> entries
;
9919 if (specified_shard_id
) {
9920 ret
= datalog_svc
->list_entries(dpp(), shard_id
, max_entries
- count
,
9922 &marker
, &truncated
,
9925 ret
= datalog_svc
->list_entries(dpp(), max_entries
- count
, entries
,
9926 log_marker
, &truncated
, null_yield
);
9929 cerr
<< "ERROR: datalog_svc->list_entries(): " << cpp_strerror(-ret
) << std::endl
;
9933 count
+= entries
.size();
9935 for (const auto& entry
: entries
) {
9937 encode_json("entry", entry
.entry
, formatter
.get());
9939 encode_json("entry", entry
, formatter
.get());
9942 formatter
.get()->flush(cout
);
9943 } while (truncated
&& count
< max_entries
);
9945 formatter
->close_section();
9946 formatter
->flush(cout
);
9949 if (opt_cmd
== OPT::DATALOG_STATUS
) {
9950 int i
= (specified_shard_id
? shard_id
: 0);
9952 formatter
->open_array_section("entries");
9953 for (; i
< g_ceph_context
->_conf
->rgw_data_log_num_shards
; i
++) {
9954 list
<cls_log_entry
> entries
;
9956 RGWDataChangesLogInfo info
;
9957 static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->
9958 datalog_rados
->get_info(dpp(), i
, &info
, null_yield
);
9960 ::encode_json("info", info
, formatter
.get());
9962 if (specified_shard_id
)
9966 formatter
->close_section();
9967 formatter
->flush(cout
);
9970 if (opt_cmd
== OPT::DATALOG_AUTOTRIM
) {
9971 RGWCoroutinesManager
crs(driver
->ctx(), driver
->get_cr_registry());
9972 RGWHTTPManager
http(driver
->ctx(), crs
.get_completion_mgr());
9973 int ret
= http
.start();
9975 cerr
<< "failed to initialize http client with " << cpp_strerror(ret
) << std::endl
;
9979 auto num_shards
= g_conf()->rgw_data_log_num_shards
;
9980 std::vector
<std::string
> markers(num_shards
);
9981 ret
= crs
.run(dpp(), create_admin_data_log_trim_cr(dpp(), static_cast<rgw::sal::RadosStore
*>(driver
), &http
, num_shards
, markers
));
9983 cerr
<< "automated datalog trim failed with " << cpp_strerror(ret
) << std::endl
;
9988 if (opt_cmd
== OPT::DATALOG_TRIM
) {
9989 if (!start_date
.empty()) {
9990 std::cerr
<< "start-date not allowed." << std::endl
;
9993 if (!end_date
.empty()) {
9994 std::cerr
<< "end-date not allowed." << std::endl
;
9997 if (!start_marker
.empty()) {
9998 std::cerr
<< "start-marker not allowed." << std::endl
;
10001 if (!end_marker
.empty()) {
10002 if (marker
.empty()) {
10003 marker
= end_marker
;
10005 std::cerr
<< "end-marker and marker not both allowed." << std::endl
;
10010 if (!specified_shard_id
) {
10011 cerr
<< "ERROR: requires a --shard-id" << std::endl
;
10015 if (marker
.empty()) {
10016 cerr
<< "ERROR: requires a --marker" << std::endl
;
10020 auto datalog
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->datalog_rados
;
10021 ret
= datalog
->trim_entries(dpp(), shard_id
, marker
, null_yield
);
10023 if (ret
< 0 && ret
!= -ENODATA
) {
10024 cerr
<< "ERROR: trim_entries(): " << cpp_strerror(-ret
) << std::endl
;
10029 if (opt_cmd
== OPT::DATALOG_TYPE
) {
10030 if (!opt_log_type
) {
10031 std::cerr
<< "log-type not specified." << std::endl
;
10034 auto datalog
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->datalog_rados
;
10035 ret
= datalog
->change_format(dpp(), *opt_log_type
, null_yield
);
10037 cerr
<< "ERROR: change_format(): " << cpp_strerror(-ret
) << std::endl
;
10042 if (opt_cmd
== OPT::DATALOG_PRUNE
) {
10043 auto datalog
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->datalog_rados
;
10044 std::optional
<uint64_t> through
;
10045 ret
= datalog
->trim_generations(dpp(), through
, null_yield
);
10048 cerr
<< "ERROR: trim_generations(): " << cpp_strerror(-ret
) << std::endl
;
10053 std::cout
<< "Pruned " << *through
<< " empty generations." << std::endl
;
10055 std::cout
<< "No empty generations." << std::endl
;
10059 bool quota_op
= (opt_cmd
== OPT::QUOTA_SET
|| opt_cmd
== OPT::QUOTA_ENABLE
|| opt_cmd
== OPT::QUOTA_DISABLE
);
10062 if (bucket_name
.empty() && rgw::sal::User::empty(user
)) {
10063 cerr
<< "ERROR: bucket name or uid is required for quota operation" << std::endl
;
10067 if (!bucket_name
.empty()) {
10068 if (!quota_scope
.empty() && quota_scope
!= "bucket") {
10069 cerr
<< "ERROR: invalid quota scope specification." << std::endl
;
10072 set_bucket_quota(driver
, opt_cmd
, tenant
, bucket_name
,
10073 max_size
, max_objects
, have_max_size
, have_max_objects
);
10074 } else if (!rgw::sal::User::empty(user
)) {
10075 if (quota_scope
== "bucket") {
10076 return set_user_bucket_quota(opt_cmd
, ruser
, user_op
, max_size
, max_objects
, have_max_size
, have_max_objects
);
10077 } else if (quota_scope
== "user") {
10078 return set_user_quota(opt_cmd
, ruser
, user_op
, max_size
, max_objects
, have_max_size
, have_max_objects
);
10080 cerr
<< "ERROR: invalid quota scope specification. Please specify either --quota-scope=bucket, or --quota-scope=user" << std::endl
;
10086 bool ratelimit_op_set
= (opt_cmd
== OPT::RATELIMIT_SET
|| opt_cmd
== OPT::RATELIMIT_ENABLE
|| opt_cmd
== OPT::RATELIMIT_DISABLE
);
10087 bool ratelimit_op_get
= opt_cmd
== OPT::RATELIMIT_GET
;
10088 if (ratelimit_op_set
) {
10089 if (bucket_name
.empty() && rgw::sal::User::empty(user
)) {
10090 cerr
<< "ERROR: bucket name or uid is required for ratelimit operation" << std::endl
;
10094 if (!bucket_name
.empty()) {
10095 if (!ratelimit_scope
.empty() && ratelimit_scope
!= "bucket") {
10096 cerr
<< "ERROR: invalid ratelimit scope specification. (bucket scope is not bucket but bucket has been specified)" << std::endl
;
10099 return set_bucket_ratelimit(driver
, opt_cmd
, tenant
, bucket_name
,
10100 max_read_ops
, max_write_ops
,
10101 max_read_bytes
, max_write_bytes
,
10102 have_max_read_ops
, have_max_write_ops
,
10103 have_max_read_bytes
, have_max_write_bytes
);
10104 } else if (!rgw::sal::User::empty(user
)) {
10105 } if (ratelimit_scope
== "user") {
10106 return set_user_ratelimit(opt_cmd
, user
, max_read_ops
, max_write_ops
,
10107 max_read_bytes
, max_write_bytes
,
10108 have_max_read_ops
, have_max_write_ops
,
10109 have_max_read_bytes
, have_max_write_bytes
);
10111 cerr
<< "ERROR: invalid ratelimit scope specification. Please specify either --ratelimit-scope=bucket, or --ratelimit-scope=user" << std::endl
;
10116 if (ratelimit_op_get
) {
10117 if (bucket_name
.empty() && rgw::sal::User::empty(user
)) {
10118 cerr
<< "ERROR: bucket name or uid is required for ratelimit operation" << std::endl
;
10122 if (!bucket_name
.empty()) {
10123 if (!ratelimit_scope
.empty() && ratelimit_scope
!= "bucket") {
10124 cerr
<< "ERROR: invalid ratelimit scope specification. (bucket scope is not bucket but bucket has been specified)" << std::endl
;
10127 return show_bucket_ratelimit(driver
, tenant
, bucket_name
, formatter
.get());
10128 } else if (!rgw::sal::User::empty(user
)) {
10129 } if (ratelimit_scope
== "user") {
10130 return show_user_ratelimit(user
, formatter
.get());
10132 cerr
<< "ERROR: invalid ratelimit scope specification. Please specify either --ratelimit-scope=bucket, or --ratelimit-scope=user" << std::endl
;
10137 if (opt_cmd
== OPT::MFA_CREATE
) {
10138 rados::cls::otp::otp_info_t config
;
10140 if (rgw::sal::User::empty(user
)) {
10141 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10145 if (totp_serial
.empty()) {
10146 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
10150 if (totp_seed
.empty()) {
10151 cerr
<< "ERROR: TOTP device seed was not provided (via --totp-seed)" << std::endl
;
10156 rados::cls::otp::SeedType seed_type
;
10157 if (totp_seed_type
== "hex") {
10158 seed_type
= rados::cls::otp::OTP_SEED_HEX
;
10159 } else if (totp_seed_type
== "base32") {
10160 seed_type
= rados::cls::otp::OTP_SEED_BASE32
;
10162 cerr
<< "ERROR: invalid seed type: " << totp_seed_type
<< std::endl
;
10166 config
.id
= totp_serial
;
10167 config
.seed
= totp_seed
;
10168 config
.seed_type
= seed_type
;
10170 if (totp_seconds
> 0) {
10171 config
.step_size
= totp_seconds
;
10174 if (totp_window
> 0) {
10175 config
.window
= totp_window
;
10178 real_time mtime
= real_clock::now();
10179 string oid
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.get_mfa_oid(user
->get_id());
10181 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->mutate(RGWSI_MetaBackend_OTP::get_meta_key(user
->get_id()),
10182 mtime
, &objv_tracker
,
10184 MDLOG_STATUS_WRITE
,
10186 return static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.create_mfa(dpp(), user
->get_id(), config
, &objv_tracker
, mtime
, null_yield
);
10189 cerr
<< "MFA creation failed, error: " << cpp_strerror(-ret
) << std::endl
;
10193 RGWUserInfo
& user_info
= user_op
.get_user_info();
10194 user_info
.mfa_ids
.insert(totp_serial
);
10195 user_op
.set_mfa_ids(user_info
.mfa_ids
);
10197 ret
= ruser
.modify(dpp(), user_op
, null_yield
, &err
);
10199 cerr
<< "ERROR: failed storing user info, error: " << err
<< std::endl
;
10204 if (opt_cmd
== OPT::MFA_REMOVE
) {
10205 if (rgw::sal::User::empty(user
)) {
10206 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10210 if (totp_serial
.empty()) {
10211 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
10215 real_time mtime
= real_clock::now();
10217 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->mutate(RGWSI_MetaBackend_OTP::get_meta_key(user
->get_id()),
10218 mtime
, &objv_tracker
,
10220 MDLOG_STATUS_WRITE
,
10222 return static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.remove_mfa(dpp(), user
->get_id(), totp_serial
, &objv_tracker
, mtime
, null_yield
);
10225 cerr
<< "MFA removal failed, error: " << cpp_strerror(-ret
) << std::endl
;
10229 RGWUserInfo
& user_info
= user_op
.get_user_info();
10230 user_info
.mfa_ids
.erase(totp_serial
);
10231 user_op
.set_mfa_ids(user_info
.mfa_ids
);
10233 ret
= ruser
.modify(dpp(), user_op
, null_yield
, &err
);
10235 cerr
<< "ERROR: failed storing user info, error: " << err
<< std::endl
;
10240 if (opt_cmd
== OPT::MFA_GET
) {
10241 if (rgw::sal::User::empty(user
)) {
10242 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10246 if (totp_serial
.empty()) {
10247 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
10251 rados::cls::otp::otp_info_t result
;
10252 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.get_mfa(dpp(), user
->get_id(), totp_serial
, &result
, null_yield
);
10254 if (ret
== -ENOENT
|| ret
== -ENODATA
) {
10255 cerr
<< "MFA serial id not found" << std::endl
;
10257 cerr
<< "MFA retrieval failed, error: " << cpp_strerror(-ret
) << std::endl
;
10261 formatter
->open_object_section("result");
10262 encode_json("entry", result
, formatter
.get());
10263 formatter
->close_section();
10264 formatter
->flush(cout
);
10267 if (opt_cmd
== OPT::MFA_LIST
) {
10268 if (rgw::sal::User::empty(user
)) {
10269 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10273 list
<rados::cls::otp::otp_info_t
> result
;
10274 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.list_mfa(dpp(), user
->get_id(), &result
, null_yield
);
10275 if (ret
< 0 && ret
!= -ENOENT
) {
10276 cerr
<< "MFA listing failed, error: " << cpp_strerror(-ret
) << std::endl
;
10279 formatter
->open_object_section("result");
10280 encode_json("entries", result
, formatter
.get());
10281 formatter
->close_section();
10282 formatter
->flush(cout
);
10285 if (opt_cmd
== OPT::MFA_CHECK
) {
10286 if (rgw::sal::User::empty(user
)) {
10287 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10291 if (totp_serial
.empty()) {
10292 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
10296 if (totp_pin
.empty()) {
10297 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-pin)" << std::endl
;
10301 list
<rados::cls::otp::otp_info_t
> result
;
10302 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.check_mfa(dpp(), user
->get_id(), totp_serial
, totp_pin
.front(), null_yield
);
10304 cerr
<< "MFA check failed, error: " << cpp_strerror(-ret
) << std::endl
;
10308 cout
<< "ok" << std::endl
;
10311 if (opt_cmd
== OPT::MFA_RESYNC
) {
10312 if (rgw::sal::User::empty(user
)) {
10313 cerr
<< "ERROR: user id was not provided (via --uid)" << std::endl
;
10317 if (totp_serial
.empty()) {
10318 cerr
<< "ERROR: TOTP device serial number was not provided (via --totp-serial)" << std::endl
;
10322 if (totp_pin
.size() != 2) {
10323 cerr
<< "ERROR: missing two --totp-pin params (--totp-pin=<first> --totp-pin=<second>)" << std::endl
;
10327 rados::cls::otp::otp_info_t config
;
10328 int ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.get_mfa(dpp(), user
->get_id(), totp_serial
, &config
, null_yield
);
10330 if (ret
== -ENOENT
|| ret
== -ENODATA
) {
10331 cerr
<< "MFA serial id not found" << std::endl
;
10333 cerr
<< "MFA retrieval failed, error: " << cpp_strerror(-ret
) << std::endl
;
10338 ceph::real_time now
;
10340 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.otp_get_current_time(dpp(), user
->get_id(), &now
, null_yield
);
10342 cerr
<< "ERROR: failed to fetch current time from osd: " << cpp_strerror(-ret
) << std::endl
;
10347 ret
= scan_totp(driver
->ctx(), now
, config
, totp_pin
, &time_ofs
);
10349 if (ret
== -ENOENT
) {
10350 cerr
<< "failed to resync, TOTP values not found in range" << std::endl
;
10352 cerr
<< "ERROR: failed to scan for TOTP values: " << cpp_strerror(-ret
) << std::endl
;
10357 config
.time_ofs
= time_ofs
;
10359 /* now update the backend */
10360 real_time mtime
= real_clock::now();
10362 ret
= static_cast<rgw::sal::RadosStore
*>(driver
)->ctl()->meta
.mgr
->mutate(RGWSI_MetaBackend_OTP::get_meta_key(user
->get_id()),
10363 mtime
, &objv_tracker
,
10365 MDLOG_STATUS_WRITE
,
10367 return static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->cls
->mfa
.create_mfa(dpp(), user
->get_id(), config
, &objv_tracker
, mtime
, null_yield
);
10370 cerr
<< "MFA update failed, error: " << cpp_strerror(-ret
) << std::endl
;
10376 if (opt_cmd
== OPT::RESHARD_STALE_INSTANCES_LIST
) {
10377 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->can_reshard() && !yes_i_really_mean_it
) {
10378 cerr
<< "Resharding disabled in a multisite env, stale instances unlikely from resharding" << std::endl
;
10379 cerr
<< "These instances may not be safe to delete." << std::endl
;
10380 cerr
<< "Use --yes-i-really-mean-it to force displaying these instances." << std::endl
;
10384 ret
= RGWBucketAdminOp::list_stale_instances(driver
, bucket_op
, stream_flusher
, dpp());
10386 cerr
<< "ERROR: listing stale instances" << cpp_strerror(-ret
) << std::endl
;
10390 if (opt_cmd
== OPT::RESHARD_STALE_INSTANCES_DELETE
) {
10391 if (!static_cast<rgw::sal::RadosStore
*>(driver
)->svc()->zone
->can_reshard()) {
10392 cerr
<< "Resharding disabled in a multisite env. Stale instances are not safe to be deleted." << std::endl
;
10396 ret
= RGWBucketAdminOp::clear_stale_instances(driver
, bucket_op
, stream_flusher
, dpp());
10398 cerr
<< "ERROR: deleting stale instances" << cpp_strerror(-ret
) << std::endl
;
10402 if (opt_cmd
== OPT::PUBSUB_TOPICS_LIST
) {
10404 RGWPubSub
ps(driver
, tenant
);
10406 if (!bucket_name
.empty()) {
10407 rgw_pubsub_bucket_topics result
;
10408 int ret
= init_bucket(user
.get(), tenant
, bucket_name
, bucket_id
, &bucket
);
10410 cerr
<< "ERROR: could not init bucket: " << cpp_strerror(-ret
) << std::endl
;
10414 const RGWPubSub::Bucket
b(ps
, bucket
.get());
10415 ret
= b
.get_topics(dpp(), result
, null_yield
);
10416 if (ret
< 0 && ret
!= -ENOENT
) {
10417 cerr
<< "ERROR: could not get topics: " << cpp_strerror(-ret
) << std::endl
;
10420 encode_json("result", result
, formatter
.get());
10422 rgw_pubsub_topics result
;
10423 int ret
= ps
.get_topics(dpp(), result
, null_yield
);
10424 if (ret
< 0 && ret
!= -ENOENT
) {
10425 cerr
<< "ERROR: could not get topics: " << cpp_strerror(-ret
) << std::endl
;
10428 encode_json("result", result
, formatter
.get());
10430 formatter
->flush(cout
);
10433 if (opt_cmd
== OPT::PUBSUB_TOPIC_GET
) {
10434 if (topic_name
.empty()) {
10435 cerr
<< "ERROR: topic name was not provided (via --topic)" << std::endl
;
10439 RGWPubSub
ps(driver
, tenant
);
10441 rgw_pubsub_topic topic
;
10442 ret
= ps
.get_topic(dpp(), topic_name
, topic
, null_yield
);
10444 cerr
<< "ERROR: could not get topic: " << cpp_strerror(-ret
) << std::endl
;
10447 encode_json("topic", topic
, formatter
.get());
10448 formatter
->flush(cout
);
10451 if (opt_cmd
== OPT::PUBSUB_TOPIC_RM
) {
10452 if (topic_name
.empty()) {
10453 cerr
<< "ERROR: topic name was not provided (via --topic)" << std::endl
;
10457 RGWPubSub
ps(driver
, tenant
);
10459 ret
= ps
.remove_topic(dpp(), topic_name
, null_yield
);
10461 cerr
<< "ERROR: could not remove topic: " << cpp_strerror(-ret
) << std::endl
;
10466 if (opt_cmd
== OPT::SCRIPT_PUT
) {
10467 if (!str_script_ctx
) {
10468 cerr
<< "ERROR: context was not provided (via --context)" << std::endl
;
10471 if (infile
.empty()) {
10472 cerr
<< "ERROR: infile was not provided (via --infile)" << std::endl
;
10476 auto rc
= read_input(infile
, bl
);
10478 cerr
<< "ERROR: failed to read script: '" << infile
<< "'. error: " << rc
<< std::endl
;
10481 const std::string script
= bl
.to_str();
10482 std::string err_msg
;
10483 if (!rgw::lua::verify(script
, err_msg
)) {
10484 cerr
<< "ERROR: script: '" << infile
<< "' has error: " << std::endl
<< err_msg
<< std::endl
;
10487 const rgw::lua::context script_ctx
= rgw::lua::to_context(*str_script_ctx
);
10488 if (script_ctx
== rgw::lua::context::none
) {
10489 cerr
<< "ERROR: invalid script context: " << *str_script_ctx
<< ". must be one of: " << LUA_CONTEXT_LIST
<< std::endl
;
10492 if (script_ctx
== rgw::lua::context::background
&& !tenant
.empty()) {
10493 cerr
<< "ERROR: cannot specify tenant in background context" << std::endl
;
10496 auto lua_manager
= driver
->get_lua_manager();
10497 rc
= rgw::lua::write_script(dpp(), lua_manager
.get(), tenant
, null_yield
, script_ctx
, script
);
10499 cerr
<< "ERROR: failed to put script. error: " << rc
<< std::endl
;
10504 if (opt_cmd
== OPT::SCRIPT_GET
) {
10505 if (!str_script_ctx
) {
10506 cerr
<< "ERROR: context was not provided (via --context)" << std::endl
;
10509 const rgw::lua::context script_ctx
= rgw::lua::to_context(*str_script_ctx
);
10510 if (script_ctx
== rgw::lua::context::none
) {
10511 cerr
<< "ERROR: invalid script context: " << *str_script_ctx
<< ". must be one of: " << LUA_CONTEXT_LIST
<< std::endl
;
10514 auto lua_manager
= driver
->get_lua_manager();
10515 std::string script
;
10516 const auto rc
= rgw::lua::read_script(dpp(), lua_manager
.get(), tenant
, null_yield
, script_ctx
, script
);
10517 if (rc
== -ENOENT
) {
10518 std::cout
<< "no script exists for context: " << *str_script_ctx
<<
10519 (tenant
.empty() ? "" : (" in tenant: " + tenant
)) << std::endl
;
10520 } else if (rc
< 0) {
10521 cerr
<< "ERROR: failed to read script. error: " << rc
<< std::endl
;
10524 std::cout
<< script
<< std::endl
;
10528 if (opt_cmd
== OPT::SCRIPT_RM
) {
10529 if (!str_script_ctx
) {
10530 cerr
<< "ERROR: context was not provided (via --context)" << std::endl
;
10533 const rgw::lua::context script_ctx
= rgw::lua::to_context(*str_script_ctx
);
10534 if (script_ctx
== rgw::lua::context::none
) {
10535 cerr
<< "ERROR: invalid script context: " << *str_script_ctx
<< ". must be one of: " << LUA_CONTEXT_LIST
<< std::endl
;
10538 auto lua_manager
= driver
->get_lua_manager();
10539 const auto rc
= rgw::lua::delete_script(dpp(), lua_manager
.get(), tenant
, null_yield
, script_ctx
);
10541 cerr
<< "ERROR: failed to remove script. error: " << rc
<< std::endl
;
10546 if (opt_cmd
== OPT::SCRIPT_PACKAGE_ADD
) {
10547 #ifdef WITH_RADOSGW_LUA_PACKAGES
10548 if (!script_package
) {
10549 cerr
<< "ERROR: lua package name was not provided (via --package)" << std::endl
;
10552 const auto rc
= rgw::lua::add_package(dpp(), driver
, null_yield
, *script_package
, bool(allow_compilation
));
10554 cerr
<< "ERROR: failed to add lua package: " << script_package
<< " .error: " << rc
<< std::endl
;
10558 cerr
<< "ERROR: adding lua packages is not permitted" << std::endl
;
10563 if (opt_cmd
== OPT::SCRIPT_PACKAGE_RM
) {
10564 #ifdef WITH_RADOSGW_LUA_PACKAGES
10565 if (!script_package
) {
10566 cerr
<< "ERROR: lua package name was not provided (via --package)" << std::endl
;
10569 const auto rc
= rgw::lua::remove_package(dpp(), driver
, null_yield
, *script_package
);
10570 if (rc
== -ENOENT
) {
10571 cerr
<< "WARNING: package " << script_package
<< " did not exists or already removed" << std::endl
;
10575 cerr
<< "ERROR: failed to remove lua package: " << script_package
<< " .error: " << rc
<< std::endl
;
10579 cerr
<< "ERROR: removing lua packages in not permitted" << std::endl
;
10584 if (opt_cmd
== OPT::SCRIPT_PACKAGE_LIST
) {
10585 #ifdef WITH_RADOSGW_LUA_PACKAGES
10586 rgw::lua::packages_t packages
;
10587 const auto rc
= rgw::lua::list_packages(dpp(), driver
, null_yield
, packages
);
10588 if (rc
== -ENOENT
) {
10589 std::cout
<< "no lua packages in allowlist" << std::endl
;
10590 } else if (rc
< 0) {
10591 cerr
<< "ERROR: failed to read lua packages allowlist. error: " << rc
<< std::endl
;
10594 for (const auto& package
: packages
) {
10595 std::cout
<< package
<< std::endl
;
10599 cerr
<< "ERROR: listing lua packages in not permitted" << std::endl
;