ceph/src/rgw/rgw-orphan-list

   1 #!/usr/bin/env bash
   2
   3 # version 2020-10-20
   4
   5 # IMPORTANT: affects order produced by 'sort' and 'ceph-diff-sorted'
   6 # relies on this ordering
   7 export LANG=C
   8
   9 out_dir="."
  10 temp_file=/tmp/temp.$$
  11 timestamp=$(date -u +%Y%m%d%H%M%S)
  12 lspools_err="${out_dir}/lspools-${timestamp}.error"
  13 rados_out="${out_dir}/rados-${timestamp}.intermediate"
  14 rados_odd="${out_dir}/rados-${timestamp}.issues"
  15 rados_err="${out_dir}/rados-${timestamp}.error"
  16 rgwadmin_out="${out_dir}/radosgw-admin-${timestamp}.intermediate"
  17 rgwadmin_err="${out_dir}/radosgw-admin-${timestamp}.error"
  18 delta_out="${out_dir}/orphan-list-${timestamp}.out"
  19
  20 error_out() {
  21     echo "An error was encountered while running '$1'. Aborting."
  22     if [ $# -gt 2 ] ;then
  23         echo "Error: $3"
  24     fi
  25     if [ $# -gt 1 ] ;then
  26         echo "Review file '$2' for details."
  27     fi
  28     echo "***"
  29     echo "*** WARNING: The results are incomplete. Do not use! ***"
  30     echo "***"
  31     exit 1
  32 }
  33
  34 prompt_pool() {
  35     # note: all prompts go to stderr so stdout contains just the result
  36     >&2 echo "Available pools:"
  37     rados lspools >"$temp_file" 2>"$lspools_err"
  38     if [ "$?" -ne 0 ] ;then
  39         error_out "rados lspools" "$lspools_err"
  40     fi
  41     >&2 sed 's/^/    /' "$temp_file" # list pools and indent
  42     >&2 printf "Which pool do you want to search for orphans? "
  43     local mypool
  44     read mypool
  45     echo $mypool
  46 }
  47
  48 if [ $# -eq 0 ] ;then
  49     pool="$(prompt_pool)"
  50 elif [ $# -eq 1 ] ;then
  51     pool="$1"
  52 else
  53     error_out "Usage: $0 [pool]"
  54 fi
  55
  56 echo "Pool is \"$pool\"."
  57
  58 echo "Note: output files produced will be tagged with the current timestamp -- ${timestamp}."
  59
  60 echo "running 'rados ls' at $(date)"
  61 # since --format is not specified, plain should be used
  62 rados ls --pool="$pool" --all >"$rados_out" 2>"$rados_err"
  63 if [ "$?" -ne 0 ] ;then
  64     error_out "rados ls" "$rados_err"
  65 fi
  66
  67 # NOTE: Each entry (line of output) of `rados ls --all` should be in
  68 # one of four formats depending on whether or not an entry has a
  69 # namespace and/or locator:
  70 #
  71 #   <TAB>oid
  72 #   <TAB>oid<TAB>locator
  73 #   namespace<TAB>oid
  74 #   namespace<TAB>oid<TAB>locator
  75 #
  76 # Any occurrences of the 2nd, 3rd, or 4th (i.e., existence of
  77 # namespace and/or locator) should cause the create of the "odd" file
  78 # and an explanation in the output, and those entries will not be
  79 # retained, and therefore they will not be called out as orphans. They
  80 # will need special handling by the end-user as we do not expect
  81 # namespaces or locators.
  82
  83 # check for namespaces -- any line that does not begin with a tab
  84 # indicates a namespace; add those to "odd" file and set flag; note:
  85 # this also picks up entries with namespace and locator
  86 grep $'^[^\t]' "$rados_out" >"$rados_odd"
  87 if [ "${PIPESTATUS[0]}" -eq 0 ] ;then
  88     namespace_found=1
  89 fi
  90
  91 # check for locators (w/o namespace); we idenitfy them by skipping
  92 # past the empty namespace (i.e., one TAB), skipping past the oid,
  93 # then looking for a TAB; note we use egrep to get the '+' character
  94 # and the $ in front of the ' allows the \t to be interpreted as a TAB
  95 egrep $'^\t[[:graph:]]+\t' "$rados_out" >>"$rados_odd"
  96 if [ "${PIPESTATUS[0]}" -eq 0 ] ;then
  97     locator_found=1
  98 fi
  99
 100 # extract the entries that are just oids (i.e., no namespace or
 101 # locator) for further processing; only look at lines that begin with
 102 # a TAB and do not contain a second TAB, and then grab everything
 103 # after the initial TAB
 104 grep $'^\t' "$rados_out" | grep -v $'^\t.*\t' | sed -E 's/^\t//' >"$temp_file"
 105 mv -f "$temp_file" "$rados_out"
 106
 107 sort -u "$rados_out" >"$temp_file"
 108 mv -f "$temp_file" "$rados_out"
 109
 110 echo "running 'radosgw-admin bucket radoslist' at $(date)"
 111 radosgw-admin bucket radoslist >"$rgwadmin_out" 2>"$rgwadmin_err"
 112 if [ "$?" -ne 0 ] ;then
 113     error_out "radosgw-admin radoslist" "$rgwadmin_err"
 114 fi
 115 sort -u "$rgwadmin_out" >"$temp_file"
 116 mv -f "$temp_file" "$rgwadmin_out"
 117
 118 echo "computing delta at $(date)"
 119 ceph-diff-sorted "$rados_out" "$rgwadmin_out" | grep "^<" | sed 's/^< *//' >"$delta_out"
 120 # use PIPESTATUS to get at exit status of first process in above pipe;
 121 # 0 means same, 1 means different, >1 means error
 122 if [ "${PIPESTATUS[0]}" -gt 1 ] ;then
 123     error_out "ceph-diff-sorted"
 124 fi
 125
 126 found=$(wc -l < "$delta_out")
 127 possible=$(wc -l < "$rados_out")
 128 percentage=0
 129 if [ $possible -ne 0 ] ;then
 130     percentage=$(expr 100 \* $found / $possible)
 131 fi
 132
 133 echo "$found potential orphans found out of a possible $possible (${percentage}%)."
 134 echo "The results can be found in '${delta_out}'."
 135 echo "    Intermediate files are '${rados_out}' and '${rgwadmin_out}'."
 136 if [ -n "$namespace_found" -o -n "$locator_found" ] ;then
 137     echo "    Note: 'rados ls' found entries that might be in a namespace or might"
 138     echo "          have a locator; see '${rados_odd}' for those entries."
 139 fi
 140 echo "***"
 141 echo "*** WARNING: This is EXPERIMENTAL code and the results should be used"
 142 echo "***          only with CAUTION!"
 143 echo "***"
 144 echo "Done at $(date)."