ceph/src/tools/rbd_recover_tool/rbd-recover-tool

   1 #!/bin/bash
   2 # file: rbd-recover-tool
   3 #
   4 # Copyright (C) 2015 Ubuntu Kylin
   5 #
   6 # Author: Min Chen <minchen@ubuntukylin.com>
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU Library Public License as published by
  10 # the Free Software Foundation; either version 2, or (at your option)
  11 # any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU Library Public License for more details.
  17 #
  18
  19 # rbd-recover-tool is an offline recover tool for rbd image in replicated pool
  20 # when ceph cluster is stopped.
  21 # it is a simple disater recovery policy, just for urgent condition
  22
  23 my_dir=$(dirname "$0")
  24
  25 . $my_dir/common_h
  26 . $my_dir/metadata_h
  27 . $my_dir/epoch_h
  28 . $my_dir/database_h
  29
  30 #scp files from admin node to osd node
  31 file1=common_h
  32 file2=metadata_h
  33 file3=epoch_h
  34 file4=osd_job
  35
  36 #------------ admin node's action -------------
  37
  38 function scp_file()
  39 {
  40   local func="scp_file"
  41   file=$1
  42   if [ "$1"x = ""x ];then
  43     echo "$func: not file input"
  44     exit
  45   fi
  46   for host in `cat $osd_host`
  47   do
  48   {
  49     echo "$func: $host"
  50     scp $ssh_option $file $host:$job_path  1>/dev/null
  51   } &
  52   done
  53 }
  54
  55 function scp_files()
  56 {
  57   local func="scp_files"
  58   for host in `cat $osd_host`
  59   do
  60   {
  61     echo "$func: $host"
  62     scp $ssh_option $file1 $host:$job_path
  63     scp $ssh_option $file2 $host:$job_path
  64     scp $ssh_option $file3 $host:$job_path
  65     scp $ssh_option $file4 $host:$job_path
  66   } &
  67   done
  68   wait
  69   echo "$func: finish"
  70 }
  71
  72 function scatter_node_jobs()
  73 {
  74   local func="scatter_node_jobs"
  75   local host=
  76   local data_path=
  77   echo "$func: flush osd journal & generate infos: omap, pg, image metadata ..."
  78
  79   trap 'echo $func failed; exit' INT HUP
  80   while read line
  81   do
  82   {
  83     host=`echo $line|awk '{print $1}'`
  84     data_path=`echo $line|awk '{print $2}'`
  85     check_osd_process $host
  86
  87     cmd="mkdir -p $job_path"
  88     ssh $ssh_option $host $cmd
  89     scp $ssh_option $file1 $host:$job_path  >/dev/null
  90     scp $ssh_option $file2 $host:$job_path  >/dev/null
  91     scp $ssh_option $file3 $host:$job_path  >/dev/null
  92     scp $ssh_option $file4 $host:$job_path  >/dev/null
  93
  94     cmd="bash $job_path/osd_job flush_osd_journal $data_path;"
  95     cmd="$cmd $job_path/osd_job do_omap_list $data_path;"
  96     cmd="$cmd bash $job_path/osd_job do_pg_epoch $data_path;"
  97     cmd="$cmd bash $job_path/osd_job do_image_list $data_path;"
  98
  99     ssh $ssh_option $host $cmd </dev/null
 100   } &
 101   done < $osd_host_path
 102   wait
 103   echo "$func: finish"
 104 }
 105
 106 function gather_node_infos()
 107 {
 108   local func="gather_node_infos"
 109   echo "$func ..."
 110   >$pg_coll
 111   >$image_coll_v1
 112   >$image_coll_v2
 113   trap 'echo $func failed; exit' INT HUP
 114   while read line
 115   do
 116   {
 117     host=`echo $line|awk '{print $1}'`
 118     data_path=`echo $line|awk '{print $2}'`
 119     echo "$func: $host"
 120     check_osd_process $host
 121
 122     #pg epoch
 123     cmd1="bash $job_path/osd_job cat_pg_epoch $data_path"
 124     ssh $ssh_option $host $cmd1 >> $pg_coll
 125     #image v1
 126     cmd2="bash $job_path/osd_job cat_image_v1 $data_path"
 127     ssh $ssh_option $host $cmd2 >> $image_coll_v1
 128     #image v2
 129     cmd3="bash $job_path/osd_job cat_image_v2 $data_path"
 130     ssh $ssh_option $host $cmd3 >> $image_coll_v2
 131   } &
 132   done < $osd_host_path
 133   wait
 134   echo "$func: finish"
 135 }
 136
 137 function scatter_gather()
 138 {
 139   local func="scatter_gather"
 140   if [ ! -s $osd_host ];then
 141     echo "$func: no osd_host input"
 142     exit
 143   fi
 144   if [ ! -s $mon_host ];then
 145     echo "$func: no mon_host input"
 146     exit
 147   fi
 148   scatter_node_jobs
 149   gather_node_infos
 150 }
 151
 152
 153 #------------- operations --------------
 154
 155 function database()
 156 {
 157   scatter_gather
 158   gen_database
 159 }
 160
 161 function list()
 162 {
 163   list_images
 164 }
 165
 166 function lookup()
 167 {
 168   lookup_image $1 $2 $3
 169 }
 170
 171 function recover()
 172 {
 173   recover_image $1 $2 $3 $4
 174 }
 175
 176 #------------- helper -------------
 177
 178 function usage()
 179 {
 180   local cmd_name="rbd-recover-tool"
 181   echo
 182   echo "$cmd_name is used to recover rbd image of replicated pool,
 183         when all ceph services are stopped"
 184   echo "Usage:"
 185   echo "$cmd_name database
 186                         gather pg info, object info, image metadata,
 187                         and epoch info from all osd nodes,
 188                         this will cosume a long time, just be patient,
 189                         especially when scale up to 1000+ osds"
 190   echo "$cmd_name list
 191                         list all rbd images of all replicated pools,
 192                         before to lookup & recover"
 193   echo "$cmd_name lookup  <pool_id>/<image_name>[@[<snap_name>]]
 194                         show image metadata: image format, rbd id, size, order, snapseq
 195                         In addtion, for image with snapshots,
 196                         this will list all snapshot infomations"
 197   echo "$cmd_name recover <pool_id>/<image_name>[@[<snap_name>]] [</path/to/store/image>]
 198                         all snapshots share one image head, to economize disk space
 199                         so there is only one snapshot at any time,
 200                         image is saved at </path/to/store/image>/pool_<pool_id>/image_name/image_name
 201                         cat <path/to/store/image>/pool_<pool_id>/image_name/@CURRENT,
 202                         will show snapid
 203                         recover to raw image/nosnap/head: <image_name>
 204                         rollback to image head:           <image_name>@
 205                         rollback to image snap:           <image_name>@<snap_name>
 206                         recover steps:
 207                         1. recover image nosnap (only one time)
 208                         2. rollback to image snap"
 209 }
 210
 211 function get_path()
 212 {
 213   local func="get_path"
 214   if [ $# -lt 1 ];then
 215     return
 216   fi
 217   if [[ $1 =~ // ]];then
 218     return # "/path//to" is invalid
 219   fi
 220   local parent=`dirname $1`
 221   local name=`basename $1`
 222   if [ "$parent"x = "/"x ];then
 223     echo "$parent$name"
 224   else
 225     echo -n "$parent/$name"
 226   fi
 227 }
 228
 229 function admin_cmd()
 230 {
 231   local func="admin_cmd"
 232   if [ $# -lt 1 ];then
 233     usage
 234     exit
 235   fi
 236   if [ "$1"x = "-h"x ] || [ "$1"x = "--help"x ];then
 237     usage
 238     exit
 239   fi
 240
 241   if [ "$1"x = "database"x ];then
 242     if [ $# -gt 1 ];then
 243       usage
 244       exit
 245     fi
 246     # remove osd_host to refresh osd_host and osd_host_mapping
 247     rm -f $osd_host
 248     init_env_admin
 249     database
 250   elif [ "$1"x = "list"x ];then
 251     if [ $# -gt 1 ];then
 252       usage
 253       exit
 254     fi
 255     init_env_admin
 256     list
 257   elif [ "$1"x = "lookup"x ];then
 258     if [ $# -gt 2 ];then
 259       usage
 260       exit
 261     fi
 262     local pool_id=-1
 263     local image_name=
 264     local snap_name=
 265     if [[ $2 =~  ^([^@/]+)/([^@/]+)$ ]];then
 266       pool_id="${BASH_REMATCH[1]}"
 267       image_name="${BASH_REMATCH[2]}"
 268     elif [[ $2 =~  ^([^@/]+)/([^@/]+)@([^@/]*)$ ]];then
 269       pool_id="${BASH_REMATCH[1]}"
 270       image_name="${BASH_REMATCH[2]}"
 271       snap_name="${BASH_REMATCH[3]}"
 272     else
 273       echo "format: $2 is invalid, use <pool_id>/<image_name>[@[<snap_name>]]"
 274       exit
 275     fi
 276     init_env_admin
 277     lookup $pool_id $image_name $snap_name
 278   elif [ "$1"x = "recover"x ];then
 279     if [ $# -lt 2 ] || [ $# -gt 3 ];then
 280       usage
 281       exit
 282     fi
 283     local pool_id=-1
 284     local image_name=
 285     local snap_name=@
 286     local image_dir=
 287     if [[ $2 =~  ^([^@/]+)/([^@/]+)$ ]];then
 288       pool_id="${BASH_REMATCH[1]}"
 289       image_name="${BASH_REMATCH[2]}"
 290     elif [[ $2 =~  ^([^@/]+)/([^@/]+)@([^@/]*)$ ]];then
 291       pool_id="${BASH_REMATCH[1]}"
 292       image_name="${BASH_REMATCH[2]}"
 293       snap_name="${BASH_REMATCH[3]}"
 294       if [ "$snap_name"x = ""x ];then
 295         snap_name=@@
 296       fi
 297     else
 298       echo "format: $2 is invalid, use <pool_id>/<image_name>[@[<snap_name>]]"
 299       exit
 300     fi
 301     if [ $# = 3 ];then
 302       image_dir=`get_path $3`
 303       if [ "image_dir"x = ""x ];then
 304         echo "$3 invalid"
 305         exit
 306       fi
 307     fi
 308     init_env_admin
 309     recover $pool_id $image_name $snap_name $image_dir
 310   elif [ "$1"x = "scp_files"x ];then
 311     if [ $# -gt 1 ];then
 312       exit
 313     fi
 314     admin_parse_osd
 315     scp_files
 316   elif [ "$1"x = "scp_file"x ];then
 317     if [ $# -gt 2 ];then
 318       exit
 319     fi
 320     admin_parse_osd
 321     scp_file $2
 322   else
 323     echo "$func: $1: command not found"
 324   fi
 325 }
 326
 327 admin_cmd $*