ceph/mirroring/mirror-ceph.sh

   1 #!/usr/bin/env bash
   2 set -e
   3 #
   4 # Script to mirror Ceph locally
   5 #
   6 # Please, choose a local source and do not sync in a shorter interval than
   7 # 3 hours.
   8 #
   9 SILENT=0
  10
  11 # All available source mirrors
  12 declare -A SOURCES
  13 SOURCES[eu]="eu.ceph.com"
  14 SOURCES[de]="de.ceph.com"
  15 SOURCES[se]="se.ceph.com"
  16 SOURCES[au]="au.ceph.com"
  17 SOURCES[us]="download.ceph.com"
  18 SOURCES[hk]="hk.ceph.com"
  19 SOURCES[fr]="fr.ceph.com"
  20 SOURCES[us-east]="us-east.ceph.com"
  21 SOURCES[us-west]="us-west.ceph.com"
  22 SOURCES[global]="download.ceph.com"
  23
  24 function print_usage() {
  25     echo "$0 [-q ] -s <source mirror> -t <target directory>"
  26 }
  27
  28 while getopts ":qhs:t:" opt; do
  29     case $opt in
  30         q)
  31             SILENT=1
  32             ;;
  33         s)
  34             SOURCE=$OPTARG
  35             ;;
  36         t)
  37             TARGET=$OPTARG
  38             ;;
  39         h)
  40             HELP=1
  41             ;;
  42         \?)
  43             print_usage
  44             exit 1
  45             ;;
  46     esac
  47 done
  48
  49 if [ ! -z "$HELP" ] || [ -z "$TARGET" ] || [ -z "$SOURCE" ]; then
  50     print_usage
  51     exit 1
  52 fi
  53
  54 if [ ! -d "$TARGET" ]; then
  55     echo "$TARGET is not a valid target directory"
  56     exit 1
  57 fi
  58
  59 for i in "${!SOURCES[@]}"; do
  60     if [ "$i" == "$SOURCE" ]; then
  61         SOURCE_HOST=${SOURCES[$i]}
  62     fi
  63 done
  64
  65 if [ -z "$SOURCE_HOST" ]; then
  66     echo -n "Please select one of the following sources:"
  67     for i in "${!SOURCES[@]}"; do
  68         echo -n " $i"
  69     done
  70     echo ""
  71     exit 1
  72 fi
  73
  74 RSYNC_OPTS="--stats --progress"
  75 if [ $SILENT -eq 1 ]; then
  76     RSYNC_OPTS="--quiet"
  77 fi
  78
  79 # We start a two-stage sync here for DEB and RPM
  80 # Based on: https://www.debian.org/mirror/ftpmirror
  81 #
  82 # The idea is to prevent temporary situations where metadata points to files
  83 # which do not exist
  84 #
  85
  86 # Exclude all metadata files
  87 rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
  88                                          --hard-links \
  89                                          --exclude Packages* \
  90                                          --exclude Sources* \
  91                                          --exclude Release* \
  92                                          --exclude InRelease \
  93                                          --exclude i18n/* \
  94                                          --exclude ls-lR* \
  95                                          --exclude repodata/* \
  96                                          ${TARGET}
  97
  98 # Now also transfer the metadata and delete afterwards
  99 rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
 100                                          --hard-links --delete-after \
 101                                          ${TARGET}