ceph/mirroring/mirror-ceph.sh

   1 #!/usr/bin/env bash
   2 set -e
   3 #
   4 # Script to mirror Ceph locally
   5 #
   6 # Please, choose a local source and do not sync in a shorter interval than
   7 # 3 hours.
   8 #
   9 SILENT=0
  10
  11 # All available source mirrors
  12 declare -A SOURCES
  13 SOURCES[eu]="eu.ceph.com"
  14 SOURCES[de]="de.ceph.com"
  15 SOURCES[se]="se.ceph.com"
  16 SOURCES[au]="au.ceph.com"
  17 SOURCES[us]="download.ceph.com"
  18 SOURCES[hk]="hk.ceph.com"
  19 SOURCES[fr]="fr.ceph.com"
  20 SOURCES[ca]="ca.ceph.com"
  21 SOURCES[us-east]="us-east.ceph.com"
  22 SOURCES[us-west]="us-west.ceph.com"
  23 SOURCES[global]="download.ceph.com"
  24
  25 function print_usage() {
  26     echo "$0 [-q ] -s <source mirror> -t <target directory>"
  27 }
  28
  29 while getopts ":qhs:t:" opt; do
  30     case $opt in
  31         q)
  32             SILENT=1
  33             ;;
  34         s)
  35             SOURCE=$OPTARG
  36             ;;
  37         t)
  38             TARGET=$OPTARG
  39             ;;
  40         h)
  41             HELP=1
  42             ;;
  43         \?)
  44             print_usage
  45             exit 1
  46             ;;
  47     esac
  48 done
  49
  50 if [ ! -z "$HELP" ] || [ -z "$TARGET" ] || [ -z "$SOURCE" ]; then
  51     print_usage
  52     exit 1
  53 fi
  54
  55 if [ ! -d "$TARGET" ]; then
  56     echo "$TARGET is not a valid target directory"
  57     exit 1
  58 fi
  59
  60 for i in "${!SOURCES[@]}"; do
  61     if [ "$i" == "$SOURCE" ]; then
  62         SOURCE_HOST=${SOURCES[$i]}
  63     fi
  64 done
  65
  66 if [ -z "$SOURCE_HOST" ]; then
  67     echo -n "Please select one of the following sources:"
  68     for i in "${!SOURCES[@]}"; do
  69         echo -n " $i"
  70     done
  71     echo ""
  72     exit 1
  73 fi
  74
  75 RSYNC_OPTS="--stats --progress"
  76 if [ $SILENT -eq 1 ]; then
  77     RSYNC_OPTS="--quiet"
  78 fi
  79
  80 # We start a two-stage sync here for DEB and RPM
  81 # Based on: https://www.debian.org/mirror/ftpmirror
  82 #
  83 # The idea is to prevent temporary situations where metadata points to files
  84 # which do not exist
  85 #
  86
  87 # Exclude all metadata files
  88 rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
  89                                          --hard-links \
  90                                          --exclude Packages* \
  91                                          --exclude Sources* \
  92                                          --exclude Release* \
  93                                          --exclude InRelease \
  94                                          --exclude i18n/* \
  95                                          --exclude ls-lR* \
  96                                          --exclude repodata/* \
  97                                          ${TARGET}
  98
  99 # Now also transfer the metadata and delete afterwards
 100 rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
 101                                          --hard-links --delete-after \
 102                                          ${TARGET}