ceph/make-dist

   1 #!/bin/bash -e
   2
   3 SCRIPTNAME="$(basename "${0}")"
   4 BASEDIR="$(readlink -f "$(dirname "${0}")")"
   5
   6 if [ ! -d .git ]; then
   7     echo "$SCRIPTNAME: Full path to the script: $BASEDIR/$SCRIPTNAME"
   8     echo "$SCRIPTNAME: No .git present. Run this from the base dir of the git checkout."
   9     exit 1
  10 fi
  11
  12 # Running the script from a directory containing a colon anywhere in the path
  13 # will expose us to the dreaded "[BUG] npm run [command] failed if the directory
  14 # path contains colon" bug https://github.com/npm/cli/issues/633
  15 # (see https://tracker.ceph.com/issues/39556 for details)
  16 if [[ "$BASEDIR" == *:* ]] ; then
  17     echo "$SCRIPTNAME: Full path to the script: $BASEDIR/$SCRIPTNAME"
  18     echo "$SCRIPTNAME: The path to the script contains a colon. Their presence has been known to break the script."
  19     exit 1
  20 fi
  21
  22 version=$1
  23 [ -z "$version" ] && version=$(git describe --long --match 'v*' | sed 's/^v//')
  24 if expr index $version '-' > /dev/null; then
  25     rpm_version=$(echo $version | cut -d - -f 1-1)
  26     rpm_release=$(echo $version | cut -d - -f 2- | sed 's/-/./')
  27 else
  28     rpm_version=$version
  29     rpm_release=0
  30 fi
  31
  32 outfile="ceph-$version"
  33 echo "version $version"
  34
  35 # update submodules
  36 echo "updating submodules..."
  37 force=$(if git submodule usage 2>&1 | grep --quiet 'update.*--force'; then echo --force ; fi)
  38 if ! git submodule sync || ! git submodule update $force --init --recursive; then
  39     echo "Error: could not initialize submodule projects"
  40     echo "  Network connectivity might be required."
  41     exit 1
  42 fi
  43
  44 download_from() {
  45     fname=$1
  46     shift
  47     sha256=$1
  48     shift
  49     set +e
  50     while true; do
  51         url_base=$1
  52         shift
  53         if [ -z $url_base ]; then
  54             echo "Error: failed to download $name."
  55             exit
  56         fi
  57         url=$url_base/$fname
  58         wget -c --no-verbose -O $fname $url
  59         if [ $? != 0 -o ! -e $fname ]; then
  60             echo "Download of $url failed"
  61         elif [ $(sha256sum $fname | awk '{print $1}') != $sha256 ]; then
  62             echo "Error: failed to download $name: SHA256 mismatch."
  63         else
  64             break
  65         fi
  66     done
  67     set -e
  68 }
  69
  70 download_boost() {
  71     boost_version=$1
  72     shift
  73     boost_sha256=$1
  74     shift
  75     boost_version_underscore=$(echo $boost_version | sed 's/\./_/g')
  76     boost_fname=boost_${boost_version_underscore}.tar.bz2
  77     download_from $boost_fname $boost_sha256 $*
  78     tar xjf $boost_fname -C src \
  79         --exclude="$boost_version_underscore/libs/*/doc" \
  80         --exclude="$boost_version_underscore/libs/*/example" \
  81         --exclude="$boost_version_underscore/libs/*/examples" \
  82         --exclude="$boost_version_underscore/libs/*/meta" \
  83         --exclude="$boost_version_underscore/libs/*/test" \
  84         --exclude="$boost_version_underscore/tools/boostbook" \
  85         --exclude="$boost_version_underscore/tools/quickbook" \
  86         --exclude="$boost_version_underscore/tools/auto_index" \
  87         --exclude='doc' --exclude='more' --exclude='status'
  88     mv src/boost_${boost_version_underscore} src/boost
  89     tar cf ${outfile}.boost.tar ${outfile}/src/boost
  90     rm -rf src/boost
  91 }
  92
  93 download_liburing() {
  94     liburing_version=$1
  95     shift
  96     liburing_sha256=$1
  97     shift
  98     liburing_fname=liburing-${liburing_version}.tar.gz
  99     download_from $liburing_fname $liburing_sha256 $*
 100     tar xzf $liburing_fname -C src  \
 101         --exclude=debian \
 102         --exclude=examples \
 103         --exclude=man \
 104         --exclude=test
 105     # normalize the names, liburing-0.7 if downloaded from git.kernel.dk,
 106     # liburing-liburing-0.7 from github.com
 107     mv src/liburing-* src/liburing
 108     tar cf ${outfile}.liburing.tar ${outfile}/src/liburing
 109     rm -rf src/liburing
 110 }
 111
 112 download_pmdk() {
 113     pmdk_version=$1
 114     shift
 115     pmdk_sha256=$1
 116     shift
 117     pmdk_fname=pmdk-${pmdk_version}.tar.gz
 118     download_from $pmdk_fname $pmdk_sha256 $*
 119     tar xzf $pmdk_fname -C src \
 120         --exclude="pmdk-${pmdk_version}/doc" \
 121         --exclude="pmdk-${pmdk_version}/src/test" \
 122         --exclude="pmdk-${pmdk_version}/src/examples" \
 123         --exclude="pmdk-${pmdk_version}/src/benchmarks"
 124     mv src/pmdk-${pmdk_version} src/pmdk
 125     tar cf ${outfile}.pmdk.tar ${outfile}/src/pmdk
 126     rm -rf src/pmdk
 127 }
 128
 129 build_dashboard_frontend() {
 130   CURR_DIR=`pwd`
 131   TEMP_DIR=`mktemp -d`
 132
 133   $CURR_DIR/src/tools/setup-virtualenv.sh $TEMP_DIR
 134   $TEMP_DIR/bin/pip install nodeenv
 135   $TEMP_DIR/bin/nodeenv --verbose -p --node=12.18.2
 136   cd src/pybind/mgr/dashboard/frontend
 137
 138   . $TEMP_DIR/bin/activate
 139   NG_CLI_ANALYTICS=false timeout 1h npm ci
 140   echo "Building ceph-dashboard frontend with build:localize script";
 141   # we need to use "--" because so that "--prod" survives accross all
 142   # scripts redirections inside package.json
 143   npm run build:localize -- --prod
 144   deactivate
 145   cd $CURR_DIR
 146   rm -rf $TEMP_DIR
 147   tar cf dashboard_frontend.tar $outfile/src/pybind/mgr/dashboard/frontend/dist
 148 }
 149
 150 generate_rook_ceph_client() {
 151   $outfile/src/pybind/mgr/rook/generate_rook_ceph_client.sh
 152   tar cf rook_ceph_client.tar $outfile/src/pybind/mgr/rook/rook_client/*.py
 153 }
 154
 155 # clean out old cruft...
 156 echo "cleanup..."
 157 rm -f $outfile*
 158
 159 # build new tarball
 160 echo "building tarball..."
 161 bin/git-archive-all.sh --prefix ceph-$version/ \
 162                        --verbose \
 163                        --ignore corpus \
 164                        $outfile.tar
 165
 166 # populate files with version strings
 167 echo "including src/.git_version, ceph.spec"
 168
 169 (git rev-parse HEAD ; echo $version) 2> /dev/null > src/.git_version
 170
 171 for spec in ceph.spec.in; do
 172     cat $spec |
 173         sed "s/@PROJECT_VERSION@/$rpm_version/g" |
 174         sed "s/@RPM_RELEASE@/$rpm_release/g" |
 175         sed "s/@TARBALL_BASENAME@/ceph-$version/g" > `echo $spec | sed 's/.in$//'`
 176 done
 177 ln -s . $outfile
 178 tar cvf $outfile.version.tar $outfile/src/.git_version $outfile/ceph.spec
 179 # NOTE: If you change this version number make sure the package is available
 180 # at the three URLs referenced below (may involve uploading to download.ceph.com)
 181 boost_version=1.75.0
 182 download_boost $boost_version 953db31e016db7bb207f11432bef7df100516eeb746843fa0486a222e3fd49cb \
 183                https://boostorg.jfrog.io/artifactory/main/release/$boost_version/source \
 184                https://downloads.sourceforge.net/project/boost/boost/$boost_version \
 185                https://download.ceph.com/qa
 186 download_liburing 0.7 8e2842cfe947f3a443af301bdd6d034455536c38a455c7a700d0c1ad165a7543 \
 187                   https://github.com/axboe/liburing/archive \
 188                   https://git.kernel.dk/cgit/liburing/snapshot
 189 pmdk_version=1.10
 190 download_pmdk $pmdk_version 08dafcf94db5ac13fac9139c92225d9aa5f3724ea74beee4e6ca19a01a2eb20c \
 191                https://github.com/pmem/pmdk/releases/download/$pmdk_version
 192 build_dashboard_frontend
 193 generate_rook_ceph_client
 194 for tarball in $outfile.version   \
 195                $outfile.boost     \
 196                $outfile.liburing  \
 197                $outfile.pmdk  \
 198                dashboard_frontend \
 199                rook_ceph_client   \
 200                $outfile; do
 201     tar --concatenate -f $outfile.all.tar $tarball.tar
 202     rm $tarball.tar
 203 done
 204 mv $outfile.all.tar $outfile.tar
 205 rm $outfile
 206
 207 echo "compressing..."
 208 bzip2 -9 $outfile.tar
 209
 210 echo "done."