ceph/win32_deps_build.sh

   1 #!/usr/bin/env bash
   2
   3 set -e
   4
   5 SCRIPT_DIR="$(dirname "$BASH_SOURCE")"
   6 SCRIPT_DIR="$(realpath "$SCRIPT_DIR")"
   7
   8 num_vcpus=$(nproc)
   9 NUM_WORKERS=${NUM_WORKERS:-$num_vcpus}
  10
  11 DEPS_DIR="${DEPS_DIR:-$SCRIPT_DIR/build.deps}"
  12 depsSrcDir="$DEPS_DIR/src"
  13 depsToolsetDir="$DEPS_DIR/mingw"
  14
  15 lz4SrcDir="${depsSrcDir}/lz4"
  16 lz4Dir="${depsToolsetDir}/lz4"
  17 lz4Tag="v1.9.2"
  18 sslTag="OpenSSL_1_1_1c"
  19 sslDir="${depsToolsetDir}/openssl"
  20 sslSrcDir="${depsSrcDir}/openssl"
  21
  22 curlTag="curl-7_66_0"
  23 curlSrcDir="${depsSrcDir}/curl"
  24 curlDir="${depsToolsetDir}/curl"
  25
  26 # For now, we'll keep the version number within the file path when not using git.
  27 boostUrl="https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz"
  28 boostSrcDir="${depsSrcDir}/boost_1_75_0"
  29 boostDir="${depsToolsetDir}/boost"
  30 zlibDir="${depsToolsetDir}/zlib"
  31 zlibSrcDir="${depsSrcDir}/zlib"
  32 backtraceDir="${depsToolsetDir}/libbacktrace"
  33 backtraceSrcDir="${depsSrcDir}/libbacktrace"
  34 snappySrcDir="${depsSrcDir}/snappy"
  35 snappyDir="${depsToolsetDir}/snappy"
  36 snappyTag="1.1.9"
  37 # Additional Windows libraries, which aren't provided by Mingw
  38 winLibDir="${depsToolsetDir}/windows/lib"
  39
  40 wnbdUrl="https://github.com/cloudbase/wnbd"
  41 wnbdTag="master"
  42 wnbdSrcDir="${depsSrcDir}/wnbd"
  43 wnbdLibDir="${depsToolsetDir}/wnbd/lib"
  44
  45 dokanUrl="https://github.com/dokan-dev/dokany"
  46 dokanTag="v1.3.1.1000"
  47 dokanSrcDir="${depsSrcDir}/dokany"
  48 dokanLibDir="${depsToolsetDir}/dokany/lib"
  49
  50 # Allow for OS specific customizations through the OS flag (normally
  51 # passed through from win32_build).
  52 # Valid options are currently "ubuntu" and "suse".
  53 OS=${OS:-"ubuntu"}
  54
  55 function _make() {
  56   make -j $NUM_WORKERS $@
  57 }
  58
  59 if [[ -d $DEPS_DIR ]]; then
  60     echo "Cleaning up dependency build dir: $DEPS_DIR"
  61     rm -rf $DEPS_DIR
  62 fi
  63
  64 mkdir -p $DEPS_DIR
  65 mkdir -p $depsToolsetDir
  66 mkdir -p $depsSrcDir
  67
  68 echo "Installing required packages."
  69 case "$OS" in
  70     ubuntu)
  71         sudo apt-get update
  72         sudo apt-get -y install mingw-w64 cmake pkg-config \
  73             python3-dev python3-pip python3-yaml \
  74                 autoconf libtool ninja-build zip
  75         sudo python3 -m pip install cython
  76         ;;
  77     suse)
  78         for PKG in mingw64-cross-gcc-c++ mingw64-libgcc_s_seh1 mingw64-libstdc++6 \
  79                 cmake pkgconf python3-devel autoconf libtool ninja zip \
  80                 python3-Cython python3-PyYAML \
  81                 gcc patch wget git; do
  82             rpm -q $PKG >/dev/null || zypper -n install $PKG
  83         done
  84         ;;
  85 esac
  86
  87 MINGW_CMAKE_FILE="$DEPS_DIR/mingw.cmake"
  88 source "$SCRIPT_DIR/mingw_conf.sh"
  89
  90 echo "Building zlib."
  91 cd $depsSrcDir
  92 if [[ ! -d $zlibSrcDir ]]; then
  93     git clone https://github.com/madler/zlib
  94 fi
  95 cd $zlibSrcDir
  96 # Apparently the configure script is broken...
  97 sed -e s/"PREFIX = *$"/"PREFIX = ${MINGW_PREFIX}"/ -i win32/Makefile.gcc
  98 _make -f win32/Makefile.gcc
  99 _make BINARY_PATH=$zlibDir \
 100      INCLUDE_PATH=$zlibDir/include \
 101      LIBRARY_PATH=$zlibDir/lib \
 102      SHARED_MODE=1 \
 103      -f win32/Makefile.gcc install
 104
 105 echo "Building lz4."
 106 cd $depsToolsetDir
 107 if [[ ! -d $lz4Dir ]]; then
 108     git clone --branch $lz4Tag --depth 1 https://github.com/lz4/lz4
 109     cd $lz4Dir
 110 fi
 111 cd $lz4Dir
 112 _make BUILD_STATIC=no CC=${MINGW_CC%-posix*} \
 113       DLLTOOL=${MINGW_DLLTOOL} \
 114       WINDRES=${MINGW_WINDRES} \
 115       TARGET_OS=Windows_NT
 116
 117 echo "Building OpenSSL."
 118 cd $depsSrcDir
 119 if [[ ! -d $sslSrcDir ]]; then
 120     git clone --branch $sslTag --depth 1 https://github.com/openssl/openssl
 121     cd $sslSrcDir
 122 fi
 123 cd $sslSrcDir
 124 mkdir -p $sslDir
 125 CROSS_COMPILE="${MINGW_PREFIX}" ./Configure \
 126     mingw64 shared --prefix=$sslDir --libdir="$sslDir/lib"
 127 _make depend
 128 _make
 129 _make install
 130
 131 echo "Building libcurl."
 132 cd $depsSrcDir
 133 if [[ ! -d $curlSrcDir ]]; then
 134     git clone --branch $curlTag --depth 1 https://github.com/curl/curl
 135     cd $curlSrcDir
 136 fi
 137 cd $curlSrcDir
 138 ./buildconf
 139 ./configure --prefix=$curlDir --with-ssl=$sslDir --with-zlib=$zlibDir \
 140             --host=${MINGW_BASE} --libdir="$curlDir/lib"
 141 _make
 142 _make install
 143
 144
 145 echo "Building boost."
 146 cd $depsSrcDir
 147 if [[ ! -d $boostSrcDir ]]; then
 148     echo "Downloading boost."
 149     wget -qO- $boostUrl | tar xz
 150 fi
 151
 152 cd $boostSrcDir
 153 echo "using gcc : mingw32 : ${MINGW_CXX} ;" > user-config.jam
 154
 155 # Workaround for https://github.com/boostorg/thread/issues/156
 156 # Older versions of mingw provided a different pthread lib.
 157 sed -i 's/lib$(libname)GC2.a/lib$(libname).a/g' ./libs/thread/build/Jamfile.v2
 158 sed -i 's/mthreads/pthreads/g' ./tools/build/src/tools/gcc.py
 159 sed -i 's/mthreads/pthreads/g' ./tools/build/src/tools/gcc.jam
 160
 161 sed -i 's/pthreads/mthreads/g' ./tools/build/src/tools/gcc.py
 162 sed -i 's/pthreads/mthreads/g' ./tools/build/src/tools/gcc.jam
 163
 164 export PTW32_INCLUDE=${PTW32Include}
 165 export PTW32_LIB=${PTW32Lib}
 166
 167 echo "Patching boost."
 168 # Fix getting Windows page size
 169 # TODO: send this upstream and maybe use a fork until it merges.
 170 # Meanwhile, we might consider moving those to ceph/cmake/modules/BuildBoost.cmake.
 171 # This cmake module will first have to be updated to support Mingw though.
 172 patch -N boost/thread/pthread/thread_data.hpp <<EOL
 173 --- boost/thread/pthread/thread_data.hpp        2019-10-11 15:26:15.678703586 +0300
 174 +++ boost/thread/pthread/thread_data.hpp.new    2019-10-11 15:26:07.321463698 +0300
 175 @@ -32,6 +32,10 @@
 176  # endif
 177  #endif
 178
 179 +#if defined(_WIN32)
 180 +#include <windows.h>
 181 +#endif
 182 +
 183  #include <pthread.h>
 184  #include <unistd.h>
 185
 186 @@ -54,6 +58,10 @@
 187            if (size==0) return;
 188  #ifdef BOOST_THREAD_USES_GETPAGESIZE
 189            std::size_t page_size = getpagesize();
 190 +#elif _WIN32
 191 +          SYSTEM_INFO system_info;
 192 +          ::GetSystemInfo (&system_info);
 193 +          std::size_t page_size = system_info.dwPageSize;
 194  #else
 195            std::size_t page_size = ::sysconf( _SC_PAGESIZE);
 196  #endif
 197 EOL
 198
 199 ./bootstrap.sh
 200
 201 ./b2 install --user-config=user-config.jam toolset=gcc-mingw32 \
 202     target-os=windows release \
 203     link=static,shared \
 204     threadapi=pthread --prefix=$boostDir \
 205     address-model=64 architecture=x86 \
 206     binary-format=pe abi=ms -j $NUM_WORKERS \
 207     -sZLIB_INCLUDE=$zlibDir/include -sZLIB_LIBRARY_PATH=$zlibDir/lib \
 208     --without-python --without-mpi --without-log --without-wave
 209
 210 echo "Building libbacktrace."
 211 cd $depsSrcDir
 212 if [[ ! -d $backtraceSrcDir ]]; then
 213     git clone https://github.com/ianlancetaylor/libbacktrace
 214 fi
 215 mkdir -p $backtraceSrcDir/build
 216 cd $backtraceSrcDir/build
 217 ../configure --prefix=$backtraceDir --exec-prefix=$backtraceDir \
 218              --host ${MINGW_BASE} --enable-host-shared \
 219              --libdir="$backtraceDir/lib"
 220 _make LDFLAGS="-no-undefined"
 221 _make install
 222
 223 echo "Building snappy."
 224 cd $depsSrcDir
 225 if [[ ! -d $snappySrcDir ]]; then
 226     git clone --branch $snappyTag --depth 1 https://github.com/google/snappy
 227     cd $snappySrcDir
 228 fi
 229 mkdir -p $snappySrcDir/build
 230 cd $snappySrcDir/build
 231
 232 cmake -DCMAKE_INSTALL_PREFIX=$snappyDir \
 233       -DCMAKE_BUILD_TYPE=Release \
 234       -DBUILD_SHARED_LIBS=ON \
 235       -DSNAPPY_BUILD_TESTS=OFF \
 236       -DSNAPPY_BUILD_BENCHMARKS=OFF \
 237       -DCMAKE_TOOLCHAIN_FILE=$MINGW_CMAKE_FILE \
 238       ../
 239 _make
 240 _make install
 241
 242 cmake -DCMAKE_INSTALL_PREFIX=$snappyDir \
 243       -DCMAKE_BUILD_TYPE=Release \
 244       -DBUILD_SHARED_LIBS=OFF \
 245       -DSNAPPY_BUILD_TESTS=OFF \
 246       -DCMAKE_TOOLCHAIN_FILE=$MINGW_CMAKE_FILE \
 247       ../
 248 _make
 249 _make install
 250
 251 echo "Generating mswsock.lib."
 252 # mswsock.lib is not provided by mingw, so we'll have to generate
 253 # it.
 254 mkdir -p $winLibDir
 255 cat > $winLibDir/mswsock.def <<EOF
 256 LIBRARY MSWSOCK.DLL
 257 EXPORTS
 258 AcceptEx@32
 259 EnumProtocolsA@12
 260 EnumProtocolsW@12
 261 GetAcceptExSockaddrs@32
 262 GetAddressByNameA@40
 263 GetAddressByNameW@40
 264 GetNameByTypeA@12
 265 GetNameByTypeW@12
 266 GetServiceA@28
 267 GetServiceW@28
 268 GetTypeByNameA@8
 269 GetTypeByNameW@8
 270 MigrateWinsockConfiguration@12
 271 NPLoadNameSpaces@12
 272 SetServiceA@24
 273 SetServiceW@24
 274 TransmitFile@28
 275 WSARecvEx@16
 276 dn_expand@20
 277 getnetbyname@4
 278 inet_network@4
 279 rcmd@24
 280 rexec@24rresvport@4
 281 s_perror@8sethostname@8
 282 EOF
 283
 284 $MINGW_DLLTOOL -d $winLibDir/mswsock.def \
 285                -l $winLibDir/libmswsock.a
 286
 287 echo "Fetching libwnbd."
 288 cd $depsSrcDir
 289 if [[ ! -d $wnbdSrcDir ]]; then
 290     git clone $wnbdUrl
 291     cd $wnbdSrcDir && git checkout $wnbdTag
 292 fi
 293 cd $wnbdSrcDir
 294 mkdir -p $wnbdLibDir
 295 $MINGW_DLLTOOL -d $wnbdSrcDir/libwnbd/libwnbd.def \
 296                -D libwnbd.dll \
 297                -l $wnbdLibDir/libwnbd.a
 298
 299 echo "Fetching dokany."
 300 cd $depsSrcDir
 301 if [[ ! -d $dokanSrcDir ]]; then
 302     git clone $dokanUrl
 303 fi
 304 cd $dokanSrcDir
 305 git checkout $dokanTag
 306
 307 mkdir -p $dokanLibDir
 308 $MINGW_DLLTOOL -d $dokanSrcDir/dokan/dokan.def \
 309                -l $dokanLibDir/libdokan.a
 310
 311 # That's probably the easiest way to deal with the dokan imports.
 312 # dokan.h is defined in both ./dokan and ./sys while both are using
 313 # sys/public.h without the "sys" prefix.
 314 cp $dokanSrcDir/sys/public.h $dokanSrcDir/dokan
 315
 316 echo "Finished building Ceph dependencies."
 317 touch $depsToolsetDir/completed