ceph/win32_deps_build.sh

   1 #!/usr/bin/env bash
   2
   3 set -e
   4
   5 SCRIPT_DIR="$(dirname "$BASH_SOURCE")"
   6 SCRIPT_DIR="$(realpath "$SCRIPT_DIR")"
   7
   8 num_vcpus=$(nproc)
   9 NUM_WORKERS=${NUM_WORKERS:-$num_vcpus}
  10
  11 DEPS_DIR="${DEPS_DIR:-$SCRIPT_DIR/build.deps}"
  12 depsSrcDir="$DEPS_DIR/src"
  13 depsToolsetDir="$DEPS_DIR/mingw"
  14
  15 lz4SrcDir="${depsSrcDir}/lz4"
  16 lz4Dir="${depsToolsetDir}/lz4"
  17 lz4Tag="v1.9.2"
  18 sslTag="OpenSSL_1_1_1c"
  19 sslDir="${depsToolsetDir}/openssl"
  20 sslSrcDir="${depsSrcDir}/openssl"
  21
  22 # For now, we'll keep the version number within the file path when not using git.
  23 boostUrl="https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.gz"
  24 boostSrcDir="${depsSrcDir}/boost_1_79_0"
  25 boostDir="${depsToolsetDir}/boost"
  26 zlibDir="${depsToolsetDir}/zlib"
  27 zlibSrcDir="${depsSrcDir}/zlib"
  28 backtraceDir="${depsToolsetDir}/libbacktrace"
  29 backtraceSrcDir="${depsSrcDir}/libbacktrace"
  30 snappySrcDir="${depsSrcDir}/snappy"
  31 snappyDir="${depsToolsetDir}/snappy"
  32 snappyTag="1.1.9"
  33 # Additional Windows libraries, which aren't provided by Mingw
  34 winLibDir="${depsToolsetDir}/windows/lib"
  35
  36 wnbdUrl="https://github.com/cloudbase/wnbd"
  37 wnbdTag="main"
  38 wnbdSrcDir="${depsSrcDir}/wnbd"
  39 wnbdLibDir="${depsToolsetDir}/wnbd/lib"
  40
  41 dokanUrl="https://github.com/dokan-dev/dokany"
  42 dokanTag="v2.0.5.1000"
  43 dokanSrcDir="${depsSrcDir}/dokany"
  44 dokanLibDir="${depsToolsetDir}/dokany/lib"
  45
  46 # Allow for OS specific customizations through the OS flag (normally
  47 # passed through from win32_build).
  48 # Valid options are currently "ubuntu", "rhel", and "suse".
  49 OS=${OS:-"ubuntu"}
  50
  51 function _make() {
  52   make -j $NUM_WORKERS $@
  53 }
  54
  55 if [[ -d $DEPS_DIR ]]; then
  56     echo "Cleaning up dependency build dir: $DEPS_DIR"
  57     rm -rf $DEPS_DIR
  58 fi
  59
  60 mkdir -p $DEPS_DIR
  61 mkdir -p $depsToolsetDir
  62 mkdir -p $depsSrcDir
  63
  64 echo "Installing required packages."
  65 case "$OS" in
  66     rhel)
  67         # pkgconf needs https://bugzilla.redhat.com/show_bug.cgi?id=1975416
  68         sudo yum -y --setopt=skip_missing_names_on_install=False install \
  69             mingw64-gcc-c++ \
  70             cmake \
  71             pkgconf \
  72             python3-devel \
  73             autoconf \
  74             libtool \
  75             ninja-build \
  76             zip \
  77             python3-PyYAML \
  78             gcc \
  79             diffutils \
  80             patch \
  81             wget \
  82             perl \
  83             git-core
  84         ;;
  85     ubuntu)
  86         sudo apt-get update
  87         sudo env DEBIAN_FRONTEND=noninteractive apt-get -y install \
  88             mingw-w64 g++ cmake pkg-config \
  89             python3-dev python3-yaml \
  90                 autoconf libtool ninja-build wget zip \
  91                 git
  92         ;;
  93     suse)
  94         for PKG in mingw64-cross-gcc-c++ mingw64-libgcc_s_seh1 mingw64-libstdc++6 \
  95                 cmake pkgconf python3-devel autoconf libtool ninja zip \
  96                 python3-PyYAML \
  97                 gcc patch wget git; do
  98             rpm -q $PKG >/dev/null || zypper -n install $PKG
  99         done
 100         ;;
 101 esac
 102
 103 MINGW_CMAKE_FILE="$DEPS_DIR/mingw.cmake"
 104 source "$SCRIPT_DIR/mingw_conf.sh"
 105
 106 echo "Building zlib."
 107 cd $depsSrcDir
 108 if [[ ! -d $zlibSrcDir ]]; then
 109     git clone --depth 1 https://github.com/madler/zlib
 110 fi
 111 cd $zlibSrcDir
 112 # Apparently the configure script is broken...
 113 sed -e s/"PREFIX = *$"/"PREFIX = ${MINGW_PREFIX}"/ -i win32/Makefile.gcc
 114 _make -f win32/Makefile.gcc
 115 _make BINARY_PATH=$zlibDir \
 116      INCLUDE_PATH=$zlibDir/include \
 117      LIBRARY_PATH=$zlibDir/lib \
 118      SHARED_MODE=1 \
 119      -f win32/Makefile.gcc install
 120
 121 echo "Building lz4."
 122 cd $depsToolsetDir
 123 if [[ ! -d $lz4Dir ]]; then
 124     git clone --branch $lz4Tag --depth 1 https://github.com/lz4/lz4
 125     cd $lz4Dir
 126 fi
 127 cd $lz4Dir
 128 _make BUILD_STATIC=no CC=${MINGW_CC%-posix*} \
 129       DLLTOOL=${MINGW_DLLTOOL} \
 130       WINDRES=${MINGW_WINDRES} \
 131       TARGET_OS=Windows_NT
 132
 133 echo "Building OpenSSL."
 134 cd $depsSrcDir
 135 if [[ ! -d $sslSrcDir ]]; then
 136     git clone --branch $sslTag --depth 1 https://github.com/openssl/openssl
 137     cd $sslSrcDir
 138 fi
 139 cd $sslSrcDir
 140 mkdir -p $sslDir
 141 CROSS_COMPILE="${MINGW_PREFIX}" ./Configure \
 142     mingw64 shared --prefix=$sslDir --libdir="$sslDir/lib"
 143 _make depend
 144 _make
 145 _make install_sw
 146
 147 echo "Building boost."
 148 cd $depsSrcDir
 149 if [[ ! -d $boostSrcDir ]]; then
 150     echo "Downloading boost."
 151     wget -qO- $boostUrl | tar xz
 152 fi
 153
 154 cd $boostSrcDir
 155 echo "using gcc : mingw32 : ${MINGW_CXX} ;" > user-config.jam
 156
 157 # Workaround for https://github.com/boostorg/thread/issues/156
 158 # Older versions of mingw provided a different pthread lib.
 159 sed -i 's/lib$(libname)GC2.a/lib$(libname).a/g' ./libs/thread/build/Jamfile.v2
 160 sed -i 's/mthreads/pthreads/g' ./tools/build/src/tools/gcc.py
 161 sed -i 's/mthreads/pthreads/g' ./tools/build/src/tools/gcc.jam
 162
 163 sed -i 's/pthreads/mthreads/g' ./tools/build/src/tools/gcc.py
 164 sed -i 's/pthreads/mthreads/g' ./tools/build/src/tools/gcc.jam
 165
 166 export PTW32_INCLUDE=${PTW32Include}
 167 export PTW32_LIB=${PTW32Lib}
 168
 169 echo "Patching boost."
 170 # Fix getting Windows page size
 171 # TODO: send this upstream and maybe use a fork until it merges.
 172 # Meanwhile, we might consider moving those to ceph/cmake/modules/BuildBoost.cmake.
 173 # This cmake module will first have to be updated to support Mingw though.
 174 patch -N boost/thread/pthread/thread_data.hpp <<EOL
 175 --- boost/thread/pthread/thread_data.hpp        2019-10-11 15:26:15.678703586 +0300
 176 +++ boost/thread/pthread/thread_data.hpp.new    2019-10-11 15:26:07.321463698 +0300
 177 @@ -32,6 +32,10 @@
 178  # endif
 179  #endif
 180
 181 +#if defined(_WIN32)
 182 +#include <windows.h>
 183 +#endif
 184 +
 185  #include <pthread.h>
 186  #include <unistd.h>
 187
 188 @@ -54,6 +58,10 @@
 189            if (size==0) return;
 190  #ifdef BOOST_THREAD_USES_GETPAGESIZE
 191            std::size_t page_size = getpagesize();
 192 +#elif _WIN32
 193 +          SYSTEM_INFO system_info;
 194 +          ::GetSystemInfo (&system_info);
 195 +          std::size_t page_size = system_info.dwPageSize;
 196  #else
 197            std::size_t page_size = ::sysconf( _SC_PAGESIZE);
 198  #endif
 199 EOL
 200
 201 ./bootstrap.sh
 202
 203 ./b2 install --user-config=user-config.jam toolset=gcc-mingw32 \
 204     target-os=windows release \
 205     link=static,shared \
 206     threadapi=win32 --prefix=$boostDir \
 207     address-model=64 architecture=x86 \
 208     binary-format=pe abi=ms -j $NUM_WORKERS \
 209     -sZLIB_INCLUDE=$zlibDir/include -sZLIB_LIBRARY_PATH=$zlibDir/lib \
 210     --without-python --without-mpi --without-log --without-wave
 211
 212 echo "Building libbacktrace."
 213 cd $depsSrcDir
 214 if [[ ! -d $backtraceSrcDir ]]; then
 215     git clone --depth 1 https://github.com/ianlancetaylor/libbacktrace
 216 fi
 217 mkdir -p $backtraceSrcDir/build
 218 cd $backtraceSrcDir/build
 219 ../configure --prefix=$backtraceDir --exec-prefix=$backtraceDir \
 220              --host ${MINGW_BASE} --enable-host-shared \
 221              --libdir="$backtraceDir/lib"
 222 _make LDFLAGS="-no-undefined"
 223 _make install
 224
 225 echo "Building snappy."
 226 cd $depsSrcDir
 227 if [[ ! -d $snappySrcDir ]]; then
 228     git clone --branch $snappyTag --depth 1 https://github.com/google/snappy
 229     cd $snappySrcDir
 230 fi
 231 mkdir -p $snappySrcDir/build
 232 cd $snappySrcDir/build
 233
 234 cmake -DCMAKE_INSTALL_PREFIX=$snappyDir \
 235       -DCMAKE_BUILD_TYPE=Release \
 236       -DBUILD_SHARED_LIBS=ON \
 237       -DSNAPPY_BUILD_TESTS=OFF \
 238       -DSNAPPY_BUILD_BENCHMARKS=OFF \
 239       -DCMAKE_TOOLCHAIN_FILE=$MINGW_CMAKE_FILE \
 240       ../
 241 _make
 242 _make install
 243
 244 cmake -DCMAKE_INSTALL_PREFIX=$snappyDir \
 245       -DCMAKE_BUILD_TYPE=Release \
 246       -DBUILD_SHARED_LIBS=OFF \
 247       -DSNAPPY_BUILD_TESTS=OFF \
 248       -DCMAKE_TOOLCHAIN_FILE=$MINGW_CMAKE_FILE \
 249       ../
 250 _make
 251 _make install
 252
 253 echo "Generating mswsock.lib."
 254 # mswsock.lib is not provided by mingw, so we'll have to generate
 255 # it.
 256 mkdir -p $winLibDir
 257 cat > $winLibDir/mswsock.def <<EOF
 258 LIBRARY MSWSOCK.DLL
 259 EXPORTS
 260 AcceptEx@32
 261 EnumProtocolsA@12
 262 EnumProtocolsW@12
 263 GetAcceptExSockaddrs@32
 264 GetAddressByNameA@40
 265 GetAddressByNameW@40
 266 GetNameByTypeA@12
 267 GetNameByTypeW@12
 268 GetServiceA@28
 269 GetServiceW@28
 270 GetTypeByNameA@8
 271 GetTypeByNameW@8
 272 MigrateWinsockConfiguration@12
 273 NPLoadNameSpaces@12
 274 SetServiceA@24
 275 SetServiceW@24
 276 TransmitFile@28
 277 WSARecvEx@16
 278 dn_expand@20
 279 getnetbyname@4
 280 inet_network@4
 281 rcmd@24
 282 rexec@24rresvport@4
 283 s_perror@8sethostname@8
 284 EOF
 285
 286 $MINGW_DLLTOOL -d $winLibDir/mswsock.def \
 287                -l $winLibDir/libmswsock.a
 288
 289 echo "Fetching libwnbd."
 290 cd $depsSrcDir
 291 if [[ ! -d $wnbdSrcDir ]]; then
 292     git clone --branch $wnbdTag --depth 1 $wnbdUrl
 293 fi
 294 cd $wnbdSrcDir
 295 mkdir -p $wnbdLibDir
 296 $MINGW_DLLTOOL -d $wnbdSrcDir/libwnbd/libwnbd.def \
 297                -D libwnbd.dll \
 298                -l $wnbdLibDir/libwnbd.a
 299
 300 echo "Fetching dokany."
 301 cd $depsSrcDir
 302 if [[ ! -d $dokanSrcDir ]]; then
 303     git clone --branch $dokanTag --depth 1 $dokanUrl
 304 fi
 305
 306 mkdir -p $dokanLibDir
 307 $MINGW_DLLTOOL -d $dokanSrcDir/dokan/dokan.def \
 308                -l $dokanLibDir/libdokan.a
 309
 310 # That's probably the easiest way to deal with the dokan imports.
 311 # dokan.h is defined in both ./dokan and ./sys while both are using
 312 # sys/public.h without the "sys" prefix.
 313 cp $dokanSrcDir/sys/public.h $dokanSrcDir/dokan
 314
 315 echo "Finished building Ceph dependencies."
 316 touch $depsToolsetDir/completed