ceph/win32_deps_build.sh

   1 #!/usr/bin/env bash
   2
   3 set -e
   4
   5 SCRIPT_DIR="$(dirname "$BASH_SOURCE")"
   6 SCRIPT_DIR="$(realpath "$SCRIPT_DIR")"
   7
   8 num_vcpus=$(( $(lscpu -p | tail -1 | cut -d "," -f 1) + 1 ))
   9 NUM_WORKERS=${NUM_WORKERS:-$num_vcpus}
  10
  11 DEPS_DIR="${DEPS_DIR:-$SCRIPT_DIR/build.deps}"
  12 depsSrcDir="$DEPS_DIR/src"
  13 depsToolsetDir="$DEPS_DIR/mingw"
  14
  15 lz4SrcDir="${depsSrcDir}/lz4"
  16 lz4Dir="${depsToolsetDir}/lz4"
  17 lz4Tag="v1.9.2"
  18 sslVersion="1.1.1c"
  19 sslDir="${depsToolsetDir}/openssl"
  20 sslSrcDir="${depsSrcDir}/openssl-${sslVersion}"
  21
  22 curlTag="curl-7_66_0"
  23 curlSrcDir="${depsSrcDir}/curl"
  24 curlDir="${depsToolsetDir}/curl"
  25
  26 # For now, we'll keep the version number within the file path when not using git.
  27 boostUrl="https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.gz"
  28 boostSrcDir="${depsSrcDir}/boost_1_70_0"
  29 boostDir="${depsToolsetDir}/boost"
  30 zlibDir="${depsToolsetDir}/zlib"
  31 zlibSrcDir="${depsSrcDir}/zlib"
  32 backtraceDir="${depsToolsetDir}/backtrace"
  33 backtraceSrcDir="${depsSrcDir}/backtrace"
  34 snappySrcDir="${depsSrcDir}/snappy"
  35 snappyDir="${depsToolsetDir}/snappy"
  36 snappyTag="1.1.7"
  37 # Additional Windows libraries, which aren't provided by Mingw
  38 winLibDir="${depsToolsetDir}/windows/lib"
  39
  40
  41 MINGW_PREFIX="x86_64-w64-mingw32-"
  42
  43 function _make() {
  44   make -j $NUM_WORKERS $@
  45 }
  46
  47 mkdir -p $DEPS_DIR
  48 mkdir -p $depsToolsetDir
  49 mkdir -p $depsSrcDir
  50
  51 MINGW_CMAKE_FILE="$DEPS_DIR/mingw.cmake"
  52 cat > $MINGW_CMAKE_FILE <<EOL
  53 set(CMAKE_SYSTEM_NAME Windows)
  54 set(TOOLCHAIN_PREFIX x86_64-w64-mingw32)
  55
  56 # We'll need to use posix threads in order to use
  57 # C++11 features, such as std::thread.
  58 set(CMAKE_C_COMPILER \${TOOLCHAIN_PREFIX}-gcc-posix)
  59 set(CMAKE_CXX_COMPILER \${TOOLCHAIN_PREFIX}-g++-posix)
  60 set(CMAKE_RC_COMPILER \${TOOLCHAIN_PREFIX}-windres)
  61
  62 set(CMAKE_FIND_ROOT_PATH /usr/\${TOOLCHAIN_PREFIX} /usr/lib/gcc/\${TOOLCHAIN_PREFIX}/7.3-posix)
  63 set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
  64 set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY BOTH)
  65 set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH)
  66 EOL
  67
  68 sudo apt-get -y install mingw-w64 cmake pkg-config python3-dev python3-pip \
  69                 autoconf libtool ninja-build
  70 sudo python3 -m pip install cython
  71
  72 cd $depsSrcDir
  73 if [[ ! -d $zlibDir ]]; then
  74     git clone https://github.com/madler/zlib
  75 fi
  76 cd zlib
  77 # Apparently the configure script is broken...
  78 sed -e s/"PREFIX ="/"PREFIX = x86_64-w64-mingw32-"/ -i win32/Makefile.gcc
  79 _make -f win32/Makefile.gcc
  80 _make BINARY_PATH=$zlibDir \
  81      INCLUDE_PATH=$zlibDir/include \
  82      LIBRARY_PATH=$zlibDir/lib \
  83      SHARED_MODE=1 \
  84      -f win32/Makefile.gcc install
  85
  86 cd $depsToolsetDir
  87 if [[ ! -d $lz4Dir ]]; then
  88     git clone https://github.com/lz4/lz4
  89 fi
  90 cd lz4
  91 git checkout $lz4Tag
  92 _make BUILD_STATIC=no CC=x86_64-w64-mingw32-gcc \
  93       DLLTOOL=x86_64-w64-mingw32-dlltool OS=Windows_NT
  94
  95 cd $depsSrcDir
  96 if [[ ! -d $sslSrcDir ]]; then
  97     curl "https://www.openssl.org/source/openssl-${sslVersion}.tar.gz" | tar xz
  98 fi
  99 cd $sslSrcDir
 100 mkdir -p $sslDir
 101 CROSS_COMPILE="x86_64-w64-mingw32-" ./Configure \
 102     mingw64 shared --prefix=$sslDir
 103 _make depend
 104 _make
 105 _make install
 106
 107 cd $depsSrcDir
 108 if [[ ! -d $curlSrcDir ]]; then
 109     git clone https://github.com/curl/curl
 110 fi
 111 cd $curlSrcDir
 112 git checkout $curlTag
 113 ./buildconf
 114 ./configure --prefix=$curlDir --with-ssl=$sslDir --with-zlib=$zlibDir \
 115             --host=x86_64-w64-mingw32
 116 _make
 117 _make install
 118
 119
 120 cd $depsSrcDir
 121 if [[ ! -d $boostSrcDir ]]; then
 122     wget -qO- $boostUrl | tar xz
 123 fi
 124
 125 cd $boostSrcDir
 126 echo "using gcc : mingw32 : x86_64-w64-mingw32-g++-posix ;" > user-config.jam
 127
 128 # Workaround for https://github.com/boostorg/thread/issues/156
 129 # Older versions of mingw provided a different pthread lib.
 130 sed -i 's/lib$(libname)GC2.a/lib$(libname).a/g' ./libs/thread/build/Jamfile.v2
 131 sed -i 's/mthreads/pthreads/g' ./tools/build/src/tools/gcc.py
 132 sed -i 's/mthreads/pthreads/g' ./tools/build/src/tools/gcc.jam
 133
 134 sed -i 's/pthreads/mthreads/g' ./tools/build/src/tools/gcc.py
 135 sed -i 's/pthreads/mthreads/g' ./tools/build/src/tools/gcc.jam
 136
 137 export PTW32_INCLUDE=/usr/share/mingw-w64/include
 138 export PTW32_LIB=/usr/x86_64-w64-mingw32/lib
 139
 140 # Fix getting Windows page size
 141 # TODO: send this upstream and maybe use a fork until it merges.
 142 # Meanwhile, we might consider moving those to ceph/cmake/modules/BuildBoost.cmake.
 143 # This cmake module will first have to be updated to support Mingw though.
 144 cat | patch -N boost/thread/pthread/thread_data.hpp <<EOL
 145 --- boost/thread/pthread/thread_data.hpp        2019-10-11 15:26:15.678703586 +0300
 146 +++ boost/thread/pthread/thread_data.hpp.new    2019-10-11 15:26:07.321463698 +0300
 147 @@ -32,6 +32,10 @@
 148  # endif
 149  #endif
 150
 151 +#if defined(_WIN32)
 152 +#include <windows.h>
 153 +#endif
 154 +
 155  #include <pthread.h>
 156  #include <unistd.h>
 157
 158 @@ -54,6 +58,10 @@
 159            if (size==0) return;
 160  #ifdef BOOST_THREAD_USES_GETPAGESIZE
 161            std::size_t page_size = getpagesize();
 162 +#elif _WIN32
 163 +          SYSTEM_INFO system_info;
 164 +          ::GetSystemInfo (&system_info);
 165 +          std::size_t page_size = system_info.dwPageSize;
 166  #else
 167            std::size_t page_size = ::sysconf( _SC_PAGESIZE);
 168  #endif
 169 EOL
 170
 171 # Use pthread if requested
 172 cat | patch -N boost/asio/detail/thread.hpp <<EOL
 173 --- boost/asio/detail/thread.hpp        2019-10-11 16:26:11.191094656 +0300
 174 +++ boost/asio/detail/thread.hpp.new    2019-10-11 16:26:03.310542438 +0300
 175 @@ -19,6 +19,8 @@
 176
 177  #if !defined(BOOST_ASIO_HAS_THREADS)
 178  # include <boost/asio/detail/null_thread.hpp>
 179 +#elif defined(BOOST_ASIO_HAS_PTHREADS)
 180 +# include <boost/asio/detail/posix_thread.hpp>
 181  #elif defined(BOOST_ASIO_WINDOWS)
 182  # if defined(UNDER_CE)
 183  #  include <boost/asio/detail/wince_thread.hpp>
 184 @@ -27,8 +29,6 @@
 185  # else
 186  #  include <boost/asio/detail/win_thread.hpp>
 187  # endif
 188 -#elif defined(BOOST_ASIO_HAS_PTHREADS)
 189 -# include <boost/asio/detail/posix_thread.hpp>
 190  #elif defined(BOOST_ASIO_HAS_STD_THREAD)
 191  # include <boost/asio/detail/std_thread.hpp>
 192  #else
 193 @@ -41,6 +41,8 @@
 194
 195  #if !defined(BOOST_ASIO_HAS_THREADS)
 196  typedef null_thread thread;
 197 +#elif defined(BOOST_ASIO_HAS_PTHREADS)
 198 +typedef posix_thread thread;
 199  #elif defined(BOOST_ASIO_WINDOWS)
 200  # if defined(UNDER_CE)
 201  typedef wince_thread thread;
 202 @@ -49,8 +51,6 @@
 203  # else
 204  typedef win_thread thread;
 205  # endif
 206 -#elif defined(BOOST_ASIO_HAS_PTHREADS)
 207 -typedef posix_thread thread;
 208  #elif defined(BOOST_ASIO_HAS_STD_THREAD)
 209  typedef std_thread thread;
 210  #endif
 211 EOL
 212
 213 # Unix socket support for Windows is currently disabled by Boost.
 214 # https://github.com/huangqinjin/asio/commit/d27a8ad1870
 215 cat | patch -N boost/asio/detail/socket_types.hpp <<EOL
 216 --- boost/asio/detail/socket_types.hpp       2019-11-29 16:50:58.647125797 +0000
 217 +++ boost/asio/detail/socket_types.hpp.new   2020-01-13 11:45:05.015104678 +0000
 218 @@ -200,6 +200,8 @@
 219  typedef ipv6_mreq in6_mreq_type;
 220  typedef sockaddr_in6 sockaddr_in6_type;
 221  typedef sockaddr_storage sockaddr_storage_type;
 222 +struct sockaddr_un_type { u_short sun_family;
 223 +  char sun_path[108]; }; /* copy from afunix.h */
 224  typedef addrinfo addrinfo_type;
 225  # endif
 226  typedef ::linger linger_type;
 227 EOL
 228 cat | patch -N boost/asio/detail/config.hpp <<EOL
 229 --- boost/asio/detail/config.hpp       2019-11-29 16:50:58.691126211 +0000
 230 +++ boost/asio/detail/config.hpp.new   2020-01-13 13:09:17.966771750 +0000
 231 @@ -1142,13 +1142,9 @@
 232  // UNIX domain sockets.
 233  #if !defined(BOOST_ASIO_HAS_LOCAL_SOCKETS)
 234  # if !defined(BOOST_ASIO_DISABLE_LOCAL_SOCKETS)
 235 -#  if !defined(BOOST_ASIO_WINDOWS) \\
 236 -  && !defined(BOOST_ASIO_WINDOWS_RUNTIME) \\
 237 -  && !defined(__CYGWIN__)
 238 +#  if !defined(BOOST_ASIO_WINDOWS_RUNTIME)
 239  #   define BOOST_ASIO_HAS_LOCAL_SOCKETS 1
 240 -#  endif // !defined(BOOST_ASIO_WINDOWS)
 241 -         //   && !defined(BOOST_ASIO_WINDOWS_RUNTIME)
 242 -         //   && !defined(__CYGWIN__)
 243 +#  endif // !defined(BOOST_ASIO_WINDOWS_RUNTIME)
 244  # endif // !defined(BOOST_ASIO_DISABLE_LOCAL_SOCKETS)
 245  #endif // !defined(BOOST_ASIO_HAS_LOCAL_SOCKETS)
 246 EOL
 247
 248 ./bootstrap.sh
 249
 250 ./b2 install --user-config=user-config.jam toolset=gcc-mingw32 \
 251     target-os=windows release \
 252     threadapi=pthread --prefix=$boostDir \
 253     address-model=64 architecture=x86 \
 254     binary-format=pe abi=ms -j $NUM_WORKERS \
 255     -sZLIB_INCLUDE=$zlibDir/include -sZLIB_LIBRARY_PATH=$zlibDir/lib \
 256     --without-python --without-mpi
 257
 258 cd $depsSrcDir
 259 if [[ ! -d $backtraceSrcDir ]]; then
 260     git clone https://github.com/ianlancetaylor/libbacktrace
 261 fi
 262 mkdir libbacktrace/build
 263 cd libbacktrace/build
 264 ../configure --prefix=$backtraceDir --exec-prefix=$backtraceDir \
 265              --host x86_64-w64-mingw32 --enable-host-shared
 266 _make LDFLAGS="-no-undefined"
 267 _make install
 268 cp $backtraceDir/lib/libbacktrace.a $backtraceDir/lib/libbacktrace.dll.a
 269
 270 cd $depsSrcDir
 271 if [[ ! -d $snappySrcDir ]]; then
 272     git clone https://github.com/google/snappy
 273 fi
 274 mkdir -p snappy/build
 275 cd snappy && git checkout $snappyTag
 276 cd build
 277
 278 cmake -DCMAKE_INSTALL_PREFIX=$snappyDir \
 279       -DCMAKE_BUILD_TYPE=Release \
 280       -DBUILD_SHARED_LIBS=ON \
 281       -DSNAPPY_BUILD_TESTS=OFF \
 282       -DCMAKE_TOOLCHAIN_FILE=$MINGW_CMAKE_FILE \
 283       ../
 284 _make
 285 _make install
 286
 287 cmake -DCMAKE_INSTALL_PREFIX=$snappyDir \
 288       -DCMAKE_BUILD_TYPE=Release \
 289       -DBUILD_SHARED_LIBS=OFF \
 290       -DSNAPPY_BUILD_TESTS=OFF \
 291       -DCMAKE_TOOLCHAIN_FILE=$MINGW_CMAKE_FILE \
 292       ../
 293 _make
 294 _make install
 295
 296 # mswsock.lib is not provided by mingw, so we'll have to generate
 297 # it.
 298 mkdir -p $winLibDir
 299 cat > $winLibDir/mswsock.def <<EOF
 300 LIBRARY MSWSOCK.DLL
 301 EXPORTS
 302 AcceptEx@32
 303 EnumProtocolsA@12
 304 EnumProtocolsW@12
 305 GetAcceptExSockaddrs@32
 306 GetAddressByNameA@40
 307 GetAddressByNameW@40
 308 GetNameByTypeA@12
 309 GetNameByTypeW@12
 310 GetServiceA@28
 311 GetServiceW@28
 312 GetTypeByNameA@8
 313 GetTypeByNameW@8
 314 MigrateWinsockConfiguration@12
 315 NPLoadNameSpaces@12
 316 SetServiceA@24
 317 SetServiceW@24
 318 TransmitFile@28
 319 WSARecvEx@16
 320 dn_expand@20
 321 getnetbyname@4
 322 inet_network@4
 323 rcmd@24
 324 rexec@24rresvport@4
 325 s_perror@8sethostname@8
 326 EOF
 327
 328 x86_64-w64-mingw32-dlltool -d $winLibDir/mswsock.def \
 329                            -l $winLibDir/libmswsock.a