src/ci/run.sh

   1 #!/usr/bin/env bash
   2
   3 set -e
   4
   5 if [ -n "$CI_JOB_NAME" ]; then
   6   echo "[CI_JOB_NAME=$CI_JOB_NAME]"
   7 fi
   8
   9 if [ "$NO_CHANGE_USER" = "" ]; then
  10   if [ "$LOCAL_USER_ID" != "" ]; then
  11     id -u user &>/dev/null || useradd --shell /bin/bash -u $LOCAL_USER_ID -o -c "" -m user
  12     export HOME=/home/user
  13     unset LOCAL_USER_ID
  14
  15     # Ensure that runners are able to execute git commands in the worktree,
  16     # overriding the typical git protections. In our docker container we're running
  17     # as root, while the user owning the checkout is not root.
  18     # This is only necessary when we change the user, otherwise we should
  19     # already be running with the right user.
  20     #
  21     # For NO_CHANGE_USER done in the small number of Dockerfiles affected.
  22     echo -e '[safe]\n\tdirectory = *' > /home/user/gitconfig
  23
  24     exec su --preserve-environment -c "env PATH=$PATH \"$0\"" user
  25   fi
  26 fi
  27
  28 # only enable core dump on Linux
  29 if [ -f /proc/sys/kernel/core_pattern ]; then
  30   ulimit -c unlimited
  31 fi
  32
  33 # There was a bad interaction between "old" 32-bit binaries on current 64-bit
  34 # kernels with selinux enabled, where ASLR mmap would sometimes choose a low
  35 # address and then block it for being below `vm.mmap_min_addr` -> `EACCES`.
  36 # This is probably a kernel bug, but setting `ulimit -Hs` works around it.
  37 # See also `dist-i686-linux` where this setting is enabled.
  38 if [ "$SET_HARD_RLIMIT_STACK" = "1" ]; then
  39   rlimit_stack=$(ulimit -Ss)
  40   if [ "$rlimit_stack" != "" ]; then
  41     ulimit -Hs "$rlimit_stack"
  42   fi
  43 fi
  44
  45 ci_dir=`cd $(dirname $0) && pwd`
  46 source "$ci_dir/shared.sh"
  47
  48 export CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
  49
  50 # suppress change-tracker warnings on CI
  51 if [ "$CI" != "" ]; then
  52     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set change-id=99999999"
  53 fi
  54
  55 if ! isCI || isCiBranch auto || isCiBranch beta || isCiBranch try || isCiBranch try-perf || \
  56   isCiBranch automation/bors/try; then
  57     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests"
  58     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.metrics"
  59     HAS_METRICS=1
  60 fi
  61
  62 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-verbose-configure"
  63 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache"
  64 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules"
  65 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-locked-deps"
  66 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-cargo-native-static"
  67 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-units-std=1"
  68 # rust-lang/promote-release will recompress CI artifacts, and while we care
  69 # about the per-commit artifact sizes, it's not as important that they're
  70 # highly compressed as it is that the process is fast. Best compression
  71 # generally implies single-threaded compression which results in wasting most
  72 # of our CPU resources.
  73 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set dist.compression-profile=balanced"
  74
  75 # When building for mingw, limit the number of parallel linker jobs during
  76 # the LLVM build, as not to run out of memory.
  77 # This is an attempt to fix the spurious build error tracked by
  78 # https://github.com/rust-lang/rust/issues/108227.
  79 if isKnownToBeMingwBuild; then
  80     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.link-jobs=1"
  81 fi
  82
  83 # Only produce xz tarballs on CI. gz tarballs will be generated by the release
  84 # process by recompressing the existing xz ones. This decreases the storage
  85 # space required for CI artifacts.
  86 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --dist-compression-formats=xz"
  87
  88 # Enable the `c` feature for compiler_builtins, but only when the `compiler-rt` source is available
  89 # (to avoid spending a lot of time cloning llvm)
  90 if [ "$EXTERNAL_LLVM" = "" ]; then
  91   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.optimized-compiler-builtins"
  92 elif [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
  93     echo "error: dist builds should always use optimized compiler-rt!" >&2
  94     exit 1
  95 fi
  96
  97 if [ "$DIST_SRC" = "" ]; then
  98   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src"
  99 fi
 100
 101 # Always set the release channel for bootstrap; this is normally not important (i.e., only dist
 102 # builds would seem to matter) but in practice bootstrap wants to know whether we're targeting
 103 # master, beta, or stable with a build to determine whether to run some checks (notably toolstate).
 104 export RUST_RELEASE_CHANNEL=$(releaseChannel)
 105 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --release-channel=$RUST_RELEASE_CHANNEL"
 106
 107 if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
 108   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-static-stdcpp"
 109   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.remap-debuginfo"
 110   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --debuginfo-level-std=1"
 111
 112   if [ "$NO_LLVM_ASSERTIONS" = "1" ]; then
 113     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-llvm-assertions"
 114   elif [ "$DEPLOY_ALT" != "" ]; then
 115     if [ "$ALT_PARALLEL_COMPILER" = "" ]; then
 116       RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.parallel-compiler=false"
 117     fi
 118     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions"
 119     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
 120   fi
 121
 122   CODEGEN_BACKENDS="${CODEGEN_BACKENDS:-llvm}"
 123   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=$CODEGEN_BACKENDS"
 124 else
 125   # We almost always want debug assertions enabled, but sometimes this takes too
 126   # long for too little benefit, so we just turn them off.
 127   if [ "$NO_DEBUG_ASSERTIONS" = "" ]; then
 128     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-debug-assertions"
 129   fi
 130
 131   # Same for overflow checks
 132   if [ "$NO_OVERFLOW_CHECKS" = "" ]; then
 133     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-overflow-checks"
 134   fi
 135
 136   # In general we always want to run tests with LLVM assertions enabled, but not
 137   # all platforms currently support that, so we have an option to disable.
 138   if [ "$NO_LLVM_ASSERTIONS" = "" ]; then
 139     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions"
 140   fi
 141
 142   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
 143
 144   # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen
 145   # tests as it will fail them.
 146   if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then
 147     # Test the Cranelift and GCC backends in CI. Bootstrap knows which targets to run tests on.
 148     CODEGEN_BACKENDS="${CODEGEN_BACKENDS:-llvm,cranelift,gcc}"
 149   else
 150     # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on.
 151     CODEGEN_BACKENDS="${CODEGEN_BACKENDS:-llvm,cranelift}"
 152   fi
 153   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=$CODEGEN_BACKENDS"
 154
 155   # We enable this for non-dist builders, since those aren't trying to produce
 156   # fresh binaries. We currently don't entirely support distributing a fresh
 157   # copy of the compiler (including llvm tools, etc.) if we haven't actually
 158   # built LLVM, since not everything necessary is copied into the
 159   # local-usage-only LLVM artifacts. If that changes, this could maybe be made
 160   # true for all builds. In practice it's probably a good idea to keep building
 161   # LLVM continuously on at least some builders to ensure it works, though.
 162   # (And PGO is its own can of worms).
 163   if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then
 164     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-unchanged"
 165   else
 166     # When building for CI we want to use the static C++ Standard library
 167     # included with LLVM, since a dynamic libstdcpp may not be available.
 168     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.static-libstdcpp"
 169   fi
 170 fi
 171
 172 # Unless we're using an older version of LLVM, check that all LLVM components
 173 # used by tests are available.
 174 if [ "$IS_NOT_LATEST_LLVM" = "" ]; then
 175   export COMPILETEST_NEEDS_ALL_LLVM_COMPONENTS=1
 176 fi
 177
 178 if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then
 179   # If `ENABLE_GCC_CODEGEN` is set and not empty, we add the `--enable-new-symbol-mangling`
 180   # argument to `RUST_CONFIGURE_ARGS` and set the `GCC_EXEC_PREFIX` environment variable.
 181   # `cg_gcc` doesn't support the legacy mangling so we need to enforce the new one
 182   # if we run `cg_gcc` tests.
 183   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-new-symbol-mangling"
 184 fi
 185
 186 # Print the date from the local machine and the date from an external source to
 187 # check for clock drifts. An HTTP URL is used instead of HTTPS since on Azure
 188 # Pipelines it happened that the certificates were marked as expired.
 189 datecheck() {
 190   # If an error has happened, we do not want to start a new group, because that will collapse
 191   # a previous group that might have contained the error log.
 192   exit_code=$?
 193
 194   if [ $exit_code -eq 0 ]
 195   then
 196     echo "::group::Clock drift check"
 197   fi
 198
 199   echo -n "  local time: "
 200   date
 201   echo -n "  network time: "
 202   curl -fs --head http://ci-caches.rust-lang.org | grep ^Date: \
 203       | sed 's/Date: //g' || true
 204
 205   if [ $exit_code -eq 0 ]
 206   then
 207     echo "::endgroup::"
 208   fi
 209 }
 210 datecheck
 211 trap datecheck EXIT
 212
 213 # We've had problems in the past of shell scripts leaking fds into the sccache
 214 # server (#48192) which causes Cargo to erroneously think that a build script
 215 # hasn't finished yet. Try to solve that problem by starting a very long-lived
 216 # sccache server at the start of the build, but no need to worry if this fails.
 217 SCCACHE_IDLE_TIMEOUT=10800 sccache --start-server || true
 218
 219 # Our build may overwrite config.toml, so we remove it here
 220 rm -f config.toml
 221
 222 $SRC/configure $RUST_CONFIGURE_ARGS
 223
 224 retry make prepare
 225
 226 # Display the CPU and memory information. This helps us know why the CI timing
 227 # is fluctuating.
 228 echo "::group::Display CPU and Memory information"
 229 if isMacOS; then
 230     system_profiler SPHardwareDataType || true
 231     sysctl hw || true
 232     ncpus=$(sysctl -n hw.ncpu)
 233 else
 234     cat /proc/cpuinfo || true
 235     cat /proc/meminfo || true
 236     ncpus=$(grep processor /proc/cpuinfo | wc -l)
 237 fi
 238 echo "::endgroup::"
 239
 240 if [ ! -z "$SCRIPT" ]; then
 241   echo "Executing ${SCRIPT}"
 242   sh -x -c "$SCRIPT"
 243 else
 244   do_make() {
 245     echo "make -j $ncpus $1"
 246     make -j $ncpus $1
 247     local retval=$?
 248     return $retval
 249   }
 250
 251   do_make "$RUST_CHECK_TARGET"
 252 fi
 253
 254 if [ "$RUN_CHECK_WITH_PARALLEL_QUERIES" != "" ]; then
 255   rm -f config.toml
 256   $SRC/configure --set change-id=99999999 --set rust.parallel-compiler
 257
 258   # Save the build metrics before we wipe the directory
 259   if [ "$HAS_METRICS" = 1 ]; then
 260     mv build/metrics.json .
 261   fi
 262   rm -rf build
 263   if [ "$HAS_METRICS" = 1 ]; then
 264     mkdir build
 265     mv metrics.json build
 266   fi
 267
 268   CARGO_INCREMENTAL=0 ../x check
 269 fi
 270
 271 echo "::group::sccache stats"
 272 sccache --show-stats || true
 273 echo "::endgroup::"