src/ci/azure-pipelines/steps/run.yml

   1 # FIXME(linux): need to configure core dumps, enable them, and then dump
   2 # backtraces on failure from all core dumps:
   3 #
   4 # - bash: sudo apt install gdb
   5 # - bash: sudo sh -c 'echo "/checkout/obj/cores/core.%p.%E" > /proc/sys/kernel/core_pattern'
   6 #
   7 # Check travis config for `gdb --batch` command to print all crash logs
   8
   9 steps:
  10
  11 # Disable automatic line ending conversion, which is enabled by default on
  12 # Azure's Windows image. Having the conversion enabled caused regressions both
  13 # in our test suite (it broke miri tests) and in the ecosystem, since we
  14 # started shipping install scripts with CRLF endings instead of the old LF.
  15 #
  16 # Note that we do this a couple times during the build as the PATH and current
  17 # user/directory change, e.g. when mingw is enabled.
  18 - bash: git config --global core.autocrlf false
  19   displayName: "Disable git automatic line ending conversion"
  20
  21 - checkout: self
  22   fetchDepth: 2
  23
  24 # Spawn a background process to collect CPU usage statistics which we'll upload
  25 # at the end of the build. See the comments in the script here for more
  26 # information.
  27 - bash: python src/ci/cpu-usage-over-time.py &> cpu-usage.csv &
  28   displayName: "Collect CPU-usage statistics in the background"
  29
  30 - bash: printenv | sort
  31   displayName: Show environment variables
  32
  33 # Log the date, even on failure. Attempting to debug SSL certificate errors.
  34 - bash: date
  35   displayName: Print out date (before build)
  36
  37 - bash: |
  38     set -e
  39     df -h
  40     du . | sort -nr | head -n100
  41   displayName: Show disk usage
  42   # FIXME: this hasn't been tested, but maybe it works on Windows? Should test!
  43   condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))
  44
  45 - template: install-sccache.yml
  46 - template: install-clang.yml
  47
  48 # Install some dependencies needed to build LLDB/Clang, currently only needed
  49 # during the `dist` target
  50 - bash: |
  51     set -e
  52     brew update
  53     brew install xz
  54     brew install swig@3
  55     brew link --force swig@3
  56   displayName: Install build dependencies (OSX)
  57   condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'), eq(variables['SCRIPT'],'./x.py dist'))
  58
  59 # Switch to XCode 9.3 on OSX since it seems to be the last version that supports
  60 # i686-apple-darwin. We'll eventually want to upgrade this and it will probably
  61 # force us to drop i686-apple-darwin, but let's keep the wheels turning for now.
  62 - bash: |
  63     set -e
  64     sudo xcode-select --switch /Applications/Xcode_9.3.app
  65   displayName: Switch to Xcode 9.3 (OSX)
  66   condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
  67
  68 - template: install-windows-build-deps.yml
  69
  70 # Looks like docker containers have IPv6 disabled by default, so let's turn it
  71 # on since libstd tests require it
  72 - bash: |
  73     set -e
  74     sudo mkdir -p /etc/docker
  75     echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' | sudo tee /etc/docker/daemon.json
  76     sudo service docker restart
  77   displayName: Enable IPv6
  78   condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
  79
  80 # Disable automatic line ending conversion (again). On Windows, when we're
  81 # installing dependencies, something switches the git configuration directory or
  82 # re-enables autocrlf. We've not tracked down the exact cause -- and there may
  83 # be multiple -- but this should ensure submodules are checked out with the
  84 # appropriate line endings.
  85 - bash: git config --replace-all --global core.autocrlf false
  86   displayName: "Disable git automatic line ending conversion"
  87
  88 # Check out all our submodules, but more quickly than using git by using one of
  89 # our custom scripts
  90 - bash: |
  91     set -e
  92     mkdir -p $HOME/rustsrc
  93     $BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc
  94   condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT'))
  95   displayName: Check out submodules (Unix)
  96 - script: |
  97     if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc
  98     sh src/ci/init_repo.sh . /d/cache/rustsrc
  99   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
 100   displayName: Check out submodules (Windows)
 101
 102 # See also the disable for autocrlf above, this just checks that it worked
 103 #
 104 # We check both in rust-lang/rust and in a submodule to make sure both are
 105 # accurate. Submodules are checked out significantly later than the main
 106 # repository in this script, so settings can (and do!) change between then.
 107 #
 108 # Linux (and maybe macOS) builders don't currently have dos2unix so just only
 109 # run this step on Windows.
 110 - bash: |
 111     set -x
 112     # print out the git configuration so we can better investigate failures in
 113     # the following
 114     git config --list --show-origin
 115     dos2unix -ih Cargo.lock src/tools/rust-installer/install-template.sh
 116     endings=$(dos2unix -ic Cargo.lock src/tools/rust-installer/install-template.sh)
 117     # if endings has non-zero length, error out
 118     if [ -n "$endings" ]; then exit 1 ; fi
 119   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
 120   displayName: Verify line endings are LF
 121
 122 # Ensure the `aws` CLI is installed so we can deploy later on, cache docker
 123 # images, etc.
 124 - bash: |
 125     set -e
 126     source src/ci/shared.sh
 127     sudo apt-get install -y python3-setuptools
 128     retry pip3 install -r src/ci/awscli-requirements.txt --upgrade --user
 129     echo "##vso[task.prependpath]$HOME/.local/bin"
 130   displayName: Install awscli (Linux)
 131   condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
 132 - script: pip install -r src/ci/awscli-requirements.txt
 133   displayName: Install awscli (non-Linux)
 134   condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux'))
 135
 136 # Configure our CI_JOB_NAME variable which log analyzers can use for the main
 137 # step to see what's going on.
 138 - bash: echo "##vso[task.setvariable variable=CI_JOB_NAME]$SYSTEM_JOBNAME"
 139   displayName: Configure Job Name
 140
 141 # As a quick smoke check on the otherwise very fast mingw-check linux builder
 142 # check our own internal scripts.
 143 - bash: |
 144     set -e
 145     git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git
 146     cd rust-toolstate
 147     python2.7 "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" ""
 148     cd ..
 149     rm -rf rust-toolstate
 150   condition: and(succeeded(), eq(variables['IMAGE'], 'mingw-check'))
 151   displayName: Verify the publish_toolstate script works
 152
 153 - bash: |
 154     set -e
 155     # Remove any preexisting rustup installation since it can interfere
 156     # with the cargotest step and its auto-detection of things like Clippy in
 157     # the environment
 158     rustup self uninstall -y || true
 159     if [ "$IMAGE" = "" ]; then
 160       src/ci/run.sh
 161     else
 162       src/ci/docker/run.sh $IMAGE
 163     fi
 164   #timeoutInMinutes: 180
 165   timeoutInMinutes: 600
 166   env:
 167     CI: true
 168     SRC: .
 169     AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
 170     TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN)
 171   displayName: Run build
 172
 173 # If we're a deploy builder, use the `aws` command to publish everything to our
 174 # bucket.
 175 - bash: |
 176     set -e
 177     source src/ci/shared.sh
 178     if [ "$AGENT_OS" = "Linux" ]; then
 179         rm -rf obj/build/dist/doc
 180         upload_dir=obj/build/dist
 181     else
 182         rm -rf build/dist/doc
 183         upload_dir=build/dist
 184     fi
 185     ls -la $upload_dir
 186     deploy_dir=rustc-builds
 187     if [ "$DEPLOY_ALT" == "1" ]; then
 188         deploy_dir=rustc-builds-alt
 189     fi
 190     retry aws s3 cp --no-progress --recursive --acl public-read ./$upload_dir s3://$DEPLOY_BUCKET/$deploy_dir/$BUILD_SOURCEVERSION
 191   env:
 192     AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
 193   condition: and(succeeded(), or(eq(variables.DEPLOY, '1'), eq(variables.DEPLOY_ALT, '1')))
 194   displayName: Upload artifacts
 195
 196 # Upload CPU usage statistics that we've been gathering this whole time. Always
 197 # execute this step in case we want to inspect failed builds, but don't let
 198 # errors here ever fail the build since this is just informational.
 199 - bash: aws s3 cp --acl public-read cpu-usage.csv s3://$DEPLOY_BUCKET/rustc-builds/$BUILD_SOURCEVERSION/cpu-$SYSTEM_JOBNAME.csv
 200   env:
 201     AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
 202   condition: variables['AWS_SECRET_ACCESS_KEY']
 203   continueOnError: true
 204   displayName: Upload CPU usage statistics
 205
 206 # Log the date, even on failure. Attempting to debug SSL certificate errors.
 207 - bash: date
 208   continueOnError: true
 209   displayName: Print out date (after build)