]>
Commit | Line | Data |
---|---|---|
abe05a73 | 1 | #!/usr/bin/env bash |
476ff2be SL |
2 | |
3 | set -e | |
4 | ||
abe05a73 XL |
5 | export MSYS_NO_PATHCONV=1 |
6 | ||
476ff2be | 7 | script=`cd $(dirname $0) && pwd`/`basename $0` |
476ff2be | 8 | |
f035d41b XL |
9 | image="" |
10 | dev=0 | |
11 | ||
12 | while [[ $# -gt 0 ]] | |
13 | do | |
14 | case "$1" in | |
15 | --dev) | |
16 | dev=1 | |
17 | ;; | |
18 | *) | |
19 | if [ -n "$image" ] | |
20 | then | |
21 | echo "expected single argument for the image name" | |
22 | exit 1 | |
23 | fi | |
24 | image="$1" | |
25 | ;; | |
26 | esac | |
27 | shift | |
28 | done | |
29 | ||
30 | script_dir="`dirname $script`" | |
31 | docker_dir="${script_dir}/host-$(uname -m)" | |
32 | ci_dir="`dirname $script_dir`" | |
476ff2be SL |
33 | src_dir="`dirname $ci_dir`" |
34 | root_dir="`dirname $src_dir`" | |
35 | ||
532ac7d7 XL |
36 | objdir=$root_dir/obj |
37 | dist=$objdir/build/dist | |
38 | ||
8bb4bdeb XL |
39 | source "$ci_dir/shared.sh" |
40 | ||
f9f354fc XL |
41 | CACHE_DOMAIN="${CACHE_DOMAIN:-ci-caches.rust-lang.org}" |
42 | ||
7cac9316 | 43 | if [ -f "$docker_dir/$image/Dockerfile" ]; then |
0531ce1d | 44 | if [ "$CI" != "" ]; then |
0731742a | 45 | hash_key=/tmp/.docker-hash-key.txt |
9fa01778 XL |
46 | rm -f "${hash_key}" |
47 | echo $image >> $hash_key | |
532ac7d7 XL |
48 | |
49 | cat "$docker_dir/$image/Dockerfile" >> $hash_key | |
50 | # Look for all source files involves in the COPY command | |
51 | copied_files=/tmp/.docker-copied-files.txt | |
52 | rm -f "$copied_files" | |
c295e0f8 XL |
53 | for i in $(sed -n -e '/^COPY --from=/! s/^COPY \(.*\) .*$/\1/p' \ |
54 | "$docker_dir/$image/Dockerfile"); do | |
532ac7d7 | 55 | # List the file names |
f035d41b | 56 | find "$script_dir/$i" -type f >> $copied_files |
532ac7d7 XL |
57 | done |
58 | # Sort the file names and cat the content into the hash key | |
59 | sort $copied_files | xargs cat >> $hash_key | |
60 | ||
f035d41b XL |
61 | # Include the architecture in the hash key, since our Linux CI does not |
62 | # only run in x86_64 machines. | |
63 | uname -m >> $hash_key | |
64 | ||
0731742a XL |
65 | docker --version >> $hash_key |
66 | cksum=$(sha512sum $hash_key | \ | |
0531ce1d | 67 | awk '{print $1}') |
dc9dc135 | 68 | |
f9f354fc | 69 | url="https://$CACHE_DOMAIN/docker/$cksum" |
dc9dc135 XL |
70 | |
71 | echo "Attempting to download $url" | |
94b46f34 | 72 | rm -f /tmp/rustci_docker_cache |
0531ce1d | 73 | set +e |
3c0e092e XL |
74 | retry curl --max-time 600 -y 30 -Y 10 --connect-timeout 30 -f -L -C - \ |
75 | -o /tmp/rustci_docker_cache "$url" | |
76 | echo "Loading images into docker" | |
77 | # docker load sometimes hangs in the CI, so time out after 10 minutes with TERM, | |
78 | # KILL after 12 minutes | |
79 | loaded_images=$(/usr/bin/timeout -k 720 600 docker load -i /tmp/rustci_docker_cache \ | |
80 | | sed 's/.* sha/sha/') | |
0531ce1d XL |
81 | set -e |
82 | echo "Downloaded containers:\n$loaded_images" | |
83 | fi | |
84 | ||
abe05a73 XL |
85 | dockerfile="$docker_dir/$image/Dockerfile" |
86 | if [ -x /usr/bin/cygpath ]; then | |
f035d41b | 87 | context="`cygpath -w $script_dir`" |
abe05a73 XL |
88 | dockerfile="`cygpath -w $dockerfile`" |
89 | else | |
f035d41b | 90 | context="$script_dir" |
abe05a73 | 91 | fi |
7cac9316 XL |
92 | retry docker \ |
93 | build \ | |
94 | --rm \ | |
95 | -t rust-ci \ | |
abe05a73 XL |
96 | -f "$dockerfile" \ |
97 | "$context" | |
0531ce1d | 98 | |
f9f354fc XL |
99 | if [ "$CI" != "" ]; then |
100 | s3url="s3://$SCCACHE_BUCKET/docker/$cksum" | |
101 | upload="aws s3 cp - $s3url" | |
0531ce1d XL |
102 | digest=$(docker inspect rust-ci --format '{{.Id}}') |
103 | echo "Built container $digest" | |
104 | if ! grep -q "$digest" <(echo "$loaded_images"); then | |
dc9dc135 | 105 | echo "Uploading finished image to $url" |
0531ce1d XL |
106 | set +e |
107 | docker history -q rust-ci | \ | |
108 | grep -v missing | \ | |
109 | xargs docker save | \ | |
110 | gzip | \ | |
dc9dc135 | 111 | $upload |
0531ce1d XL |
112 | set -e |
113 | else | |
114 | echo "Looks like docker image is the same as before, not uploading" | |
115 | fi | |
532ac7d7 XL |
116 | # Record the container image for reuse, e.g. by rustup.rs builds |
117 | info="$dist/image-$image.txt" | |
118 | mkdir -p "$dist" | |
119 | echo "$url" >"$info" | |
120 | echo "$digest" >>"$info" | |
0531ce1d | 121 | fi |
7cac9316 | 122 | elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then |
dc9dc135 XL |
123 | if isCI; then |
124 | echo Cannot run disabled images on CI! | |
7cac9316 XL |
125 | exit 1 |
126 | fi | |
ea8adc8c | 127 | # Transform changes the context of disabled Dockerfiles to match the enabled ones |
f035d41b | 128 | tar --transform 's#disabled/#./#' -C $script_dir -c . | docker \ |
7cac9316 XL |
129 | build \ |
130 | --rm \ | |
131 | -t rust-ci \ | |
f035d41b | 132 | -f "host-$(uname -m)/$image/Dockerfile" \ |
ea8adc8c | 133 | - |
7cac9316 XL |
134 | else |
135 | echo Invalid image: $image | |
f035d41b XL |
136 | |
137 | # Check whether the image exists for other architectures | |
138 | for arch_dir in "${script_dir}"/host-*; do | |
139 | # Avoid checking non-directories and the current host architecture directory | |
140 | if ! [[ -d "${arch_dir}" ]]; then | |
141 | continue | |
142 | fi | |
143 | if [[ "${arch_dir}" = "${docker_dir}" ]]; then | |
144 | continue | |
145 | fi | |
146 | ||
147 | arch_name="$(basename "${arch_dir}" | sed 's/^host-//')" | |
148 | if [[ -f "${arch_dir}/${image}/Dockerfile" ]]; then | |
149 | echo "Note: the image exists for the ${arch_name} host architecture" | |
150 | elif [[ -f "${arch_dir}/disabled/${image}/Dockerfile" ]]; then | |
151 | echo "Note: the disabled image exists for the ${arch_name} host architecture" | |
152 | else | |
153 | continue | |
154 | fi | |
155 | echo "Note: the current host architecture is $(uname -m)" | |
156 | done | |
157 | ||
7cac9316 XL |
158 | exit 1 |
159 | fi | |
160 | ||
476ff2be | 161 | mkdir -p $HOME/.cargo |
8bb4bdeb | 162 | mkdir -p $objdir/tmp |
8faf50e0 | 163 | mkdir -p $objdir/cores |
e74abb32 | 164 | mkdir -p /tmp/toolstate |
32a655c1 SL |
165 | |
166 | args= | |
167 | if [ "$SCCACHE_BUCKET" != "" ]; then | |
3b2f2976 XL |
168 | args="$args --env SCCACHE_BUCKET" |
169 | args="$args --env SCCACHE_REGION" | |
170 | args="$args --env AWS_ACCESS_KEY_ID" | |
171 | args="$args --env AWS_SECRET_ACCESS_KEY" | |
32a655c1 SL |
172 | else |
173 | mkdir -p $HOME/.cache/sccache | |
174 | args="$args --env SCCACHE_DIR=/sccache --volume $HOME/.cache/sccache:/sccache" | |
175 | fi | |
476ff2be | 176 | |
3b2f2976 XL |
177 | # Run containers as privileged as it should give them access to some more |
178 | # syscalls such as ptrace and whatnot. In the upgrade to LLVM 5.0 it was | |
179 | # discovered that the leak sanitizer apparently needs these syscalls nowadays so | |
180 | # we'll need `--privileged` for at least the `x86_64-gnu` builder, so this just | |
181 | # goes ahead and sets it for all builders. | |
182 | args="$args --privileged" | |
183 | ||
416331ca XL |
184 | # Things get a little weird if this script is already running in a docker |
185 | # container. If we're already in a docker container then we assume it's set up | |
186 | # to do docker-in-docker where we have access to a working `docker` command. | |
187 | # | |
188 | # If this is the case (we check via the presence of `/.dockerenv`) | |
189 | # then we can't actually use the `--volume` argument. Typically we use | |
190 | # `--volume` to efficiently share the build and source directory between this | |
191 | # script and the container we're about to spawn. If we're inside docker already | |
192 | # though the `--volume` argument maps the *host's* folder to the container we're | |
193 | # about to spawn, when in fact we want the folder in this container itself. To | |
194 | # work around this we use a recipe cribbed from | |
195 | # https://circleci.com/docs/2.0/building-docker-images/#mounting-folders to | |
196 | # create a temporary container with a volume. We then copy the entire source | |
197 | # directory into this container, and then use that copy in the container we're | |
198 | # about to spawn. Finally after the build finishes we re-extract the object | |
199 | # directory. | |
200 | # | |
201 | # Note that none of this is necessary if we're *not* in a docker-in-docker | |
202 | # scenario. If this script is run on a bare metal host then we share a bunch of | |
203 | # data directories to share as much data as possible. Note that we also use | |
204 | # `LOCAL_USER_ID` (recognized in `src/ci/run.sh`) to ensure that files are all | |
205 | # read/written as the same user as the bare-metal user. | |
206 | if [ -f /.dockerenv ]; then | |
207 | docker create -v /checkout --name checkout alpine:3.4 /bin/true | |
208 | docker cp . checkout:/checkout | |
209 | args="$args --volumes-from checkout" | |
210 | else | |
211 | args="$args --volume $root_dir:/checkout:ro" | |
212 | args="$args --volume $objdir:/checkout/obj" | |
213 | args="$args --volume $HOME/.cargo:/cargo" | |
214 | args="$args --volume $HOME/rustsrc:$HOME/rustsrc" | |
e74abb32 | 215 | args="$args --volume /tmp/toolstate:/tmp/toolstate" |
416331ca XL |
216 | args="$args --env LOCAL_USER_ID=`id -u`" |
217 | fi | |
218 | ||
f035d41b XL |
219 | if [ "$dev" = "1" ] |
220 | then | |
221 | # Interactive + TTY | |
222 | args="$args -it" | |
223 | command="/bin/bash" | |
224 | else | |
225 | command="/checkout/src/ci/run.sh" | |
226 | fi | |
227 | ||
94222f64 XL |
228 | if [ "$CI" != "" ]; then |
229 | # Get some needed information for $BASE_COMMIT | |
230 | # | |
231 | # This command gets the last merge commit which we'll use as base to list | |
232 | # deleted files since then. | |
233 | BASE_COMMIT="$(git log --author=bors@rust-lang.org -n 2 --pretty=format:%H | tail -n 1)" | |
234 | else | |
235 | BASE_COMMIT="" | |
236 | fi | |
237 | ||
416331ca | 238 | docker \ |
476ff2be | 239 | run \ |
476ff2be SL |
240 | --workdir /checkout/obj \ |
241 | --env SRC=/checkout \ | |
32a655c1 | 242 | $args \ |
476ff2be | 243 | --env CARGO_HOME=/cargo \ |
3b2f2976 XL |
244 | --env DEPLOY \ |
245 | --env DEPLOY_ALT \ | |
dc9dc135 | 246 | --env CI \ |
dc9dc135 XL |
247 | --env TF_BUILD \ |
248 | --env BUILD_SOURCEBRANCHNAME \ | |
60c5eb7d XL |
249 | --env GITHUB_ACTIONS \ |
250 | --env GITHUB_REF \ | |
ff7c6d11 | 251 | --env TOOLSTATE_REPO_ACCESS_TOKEN \ |
dc9dc135 | 252 | --env TOOLSTATE_REPO \ |
416331ca | 253 | --env TOOLSTATE_PUBLISH \ |
17df50a5 | 254 | --env RUST_CI_OVERRIDE_RELEASE_CHANNEL \ |
83c7162d | 255 | --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \ |
94222f64 | 256 | --env BASE_COMMIT="$BASE_COMMIT" \ |
3b2f2976 | 257 | --init \ |
32a655c1 | 258 | --rm \ |
476ff2be | 259 | rust-ci \ |
f035d41b | 260 | $command |
416331ca XL |
261 | |
262 | if [ -f /.dockerenv ]; then | |
263 | rm -rf $objdir | |
264 | docker cp checkout:/checkout/obj $objdir | |
265 | fi |