]>
Commit | Line | Data |
---|---|---|
7453a54e SL |
1 | # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT |
2 | # file at the top-level directory of this distribution and at | |
3 | # http://rust-lang.org/COPYRIGHT. | |
4 | # | |
5 | # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or | |
6 | # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license | |
7 | # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your | |
8 | # option. This file may not be copied, modified, or distributed | |
9 | # except according to those terms. | |
10 | ||
11 | import argparse | |
12 | import contextlib | |
3157f602 | 13 | import datetime |
a7813a04 | 14 | import hashlib |
7453a54e SL |
15 | import os |
16 | import shutil | |
17 | import subprocess | |
18 | import sys | |
19 | import tarfile | |
a7813a04 XL |
20 | import tempfile |
21 | ||
3157f602 XL |
22 | from time import time |
23 | ||
7453a54e SL |
24 | |
25 | def get(url, path, verbose=False): | |
a7813a04 XL |
26 | sha_url = url + ".sha256" |
27 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: | |
28 | temp_path = temp_file.name | |
29 | with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file: | |
30 | sha_path = sha_file.name | |
31 | ||
32 | try: | |
33 | download(sha_path, sha_url, verbose) | |
5bcae85e SL |
34 | if os.path.exists(path): |
35 | if verify(path, sha_path, False): | |
36 | print("using already-download file " + path) | |
37 | return | |
38 | else: | |
39 | print("ignoring already-download file " + path + " due to failed verification") | |
40 | os.unlink(path) | |
a7813a04 | 41 | download(temp_path, url, verbose) |
5bcae85e SL |
42 | if not verify(temp_path, sha_path, True): |
43 | raise RuntimeError("failed verification") | |
3157f602 | 44 | print("moving {} to {}".format(temp_path, path)) |
a7813a04 XL |
45 | shutil.move(temp_path, path) |
46 | finally: | |
47 | delete_if_present(sha_path) | |
48 | delete_if_present(temp_path) | |
49 | ||
50 | ||
51 | def delete_if_present(path): | |
52 | if os.path.isfile(path): | |
53 | print("removing " + path) | |
54 | os.unlink(path) | |
55 | ||
56 | ||
57 | def download(path, url, verbose): | |
3157f602 | 58 | print("downloading {} to {}".format(url, path)) |
7453a54e SL |
59 | # see http://serverfault.com/questions/301128/how-to-download |
60 | if sys.platform == 'win32': | |
61 | run(["PowerShell.exe", "/nologo", "-Command", | |
a7813a04 XL |
62 | "(New-Object System.Net.WebClient)" |
63 | ".DownloadFile('{}', '{}')".format(url, path)], | |
64 | verbose=verbose) | |
7453a54e SL |
65 | else: |
66 | run(["curl", "-o", path, url], verbose=verbose) | |
67 | ||
a7813a04 XL |
68 | |
69 | def verify(path, sha_path, verbose): | |
70 | print("verifying " + path) | |
71 | with open(path, "rb") as f: | |
72 | found = hashlib.sha256(f.read()).hexdigest() | |
73 | with open(sha_path, "r") as f: | |
74 | expected, _ = f.readline().split() | |
5bcae85e SL |
75 | verified = found == expected |
76 | if not verified and verbose: | |
77 | print("invalid checksum:\n" | |
a7813a04 XL |
78 | " found: {}\n" |
79 | " expected: {}".format(found, expected)) | |
5bcae85e | 80 | return verified |
a7813a04 XL |
81 | |
82 | ||
7453a54e SL |
83 | def unpack(tarball, dst, verbose=False, match=None): |
84 | print("extracting " + tarball) | |
85 | fname = os.path.basename(tarball).replace(".tar.gz", "") | |
86 | with contextlib.closing(tarfile.open(tarball)) as tar: | |
87 | for p in tar.getnames(): | |
88 | if "/" not in p: | |
89 | continue | |
90 | name = p.replace(fname + "/", "", 1) | |
91 | if match is not None and not name.startswith(match): | |
92 | continue | |
93 | name = name[len(match) + 1:] | |
94 | ||
95 | fp = os.path.join(dst, name) | |
96 | if verbose: | |
97 | print(" extracting " + p) | |
98 | tar.extract(p, dst) | |
99 | tp = os.path.join(dst, p) | |
100 | if os.path.isdir(tp) and os.path.exists(fp): | |
101 | continue | |
102 | shutil.move(tp, fp) | |
103 | shutil.rmtree(os.path.join(dst, fname)) | |
104 | ||
105 | def run(args, verbose=False): | |
106 | if verbose: | |
107 | print("running: " + ' '.join(args)) | |
108 | sys.stdout.flush() | |
109 | # Use Popen here instead of call() as it apparently allows powershell on | |
110 | # Windows to not lock up waiting for input presumably. | |
111 | ret = subprocess.Popen(args) | |
112 | code = ret.wait() | |
113 | if code != 0: | |
a7813a04 XL |
114 | err = "failed to run: " + ' '.join(args) |
115 | if verbose: | |
116 | raise RuntimeError(err) | |
117 | sys.exit(err) | |
118 | ||
119 | def stage0_data(rust_root): | |
120 | nightlies = os.path.join(rust_root, "src/stage0.txt") | |
3157f602 | 121 | data = {} |
a7813a04 | 122 | with open(nightlies, 'r') as nightlies: |
3157f602 XL |
123 | for line in nightlies: |
124 | line = line.rstrip() # Strip newline character, '\n' | |
a7813a04 XL |
125 | if line.startswith("#") or line == '': |
126 | continue | |
127 | a, b = line.split(": ", 1) | |
128 | data[a] = b | |
3157f602 XL |
129 | return data |
130 | ||
131 | def format_build_time(duration): | |
132 | return str(datetime.timedelta(seconds=int(duration))) | |
7453a54e | 133 | |
9e0c209e SL |
134 | |
135 | class RustBuild(object): | |
a7813a04 | 136 | def download_stage0(self): |
7453a54e | 137 | cache_dst = os.path.join(self.build_dir, "cache") |
a7813a04 XL |
138 | rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date()) |
139 | cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date()) | |
7453a54e SL |
140 | if not os.path.exists(rustc_cache): |
141 | os.makedirs(rustc_cache) | |
142 | if not os.path.exists(cargo_cache): | |
143 | os.makedirs(cargo_cache) | |
144 | ||
145 | if self.rustc().startswith(self.bin_root()) and \ | |
9e0c209e | 146 | (not os.path.exists(self.rustc()) or self.rustc_out_of_date()): |
54a0048b SL |
147 | if os.path.exists(self.bin_root()): |
148 | shutil.rmtree(self.bin_root()) | |
a7813a04 | 149 | channel = self.stage0_rustc_channel() |
3157f602 | 150 | filename = "rust-std-{}-{}.tar.gz".format(channel, self.build) |
a7813a04 | 151 | url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date() |
7453a54e SL |
152 | tarball = os.path.join(rustc_cache, filename) |
153 | if not os.path.exists(tarball): | |
3157f602 | 154 | get("{}/{}".format(url, filename), tarball, verbose=self.verbose) |
7453a54e SL |
155 | unpack(tarball, self.bin_root(), |
156 | match="rust-std-" + self.build, | |
157 | verbose=self.verbose) | |
158 | ||
3157f602 | 159 | filename = "rustc-{}-{}.tar.gz".format(channel, self.build) |
a7813a04 | 160 | url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date() |
7453a54e SL |
161 | tarball = os.path.join(rustc_cache, filename) |
162 | if not os.path.exists(tarball): | |
3157f602 | 163 | get("{}/{}".format(url, filename), tarball, verbose=self.verbose) |
7453a54e SL |
164 | unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose) |
165 | with open(self.rustc_stamp(), 'w') as f: | |
a7813a04 | 166 | f.write(self.stage0_rustc_date()) |
7453a54e SL |
167 | |
168 | if self.cargo().startswith(self.bin_root()) and \ | |
9e0c209e | 169 | (not os.path.exists(self.cargo()) or self.cargo_out_of_date()): |
a7813a04 | 170 | channel = self.stage0_cargo_channel() |
3157f602 | 171 | filename = "cargo-{}-{}.tar.gz".format(channel, self.build) |
a7813a04 | 172 | url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date() |
7453a54e SL |
173 | tarball = os.path.join(cargo_cache, filename) |
174 | if not os.path.exists(tarball): | |
3157f602 | 175 | get("{}/{}".format(url, filename), tarball, verbose=self.verbose) |
7453a54e SL |
176 | unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose) |
177 | with open(self.cargo_stamp(), 'w') as f: | |
a7813a04 | 178 | f.write(self.stage0_cargo_date()) |
7453a54e | 179 | |
a7813a04 | 180 | def stage0_cargo_date(self): |
7453a54e SL |
181 | return self._cargo_date |
182 | ||
a7813a04 XL |
183 | def stage0_cargo_channel(self): |
184 | return self._cargo_channel | |
185 | ||
186 | def stage0_rustc_date(self): | |
7453a54e SL |
187 | return self._rustc_date |
188 | ||
a7813a04 XL |
189 | def stage0_rustc_channel(self): |
190 | return self._rustc_channel | |
191 | ||
7453a54e SL |
192 | def rustc_stamp(self): |
193 | return os.path.join(self.bin_root(), '.rustc-stamp') | |
194 | ||
195 | def cargo_stamp(self): | |
196 | return os.path.join(self.bin_root(), '.cargo-stamp') | |
197 | ||
198 | def rustc_out_of_date(self): | |
3157f602 | 199 | if not os.path.exists(self.rustc_stamp()) or self.clean: |
7453a54e SL |
200 | return True |
201 | with open(self.rustc_stamp(), 'r') as f: | |
a7813a04 | 202 | return self.stage0_rustc_date() != f.read() |
7453a54e SL |
203 | |
204 | def cargo_out_of_date(self): | |
3157f602 | 205 | if not os.path.exists(self.cargo_stamp()) or self.clean: |
7453a54e SL |
206 | return True |
207 | with open(self.cargo_stamp(), 'r') as f: | |
a7813a04 | 208 | return self.stage0_cargo_date() != f.read() |
7453a54e SL |
209 | |
210 | def bin_root(self): | |
211 | return os.path.join(self.build_dir, self.build, "stage0") | |
212 | ||
213 | def get_toml(self, key): | |
214 | for line in self.config_toml.splitlines(): | |
215 | if line.startswith(key + ' ='): | |
216 | return self.get_string(line) | |
217 | return None | |
218 | ||
219 | def get_mk(self, key): | |
220 | for line in iter(self.config_mk.splitlines()): | |
221 | if line.startswith(key): | |
222 | return line[line.find(':=') + 2:].strip() | |
223 | return None | |
224 | ||
225 | def cargo(self): | |
226 | config = self.get_toml('cargo') | |
227 | if config: | |
228 | return config | |
229 | return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix()) | |
230 | ||
231 | def rustc(self): | |
232 | config = self.get_toml('rustc') | |
233 | if config: | |
234 | return config | |
235 | config = self.get_mk('CFG_LOCAL_RUST') | |
236 | if config: | |
237 | return config + '/bin/rustc' + self.exe_suffix() | |
238 | return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix()) | |
239 | ||
240 | def get_string(self, line): | |
241 | start = line.find('"') | |
9e0c209e SL |
242 | end = start + 1 + line[start + 1:].find('"') |
243 | return line[start + 1:end] | |
7453a54e SL |
244 | |
245 | def exe_suffix(self): | |
246 | if sys.platform == 'win32': | |
247 | return '.exe' | |
248 | else: | |
249 | return '' | |
250 | ||
7453a54e | 251 | def build_bootstrap(self): |
3157f602 XL |
252 | build_dir = os.path.join(self.build_dir, "bootstrap") |
253 | if self.clean and os.path.exists(build_dir): | |
254 | shutil.rmtree(build_dir) | |
7453a54e | 255 | env = os.environ.copy() |
3157f602 | 256 | env["CARGO_TARGET_DIR"] = build_dir |
7453a54e SL |
257 | env["RUSTC"] = self.rustc() |
258 | env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") | |
259 | env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") | |
260 | env["PATH"] = os.path.join(self.bin_root(), "bin") + \ | |
261 | os.pathsep + env["PATH"] | |
262 | self.run([self.cargo(), "build", "--manifest-path", | |
263 | os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")], | |
264 | env) | |
265 | ||
266 | def run(self, args, env): | |
3157f602 | 267 | proc = subprocess.Popen(args, env=env) |
7453a54e SL |
268 | ret = proc.wait() |
269 | if ret != 0: | |
270 | sys.exit(ret) | |
271 | ||
272 | def build_triple(self): | |
9e0c209e | 273 | default_encoding = sys.getdefaultencoding() |
7453a54e SL |
274 | config = self.get_toml('build') |
275 | if config: | |
276 | return config | |
277 | config = self.get_mk('CFG_BUILD') | |
278 | if config: | |
279 | return config | |
280 | try: | |
9e0c209e SL |
281 | ostype = subprocess.check_output(['uname', '-s']).strip().decode(default_encoding) |
282 | cputype = subprocess.check_output(['uname', '-m']).strip().decode(default_encoding) | |
3157f602 | 283 | except (subprocess.CalledProcessError, WindowsError): |
7453a54e SL |
284 | if sys.platform == 'win32': |
285 | return 'x86_64-pc-windows-msvc' | |
3157f602 XL |
286 | err = "uname not found" |
287 | if self.verbose: | |
288 | raise Exception(err) | |
289 | sys.exit(err) | |
7453a54e SL |
290 | |
291 | # Darwin's `uname -s` lies and always returns i386. We have to use | |
292 | # sysctl instead. | |
293 | if ostype == 'Darwin' and cputype == 'i686': | |
9e0c209e SL |
294 | args = ['sysctl', 'hw.optional.x86_64'] |
295 | sysctl = subprocess.check_output(args).decode(default_encoding) | |
3157f602 | 296 | if ': 1' in sysctl: |
7453a54e SL |
297 | cputype = 'x86_64' |
298 | ||
299 | # The goal here is to come up with the same triple as LLVM would, | |
300 | # at least for the subset of platforms we're willing to target. | |
301 | if ostype == 'Linux': | |
302 | ostype = 'unknown-linux-gnu' | |
303 | elif ostype == 'FreeBSD': | |
304 | ostype = 'unknown-freebsd' | |
305 | elif ostype == 'DragonFly': | |
306 | ostype = 'unknown-dragonfly' | |
307 | elif ostype == 'Bitrig': | |
308 | ostype = 'unknown-bitrig' | |
309 | elif ostype == 'OpenBSD': | |
310 | ostype = 'unknown-openbsd' | |
311 | elif ostype == 'NetBSD': | |
312 | ostype = 'unknown-netbsd' | |
313 | elif ostype == 'Darwin': | |
314 | ostype = 'apple-darwin' | |
315 | elif ostype.startswith('MINGW'): | |
316 | # msys' `uname` does not print gcc configuration, but prints msys | |
317 | # configuration. so we cannot believe `uname -m`: | |
318 | # msys1 is always i686 and msys2 is always x86_64. | |
319 | # instead, msys defines $MSYSTEM which is MINGW32 on i686 and | |
320 | # MINGW64 on x86_64. | |
321 | ostype = 'pc-windows-gnu' | |
322 | cputype = 'i686' | |
323 | if os.environ.get('MSYSTEM') == 'MINGW64': | |
324 | cputype = 'x86_64' | |
325 | elif ostype.startswith('MSYS'): | |
326 | ostype = 'pc-windows-gnu' | |
327 | elif ostype.startswith('CYGWIN_NT'): | |
328 | cputype = 'i686' | |
329 | if ostype.endswith('WOW64'): | |
330 | cputype = 'x86_64' | |
331 | ostype = 'pc-windows-gnu' | |
332 | else: | |
a7813a04 XL |
333 | err = "unknown OS type: " + ostype |
334 | if self.verbose: | |
335 | raise ValueError(err) | |
336 | sys.exit(err) | |
7453a54e SL |
337 | |
338 | if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}: | |
339 | cputype = 'i686' | |
340 | elif cputype in {'xscale', 'arm'}: | |
341 | cputype = 'arm' | |
342 | elif cputype == 'armv7l': | |
343 | cputype = 'arm' | |
344 | ostype += 'eabihf' | |
345 | elif cputype == 'aarch64': | |
346 | cputype = 'aarch64' | |
c30ab7b3 SL |
347 | elif cputype == 'mips': |
348 | if sys.byteorder == 'big': | |
349 | cputype = 'mips' | |
350 | elif sys.byteorder == 'little': | |
351 | cputype = 'mipsel' | |
352 | else: | |
353 | raise ValueError('unknown byteorder: ' + sys.byteorder) | |
354 | elif cputype == 'mips64': | |
355 | if sys.byteorder == 'big': | |
356 | cputype = 'mips64' | |
357 | elif sys.byteorder == 'little': | |
358 | cputype = 'mips64el' | |
359 | else: | |
360 | raise ValueError('unknown byteorder: ' + sys.byteorder) | |
361 | # only the n64 ABI is supported, indicate it | |
362 | ostype += 'abi64' | |
7453a54e SL |
363 | elif cputype in {'powerpc', 'ppc', 'ppc64'}: |
364 | cputype = 'powerpc' | |
365 | elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}: | |
366 | cputype = 'x86_64' | |
367 | else: | |
a7813a04 XL |
368 | err = "unknown cpu type: " + cputype |
369 | if self.verbose: | |
370 | raise ValueError(err) | |
371 | sys.exit(err) | |
7453a54e | 372 | |
3157f602 | 373 | return "{}-{}".format(cputype, ostype) |
7453a54e | 374 | |
a7813a04 XL |
375 | def main(): |
376 | parser = argparse.ArgumentParser(description='Build rust') | |
377 | parser.add_argument('--config') | |
3157f602 | 378 | parser.add_argument('--clean', action='store_true') |
a7813a04 XL |
379 | parser.add_argument('-v', '--verbose', action='store_true') |
380 | ||
5bcae85e | 381 | args = [a for a in sys.argv if a != '-h' and a != '--help'] |
a7813a04 XL |
382 | args, _ = parser.parse_known_args(args) |
383 | ||
384 | # Configure initial bootstrap | |
385 | rb = RustBuild() | |
386 | rb.config_toml = '' | |
387 | rb.config_mk = '' | |
388 | rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..')) | |
389 | rb.build_dir = os.path.join(os.getcwd(), "build") | |
390 | rb.verbose = args.verbose | |
3157f602 | 391 | rb.clean = args.clean |
a7813a04 XL |
392 | |
393 | try: | |
394 | with open(args.config or 'config.toml') as config: | |
395 | rb.config_toml = config.read() | |
396 | except: | |
397 | pass | |
398 | try: | |
399 | rb.config_mk = open('config.mk').read() | |
400 | except: | |
401 | pass | |
402 | ||
403 | data = stage0_data(rb.rust_root) | |
404 | rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1) | |
405 | rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1) | |
406 | ||
3157f602 XL |
407 | start_time = time() |
408 | ||
a7813a04 XL |
409 | # Fetch/build the bootstrap |
410 | rb.build = rb.build_triple() | |
411 | rb.download_stage0() | |
412 | sys.stdout.flush() | |
413 | rb.build_bootstrap() | |
414 | sys.stdout.flush() | |
415 | ||
416 | # Run the bootstrap | |
417 | args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")] | |
a7813a04 XL |
418 | args.extend(sys.argv[1:]) |
419 | env = os.environ.copy() | |
c30ab7b3 SL |
420 | env["BUILD"] = rb.build |
421 | env["SRC"] = rb.rust_root | |
a7813a04 XL |
422 | env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) |
423 | rb.run(args, env) | |
424 | ||
3157f602 XL |
425 | end_time = time() |
426 | ||
427 | print("Build completed in %s" % format_build_time(end_time - start_time)) | |
428 | ||
a7813a04 XL |
429 | if __name__ == '__main__': |
430 | main() |