]>
Commit | Line | Data |
---|---|---|
7453a54e SL |
1 | # Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT |
2 | # file at the top-level directory of this distribution and at | |
3 | # http://rust-lang.org/COPYRIGHT. | |
4 | # | |
5 | # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or | |
6 | # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license | |
7 | # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your | |
8 | # option. This file may not be copied, modified, or distributed | |
9 | # except according to those terms. | |
10 | ||
11 | import argparse | |
12 | import contextlib | |
3157f602 | 13 | import datetime |
a7813a04 | 14 | import hashlib |
7453a54e SL |
15 | import os |
16 | import shutil | |
17 | import subprocess | |
18 | import sys | |
19 | import tarfile | |
a7813a04 XL |
20 | import tempfile |
21 | ||
3157f602 XL |
22 | from time import time |
23 | ||
7453a54e SL |
24 | |
25 | def get(url, path, verbose=False): | |
a7813a04 XL |
26 | sha_url = url + ".sha256" |
27 | with tempfile.NamedTemporaryFile(delete=False) as temp_file: | |
28 | temp_path = temp_file.name | |
29 | with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file: | |
30 | sha_path = sha_file.name | |
31 | ||
32 | try: | |
33 | download(sha_path, sha_url, verbose) | |
34 | download(temp_path, url, verbose) | |
35 | verify(temp_path, sha_path, verbose) | |
3157f602 | 36 | print("moving {} to {}".format(temp_path, path)) |
a7813a04 XL |
37 | shutil.move(temp_path, path) |
38 | finally: | |
39 | delete_if_present(sha_path) | |
40 | delete_if_present(temp_path) | |
41 | ||
42 | ||
43 | def delete_if_present(path): | |
44 | if os.path.isfile(path): | |
45 | print("removing " + path) | |
46 | os.unlink(path) | |
47 | ||
48 | ||
49 | def download(path, url, verbose): | |
3157f602 | 50 | print("downloading {} to {}".format(url, path)) |
7453a54e SL |
51 | # see http://serverfault.com/questions/301128/how-to-download |
52 | if sys.platform == 'win32': | |
53 | run(["PowerShell.exe", "/nologo", "-Command", | |
a7813a04 XL |
54 | "(New-Object System.Net.WebClient)" |
55 | ".DownloadFile('{}', '{}')".format(url, path)], | |
56 | verbose=verbose) | |
7453a54e SL |
57 | else: |
58 | run(["curl", "-o", path, url], verbose=verbose) | |
59 | ||
a7813a04 XL |
60 | |
61 | def verify(path, sha_path, verbose): | |
62 | print("verifying " + path) | |
63 | with open(path, "rb") as f: | |
64 | found = hashlib.sha256(f.read()).hexdigest() | |
65 | with open(sha_path, "r") as f: | |
66 | expected, _ = f.readline().split() | |
67 | if found != expected: | |
68 | err = ("invalid checksum:\n" | |
69 | " found: {}\n" | |
70 | " expected: {}".format(found, expected)) | |
71 | if verbose: | |
72 | raise RuntimeError(err) | |
73 | sys.exit(err) | |
74 | ||
75 | ||
7453a54e SL |
76 | def unpack(tarball, dst, verbose=False, match=None): |
77 | print("extracting " + tarball) | |
78 | fname = os.path.basename(tarball).replace(".tar.gz", "") | |
79 | with contextlib.closing(tarfile.open(tarball)) as tar: | |
80 | for p in tar.getnames(): | |
81 | if "/" not in p: | |
82 | continue | |
83 | name = p.replace(fname + "/", "", 1) | |
84 | if match is not None and not name.startswith(match): | |
85 | continue | |
86 | name = name[len(match) + 1:] | |
87 | ||
88 | fp = os.path.join(dst, name) | |
89 | if verbose: | |
90 | print(" extracting " + p) | |
91 | tar.extract(p, dst) | |
92 | tp = os.path.join(dst, p) | |
93 | if os.path.isdir(tp) and os.path.exists(fp): | |
94 | continue | |
95 | shutil.move(tp, fp) | |
96 | shutil.rmtree(os.path.join(dst, fname)) | |
97 | ||
98 | def run(args, verbose=False): | |
99 | if verbose: | |
100 | print("running: " + ' '.join(args)) | |
101 | sys.stdout.flush() | |
102 | # Use Popen here instead of call() as it apparently allows powershell on | |
103 | # Windows to not lock up waiting for input presumably. | |
104 | ret = subprocess.Popen(args) | |
105 | code = ret.wait() | |
106 | if code != 0: | |
a7813a04 XL |
107 | err = "failed to run: " + ' '.join(args) |
108 | if verbose: | |
109 | raise RuntimeError(err) | |
110 | sys.exit(err) | |
111 | ||
112 | def stage0_data(rust_root): | |
113 | nightlies = os.path.join(rust_root, "src/stage0.txt") | |
3157f602 | 114 | data = {} |
a7813a04 | 115 | with open(nightlies, 'r') as nightlies: |
3157f602 XL |
116 | for line in nightlies: |
117 | line = line.rstrip() # Strip newline character, '\n' | |
a7813a04 XL |
118 | if line.startswith("#") or line == '': |
119 | continue | |
120 | a, b = line.split(": ", 1) | |
121 | data[a] = b | |
3157f602 XL |
122 | return data |
123 | ||
124 | def format_build_time(duration): | |
125 | return str(datetime.timedelta(seconds=int(duration))) | |
7453a54e SL |
126 | |
127 | class RustBuild: | |
a7813a04 | 128 | def download_stage0(self): |
7453a54e | 129 | cache_dst = os.path.join(self.build_dir, "cache") |
a7813a04 XL |
130 | rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date()) |
131 | cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date()) | |
7453a54e SL |
132 | if not os.path.exists(rustc_cache): |
133 | os.makedirs(rustc_cache) | |
134 | if not os.path.exists(cargo_cache): | |
135 | os.makedirs(cargo_cache) | |
136 | ||
137 | if self.rustc().startswith(self.bin_root()) and \ | |
138 | (not os.path.exists(self.rustc()) or self.rustc_out_of_date()): | |
54a0048b SL |
139 | if os.path.exists(self.bin_root()): |
140 | shutil.rmtree(self.bin_root()) | |
a7813a04 | 141 | channel = self.stage0_rustc_channel() |
3157f602 | 142 | filename = "rust-std-{}-{}.tar.gz".format(channel, self.build) |
a7813a04 | 143 | url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date() |
7453a54e SL |
144 | tarball = os.path.join(rustc_cache, filename) |
145 | if not os.path.exists(tarball): | |
3157f602 | 146 | get("{}/{}".format(url, filename), tarball, verbose=self.verbose) |
7453a54e SL |
147 | unpack(tarball, self.bin_root(), |
148 | match="rust-std-" + self.build, | |
149 | verbose=self.verbose) | |
150 | ||
3157f602 | 151 | filename = "rustc-{}-{}.tar.gz".format(channel, self.build) |
a7813a04 | 152 | url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date() |
7453a54e SL |
153 | tarball = os.path.join(rustc_cache, filename) |
154 | if not os.path.exists(tarball): | |
3157f602 | 155 | get("{}/{}".format(url, filename), tarball, verbose=self.verbose) |
7453a54e SL |
156 | unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose) |
157 | with open(self.rustc_stamp(), 'w') as f: | |
a7813a04 | 158 | f.write(self.stage0_rustc_date()) |
7453a54e SL |
159 | |
160 | if self.cargo().startswith(self.bin_root()) and \ | |
161 | (not os.path.exists(self.cargo()) or self.cargo_out_of_date()): | |
a7813a04 | 162 | channel = self.stage0_cargo_channel() |
3157f602 | 163 | filename = "cargo-{}-{}.tar.gz".format(channel, self.build) |
a7813a04 | 164 | url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date() |
7453a54e SL |
165 | tarball = os.path.join(cargo_cache, filename) |
166 | if not os.path.exists(tarball): | |
3157f602 | 167 | get("{}/{}".format(url, filename), tarball, verbose=self.verbose) |
7453a54e SL |
168 | unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose) |
169 | with open(self.cargo_stamp(), 'w') as f: | |
a7813a04 | 170 | f.write(self.stage0_cargo_date()) |
7453a54e | 171 | |
a7813a04 | 172 | def stage0_cargo_date(self): |
7453a54e SL |
173 | return self._cargo_date |
174 | ||
a7813a04 XL |
175 | def stage0_cargo_channel(self): |
176 | return self._cargo_channel | |
177 | ||
178 | def stage0_rustc_date(self): | |
7453a54e SL |
179 | return self._rustc_date |
180 | ||
a7813a04 XL |
181 | def stage0_rustc_channel(self): |
182 | return self._rustc_channel | |
183 | ||
7453a54e SL |
184 | def rustc_stamp(self): |
185 | return os.path.join(self.bin_root(), '.rustc-stamp') | |
186 | ||
187 | def cargo_stamp(self): | |
188 | return os.path.join(self.bin_root(), '.cargo-stamp') | |
189 | ||
190 | def rustc_out_of_date(self): | |
3157f602 | 191 | if not os.path.exists(self.rustc_stamp()) or self.clean: |
7453a54e SL |
192 | return True |
193 | with open(self.rustc_stamp(), 'r') as f: | |
a7813a04 | 194 | return self.stage0_rustc_date() != f.read() |
7453a54e SL |
195 | |
196 | def cargo_out_of_date(self): | |
3157f602 | 197 | if not os.path.exists(self.cargo_stamp()) or self.clean: |
7453a54e SL |
198 | return True |
199 | with open(self.cargo_stamp(), 'r') as f: | |
a7813a04 | 200 | return self.stage0_cargo_date() != f.read() |
7453a54e SL |
201 | |
202 | def bin_root(self): | |
203 | return os.path.join(self.build_dir, self.build, "stage0") | |
204 | ||
205 | def get_toml(self, key): | |
206 | for line in self.config_toml.splitlines(): | |
207 | if line.startswith(key + ' ='): | |
208 | return self.get_string(line) | |
209 | return None | |
210 | ||
211 | def get_mk(self, key): | |
212 | for line in iter(self.config_mk.splitlines()): | |
213 | if line.startswith(key): | |
214 | return line[line.find(':=') + 2:].strip() | |
215 | return None | |
216 | ||
217 | def cargo(self): | |
218 | config = self.get_toml('cargo') | |
219 | if config: | |
220 | return config | |
221 | return os.path.join(self.bin_root(), "bin/cargo" + self.exe_suffix()) | |
222 | ||
223 | def rustc(self): | |
224 | config = self.get_toml('rustc') | |
225 | if config: | |
226 | return config | |
227 | config = self.get_mk('CFG_LOCAL_RUST') | |
228 | if config: | |
229 | return config + '/bin/rustc' + self.exe_suffix() | |
230 | return os.path.join(self.bin_root(), "bin/rustc" + self.exe_suffix()) | |
231 | ||
232 | def get_string(self, line): | |
233 | start = line.find('"') | |
234 | end = start + 1 + line[start+1:].find('"') | |
235 | return line[start+1:end] | |
236 | ||
237 | def exe_suffix(self): | |
238 | if sys.platform == 'win32': | |
239 | return '.exe' | |
240 | else: | |
241 | return '' | |
242 | ||
7453a54e | 243 | def build_bootstrap(self): |
3157f602 XL |
244 | build_dir = os.path.join(self.build_dir, "bootstrap") |
245 | if self.clean and os.path.exists(build_dir): | |
246 | shutil.rmtree(build_dir) | |
7453a54e | 247 | env = os.environ.copy() |
3157f602 | 248 | env["CARGO_TARGET_DIR"] = build_dir |
7453a54e SL |
249 | env["RUSTC"] = self.rustc() |
250 | env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") | |
251 | env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") | |
252 | env["PATH"] = os.path.join(self.bin_root(), "bin") + \ | |
253 | os.pathsep + env["PATH"] | |
254 | self.run([self.cargo(), "build", "--manifest-path", | |
255 | os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")], | |
256 | env) | |
257 | ||
258 | def run(self, args, env): | |
3157f602 | 259 | proc = subprocess.Popen(args, env=env) |
7453a54e SL |
260 | ret = proc.wait() |
261 | if ret != 0: | |
262 | sys.exit(ret) | |
263 | ||
264 | def build_triple(self): | |
265 | config = self.get_toml('build') | |
266 | if config: | |
267 | return config | |
268 | config = self.get_mk('CFG_BUILD') | |
269 | if config: | |
270 | return config | |
271 | try: | |
272 | ostype = subprocess.check_output(['uname', '-s']).strip() | |
273 | cputype = subprocess.check_output(['uname', '-m']).strip() | |
3157f602 | 274 | except (subprocess.CalledProcessError, WindowsError): |
7453a54e SL |
275 | if sys.platform == 'win32': |
276 | return 'x86_64-pc-windows-msvc' | |
3157f602 XL |
277 | err = "uname not found" |
278 | if self.verbose: | |
279 | raise Exception(err) | |
280 | sys.exit(err) | |
7453a54e SL |
281 | |
282 | # Darwin's `uname -s` lies and always returns i386. We have to use | |
283 | # sysctl instead. | |
284 | if ostype == 'Darwin' and cputype == 'i686': | |
285 | sysctl = subprocess.check_output(['sysctl', 'hw.optional.x86_64']) | |
3157f602 | 286 | if ': 1' in sysctl: |
7453a54e SL |
287 | cputype = 'x86_64' |
288 | ||
289 | # The goal here is to come up with the same triple as LLVM would, | |
290 | # at least for the subset of platforms we're willing to target. | |
291 | if ostype == 'Linux': | |
292 | ostype = 'unknown-linux-gnu' | |
293 | elif ostype == 'FreeBSD': | |
294 | ostype = 'unknown-freebsd' | |
295 | elif ostype == 'DragonFly': | |
296 | ostype = 'unknown-dragonfly' | |
297 | elif ostype == 'Bitrig': | |
298 | ostype = 'unknown-bitrig' | |
299 | elif ostype == 'OpenBSD': | |
300 | ostype = 'unknown-openbsd' | |
301 | elif ostype == 'NetBSD': | |
302 | ostype = 'unknown-netbsd' | |
303 | elif ostype == 'Darwin': | |
304 | ostype = 'apple-darwin' | |
305 | elif ostype.startswith('MINGW'): | |
306 | # msys' `uname` does not print gcc configuration, but prints msys | |
307 | # configuration. so we cannot believe `uname -m`: | |
308 | # msys1 is always i686 and msys2 is always x86_64. | |
309 | # instead, msys defines $MSYSTEM which is MINGW32 on i686 and | |
310 | # MINGW64 on x86_64. | |
311 | ostype = 'pc-windows-gnu' | |
312 | cputype = 'i686' | |
313 | if os.environ.get('MSYSTEM') == 'MINGW64': | |
314 | cputype = 'x86_64' | |
315 | elif ostype.startswith('MSYS'): | |
316 | ostype = 'pc-windows-gnu' | |
317 | elif ostype.startswith('CYGWIN_NT'): | |
318 | cputype = 'i686' | |
319 | if ostype.endswith('WOW64'): | |
320 | cputype = 'x86_64' | |
321 | ostype = 'pc-windows-gnu' | |
322 | else: | |
a7813a04 XL |
323 | err = "unknown OS type: " + ostype |
324 | if self.verbose: | |
325 | raise ValueError(err) | |
326 | sys.exit(err) | |
7453a54e SL |
327 | |
328 | if cputype in {'i386', 'i486', 'i686', 'i786', 'x86'}: | |
329 | cputype = 'i686' | |
330 | elif cputype in {'xscale', 'arm'}: | |
331 | cputype = 'arm' | |
332 | elif cputype == 'armv7l': | |
333 | cputype = 'arm' | |
334 | ostype += 'eabihf' | |
335 | elif cputype == 'aarch64': | |
336 | cputype = 'aarch64' | |
337 | elif cputype in {'powerpc', 'ppc', 'ppc64'}: | |
338 | cputype = 'powerpc' | |
339 | elif cputype in {'amd64', 'x86_64', 'x86-64', 'x64'}: | |
340 | cputype = 'x86_64' | |
341 | else: | |
a7813a04 XL |
342 | err = "unknown cpu type: " + cputype |
343 | if self.verbose: | |
344 | raise ValueError(err) | |
345 | sys.exit(err) | |
7453a54e | 346 | |
3157f602 | 347 | return "{}-{}".format(cputype, ostype) |
7453a54e | 348 | |
a7813a04 XL |
349 | def main(): |
350 | parser = argparse.ArgumentParser(description='Build rust') | |
351 | parser.add_argument('--config') | |
3157f602 | 352 | parser.add_argument('--clean', action='store_true') |
a7813a04 XL |
353 | parser.add_argument('-v', '--verbose', action='store_true') |
354 | ||
355 | args = [a for a in sys.argv if a != '-h'] | |
356 | args, _ = parser.parse_known_args(args) | |
357 | ||
358 | # Configure initial bootstrap | |
359 | rb = RustBuild() | |
360 | rb.config_toml = '' | |
361 | rb.config_mk = '' | |
362 | rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..')) | |
363 | rb.build_dir = os.path.join(os.getcwd(), "build") | |
364 | rb.verbose = args.verbose | |
3157f602 | 365 | rb.clean = args.clean |
a7813a04 XL |
366 | |
367 | try: | |
368 | with open(args.config or 'config.toml') as config: | |
369 | rb.config_toml = config.read() | |
370 | except: | |
371 | pass | |
372 | try: | |
373 | rb.config_mk = open('config.mk').read() | |
374 | except: | |
375 | pass | |
376 | ||
377 | data = stage0_data(rb.rust_root) | |
378 | rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1) | |
379 | rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1) | |
380 | ||
3157f602 XL |
381 | start_time = time() |
382 | ||
a7813a04 XL |
383 | # Fetch/build the bootstrap |
384 | rb.build = rb.build_triple() | |
385 | rb.download_stage0() | |
386 | sys.stdout.flush() | |
387 | rb.build_bootstrap() | |
388 | sys.stdout.flush() | |
389 | ||
390 | # Run the bootstrap | |
391 | args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")] | |
392 | args.append('--src') | |
393 | args.append(rb.rust_root) | |
394 | args.append('--build') | |
395 | args.append(rb.build) | |
396 | args.extend(sys.argv[1:]) | |
397 | env = os.environ.copy() | |
398 | env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) | |
399 | rb.run(args, env) | |
400 | ||
3157f602 XL |
401 | end_time = time() |
402 | ||
403 | print("Build completed in %s" % format_build_time(end_time - start_time)) | |
404 | ||
a7813a04 XL |
405 | if __name__ == '__main__': |
406 | main() |