]>
Commit | Line | Data |
---|---|---|
dd84028f JS |
1 | """ |
2 | mkvenv - QEMU pyvenv bootstrapping utility | |
3 | ||
4 | usage: mkvenv [-h] command ... | |
5 | ||
6 | QEMU pyvenv bootstrapping utility | |
7 | ||
8 | options: | |
9 | -h, --help show this help message and exit | |
10 | ||
11 | Commands: | |
12 | command Description | |
13 | create create a venv | |
c5538eed | 14 | ensure Ensure that the specified package is installed. |
dd84028f JS |
15 | |
16 | -------------------------------------------------- | |
17 | ||
18 | usage: mkvenv create [-h] target | |
19 | ||
20 | positional arguments: | |
21 | target Target directory to install virtual environment into. | |
22 | ||
23 | options: | |
24 | -h, --help show this help message and exit | |
25 | ||
c5538eed JS |
26 | -------------------------------------------------- |
27 | ||
28 | usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec... | |
29 | ||
30 | positional arguments: | |
31 | dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5' | |
32 | ||
33 | options: | |
34 | -h, --help show this help message and exit | |
35 | --online Install packages from PyPI, if necessary. | |
36 | --dir DIR Path to vendored packages where we may install from. | |
37 | ||
dd84028f JS |
38 | """ |
39 | ||
40 | # Copyright (C) 2022-2023 Red Hat, Inc. | |
41 | # | |
42 | # Authors: | |
43 | # John Snow <jsnow@redhat.com> | |
44 | # Paolo Bonzini <pbonzini@redhat.com> | |
45 | # | |
46 | # This work is licensed under the terms of the GNU GPL, version 2 or | |
47 | # later. See the COPYING file in the top-level directory. | |
48 | ||
49 | import argparse | |
a9dbde71 | 50 | from importlib.util import find_spec |
dd84028f JS |
51 | import logging |
52 | import os | |
53 | from pathlib import Path | |
4695a22e JS |
54 | import re |
55 | import shutil | |
dee01b82 | 56 | import site |
dd84028f JS |
57 | import subprocess |
58 | import sys | |
dee01b82 | 59 | import sysconfig |
dd84028f | 60 | from types import SimpleNamespace |
c5538eed JS |
61 | from typing import ( |
62 | Any, | |
92834894 | 63 | Iterator, |
c5538eed JS |
64 | Optional, |
65 | Sequence, | |
4695a22e | 66 | Tuple, |
c5538eed JS |
67 | Union, |
68 | ) | |
dd84028f | 69 | import venv |
c5538eed JS |
70 | import warnings |
71 | ||
72 | import distlib.database | |
92834894 | 73 | import distlib.scripts |
c5538eed | 74 | import distlib.version |
dd84028f JS |
75 | |
76 | ||
77 | # Do not add any mandatory dependencies from outside the stdlib: | |
78 | # This script *must* be usable standalone! | |
79 | ||
80 | DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] | |
81 | logger = logging.getLogger("mkvenv") | |
82 | ||
83 | ||
dee01b82 JS |
84 | def inside_a_venv() -> bool: |
85 | """Returns True if it is executed inside of a virtual environment.""" | |
86 | return sys.prefix != sys.base_prefix | |
87 | ||
88 | ||
dd84028f JS |
89 | class Ouch(RuntimeError): |
90 | """An Exception class we can't confuse with a builtin.""" | |
91 | ||
92 | ||
93 | class QemuEnvBuilder(venv.EnvBuilder): | |
94 | """ | |
95 | An extension of venv.EnvBuilder for building QEMU's configure-time venv. | |
96 | ||
dee01b82 JS |
97 | The primary difference is that it emulates a "nested" virtual |
98 | environment when invoked from inside of an existing virtual | |
99 | environment by including packages from the parent. | |
dd84028f JS |
100 | |
101 | Parameters for base class init: | |
102 | - system_site_packages: bool = False | |
103 | - clear: bool = False | |
104 | - symlinks: bool = False | |
105 | - upgrade: bool = False | |
106 | - with_pip: bool = False | |
107 | - prompt: Optional[str] = None | |
108 | - upgrade_deps: bool = False (Since 3.9) | |
109 | """ | |
110 | ||
111 | def __init__(self, *args: Any, **kwargs: Any) -> None: | |
112 | logger.debug("QemuEnvBuilder.__init__(...)") | |
a9dbde71 | 113 | |
dee01b82 JS |
114 | # For nested venv emulation: |
115 | self.use_parent_packages = False | |
116 | if inside_a_venv(): | |
117 | # Include parent packages only if we're in a venv and | |
118 | # system_site_packages was True. | |
119 | self.use_parent_packages = kwargs.pop( | |
120 | "system_site_packages", False | |
121 | ) | |
122 | # Include system_site_packages only when the parent, | |
123 | # The venv we are currently in, also does so. | |
124 | kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES | |
125 | ||
a9dbde71 JS |
126 | if kwargs.get("with_pip", False): |
127 | check_ensurepip() | |
128 | ||
dd84028f JS |
129 | super().__init__(*args, **kwargs) |
130 | ||
131 | # Make the context available post-creation: | |
132 | self._context: Optional[SimpleNamespace] = None | |
133 | ||
dee01b82 JS |
134 | def get_parent_libpath(self) -> Optional[str]: |
135 | """Return the libpath of the parent venv, if applicable.""" | |
136 | if self.use_parent_packages: | |
137 | return sysconfig.get_path("purelib") | |
138 | return None | |
139 | ||
140 | @staticmethod | |
141 | def compute_venv_libpath(context: SimpleNamespace) -> str: | |
142 | """ | |
143 | Compatibility wrapper for context.lib_path for Python < 3.12 | |
144 | """ | |
145 | # Python 3.12+, not strictly necessary because it's documented | |
146 | # to be the same as 3.10 code below: | |
147 | if sys.version_info >= (3, 12): | |
148 | return context.lib_path | |
149 | ||
150 | # Python 3.10+ | |
151 | if "venv" in sysconfig.get_scheme_names(): | |
152 | lib_path = sysconfig.get_path( | |
153 | "purelib", scheme="venv", vars={"base": context.env_dir} | |
154 | ) | |
155 | assert lib_path is not None | |
156 | return lib_path | |
157 | ||
158 | # For Python <= 3.9 we need to hardcode this. Fortunately the | |
159 | # code below was the same in Python 3.6-3.10, so there is only | |
160 | # one case. | |
161 | if sys.platform == "win32": | |
162 | return os.path.join(context.env_dir, "Lib", "site-packages") | |
163 | return os.path.join( | |
164 | context.env_dir, | |
165 | "lib", | |
166 | "python%d.%d" % sys.version_info[:2], | |
167 | "site-packages", | |
168 | ) | |
169 | ||
dd84028f JS |
170 | def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: |
171 | logger.debug("ensure_directories(env_dir=%s)", env_dir) | |
172 | self._context = super().ensure_directories(env_dir) | |
173 | return self._context | |
174 | ||
dee01b82 JS |
175 | def create(self, env_dir: DirType) -> None: |
176 | logger.debug("create(env_dir=%s)", env_dir) | |
177 | super().create(env_dir) | |
178 | assert self._context is not None | |
179 | self.post_post_setup(self._context) | |
180 | ||
181 | def post_post_setup(self, context: SimpleNamespace) -> None: | |
182 | """ | |
183 | The final, final hook. Enter the venv and run commands inside of it. | |
184 | """ | |
185 | if self.use_parent_packages: | |
186 | # We're inside of a venv and we want to include the parent | |
187 | # venv's packages. | |
188 | parent_libpath = self.get_parent_libpath() | |
189 | assert parent_libpath is not None | |
190 | logger.debug("parent_libpath: %s", parent_libpath) | |
191 | ||
192 | our_libpath = self.compute_venv_libpath(context) | |
193 | logger.debug("our_libpath: %s", our_libpath) | |
194 | ||
195 | pth_file = os.path.join(our_libpath, "nested.pth") | |
196 | with open(pth_file, "w", encoding="UTF-8") as file: | |
197 | file.write(parent_libpath + os.linesep) | |
198 | ||
dd84028f JS |
199 | def get_value(self, field: str) -> str: |
200 | """ | |
201 | Get a string value from the context namespace after a call to build. | |
202 | ||
203 | For valid field names, see: | |
204 | https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories | |
205 | """ | |
206 | ret = getattr(self._context, field) | |
207 | assert isinstance(ret, str) | |
208 | return ret | |
209 | ||
210 | ||
a9dbde71 JS |
211 | def check_ensurepip() -> None: |
212 | """ | |
213 | Check that we have ensurepip. | |
214 | ||
215 | Raise a fatal exception with a helpful hint if it isn't available. | |
216 | """ | |
217 | if not find_spec("ensurepip"): | |
218 | msg = ( | |
219 | "Python's ensurepip module is not found.\n" | |
220 | "It's normally part of the Python standard library, " | |
221 | "maybe your distribution packages it separately?\n" | |
222 | "Either install ensurepip, or alleviate the need for it in the " | |
223 | "first place by installing pip and setuptools for " | |
224 | f"'{sys.executable}'.\n" | |
225 | "(Hint: Debian puts ensurepip in its python3-venv package.)" | |
226 | ) | |
227 | raise Ouch(msg) | |
228 | ||
229 | # ensurepip uses pyexpat, which can also go missing on us: | |
230 | if not find_spec("pyexpat"): | |
231 | msg = ( | |
232 | "Python's pyexpat module is not found.\n" | |
233 | "It's normally part of the Python standard library, " | |
234 | "maybe your distribution packages it separately?\n" | |
235 | "Either install pyexpat, or alleviate the need for it in the " | |
236 | "first place by installing pip and setuptools for " | |
237 | f"'{sys.executable}'.\n\n" | |
238 | "(Hint: NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)" | |
239 | ) | |
240 | raise Ouch(msg) | |
241 | ||
242 | ||
dd84028f JS |
243 | def make_venv( # pylint: disable=too-many-arguments |
244 | env_dir: Union[str, Path], | |
245 | system_site_packages: bool = False, | |
246 | clear: bool = True, | |
247 | symlinks: Optional[bool] = None, | |
248 | with_pip: bool = True, | |
249 | ) -> None: | |
250 | """ | |
251 | Create a venv using `QemuEnvBuilder`. | |
252 | ||
253 | This is analogous to the `venv.create` module-level convenience | |
254 | function that is part of the Python stdblib, except it uses | |
255 | `QemuEnvBuilder` instead. | |
256 | ||
257 | :param env_dir: The directory to create/install to. | |
258 | :param system_site_packages: | |
259 | Allow inheriting packages from the system installation. | |
260 | :param clear: When True, fully remove any prior venv and files. | |
261 | :param symlinks: | |
262 | Whether to use symlinks to the target interpreter or not. If | |
263 | left unspecified, it will use symlinks except on Windows to | |
264 | match behavior with the "venv" CLI tool. | |
265 | :param with_pip: | |
266 | Whether to install "pip" binaries or not. | |
267 | """ | |
268 | logger.debug( | |
269 | "%s: make_venv(env_dir=%s, system_site_packages=%s, " | |
270 | "clear=%s, symlinks=%s, with_pip=%s)", | |
271 | __file__, | |
272 | str(env_dir), | |
273 | system_site_packages, | |
274 | clear, | |
275 | symlinks, | |
276 | with_pip, | |
277 | ) | |
278 | ||
279 | if symlinks is None: | |
280 | # Default behavior of standard venv CLI | |
281 | symlinks = os.name != "nt" | |
282 | ||
283 | builder = QemuEnvBuilder( | |
284 | system_site_packages=system_site_packages, | |
285 | clear=clear, | |
286 | symlinks=symlinks, | |
287 | with_pip=with_pip, | |
288 | ) | |
289 | ||
290 | style = "non-isolated" if builder.system_site_packages else "isolated" | |
dee01b82 JS |
291 | nested = "" |
292 | if builder.use_parent_packages: | |
293 | nested = f"(with packages from '{builder.get_parent_libpath()}') " | |
dd84028f JS |
294 | print( |
295 | f"mkvenv: Creating {style} virtual environment" | |
dee01b82 | 296 | f" {nested}at '{str(env_dir)}'", |
dd84028f JS |
297 | file=sys.stderr, |
298 | ) | |
299 | ||
300 | try: | |
301 | logger.debug("Invoking builder.create()") | |
302 | try: | |
303 | builder.create(str(env_dir)) | |
304 | except SystemExit as exc: | |
305 | # Some versions of the venv module raise SystemExit; *nasty*! | |
306 | # We want the exception that prompted it. It might be a subprocess | |
307 | # error that has output we *really* want to see. | |
308 | logger.debug("Intercepted SystemExit from EnvBuilder.create()") | |
309 | raise exc.__cause__ or exc.__context__ or exc | |
310 | logger.debug("builder.create() finished") | |
311 | except subprocess.CalledProcessError as exc: | |
312 | logger.error("mkvenv subprocess failed:") | |
313 | logger.error("cmd: %s", exc.cmd) | |
314 | logger.error("returncode: %d", exc.returncode) | |
315 | ||
316 | def _stringify(data: Union[str, bytes]) -> str: | |
317 | if isinstance(data, bytes): | |
318 | return data.decode() | |
319 | return data | |
320 | ||
321 | lines = [] | |
322 | if exc.stdout: | |
323 | lines.append("========== stdout ==========") | |
324 | lines.append(_stringify(exc.stdout)) | |
325 | lines.append("============================") | |
326 | if exc.stderr: | |
327 | lines.append("========== stderr ==========") | |
328 | lines.append(_stringify(exc.stderr)) | |
329 | lines.append("============================") | |
330 | if lines: | |
331 | logger.error(os.linesep.join(lines)) | |
332 | ||
333 | raise Ouch("VENV creation subprocess failed.") from exc | |
334 | ||
335 | # print the python executable to stdout for configure. | |
336 | print(builder.get_value("env_exe")) | |
337 | ||
338 | ||
92834894 JS |
339 | def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: |
340 | # pylint: disable=import-outside-toplevel | |
341 | # pylint: disable=no-name-in-module | |
342 | # pylint: disable=import-error | |
343 | try: | |
344 | # First preference: Python 3.8+ stdlib | |
345 | from importlib.metadata import ( # type: ignore | |
346 | PackageNotFoundError, | |
347 | distribution, | |
348 | ) | |
349 | except ImportError as exc: | |
350 | logger.debug("%s", str(exc)) | |
351 | # Second preference: Commonly available PyPI backport | |
352 | from importlib_metadata import ( # type: ignore | |
353 | PackageNotFoundError, | |
354 | distribution, | |
355 | ) | |
356 | ||
357 | def _generator() -> Iterator[str]: | |
358 | for package in packages: | |
359 | try: | |
360 | entry_points = distribution(package).entry_points | |
361 | except PackageNotFoundError: | |
362 | continue | |
363 | ||
364 | # The EntryPoints type is only available in 3.10+, | |
365 | # treat this as a vanilla list and filter it ourselves. | |
366 | entry_points = filter( | |
367 | lambda ep: ep.group == "console_scripts", entry_points | |
368 | ) | |
369 | ||
370 | for entry_point in entry_points: | |
371 | yield f"{entry_point.name} = {entry_point.value}" | |
372 | ||
373 | return _generator() | |
374 | ||
375 | ||
376 | def _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]: | |
377 | # pylint: disable=import-outside-toplevel | |
378 | # Bundled with setuptools; has a good chance of being available. | |
379 | import pkg_resources | |
380 | ||
381 | def _generator() -> Iterator[str]: | |
382 | for package in packages: | |
383 | try: | |
384 | eps = pkg_resources.get_entry_map(package, "console_scripts") | |
385 | except pkg_resources.DistributionNotFound: | |
386 | continue | |
387 | ||
388 | for entry_point in eps.values(): | |
389 | yield str(entry_point) | |
390 | ||
391 | return _generator() | |
392 | ||
393 | ||
394 | def generate_console_scripts( | |
395 | packages: Sequence[str], | |
396 | python_path: Optional[str] = None, | |
397 | bin_path: Optional[str] = None, | |
398 | ) -> None: | |
399 | """ | |
400 | Generate script shims for console_script entry points in @packages. | |
401 | """ | |
402 | if python_path is None: | |
403 | python_path = sys.executable | |
404 | if bin_path is None: | |
405 | bin_path = sysconfig.get_path("scripts") | |
406 | assert bin_path is not None | |
407 | ||
408 | logger.debug( | |
409 | "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", | |
410 | packages, | |
411 | python_path, | |
412 | bin_path, | |
413 | ) | |
414 | ||
415 | if not packages: | |
416 | return | |
417 | ||
418 | def _get_entry_points() -> Iterator[str]: | |
419 | """Python 3.7 compatibility shim for iterating entry points.""" | |
420 | # Python 3.8+, or Python 3.7 with importlib_metadata installed. | |
421 | try: | |
422 | return _gen_importlib(packages) | |
423 | except ImportError as exc: | |
424 | logger.debug("%s", str(exc)) | |
425 | ||
426 | # Python 3.7 with setuptools installed. | |
427 | try: | |
428 | return _gen_pkg_resources(packages) | |
429 | except ImportError as exc: | |
430 | logger.debug("%s", str(exc)) | |
431 | raise Ouch( | |
432 | "Neither importlib.metadata nor pkg_resources found, " | |
433 | "can't generate console script shims.\n" | |
434 | "Use Python 3.8+, or install importlib-metadata or setuptools." | |
435 | ) from exc | |
436 | ||
437 | maker = distlib.scripts.ScriptMaker(None, bin_path) | |
438 | maker.variants = {""} | |
439 | maker.clobber = False | |
440 | ||
441 | for entry_point in _get_entry_points(): | |
442 | for filename in maker.make(entry_point): | |
443 | logger.debug("wrote console_script '%s'", filename) | |
444 | ||
445 | ||
4695a22e JS |
446 | def pkgname_from_depspec(dep_spec: str) -> str: |
447 | """ | |
448 | Parse package name out of a PEP-508 depspec. | |
449 | ||
450 | See https://peps.python.org/pep-0508/#names | |
451 | """ | |
452 | match = re.match( | |
453 | r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE | |
454 | ) | |
455 | if not match: | |
456 | raise ValueError( | |
457 | f"dep_spec '{dep_spec}'" | |
458 | " does not appear to contain a valid package name" | |
459 | ) | |
460 | return match.group(0) | |
461 | ||
462 | ||
463 | def diagnose( | |
464 | dep_spec: str, | |
465 | online: bool, | |
466 | wheels_dir: Optional[Union[str, Path]], | |
467 | prog: Optional[str], | |
468 | ) -> Tuple[str, bool]: | |
469 | """ | |
470 | Offer a summary to the user as to why a package failed to be installed. | |
471 | ||
472 | :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' | |
473 | :param online: Did we allow PyPI access? | |
474 | :param prog: | |
475 | Optionally, a shell program name that can be used as a | |
476 | bellwether to detect if this program is installed elsewhere on | |
477 | the system. This is used to offer advice when a program is | |
478 | detected for a different python version. | |
479 | :param wheels_dir: | |
480 | Optionally, a directory that was searched for vendored packages. | |
481 | """ | |
482 | # pylint: disable=too-many-branches | |
483 | ||
484 | # Some errors are not particularly serious | |
485 | bad = False | |
486 | ||
487 | pkg_name = pkgname_from_depspec(dep_spec) | |
488 | pkg_version = None | |
489 | ||
490 | has_importlib = False | |
491 | try: | |
492 | # Python 3.8+ stdlib | |
493 | # pylint: disable=import-outside-toplevel | |
494 | # pylint: disable=no-name-in-module | |
495 | # pylint: disable=import-error | |
496 | from importlib.metadata import ( # type: ignore | |
497 | PackageNotFoundError, | |
498 | version, | |
499 | ) | |
500 | ||
501 | has_importlib = True | |
502 | try: | |
503 | pkg_version = version(pkg_name) | |
504 | except PackageNotFoundError: | |
505 | pass | |
506 | except ModuleNotFoundError: | |
507 | pass | |
508 | ||
509 | lines = [] | |
510 | ||
511 | if pkg_version: | |
512 | lines.append( | |
513 | f"Python package '{pkg_name}' version '{pkg_version}' was found," | |
514 | " but isn't suitable." | |
515 | ) | |
516 | elif has_importlib: | |
517 | lines.append( | |
518 | f"Python package '{pkg_name}' was not found nor installed." | |
519 | ) | |
520 | else: | |
521 | lines.append( | |
522 | f"Python package '{pkg_name}' is either not found or" | |
523 | " not a suitable version." | |
524 | ) | |
525 | ||
526 | if wheels_dir: | |
527 | lines.append( | |
528 | "No suitable version found in, or failed to install from" | |
529 | f" '{wheels_dir}'." | |
530 | ) | |
531 | bad = True | |
532 | ||
533 | if online: | |
534 | lines.append("A suitable version could not be obtained from PyPI.") | |
535 | bad = True | |
536 | else: | |
537 | lines.append( | |
538 | "mkvenv was configured to operate offline and did not check PyPI." | |
539 | ) | |
540 | ||
541 | if prog and not pkg_version: | |
542 | which = shutil.which(prog) | |
543 | if which: | |
544 | if sys.base_prefix in site.PREFIXES: | |
545 | pypath = Path(sys.executable).resolve() | |
546 | lines.append( | |
547 | f"'{prog}' was detected on your system at '{which}', " | |
548 | f"but the Python package '{pkg_name}' was not found by " | |
549 | f"this Python interpreter ('{pypath}'). " | |
550 | f"Typically this means that '{prog}' has been installed " | |
551 | "against a different Python interpreter on your system." | |
552 | ) | |
553 | else: | |
554 | lines.append( | |
555 | f"'{prog}' was detected on your system at '{which}', " | |
556 | "but the build is using an isolated virtual environment." | |
557 | ) | |
558 | bad = True | |
559 | ||
560 | lines = [f" • {line}" for line in lines] | |
561 | if bad: | |
562 | lines.insert(0, f"Could not provide build dependency '{dep_spec}':") | |
563 | else: | |
564 | lines.insert(0, f"'{dep_spec}' not found:") | |
565 | return os.linesep.join(lines), bad | |
566 | ||
567 | ||
c5538eed JS |
568 | def pip_install( |
569 | args: Sequence[str], | |
570 | online: bool = False, | |
571 | wheels_dir: Optional[Union[str, Path]] = None, | |
572 | ) -> None: | |
573 | """ | |
574 | Use pip to install a package or package(s) as specified in @args. | |
575 | """ | |
576 | loud = bool( | |
577 | os.environ.get("DEBUG") | |
578 | or os.environ.get("GITLAB_CI") | |
579 | or os.environ.get("V") | |
580 | ) | |
581 | ||
582 | full_args = [ | |
583 | sys.executable, | |
584 | "-m", | |
585 | "pip", | |
586 | "install", | |
587 | "--disable-pip-version-check", | |
588 | "-v" if loud else "-q", | |
589 | ] | |
590 | if not online: | |
591 | full_args += ["--no-index"] | |
592 | if wheels_dir: | |
593 | full_args += ["--find-links", f"file://{str(wheels_dir)}"] | |
594 | full_args += list(args) | |
595 | subprocess.run( | |
596 | full_args, | |
597 | check=True, | |
598 | ) | |
599 | ||
600 | ||
4695a22e | 601 | def _do_ensure( |
c5538eed JS |
602 | dep_specs: Sequence[str], |
603 | online: bool = False, | |
604 | wheels_dir: Optional[Union[str, Path]] = None, | |
605 | ) -> None: | |
606 | """ | |
607 | Use pip to ensure we have the package specified by @dep_specs. | |
608 | ||
609 | If the package is already installed, do nothing. If online and | |
610 | wheels_dir are both provided, prefer packages found in wheels_dir | |
611 | first before connecting to PyPI. | |
612 | ||
613 | :param dep_specs: | |
614 | PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. | |
615 | :param online: If True, fall back to PyPI. | |
616 | :param wheels_dir: If specified, search this path for packages. | |
617 | """ | |
618 | with warnings.catch_warnings(): | |
619 | warnings.filterwarnings( | |
620 | "ignore", category=UserWarning, module="distlib" | |
621 | ) | |
622 | dist_path = distlib.database.DistributionPath(include_egg=True) | |
623 | absent = [] | |
92834894 | 624 | present = [] |
c5538eed JS |
625 | for spec in dep_specs: |
626 | matcher = distlib.version.LegacyMatcher(spec) | |
627 | dist = dist_path.get_distribution(matcher.name) | |
628 | if dist is None or not matcher.match(dist.version): | |
629 | absent.append(spec) | |
630 | else: | |
631 | logger.info("found %s", dist) | |
92834894 JS |
632 | present.append(matcher.name) |
633 | ||
634 | if present: | |
635 | generate_console_scripts(present) | |
c5538eed JS |
636 | |
637 | if absent: | |
638 | # Some packages are missing or aren't a suitable version, | |
639 | # install a suitable (possibly vendored) package. | |
640 | print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) | |
641 | pip_install(args=absent, online=online, wheels_dir=wheels_dir) | |
642 | ||
643 | ||
4695a22e JS |
644 | def ensure( |
645 | dep_specs: Sequence[str], | |
646 | online: bool = False, | |
647 | wheels_dir: Optional[Union[str, Path]] = None, | |
648 | prog: Optional[str] = None, | |
649 | ) -> None: | |
650 | """ | |
651 | Use pip to ensure we have the package specified by @dep_specs. | |
652 | ||
653 | If the package is already installed, do nothing. If online and | |
654 | wheels_dir are both provided, prefer packages found in wheels_dir | |
655 | first before connecting to PyPI. | |
656 | ||
657 | :param dep_specs: | |
658 | PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. | |
659 | :param online: If True, fall back to PyPI. | |
660 | :param wheels_dir: If specified, search this path for packages. | |
661 | :param prog: | |
662 | If specified, use this program name for error diagnostics that will | |
663 | be presented to the user. e.g., 'sphinx-build' can be used as a | |
664 | bellwether for the presence of 'sphinx'. | |
665 | """ | |
666 | print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr) | |
667 | try: | |
668 | _do_ensure(dep_specs, online, wheels_dir) | |
669 | except subprocess.CalledProcessError as exc: | |
670 | # Well, that's not good. | |
671 | msg, bad = diagnose(dep_specs[0], online, wheels_dir, prog) | |
672 | if bad: | |
673 | raise Ouch(msg) from exc | |
674 | raise SystemExit(f"\n{msg}\n\n") from exc | |
675 | ||
676 | ||
dd84028f JS |
677 | def _add_create_subcommand(subparsers: Any) -> None: |
678 | subparser = subparsers.add_parser("create", help="create a venv") | |
679 | subparser.add_argument( | |
680 | "target", | |
681 | type=str, | |
682 | action="store", | |
683 | help="Target directory to install virtual environment into.", | |
684 | ) | |
685 | ||
686 | ||
c5538eed JS |
687 | def _add_ensure_subcommand(subparsers: Any) -> None: |
688 | subparser = subparsers.add_parser( | |
689 | "ensure", help="Ensure that the specified package is installed." | |
690 | ) | |
691 | subparser.add_argument( | |
692 | "--online", | |
693 | action="store_true", | |
694 | help="Install packages from PyPI, if necessary.", | |
695 | ) | |
696 | subparser.add_argument( | |
697 | "--dir", | |
698 | type=str, | |
699 | action="store", | |
700 | help="Path to vendored packages where we may install from.", | |
701 | ) | |
4695a22e JS |
702 | subparser.add_argument( |
703 | "--diagnose", | |
704 | type=str, | |
705 | action="store", | |
706 | help=( | |
707 | "Name of a shell utility to use for " | |
708 | "diagnostics if this command fails." | |
709 | ), | |
710 | ) | |
c5538eed JS |
711 | subparser.add_argument( |
712 | "dep_specs", | |
713 | type=str, | |
714 | action="store", | |
715 | help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'", | |
716 | nargs="+", | |
717 | ) | |
718 | ||
719 | ||
dd84028f JS |
720 | def main() -> int: |
721 | """CLI interface to make_qemu_venv. See module docstring.""" | |
722 | if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): | |
723 | # You're welcome. | |
724 | logging.basicConfig(level=logging.DEBUG) | |
c5538eed JS |
725 | else: |
726 | if os.environ.get("V"): | |
727 | logging.basicConfig(level=logging.INFO) | |
728 | ||
729 | # These are incredibly noisy even for V=1 | |
730 | logging.getLogger("distlib.metadata").addFilter(lambda record: False) | |
731 | logging.getLogger("distlib.database").addFilter(lambda record: False) | |
dd84028f JS |
732 | |
733 | parser = argparse.ArgumentParser( | |
734 | prog="mkvenv", | |
735 | description="QEMU pyvenv bootstrapping utility", | |
736 | ) | |
737 | subparsers = parser.add_subparsers( | |
738 | title="Commands", | |
739 | dest="command", | |
740 | metavar="command", | |
741 | help="Description", | |
742 | ) | |
743 | ||
744 | _add_create_subcommand(subparsers) | |
c5538eed | 745 | _add_ensure_subcommand(subparsers) |
dd84028f JS |
746 | |
747 | args = parser.parse_args() | |
748 | try: | |
749 | if args.command == "create": | |
750 | make_venv( | |
751 | args.target, | |
752 | system_site_packages=True, | |
753 | clear=True, | |
754 | ) | |
c5538eed JS |
755 | if args.command == "ensure": |
756 | ensure( | |
757 | dep_specs=args.dep_specs, | |
758 | online=args.online, | |
759 | wheels_dir=args.dir, | |
4695a22e | 760 | prog=args.diagnose, |
c5538eed | 761 | ) |
dd84028f JS |
762 | logger.debug("mkvenv.py %s: exiting", args.command) |
763 | except Ouch as exc: | |
764 | print("\n*** Ouch! ***\n", file=sys.stderr) | |
765 | print(str(exc), "\n\n", file=sys.stderr) | |
766 | return 1 | |
767 | except SystemExit: | |
768 | raise | |
769 | except: # pylint: disable=bare-except | |
770 | logger.exception("mkvenv did not complete successfully:") | |
771 | return 2 | |
772 | return 0 | |
773 | ||
774 | ||
775 | if __name__ == "__main__": | |
776 | sys.exit(main()) |