]>
Commit | Line | Data |
---|---|---|
dd84028f JS |
1 | """ |
2 | mkvenv - QEMU pyvenv bootstrapping utility | |
3 | ||
4 | usage: mkvenv [-h] command ... | |
5 | ||
6 | QEMU pyvenv bootstrapping utility | |
7 | ||
8 | options: | |
9 | -h, --help show this help message and exit | |
10 | ||
11 | Commands: | |
12 | command Description | |
13 | create create a venv | |
f1ad527f JS |
14 | post_init |
15 | post-venv initialization | |
c5538eed | 16 | ensure Ensure that the specified package is installed. |
dd84028f JS |
17 | |
18 | -------------------------------------------------- | |
19 | ||
20 | usage: mkvenv create [-h] target | |
21 | ||
22 | positional arguments: | |
23 | target Target directory to install virtual environment into. | |
24 | ||
25 | options: | |
26 | -h, --help show this help message and exit | |
27 | ||
c5538eed JS |
28 | -------------------------------------------------- |
29 | ||
f1ad527f JS |
30 | usage: mkvenv post_init [-h] |
31 | ||
32 | options: | |
33 | -h, --help show this help message and exit | |
34 | ||
35 | -------------------------------------------------- | |
36 | ||
c5538eed JS |
37 | usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec... |
38 | ||
39 | positional arguments: | |
40 | dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5' | |
41 | ||
42 | options: | |
43 | -h, --help show this help message and exit | |
44 | --online Install packages from PyPI, if necessary. | |
45 | --dir DIR Path to vendored packages where we may install from. | |
46 | ||
dd84028f JS |
47 | """ |
48 | ||
49 | # Copyright (C) 2022-2023 Red Hat, Inc. | |
50 | # | |
51 | # Authors: | |
52 | # John Snow <jsnow@redhat.com> | |
53 | # Paolo Bonzini <pbonzini@redhat.com> | |
54 | # | |
55 | # This work is licensed under the terms of the GNU GPL, version 2 or | |
56 | # later. See the COPYING file in the top-level directory. | |
57 | ||
58 | import argparse | |
a9dbde71 | 59 | from importlib.util import find_spec |
dd84028f JS |
60 | import logging |
61 | import os | |
62 | from pathlib import Path | |
4695a22e JS |
63 | import re |
64 | import shutil | |
dee01b82 | 65 | import site |
dd84028f JS |
66 | import subprocess |
67 | import sys | |
dee01b82 | 68 | import sysconfig |
dd84028f | 69 | from types import SimpleNamespace |
c5538eed JS |
70 | from typing import ( |
71 | Any, | |
92834894 | 72 | Iterator, |
c5538eed JS |
73 | Optional, |
74 | Sequence, | |
4695a22e | 75 | Tuple, |
c5538eed JS |
76 | Union, |
77 | ) | |
dd84028f | 78 | import venv |
c5538eed | 79 | |
dd84028f | 80 | |
68ea6d17 JS |
81 | # Try to load distlib, with a fallback to pip's vendored version. |
82 | # HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail | |
83 | # outside the venv or before a potential call to ensurepip in checkpip(). | |
84 | HAVE_DISTLIB = True | |
85 | try: | |
68ea6d17 JS |
86 | import distlib.scripts |
87 | import distlib.version | |
88 | except ImportError: | |
89 | try: | |
90 | # Reach into pip's cookie jar. pylint and flake8 don't understand | |
91 | # that these imports will be used via distlib.xxx. | |
92 | from pip._vendor import distlib | |
68ea6d17 JS |
93 | import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import |
94 | import pip._vendor.distlib.version # noqa, pylint: disable=unused-import | |
95 | except ImportError: | |
96 | HAVE_DISTLIB = False | |
dd84028f JS |
97 | |
98 | # Do not add any mandatory dependencies from outside the stdlib: | |
99 | # This script *must* be usable standalone! | |
100 | ||
101 | DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] | |
102 | logger = logging.getLogger("mkvenv") | |
103 | ||
104 | ||
dee01b82 JS |
105 | def inside_a_venv() -> bool: |
106 | """Returns True if it is executed inside of a virtual environment.""" | |
107 | return sys.prefix != sys.base_prefix | |
108 | ||
109 | ||
dd84028f JS |
110 | class Ouch(RuntimeError): |
111 | """An Exception class we can't confuse with a builtin.""" | |
112 | ||
113 | ||
114 | class QemuEnvBuilder(venv.EnvBuilder): | |
115 | """ | |
116 | An extension of venv.EnvBuilder for building QEMU's configure-time venv. | |
117 | ||
dee01b82 JS |
118 | The primary difference is that it emulates a "nested" virtual |
119 | environment when invoked from inside of an existing virtual | |
f1ad527f JS |
120 | environment by including packages from the parent. Also, |
121 | "ensurepip" is replaced if possible with just recreating pip's | |
122 | console_scripts inside the virtual environment. | |
dd84028f JS |
123 | |
124 | Parameters for base class init: | |
125 | - system_site_packages: bool = False | |
126 | - clear: bool = False | |
127 | - symlinks: bool = False | |
128 | - upgrade: bool = False | |
129 | - with_pip: bool = False | |
130 | - prompt: Optional[str] = None | |
131 | - upgrade_deps: bool = False (Since 3.9) | |
132 | """ | |
133 | ||
134 | def __init__(self, *args: Any, **kwargs: Any) -> None: | |
135 | logger.debug("QemuEnvBuilder.__init__(...)") | |
a9dbde71 | 136 | |
dee01b82 JS |
137 | # For nested venv emulation: |
138 | self.use_parent_packages = False | |
139 | if inside_a_venv(): | |
140 | # Include parent packages only if we're in a venv and | |
141 | # system_site_packages was True. | |
142 | self.use_parent_packages = kwargs.pop( | |
143 | "system_site_packages", False | |
144 | ) | |
145 | # Include system_site_packages only when the parent, | |
146 | # The venv we are currently in, also does so. | |
147 | kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES | |
148 | ||
f1ad527f JS |
149 | # ensurepip is slow: venv creation can be very fast for cases where |
150 | # we allow the use of system_site_packages. Therefore, ensurepip is | |
151 | # replaced with our own script generation once the virtual environment | |
152 | # is setup. | |
153 | self.want_pip = kwargs.get("with_pip", False) | |
154 | if self.want_pip: | |
155 | if ( | |
156 | kwargs.get("system_site_packages", False) | |
157 | and not need_ensurepip() | |
158 | ): | |
159 | kwargs["with_pip"] = False | |
160 | else: | |
c8049626 | 161 | check_ensurepip(suggest_remedy=True) |
a9dbde71 | 162 | |
dd84028f JS |
163 | super().__init__(*args, **kwargs) |
164 | ||
165 | # Make the context available post-creation: | |
166 | self._context: Optional[SimpleNamespace] = None | |
167 | ||
dee01b82 JS |
168 | def get_parent_libpath(self) -> Optional[str]: |
169 | """Return the libpath of the parent venv, if applicable.""" | |
170 | if self.use_parent_packages: | |
171 | return sysconfig.get_path("purelib") | |
172 | return None | |
173 | ||
174 | @staticmethod | |
175 | def compute_venv_libpath(context: SimpleNamespace) -> str: | |
176 | """ | |
177 | Compatibility wrapper for context.lib_path for Python < 3.12 | |
178 | """ | |
179 | # Python 3.12+, not strictly necessary because it's documented | |
180 | # to be the same as 3.10 code below: | |
181 | if sys.version_info >= (3, 12): | |
182 | return context.lib_path | |
183 | ||
184 | # Python 3.10+ | |
185 | if "venv" in sysconfig.get_scheme_names(): | |
186 | lib_path = sysconfig.get_path( | |
187 | "purelib", scheme="venv", vars={"base": context.env_dir} | |
188 | ) | |
189 | assert lib_path is not None | |
190 | return lib_path | |
191 | ||
192 | # For Python <= 3.9 we need to hardcode this. Fortunately the | |
193 | # code below was the same in Python 3.6-3.10, so there is only | |
194 | # one case. | |
195 | if sys.platform == "win32": | |
196 | return os.path.join(context.env_dir, "Lib", "site-packages") | |
197 | return os.path.join( | |
198 | context.env_dir, | |
199 | "lib", | |
200 | "python%d.%d" % sys.version_info[:2], | |
201 | "site-packages", | |
202 | ) | |
203 | ||
dd84028f JS |
204 | def ensure_directories(self, env_dir: DirType) -> SimpleNamespace: |
205 | logger.debug("ensure_directories(env_dir=%s)", env_dir) | |
206 | self._context = super().ensure_directories(env_dir) | |
207 | return self._context | |
208 | ||
dee01b82 JS |
209 | def create(self, env_dir: DirType) -> None: |
210 | logger.debug("create(env_dir=%s)", env_dir) | |
211 | super().create(env_dir) | |
212 | assert self._context is not None | |
213 | self.post_post_setup(self._context) | |
214 | ||
215 | def post_post_setup(self, context: SimpleNamespace) -> None: | |
216 | """ | |
217 | The final, final hook. Enter the venv and run commands inside of it. | |
218 | """ | |
219 | if self.use_parent_packages: | |
220 | # We're inside of a venv and we want to include the parent | |
221 | # venv's packages. | |
222 | parent_libpath = self.get_parent_libpath() | |
223 | assert parent_libpath is not None | |
224 | logger.debug("parent_libpath: %s", parent_libpath) | |
225 | ||
226 | our_libpath = self.compute_venv_libpath(context) | |
227 | logger.debug("our_libpath: %s", our_libpath) | |
228 | ||
229 | pth_file = os.path.join(our_libpath, "nested.pth") | |
230 | with open(pth_file, "w", encoding="UTF-8") as file: | |
231 | file.write(parent_libpath + os.linesep) | |
232 | ||
f1ad527f JS |
233 | if self.want_pip: |
234 | args = [ | |
235 | context.env_exe, | |
236 | __file__, | |
237 | "post_init", | |
238 | ] | |
239 | subprocess.run(args, check=True) | |
240 | ||
dd84028f JS |
241 | def get_value(self, field: str) -> str: |
242 | """ | |
243 | Get a string value from the context namespace after a call to build. | |
244 | ||
245 | For valid field names, see: | |
246 | https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories | |
247 | """ | |
248 | ret = getattr(self._context, field) | |
249 | assert isinstance(ret, str) | |
250 | return ret | |
251 | ||
252 | ||
f1ad527f JS |
253 | def need_ensurepip() -> bool: |
254 | """ | |
255 | Tests for the presence of setuptools and pip. | |
256 | ||
257 | :return: `True` if we do not detect both packages. | |
258 | """ | |
259 | # Don't try to actually import them, it's fraught with danger: | |
260 | # https://github.com/pypa/setuptools/issues/2993 | |
261 | if find_spec("setuptools") and find_spec("pip"): | |
262 | return False | |
263 | return True | |
264 | ||
265 | ||
c8049626 | 266 | def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None: |
a9dbde71 JS |
267 | """ |
268 | Check that we have ensurepip. | |
269 | ||
270 | Raise a fatal exception with a helpful hint if it isn't available. | |
271 | """ | |
272 | if not find_spec("ensurepip"): | |
273 | msg = ( | |
274 | "Python's ensurepip module is not found.\n" | |
275 | "It's normally part of the Python standard library, " | |
276 | "maybe your distribution packages it separately?\n" | |
c8049626 | 277 | "(Debian puts ensurepip in its python3-venv package.)\n" |
a9dbde71 | 278 | ) |
c8049626 JS |
279 | if suggest_remedy: |
280 | msg += ( | |
281 | "Either install ensurepip, or alleviate the need for it in the" | |
282 | " first place by installing pip and setuptools for " | |
283 | f"'{sys.executable}'.\n" | |
284 | ) | |
285 | raise Ouch(prefix + msg) | |
a9dbde71 JS |
286 | |
287 | # ensurepip uses pyexpat, which can also go missing on us: | |
288 | if not find_spec("pyexpat"): | |
289 | msg = ( | |
290 | "Python's pyexpat module is not found.\n" | |
291 | "It's normally part of the Python standard library, " | |
292 | "maybe your distribution packages it separately?\n" | |
c8049626 | 293 | "(NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)\n" |
a9dbde71 | 294 | ) |
c8049626 JS |
295 | if suggest_remedy: |
296 | msg += ( | |
297 | "Either install pyexpat, or alleviate the need for it in the " | |
298 | "first place by installing pip and setuptools for " | |
299 | f"'{sys.executable}'.\n" | |
300 | ) | |
301 | raise Ouch(prefix + msg) | |
a9dbde71 JS |
302 | |
303 | ||
dd84028f JS |
304 | def make_venv( # pylint: disable=too-many-arguments |
305 | env_dir: Union[str, Path], | |
306 | system_site_packages: bool = False, | |
307 | clear: bool = True, | |
308 | symlinks: Optional[bool] = None, | |
309 | with_pip: bool = True, | |
310 | ) -> None: | |
311 | """ | |
312 | Create a venv using `QemuEnvBuilder`. | |
313 | ||
314 | This is analogous to the `venv.create` module-level convenience | |
315 | function that is part of the Python stdblib, except it uses | |
316 | `QemuEnvBuilder` instead. | |
317 | ||
318 | :param env_dir: The directory to create/install to. | |
319 | :param system_site_packages: | |
320 | Allow inheriting packages from the system installation. | |
321 | :param clear: When True, fully remove any prior venv and files. | |
322 | :param symlinks: | |
323 | Whether to use symlinks to the target interpreter or not. If | |
324 | left unspecified, it will use symlinks except on Windows to | |
325 | match behavior with the "venv" CLI tool. | |
326 | :param with_pip: | |
327 | Whether to install "pip" binaries or not. | |
328 | """ | |
329 | logger.debug( | |
330 | "%s: make_venv(env_dir=%s, system_site_packages=%s, " | |
331 | "clear=%s, symlinks=%s, with_pip=%s)", | |
332 | __file__, | |
333 | str(env_dir), | |
334 | system_site_packages, | |
335 | clear, | |
336 | symlinks, | |
337 | with_pip, | |
338 | ) | |
339 | ||
340 | if symlinks is None: | |
341 | # Default behavior of standard venv CLI | |
342 | symlinks = os.name != "nt" | |
343 | ||
344 | builder = QemuEnvBuilder( | |
345 | system_site_packages=system_site_packages, | |
346 | clear=clear, | |
347 | symlinks=symlinks, | |
348 | with_pip=with_pip, | |
349 | ) | |
350 | ||
351 | style = "non-isolated" if builder.system_site_packages else "isolated" | |
dee01b82 JS |
352 | nested = "" |
353 | if builder.use_parent_packages: | |
354 | nested = f"(with packages from '{builder.get_parent_libpath()}') " | |
dd84028f JS |
355 | print( |
356 | f"mkvenv: Creating {style} virtual environment" | |
dee01b82 | 357 | f" {nested}at '{str(env_dir)}'", |
dd84028f JS |
358 | file=sys.stderr, |
359 | ) | |
360 | ||
361 | try: | |
362 | logger.debug("Invoking builder.create()") | |
363 | try: | |
364 | builder.create(str(env_dir)) | |
365 | except SystemExit as exc: | |
366 | # Some versions of the venv module raise SystemExit; *nasty*! | |
367 | # We want the exception that prompted it. It might be a subprocess | |
368 | # error that has output we *really* want to see. | |
369 | logger.debug("Intercepted SystemExit from EnvBuilder.create()") | |
370 | raise exc.__cause__ or exc.__context__ or exc | |
371 | logger.debug("builder.create() finished") | |
372 | except subprocess.CalledProcessError as exc: | |
373 | logger.error("mkvenv subprocess failed:") | |
374 | logger.error("cmd: %s", exc.cmd) | |
375 | logger.error("returncode: %d", exc.returncode) | |
376 | ||
377 | def _stringify(data: Union[str, bytes]) -> str: | |
378 | if isinstance(data, bytes): | |
379 | return data.decode() | |
380 | return data | |
381 | ||
382 | lines = [] | |
383 | if exc.stdout: | |
384 | lines.append("========== stdout ==========") | |
385 | lines.append(_stringify(exc.stdout)) | |
386 | lines.append("============================") | |
387 | if exc.stderr: | |
388 | lines.append("========== stderr ==========") | |
389 | lines.append(_stringify(exc.stderr)) | |
390 | lines.append("============================") | |
391 | if lines: | |
392 | logger.error(os.linesep.join(lines)) | |
393 | ||
394 | raise Ouch("VENV creation subprocess failed.") from exc | |
395 | ||
396 | # print the python executable to stdout for configure. | |
397 | print(builder.get_value("env_exe")) | |
398 | ||
399 | ||
92834894 JS |
400 | def _gen_importlib(packages: Sequence[str]) -> Iterator[str]: |
401 | # pylint: disable=import-outside-toplevel | |
402 | # pylint: disable=no-name-in-module | |
403 | # pylint: disable=import-error | |
404 | try: | |
405 | # First preference: Python 3.8+ stdlib | |
406 | from importlib.metadata import ( # type: ignore | |
407 | PackageNotFoundError, | |
408 | distribution, | |
409 | ) | |
410 | except ImportError as exc: | |
411 | logger.debug("%s", str(exc)) | |
412 | # Second preference: Commonly available PyPI backport | |
413 | from importlib_metadata import ( # type: ignore | |
414 | PackageNotFoundError, | |
415 | distribution, | |
416 | ) | |
417 | ||
418 | def _generator() -> Iterator[str]: | |
419 | for package in packages: | |
420 | try: | |
421 | entry_points = distribution(package).entry_points | |
422 | except PackageNotFoundError: | |
423 | continue | |
424 | ||
425 | # The EntryPoints type is only available in 3.10+, | |
426 | # treat this as a vanilla list and filter it ourselves. | |
427 | entry_points = filter( | |
428 | lambda ep: ep.group == "console_scripts", entry_points | |
429 | ) | |
430 | ||
431 | for entry_point in entry_points: | |
432 | yield f"{entry_point.name} = {entry_point.value}" | |
433 | ||
434 | return _generator() | |
435 | ||
436 | ||
437 | def _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]: | |
438 | # pylint: disable=import-outside-toplevel | |
439 | # Bundled with setuptools; has a good chance of being available. | |
440 | import pkg_resources | |
441 | ||
442 | def _generator() -> Iterator[str]: | |
443 | for package in packages: | |
444 | try: | |
445 | eps = pkg_resources.get_entry_map(package, "console_scripts") | |
446 | except pkg_resources.DistributionNotFound: | |
447 | continue | |
448 | ||
449 | for entry_point in eps.values(): | |
450 | yield str(entry_point) | |
451 | ||
452 | return _generator() | |
453 | ||
454 | ||
455 | def generate_console_scripts( | |
456 | packages: Sequence[str], | |
457 | python_path: Optional[str] = None, | |
458 | bin_path: Optional[str] = None, | |
459 | ) -> None: | |
460 | """ | |
461 | Generate script shims for console_script entry points in @packages. | |
462 | """ | |
463 | if python_path is None: | |
464 | python_path = sys.executable | |
465 | if bin_path is None: | |
466 | bin_path = sysconfig.get_path("scripts") | |
467 | assert bin_path is not None | |
468 | ||
469 | logger.debug( | |
470 | "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)", | |
471 | packages, | |
472 | python_path, | |
473 | bin_path, | |
474 | ) | |
475 | ||
476 | if not packages: | |
477 | return | |
478 | ||
479 | def _get_entry_points() -> Iterator[str]: | |
480 | """Python 3.7 compatibility shim for iterating entry points.""" | |
481 | # Python 3.8+, or Python 3.7 with importlib_metadata installed. | |
482 | try: | |
483 | return _gen_importlib(packages) | |
484 | except ImportError as exc: | |
485 | logger.debug("%s", str(exc)) | |
486 | ||
487 | # Python 3.7 with setuptools installed. | |
488 | try: | |
489 | return _gen_pkg_resources(packages) | |
490 | except ImportError as exc: | |
491 | logger.debug("%s", str(exc)) | |
492 | raise Ouch( | |
493 | "Neither importlib.metadata nor pkg_resources found, " | |
494 | "can't generate console script shims.\n" | |
495 | "Use Python 3.8+, or install importlib-metadata or setuptools." | |
496 | ) from exc | |
497 | ||
498 | maker = distlib.scripts.ScriptMaker(None, bin_path) | |
499 | maker.variants = {""} | |
500 | maker.clobber = False | |
501 | ||
502 | for entry_point in _get_entry_points(): | |
503 | for filename in maker.make(entry_point): | |
504 | logger.debug("wrote console_script '%s'", filename) | |
505 | ||
506 | ||
c8049626 JS |
507 | def checkpip() -> bool: |
508 | """ | |
509 | Debian10 has a pip that's broken when used inside of a virtual environment. | |
510 | ||
511 | We try to detect and correct that case here. | |
512 | """ | |
513 | try: | |
514 | # pylint: disable=import-outside-toplevel,unused-import,import-error | |
515 | # pylint: disable=redefined-outer-name | |
516 | import pip._internal # type: ignore # noqa: F401 | |
517 | ||
518 | logger.debug("pip appears to be working correctly.") | |
519 | return False | |
520 | except ModuleNotFoundError as exc: | |
521 | if exc.name == "pip._internal": | |
522 | # Uh, fair enough. They did say "internal". | |
523 | # Let's just assume it's fine. | |
524 | return False | |
525 | logger.warning("pip appears to be malfunctioning: %s", str(exc)) | |
526 | ||
527 | check_ensurepip("pip appears to be non-functional, and ") | |
528 | ||
529 | logger.debug("Attempting to repair pip ...") | |
530 | subprocess.run( | |
531 | (sys.executable, "-m", "ensurepip"), | |
532 | stdout=subprocess.DEVNULL, | |
533 | check=True, | |
534 | ) | |
535 | logger.debug("Pip is now (hopefully) repaired!") | |
536 | return True | |
537 | ||
538 | ||
4695a22e JS |
539 | def pkgname_from_depspec(dep_spec: str) -> str: |
540 | """ | |
541 | Parse package name out of a PEP-508 depspec. | |
542 | ||
543 | See https://peps.python.org/pep-0508/#names | |
544 | """ | |
545 | match = re.match( | |
546 | r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE | |
547 | ) | |
548 | if not match: | |
549 | raise ValueError( | |
550 | f"dep_spec '{dep_spec}'" | |
551 | " does not appear to contain a valid package name" | |
552 | ) | |
553 | return match.group(0) | |
554 | ||
555 | ||
c673f3d0 PB |
556 | def _get_version_importlib(package: str) -> Optional[str]: |
557 | # pylint: disable=import-outside-toplevel | |
558 | # pylint: disable=no-name-in-module | |
559 | # pylint: disable=import-error | |
560 | try: | |
561 | # First preference: Python 3.8+ stdlib | |
562 | from importlib.metadata import ( # type: ignore | |
563 | PackageNotFoundError, | |
564 | distribution, | |
565 | ) | |
566 | except ImportError as exc: | |
567 | logger.debug("%s", str(exc)) | |
568 | # Second preference: Commonly available PyPI backport | |
569 | from importlib_metadata import ( # type: ignore | |
570 | PackageNotFoundError, | |
571 | distribution, | |
572 | ) | |
573 | ||
574 | try: | |
575 | return str(distribution(package).version) | |
576 | except PackageNotFoundError: | |
577 | return None | |
578 | ||
579 | ||
580 | def _get_version_pkg_resources(package: str) -> Optional[str]: | |
581 | # pylint: disable=import-outside-toplevel | |
582 | # Bundled with setuptools; has a good chance of being available. | |
583 | import pkg_resources | |
584 | ||
585 | try: | |
586 | return str(pkg_resources.get_distribution(package).version) | |
587 | except pkg_resources.DistributionNotFound: | |
588 | return None | |
589 | ||
590 | ||
591 | def _get_version(package: str) -> Optional[str]: | |
592 | try: | |
593 | return _get_version_importlib(package) | |
594 | except ImportError as exc: | |
595 | logger.debug("%s", str(exc)) | |
596 | ||
597 | try: | |
598 | return _get_version_pkg_resources(package) | |
599 | except ImportError as exc: | |
600 | logger.debug("%s", str(exc)) | |
601 | raise Ouch( | |
602 | "Neither importlib.metadata nor pkg_resources found. " | |
603 | "Use Python 3.8+, or install importlib-metadata or setuptools." | |
604 | ) from exc | |
605 | ||
606 | ||
4695a22e JS |
607 | def diagnose( |
608 | dep_spec: str, | |
609 | online: bool, | |
610 | wheels_dir: Optional[Union[str, Path]], | |
611 | prog: Optional[str], | |
612 | ) -> Tuple[str, bool]: | |
613 | """ | |
614 | Offer a summary to the user as to why a package failed to be installed. | |
615 | ||
616 | :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5' | |
617 | :param online: Did we allow PyPI access? | |
618 | :param prog: | |
619 | Optionally, a shell program name that can be used as a | |
620 | bellwether to detect if this program is installed elsewhere on | |
621 | the system. This is used to offer advice when a program is | |
622 | detected for a different python version. | |
623 | :param wheels_dir: | |
624 | Optionally, a directory that was searched for vendored packages. | |
625 | """ | |
626 | # pylint: disable=too-many-branches | |
627 | ||
628 | # Some errors are not particularly serious | |
629 | bad = False | |
630 | ||
631 | pkg_name = pkgname_from_depspec(dep_spec) | |
c673f3d0 | 632 | pkg_version = _get_version(pkg_name) |
4695a22e JS |
633 | |
634 | lines = [] | |
635 | ||
636 | if pkg_version: | |
637 | lines.append( | |
638 | f"Python package '{pkg_name}' version '{pkg_version}' was found," | |
639 | " but isn't suitable." | |
640 | ) | |
4695a22e JS |
641 | else: |
642 | lines.append( | |
c673f3d0 | 643 | f"Python package '{pkg_name}' was not found nor installed." |
4695a22e JS |
644 | ) |
645 | ||
646 | if wheels_dir: | |
647 | lines.append( | |
648 | "No suitable version found in, or failed to install from" | |
649 | f" '{wheels_dir}'." | |
650 | ) | |
651 | bad = True | |
652 | ||
653 | if online: | |
654 | lines.append("A suitable version could not be obtained from PyPI.") | |
655 | bad = True | |
656 | else: | |
657 | lines.append( | |
658 | "mkvenv was configured to operate offline and did not check PyPI." | |
659 | ) | |
660 | ||
661 | if prog and not pkg_version: | |
662 | which = shutil.which(prog) | |
663 | if which: | |
664 | if sys.base_prefix in site.PREFIXES: | |
665 | pypath = Path(sys.executable).resolve() | |
666 | lines.append( | |
667 | f"'{prog}' was detected on your system at '{which}', " | |
668 | f"but the Python package '{pkg_name}' was not found by " | |
669 | f"this Python interpreter ('{pypath}'). " | |
670 | f"Typically this means that '{prog}' has been installed " | |
671 | "against a different Python interpreter on your system." | |
672 | ) | |
673 | else: | |
674 | lines.append( | |
675 | f"'{prog}' was detected on your system at '{which}', " | |
676 | "but the build is using an isolated virtual environment." | |
677 | ) | |
678 | bad = True | |
679 | ||
680 | lines = [f" • {line}" for line in lines] | |
681 | if bad: | |
682 | lines.insert(0, f"Could not provide build dependency '{dep_spec}':") | |
683 | else: | |
684 | lines.insert(0, f"'{dep_spec}' not found:") | |
685 | return os.linesep.join(lines), bad | |
686 | ||
687 | ||
c5538eed JS |
688 | def pip_install( |
689 | args: Sequence[str], | |
690 | online: bool = False, | |
691 | wheels_dir: Optional[Union[str, Path]] = None, | |
692 | ) -> None: | |
693 | """ | |
694 | Use pip to install a package or package(s) as specified in @args. | |
695 | """ | |
696 | loud = bool( | |
697 | os.environ.get("DEBUG") | |
698 | or os.environ.get("GITLAB_CI") | |
699 | or os.environ.get("V") | |
700 | ) | |
701 | ||
702 | full_args = [ | |
703 | sys.executable, | |
704 | "-m", | |
705 | "pip", | |
706 | "install", | |
707 | "--disable-pip-version-check", | |
708 | "-v" if loud else "-q", | |
709 | ] | |
710 | if not online: | |
711 | full_args += ["--no-index"] | |
712 | if wheels_dir: | |
713 | full_args += ["--find-links", f"file://{str(wheels_dir)}"] | |
714 | full_args += list(args) | |
715 | subprocess.run( | |
716 | full_args, | |
717 | check=True, | |
718 | ) | |
719 | ||
720 | ||
4695a22e | 721 | def _do_ensure( |
c5538eed JS |
722 | dep_specs: Sequence[str], |
723 | online: bool = False, | |
724 | wheels_dir: Optional[Union[str, Path]] = None, | |
d37c21b5 PB |
725 | prog: Optional[str] = None, |
726 | ) -> Optional[Tuple[str, bool]]: | |
c5538eed JS |
727 | """ |
728 | Use pip to ensure we have the package specified by @dep_specs. | |
729 | ||
730 | If the package is already installed, do nothing. If online and | |
731 | wheels_dir are both provided, prefer packages found in wheels_dir | |
732 | first before connecting to PyPI. | |
733 | ||
734 | :param dep_specs: | |
735 | PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. | |
736 | :param online: If True, fall back to PyPI. | |
737 | :param wheels_dir: If specified, search this path for packages. | |
738 | """ | |
c673f3d0 PB |
739 | absent = [] |
740 | present = [] | |
741 | for spec in dep_specs: | |
742 | matcher = distlib.version.LegacyMatcher(spec) | |
743 | ver = _get_version(matcher.name) | |
744 | if ver is None or not matcher.match( | |
745 | distlib.version.LegacyVersion(ver) | |
746 | ): | |
747 | absent.append(spec) | |
748 | else: | |
749 | logger.info("found %s %s", matcher.name, ver) | |
750 | present.append(matcher.name) | |
92834894 JS |
751 | |
752 | if present: | |
753 | generate_console_scripts(present) | |
c5538eed JS |
754 | |
755 | if absent: | |
d37c21b5 PB |
756 | if online or wheels_dir: |
757 | # Some packages are missing or aren't a suitable version, | |
758 | # install a suitable (possibly vendored) package. | |
759 | print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr) | |
760 | try: | |
761 | pip_install(args=absent, online=online, wheels_dir=wheels_dir) | |
762 | return None | |
763 | except subprocess.CalledProcessError: | |
764 | pass | |
765 | ||
766 | return diagnose( | |
767 | absent[0], | |
768 | online, | |
769 | wheels_dir, | |
770 | prog if absent[0] == dep_specs[0] else None, | |
771 | ) | |
772 | ||
773 | return None | |
c5538eed JS |
774 | |
775 | ||
4695a22e JS |
776 | def ensure( |
777 | dep_specs: Sequence[str], | |
778 | online: bool = False, | |
779 | wheels_dir: Optional[Union[str, Path]] = None, | |
780 | prog: Optional[str] = None, | |
781 | ) -> None: | |
782 | """ | |
783 | Use pip to ensure we have the package specified by @dep_specs. | |
784 | ||
785 | If the package is already installed, do nothing. If online and | |
786 | wheels_dir are both provided, prefer packages found in wheels_dir | |
787 | first before connecting to PyPI. | |
788 | ||
789 | :param dep_specs: | |
790 | PEP 508 dependency specifications. e.g. ['meson>=0.61.5']. | |
791 | :param online: If True, fall back to PyPI. | |
792 | :param wheels_dir: If specified, search this path for packages. | |
793 | :param prog: | |
794 | If specified, use this program name for error diagnostics that will | |
795 | be presented to the user. e.g., 'sphinx-build' can be used as a | |
796 | bellwether for the presence of 'sphinx'. | |
797 | """ | |
798 | print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr) | |
68ea6d17 JS |
799 | |
800 | if not HAVE_DISTLIB: | |
801 | raise Ouch("a usable distlib could not be found, please install it") | |
802 | ||
d37c21b5 PB |
803 | result = _do_ensure(dep_specs, online, wheels_dir, prog) |
804 | if result: | |
4695a22e | 805 | # Well, that's not good. |
d37c21b5 PB |
806 | if result[1]: |
807 | raise Ouch(result[0]) | |
808 | raise SystemExit(f"\n{result[0]}\n\n") | |
4695a22e JS |
809 | |
810 | ||
f1ad527f JS |
811 | def post_venv_setup() -> None: |
812 | """ | |
813 | This is intended to be run *inside the venv* after it is created. | |
814 | """ | |
815 | logger.debug("post_venv_setup()") | |
c8049626 JS |
816 | # Test for a broken pip (Debian 10 or derivative?) and fix it if needed |
817 | if not checkpip(): | |
818 | # Finally, generate a 'pip' script so the venv is usable in a normal | |
819 | # way from the CLI. This only happens when we inherited pip from a | |
820 | # parent/system-site and haven't run ensurepip in some way. | |
821 | generate_console_scripts(["pip"]) | |
f1ad527f JS |
822 | |
823 | ||
dd84028f JS |
824 | def _add_create_subcommand(subparsers: Any) -> None: |
825 | subparser = subparsers.add_parser("create", help="create a venv") | |
826 | subparser.add_argument( | |
827 | "target", | |
828 | type=str, | |
829 | action="store", | |
830 | help="Target directory to install virtual environment into.", | |
831 | ) | |
832 | ||
833 | ||
f1ad527f JS |
834 | def _add_post_init_subcommand(subparsers: Any) -> None: |
835 | subparsers.add_parser("post_init", help="post-venv initialization") | |
836 | ||
837 | ||
c5538eed JS |
838 | def _add_ensure_subcommand(subparsers: Any) -> None: |
839 | subparser = subparsers.add_parser( | |
840 | "ensure", help="Ensure that the specified package is installed." | |
841 | ) | |
842 | subparser.add_argument( | |
843 | "--online", | |
844 | action="store_true", | |
845 | help="Install packages from PyPI, if necessary.", | |
846 | ) | |
847 | subparser.add_argument( | |
848 | "--dir", | |
849 | type=str, | |
850 | action="store", | |
851 | help="Path to vendored packages where we may install from.", | |
852 | ) | |
4695a22e JS |
853 | subparser.add_argument( |
854 | "--diagnose", | |
855 | type=str, | |
856 | action="store", | |
857 | help=( | |
858 | "Name of a shell utility to use for " | |
859 | "diagnostics if this command fails." | |
860 | ), | |
861 | ) | |
c5538eed JS |
862 | subparser.add_argument( |
863 | "dep_specs", | |
864 | type=str, | |
865 | action="store", | |
866 | help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'", | |
867 | nargs="+", | |
868 | ) | |
869 | ||
870 | ||
dd84028f JS |
871 | def main() -> int: |
872 | """CLI interface to make_qemu_venv. See module docstring.""" | |
873 | if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"): | |
874 | # You're welcome. | |
875 | logging.basicConfig(level=logging.DEBUG) | |
c5538eed JS |
876 | else: |
877 | if os.environ.get("V"): | |
878 | logging.basicConfig(level=logging.INFO) | |
879 | ||
dd84028f JS |
880 | parser = argparse.ArgumentParser( |
881 | prog="mkvenv", | |
882 | description="QEMU pyvenv bootstrapping utility", | |
883 | ) | |
884 | subparsers = parser.add_subparsers( | |
885 | title="Commands", | |
886 | dest="command", | |
02312f1a | 887 | required=True, |
dd84028f JS |
888 | metavar="command", |
889 | help="Description", | |
890 | ) | |
891 | ||
892 | _add_create_subcommand(subparsers) | |
f1ad527f | 893 | _add_post_init_subcommand(subparsers) |
c5538eed | 894 | _add_ensure_subcommand(subparsers) |
dd84028f JS |
895 | |
896 | args = parser.parse_args() | |
897 | try: | |
898 | if args.command == "create": | |
899 | make_venv( | |
900 | args.target, | |
901 | system_site_packages=True, | |
902 | clear=True, | |
903 | ) | |
f1ad527f JS |
904 | if args.command == "post_init": |
905 | post_venv_setup() | |
c5538eed JS |
906 | if args.command == "ensure": |
907 | ensure( | |
908 | dep_specs=args.dep_specs, | |
909 | online=args.online, | |
910 | wheels_dir=args.dir, | |
4695a22e | 911 | prog=args.diagnose, |
c5538eed | 912 | ) |
dd84028f JS |
913 | logger.debug("mkvenv.py %s: exiting", args.command) |
914 | except Ouch as exc: | |
915 | print("\n*** Ouch! ***\n", file=sys.stderr) | |
916 | print(str(exc), "\n\n", file=sys.stderr) | |
917 | return 1 | |
918 | except SystemExit: | |
919 | raise | |
920 | except: # pylint: disable=bare-except | |
921 | logger.exception("mkvenv did not complete successfully:") | |
922 | return 2 | |
923 | return 0 | |
924 | ||
925 | ||
926 | if __name__ == "__main__": | |
927 | sys.exit(main()) |