]> git.proxmox.com Git - mirror_qemu.git/blame - python/scripts/mkvenv.py
mkvenv: pass first missing package to diagnose()
[mirror_qemu.git] / python / scripts / mkvenv.py
CommitLineData
dd84028f
JS
1"""
2mkvenv - QEMU pyvenv bootstrapping utility
3
4usage: mkvenv [-h] command ...
5
6QEMU pyvenv bootstrapping utility
7
8options:
9 -h, --help show this help message and exit
10
11Commands:
12 command Description
13 create create a venv
f1ad527f
JS
14 post_init
15 post-venv initialization
c5538eed 16 ensure Ensure that the specified package is installed.
dd84028f
JS
17
18--------------------------------------------------
19
20usage: mkvenv create [-h] target
21
22positional arguments:
23 target Target directory to install virtual environment into.
24
25options:
26 -h, --help show this help message and exit
27
c5538eed
JS
28--------------------------------------------------
29
f1ad527f
JS
30usage: mkvenv post_init [-h]
31
32options:
33 -h, --help show this help message and exit
34
35--------------------------------------------------
36
c5538eed
JS
37usage: mkvenv ensure [-h] [--online] [--dir DIR] dep_spec...
38
39positional arguments:
40 dep_spec PEP 508 Dependency specification, e.g. 'meson>=0.61.5'
41
42options:
43 -h, --help show this help message and exit
44 --online Install packages from PyPI, if necessary.
45 --dir DIR Path to vendored packages where we may install from.
46
dd84028f
JS
47"""
48
49# Copyright (C) 2022-2023 Red Hat, Inc.
50#
51# Authors:
52# John Snow <jsnow@redhat.com>
53# Paolo Bonzini <pbonzini@redhat.com>
54#
55# This work is licensed under the terms of the GNU GPL, version 2 or
56# later. See the COPYING file in the top-level directory.
57
58import argparse
a9dbde71 59from importlib.util import find_spec
dd84028f
JS
60import logging
61import os
62from pathlib import Path
4695a22e
JS
63import re
64import shutil
dee01b82 65import site
dd84028f
JS
66import subprocess
67import sys
dee01b82 68import sysconfig
dd84028f 69from types import SimpleNamespace
c5538eed
JS
70from typing import (
71 Any,
92834894 72 Iterator,
c5538eed
JS
73 Optional,
74 Sequence,
4695a22e 75 Tuple,
c5538eed
JS
76 Union,
77)
dd84028f 78import venv
c5538eed 79
dd84028f 80
68ea6d17
JS
81# Try to load distlib, with a fallback to pip's vendored version.
82# HAVE_DISTLIB is checked below, just-in-time, so that mkvenv does not fail
83# outside the venv or before a potential call to ensurepip in checkpip().
84HAVE_DISTLIB = True
85try:
68ea6d17
JS
86 import distlib.scripts
87 import distlib.version
88except ImportError:
89 try:
90 # Reach into pip's cookie jar. pylint and flake8 don't understand
91 # that these imports will be used via distlib.xxx.
92 from pip._vendor import distlib
68ea6d17
JS
93 import pip._vendor.distlib.scripts # noqa, pylint: disable=unused-import
94 import pip._vendor.distlib.version # noqa, pylint: disable=unused-import
95 except ImportError:
96 HAVE_DISTLIB = False
dd84028f
JS
97
98# Do not add any mandatory dependencies from outside the stdlib:
99# This script *must* be usable standalone!
100
101DirType = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"]
102logger = logging.getLogger("mkvenv")
103
104
dee01b82
JS
105def inside_a_venv() -> bool:
106 """Returns True if it is executed inside of a virtual environment."""
107 return sys.prefix != sys.base_prefix
108
109
dd84028f
JS
110class Ouch(RuntimeError):
111 """An Exception class we can't confuse with a builtin."""
112
113
114class QemuEnvBuilder(venv.EnvBuilder):
115 """
116 An extension of venv.EnvBuilder for building QEMU's configure-time venv.
117
dee01b82
JS
118 The primary difference is that it emulates a "nested" virtual
119 environment when invoked from inside of an existing virtual
f1ad527f
JS
120 environment by including packages from the parent. Also,
121 "ensurepip" is replaced if possible with just recreating pip's
122 console_scripts inside the virtual environment.
dd84028f
JS
123
124 Parameters for base class init:
125 - system_site_packages: bool = False
126 - clear: bool = False
127 - symlinks: bool = False
128 - upgrade: bool = False
129 - with_pip: bool = False
130 - prompt: Optional[str] = None
131 - upgrade_deps: bool = False (Since 3.9)
132 """
133
134 def __init__(self, *args: Any, **kwargs: Any) -> None:
135 logger.debug("QemuEnvBuilder.__init__(...)")
a9dbde71 136
dee01b82
JS
137 # For nested venv emulation:
138 self.use_parent_packages = False
139 if inside_a_venv():
140 # Include parent packages only if we're in a venv and
141 # system_site_packages was True.
142 self.use_parent_packages = kwargs.pop(
143 "system_site_packages", False
144 )
145 # Include system_site_packages only when the parent,
146 # The venv we are currently in, also does so.
147 kwargs["system_site_packages"] = sys.base_prefix in site.PREFIXES
148
f1ad527f
JS
149 # ensurepip is slow: venv creation can be very fast for cases where
150 # we allow the use of system_site_packages. Therefore, ensurepip is
151 # replaced with our own script generation once the virtual environment
152 # is setup.
153 self.want_pip = kwargs.get("with_pip", False)
154 if self.want_pip:
155 if (
156 kwargs.get("system_site_packages", False)
157 and not need_ensurepip()
158 ):
159 kwargs["with_pip"] = False
160 else:
c8049626 161 check_ensurepip(suggest_remedy=True)
a9dbde71 162
dd84028f
JS
163 super().__init__(*args, **kwargs)
164
165 # Make the context available post-creation:
166 self._context: Optional[SimpleNamespace] = None
167
dee01b82
JS
168 def get_parent_libpath(self) -> Optional[str]:
169 """Return the libpath of the parent venv, if applicable."""
170 if self.use_parent_packages:
171 return sysconfig.get_path("purelib")
172 return None
173
174 @staticmethod
175 def compute_venv_libpath(context: SimpleNamespace) -> str:
176 """
177 Compatibility wrapper for context.lib_path for Python < 3.12
178 """
179 # Python 3.12+, not strictly necessary because it's documented
180 # to be the same as 3.10 code below:
181 if sys.version_info >= (3, 12):
182 return context.lib_path
183
184 # Python 3.10+
185 if "venv" in sysconfig.get_scheme_names():
186 lib_path = sysconfig.get_path(
187 "purelib", scheme="venv", vars={"base": context.env_dir}
188 )
189 assert lib_path is not None
190 return lib_path
191
192 # For Python <= 3.9 we need to hardcode this. Fortunately the
193 # code below was the same in Python 3.6-3.10, so there is only
194 # one case.
195 if sys.platform == "win32":
196 return os.path.join(context.env_dir, "Lib", "site-packages")
197 return os.path.join(
198 context.env_dir,
199 "lib",
200 "python%d.%d" % sys.version_info[:2],
201 "site-packages",
202 )
203
dd84028f
JS
204 def ensure_directories(self, env_dir: DirType) -> SimpleNamespace:
205 logger.debug("ensure_directories(env_dir=%s)", env_dir)
206 self._context = super().ensure_directories(env_dir)
207 return self._context
208
dee01b82
JS
209 def create(self, env_dir: DirType) -> None:
210 logger.debug("create(env_dir=%s)", env_dir)
211 super().create(env_dir)
212 assert self._context is not None
213 self.post_post_setup(self._context)
214
215 def post_post_setup(self, context: SimpleNamespace) -> None:
216 """
217 The final, final hook. Enter the venv and run commands inside of it.
218 """
219 if self.use_parent_packages:
220 # We're inside of a venv and we want to include the parent
221 # venv's packages.
222 parent_libpath = self.get_parent_libpath()
223 assert parent_libpath is not None
224 logger.debug("parent_libpath: %s", parent_libpath)
225
226 our_libpath = self.compute_venv_libpath(context)
227 logger.debug("our_libpath: %s", our_libpath)
228
229 pth_file = os.path.join(our_libpath, "nested.pth")
230 with open(pth_file, "w", encoding="UTF-8") as file:
231 file.write(parent_libpath + os.linesep)
232
f1ad527f
JS
233 if self.want_pip:
234 args = [
235 context.env_exe,
236 __file__,
237 "post_init",
238 ]
239 subprocess.run(args, check=True)
240
dd84028f
JS
241 def get_value(self, field: str) -> str:
242 """
243 Get a string value from the context namespace after a call to build.
244
245 For valid field names, see:
246 https://docs.python.org/3/library/venv.html#venv.EnvBuilder.ensure_directories
247 """
248 ret = getattr(self._context, field)
249 assert isinstance(ret, str)
250 return ret
251
252
f1ad527f
JS
253def need_ensurepip() -> bool:
254 """
255 Tests for the presence of setuptools and pip.
256
257 :return: `True` if we do not detect both packages.
258 """
259 # Don't try to actually import them, it's fraught with danger:
260 # https://github.com/pypa/setuptools/issues/2993
261 if find_spec("setuptools") and find_spec("pip"):
262 return False
263 return True
264
265
c8049626 266def check_ensurepip(prefix: str = "", suggest_remedy: bool = False) -> None:
a9dbde71
JS
267 """
268 Check that we have ensurepip.
269
270 Raise a fatal exception with a helpful hint if it isn't available.
271 """
272 if not find_spec("ensurepip"):
273 msg = (
274 "Python's ensurepip module is not found.\n"
275 "It's normally part of the Python standard library, "
276 "maybe your distribution packages it separately?\n"
c8049626 277 "(Debian puts ensurepip in its python3-venv package.)\n"
a9dbde71 278 )
c8049626
JS
279 if suggest_remedy:
280 msg += (
281 "Either install ensurepip, or alleviate the need for it in the"
282 " first place by installing pip and setuptools for "
283 f"'{sys.executable}'.\n"
284 )
285 raise Ouch(prefix + msg)
a9dbde71
JS
286
287 # ensurepip uses pyexpat, which can also go missing on us:
288 if not find_spec("pyexpat"):
289 msg = (
290 "Python's pyexpat module is not found.\n"
291 "It's normally part of the Python standard library, "
292 "maybe your distribution packages it separately?\n"
c8049626 293 "(NetBSD's pkgsrc debundles this to e.g. 'py310-expat'.)\n"
a9dbde71 294 )
c8049626
JS
295 if suggest_remedy:
296 msg += (
297 "Either install pyexpat, or alleviate the need for it in the "
298 "first place by installing pip and setuptools for "
299 f"'{sys.executable}'.\n"
300 )
301 raise Ouch(prefix + msg)
a9dbde71
JS
302
303
dd84028f
JS
304def make_venv( # pylint: disable=too-many-arguments
305 env_dir: Union[str, Path],
306 system_site_packages: bool = False,
307 clear: bool = True,
308 symlinks: Optional[bool] = None,
309 with_pip: bool = True,
310) -> None:
311 """
312 Create a venv using `QemuEnvBuilder`.
313
314 This is analogous to the `venv.create` module-level convenience
315 function that is part of the Python stdblib, except it uses
316 `QemuEnvBuilder` instead.
317
318 :param env_dir: The directory to create/install to.
319 :param system_site_packages:
320 Allow inheriting packages from the system installation.
321 :param clear: When True, fully remove any prior venv and files.
322 :param symlinks:
323 Whether to use symlinks to the target interpreter or not. If
324 left unspecified, it will use symlinks except on Windows to
325 match behavior with the "venv" CLI tool.
326 :param with_pip:
327 Whether to install "pip" binaries or not.
328 """
329 logger.debug(
330 "%s: make_venv(env_dir=%s, system_site_packages=%s, "
331 "clear=%s, symlinks=%s, with_pip=%s)",
332 __file__,
333 str(env_dir),
334 system_site_packages,
335 clear,
336 symlinks,
337 with_pip,
338 )
339
340 if symlinks is None:
341 # Default behavior of standard venv CLI
342 symlinks = os.name != "nt"
343
344 builder = QemuEnvBuilder(
345 system_site_packages=system_site_packages,
346 clear=clear,
347 symlinks=symlinks,
348 with_pip=with_pip,
349 )
350
351 style = "non-isolated" if builder.system_site_packages else "isolated"
dee01b82
JS
352 nested = ""
353 if builder.use_parent_packages:
354 nested = f"(with packages from '{builder.get_parent_libpath()}') "
dd84028f
JS
355 print(
356 f"mkvenv: Creating {style} virtual environment"
dee01b82 357 f" {nested}at '{str(env_dir)}'",
dd84028f
JS
358 file=sys.stderr,
359 )
360
361 try:
362 logger.debug("Invoking builder.create()")
363 try:
364 builder.create(str(env_dir))
365 except SystemExit as exc:
366 # Some versions of the venv module raise SystemExit; *nasty*!
367 # We want the exception that prompted it. It might be a subprocess
368 # error that has output we *really* want to see.
369 logger.debug("Intercepted SystemExit from EnvBuilder.create()")
370 raise exc.__cause__ or exc.__context__ or exc
371 logger.debug("builder.create() finished")
372 except subprocess.CalledProcessError as exc:
373 logger.error("mkvenv subprocess failed:")
374 logger.error("cmd: %s", exc.cmd)
375 logger.error("returncode: %d", exc.returncode)
376
377 def _stringify(data: Union[str, bytes]) -> str:
378 if isinstance(data, bytes):
379 return data.decode()
380 return data
381
382 lines = []
383 if exc.stdout:
384 lines.append("========== stdout ==========")
385 lines.append(_stringify(exc.stdout))
386 lines.append("============================")
387 if exc.stderr:
388 lines.append("========== stderr ==========")
389 lines.append(_stringify(exc.stderr))
390 lines.append("============================")
391 if lines:
392 logger.error(os.linesep.join(lines))
393
394 raise Ouch("VENV creation subprocess failed.") from exc
395
396 # print the python executable to stdout for configure.
397 print(builder.get_value("env_exe"))
398
399
92834894
JS
400def _gen_importlib(packages: Sequence[str]) -> Iterator[str]:
401 # pylint: disable=import-outside-toplevel
402 # pylint: disable=no-name-in-module
403 # pylint: disable=import-error
404 try:
405 # First preference: Python 3.8+ stdlib
406 from importlib.metadata import ( # type: ignore
407 PackageNotFoundError,
408 distribution,
409 )
410 except ImportError as exc:
411 logger.debug("%s", str(exc))
412 # Second preference: Commonly available PyPI backport
413 from importlib_metadata import ( # type: ignore
414 PackageNotFoundError,
415 distribution,
416 )
417
418 def _generator() -> Iterator[str]:
419 for package in packages:
420 try:
421 entry_points = distribution(package).entry_points
422 except PackageNotFoundError:
423 continue
424
425 # The EntryPoints type is only available in 3.10+,
426 # treat this as a vanilla list and filter it ourselves.
427 entry_points = filter(
428 lambda ep: ep.group == "console_scripts", entry_points
429 )
430
431 for entry_point in entry_points:
432 yield f"{entry_point.name} = {entry_point.value}"
433
434 return _generator()
435
436
437def _gen_pkg_resources(packages: Sequence[str]) -> Iterator[str]:
438 # pylint: disable=import-outside-toplevel
439 # Bundled with setuptools; has a good chance of being available.
440 import pkg_resources
441
442 def _generator() -> Iterator[str]:
443 for package in packages:
444 try:
445 eps = pkg_resources.get_entry_map(package, "console_scripts")
446 except pkg_resources.DistributionNotFound:
447 continue
448
449 for entry_point in eps.values():
450 yield str(entry_point)
451
452 return _generator()
453
454
455def generate_console_scripts(
456 packages: Sequence[str],
457 python_path: Optional[str] = None,
458 bin_path: Optional[str] = None,
459) -> None:
460 """
461 Generate script shims for console_script entry points in @packages.
462 """
463 if python_path is None:
464 python_path = sys.executable
465 if bin_path is None:
466 bin_path = sysconfig.get_path("scripts")
467 assert bin_path is not None
468
469 logger.debug(
470 "generate_console_scripts(packages=%s, python_path=%s, bin_path=%s)",
471 packages,
472 python_path,
473 bin_path,
474 )
475
476 if not packages:
477 return
478
479 def _get_entry_points() -> Iterator[str]:
480 """Python 3.7 compatibility shim for iterating entry points."""
481 # Python 3.8+, or Python 3.7 with importlib_metadata installed.
482 try:
483 return _gen_importlib(packages)
484 except ImportError as exc:
485 logger.debug("%s", str(exc))
486
487 # Python 3.7 with setuptools installed.
488 try:
489 return _gen_pkg_resources(packages)
490 except ImportError as exc:
491 logger.debug("%s", str(exc))
492 raise Ouch(
493 "Neither importlib.metadata nor pkg_resources found, "
494 "can't generate console script shims.\n"
495 "Use Python 3.8+, or install importlib-metadata or setuptools."
496 ) from exc
497
498 maker = distlib.scripts.ScriptMaker(None, bin_path)
499 maker.variants = {""}
500 maker.clobber = False
501
502 for entry_point in _get_entry_points():
503 for filename in maker.make(entry_point):
504 logger.debug("wrote console_script '%s'", filename)
505
506
c8049626
JS
507def checkpip() -> bool:
508 """
509 Debian10 has a pip that's broken when used inside of a virtual environment.
510
511 We try to detect and correct that case here.
512 """
513 try:
514 # pylint: disable=import-outside-toplevel,unused-import,import-error
515 # pylint: disable=redefined-outer-name
516 import pip._internal # type: ignore # noqa: F401
517
518 logger.debug("pip appears to be working correctly.")
519 return False
520 except ModuleNotFoundError as exc:
521 if exc.name == "pip._internal":
522 # Uh, fair enough. They did say "internal".
523 # Let's just assume it's fine.
524 return False
525 logger.warning("pip appears to be malfunctioning: %s", str(exc))
526
527 check_ensurepip("pip appears to be non-functional, and ")
528
529 logger.debug("Attempting to repair pip ...")
530 subprocess.run(
531 (sys.executable, "-m", "ensurepip"),
532 stdout=subprocess.DEVNULL,
533 check=True,
534 )
535 logger.debug("Pip is now (hopefully) repaired!")
536 return True
537
538
4695a22e
JS
539def pkgname_from_depspec(dep_spec: str) -> str:
540 """
541 Parse package name out of a PEP-508 depspec.
542
543 See https://peps.python.org/pep-0508/#names
544 """
545 match = re.match(
546 r"^([A-Z0-9]([A-Z0-9._-]*[A-Z0-9])?)", dep_spec, re.IGNORECASE
547 )
548 if not match:
549 raise ValueError(
550 f"dep_spec '{dep_spec}'"
551 " does not appear to contain a valid package name"
552 )
553 return match.group(0)
554
555
c673f3d0
PB
556def _get_version_importlib(package: str) -> Optional[str]:
557 # pylint: disable=import-outside-toplevel
558 # pylint: disable=no-name-in-module
559 # pylint: disable=import-error
560 try:
561 # First preference: Python 3.8+ stdlib
562 from importlib.metadata import ( # type: ignore
563 PackageNotFoundError,
564 distribution,
565 )
566 except ImportError as exc:
567 logger.debug("%s", str(exc))
568 # Second preference: Commonly available PyPI backport
569 from importlib_metadata import ( # type: ignore
570 PackageNotFoundError,
571 distribution,
572 )
573
574 try:
575 return str(distribution(package).version)
576 except PackageNotFoundError:
577 return None
578
579
580def _get_version_pkg_resources(package: str) -> Optional[str]:
581 # pylint: disable=import-outside-toplevel
582 # Bundled with setuptools; has a good chance of being available.
583 import pkg_resources
584
585 try:
586 return str(pkg_resources.get_distribution(package).version)
587 except pkg_resources.DistributionNotFound:
588 return None
589
590
591def _get_version(package: str) -> Optional[str]:
592 try:
593 return _get_version_importlib(package)
594 except ImportError as exc:
595 logger.debug("%s", str(exc))
596
597 try:
598 return _get_version_pkg_resources(package)
599 except ImportError as exc:
600 logger.debug("%s", str(exc))
601 raise Ouch(
602 "Neither importlib.metadata nor pkg_resources found. "
603 "Use Python 3.8+, or install importlib-metadata or setuptools."
604 ) from exc
605
606
4695a22e
JS
607def diagnose(
608 dep_spec: str,
609 online: bool,
610 wheels_dir: Optional[Union[str, Path]],
611 prog: Optional[str],
612) -> Tuple[str, bool]:
613 """
614 Offer a summary to the user as to why a package failed to be installed.
615
616 :param dep_spec: The package we tried to ensure, e.g. 'meson>=0.61.5'
617 :param online: Did we allow PyPI access?
618 :param prog:
619 Optionally, a shell program name that can be used as a
620 bellwether to detect if this program is installed elsewhere on
621 the system. This is used to offer advice when a program is
622 detected for a different python version.
623 :param wheels_dir:
624 Optionally, a directory that was searched for vendored packages.
625 """
626 # pylint: disable=too-many-branches
627
628 # Some errors are not particularly serious
629 bad = False
630
631 pkg_name = pkgname_from_depspec(dep_spec)
c673f3d0 632 pkg_version = _get_version(pkg_name)
4695a22e
JS
633
634 lines = []
635
636 if pkg_version:
637 lines.append(
638 f"Python package '{pkg_name}' version '{pkg_version}' was found,"
639 " but isn't suitable."
640 )
4695a22e
JS
641 else:
642 lines.append(
c673f3d0 643 f"Python package '{pkg_name}' was not found nor installed."
4695a22e
JS
644 )
645
646 if wheels_dir:
647 lines.append(
648 "No suitable version found in, or failed to install from"
649 f" '{wheels_dir}'."
650 )
651 bad = True
652
653 if online:
654 lines.append("A suitable version could not be obtained from PyPI.")
655 bad = True
656 else:
657 lines.append(
658 "mkvenv was configured to operate offline and did not check PyPI."
659 )
660
661 if prog and not pkg_version:
662 which = shutil.which(prog)
663 if which:
664 if sys.base_prefix in site.PREFIXES:
665 pypath = Path(sys.executable).resolve()
666 lines.append(
667 f"'{prog}' was detected on your system at '{which}', "
668 f"but the Python package '{pkg_name}' was not found by "
669 f"this Python interpreter ('{pypath}'). "
670 f"Typically this means that '{prog}' has been installed "
671 "against a different Python interpreter on your system."
672 )
673 else:
674 lines.append(
675 f"'{prog}' was detected on your system at '{which}', "
676 "but the build is using an isolated virtual environment."
677 )
678 bad = True
679
680 lines = [f" • {line}" for line in lines]
681 if bad:
682 lines.insert(0, f"Could not provide build dependency '{dep_spec}':")
683 else:
684 lines.insert(0, f"'{dep_spec}' not found:")
685 return os.linesep.join(lines), bad
686
687
c5538eed
JS
688def pip_install(
689 args: Sequence[str],
690 online: bool = False,
691 wheels_dir: Optional[Union[str, Path]] = None,
692) -> None:
693 """
694 Use pip to install a package or package(s) as specified in @args.
695 """
696 loud = bool(
697 os.environ.get("DEBUG")
698 or os.environ.get("GITLAB_CI")
699 or os.environ.get("V")
700 )
701
702 full_args = [
703 sys.executable,
704 "-m",
705 "pip",
706 "install",
707 "--disable-pip-version-check",
708 "-v" if loud else "-q",
709 ]
710 if not online:
711 full_args += ["--no-index"]
712 if wheels_dir:
713 full_args += ["--find-links", f"file://{str(wheels_dir)}"]
714 full_args += list(args)
715 subprocess.run(
716 full_args,
717 check=True,
718 )
719
720
4695a22e 721def _do_ensure(
c5538eed
JS
722 dep_specs: Sequence[str],
723 online: bool = False,
724 wheels_dir: Optional[Union[str, Path]] = None,
d37c21b5
PB
725 prog: Optional[str] = None,
726) -> Optional[Tuple[str, bool]]:
c5538eed
JS
727 """
728 Use pip to ensure we have the package specified by @dep_specs.
729
730 If the package is already installed, do nothing. If online and
731 wheels_dir are both provided, prefer packages found in wheels_dir
732 first before connecting to PyPI.
733
734 :param dep_specs:
735 PEP 508 dependency specifications. e.g. ['meson>=0.61.5'].
736 :param online: If True, fall back to PyPI.
737 :param wheels_dir: If specified, search this path for packages.
738 """
c673f3d0
PB
739 absent = []
740 present = []
741 for spec in dep_specs:
742 matcher = distlib.version.LegacyMatcher(spec)
743 ver = _get_version(matcher.name)
744 if ver is None or not matcher.match(
745 distlib.version.LegacyVersion(ver)
746 ):
747 absent.append(spec)
748 else:
749 logger.info("found %s %s", matcher.name, ver)
750 present.append(matcher.name)
92834894
JS
751
752 if present:
753 generate_console_scripts(present)
c5538eed
JS
754
755 if absent:
d37c21b5
PB
756 if online or wheels_dir:
757 # Some packages are missing or aren't a suitable version,
758 # install a suitable (possibly vendored) package.
759 print(f"mkvenv: installing {', '.join(absent)}", file=sys.stderr)
760 try:
761 pip_install(args=absent, online=online, wheels_dir=wheels_dir)
762 return None
763 except subprocess.CalledProcessError:
764 pass
765
766 return diagnose(
767 absent[0],
768 online,
769 wheels_dir,
770 prog if absent[0] == dep_specs[0] else None,
771 )
772
773 return None
c5538eed
JS
774
775
4695a22e
JS
776def ensure(
777 dep_specs: Sequence[str],
778 online: bool = False,
779 wheels_dir: Optional[Union[str, Path]] = None,
780 prog: Optional[str] = None,
781) -> None:
782 """
783 Use pip to ensure we have the package specified by @dep_specs.
784
785 If the package is already installed, do nothing. If online and
786 wheels_dir are both provided, prefer packages found in wheels_dir
787 first before connecting to PyPI.
788
789 :param dep_specs:
790 PEP 508 dependency specifications. e.g. ['meson>=0.61.5'].
791 :param online: If True, fall back to PyPI.
792 :param wheels_dir: If specified, search this path for packages.
793 :param prog:
794 If specified, use this program name for error diagnostics that will
795 be presented to the user. e.g., 'sphinx-build' can be used as a
796 bellwether for the presence of 'sphinx'.
797 """
798 print(f"mkvenv: checking for {', '.join(dep_specs)}", file=sys.stderr)
68ea6d17
JS
799
800 if not HAVE_DISTLIB:
801 raise Ouch("a usable distlib could not be found, please install it")
802
d37c21b5
PB
803 result = _do_ensure(dep_specs, online, wheels_dir, prog)
804 if result:
4695a22e 805 # Well, that's not good.
d37c21b5
PB
806 if result[1]:
807 raise Ouch(result[0])
808 raise SystemExit(f"\n{result[0]}\n\n")
4695a22e
JS
809
810
f1ad527f
JS
811def post_venv_setup() -> None:
812 """
813 This is intended to be run *inside the venv* after it is created.
814 """
815 logger.debug("post_venv_setup()")
c8049626
JS
816 # Test for a broken pip (Debian 10 or derivative?) and fix it if needed
817 if not checkpip():
818 # Finally, generate a 'pip' script so the venv is usable in a normal
819 # way from the CLI. This only happens when we inherited pip from a
820 # parent/system-site and haven't run ensurepip in some way.
821 generate_console_scripts(["pip"])
f1ad527f
JS
822
823
dd84028f
JS
824def _add_create_subcommand(subparsers: Any) -> None:
825 subparser = subparsers.add_parser("create", help="create a venv")
826 subparser.add_argument(
827 "target",
828 type=str,
829 action="store",
830 help="Target directory to install virtual environment into.",
831 )
832
833
f1ad527f
JS
834def _add_post_init_subcommand(subparsers: Any) -> None:
835 subparsers.add_parser("post_init", help="post-venv initialization")
836
837
c5538eed
JS
838def _add_ensure_subcommand(subparsers: Any) -> None:
839 subparser = subparsers.add_parser(
840 "ensure", help="Ensure that the specified package is installed."
841 )
842 subparser.add_argument(
843 "--online",
844 action="store_true",
845 help="Install packages from PyPI, if necessary.",
846 )
847 subparser.add_argument(
848 "--dir",
849 type=str,
850 action="store",
851 help="Path to vendored packages where we may install from.",
852 )
4695a22e
JS
853 subparser.add_argument(
854 "--diagnose",
855 type=str,
856 action="store",
857 help=(
858 "Name of a shell utility to use for "
859 "diagnostics if this command fails."
860 ),
861 )
c5538eed
JS
862 subparser.add_argument(
863 "dep_specs",
864 type=str,
865 action="store",
866 help="PEP 508 Dependency specification, e.g. 'meson>=0.61.5'",
867 nargs="+",
868 )
869
870
dd84028f
JS
871def main() -> int:
872 """CLI interface to make_qemu_venv. See module docstring."""
873 if os.environ.get("DEBUG") or os.environ.get("GITLAB_CI"):
874 # You're welcome.
875 logging.basicConfig(level=logging.DEBUG)
c5538eed
JS
876 else:
877 if os.environ.get("V"):
878 logging.basicConfig(level=logging.INFO)
879
dd84028f
JS
880 parser = argparse.ArgumentParser(
881 prog="mkvenv",
882 description="QEMU pyvenv bootstrapping utility",
883 )
884 subparsers = parser.add_subparsers(
885 title="Commands",
886 dest="command",
02312f1a 887 required=True,
dd84028f
JS
888 metavar="command",
889 help="Description",
890 )
891
892 _add_create_subcommand(subparsers)
f1ad527f 893 _add_post_init_subcommand(subparsers)
c5538eed 894 _add_ensure_subcommand(subparsers)
dd84028f
JS
895
896 args = parser.parse_args()
897 try:
898 if args.command == "create":
899 make_venv(
900 args.target,
901 system_site_packages=True,
902 clear=True,
903 )
f1ad527f
JS
904 if args.command == "post_init":
905 post_venv_setup()
c5538eed
JS
906 if args.command == "ensure":
907 ensure(
908 dep_specs=args.dep_specs,
909 online=args.online,
910 wheels_dir=args.dir,
4695a22e 911 prog=args.diagnose,
c5538eed 912 )
dd84028f
JS
913 logger.debug("mkvenv.py %s: exiting", args.command)
914 except Ouch as exc:
915 print("\n*** Ouch! ***\n", file=sys.stderr)
916 print(str(exc), "\n\n", file=sys.stderr)
917 return 1
918 except SystemExit:
919 raise
920 except: # pylint: disable=bare-except
921 logger.exception("mkvenv did not complete successfully:")
922 return 2
923 return 0
924
925
926if __name__ == "__main__":
927 sys.exit(main())