edk2 build scripts: work around TianoCore#1607 without forcing Python 2

It turns out that forcing python2 for running the edk2 "build" utility is
neither necessary nor sufficient.

Forcing python2 is not sufficient for two reasons:

- QEMU is moving away from python2, with python2 nearing EOL,

- according to my most recent testing, the lacking dependency information
  in the makefiles that are generated by edk2's "build" utility can cause
  parallel build failures even when "build" is executed by python2.

And forcing python2 is not necessary because we can still return to the
original idea of filtering out jobserver-related options from MAKEFLAGS.
So do that.

While at it, cut short edk2's auto-detection of the python3.* minor
version, by setting PYTHON_COMMAND to "python3" (which we expect to be
available wherever we intend to build edk2).

With this patch, the guest UEFI binaries that are used as part of the BIOS
tables test, and the OVMF and ArmVirtQemu platform firmwares, will be
built strictly in a single job, regardless of an outermost "-jN" make
option. Alas, there appears to be no reliable way to build edk2 in an
(outer make, inner make) environment, with a jobserver enabled.

Cc: Eduardo Habkost <ehabkost@redhat.com>
Cc: John Snow <jsnow@redhat.com>
Cc: Philippe Mathieu-Daudé <philmd@redhat.com>
Reported-by: John Snow <jsnow@redhat.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Tested-by: Philippe Mathieu-Daudé <philmd@redhat.com>
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Laszlo Ersek <lersek@redhat.com>
Message-Id: <20190920083808.21399-3-lersek@redhat.com>
Signed-off-by: Philippe Mathieu-Daudé <philmd@redhat.com>

diff --git a/roms/Makefile b/roms/Makefile
index 3ffd13cc7e45e3d6385e5eb714f5ca6cddff708b..da4efa47a9282ee0c044a8c4db9520a125143ddd 100644 (file)
--- a/roms/Makefile
+++ b/roms/Makefile
@@ -141,6 +141,7 @@ build-efi-roms: build-pxe-roms
 #
 edk2-basetools:
        $(MAKE) -C edk2/BaseTools \
+               PYTHON_COMMAND=$${EDK2_PYTHON_COMMAND:-python3} \
                EXTRA_OPTFLAGS='$(EDK2_BASETOOLS_OPTFLAGS)' \
                EXTRA_LDFLAGS='$(EDK2_BASETOOLS_LDFLAGS)'
 

diff --git a/roms/edk2-build.sh b/roms/edk2-build.sh
index 4f46f8a6a217e4e0eefd0878883d2271bb11bc6c..d5391c7637284bd6752e0d047e4a1c910d6532e6 100755 (executable)
--- a/roms/edk2-build.sh
+++ b/roms/edk2-build.sh
@@ -27,8 +27,7 @@ shift $num_args
 
 cd edk2
 
-# Work around <https://bugzilla.tianocore.org/show_bug.cgi?id=1607>.
-export PYTHON_COMMAND=python2
+export PYTHON_COMMAND=${EDK2_PYTHON_COMMAND:-python3}
 
 # Source "edksetup.sh" carefully.
 set +e +u +C
@@ -43,6 +42,7 @@ fi
 # any), for the edk2 "build" utility.
 source ../edk2-funcs.sh
 edk2_toolchain=$(qemu_edk2_get_toolchain "$emulation_target")
+MAKEFLAGS=$(qemu_edk2_quirk_tianocore_1607 "$MAKEFLAGS")
 edk2_thread_count=$(qemu_edk2_get_thread_count "$MAKEFLAGS")
 qemu_edk2_set_cross_env "$emulation_target"
 

diff --git a/roms/edk2-funcs.sh b/roms/edk2-funcs.sh
index a9fae7ee891bc4cadbe1a3cb3f5486d2a5df40c6..3f4485b201f1f6f8cff47a9933da569e7fde07e5 100644 (file)
--- a/roms/edk2-funcs.sh
+++ b/roms/edk2-funcs.sh
@@ -251,3 +251,20 @@ qemu_edk2_get_thread_count()
     printf '1\n'
   fi
 }
+
+
+# Work around <https://bugzilla.tianocore.org/show_bug.cgi?id=1607> by
+# filtering jobserver-related flags out of MAKEFLAGS. Print the result to the
+# standard output.
+#
+# Parameters:
+#   $1: the value of the MAKEFLAGS variable
+qemu_edk2_quirk_tianocore_1607()
+{
+  local makeflags="$1"
+
+  printf %s "$makeflags" \
+  | LC_ALL=C sed --regexp-extended \
+      --expression='s/--jobserver-(auth|fds)=[0-9]+,[0-9]+//' \
+      --expression='s/-j([0-9]+)?//'
+}

diff --git a/tests/uefi-test-tools/Makefile b/tests/uefi-test-tools/Makefile
index 7e0177d7337e831d81c84ef65ced49c4587730f6..1dcddcdbbabf1c7dbea9c233d0ef45cd5ea3325d 100644 (file)
--- a/tests/uefi-test-tools/Makefile
+++ b/tests/uefi-test-tools/Makefile
@@ -100,6 +100,7 @@ Build/bios-tables-test.%.efi: build-edk2-tools
 
 build-edk2-tools:
        $(MAKE) -C $(edk2_dir)/BaseTools \
+               PYTHON_COMMAND=$${EDK2_PYTHON_COMMAND:-python3} \
                EXTRA_OPTFLAGS='$(EDK2_BASETOOLS_OPTFLAGS)' \
                EXTRA_LDFLAGS='$(EDK2_BASETOOLS_LDFLAGS)'
 

diff --git a/tests/uefi-test-tools/build.sh b/tests/uefi-test-tools/build.sh
index 8aa7935c43bb977d04f67420303c92cf0dd586cd..3b78f30840692c1ce17989bcb63f87d99aca4004 100755 (executable)
--- a/tests/uefi-test-tools/build.sh
+++ b/tests/uefi-test-tools/build.sh
@@ -29,8 +29,7 @@ export PACKAGES_PATH=$(realpath -- "$edk2_dir")
 export WORKSPACE=$PWD
 mkdir -p Conf
 
-# Work around <https://bugzilla.tianocore.org/show_bug.cgi?id=1607>.
-export PYTHON_COMMAND=python2
+export PYTHON_COMMAND=${EDK2_PYTHON_COMMAND:-python3}
 
 # Source "edksetup.sh" carefully.
 set +e +u +C
@@ -46,12 +45,15 @@ fi
 source "$edk2_dir/../edk2-funcs.sh"
 edk2_arch=$(qemu_edk2_get_arch "$emulation_target")
 edk2_toolchain=$(qemu_edk2_get_toolchain "$emulation_target")
+MAKEFLAGS=$(qemu_edk2_quirk_tianocore_1607 "$MAKEFLAGS")
+edk2_thread_count=$(qemu_edk2_get_thread_count "$MAKEFLAGS")
 qemu_edk2_set_cross_env "$emulation_target"
 
 # Build the UEFI binary
 mkdir -p log
 build \
   --arch="$edk2_arch" \
+  -n "$edk2_thread_count" \
   --buildtarget=DEBUG \
   --platform=UefiTestToolsPkg/UefiTestToolsPkg.dsc \
   --tagname="$edk2_toolchain" \