From 1aa6bf528926f8f012a0f2a77bdf53374551a9ec Mon Sep 17 00:00:00 2001
From: Michael Kinney <michael.d.kinney@intel.com>
Date: Mon, 19 Oct 2015 19:08:28 +0000
Subject: [PATCH] UefiCpuPkg: CpuDxe: Wait for APs to enter idle loop

Address a race condition in first call to StartupAllAPs() with
SingleThread set to TRUE in the MP initialization.  If the APs
have not entered their idle loop before StartupAllAPs() is called,
then some of the APs will be in an unexpected state, and
StartupAllAPs() will hang.  This is the hang condition that is
only seen when SingleThread parameter is set to TRUE and
StartupAllAPs() is called very shortly after mAPsAlreadyInitFinished
is set to TRUE that releases the APs to complete their initialization.

An internal function has been added to check if all APs are in the
sleeping state in their idle loop.  On the first call to
StartupAllAPs(), this internal function is used in a loop to make
sure the APs are in a state that is compatible with the use of
StartupAllAPs().  PcdCpuApInitTimeOutInMicroSeconds is used as the
maximum wait time for the APs to enter their idle loop.  If all
APs have not entered their idle loop within the timeout, then
an ASSERT() is generated.

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Michael Kinney <michael.d.kinney@intel.com>
Cc: Jeff Fan <jeff.fan@intel.com>
Cc: Laszlo Ersek <lersek@redhat.com>
Reviewed-by: Laszlo Ersek <lersek@redhat.com>
Reviewed-by: Jeff Fan <jeff.fan@intel.com>

git-svn-id: https://svn.code.sf.net/p/edk2/code/trunk/edk2@18631 6f19259b-4bc3-4df7-8a09-765794883524
---
 UefiCpuPkg/CpuDxe/CpuMp.c | 60 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 59 insertions(+), 1 deletion(-)

diff --git a/UefiCpuPkg/CpuDxe/CpuMp.c b/UefiCpuPkg/CpuDxe/CpuMp.c
index da3686e278..f3a5a24b0f 100644
--- a/UefiCpuPkg/CpuDxe/CpuMp.c
+++ b/UefiCpuPkg/CpuDxe/CpuMp.c
@@ -312,6 +312,47 @@ CheckAndUpdateAllAPsToIdleState (
   }
 }
 
+/**
+  Check if all APs are in state CpuStateSleeping.
+
+  Return TRUE if all APs are in the CpuStateSleeping state.  Do not
+  check the state of the BSP or any disabled APs.
+
+  @retval TRUE   All APs are in CpuStateSleeping state.
+  @retval FALSE  One or more APs are not in CpuStateSleeping state.
+
+**/
+BOOLEAN
+CheckAllAPsSleeping (
+  VOID
+  )
+{
+  UINTN           ProcessorNumber;
+  CPU_DATA_BLOCK  *CpuData;
+
+  for (ProcessorNumber = 0; ProcessorNumber < mMpSystemData.NumberOfProcessors; ProcessorNumber++) {
+    CpuData = &mMpSystemData.CpuDatas[ProcessorNumber];
+    if (TestCpuStatusFlag (CpuData, PROCESSOR_AS_BSP_BIT)) {
+      //
+      // Skip BSP
+      //
+      continue;
+    }
+
+    if (!TestCpuStatusFlag (CpuData, PROCESSOR_ENABLED_BIT)) {
+      //
+      // Skip Disabled processors
+      //
+      continue;
+    }
+
+    if (GetApState (CpuData) != CpuStateSleeping) {
+      return FALSE;
+    }
+  }
+  return TRUE;
+}
+
 /**
   If the timeout expires before all APs returns from Procedure,
   we should forcibly terminate the executing AP and fill FailedList back
@@ -1634,7 +1675,8 @@ InitializeMpSupport (
   VOID
   )
 {
-  EFI_STATUS Status;
+  EFI_STATUS     Status;
+  UINTN          Timeout;
 
   gMaxLogicalProcessorNumber = (UINTN) PcdGet32 (PcdCpuMaxLogicalProcessorNumber);
   if (gMaxLogicalProcessorNumber < 1) {
@@ -1683,8 +1725,24 @@ InitializeMpSupport (
                              sizeof (CPU_DATA_BLOCK) * mMpSystemData.NumberOfProcessors,
                              mMpSystemData.CpuDatas);
 
+  //
+  // Release all APs to complete initialization and enter idle loop
+  //
   mAPsAlreadyInitFinished = TRUE;
 
+  //
+  // Wait for all APs to enter idle loop.
+  //
+  Timeout = 0;
+  do {
+    if (CheckAllAPsSleeping ()) {
+      break;
+    }
+    gBS->Stall (gPollInterval);
+    Timeout += gPollInterval;
+  } while (Timeout <= PcdGet32 (PcdCpuApInitTimeOutInMicroSeconds));
+  ASSERT (Timeout <= PcdGet32 (PcdCpuApInitTimeOutInMicroSeconds));
+
   //
   // Update CPU healthy information from Guided HOB
   //
-- 
2.39.2