} MP_ASSEMBLY_ADDRESS_MAP;\r
\r
//\r
-// Spin lock used to serialize MemoryMapped operation\r
+// Flags used when program the register.\r
//\r
-SPIN_LOCK *mMemoryMappedLock = NULL;\r
+typedef struct {\r
+ volatile UINTN ConsoleLogLock; // Spinlock used to control console.\r
+ volatile UINTN MemoryMappedLock; // Spinlock used to program mmio\r
+ volatile UINT32 *SemaphoreCount; // Semaphore used to program semaphore.\r
+} PROGRAM_CPU_REGISTER_FLAGS;\r
\r
//\r
// Signal that SMM BASE relocation is complete.\r
#define LEGACY_REGION_SIZE (2 * 0x1000)\r
#define LEGACY_REGION_BASE (0xA0000 - LEGACY_REGION_SIZE)\r
\r
+PROGRAM_CPU_REGISTER_FLAGS mCpuFlags;\r
ACPI_CPU_DATA mAcpiCpuData;\r
volatile UINT32 mNumberToFinish;\r
MP_CPU_EXCHANGE_INFO *mExchangeInfo;\r
BOOLEAN mRestoreSmmConfigurationInS3 = FALSE;\r
-MP_MSR_LOCK *mMsrSpinLocks = NULL;\r
-UINTN mMsrSpinLockCount;\r
-UINTN mMsrCount = 0;\r
\r
//\r
// S3 boot flag\r
0xEB, 0xFC // jmp $-2\r
};\r
\r
-/**\r
- Get MSR spin lock by MSR index.\r
-\r
- @param MsrIndex MSR index value.\r
-\r
- @return Pointer to MSR spin lock.\r
-\r
-**/\r
-SPIN_LOCK *\r
-GetMsrSpinLockByIndex (\r
- IN UINT32 MsrIndex\r
- )\r
-{\r
- UINTN Index;\r
- for (Index = 0; Index < mMsrCount; Index++) {\r
- if (MsrIndex == mMsrSpinLocks[Index].MsrIndex) {\r
- return mMsrSpinLocks[Index].SpinLock;\r
- }\r
- }\r
- return NULL;\r
-}\r
-\r
-/**\r
- Initialize MSR spin lock by MSR index.\r
-\r
- @param MsrIndex MSR index value.\r
-\r
-**/\r
-VOID\r
-InitMsrSpinLockByIndex (\r
- IN UINT32 MsrIndex\r
- )\r
-{\r
- UINTN MsrSpinLockCount;\r
- UINTN NewMsrSpinLockCount;\r
- UINTN Index;\r
- UINTN AddedSize;\r
-\r
- if (mMsrSpinLocks == NULL) {\r
- MsrSpinLockCount = mSmmCpuSemaphores.SemaphoreMsr.AvailableCounter;\r
- mMsrSpinLocks = (MP_MSR_LOCK *) AllocatePool (sizeof (MP_MSR_LOCK) * MsrSpinLockCount);\r
- ASSERT (mMsrSpinLocks != NULL);\r
- for (Index = 0; Index < MsrSpinLockCount; Index++) {\r
- mMsrSpinLocks[Index].SpinLock =\r
- (SPIN_LOCK *)((UINTN)mSmmCpuSemaphores.SemaphoreMsr.Msr + Index * mSemaphoreSize);\r
- mMsrSpinLocks[Index].MsrIndex = (UINT32)-1;\r
- }\r
- mMsrSpinLockCount = MsrSpinLockCount;\r
- mSmmCpuSemaphores.SemaphoreMsr.AvailableCounter = 0;\r
- }\r
- if (GetMsrSpinLockByIndex (MsrIndex) == NULL) {\r
- //\r
- // Initialize spin lock for MSR programming\r
- //\r
- mMsrSpinLocks[mMsrCount].MsrIndex = MsrIndex;\r
- InitializeSpinLock (mMsrSpinLocks[mMsrCount].SpinLock);\r
- mMsrCount ++;\r
- if (mMsrCount == mMsrSpinLockCount) {\r
- //\r
- // If MSR spin lock buffer is full, enlarge it\r
- //\r
- AddedSize = SIZE_4KB;\r
- mSmmCpuSemaphores.SemaphoreMsr.Msr =\r
- AllocatePages (EFI_SIZE_TO_PAGES(AddedSize));\r
- ASSERT (mSmmCpuSemaphores.SemaphoreMsr.Msr != NULL);\r
- NewMsrSpinLockCount = mMsrSpinLockCount + AddedSize / mSemaphoreSize;\r
- mMsrSpinLocks = ReallocatePool (\r
- sizeof (MP_MSR_LOCK) * mMsrSpinLockCount,\r
- sizeof (MP_MSR_LOCK) * NewMsrSpinLockCount,\r
- mMsrSpinLocks\r
- );\r
- ASSERT (mMsrSpinLocks != NULL);\r
- mMsrSpinLockCount = NewMsrSpinLockCount;\r
- for (Index = mMsrCount; Index < mMsrSpinLockCount; Index++) {\r
- mMsrSpinLocks[Index].SpinLock =\r
- (SPIN_LOCK *)((UINTN)mSmmCpuSemaphores.SemaphoreMsr.Msr +\r
- (Index - mMsrCount) * mSemaphoreSize);\r
- mMsrSpinLocks[Index].MsrIndex = (UINT32)-1;\r
- }\r
- }\r
- }\r
-}\r
+CHAR16 *mRegisterTypeStr[] = {L"MSR", L"CR", L"MMIO", L"CACHE", L"SEMAP", L"INVALID" };\r
\r
/**\r
Sync up the MTRR values for all processors.\r
}\r
\r
/**\r
- Programs registers for the calling processor.\r
+ Increment semaphore by 1.\r
\r
- This function programs registers for the calling processor.\r
+ @param Sem IN: 32-bit unsigned integer\r
\r
- @param RegisterTables Pointer to register table of the running processor.\r
- @param RegisterTableCount Register table count.\r
+**/\r
+VOID\r
+S3ReleaseSemaphore (\r
+ IN OUT volatile UINT32 *Sem\r
+ )\r
+{\r
+ InterlockedIncrement (Sem);\r
+}\r
+\r
+/**\r
+ Decrement the semaphore by 1 if it is not zero.\r
\r
+ Performs an atomic decrement operation for semaphore.\r
+ The compare exchange operation must be performed using\r
+ MP safe mechanisms.\r
+\r
+ @param Sem IN: 32-bit unsigned integer\r
+\r
+**/\r
+VOID\r
+S3WaitForSemaphore (\r
+ IN OUT volatile UINT32 *Sem\r
+ )\r
+{\r
+ UINT32 Value;\r
+\r
+ do {\r
+ Value = *Sem;\r
+ } while (Value == 0 ||\r
+ InterlockedCompareExchange32 (\r
+ Sem,\r
+ Value,\r
+ Value - 1\r
+ ) != Value);\r
+}\r
+\r
+/**\r
+ Initialize the CPU registers from a register table.\r
+\r
+ @param[in] RegisterTable The register table for this AP.\r
+ @param[in] ApLocation AP location info for this ap.\r
+ @param[in] CpuStatus CPU status info for this CPU.\r
+ @param[in] CpuFlags Flags data structure used when program the register.\r
+\r
+ @note This service could be called by BSP/APs.\r
**/\r
VOID\r
-SetProcessorRegister (\r
- IN CPU_REGISTER_TABLE *RegisterTables,\r
- IN UINTN RegisterTableCount\r
+ProgramProcessorRegister (\r
+ IN CPU_REGISTER_TABLE *RegisterTable,\r
+ IN EFI_CPU_PHYSICAL_LOCATION *ApLocation,\r
+ IN CPU_STATUS_INFORMATION *CpuStatus,\r
+ IN PROGRAM_CPU_REGISTER_FLAGS *CpuFlags\r
)\r
{\r
CPU_REGISTER_TABLE_ENTRY *RegisterTableEntry;\r
UINTN Index;\r
UINTN Value;\r
- SPIN_LOCK *MsrSpinLock;\r
- UINT32 InitApicId;\r
- CPU_REGISTER_TABLE *RegisterTable;\r
-\r
- InitApicId = GetInitialApicId ();\r
- RegisterTable = NULL;\r
- for (Index = 0; Index < RegisterTableCount; Index++) {\r
- if (RegisterTables[Index].InitialApicId == InitApicId) {\r
- RegisterTable = &RegisterTables[Index];\r
- break;\r
- }\r
- }\r
- ASSERT (RegisterTable != NULL);\r
+ CPU_REGISTER_TABLE_ENTRY *RegisterTableEntryHead;\r
+ volatile UINT32 *SemaphorePtr;\r
+ UINT32 FirstThread;\r
+ UINT32 PackageThreadsCount;\r
+ UINT32 CurrentThread;\r
+ UINTN ProcessorIndex;\r
+ UINTN ThreadIndex;\r
+ UINTN ValidThreadCount;\r
+ UINT32 *ValidCoreCountPerPackage;\r
\r
//\r
// Traverse Register Table of this logical processor\r
//\r
- RegisterTableEntry = (CPU_REGISTER_TABLE_ENTRY *) (UINTN) RegisterTable->RegisterTableEntry;\r
- for (Index = 0; Index < RegisterTable->TableLength; Index++, RegisterTableEntry++) {\r
+ RegisterTableEntryHead = (CPU_REGISTER_TABLE_ENTRY *) (UINTN) RegisterTable->RegisterTableEntry;\r
+\r
+ for (Index = 0; Index < RegisterTable->TableLength; Index++) {\r
+\r
+ RegisterTableEntry = &RegisterTableEntryHead[Index];\r
+\r
+ DEBUG_CODE_BEGIN ();\r
+ if (ApLocation != NULL) {\r
+ AcquireSpinLock (&CpuFlags->ConsoleLogLock);\r
+ ThreadIndex = ApLocation->Package * CpuStatus->MaxCoreCount * CpuStatus->MaxThreadCount +\r
+ ApLocation->Core * CpuStatus->MaxThreadCount +\r
+ ApLocation->Thread;\r
+ DEBUG ((\r
+ DEBUG_INFO,\r
+ "Processor = %lu, Entry Index %lu, Type = %s!\n",\r
+ (UINT64)ThreadIndex,\r
+ (UINT64)Index,\r
+ mRegisterTypeStr[MIN ((REGISTER_TYPE)RegisterTableEntry->RegisterType, InvalidReg)]\r
+ ));\r
+ ReleaseSpinLock (&CpuFlags->ConsoleLogLock);\r
+ }\r
+ DEBUG_CODE_END ();\r
+\r
//\r
// Check the type of specified register\r
//\r
RegisterTableEntry->Value\r
);\r
} else {\r
- //\r
- // Get lock to avoid Package/Core scope MSRs programming issue in parallel execution mode\r
- // to make sure MSR read/write operation is atomic.\r
- //\r
- MsrSpinLock = GetMsrSpinLockByIndex (RegisterTableEntry->Index);\r
- AcquireSpinLock (MsrSpinLock);\r
//\r
// Set the bit section according to bit start and length\r
//\r
RegisterTableEntry->ValidBitStart + RegisterTableEntry->ValidBitLength - 1,\r
RegisterTableEntry->Value\r
);\r
- ReleaseSpinLock (MsrSpinLock);\r
}\r
break;\r
//\r
// MemoryMapped operations\r
//\r
case MemoryMapped:\r
- AcquireSpinLock (mMemoryMappedLock);\r
+ AcquireSpinLock (&CpuFlags->MemoryMappedLock);\r
MmioBitFieldWrite32 (\r
(UINTN)(RegisterTableEntry->Index | LShiftU64 (RegisterTableEntry->HighIndex, 32)),\r
RegisterTableEntry->ValidBitStart,\r
RegisterTableEntry->ValidBitStart + RegisterTableEntry->ValidBitLength - 1,\r
(UINT32)RegisterTableEntry->Value\r
);\r
- ReleaseSpinLock (mMemoryMappedLock);\r
+ ReleaseSpinLock (&CpuFlags->MemoryMappedLock);\r
break;\r
//\r
// Enable or disable cache\r
}\r
break;\r
\r
+ case Semaphore:\r
+ // Semaphore works logic like below:\r
+ //\r
+ // V(x) = LibReleaseSemaphore (Semaphore[FirstThread + x]);\r
+ // P(x) = LibWaitForSemaphore (Semaphore[FirstThread + x]);\r
+ //\r
+ // All threads (T0...Tn) waits in P() line and continues running\r
+ // together.\r
+ //\r
+ //\r
+ // T0 T1 ... Tn\r
+ //\r
+ // V(0...n) V(0...n) ... V(0...n)\r
+ // n * P(0) n * P(1) ... n * P(n)\r
+ //\r
+ ASSERT (\r
+ (ApLocation != NULL) && \r
+ (CpuStatus->ValidCoreCountPerPackage != 0) &&\r
+ (CpuFlags->SemaphoreCount) != NULL\r
+ );\r
+ SemaphorePtr = CpuFlags->SemaphoreCount;\r
+ switch (RegisterTableEntry->Value) {\r
+ case CoreDepType:\r
+ //\r
+ // Get Offset info for the first thread in the core which current thread belongs to.\r
+ //\r
+ FirstThread = (ApLocation->Package * CpuStatus->MaxCoreCount + ApLocation->Core) * CpuStatus->MaxThreadCount;\r
+ CurrentThread = FirstThread + ApLocation->Thread;\r
+ //\r
+ // First Notify all threads in current Core that this thread has ready.\r
+ //\r
+ for (ProcessorIndex = 0; ProcessorIndex < CpuStatus->MaxThreadCount; ProcessorIndex ++) {\r
+ S3ReleaseSemaphore (&SemaphorePtr[FirstThread + ProcessorIndex]);\r
+ }\r
+ //\r
+ // Second, check whether all valid threads in current core have ready.\r
+ //\r
+ for (ProcessorIndex = 0; ProcessorIndex < CpuStatus->MaxThreadCount; ProcessorIndex ++) {\r
+ S3WaitForSemaphore (&SemaphorePtr[CurrentThread]);\r
+ }\r
+ break;\r
+\r
+ case PackageDepType:\r
+ ValidCoreCountPerPackage = (UINT32 *)(UINTN)CpuStatus->ValidCoreCountPerPackage;\r
+ //\r
+ // Get Offset info for the first thread in the package which current thread belongs to.\r
+ //\r
+ FirstThread = ApLocation->Package * CpuStatus->MaxCoreCount * CpuStatus->MaxThreadCount;\r
+ //\r
+ // Get the possible threads count for current package.\r
+ //\r
+ PackageThreadsCount = CpuStatus->MaxThreadCount * CpuStatus->MaxCoreCount;\r
+ CurrentThread = FirstThread + CpuStatus->MaxThreadCount * ApLocation->Core + ApLocation->Thread;\r
+ //\r
+ // Get the valid thread count for current package.\r
+ //\r
+ ValidThreadCount = CpuStatus->MaxThreadCount * ValidCoreCountPerPackage[ApLocation->Package];\r
+\r
+ //\r
+ // Different packages may have different valid cores in them. If driver maintail clearly\r
+ // cores number in different packages, the logic will be much complicated.\r
+ // Here driver just simply records the max core number in all packages and use it as expect\r
+ // core number for all packages.\r
+ // In below two steps logic, first current thread will Release semaphore for each thread\r
+ // in current package. Maybe some threads are not valid in this package, but driver don't\r
+ // care. Second, driver will let current thread wait semaphore for all valid threads in\r
+ // current package. Because only the valid threads will do release semaphore for this\r
+ // thread, driver here only need to wait the valid thread count.\r
+ //\r
+\r
+ //\r
+ // First Notify all threads in current package that this thread has ready.\r
+ //\r
+ for (ProcessorIndex = 0; ProcessorIndex < PackageThreadsCount ; ProcessorIndex ++) {\r
+ S3ReleaseSemaphore (&SemaphorePtr[FirstThread + ProcessorIndex]);\r
+ }\r
+ //\r
+ // Second, check whether all valid threads in current package have ready.\r
+ //\r
+ for (ProcessorIndex = 0; ProcessorIndex < ValidThreadCount; ProcessorIndex ++) {\r
+ S3WaitForSemaphore (&SemaphorePtr[CurrentThread]);\r
+ }\r
+ break;\r
+\r
+ default:\r
+ break;\r
+ }\r
+ break;\r
+\r
default:\r
break;\r
}\r
}\r
}\r
\r
+/**\r
+\r
+ Set Processor register for one AP.\r
+ \r
+ @param PreSmmRegisterTable Use pre Smm register table or register table.\r
+\r
+**/\r
+VOID\r
+SetRegister (\r
+ IN BOOLEAN PreSmmRegisterTable\r
+ )\r
+{\r
+ CPU_REGISTER_TABLE *RegisterTable;\r
+ CPU_REGISTER_TABLE *RegisterTables;\r
+ UINT32 InitApicId;\r
+ UINTN ProcIndex;\r
+ UINTN Index;\r
+\r
+ if (PreSmmRegisterTable) {\r
+ RegisterTables = (CPU_REGISTER_TABLE *)(UINTN)mAcpiCpuData.PreSmmInitRegisterTable;\r
+ } else {\r
+ RegisterTables = (CPU_REGISTER_TABLE *)(UINTN)mAcpiCpuData.RegisterTable;\r
+ }\r
+\r
+ InitApicId = GetInitialApicId ();\r
+ RegisterTable = NULL;\r
+ for (Index = 0; Index < mAcpiCpuData.NumberOfCpus; Index++) {\r
+ if (RegisterTables[Index].InitialApicId == InitApicId) {\r
+ RegisterTable = &RegisterTables[Index];\r
+ ProcIndex = Index;\r
+ break;\r
+ }\r
+ }\r
+ ASSERT (RegisterTable != NULL);\r
+\r
+ if (mAcpiCpuData.ApLocation != 0) {\r
+ ProgramProcessorRegister (\r
+ RegisterTable,\r
+ (EFI_CPU_PHYSICAL_LOCATION *)(UINTN)mAcpiCpuData.ApLocation + ProcIndex,\r
+ &mAcpiCpuData.CpuStatus,\r
+ &mCpuFlags\r
+ );\r
+ } else {\r
+ ProgramProcessorRegister (\r
+ RegisterTable,\r
+ NULL,\r
+ &mAcpiCpuData.CpuStatus,\r
+ &mCpuFlags\r
+ );\r
+ }\r
+}\r
+\r
/**\r
AP initialization before then after SMBASE relocation in the S3 boot path.\r
**/\r
\r
LoadMtrrData (mAcpiCpuData.MtrrTable);\r
\r
- SetProcessorRegister ((CPU_REGISTER_TABLE *) (UINTN) mAcpiCpuData.PreSmmInitRegisterTable, mAcpiCpuData.NumberOfCpus);\r
+ SetRegister (TRUE);\r
\r
//\r
// Count down the number with lock mechanism.\r
ProgramVirtualWireMode ();\r
DisableLvtInterrupts ();\r
\r
- SetProcessorRegister ((CPU_REGISTER_TABLE *) (UINTN) mAcpiCpuData.RegisterTable, mAcpiCpuData.NumberOfCpus);\r
+ SetRegister (FALSE);\r
\r
//\r
// Place AP into the safe code, count down the number with lock mechanism in the safe code.\r
{\r
LoadMtrrData (mAcpiCpuData.MtrrTable);\r
\r
- SetProcessorRegister ((CPU_REGISTER_TABLE *) (UINTN) mAcpiCpuData.PreSmmInitRegisterTable, mAcpiCpuData.NumberOfCpus);\r
+ SetRegister (TRUE);\r
\r
ProgramVirtualWireMode ();\r
\r
VOID\r
)\r
{\r
- SetProcessorRegister ((CPU_REGISTER_TABLE *) (UINTN) mAcpiCpuData.RegisterTable, mAcpiCpuData.NumberOfCpus);\r
-\r
mNumberToFinish = mAcpiCpuData.NumberOfCpus - 1;\r
\r
//\r
- // Signal that SMM base relocation is complete and to continue initialization.\r
+ // Signal that SMM base relocation is complete and to continue initialization for all APs.\r
//\r
mInitApsAfterSmmBaseReloc = TRUE;\r
\r
+ //\r
+ // Must begin set register after all APs have continue their initialization.\r
+ // This is a requirement to support semaphore mechanism in register table.\r
+ // Because if semaphore's dependence type is package type, semaphore will wait\r
+ // for all Aps in one package finishing their tasks before set next register\r
+ // for all APs. If the Aps not begin its task during BSP doing its task, the\r
+ // BSP thread will hang because it is waiting for other Aps in the same\r
+ // package finishing their task.\r
+ //\r
+ SetRegister (FALSE);\r
+\r
while (mNumberToFinish > 0) {\r
CpuPause ();\r
}\r
\r
mSmmS3Flag = TRUE;\r
\r
- InitializeSpinLock (mMemoryMappedLock);\r
-\r
//\r
// See if there is enough context to resume PEI Phase\r
//\r
)\r
{\r
UINTN Index;\r
- UINTN Index1;\r
CPU_REGISTER_TABLE_ENTRY *RegisterTableEntry;\r
\r
CopyMem (DestinationRegisterTableList, SourceRegisterTableList, NumberOfCpus * sizeof (CPU_REGISTER_TABLE));\r
);\r
ASSERT (RegisterTableEntry != NULL);\r
DestinationRegisterTableList[Index].RegisterTableEntry = (EFI_PHYSICAL_ADDRESS)(UINTN)RegisterTableEntry;\r
- //\r
- // Go though all MSRs in register table to initialize MSR spin lock\r
- //\r
- for (Index1 = 0; Index1 < DestinationRegisterTableList[Index].TableLength; Index1++, RegisterTableEntry++) {\r
- if ((RegisterTableEntry->RegisterType == Msr) && (RegisterTableEntry->ValidBitLength < 64)) {\r
- //\r
- // Initialize MSR spin lock only for those MSRs need bit field writing\r
- //\r
- InitMsrSpinLockByIndex (RegisterTableEntry->Index);\r
- }\r
- }\r
}\r
}\r
}\r
VOID *GdtForAp;\r
VOID *IdtForAp;\r
VOID *MachineCheckHandlerForAp;\r
+ CPU_STATUS_INFORMATION *CpuStatus;\r
\r
if (!mAcpiS3Enable) {\r
return;\r
Gdtr->Base = (UINTN)GdtForAp;\r
Idtr->Base = (UINTN)IdtForAp;\r
mAcpiCpuData.ApMachineCheckHandlerBase = (EFI_PHYSICAL_ADDRESS)(UINTN)MachineCheckHandlerForAp;\r
+\r
+ CpuStatus = &mAcpiCpuData.CpuStatus;\r
+ CopyMem (CpuStatus, &AcpiCpuData->CpuStatus, sizeof (CPU_STATUS_INFORMATION));\r
+ if (AcpiCpuData->CpuStatus.ValidCoreCountPerPackage != 0) {\r
+ CpuStatus->ValidCoreCountPerPackage = (EFI_PHYSICAL_ADDRESS)(UINTN)AllocateCopyPool (\r
+ sizeof (UINT32) * CpuStatus->PackageCount,\r
+ (UINT32 *)(UINTN)AcpiCpuData->CpuStatus.ValidCoreCountPerPackage\r
+ );\r
+ ASSERT (CpuStatus->ValidCoreCountPerPackage != 0);\r
+ }\r
+ if (AcpiCpuData->ApLocation != 0) {\r
+ mAcpiCpuData.ApLocation = (EFI_PHYSICAL_ADDRESS)(UINTN)AllocateCopyPool (\r
+ mAcpiCpuData.NumberOfCpus * sizeof (EFI_CPU_PHYSICAL_LOCATION),\r
+ (EFI_CPU_PHYSICAL_LOCATION *)(UINTN)AcpiCpuData->ApLocation\r
+ );\r
+ ASSERT (mAcpiCpuData.ApLocation != 0);\r
+ }\r
+ if (CpuStatus->PackageCount != 0) {\r
+ mCpuFlags.SemaphoreCount = AllocateZeroPool (\r
+ sizeof (UINT32) * CpuStatus->PackageCount *\r
+ CpuStatus->MaxCoreCount * CpuStatus->MaxThreadCount);\r
+ ASSERT (mCpuFlags.SemaphoreCount != NULL);\r
+ }\r
+ InitializeSpinLock((SPIN_LOCK*) &mCpuFlags.MemoryMappedLock);\r
+ InitializeSpinLock((SPIN_LOCK*) &mCpuFlags.ConsoleLogLock);\r
}\r
\r
/**\r
projects / mirror_edk2.git / commitdiff