/** @file\r
SMM MP service implementation\r
\r
-Copyright (c) 2009 - 2019, Intel Corporation. All rights reserved.<BR>\r
+Copyright (c) 2009 - 2021, Intel Corporation. All rights reserved.<BR>\r
Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>\r
\r
SPDX-License-Identifier: BSD-2-Clause-Patent\r
SPIN_LOCK *mPFLock = NULL;\r
SMM_CPU_SYNC_MODE mCpuSmmSyncMode;\r
BOOLEAN mMachineCheckSupported = FALSE;\r
+MM_COMPLETION mSmmStartupThisApToken;\r
+\r
+extern UINTN mSmmShadowStackSize;\r
\r
/**\r
Performs an atomic compare exchange operation to get semaphore.\r
{\r
UINT32 Value;\r
\r
- do {\r
+ for (;;) {\r
Value = *Sem;\r
- } while (Value == 0 ||\r
- InterlockedCompareExchange32 (\r
- (UINT32*)Sem,\r
- Value,\r
- Value - 1\r
- ) != Value);\r
+ if (Value != 0 &&\r
+ InterlockedCompareExchange32 (\r
+ (UINT32*)Sem,\r
+ Value,\r
+ Value - 1\r
+ ) == Value) {\r
+ break;\r
+ }\r
+ CpuPause ();\r
+ }\r
return Value - 1;\r
}\r
\r
\r
**/\r
VOID\r
-FreeTokens (\r
+ResetTokens (\r
VOID\r
)\r
{\r
- LIST_ENTRY *Link;\r
- PROCEDURE_TOKEN *ProcToken;\r
- TOKEN_BUFFER *TokenBuf;\r
-\r
//\r
- // Only free the token buffer recorded in the OldTOkenBufList\r
- // upon exiting SMI. Current token buffer stays allocated so\r
- // next SMI doesn't need to re-allocate.\r
+ // Reset the FirstFreeToken to the beginning of token list upon exiting SMI.\r
//\r
- gSmmCpuPrivate->UsedTokenNum = 0;\r
-\r
- Link = GetFirstNode (&gSmmCpuPrivate->OldTokenBufList);\r
- while (!IsNull (&gSmmCpuPrivate->OldTokenBufList, Link)) {\r
- TokenBuf = TOKEN_BUFFER_FROM_LINK (Link);\r
-\r
- Link = RemoveEntryList (&TokenBuf->Link);\r
-\r
- FreePool (TokenBuf->Buffer);\r
- FreePool (TokenBuf);\r
- }\r
-\r
- while (!IsListEmpty (&gSmmCpuPrivate->TokenList)) {\r
- Link = GetFirstNode (&gSmmCpuPrivate->TokenList);\r
- ProcToken = PROCEDURE_TOKEN_FROM_LINK (Link);\r
-\r
- RemoveEntryList (&ProcToken->Link);\r
-\r
- FreePool (ProcToken);\r
- }\r
+ gSmmCpuPrivate->FirstFreeToken = GetFirstNode (&gSmmCpuPrivate->TokenList);\r
}\r
\r
/**\r
WaitForAllAPs (ApCount);\r
\r
//\r
- // Clean the tokens buffer.\r
+ // Reset the tokens buffer.\r
//\r
- FreeTokens ();\r
+ ResetTokens ();\r
\r
//\r
// Reset BspIndex to -1, meaning BSP has not been elected.\r
// Add two more pages for known good stack and stack guard page,\r
// then find the lower 2MB aligned address.\r
//\r
- High2MBoundary = (mSmmStackArrayEnd - mSmmStackSize + EFI_PAGE_SIZE * 2) & ~(SIZE_2MB-1);\r
+ High2MBoundary = (mSmmStackArrayEnd - mSmmStackSize - mSmmShadowStackSize + EFI_PAGE_SIZE * 2) & ~(SIZE_2MB-1);\r
PagesNeeded = ((High2MBoundary - Low2MBoundary) / SIZE_2MB) + 1;\r
}\r
//\r
// Mark the guard page as non-present\r
//\r
Pte[Index] = PageAddress | mAddressEncMask;\r
- GuardPage += mSmmStackSize;\r
+ GuardPage += (mSmmStackSize + mSmmShadowStackSize);\r
if (GuardPage > mSmmStackArrayEnd) {\r
GuardPage = 0;\r
}\r
}\r
\r
Link = GetFirstNode (&gSmmCpuPrivate->TokenList);\r
- while (!IsNull (&gSmmCpuPrivate->TokenList, Link)) {\r
+ //\r
+ // Only search used tokens.\r
+ //\r
+ while (Link != gSmmCpuPrivate->FirstFreeToken) {\r
ProcToken = PROCEDURE_TOKEN_FROM_LINK (Link);\r
\r
if (ProcToken->SpinLock == Token) {\r
}\r
\r
/**\r
- create token and save it to the maintain list.\r
-\r
- @param RunningApCount Input running AP count.\r
-\r
- @retval return the spin lock used as token.\r
+ Allocate buffer for the SPIN_LOCK and PROCEDURE_TOKEN.\r
\r
+ @return First token of the token buffer.\r
**/\r
-PROCEDURE_TOKEN *\r
-CreateToken (\r
- IN UINT32 RunningApCount\r
+LIST_ENTRY *\r
+AllocateTokenBuffer (\r
+ VOID\r
)\r
{\r
- PROCEDURE_TOKEN *ProcToken;\r
- SPIN_LOCK *SpinLock;\r
UINTN SpinLockSize;\r
- TOKEN_BUFFER *TokenBuf;\r
UINT32 TokenCountPerChunk;\r
+ UINTN Index;\r
+ SPIN_LOCK *SpinLock;\r
+ UINT8 *SpinLockBuffer;\r
+ PROCEDURE_TOKEN *ProcTokens;\r
\r
SpinLockSize = GetSpinLockProperties ();\r
+\r
TokenCountPerChunk = FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk);\r
+ ASSERT (TokenCountPerChunk != 0);\r
+ if (TokenCountPerChunk == 0) {\r
+ DEBUG ((DEBUG_ERROR, "PcdCpuSmmMpTokenCountPerChunk should not be Zero!\n"));\r
+ CpuDeadLoop ();\r
+ }\r
+ DEBUG ((DEBUG_INFO, "CpuSmm: SpinLock Size = 0x%x, PcdCpuSmmMpTokenCountPerChunk = 0x%x\n", SpinLockSize, TokenCountPerChunk));\r
\r
- if (gSmmCpuPrivate->UsedTokenNum == TokenCountPerChunk) {\r
- DEBUG ((DEBUG_VERBOSE, "CpuSmm: No free token buffer, allocate new buffer!\n"));\r
+ //\r
+ // Separate the Spin_lock and Proc_token because the alignment requires by Spin_Lock.\r
+ //\r
+ SpinLockBuffer = AllocatePool (SpinLockSize * TokenCountPerChunk);\r
+ ASSERT (SpinLockBuffer != NULL);\r
\r
- //\r
- // Record current token buffer for later free action usage.\r
- // Current used token buffer not in this list.\r
- //\r
- TokenBuf = AllocatePool (sizeof (TOKEN_BUFFER));\r
- ASSERT (TokenBuf != NULL);\r
- TokenBuf->Signature = TOKEN_BUFFER_SIGNATURE;\r
- TokenBuf->Buffer = gSmmCpuPrivate->CurrentTokenBuf;\r
+ ProcTokens = AllocatePool (sizeof (PROCEDURE_TOKEN) * TokenCountPerChunk);\r
+ ASSERT (ProcTokens != NULL);\r
+\r
+ for (Index = 0; Index < TokenCountPerChunk; Index++) {\r
+ SpinLock = (SPIN_LOCK *)(SpinLockBuffer + SpinLockSize * Index);\r
+ InitializeSpinLock (SpinLock);\r
\r
- InsertTailList (&gSmmCpuPrivate->OldTokenBufList, &TokenBuf->Link);\r
+ ProcTokens[Index].Signature = PROCEDURE_TOKEN_SIGNATURE;\r
+ ProcTokens[Index].SpinLock = SpinLock;\r
+ ProcTokens[Index].RunningApCount = 0;\r
\r
- gSmmCpuPrivate->CurrentTokenBuf = AllocatePool (SpinLockSize * TokenCountPerChunk);\r
- ASSERT (gSmmCpuPrivate->CurrentTokenBuf != NULL);\r
- gSmmCpuPrivate->UsedTokenNum = 0;\r
+ InsertTailList (&gSmmCpuPrivate->TokenList, &ProcTokens[Index].Link);\r
}\r
\r
- SpinLock = (SPIN_LOCK *)(gSmmCpuPrivate->CurrentTokenBuf + SpinLockSize * gSmmCpuPrivate->UsedTokenNum);\r
- gSmmCpuPrivate->UsedTokenNum++;\r
+ return &ProcTokens[0].Link;\r
+}\r
+\r
+/**\r
+ Get the free token.\r
+\r
+ If no free token, allocate new tokens then return the free one.\r
+\r
+ @param RunningApsCount The Running Aps count for this token.\r
+\r
+ @retval return the first free PROCEDURE_TOKEN.\r
\r
- InitializeSpinLock (SpinLock);\r
- AcquireSpinLock (SpinLock);\r
+**/\r
+PROCEDURE_TOKEN *\r
+GetFreeToken (\r
+ IN UINT32 RunningApsCount\r
+ )\r
+{\r
+ PROCEDURE_TOKEN *NewToken;\r
\r
- ProcToken = AllocatePool (sizeof (PROCEDURE_TOKEN));\r
- ASSERT (ProcToken != NULL);\r
- ProcToken->Signature = PROCEDURE_TOKEN_SIGNATURE;\r
- ProcToken->SpinLock = SpinLock;\r
- ProcToken->RunningApCount = RunningApCount;\r
+ //\r
+ // If FirstFreeToken meets the end of token list, enlarge the token list.\r
+ // Set FirstFreeToken to the first free token.\r
+ //\r
+ if (gSmmCpuPrivate->FirstFreeToken == &gSmmCpuPrivate->TokenList) {\r
+ gSmmCpuPrivate->FirstFreeToken = AllocateTokenBuffer ();\r
+ }\r
+ NewToken = PROCEDURE_TOKEN_FROM_LINK (gSmmCpuPrivate->FirstFreeToken);\r
+ gSmmCpuPrivate->FirstFreeToken = GetNextNode (&gSmmCpuPrivate->TokenList, gSmmCpuPrivate->FirstFreeToken);\r
\r
- InsertTailList (&gSmmCpuPrivate->TokenList, &ProcToken->Link);\r
+ NewToken->RunningApCount = RunningApsCount;\r
+ AcquireSpinLock (NewToken->SpinLock);\r
\r
- return ProcToken;\r
+ return NewToken;\r
}\r
\r
/**\r
mSmmMpSyncData->CpuData[CpuIndex].Procedure = Procedure;\r
mSmmMpSyncData->CpuData[CpuIndex].Parameter = ProcArguments;\r
if (Token != NULL) {\r
- ProcToken= CreateToken (1);\r
- mSmmMpSyncData->CpuData[CpuIndex].Token = ProcToken;\r
- *Token = (MM_COMPLETION)ProcToken->SpinLock;\r
+ if (Token != &mSmmStartupThisApToken) {\r
+ //\r
+ // When Token points to mSmmStartupThisApToken, this routine is called\r
+ // from SmmStartupThisAp() in non-blocking mode (PcdCpuSmmBlockStartupThisAp == FALSE).\r
+ //\r
+ // In this case, caller wants to startup AP procedure in non-blocking\r
+ // mode and cannot get the completion status from the Token because there\r
+ // is no way to return the Token to caller from SmmStartupThisAp().\r
+ // Caller needs to use its implementation specific way to query the completion status.\r
+ //\r
+ // There is no need to allocate a token for such case so the 3 overheads\r
+ // can be avoided:\r
+ // 1. Call AllocateTokenBuffer() when there is no free token.\r
+ // 2. Get a free token from the token buffer.\r
+ // 3. Call ReleaseToken() in APHandler().\r
+ //\r
+ ProcToken = GetFreeToken (1);\r
+ mSmmMpSyncData->CpuData[CpuIndex].Token = ProcToken;\r
+ *Token = (MM_COMPLETION)ProcToken->SpinLock;\r
+ }\r
}\r
mSmmMpSyncData->CpuData[CpuIndex].Status = CpuStatus;\r
if (mSmmMpSyncData->CpuData[CpuIndex].Status != NULL) {\r
}\r
\r
if (Token != NULL) {\r
- ProcToken = CreateToken ((UINT32)mMaxNumberOfCpus);\r
+ ProcToken = GetFreeToken ((UINT32)mMaxNumberOfCpus);\r
*Token = (MM_COMPLETION)ProcToken->SpinLock;\r
} else {\r
ProcToken = NULL;\r
IN OUT VOID *ProcArguments OPTIONAL\r
)\r
{\r
- MM_COMPLETION Token;\r
-\r
gSmmCpuPrivate->ApWrapperFunc[CpuIndex].Procedure = Procedure;\r
gSmmCpuPrivate->ApWrapperFunc[CpuIndex].ProcedureArgument = ProcArguments;\r
\r
ProcedureWrapper,\r
CpuIndex,\r
&gSmmCpuPrivate->ApWrapperFunc[CpuIndex],\r
- FeaturePcdGet (PcdCpuSmmBlockStartupThisAp) ? NULL : &Token,\r
+ FeaturePcdGet (PcdCpuSmmBlockStartupThisAp) ? NULL : &mSmmStartupThisApToken,\r
0,\r
NULL\r
);\r
VOID\r
)\r
{\r
- UINTN SpinLockSize;\r
- UINT32 TokenCountPerChunk;\r
-\r
- SpinLockSize = GetSpinLockProperties ();\r
- TokenCountPerChunk = FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk);\r
- ASSERT (TokenCountPerChunk != 0);\r
- if (TokenCountPerChunk == 0) {\r
- DEBUG ((DEBUG_ERROR, "PcdCpuSmmMpTokenCountPerChunk should not be Zero!\n"));\r
- CpuDeadLoop ();\r
- }\r
- DEBUG ((DEBUG_INFO, "CpuSmm: SpinLock Size = 0x%x, PcdCpuSmmMpTokenCountPerChunk = 0x%x\n", SpinLockSize, TokenCountPerChunk));\r
-\r
- gSmmCpuPrivate->CurrentTokenBuf = AllocatePool (SpinLockSize * TokenCountPerChunk);\r
- ASSERT (gSmmCpuPrivate->CurrentTokenBuf != NULL);\r
-\r
- gSmmCpuPrivate->UsedTokenNum = 0;\r
-\r
gSmmCpuPrivate->ApWrapperFunc = AllocatePool (sizeof (PROCEDURE_WRAPPER) * gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus);\r
ASSERT (gSmmCpuPrivate->ApWrapperFunc != NULL);\r
\r
InitializeListHead (&gSmmCpuPrivate->TokenList);\r
- InitializeListHead (&gSmmCpuPrivate->OldTokenBufList);\r
+\r
+ gSmmCpuPrivate->FirstFreeToken = AllocateTokenBuffer ();\r
}\r
\r
/**\r
GlobalSemaphoresSize = (sizeof (SMM_CPU_SEMAPHORE_GLOBAL) / sizeof (VOID *)) * SemaphoreSize;\r
CpuSemaphoresSize = (sizeof (SMM_CPU_SEMAPHORE_CPU) / sizeof (VOID *)) * ProcessorCount * SemaphoreSize;\r
TotalSize = GlobalSemaphoresSize + CpuSemaphoresSize;\r
- DEBUG((EFI_D_INFO, "One Semaphore Size = 0x%x\n", SemaphoreSize));\r
- DEBUG((EFI_D_INFO, "Total Semaphores Size = 0x%x\n", TotalSize));\r
+ DEBUG((DEBUG_INFO, "One Semaphore Size = 0x%x\n", SemaphoreSize));\r
+ DEBUG((DEBUG_INFO, "Total Semaphores Size = 0x%x\n", TotalSize));\r
Pages = EFI_SIZE_TO_PAGES (TotalSize);\r
SemaphoreBlock = AllocatePages (Pages);\r
ASSERT (SemaphoreBlock != NULL);\r
IN UINTN ShadowStackSize\r
)\r
{\r
- UINT32 Cr3;\r
- UINTN Index;\r
- UINT8 *GdtTssTables;\r
- UINTN GdtTableStepSize;\r
- CPUID_VERSION_INFO_EDX RegEdx;\r
+ UINT32 Cr3;\r
+ UINTN Index;\r
+ UINT8 *GdtTssTables;\r
+ UINTN GdtTableStepSize;\r
+ CPUID_VERSION_INFO_EDX RegEdx;\r
+ UINT32 MaxExtendedFunction;\r
+ CPUID_VIR_PHY_ADDRESS_SIZE_EAX VirPhyAddressSize;\r
\r
//\r
// Determine if this CPU supports machine check\r
// Initialize physical address mask\r
// NOTE: Physical memory above virtual address limit is not supported !!!\r
//\r
- AsmCpuid (0x80000008, (UINT32*)&Index, NULL, NULL, NULL);\r
- gPhyMask = LShiftU64 (1, (UINT8)Index) - 1;\r
- gPhyMask &= (1ull << 48) - EFI_PAGE_SIZE;\r
+ AsmCpuid (CPUID_EXTENDED_FUNCTION, &MaxExtendedFunction, NULL, NULL, NULL);\r
+ if (MaxExtendedFunction >= CPUID_VIR_PHY_ADDRESS_SIZE) {\r
+ AsmCpuid (CPUID_VIR_PHY_ADDRESS_SIZE, &VirPhyAddressSize.Uint32, NULL, NULL, NULL);\r
+ } else {\r
+ VirPhyAddressSize.Bits.PhysicalAddressBits = 36;\r
+ }\r
+ gPhyMask = LShiftU64 (1, VirPhyAddressSize.Bits.PhysicalAddressBits) - 1;\r
+ //\r
+ // Clear the low 12 bits\r
+ //\r
+ gPhyMask &= 0xfffffffffffff000ULL;\r
\r
//\r
// Create page tables\r
projects / mirror_edk2.git / blobdiff