2 SMM MP service implementation
4 Copyright (c) 2009 - 2019, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "PiSmmCpuDxeSmm.h"
14 // Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
16 MTRR_SETTINGS gSmiMtrrs
;
18 SMM_DISPATCHER_MP_SYNC_DATA
*mSmmMpSyncData
= NULL
;
19 UINTN mSmmMpSyncDataSize
;
20 SMM_CPU_SEMAPHORES mSmmCpuSemaphores
;
22 SPIN_LOCK
*mPFLock
= NULL
;
23 SMM_CPU_SYNC_MODE mCpuSmmSyncMode
;
24 BOOLEAN mMachineCheckSupported
= FALSE
;
27 Performs an atomic compare exchange operation to get semaphore.
28 The compare exchange operation must be performed using
31 @param Sem IN: 32-bit unsigned integer
32 OUT: original integer - 1
33 @return Original integer - 1
38 IN OUT
volatile UINT32
*Sem
45 } while (Value
== 0 ||
46 InterlockedCompareExchange32 (
56 Performs an atomic compare exchange operation to release semaphore.
57 The compare exchange operation must be performed using
60 @param Sem IN: 32-bit unsigned integer
61 OUT: original integer + 1
62 @return Original integer + 1
67 IN OUT
volatile UINT32
*Sem
74 } while (Value
+ 1 != 0 &&
75 InterlockedCompareExchange32 (
84 Performs an atomic compare exchange operation to lock semaphore.
85 The compare exchange operation must be performed using
88 @param Sem IN: 32-bit unsigned integer
90 @return Original integer
95 IN OUT
volatile UINT32
*Sem
102 } while (InterlockedCompareExchange32 (
110 Wait all APs to performs an atomic compare exchange operation to release semaphore.
112 @param NumberOfAPs AP number
122 BspIndex
= mSmmMpSyncData
->BspIndex
;
123 while (NumberOfAPs
-- > 0) {
124 WaitForSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
129 Performs an atomic compare exchange operation to release semaphore
140 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
141 if (IsPresentAp (Index
)) {
142 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[Index
].Run
);
148 Checks if all CPUs (with certain exceptions) have checked in for this SMI run
150 @param Exceptions CPU Arrival exception flags.
152 @retval TRUE if all CPUs the have checked in.
153 @retval FALSE if at least one Normal AP hasn't checked in.
157 AllCpusInSmmWithExceptions (
158 SMM_CPU_ARRIVAL_EXCEPTIONS Exceptions
162 SMM_CPU_DATA_BLOCK
*CpuData
;
163 EFI_PROCESSOR_INFORMATION
*ProcessorInfo
;
165 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
167 if (*mSmmMpSyncData
->Counter
== mNumberOfCpus
) {
171 CpuData
= mSmmMpSyncData
->CpuData
;
172 ProcessorInfo
= gSmmCpuPrivate
->ProcessorInfo
;
173 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
174 if (!(*(CpuData
[Index
].Present
)) && ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
175 if (((Exceptions
& ARRIVAL_EXCEPTION_DELAYED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmDelayed
) != 0) {
178 if (((Exceptions
& ARRIVAL_EXCEPTION_BLOCKED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmBlocked
) != 0) {
181 if (((Exceptions
& ARRIVAL_EXCEPTION_SMI_DISABLED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmEnable
) != 0) {
193 Has OS enabled Lmce in the MSR_IA32_MCG_EXT_CTL
195 @retval TRUE Os enable lmce.
196 @retval FALSE Os not enable lmce.
204 MSR_IA32_MCG_CAP_REGISTER McgCap
;
205 MSR_IA32_FEATURE_CONTROL_REGISTER FeatureCtrl
;
206 MSR_IA32_MCG_EXT_CTL_REGISTER McgExtCtrl
;
208 McgCap
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_CAP
);
209 if (McgCap
.Bits
.MCG_LMCE_P
== 0) {
213 FeatureCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_FEATURE_CONTROL
);
214 if (FeatureCtrl
.Bits
.LmceOn
== 0) {
218 McgExtCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_EXT_CTL
);
219 return (BOOLEAN
) (McgExtCtrl
.Bits
.LMCE_EN
== 1);
223 Return if Local machine check exception signaled.
225 Indicates (when set) that a local machine check exception was generated. This indicates that the current machine-check event was
226 delivered to only the logical processor.
228 @retval TRUE LMCE was signaled.
229 @retval FALSE LMCE was not signaled.
237 MSR_IA32_MCG_STATUS_REGISTER McgStatus
;
239 McgStatus
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_STATUS
);
240 return (BOOLEAN
) (McgStatus
.Bits
.LMCE_S
== 1);
244 Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
245 entering SMM, except SMI disabled APs.
249 SmmWaitForApArrival (
258 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
262 if (mMachineCheckSupported
) {
263 LmceEn
= IsLmceOsEnabled ();
264 LmceSignal
= IsLmceSignaled();
268 // Platform implementor should choose a timeout value appropriately:
269 // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note
270 // the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.
271 // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI
272 // and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will
273 // be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the
274 // SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.
275 // - The timeout value must be longer than longest possible IO operation in the system
279 // Sync with APs 1st timeout
281 for (Timer
= StartSyncTimer ();
282 !IsSyncTimerTimeout (Timer
) && !(LmceEn
&& LmceSignal
) &&
283 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
289 // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,
291 // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running
292 // normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they
293 // enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode
294 // work while SMI handling is on-going.
295 // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.
296 // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state
297 // will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal
298 // mode work while SMI handling is on-going.
299 // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:
300 // - In traditional flow, SMI disabling is discouraged.
301 // - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
302 // In both cases, adding SMI-disabling checking code increases overhead.
304 if (*mSmmMpSyncData
->Counter
< mNumberOfCpus
) {
306 // Send SMI IPIs to bring outside processors in
308 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
309 if (!(*(mSmmMpSyncData
->CpuData
[Index
].Present
)) && gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
310 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
);
315 // Sync with APs 2nd timeout.
317 for (Timer
= StartSyncTimer ();
318 !IsSyncTimerTimeout (Timer
) &&
319 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
330 Replace OS MTRR's with SMI MTRR's.
332 @param CpuIndex Processor Index
340 SmmCpuFeaturesDisableSmrr ();
343 // Replace all MTRRs registers
345 MtrrSetAllMtrrs (&gSmiMtrrs
);
349 Wheck whether task has been finished by all APs.
351 @param BlockMode Whether did it in block mode or non-block mode.
353 @retval TRUE Task has been finished by all APs.
354 @retval FALSE Task not has been finished by all APs.
358 WaitForAllAPsNotBusy (
364 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
366 // Ignore BSP and APs which not call in SMM.
368 if (!IsPresentAp(Index
)) {
373 AcquireSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
374 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
376 if (AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
377 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
388 Check whether it is an present AP.
390 @param CpuIndex The AP index which calls this function.
392 @retval TRUE It's a present AP.
393 @retval TRUE This is not an AP or it is not present.
401 return ((CpuIndex
!= gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) &&
402 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
));
406 Clean up the status flags used during executing the procedure.
408 @param CpuIndex The AP index which calls this function.
416 PROCEDURE_TOKEN
*Token
;
418 Token
= mSmmMpSyncData
->CpuData
[CpuIndex
].Token
;
420 if (InterlockedDecrement (&Token
->RunningApCount
) == 0) {
421 ReleaseSpinLock (Token
->SpinLock
);
424 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= NULL
;
428 Free the tokens in the maintained list.
437 PROCEDURE_TOKEN
*ProcToken
;
438 TOKEN_BUFFER
*TokenBuf
;
441 // Only free the token buffer recorded in the OldTOkenBufList
442 // upon exiting SMI. Current token buffer stays allocated so
443 // next SMI doesn't need to re-allocate.
445 gSmmCpuPrivate
->UsedTokenNum
= 0;
447 Link
= GetFirstNode (&gSmmCpuPrivate
->OldTokenBufList
);
448 while (!IsNull (&gSmmCpuPrivate
->OldTokenBufList
, Link
)) {
449 TokenBuf
= TOKEN_BUFFER_FROM_LINK (Link
);
451 Link
= RemoveEntryList (&TokenBuf
->Link
);
453 FreePool (TokenBuf
->Buffer
);
457 while (!IsListEmpty (&gSmmCpuPrivate
->TokenList
)) {
458 Link
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
459 ProcToken
= PROCEDURE_TOKEN_FROM_LINK (Link
);
461 RemoveEntryList (&ProcToken
->Link
);
463 FreePool (ProcToken
);
470 @param CpuIndex BSP processor Index
471 @param SyncMode SMM MP sync mode
477 IN SMM_CPU_SYNC_MODE SyncMode
483 BOOLEAN ClearTopLevelSmiResult
;
486 ASSERT (CpuIndex
== mSmmMpSyncData
->BspIndex
);
490 // Flag BSP's presence
492 *mSmmMpSyncData
->InsideSmm
= TRUE
;
495 // Initialize Debug Agent to start source level debug in BSP handler
497 InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI
, NULL
, NULL
);
500 // Mark this processor's presence
502 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
505 // Clear platform top level SMI status bit before calling SMI handlers. If
506 // we cleared it after SMI handlers are run, we would miss the SMI that
507 // occurs after SMI handlers are done and before SMI status bit is cleared.
509 ClearTopLevelSmiResult
= ClearTopLevelSmiStatus();
510 ASSERT (ClearTopLevelSmiResult
== TRUE
);
513 // Set running processor index
515 gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
= CpuIndex
;
518 // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
520 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
523 // Wait for APs to arrive
525 SmmWaitForApArrival();
528 // Lock the counter down and retrieve the number of APs
530 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
531 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
534 // Wait for all APs to get ready for programming MTRRs
536 WaitForAllAPs (ApCount
);
538 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
540 // Signal all APs it's time for backup MTRRs
545 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
546 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
547 // to a large enough value to avoid this situation.
548 // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
549 // We do the backup first and then set MTRR to avoid race condition for threads
552 MtrrGetAllMtrrs(&Mtrrs
);
555 // Wait for all APs to complete their MTRR saving
557 WaitForAllAPs (ApCount
);
560 // Let all processors program SMM MTRRs together
565 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
566 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
567 // to a large enough value to avoid this situation.
569 ReplaceOSMtrrs (CpuIndex
);
572 // Wait for all APs to complete their MTRR programming
574 WaitForAllAPs (ApCount
);
579 // The BUSY lock is initialized to Acquired state
581 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
584 // Perform the pre tasks
589 // Invoke SMM Foundation EntryPoint with the processor information context.
591 gSmmCpuPrivate
->SmmCoreEntry (&gSmmCpuPrivate
->SmmCoreEntryContext
);
594 // Make sure all APs have completed their pending none-block tasks
596 WaitForAllAPsNotBusy (TRUE
);
599 // Perform the remaining tasks
601 PerformRemainingTasks ();
604 // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
605 // make those APs to exit SMI synchronously. APs which arrive later will be excluded and
606 // will run through freely.
608 if (SyncMode
!= SmmCpuSyncModeTradition
&& !SmmCpuFeaturesNeedConfigureMtrrs()) {
611 // Lock the counter down and retrieve the number of APs
613 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
614 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
616 // Make sure all APs have their Present flag set
620 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
621 if (*(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
625 if (PresentCount
> ApCount
) {
632 // Notify all APs to exit
634 *mSmmMpSyncData
->InsideSmm
= FALSE
;
638 // Wait for all APs to complete their pending tasks
640 WaitForAllAPs (ApCount
);
642 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
644 // Signal APs to restore MTRRs
651 SmmCpuFeaturesReenableSmrr ();
652 MtrrSetAllMtrrs(&Mtrrs
);
655 // Wait for all APs to complete MTRR programming
657 WaitForAllAPs (ApCount
);
661 // Stop source level debug in BSP handler, the code below will not be
664 InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI
, NULL
, NULL
);
667 // Signal APs to Reset states/semaphore for this processor
672 // Perform pending operations for hot-plug
677 // Clear the Present flag of BSP
679 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
682 // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
683 // WaitForAllAps does not depend on the Present flag.
685 WaitForAllAPs (ApCount
);
688 // Clean the tokens buffer.
693 // Reset BspIndex to -1, meaning BSP has not been elected.
695 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
696 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
700 // Allow APs to check in from this point on
702 *mSmmMpSyncData
->Counter
= 0;
703 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
709 @param CpuIndex AP processor Index.
710 @param ValidSmi Indicates that current SMI is a valid SMI or not.
711 @param SyncMode SMM MP sync mode.
718 IN SMM_CPU_SYNC_MODE SyncMode
724 EFI_STATUS ProcedureStatus
;
729 for (Timer
= StartSyncTimer ();
730 !IsSyncTimerTimeout (Timer
) &&
731 !(*mSmmMpSyncData
->InsideSmm
);
736 if (!(*mSmmMpSyncData
->InsideSmm
)) {
738 // BSP timeout in the first round
740 if (mSmmMpSyncData
->BspIndex
!= -1) {
742 // BSP Index is known
744 BspIndex
= mSmmMpSyncData
->BspIndex
;
745 ASSERT (CpuIndex
!= BspIndex
);
748 // Send SMI IPI to bring BSP in
750 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[BspIndex
].ProcessorId
);
753 // Now clock BSP for the 2nd time
755 for (Timer
= StartSyncTimer ();
756 !IsSyncTimerTimeout (Timer
) &&
757 !(*mSmmMpSyncData
->InsideSmm
);
762 if (!(*mSmmMpSyncData
->InsideSmm
)) {
764 // Give up since BSP is unable to enter SMM
765 // and signal the completion of this AP
766 WaitForSemaphore (mSmmMpSyncData
->Counter
);
771 // Don't know BSP index. Give up without sending IPI to BSP.
773 WaitForSemaphore (mSmmMpSyncData
->Counter
);
781 BspIndex
= mSmmMpSyncData
->BspIndex
;
782 ASSERT (CpuIndex
!= BspIndex
);
785 // Mark this processor's presence
787 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
789 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
791 // Notify BSP of arrival at this point
793 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
796 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
798 // Wait for the signal from BSP to backup MTRRs
800 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
805 MtrrGetAllMtrrs(&Mtrrs
);
808 // Signal BSP the completion of this AP
810 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
813 // Wait for BSP's signal to program MTRRs
815 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
818 // Replace OS MTRRs with SMI MTRRs
820 ReplaceOSMtrrs (CpuIndex
);
823 // Signal BSP the completion of this AP
825 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
830 // Wait for something to happen
832 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
835 // Check if BSP wants to exit SMM
837 if (!(*mSmmMpSyncData
->InsideSmm
)) {
842 // BUSY should be acquired by SmmStartupThisAp()
845 !AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)
849 // Invoke the scheduled procedure
851 ProcedureStatus
= (*mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
) (
852 (VOID
*)mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
854 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
855 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= ProcedureStatus
;
858 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Token
!= NULL
) {
859 ReleaseToken (CpuIndex
);
865 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
868 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
870 // Notify BSP the readiness of this AP to program MTRRs
872 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
875 // Wait for the signal from BSP to program MTRRs
877 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
882 SmmCpuFeaturesReenableSmrr ();
883 MtrrSetAllMtrrs(&Mtrrs
);
887 // Notify BSP the readiness of this AP to Reset states/semaphore for this processor
889 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
892 // Wait for the signal from BSP to Reset states/semaphore for this processor
894 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
897 // Reset states/semaphore for this processor
899 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
902 // Notify BSP the readiness of this AP to exit SMM
904 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
909 Create 4G PageTable in SMRAM.
911 @param[in] Is32BitPageTable Whether the page table is 32-bit PAE
912 @return PageTable Address
917 IN BOOLEAN Is32BitPageTable
925 UINTN High2MBoundary
;
935 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
937 // Add one more page for known good stack, then find the lower 2MB aligned address.
939 Low2MBoundary
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
) & ~(SIZE_2MB
-1);
941 // Add two more pages for known good stack and stack guard page,
942 // then find the lower 2MB aligned address.
944 High2MBoundary
= (mSmmStackArrayEnd
- mSmmStackSize
+ EFI_PAGE_SIZE
* 2) & ~(SIZE_2MB
-1);
945 PagesNeeded
= ((High2MBoundary
- Low2MBoundary
) / SIZE_2MB
) + 1;
948 // Allocate the page table
950 PageTable
= AllocatePageTableMemory (5 + PagesNeeded
);
951 ASSERT (PageTable
!= NULL
);
953 PageTable
= (VOID
*)((UINTN
)PageTable
);
954 Pte
= (UINT64
*)PageTable
;
957 // Zero out all page table entries first
959 ZeroMem (Pte
, EFI_PAGES_TO_SIZE (1));
962 // Set Page Directory Pointers
964 for (Index
= 0; Index
< 4; Index
++) {
965 Pte
[Index
] = ((UINTN
)PageTable
+ EFI_PAGE_SIZE
* (Index
+ 1)) | mAddressEncMask
|
966 (Is32BitPageTable
? IA32_PAE_PDPTE_ATTRIBUTE_BITS
: PAGE_ATTRIBUTE_BITS
);
968 Pte
+= EFI_PAGE_SIZE
/ sizeof (*Pte
);
971 // Fill in Page Directory Entries
973 for (Index
= 0; Index
< EFI_PAGE_SIZE
* 4 / sizeof (*Pte
); Index
++) {
974 Pte
[Index
] = (Index
<< 21) | mAddressEncMask
| IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
977 Pdpte
= (UINT64
*)PageTable
;
978 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
979 Pages
= (UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (5);
980 GuardPage
= mSmmStackArrayBase
+ EFI_PAGE_SIZE
;
981 for (PageIndex
= Low2MBoundary
; PageIndex
<= High2MBoundary
; PageIndex
+= SIZE_2MB
) {
982 Pte
= (UINT64
*)(UINTN
)(Pdpte
[BitFieldRead32 ((UINT32
)PageIndex
, 30, 31)] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
983 Pte
[BitFieldRead32 ((UINT32
)PageIndex
, 21, 29)] = (UINT64
)Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
985 // Fill in Page Table Entries
987 Pte
= (UINT64
*)Pages
;
988 PageAddress
= PageIndex
;
989 for (Index
= 0; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
990 if (PageAddress
== GuardPage
) {
992 // Mark the guard page as non-present
994 Pte
[Index
] = PageAddress
| mAddressEncMask
;
995 GuardPage
+= mSmmStackSize
;
996 if (GuardPage
> mSmmStackArrayEnd
) {
1000 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
1002 PageAddress
+= EFI_PAGE_SIZE
;
1004 Pages
+= EFI_PAGE_SIZE
;
1008 if ((PcdGet8 (PcdNullPointerDetectionPropertyMask
) & BIT1
) != 0) {
1009 Pte
= (UINT64
*)(UINTN
)(Pdpte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
1010 if ((Pte
[0] & IA32_PG_PS
) == 0) {
1011 // 4K-page entries are already mapped. Just hide the first one anyway.
1012 Pte
= (UINT64
*)(UINTN
)(Pte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
1013 Pte
[0] &= ~(UINT64
)IA32_PG_P
; // Hide page 0
1015 // Create 4K-page entries
1016 Pages
= (UINTN
)AllocatePageTableMemory (1);
1017 ASSERT (Pages
!= 0);
1019 Pte
[0] = (UINT64
)(Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
);
1021 Pte
= (UINT64
*)Pages
;
1023 Pte
[0] = PageAddress
| mAddressEncMask
; // Hide page 0 but present left
1024 for (Index
= 1; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
1025 PageAddress
+= EFI_PAGE_SIZE
;
1026 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
1031 return (UINT32
)(UINTN
)PageTable
;
1035 Checks whether the input token is the current used token.
1037 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1040 @retval TRUE The input token is the current used token.
1041 @retval FALSE The input token is not the current used token.
1049 PROCEDURE_TOKEN
*ProcToken
;
1051 if (Token
== NULL
) {
1055 Link
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
1056 while (!IsNull (&gSmmCpuPrivate
->TokenList
, Link
)) {
1057 ProcToken
= PROCEDURE_TOKEN_FROM_LINK (Link
);
1059 if (ProcToken
->SpinLock
== Token
) {
1063 Link
= GetNextNode (&gSmmCpuPrivate
->TokenList
, Link
);
1070 create token and save it to the maintain list.
1072 @param RunningApCount Input running AP count.
1074 @retval return the spin lock used as token.
1079 IN UINT32 RunningApCount
1082 PROCEDURE_TOKEN
*ProcToken
;
1083 SPIN_LOCK
*SpinLock
;
1085 TOKEN_BUFFER
*TokenBuf
;
1086 UINT32 TokenCountPerChunk
;
1088 SpinLockSize
= GetSpinLockProperties ();
1089 TokenCountPerChunk
= FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk
);
1091 if (gSmmCpuPrivate
->UsedTokenNum
== TokenCountPerChunk
) {
1092 DEBUG ((DEBUG_VERBOSE
, "CpuSmm: No free token buffer, allocate new buffer!\n"));
1095 // Record current token buffer for later free action usage.
1096 // Current used token buffer not in this list.
1098 TokenBuf
= AllocatePool (sizeof (TOKEN_BUFFER
));
1099 ASSERT (TokenBuf
!= NULL
);
1100 TokenBuf
->Signature
= TOKEN_BUFFER_SIGNATURE
;
1101 TokenBuf
->Buffer
= gSmmCpuPrivate
->CurrentTokenBuf
;
1103 InsertTailList (&gSmmCpuPrivate
->OldTokenBufList
, &TokenBuf
->Link
);
1105 gSmmCpuPrivate
->CurrentTokenBuf
= AllocatePool (SpinLockSize
* TokenCountPerChunk
);
1106 ASSERT (gSmmCpuPrivate
->CurrentTokenBuf
!= NULL
);
1107 gSmmCpuPrivate
->UsedTokenNum
= 0;
1110 SpinLock
= (SPIN_LOCK
*)(gSmmCpuPrivate
->CurrentTokenBuf
+ SpinLockSize
* gSmmCpuPrivate
->UsedTokenNum
);
1111 gSmmCpuPrivate
->UsedTokenNum
++;
1113 InitializeSpinLock (SpinLock
);
1114 AcquireSpinLock (SpinLock
);
1116 ProcToken
= AllocatePool (sizeof (PROCEDURE_TOKEN
));
1117 ASSERT (ProcToken
!= NULL
);
1118 ProcToken
->Signature
= PROCEDURE_TOKEN_SIGNATURE
;
1119 ProcToken
->SpinLock
= SpinLock
;
1120 ProcToken
->RunningApCount
= RunningApCount
;
1122 InsertTailList (&gSmmCpuPrivate
->TokenList
, &ProcToken
->Link
);
1128 Checks status of specified AP.
1130 This function checks whether the specified AP has finished the task assigned
1131 by StartupThisAP(), and whether timeout expires.
1133 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1136 @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
1137 @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
1144 if (AcquireSpinLockOrFail (Token
)) {
1145 ReleaseSpinLock (Token
);
1149 return EFI_NOT_READY
;
1153 Schedule a procedure to run on the specified CPU.
1155 @param[in] Procedure The address of the procedure to run
1156 @param[in] CpuIndex Target CPU Index
1157 @param[in,out] ProcArguments The parameter to pass to the procedure
1158 @param[in] Token This is an optional parameter that allows the caller to execute the
1159 procedure in a blocking or non-blocking fashion. If it is NULL the
1160 call is blocking, and the call will not return until the AP has
1161 completed the procedure. If the token is not NULL, the call will
1162 return immediately. The caller can check whether the procedure has
1163 completed with CheckOnProcedure or WaitForProcedure.
1164 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for the APs to finish
1165 execution of Procedure, either for blocking or non-blocking mode.
1166 Zero means infinity. If the timeout expires before all APs return
1167 from Procedure, then Procedure on the failed APs is terminated. If
1168 the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
1169 If the timeout expires in non-blocking mode, the timeout determined
1170 can be through CheckOnProcedure or WaitForProcedure.
1171 Note that timeout support is optional. Whether an implementation
1172 supports this feature can be determined via the Attributes data
1174 @param[in,out] CpuStatus This optional pointer may be used to get the status code returned
1175 by Procedure when it completes execution on the target AP, or with
1176 EFI_TIMEOUT if the Procedure fails to complete within the optional
1177 timeout. The implementation will update this variable with
1178 EFI_NOT_READY prior to starting Procedure on the target AP.
1180 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1181 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1182 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1183 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1184 @retval EFI_SUCCESS The procedure has been successfully scheduled
1188 InternalSmmStartupThisAp (
1189 IN EFI_AP_PROCEDURE2 Procedure
,
1191 IN OUT VOID
*ProcArguments OPTIONAL
,
1192 IN MM_COMPLETION
*Token
,
1193 IN UINTN TimeoutInMicroseconds
,
1194 IN OUT EFI_STATUS
*CpuStatus
1197 PROCEDURE_TOKEN
*ProcToken
;
1199 if (CpuIndex
>= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
) {
1200 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus(%d)\n", CpuIndex
, gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
));
1201 return EFI_INVALID_PARAMETER
;
1203 if (CpuIndex
== gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) {
1204 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu\n", CpuIndex
));
1205 return EFI_INVALID_PARAMETER
;
1207 if (gSmmCpuPrivate
->ProcessorInfo
[CpuIndex
].ProcessorId
== INVALID_APIC_ID
) {
1208 return EFI_INVALID_PARAMETER
;
1210 if (!(*(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
))) {
1211 if (mSmmMpSyncData
->EffectiveSyncMode
== SmmCpuSyncModeTradition
) {
1212 DEBUG((DEBUG_ERROR
, "!mSmmMpSyncData->CpuData[%d].Present\n", CpuIndex
));
1214 return EFI_INVALID_PARAMETER
;
1216 if (gSmmCpuPrivate
->Operation
[CpuIndex
] == SmmCpuRemove
) {
1217 if (!FeaturePcdGet (PcdCpuHotPlugSupport
)) {
1218 DEBUG((DEBUG_ERROR
, "gSmmCpuPrivate->Operation[%d] == SmmCpuRemove\n", CpuIndex
));
1220 return EFI_INVALID_PARAMETER
;
1222 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1223 return EFI_INVALID_PARAMETER
;
1225 if (Procedure
== NULL
) {
1226 return EFI_INVALID_PARAMETER
;
1229 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1231 mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
= Procedure
;
1232 mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
= ProcArguments
;
1233 if (Token
!= NULL
) {
1234 ProcToken
= CreateToken (1);
1235 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= ProcToken
;
1236 *Token
= (MM_COMPLETION
)ProcToken
->SpinLock
;
1238 mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= CpuStatus
;
1239 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
1240 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= EFI_NOT_READY
;
1243 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
1245 if (Token
== NULL
) {
1246 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1247 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1254 Worker function to execute a caller provided function on all enabled APs.
1256 @param[in] Procedure A pointer to the function to be run on
1257 enabled APs of the system.
1258 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
1259 APs to return from Procedure, either for
1260 blocking or non-blocking mode.
1261 @param[in,out] ProcedureArguments The parameter passed into Procedure for
1263 @param[in,out] Token This is an optional parameter that allows the caller to execute the
1264 procedure in a blocking or non-blocking fashion. If it is NULL the
1265 call is blocking, and the call will not return until the AP has
1266 completed the procedure. If the token is not NULL, the call will
1267 return immediately. The caller can check whether the procedure has
1268 completed with CheckOnProcedure or WaitForProcedure.
1269 @param[in,out] CPUStatus This optional pointer may be used to get the status code returned
1270 by Procedure when it completes execution on the target AP, or with
1271 EFI_TIMEOUT if the Procedure fails to complete within the optional
1272 timeout. The implementation will update this variable with
1273 EFI_NOT_READY prior to starting Procedure on the target AP.
1276 @retval EFI_SUCCESS In blocking mode, all APs have finished before
1277 the timeout expired.
1278 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
1280 @retval others Failed to Startup all APs.
1284 InternalSmmStartupAllAPs (
1285 IN EFI_AP_PROCEDURE2 Procedure
,
1286 IN UINTN TimeoutInMicroseconds
,
1287 IN OUT VOID
*ProcedureArguments OPTIONAL
,
1288 IN OUT MM_COMPLETION
*Token
,
1289 IN OUT EFI_STATUS
*CPUStatus
1294 PROCEDURE_TOKEN
*ProcToken
;
1296 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1297 return EFI_INVALID_PARAMETER
;
1299 if (Procedure
== NULL
) {
1300 return EFI_INVALID_PARAMETER
;
1304 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1305 if (IsPresentAp (Index
)) {
1308 if (gSmmCpuPrivate
->Operation
[Index
] == SmmCpuRemove
) {
1309 return EFI_INVALID_PARAMETER
;
1312 if (!AcquireSpinLockOrFail(mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
1313 return EFI_NOT_READY
;
1315 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1318 if (CpuCount
== 0) {
1319 return EFI_NOT_STARTED
;
1322 if (Token
!= NULL
) {
1323 ProcToken
= CreateToken ((UINT32
)mMaxNumberOfCpus
);
1324 *Token
= (MM_COMPLETION
)ProcToken
->SpinLock
;
1330 // Make sure all BUSY should be acquired.
1332 // Because former code already check mSmmMpSyncData->CpuData[***].Busy for each AP.
1333 // Here code always use AcquireSpinLock instead of AcquireSpinLockOrFail for not
1336 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1337 if (IsPresentAp (Index
)) {
1338 AcquireSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1342 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1343 if (IsPresentAp (Index
)) {
1344 mSmmMpSyncData
->CpuData
[Index
].Procedure
= (EFI_AP_PROCEDURE2
) Procedure
;
1345 mSmmMpSyncData
->CpuData
[Index
].Parameter
= ProcedureArguments
;
1346 if (ProcToken
!= NULL
) {
1347 mSmmMpSyncData
->CpuData
[Index
].Token
= ProcToken
;
1349 if (CPUStatus
!= NULL
) {
1350 mSmmMpSyncData
->CpuData
[Index
].Status
= &CPUStatus
[Index
];
1351 if (mSmmMpSyncData
->CpuData
[Index
].Status
!= NULL
) {
1352 *mSmmMpSyncData
->CpuData
[Index
].Status
= EFI_NOT_READY
;
1357 // PI spec requirement:
1358 // For every excluded processor, the array entry must contain a value of EFI_NOT_STARTED.
1360 if (CPUStatus
!= NULL
) {
1361 CPUStatus
[Index
] = EFI_NOT_STARTED
;
1365 // Decrease the count to mark this processor(AP or BSP) as finished.
1367 if (ProcToken
!= NULL
) {
1368 WaitForSemaphore (&ProcToken
->RunningApCount
);
1375 if (Token
== NULL
) {
1377 // Make sure all APs have completed their tasks.
1379 WaitForAllAPsNotBusy (TRUE
);
1386 ISO C99 6.5.2.2 "Function calls", paragraph 9:
1387 If the function is defined with a type that is not compatible with
1388 the type (of the expression) pointed to by the expression that
1389 denotes the called function, the behavior is undefined.
1391 So add below wrapper function to convert between EFI_AP_PROCEDURE
1392 and EFI_AP_PROCEDURE2.
1394 Wrapper for Procedures.
1396 @param[in] Buffer Pointer to PROCEDURE_WRAPPER buffer.
1405 PROCEDURE_WRAPPER
*Wrapper
;
1408 Wrapper
->Procedure (Wrapper
->ProcedureArgument
);
1414 Schedule a procedure to run on the specified CPU in blocking mode.
1416 @param[in] Procedure The address of the procedure to run
1417 @param[in] CpuIndex Target CPU Index
1418 @param[in, out] ProcArguments The parameter to pass to the procedure
1420 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1421 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1422 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1423 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1424 @retval EFI_SUCCESS The procedure has been successfully scheduled
1429 SmmBlockingStartupThisAp (
1430 IN EFI_AP_PROCEDURE Procedure
,
1432 IN OUT VOID
*ProcArguments OPTIONAL
1435 PROCEDURE_WRAPPER Wrapper
;
1437 Wrapper
.Procedure
= Procedure
;
1438 Wrapper
.ProcedureArgument
= ProcArguments
;
1441 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1443 return InternalSmmStartupThisAp (ProcedureWrapper
, CpuIndex
, &Wrapper
, NULL
, 0, NULL
);
1447 Schedule a procedure to run on the specified CPU.
1449 @param Procedure The address of the procedure to run
1450 @param CpuIndex Target CPU Index
1451 @param ProcArguments The parameter to pass to the procedure
1453 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1454 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1455 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1456 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1457 @retval EFI_SUCCESS The procedure has been successfully scheduled
1463 IN EFI_AP_PROCEDURE Procedure
,
1465 IN OUT VOID
*ProcArguments OPTIONAL
1468 MM_COMPLETION Token
;
1470 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].Procedure
= Procedure
;
1471 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].ProcedureArgument
= ProcArguments
;
1474 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1476 return InternalSmmStartupThisAp (
1479 &gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
],
1480 FeaturePcdGet (PcdCpuSmmBlockStartupThisAp
) ? NULL
: &Token
,
1487 This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
1488 They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
1490 NOTE: It might not be appreciated in runtime since it might
1491 conflict with OS debugging facilities. Turn them off in RELEASE.
1493 @param CpuIndex CPU Index
1502 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1504 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1505 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1506 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1507 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1508 AsmWriteDr6 (CpuSaveState
->x86
._DR6
);
1509 AsmWriteDr7 (CpuSaveState
->x86
._DR7
);
1511 AsmWriteDr6 ((UINTN
)CpuSaveState
->x64
._DR6
);
1512 AsmWriteDr7 ((UINTN
)CpuSaveState
->x64
._DR7
);
1518 This function restores DR6 & DR7 to SMM save state.
1520 NOTE: It might not be appreciated in runtime since it might
1521 conflict with OS debugging facilities. Turn them off in RELEASE.
1523 @param CpuIndex CPU Index
1532 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1534 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1535 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1536 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1537 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1538 CpuSaveState
->x86
._DR7
= (UINT32
)AsmReadDr7 ();
1539 CpuSaveState
->x86
._DR6
= (UINT32
)AsmReadDr6 ();
1541 CpuSaveState
->x64
._DR7
= AsmReadDr7 ();
1542 CpuSaveState
->x64
._DR6
= AsmReadDr6 ();
1548 C function for SMI entry, each processor comes here upon SMI trigger.
1550 @param CpuIndex CPU Index
1562 BOOLEAN BspInProgress
;
1566 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1569 // Save Cr2 because Page Fault exception in SMM may override its value,
1570 // when using on-demand paging for above 4G memory.
1576 // Call the user register Startup function first.
1578 if (mSmmMpSyncData
->StartupProcedure
!= NULL
) {
1579 mSmmMpSyncData
->StartupProcedure (mSmmMpSyncData
->StartupProcArgs
);
1583 // Perform CPU specific entry hooks
1585 SmmCpuFeaturesRendezvousEntry (CpuIndex
);
1588 // Determine if this is a valid SMI
1590 ValidSmi
= PlatformValidSmi();
1593 // Determine if BSP has been already in progress. Note this must be checked after
1594 // ValidSmi because BSP may clear a valid SMI source after checking in.
1596 BspInProgress
= *mSmmMpSyncData
->InsideSmm
;
1598 if (!BspInProgress
&& !ValidSmi
) {
1600 // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not
1601 // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI
1602 // status had been cleared by BSP and an existing SMI run has almost ended. (Note
1603 // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there
1604 // is nothing we need to do.
1609 // Signal presence of this processor
1611 if (ReleaseSemaphore (mSmmMpSyncData
->Counter
) == 0) {
1613 // BSP has already ended the synchronization, so QUIT!!!
1617 // Wait for BSP's signal to finish SMI
1619 while (*mSmmMpSyncData
->AllCpusInSync
) {
1626 // The BUSY lock is initialized to Released state.
1627 // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.
1628 // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
1629 // after AP's present flag is detected.
1631 InitializeSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1634 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1635 ActivateSmmProfile (CpuIndex
);
1638 if (BspInProgress
) {
1640 // BSP has been elected. Follow AP path, regardless of ValidSmi flag
1641 // as BSP may have cleared the SMI status
1643 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1646 // We have a valid SMI
1653 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1654 if (!mSmmMpSyncData
->SwitchBsp
|| mSmmMpSyncData
->CandidateBsp
[CpuIndex
]) {
1656 // Call platform hook to do BSP election
1658 Status
= PlatformSmmBspElection (&IsBsp
);
1659 if (EFI_SUCCESS
== Status
) {
1661 // Platform hook determines successfully
1664 mSmmMpSyncData
->BspIndex
= (UINT32
)CpuIndex
;
1668 // Platform hook fails to determine, use default BSP election method
1670 InterlockedCompareExchange32 (
1671 (UINT32
*)&mSmmMpSyncData
->BspIndex
,
1680 // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP
1682 if (mSmmMpSyncData
->BspIndex
== CpuIndex
) {
1685 // Clear last request for SwitchBsp.
1687 if (mSmmMpSyncData
->SwitchBsp
) {
1688 mSmmMpSyncData
->SwitchBsp
= FALSE
;
1689 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1690 mSmmMpSyncData
->CandidateBsp
[Index
] = FALSE
;
1694 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1695 SmmProfileRecordSmiNum ();
1699 // BSP Handler is always called with a ValidSmi == TRUE
1701 BSPHandler (CpuIndex
, mSmmMpSyncData
->EffectiveSyncMode
);
1703 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1707 ASSERT (*mSmmMpSyncData
->CpuData
[CpuIndex
].Run
== 0);
1710 // Wait for BSP's signal to exit SMI
1712 while (*mSmmMpSyncData
->AllCpusInSync
) {
1718 SmmCpuFeaturesRendezvousExit (CpuIndex
);
1727 Allocate buffer for SpinLock and Wrapper function buffer.
1731 InitializeDataForMmMp (
1736 UINT32 TokenCountPerChunk
;
1738 SpinLockSize
= GetSpinLockProperties ();
1739 TokenCountPerChunk
= FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk
);
1740 ASSERT (TokenCountPerChunk
!= 0);
1741 if (TokenCountPerChunk
== 0) {
1742 DEBUG ((DEBUG_ERROR
, "PcdCpuSmmMpTokenCountPerChunk should not be Zero!\n"));
1745 DEBUG ((DEBUG_INFO
, "CpuSmm: SpinLock Size = 0x%x, PcdCpuSmmMpTokenCountPerChunk = 0x%x\n", SpinLockSize
, TokenCountPerChunk
));
1747 gSmmCpuPrivate
->CurrentTokenBuf
= AllocatePool (SpinLockSize
* TokenCountPerChunk
);
1748 ASSERT (gSmmCpuPrivate
->CurrentTokenBuf
!= NULL
);
1750 gSmmCpuPrivate
->UsedTokenNum
= 0;
1752 gSmmCpuPrivate
->ApWrapperFunc
= AllocatePool (sizeof (PROCEDURE_WRAPPER
) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1753 ASSERT (gSmmCpuPrivate
->ApWrapperFunc
!= NULL
);
1755 InitializeListHead (&gSmmCpuPrivate
->TokenList
);
1756 InitializeListHead (&gSmmCpuPrivate
->OldTokenBufList
);
1760 Allocate buffer for all semaphores and spin locks.
1764 InitializeSmmCpuSemaphores (
1768 UINTN ProcessorCount
;
1770 UINTN GlobalSemaphoresSize
;
1771 UINTN CpuSemaphoresSize
;
1772 UINTN SemaphoreSize
;
1774 UINTN
*SemaphoreBlock
;
1775 UINTN SemaphoreAddr
;
1777 SemaphoreSize
= GetSpinLockProperties ();
1778 ProcessorCount
= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1779 GlobalSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_GLOBAL
) / sizeof (VOID
*)) * SemaphoreSize
;
1780 CpuSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_CPU
) / sizeof (VOID
*)) * ProcessorCount
* SemaphoreSize
;
1781 TotalSize
= GlobalSemaphoresSize
+ CpuSemaphoresSize
;
1782 DEBUG((EFI_D_INFO
, "One Semaphore Size = 0x%x\n", SemaphoreSize
));
1783 DEBUG((EFI_D_INFO
, "Total Semaphores Size = 0x%x\n", TotalSize
));
1784 Pages
= EFI_SIZE_TO_PAGES (TotalSize
);
1785 SemaphoreBlock
= AllocatePages (Pages
);
1786 ASSERT (SemaphoreBlock
!= NULL
);
1787 ZeroMem (SemaphoreBlock
, TotalSize
);
1789 SemaphoreAddr
= (UINTN
)SemaphoreBlock
;
1790 mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
= (UINT32
*)SemaphoreAddr
;
1791 SemaphoreAddr
+= SemaphoreSize
;
1792 mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
= (BOOLEAN
*)SemaphoreAddr
;
1793 SemaphoreAddr
+= SemaphoreSize
;
1794 mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
= (BOOLEAN
*)SemaphoreAddr
;
1795 SemaphoreAddr
+= SemaphoreSize
;
1796 mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
= (SPIN_LOCK
*)SemaphoreAddr
;
1797 SemaphoreAddr
+= SemaphoreSize
;
1798 mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
1799 = (SPIN_LOCK
*)SemaphoreAddr
;
1800 SemaphoreAddr
+= SemaphoreSize
;
1802 SemaphoreAddr
= (UINTN
)SemaphoreBlock
+ GlobalSemaphoresSize
;
1803 mSmmCpuSemaphores
.SemaphoreCpu
.Busy
= (SPIN_LOCK
*)SemaphoreAddr
;
1804 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1805 mSmmCpuSemaphores
.SemaphoreCpu
.Run
= (UINT32
*)SemaphoreAddr
;
1806 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1807 mSmmCpuSemaphores
.SemaphoreCpu
.Present
= (BOOLEAN
*)SemaphoreAddr
;
1809 mPFLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
;
1810 mConfigSmmCodeAccessCheckLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
;
1812 mSemaphoreSize
= SemaphoreSize
;
1816 Initialize un-cacheable data.
1821 InitializeMpSyncData (
1827 if (mSmmMpSyncData
!= NULL
) {
1829 // mSmmMpSyncDataSize includes one structure of SMM_DISPATCHER_MP_SYNC_DATA, one
1830 // CpuData array of SMM_CPU_DATA_BLOCK and one CandidateBsp array of BOOLEAN.
1832 ZeroMem (mSmmMpSyncData
, mSmmMpSyncDataSize
);
1833 mSmmMpSyncData
->CpuData
= (SMM_CPU_DATA_BLOCK
*)((UINT8
*)mSmmMpSyncData
+ sizeof (SMM_DISPATCHER_MP_SYNC_DATA
));
1834 mSmmMpSyncData
->CandidateBsp
= (BOOLEAN
*)(mSmmMpSyncData
->CpuData
+ gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1835 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1837 // Enable BSP election by setting BspIndex to -1
1839 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
1841 mSmmMpSyncData
->EffectiveSyncMode
= mCpuSmmSyncMode
;
1843 mSmmMpSyncData
->Counter
= mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
;
1844 mSmmMpSyncData
->InsideSmm
= mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
;
1845 mSmmMpSyncData
->AllCpusInSync
= mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
;
1846 ASSERT (mSmmMpSyncData
->Counter
!= NULL
&& mSmmMpSyncData
->InsideSmm
!= NULL
&&
1847 mSmmMpSyncData
->AllCpusInSync
!= NULL
);
1848 *mSmmMpSyncData
->Counter
= 0;
1849 *mSmmMpSyncData
->InsideSmm
= FALSE
;
1850 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
1852 for (CpuIndex
= 0; CpuIndex
< gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
; CpuIndex
++) {
1853 mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
=
1854 (SPIN_LOCK
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Busy
+ mSemaphoreSize
* CpuIndex
);
1855 mSmmMpSyncData
->CpuData
[CpuIndex
].Run
=
1856 (UINT32
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Run
+ mSemaphoreSize
* CpuIndex
);
1857 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
=
1858 (BOOLEAN
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Present
+ mSemaphoreSize
* CpuIndex
);
1859 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
) = 0;
1860 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Run
) = 0;
1861 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
1867 Initialize global data for MP synchronization.
1869 @param Stacks Base address of SMI stack buffer for all processors.
1870 @param StackSize Stack size for each processor in SMM.
1871 @param ShadowStackSize Shadow Stack size for each processor in SMM.
1875 InitializeMpServiceData (
1878 IN UINTN ShadowStackSize
1883 UINT8
*GdtTssTables
;
1884 UINTN GdtTableStepSize
;
1885 CPUID_VERSION_INFO_EDX RegEdx
;
1888 // Determine if this CPU supports machine check
1890 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
1891 mMachineCheckSupported
= (BOOLEAN
)(RegEdx
.Bits
.MCA
== 1);
1894 // Allocate memory for all locks and semaphores
1896 InitializeSmmCpuSemaphores ();
1899 // Initialize mSmmMpSyncData
1901 mSmmMpSyncDataSize
= sizeof (SMM_DISPATCHER_MP_SYNC_DATA
) +
1902 (sizeof (SMM_CPU_DATA_BLOCK
) + sizeof (BOOLEAN
)) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1903 mSmmMpSyncData
= (SMM_DISPATCHER_MP_SYNC_DATA
*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize
));
1904 ASSERT (mSmmMpSyncData
!= NULL
);
1905 mCpuSmmSyncMode
= (SMM_CPU_SYNC_MODE
)PcdGet8 (PcdCpuSmmSyncMode
);
1906 InitializeMpSyncData ();
1909 // Initialize physical address mask
1910 // NOTE: Physical memory above virtual address limit is not supported !!!
1912 AsmCpuid (0x80000008, (UINT32
*)&Index
, NULL
, NULL
, NULL
);
1913 gPhyMask
= LShiftU64 (1, (UINT8
)Index
) - 1;
1914 gPhyMask
&= (1ull << 48) - EFI_PAGE_SIZE
;
1917 // Create page tables
1919 Cr3
= SmmInitPageTable ();
1921 GdtTssTables
= InitGdt (Cr3
, &GdtTableStepSize
);
1924 // Install SMI handler for each CPU
1926 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1929 (UINT32
)mCpuHotPlugData
.SmBase
[Index
],
1930 (VOID
*)((UINTN
)Stacks
+ (StackSize
+ ShadowStackSize
) * Index
),
1932 (UINTN
)(GdtTssTables
+ GdtTableStepSize
* Index
),
1933 gcSmiGdtr
.Limit
+ 1,
1935 gcSmiIdtr
.Limit
+ 1,
1941 // Record current MTRR settings
1943 ZeroMem (&gSmiMtrrs
, sizeof (gSmiMtrrs
));
1944 MtrrGetAllMtrrs (&gSmiMtrrs
);
1951 Register the SMM Foundation entry point.
1953 @param This Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance
1954 @param SmmEntryPoint SMM Foundation EntryPoint
1956 @retval EFI_SUCCESS Successfully to register SMM foundation entry point
1962 IN CONST EFI_SMM_CONFIGURATION_PROTOCOL
*This
,
1963 IN EFI_SMM_ENTRY_POINT SmmEntryPoint
1967 // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.
1969 gSmmCpuPrivate
->SmmCoreEntry
= SmmEntryPoint
;
1975 Register the SMM Foundation entry point.
1977 @param[in] Procedure A pointer to the code stream to be run on the designated target AP
1978 of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
1979 with the related definitions of
1980 EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
1981 If caller may pass a value of NULL to deregister any existing
1983 @param[in,out] ProcedureArguments Allows the caller to pass a list of parameters to the code that is
1984 run by the AP. It is an optional common mailbox between APs and
1985 the caller to share information
1987 @retval EFI_SUCCESS The Procedure has been set successfully.
1988 @retval EFI_INVALID_PARAMETER The Procedure is NULL but ProcedureArguments not NULL.
1992 RegisterStartupProcedure (
1993 IN EFI_AP_PROCEDURE Procedure
,
1994 IN OUT VOID
*ProcedureArguments OPTIONAL
1997 if (Procedure
== NULL
&& ProcedureArguments
!= NULL
) {
1998 return EFI_INVALID_PARAMETER
;
2000 if (mSmmMpSyncData
== NULL
) {
2001 return EFI_NOT_READY
;
2004 mSmmMpSyncData
->StartupProcedure
= Procedure
;
2005 mSmmMpSyncData
->StartupProcArgs
= ProcedureArguments
;