2 SMM MP service implementation
4 Copyright (c) 2009 - 2019, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "PiSmmCpuDxeSmm.h"
14 // Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
16 MTRR_SETTINGS gSmiMtrrs
;
18 SMM_DISPATCHER_MP_SYNC_DATA
*mSmmMpSyncData
= NULL
;
19 UINTN mSmmMpSyncDataSize
;
20 SMM_CPU_SEMAPHORES mSmmCpuSemaphores
;
22 SPIN_LOCK
*mPFLock
= NULL
;
23 SMM_CPU_SYNC_MODE mCpuSmmSyncMode
;
24 BOOLEAN mMachineCheckSupported
= FALSE
;
27 Performs an atomic compare exchange operation to get semaphore.
28 The compare exchange operation must be performed using
31 @param Sem IN: 32-bit unsigned integer
32 OUT: original integer - 1
33 @return Original integer - 1
38 IN OUT
volatile UINT32
*Sem
45 } while (Value
== 0 ||
46 InterlockedCompareExchange32 (
56 Performs an atomic compare exchange operation to release semaphore.
57 The compare exchange operation must be performed using
60 @param Sem IN: 32-bit unsigned integer
61 OUT: original integer + 1
62 @return Original integer + 1
67 IN OUT
volatile UINT32
*Sem
74 } while (Value
+ 1 != 0 &&
75 InterlockedCompareExchange32 (
84 Performs an atomic compare exchange operation to lock semaphore.
85 The compare exchange operation must be performed using
88 @param Sem IN: 32-bit unsigned integer
90 @return Original integer
95 IN OUT
volatile UINT32
*Sem
102 } while (InterlockedCompareExchange32 (
110 Wait all APs to performs an atomic compare exchange operation to release semaphore.
112 @param NumberOfAPs AP number
122 BspIndex
= mSmmMpSyncData
->BspIndex
;
123 while (NumberOfAPs
-- > 0) {
124 WaitForSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
129 Performs an atomic compare exchange operation to release semaphore
140 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
141 if (IsPresentAp (Index
)) {
142 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[Index
].Run
);
148 Checks if all CPUs (with certain exceptions) have checked in for this SMI run
150 @param Exceptions CPU Arrival exception flags.
152 @retval TRUE if all CPUs the have checked in.
153 @retval FALSE if at least one Normal AP hasn't checked in.
157 AllCpusInSmmWithExceptions (
158 SMM_CPU_ARRIVAL_EXCEPTIONS Exceptions
162 SMM_CPU_DATA_BLOCK
*CpuData
;
163 EFI_PROCESSOR_INFORMATION
*ProcessorInfo
;
165 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
167 if (*mSmmMpSyncData
->Counter
== mNumberOfCpus
) {
171 CpuData
= mSmmMpSyncData
->CpuData
;
172 ProcessorInfo
= gSmmCpuPrivate
->ProcessorInfo
;
173 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
174 if (!(*(CpuData
[Index
].Present
)) && ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
175 if (((Exceptions
& ARRIVAL_EXCEPTION_DELAYED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmDelayed
) != 0) {
178 if (((Exceptions
& ARRIVAL_EXCEPTION_BLOCKED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmBlocked
) != 0) {
181 if (((Exceptions
& ARRIVAL_EXCEPTION_SMI_DISABLED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmEnable
) != 0) {
193 Has OS enabled Lmce in the MSR_IA32_MCG_EXT_CTL
195 @retval TRUE Os enable lmce.
196 @retval FALSE Os not enable lmce.
204 MSR_IA32_MCG_CAP_REGISTER McgCap
;
205 MSR_IA32_FEATURE_CONTROL_REGISTER FeatureCtrl
;
206 MSR_IA32_MCG_EXT_CTL_REGISTER McgExtCtrl
;
208 McgCap
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_CAP
);
209 if (McgCap
.Bits
.MCG_LMCE_P
== 0) {
213 FeatureCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_FEATURE_CONTROL
);
214 if (FeatureCtrl
.Bits
.LmceOn
== 0) {
218 McgExtCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_EXT_CTL
);
219 return (BOOLEAN
) (McgExtCtrl
.Bits
.LMCE_EN
== 1);
223 Return if Local machine check exception signaled.
225 Indicates (when set) that a local machine check exception was generated. This indicates that the current machine-check event was
226 delivered to only the logical processor.
228 @retval TRUE LMCE was signaled.
229 @retval FALSE LMCE was not signaled.
237 MSR_IA32_MCG_STATUS_REGISTER McgStatus
;
239 McgStatus
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_STATUS
);
240 return (BOOLEAN
) (McgStatus
.Bits
.LMCE_S
== 1);
244 Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
245 entering SMM, except SMI disabled APs.
249 SmmWaitForApArrival (
258 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
262 if (mMachineCheckSupported
) {
263 LmceEn
= IsLmceOsEnabled ();
264 LmceSignal
= IsLmceSignaled();
268 // Platform implementor should choose a timeout value appropriately:
269 // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note
270 // the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.
271 // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI
272 // and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will
273 // be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the
274 // SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.
275 // - The timeout value must be longer than longest possible IO operation in the system
279 // Sync with APs 1st timeout
281 for (Timer
= StartSyncTimer ();
282 !IsSyncTimerTimeout (Timer
) && !(LmceEn
&& LmceSignal
) &&
283 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
289 // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,
291 // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running
292 // normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they
293 // enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode
294 // work while SMI handling is on-going.
295 // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.
296 // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state
297 // will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal
298 // mode work while SMI handling is on-going.
299 // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:
300 // - In traditional flow, SMI disabling is discouraged.
301 // - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
302 // In both cases, adding SMI-disabling checking code increases overhead.
304 if (*mSmmMpSyncData
->Counter
< mNumberOfCpus
) {
306 // Send SMI IPIs to bring outside processors in
308 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
309 if (!(*(mSmmMpSyncData
->CpuData
[Index
].Present
)) && gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
310 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
);
315 // Sync with APs 2nd timeout.
317 for (Timer
= StartSyncTimer ();
318 !IsSyncTimerTimeout (Timer
) &&
319 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
330 Replace OS MTRR's with SMI MTRR's.
332 @param CpuIndex Processor Index
340 SmmCpuFeaturesDisableSmrr ();
343 // Replace all MTRRs registers
345 MtrrSetAllMtrrs (&gSmiMtrrs
);
349 Wheck whether task has been finished by all APs.
351 @param BlockMode Whether did it in block mode or non-block mode.
353 @retval TRUE Task has been finished by all APs.
354 @retval FALSE Task not has been finished by all APs.
358 WaitForAllAPsNotBusy (
364 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
366 // Ignore BSP and APs which not call in SMM.
368 if (!IsPresentAp(Index
)) {
373 AcquireSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
374 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
376 if (AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
377 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
388 Check whether it is an present AP.
390 @param CpuIndex The AP index which calls this function.
392 @retval TRUE It's a present AP.
393 @retval TRUE This is not an AP or it is not present.
401 return ((CpuIndex
!= gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) &&
402 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
));
406 Check whether execute in single AP or all APs.
408 Compare two Tokens used by different APs to know whether in StartAllAps call.
410 Whether is an valid AP base on AP's Present flag.
412 @retval TRUE IN StartAllAps call.
413 @retval FALSE Not in StartAllAps call.
424 for (ApIndex
= mMaxNumberOfCpus
; ApIndex
-- > 0;) {
425 if (IsPresentAp (ApIndex
) && (mSmmMpSyncData
->CpuData
[ApIndex
].Token
!= NULL
)) {
426 for (ApIndex2
= ApIndex
; ApIndex2
-- > 0;) {
427 if (IsPresentAp (ApIndex2
) && (mSmmMpSyncData
->CpuData
[ApIndex2
].Token
!= NULL
)) {
428 return mSmmMpSyncData
->CpuData
[ApIndex2
].Token
== mSmmMpSyncData
->CpuData
[ApIndex
].Token
;
438 Clean up the status flags used during executing the procedure.
440 @param CpuIndex The AP index which calls this function.
451 if (InStartAllApsCall ()) {
453 // In Start All APs mode, make sure all APs have finished task.
455 if (WaitForAllAPsNotBusy (FALSE
)) {
457 // Clean the flags update in the function call.
460 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
462 // Only In SMM APs need to be clean up.
464 if (mSmmMpSyncData
->CpuData
[Index
].Present
&& mSmmMpSyncData
->CpuData
[Index
].Token
!= NULL
) {
466 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[Index
].Token
);
469 mSmmMpSyncData
->CpuData
[Index
].Token
= NULL
;
475 // In single AP mode.
477 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Token
!= NULL
) {
478 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Token
);
479 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= NULL
;
485 Free the tokens in the maintained list.
494 PROCEDURE_TOKEN
*ProcToken
;
495 TOKEN_BUFFER
*TokenBuf
;
498 // Only free the token buffer recorded in the OldTOkenBufList
499 // upon exiting SMI. Current token buffer stays allocated so
500 // next SMI doesn't need to re-allocate.
502 gSmmCpuPrivate
->UsedTokenNum
= 0;
504 Link
= GetFirstNode (&gSmmCpuPrivate
->OldTokenBufList
);
505 while (!IsNull (&gSmmCpuPrivate
->OldTokenBufList
, Link
)) {
506 TokenBuf
= TOKEN_BUFFER_FROM_LINK (Link
);
508 Link
= RemoveEntryList (&TokenBuf
->Link
);
510 FreePool (TokenBuf
->Buffer
);
514 while (!IsListEmpty (&gSmmCpuPrivate
->TokenList
)) {
515 Link
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
516 ProcToken
= PROCEDURE_TOKEN_FROM_LINK (Link
);
518 RemoveEntryList (&ProcToken
->Link
);
520 FreePool (ProcToken
);
527 @param CpuIndex BSP processor Index
528 @param SyncMode SMM MP sync mode
534 IN SMM_CPU_SYNC_MODE SyncMode
540 BOOLEAN ClearTopLevelSmiResult
;
543 ASSERT (CpuIndex
== mSmmMpSyncData
->BspIndex
);
547 // Flag BSP's presence
549 *mSmmMpSyncData
->InsideSmm
= TRUE
;
552 // Initialize Debug Agent to start source level debug in BSP handler
554 InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI
, NULL
, NULL
);
557 // Mark this processor's presence
559 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
562 // Clear platform top level SMI status bit before calling SMI handlers. If
563 // we cleared it after SMI handlers are run, we would miss the SMI that
564 // occurs after SMI handlers are done and before SMI status bit is cleared.
566 ClearTopLevelSmiResult
= ClearTopLevelSmiStatus();
567 ASSERT (ClearTopLevelSmiResult
== TRUE
);
570 // Set running processor index
572 gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
= CpuIndex
;
575 // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
577 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
580 // Wait for APs to arrive
582 SmmWaitForApArrival();
585 // Lock the counter down and retrieve the number of APs
587 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
588 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
591 // Wait for all APs to get ready for programming MTRRs
593 WaitForAllAPs (ApCount
);
595 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
597 // Signal all APs it's time for backup MTRRs
602 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
603 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
604 // to a large enough value to avoid this situation.
605 // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
606 // We do the backup first and then set MTRR to avoid race condition for threads
609 MtrrGetAllMtrrs(&Mtrrs
);
612 // Wait for all APs to complete their MTRR saving
614 WaitForAllAPs (ApCount
);
617 // Let all processors program SMM MTRRs together
622 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
623 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
624 // to a large enough value to avoid this situation.
626 ReplaceOSMtrrs (CpuIndex
);
629 // Wait for all APs to complete their MTRR programming
631 WaitForAllAPs (ApCount
);
636 // The BUSY lock is initialized to Acquired state
638 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
641 // Perform the pre tasks
646 // Invoke SMM Foundation EntryPoint with the processor information context.
648 gSmmCpuPrivate
->SmmCoreEntry (&gSmmCpuPrivate
->SmmCoreEntryContext
);
651 // Make sure all APs have completed their pending none-block tasks
653 WaitForAllAPsNotBusy (TRUE
);
656 // Perform the remaining tasks
658 PerformRemainingTasks ();
661 // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
662 // make those APs to exit SMI synchronously. APs which arrive later will be excluded and
663 // will run through freely.
665 if (SyncMode
!= SmmCpuSyncModeTradition
&& !SmmCpuFeaturesNeedConfigureMtrrs()) {
668 // Lock the counter down and retrieve the number of APs
670 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
671 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
673 // Make sure all APs have their Present flag set
677 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
678 if (*(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
682 if (PresentCount
> ApCount
) {
689 // Notify all APs to exit
691 *mSmmMpSyncData
->InsideSmm
= FALSE
;
695 // Wait for all APs to complete their pending tasks
697 WaitForAllAPs (ApCount
);
699 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
701 // Signal APs to restore MTRRs
708 SmmCpuFeaturesReenableSmrr ();
709 MtrrSetAllMtrrs(&Mtrrs
);
712 // Wait for all APs to complete MTRR programming
714 WaitForAllAPs (ApCount
);
718 // Stop source level debug in BSP handler, the code below will not be
721 InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI
, NULL
, NULL
);
724 // Signal APs to Reset states/semaphore for this processor
729 // Perform pending operations for hot-plug
734 // Clear the Present flag of BSP
736 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
739 // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
740 // WaitForAllAps does not depend on the Present flag.
742 WaitForAllAPs (ApCount
);
745 // Clean the tokens buffer.
750 // Reset BspIndex to -1, meaning BSP has not been elected.
752 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
753 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
757 // Allow APs to check in from this point on
759 *mSmmMpSyncData
->Counter
= 0;
760 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
766 @param CpuIndex AP processor Index.
767 @param ValidSmi Indicates that current SMI is a valid SMI or not.
768 @param SyncMode SMM MP sync mode.
775 IN SMM_CPU_SYNC_MODE SyncMode
781 EFI_STATUS ProcedureStatus
;
786 for (Timer
= StartSyncTimer ();
787 !IsSyncTimerTimeout (Timer
) &&
788 !(*mSmmMpSyncData
->InsideSmm
);
793 if (!(*mSmmMpSyncData
->InsideSmm
)) {
795 // BSP timeout in the first round
797 if (mSmmMpSyncData
->BspIndex
!= -1) {
799 // BSP Index is known
801 BspIndex
= mSmmMpSyncData
->BspIndex
;
802 ASSERT (CpuIndex
!= BspIndex
);
805 // Send SMI IPI to bring BSP in
807 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[BspIndex
].ProcessorId
);
810 // Now clock BSP for the 2nd time
812 for (Timer
= StartSyncTimer ();
813 !IsSyncTimerTimeout (Timer
) &&
814 !(*mSmmMpSyncData
->InsideSmm
);
819 if (!(*mSmmMpSyncData
->InsideSmm
)) {
821 // Give up since BSP is unable to enter SMM
822 // and signal the completion of this AP
823 WaitForSemaphore (mSmmMpSyncData
->Counter
);
828 // Don't know BSP index. Give up without sending IPI to BSP.
830 WaitForSemaphore (mSmmMpSyncData
->Counter
);
838 BspIndex
= mSmmMpSyncData
->BspIndex
;
839 ASSERT (CpuIndex
!= BspIndex
);
842 // Mark this processor's presence
844 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
846 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
848 // Notify BSP of arrival at this point
850 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
853 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
855 // Wait for the signal from BSP to backup MTRRs
857 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
862 MtrrGetAllMtrrs(&Mtrrs
);
865 // Signal BSP the completion of this AP
867 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
870 // Wait for BSP's signal to program MTRRs
872 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
875 // Replace OS MTRRs with SMI MTRRs
877 ReplaceOSMtrrs (CpuIndex
);
880 // Signal BSP the completion of this AP
882 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
887 // Wait for something to happen
889 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
892 // Check if BSP wants to exit SMM
894 if (!(*mSmmMpSyncData
->InsideSmm
)) {
899 // BUSY should be acquired by SmmStartupThisAp()
902 !AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)
906 // Invoke the scheduled procedure
908 ProcedureStatus
= (*mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
) (
909 (VOID
*)mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
911 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
912 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= ProcedureStatus
;
918 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
920 ReleaseToken (CpuIndex
);
923 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
925 // Notify BSP the readiness of this AP to program MTRRs
927 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
930 // Wait for the signal from BSP to program MTRRs
932 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
937 SmmCpuFeaturesReenableSmrr ();
938 MtrrSetAllMtrrs(&Mtrrs
);
942 // Notify BSP the readiness of this AP to Reset states/semaphore for this processor
944 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
947 // Wait for the signal from BSP to Reset states/semaphore for this processor
949 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
952 // Reset states/semaphore for this processor
954 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
957 // Notify BSP the readiness of this AP to exit SMM
959 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
964 Create 4G PageTable in SMRAM.
966 @param[in] Is32BitPageTable Whether the page table is 32-bit PAE
967 @return PageTable Address
972 IN BOOLEAN Is32BitPageTable
980 UINTN High2MBoundary
;
990 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
992 // Add one more page for known good stack, then find the lower 2MB aligned address.
994 Low2MBoundary
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
) & ~(SIZE_2MB
-1);
996 // Add two more pages for known good stack and stack guard page,
997 // then find the lower 2MB aligned address.
999 High2MBoundary
= (mSmmStackArrayEnd
- mSmmStackSize
+ EFI_PAGE_SIZE
* 2) & ~(SIZE_2MB
-1);
1000 PagesNeeded
= ((High2MBoundary
- Low2MBoundary
) / SIZE_2MB
) + 1;
1003 // Allocate the page table
1005 PageTable
= AllocatePageTableMemory (5 + PagesNeeded
);
1006 ASSERT (PageTable
!= NULL
);
1008 PageTable
= (VOID
*)((UINTN
)PageTable
);
1009 Pte
= (UINT64
*)PageTable
;
1012 // Zero out all page table entries first
1014 ZeroMem (Pte
, EFI_PAGES_TO_SIZE (1));
1017 // Set Page Directory Pointers
1019 for (Index
= 0; Index
< 4; Index
++) {
1020 Pte
[Index
] = ((UINTN
)PageTable
+ EFI_PAGE_SIZE
* (Index
+ 1)) | mAddressEncMask
|
1021 (Is32BitPageTable
? IA32_PAE_PDPTE_ATTRIBUTE_BITS
: PAGE_ATTRIBUTE_BITS
);
1023 Pte
+= EFI_PAGE_SIZE
/ sizeof (*Pte
);
1026 // Fill in Page Directory Entries
1028 for (Index
= 0; Index
< EFI_PAGE_SIZE
* 4 / sizeof (*Pte
); Index
++) {
1029 Pte
[Index
] = (Index
<< 21) | mAddressEncMask
| IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
1032 Pdpte
= (UINT64
*)PageTable
;
1033 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
1034 Pages
= (UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (5);
1035 GuardPage
= mSmmStackArrayBase
+ EFI_PAGE_SIZE
;
1036 for (PageIndex
= Low2MBoundary
; PageIndex
<= High2MBoundary
; PageIndex
+= SIZE_2MB
) {
1037 Pte
= (UINT64
*)(UINTN
)(Pdpte
[BitFieldRead32 ((UINT32
)PageIndex
, 30, 31)] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
1038 Pte
[BitFieldRead32 ((UINT32
)PageIndex
, 21, 29)] = (UINT64
)Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
1040 // Fill in Page Table Entries
1042 Pte
= (UINT64
*)Pages
;
1043 PageAddress
= PageIndex
;
1044 for (Index
= 0; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
1045 if (PageAddress
== GuardPage
) {
1047 // Mark the guard page as non-present
1049 Pte
[Index
] = PageAddress
| mAddressEncMask
;
1050 GuardPage
+= mSmmStackSize
;
1051 if (GuardPage
> mSmmStackArrayEnd
) {
1055 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
1057 PageAddress
+= EFI_PAGE_SIZE
;
1059 Pages
+= EFI_PAGE_SIZE
;
1063 if ((PcdGet8 (PcdNullPointerDetectionPropertyMask
) & BIT1
) != 0) {
1064 Pte
= (UINT64
*)(UINTN
)(Pdpte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
1065 if ((Pte
[0] & IA32_PG_PS
) == 0) {
1066 // 4K-page entries are already mapped. Just hide the first one anyway.
1067 Pte
= (UINT64
*)(UINTN
)(Pte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
1068 Pte
[0] &= ~(UINT64
)IA32_PG_P
; // Hide page 0
1070 // Create 4K-page entries
1071 Pages
= (UINTN
)AllocatePageTableMemory (1);
1072 ASSERT (Pages
!= 0);
1074 Pte
[0] = (UINT64
)(Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
);
1076 Pte
= (UINT64
*)Pages
;
1078 Pte
[0] = PageAddress
| mAddressEncMask
; // Hide page 0 but present left
1079 for (Index
= 1; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
1080 PageAddress
+= EFI_PAGE_SIZE
;
1081 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
1086 return (UINT32
)(UINTN
)PageTable
;
1090 Checks whether the input token is the current used token.
1092 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1095 @retval TRUE The input token is the current used token.
1096 @retval FALSE The input token is not the current used token.
1104 PROCEDURE_TOKEN
*ProcToken
;
1106 if (Token
== NULL
) {
1110 Link
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
1111 while (!IsNull (&gSmmCpuPrivate
->TokenList
, Link
)) {
1112 ProcToken
= PROCEDURE_TOKEN_FROM_LINK (Link
);
1114 if (ProcToken
->ProcedureToken
== Token
) {
1118 Link
= GetNextNode (&gSmmCpuPrivate
->TokenList
, Link
);
1125 create token and save it to the maintain list.
1127 @retval return the spin lock used as token.
1135 PROCEDURE_TOKEN
*ProcToken
;
1136 SPIN_LOCK
*CpuToken
;
1138 TOKEN_BUFFER
*TokenBuf
;
1139 UINT32 TokenCountPerChunk
;
1141 SpinLockSize
= GetSpinLockProperties ();
1142 TokenCountPerChunk
= FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk
);
1144 if (gSmmCpuPrivate
->UsedTokenNum
== TokenCountPerChunk
) {
1145 DEBUG ((DEBUG_VERBOSE
, "CpuSmm: No free token buffer, allocate new buffer!\n"));
1148 // Record current token buffer for later free action usage.
1149 // Current used token buffer not in this list.
1151 TokenBuf
= AllocatePool (sizeof (TOKEN_BUFFER
));
1152 ASSERT (TokenBuf
!= NULL
);
1153 TokenBuf
->Signature
= TOKEN_BUFFER_SIGNATURE
;
1154 TokenBuf
->Buffer
= gSmmCpuPrivate
->CurrentTokenBuf
;
1156 InsertTailList (&gSmmCpuPrivate
->OldTokenBufList
, &TokenBuf
->Link
);
1158 gSmmCpuPrivate
->CurrentTokenBuf
= AllocatePool (SpinLockSize
* TokenCountPerChunk
);
1159 ASSERT (gSmmCpuPrivate
->CurrentTokenBuf
!= NULL
);
1160 gSmmCpuPrivate
->UsedTokenNum
= 0;
1163 CpuToken
= (SPIN_LOCK
*)(gSmmCpuPrivate
->CurrentTokenBuf
+ SpinLockSize
* gSmmCpuPrivate
->UsedTokenNum
);
1164 gSmmCpuPrivate
->UsedTokenNum
++;
1166 InitializeSpinLock (CpuToken
);
1167 AcquireSpinLock (CpuToken
);
1169 ProcToken
= AllocatePool (sizeof (PROCEDURE_TOKEN
));
1170 ASSERT (ProcToken
!= NULL
);
1171 ProcToken
->Signature
= PROCEDURE_TOKEN_SIGNATURE
;
1172 ProcToken
->ProcedureToken
= CpuToken
;
1174 InsertTailList (&gSmmCpuPrivate
->TokenList
, &ProcToken
->Link
);
1180 Checks status of specified AP.
1182 This function checks whether the specified AP has finished the task assigned
1183 by StartupThisAP(), and whether timeout expires.
1185 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1188 @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
1189 @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
1196 if (AcquireSpinLockOrFail (Token
)) {
1197 ReleaseSpinLock (Token
);
1201 return EFI_NOT_READY
;
1205 Schedule a procedure to run on the specified CPU.
1207 @param[in] Procedure The address of the procedure to run
1208 @param[in] CpuIndex Target CPU Index
1209 @param[in,out] ProcArguments The parameter to pass to the procedure
1210 @param[in] Token This is an optional parameter that allows the caller to execute the
1211 procedure in a blocking or non-blocking fashion. If it is NULL the
1212 call is blocking, and the call will not return until the AP has
1213 completed the procedure. If the token is not NULL, the call will
1214 return immediately. The caller can check whether the procedure has
1215 completed with CheckOnProcedure or WaitForProcedure.
1216 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for the APs to finish
1217 execution of Procedure, either for blocking or non-blocking mode.
1218 Zero means infinity. If the timeout expires before all APs return
1219 from Procedure, then Procedure on the failed APs is terminated. If
1220 the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
1221 If the timeout expires in non-blocking mode, the timeout determined
1222 can be through CheckOnProcedure or WaitForProcedure.
1223 Note that timeout support is optional. Whether an implementation
1224 supports this feature can be determined via the Attributes data
1226 @param[in,out] CpuStatus This optional pointer may be used to get the status code returned
1227 by Procedure when it completes execution on the target AP, or with
1228 EFI_TIMEOUT if the Procedure fails to complete within the optional
1229 timeout. The implementation will update this variable with
1230 EFI_NOT_READY prior to starting Procedure on the target AP.
1232 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1233 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1234 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1235 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1236 @retval EFI_SUCCESS The procedure has been successfully scheduled
1240 InternalSmmStartupThisAp (
1241 IN EFI_AP_PROCEDURE2 Procedure
,
1243 IN OUT VOID
*ProcArguments OPTIONAL
,
1244 IN MM_COMPLETION
*Token
,
1245 IN UINTN TimeoutInMicroseconds
,
1246 IN OUT EFI_STATUS
*CpuStatus
1249 if (CpuIndex
>= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
) {
1250 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus(%d)\n", CpuIndex
, gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
));
1251 return EFI_INVALID_PARAMETER
;
1253 if (CpuIndex
== gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) {
1254 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu\n", CpuIndex
));
1255 return EFI_INVALID_PARAMETER
;
1257 if (gSmmCpuPrivate
->ProcessorInfo
[CpuIndex
].ProcessorId
== INVALID_APIC_ID
) {
1258 return EFI_INVALID_PARAMETER
;
1260 if (!(*(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
))) {
1261 if (mSmmMpSyncData
->EffectiveSyncMode
== SmmCpuSyncModeTradition
) {
1262 DEBUG((DEBUG_ERROR
, "!mSmmMpSyncData->CpuData[%d].Present\n", CpuIndex
));
1264 return EFI_INVALID_PARAMETER
;
1266 if (gSmmCpuPrivate
->Operation
[CpuIndex
] == SmmCpuRemove
) {
1267 if (!FeaturePcdGet (PcdCpuHotPlugSupport
)) {
1268 DEBUG((DEBUG_ERROR
, "gSmmCpuPrivate->Operation[%d] == SmmCpuRemove\n", CpuIndex
));
1270 return EFI_INVALID_PARAMETER
;
1272 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1273 return EFI_INVALID_PARAMETER
;
1275 if (Procedure
== NULL
) {
1276 return EFI_INVALID_PARAMETER
;
1279 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1281 if (Token
!= NULL
) {
1282 *Token
= (MM_COMPLETION
) CreateToken ();
1285 mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
= Procedure
;
1286 mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
= ProcArguments
;
1287 if (Token
!= NULL
) {
1288 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= (SPIN_LOCK
*)(*Token
);
1290 mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= CpuStatus
;
1291 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
1292 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= EFI_NOT_READY
;
1295 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
1297 if (Token
== NULL
) {
1298 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1299 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1306 Worker function to execute a caller provided function on all enabled APs.
1308 @param[in] Procedure A pointer to the function to be run on
1309 enabled APs of the system.
1310 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
1311 APs to return from Procedure, either for
1312 blocking or non-blocking mode.
1313 @param[in,out] ProcedureArguments The parameter passed into Procedure for
1315 @param[in,out] Token This is an optional parameter that allows the caller to execute the
1316 procedure in a blocking or non-blocking fashion. If it is NULL the
1317 call is blocking, and the call will not return until the AP has
1318 completed the procedure. If the token is not NULL, the call will
1319 return immediately. The caller can check whether the procedure has
1320 completed with CheckOnProcedure or WaitForProcedure.
1321 @param[in,out] CPUStatus This optional pointer may be used to get the status code returned
1322 by Procedure when it completes execution on the target AP, or with
1323 EFI_TIMEOUT if the Procedure fails to complete within the optional
1324 timeout. The implementation will update this variable with
1325 EFI_NOT_READY prior to starting Procedure on the target AP.
1328 @retval EFI_SUCCESS In blocking mode, all APs have finished before
1329 the timeout expired.
1330 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
1332 @retval others Failed to Startup all APs.
1336 InternalSmmStartupAllAPs (
1337 IN EFI_AP_PROCEDURE2 Procedure
,
1338 IN UINTN TimeoutInMicroseconds
,
1339 IN OUT VOID
*ProcedureArguments OPTIONAL
,
1340 IN OUT MM_COMPLETION
*Token
,
1341 IN OUT EFI_STATUS
*CPUStatus
1347 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1348 return EFI_INVALID_PARAMETER
;
1350 if (Procedure
== NULL
) {
1351 return EFI_INVALID_PARAMETER
;
1355 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
1356 if (IsPresentAp (Index
)) {
1359 if (gSmmCpuPrivate
->Operation
[Index
] == SmmCpuRemove
) {
1360 return EFI_INVALID_PARAMETER
;
1363 if (!AcquireSpinLockOrFail(mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
1364 return EFI_NOT_READY
;
1366 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1369 if (CpuCount
== 0) {
1370 return EFI_NOT_STARTED
;
1373 if (Token
!= NULL
) {
1374 *Token
= (MM_COMPLETION
) CreateToken ();
1378 // Make sure all BUSY should be acquired.
1380 // Because former code already check mSmmMpSyncData->CpuData[***].Busy for each AP.
1381 // Here code always use AcquireSpinLock instead of AcquireSpinLockOrFail for not
1384 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
1385 if (IsPresentAp (Index
)) {
1386 AcquireSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1390 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
1391 if (IsPresentAp (Index
)) {
1392 mSmmMpSyncData
->CpuData
[Index
].Procedure
= (EFI_AP_PROCEDURE2
) Procedure
;
1393 mSmmMpSyncData
->CpuData
[Index
].Parameter
= ProcedureArguments
;
1394 if (Token
!= NULL
) {
1395 mSmmMpSyncData
->CpuData
[Index
].Token
= (SPIN_LOCK
*)(*Token
);
1397 if (CPUStatus
!= NULL
) {
1398 mSmmMpSyncData
->CpuData
[Index
].Status
= &CPUStatus
[Index
];
1399 if (mSmmMpSyncData
->CpuData
[Index
].Status
!= NULL
) {
1400 *mSmmMpSyncData
->CpuData
[Index
].Status
= EFI_NOT_READY
;
1405 // PI spec requirement:
1406 // For every excluded processor, the array entry must contain a value of EFI_NOT_STARTED.
1408 if (CPUStatus
!= NULL
) {
1409 CPUStatus
[Index
] = EFI_NOT_STARTED
;
1416 if (Token
== NULL
) {
1418 // Make sure all APs have completed their tasks.
1420 WaitForAllAPsNotBusy (TRUE
);
1427 ISO C99 6.5.2.2 "Function calls", paragraph 9:
1428 If the function is defined with a type that is not compatible with
1429 the type (of the expression) pointed to by the expression that
1430 denotes the called function, the behavior is undefined.
1432 So add below wrapper function to convert between EFI_AP_PROCEDURE
1433 and EFI_AP_PROCEDURE2.
1435 Wrapper for Procedures.
1437 @param[in] Buffer Pointer to PROCEDURE_WRAPPER buffer.
1446 PROCEDURE_WRAPPER
*Wrapper
;
1449 Wrapper
->Procedure (Wrapper
->ProcedureArgument
);
1455 Schedule a procedure to run on the specified CPU in blocking mode.
1457 @param[in] Procedure The address of the procedure to run
1458 @param[in] CpuIndex Target CPU Index
1459 @param[in, out] ProcArguments The parameter to pass to the procedure
1461 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1462 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1463 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1464 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1465 @retval EFI_SUCCESS The procedure has been successfully scheduled
1470 SmmBlockingStartupThisAp (
1471 IN EFI_AP_PROCEDURE Procedure
,
1473 IN OUT VOID
*ProcArguments OPTIONAL
1476 PROCEDURE_WRAPPER Wrapper
;
1478 Wrapper
.Procedure
= Procedure
;
1479 Wrapper
.ProcedureArgument
= ProcArguments
;
1482 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1484 return InternalSmmStartupThisAp (ProcedureWrapper
, CpuIndex
, &Wrapper
, NULL
, 0, NULL
);
1488 Schedule a procedure to run on the specified CPU.
1490 @param Procedure The address of the procedure to run
1491 @param CpuIndex Target CPU Index
1492 @param ProcArguments The parameter to pass to the procedure
1494 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1495 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1496 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1497 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1498 @retval EFI_SUCCESS The procedure has been successfully scheduled
1504 IN EFI_AP_PROCEDURE Procedure
,
1506 IN OUT VOID
*ProcArguments OPTIONAL
1509 MM_COMPLETION Token
;
1511 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].Procedure
= Procedure
;
1512 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].ProcedureArgument
= ProcArguments
;
1515 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1517 return InternalSmmStartupThisAp (
1520 &gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
],
1521 FeaturePcdGet (PcdCpuSmmBlockStartupThisAp
) ? NULL
: &Token
,
1528 This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
1529 They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
1531 NOTE: It might not be appreciated in runtime since it might
1532 conflict with OS debugging facilities. Turn them off in RELEASE.
1534 @param CpuIndex CPU Index
1543 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1545 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1546 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1547 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1548 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1549 AsmWriteDr6 (CpuSaveState
->x86
._DR6
);
1550 AsmWriteDr7 (CpuSaveState
->x86
._DR7
);
1552 AsmWriteDr6 ((UINTN
)CpuSaveState
->x64
._DR6
);
1553 AsmWriteDr7 ((UINTN
)CpuSaveState
->x64
._DR7
);
1559 This function restores DR6 & DR7 to SMM save state.
1561 NOTE: It might not be appreciated in runtime since it might
1562 conflict with OS debugging facilities. Turn them off in RELEASE.
1564 @param CpuIndex CPU Index
1573 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1575 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1576 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1577 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1578 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1579 CpuSaveState
->x86
._DR7
= (UINT32
)AsmReadDr7 ();
1580 CpuSaveState
->x86
._DR6
= (UINT32
)AsmReadDr6 ();
1582 CpuSaveState
->x64
._DR7
= AsmReadDr7 ();
1583 CpuSaveState
->x64
._DR6
= AsmReadDr6 ();
1589 C function for SMI entry, each processor comes here upon SMI trigger.
1591 @param CpuIndex CPU Index
1603 BOOLEAN BspInProgress
;
1607 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1610 // Save Cr2 because Page Fault exception in SMM may override its value,
1611 // when using on-demand paging for above 4G memory.
1617 // Call the user register Startup function first.
1619 if (mSmmMpSyncData
->StartupProcedure
!= NULL
) {
1620 mSmmMpSyncData
->StartupProcedure (mSmmMpSyncData
->StartupProcArgs
);
1624 // Perform CPU specific entry hooks
1626 SmmCpuFeaturesRendezvousEntry (CpuIndex
);
1629 // Determine if this is a valid SMI
1631 ValidSmi
= PlatformValidSmi();
1634 // Determine if BSP has been already in progress. Note this must be checked after
1635 // ValidSmi because BSP may clear a valid SMI source after checking in.
1637 BspInProgress
= *mSmmMpSyncData
->InsideSmm
;
1639 if (!BspInProgress
&& !ValidSmi
) {
1641 // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not
1642 // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI
1643 // status had been cleared by BSP and an existing SMI run has almost ended. (Note
1644 // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there
1645 // is nothing we need to do.
1650 // Signal presence of this processor
1652 if (ReleaseSemaphore (mSmmMpSyncData
->Counter
) == 0) {
1654 // BSP has already ended the synchronization, so QUIT!!!
1658 // Wait for BSP's signal to finish SMI
1660 while (*mSmmMpSyncData
->AllCpusInSync
) {
1667 // The BUSY lock is initialized to Released state.
1668 // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.
1669 // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
1670 // after AP's present flag is detected.
1672 InitializeSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1675 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1676 ActivateSmmProfile (CpuIndex
);
1679 if (BspInProgress
) {
1681 // BSP has been elected. Follow AP path, regardless of ValidSmi flag
1682 // as BSP may have cleared the SMI status
1684 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1687 // We have a valid SMI
1694 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1695 if (!mSmmMpSyncData
->SwitchBsp
|| mSmmMpSyncData
->CandidateBsp
[CpuIndex
]) {
1697 // Call platform hook to do BSP election
1699 Status
= PlatformSmmBspElection (&IsBsp
);
1700 if (EFI_SUCCESS
== Status
) {
1702 // Platform hook determines successfully
1705 mSmmMpSyncData
->BspIndex
= (UINT32
)CpuIndex
;
1709 // Platform hook fails to determine, use default BSP election method
1711 InterlockedCompareExchange32 (
1712 (UINT32
*)&mSmmMpSyncData
->BspIndex
,
1721 // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP
1723 if (mSmmMpSyncData
->BspIndex
== CpuIndex
) {
1726 // Clear last request for SwitchBsp.
1728 if (mSmmMpSyncData
->SwitchBsp
) {
1729 mSmmMpSyncData
->SwitchBsp
= FALSE
;
1730 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1731 mSmmMpSyncData
->CandidateBsp
[Index
] = FALSE
;
1735 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1736 SmmProfileRecordSmiNum ();
1740 // BSP Handler is always called with a ValidSmi == TRUE
1742 BSPHandler (CpuIndex
, mSmmMpSyncData
->EffectiveSyncMode
);
1744 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1748 ASSERT (*mSmmMpSyncData
->CpuData
[CpuIndex
].Run
== 0);
1751 // Wait for BSP's signal to exit SMI
1753 while (*mSmmMpSyncData
->AllCpusInSync
) {
1759 SmmCpuFeaturesRendezvousExit (CpuIndex
);
1768 Allocate buffer for SpinLock and Wrapper function buffer.
1772 InitializeDataForMmMp (
1777 UINT32 TokenCountPerChunk
;
1779 SpinLockSize
= GetSpinLockProperties ();
1780 TokenCountPerChunk
= FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk
);
1781 ASSERT (TokenCountPerChunk
!= 0);
1782 if (TokenCountPerChunk
== 0) {
1783 DEBUG ((DEBUG_ERROR
, "PcdCpuSmmMpTokenCountPerChunk should not be Zero!\n"));
1786 DEBUG ((DEBUG_INFO
, "CpuSmm: SpinLock Size = 0x%x, PcdCpuSmmMpTokenCountPerChunk = 0x%x\n", SpinLockSize
, TokenCountPerChunk
));
1788 gSmmCpuPrivate
->CurrentTokenBuf
= AllocatePool (SpinLockSize
* TokenCountPerChunk
);
1789 ASSERT (gSmmCpuPrivate
->CurrentTokenBuf
!= NULL
);
1791 gSmmCpuPrivate
->UsedTokenNum
= 0;
1793 gSmmCpuPrivate
->ApWrapperFunc
= AllocatePool (sizeof (PROCEDURE_WRAPPER
) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1794 ASSERT (gSmmCpuPrivate
->ApWrapperFunc
!= NULL
);
1796 InitializeListHead (&gSmmCpuPrivate
->TokenList
);
1797 InitializeListHead (&gSmmCpuPrivate
->OldTokenBufList
);
1801 Allocate buffer for all semaphores and spin locks.
1805 InitializeSmmCpuSemaphores (
1809 UINTN ProcessorCount
;
1811 UINTN GlobalSemaphoresSize
;
1812 UINTN CpuSemaphoresSize
;
1813 UINTN SemaphoreSize
;
1815 UINTN
*SemaphoreBlock
;
1816 UINTN SemaphoreAddr
;
1818 SemaphoreSize
= GetSpinLockProperties ();
1819 ProcessorCount
= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1820 GlobalSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_GLOBAL
) / sizeof (VOID
*)) * SemaphoreSize
;
1821 CpuSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_CPU
) / sizeof (VOID
*)) * ProcessorCount
* SemaphoreSize
;
1822 TotalSize
= GlobalSemaphoresSize
+ CpuSemaphoresSize
;
1823 DEBUG((EFI_D_INFO
, "One Semaphore Size = 0x%x\n", SemaphoreSize
));
1824 DEBUG((EFI_D_INFO
, "Total Semaphores Size = 0x%x\n", TotalSize
));
1825 Pages
= EFI_SIZE_TO_PAGES (TotalSize
);
1826 SemaphoreBlock
= AllocatePages (Pages
);
1827 ASSERT (SemaphoreBlock
!= NULL
);
1828 ZeroMem (SemaphoreBlock
, TotalSize
);
1830 SemaphoreAddr
= (UINTN
)SemaphoreBlock
;
1831 mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
= (UINT32
*)SemaphoreAddr
;
1832 SemaphoreAddr
+= SemaphoreSize
;
1833 mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
= (BOOLEAN
*)SemaphoreAddr
;
1834 SemaphoreAddr
+= SemaphoreSize
;
1835 mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
= (BOOLEAN
*)SemaphoreAddr
;
1836 SemaphoreAddr
+= SemaphoreSize
;
1837 mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
= (SPIN_LOCK
*)SemaphoreAddr
;
1838 SemaphoreAddr
+= SemaphoreSize
;
1839 mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
1840 = (SPIN_LOCK
*)SemaphoreAddr
;
1841 SemaphoreAddr
+= SemaphoreSize
;
1843 SemaphoreAddr
= (UINTN
)SemaphoreBlock
+ GlobalSemaphoresSize
;
1844 mSmmCpuSemaphores
.SemaphoreCpu
.Busy
= (SPIN_LOCK
*)SemaphoreAddr
;
1845 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1846 mSmmCpuSemaphores
.SemaphoreCpu
.Run
= (UINT32
*)SemaphoreAddr
;
1847 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1848 mSmmCpuSemaphores
.SemaphoreCpu
.Present
= (BOOLEAN
*)SemaphoreAddr
;
1850 mPFLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
;
1851 mConfigSmmCodeAccessCheckLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
;
1853 mSemaphoreSize
= SemaphoreSize
;
1857 Initialize un-cacheable data.
1862 InitializeMpSyncData (
1868 if (mSmmMpSyncData
!= NULL
) {
1870 // mSmmMpSyncDataSize includes one structure of SMM_DISPATCHER_MP_SYNC_DATA, one
1871 // CpuData array of SMM_CPU_DATA_BLOCK and one CandidateBsp array of BOOLEAN.
1873 ZeroMem (mSmmMpSyncData
, mSmmMpSyncDataSize
);
1874 mSmmMpSyncData
->CpuData
= (SMM_CPU_DATA_BLOCK
*)((UINT8
*)mSmmMpSyncData
+ sizeof (SMM_DISPATCHER_MP_SYNC_DATA
));
1875 mSmmMpSyncData
->CandidateBsp
= (BOOLEAN
*)(mSmmMpSyncData
->CpuData
+ gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1876 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1878 // Enable BSP election by setting BspIndex to -1
1880 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
1882 mSmmMpSyncData
->EffectiveSyncMode
= mCpuSmmSyncMode
;
1884 mSmmMpSyncData
->Counter
= mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
;
1885 mSmmMpSyncData
->InsideSmm
= mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
;
1886 mSmmMpSyncData
->AllCpusInSync
= mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
;
1887 ASSERT (mSmmMpSyncData
->Counter
!= NULL
&& mSmmMpSyncData
->InsideSmm
!= NULL
&&
1888 mSmmMpSyncData
->AllCpusInSync
!= NULL
);
1889 *mSmmMpSyncData
->Counter
= 0;
1890 *mSmmMpSyncData
->InsideSmm
= FALSE
;
1891 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
1893 for (CpuIndex
= 0; CpuIndex
< gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
; CpuIndex
++) {
1894 mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
=
1895 (SPIN_LOCK
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Busy
+ mSemaphoreSize
* CpuIndex
);
1896 mSmmMpSyncData
->CpuData
[CpuIndex
].Run
=
1897 (UINT32
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Run
+ mSemaphoreSize
* CpuIndex
);
1898 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
=
1899 (BOOLEAN
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Present
+ mSemaphoreSize
* CpuIndex
);
1900 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
) = 0;
1901 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Run
) = 0;
1902 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
1908 Initialize global data for MP synchronization.
1910 @param Stacks Base address of SMI stack buffer for all processors.
1911 @param StackSize Stack size for each processor in SMM.
1912 @param ShadowStackSize Shadow Stack size for each processor in SMM.
1916 InitializeMpServiceData (
1919 IN UINTN ShadowStackSize
1924 UINT8
*GdtTssTables
;
1925 UINTN GdtTableStepSize
;
1926 CPUID_VERSION_INFO_EDX RegEdx
;
1929 // Determine if this CPU supports machine check
1931 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
1932 mMachineCheckSupported
= (BOOLEAN
)(RegEdx
.Bits
.MCA
== 1);
1935 // Allocate memory for all locks and semaphores
1937 InitializeSmmCpuSemaphores ();
1940 // Initialize mSmmMpSyncData
1942 mSmmMpSyncDataSize
= sizeof (SMM_DISPATCHER_MP_SYNC_DATA
) +
1943 (sizeof (SMM_CPU_DATA_BLOCK
) + sizeof (BOOLEAN
)) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1944 mSmmMpSyncData
= (SMM_DISPATCHER_MP_SYNC_DATA
*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize
));
1945 ASSERT (mSmmMpSyncData
!= NULL
);
1946 mCpuSmmSyncMode
= (SMM_CPU_SYNC_MODE
)PcdGet8 (PcdCpuSmmSyncMode
);
1947 InitializeMpSyncData ();
1950 // Initialize physical address mask
1951 // NOTE: Physical memory above virtual address limit is not supported !!!
1953 AsmCpuid (0x80000008, (UINT32
*)&Index
, NULL
, NULL
, NULL
);
1954 gPhyMask
= LShiftU64 (1, (UINT8
)Index
) - 1;
1955 gPhyMask
&= (1ull << 48) - EFI_PAGE_SIZE
;
1958 // Create page tables
1960 Cr3
= SmmInitPageTable ();
1962 GdtTssTables
= InitGdt (Cr3
, &GdtTableStepSize
);
1965 // Install SMI handler for each CPU
1967 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1970 (UINT32
)mCpuHotPlugData
.SmBase
[Index
],
1971 (VOID
*)((UINTN
)Stacks
+ (StackSize
+ ShadowStackSize
) * Index
),
1973 (UINTN
)(GdtTssTables
+ GdtTableStepSize
* Index
),
1974 gcSmiGdtr
.Limit
+ 1,
1976 gcSmiIdtr
.Limit
+ 1,
1982 // Record current MTRR settings
1984 ZeroMem (&gSmiMtrrs
, sizeof (gSmiMtrrs
));
1985 MtrrGetAllMtrrs (&gSmiMtrrs
);
1992 Register the SMM Foundation entry point.
1994 @param This Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance
1995 @param SmmEntryPoint SMM Foundation EntryPoint
1997 @retval EFI_SUCCESS Successfully to register SMM foundation entry point
2003 IN CONST EFI_SMM_CONFIGURATION_PROTOCOL
*This
,
2004 IN EFI_SMM_ENTRY_POINT SmmEntryPoint
2008 // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.
2010 gSmmCpuPrivate
->SmmCoreEntry
= SmmEntryPoint
;
2016 Register the SMM Foundation entry point.
2018 @param[in] Procedure A pointer to the code stream to be run on the designated target AP
2019 of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
2020 with the related definitions of
2021 EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
2022 If caller may pass a value of NULL to deregister any existing
2024 @param[in,out] ProcedureArguments Allows the caller to pass a list of parameters to the code that is
2025 run by the AP. It is an optional common mailbox between APs and
2026 the caller to share information
2028 @retval EFI_SUCCESS The Procedure has been set successfully.
2029 @retval EFI_INVALID_PARAMETER The Procedure is NULL but ProcedureArguments not NULL.
2033 RegisterStartupProcedure (
2034 IN EFI_AP_PROCEDURE Procedure
,
2035 IN OUT VOID
*ProcedureArguments OPTIONAL
2038 if (Procedure
== NULL
&& ProcedureArguments
!= NULL
) {
2039 return EFI_INVALID_PARAMETER
;
2041 if (mSmmMpSyncData
== NULL
) {
2042 return EFI_NOT_READY
;
2045 mSmmMpSyncData
->StartupProcedure
= Procedure
;
2046 mSmmMpSyncData
->StartupProcArgs
= ProcedureArguments
;