2 SMM MP service implementation
4 Copyright (c) 2009 - 2020, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "PiSmmCpuDxeSmm.h"
14 // Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
16 MTRR_SETTINGS gSmiMtrrs
;
18 SMM_DISPATCHER_MP_SYNC_DATA
*mSmmMpSyncData
= NULL
;
19 UINTN mSmmMpSyncDataSize
;
20 SMM_CPU_SEMAPHORES mSmmCpuSemaphores
;
22 SPIN_LOCK
*mPFLock
= NULL
;
23 SMM_CPU_SYNC_MODE mCpuSmmSyncMode
;
24 BOOLEAN mMachineCheckSupported
= FALSE
;
26 extern UINTN mSmmShadowStackSize
;
29 Performs an atomic compare exchange operation to get semaphore.
30 The compare exchange operation must be performed using
33 @param Sem IN: 32-bit unsigned integer
34 OUT: original integer - 1
35 @return Original integer - 1
40 IN OUT
volatile UINT32
*Sem
48 InterlockedCompareExchange32 (
62 Performs an atomic compare exchange operation to release semaphore.
63 The compare exchange operation must be performed using
66 @param Sem IN: 32-bit unsigned integer
67 OUT: original integer + 1
68 @return Original integer + 1
73 IN OUT
volatile UINT32
*Sem
80 } while (Value
+ 1 != 0 &&
81 InterlockedCompareExchange32 (
90 Performs an atomic compare exchange operation to lock semaphore.
91 The compare exchange operation must be performed using
94 @param Sem IN: 32-bit unsigned integer
96 @return Original integer
101 IN OUT
volatile UINT32
*Sem
108 } while (InterlockedCompareExchange32 (
116 Wait all APs to performs an atomic compare exchange operation to release semaphore.
118 @param NumberOfAPs AP number
128 BspIndex
= mSmmMpSyncData
->BspIndex
;
129 while (NumberOfAPs
-- > 0) {
130 WaitForSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
135 Performs an atomic compare exchange operation to release semaphore
146 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
147 if (IsPresentAp (Index
)) {
148 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[Index
].Run
);
154 Checks if all CPUs (with certain exceptions) have checked in for this SMI run
156 @param Exceptions CPU Arrival exception flags.
158 @retval TRUE if all CPUs the have checked in.
159 @retval FALSE if at least one Normal AP hasn't checked in.
163 AllCpusInSmmWithExceptions (
164 SMM_CPU_ARRIVAL_EXCEPTIONS Exceptions
168 SMM_CPU_DATA_BLOCK
*CpuData
;
169 EFI_PROCESSOR_INFORMATION
*ProcessorInfo
;
171 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
173 if (*mSmmMpSyncData
->Counter
== mNumberOfCpus
) {
177 CpuData
= mSmmMpSyncData
->CpuData
;
178 ProcessorInfo
= gSmmCpuPrivate
->ProcessorInfo
;
179 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
180 if (!(*(CpuData
[Index
].Present
)) && ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
181 if (((Exceptions
& ARRIVAL_EXCEPTION_DELAYED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmDelayed
) != 0) {
184 if (((Exceptions
& ARRIVAL_EXCEPTION_BLOCKED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmBlocked
) != 0) {
187 if (((Exceptions
& ARRIVAL_EXCEPTION_SMI_DISABLED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmEnable
) != 0) {
199 Has OS enabled Lmce in the MSR_IA32_MCG_EXT_CTL
201 @retval TRUE Os enable lmce.
202 @retval FALSE Os not enable lmce.
210 MSR_IA32_MCG_CAP_REGISTER McgCap
;
211 MSR_IA32_FEATURE_CONTROL_REGISTER FeatureCtrl
;
212 MSR_IA32_MCG_EXT_CTL_REGISTER McgExtCtrl
;
214 McgCap
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_CAP
);
215 if (McgCap
.Bits
.MCG_LMCE_P
== 0) {
219 FeatureCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_FEATURE_CONTROL
);
220 if (FeatureCtrl
.Bits
.LmceOn
== 0) {
224 McgExtCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_EXT_CTL
);
225 return (BOOLEAN
) (McgExtCtrl
.Bits
.LMCE_EN
== 1);
229 Return if Local machine check exception signaled.
231 Indicates (when set) that a local machine check exception was generated. This indicates that the current machine-check event was
232 delivered to only the logical processor.
234 @retval TRUE LMCE was signaled.
235 @retval FALSE LMCE was not signaled.
243 MSR_IA32_MCG_STATUS_REGISTER McgStatus
;
245 McgStatus
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_STATUS
);
246 return (BOOLEAN
) (McgStatus
.Bits
.LMCE_S
== 1);
250 Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
251 entering SMM, except SMI disabled APs.
255 SmmWaitForApArrival (
264 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
268 if (mMachineCheckSupported
) {
269 LmceEn
= IsLmceOsEnabled ();
270 LmceSignal
= IsLmceSignaled();
274 // Platform implementor should choose a timeout value appropriately:
275 // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note
276 // the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.
277 // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI
278 // and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will
279 // be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the
280 // SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.
281 // - The timeout value must be longer than longest possible IO operation in the system
285 // Sync with APs 1st timeout
287 for (Timer
= StartSyncTimer ();
288 !IsSyncTimerTimeout (Timer
) && !(LmceEn
&& LmceSignal
) &&
289 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
295 // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,
297 // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running
298 // normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they
299 // enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode
300 // work while SMI handling is on-going.
301 // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.
302 // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state
303 // will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal
304 // mode work while SMI handling is on-going.
305 // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:
306 // - In traditional flow, SMI disabling is discouraged.
307 // - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
308 // In both cases, adding SMI-disabling checking code increases overhead.
310 if (*mSmmMpSyncData
->Counter
< mNumberOfCpus
) {
312 // Send SMI IPIs to bring outside processors in
314 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
315 if (!(*(mSmmMpSyncData
->CpuData
[Index
].Present
)) && gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
316 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
);
321 // Sync with APs 2nd timeout.
323 for (Timer
= StartSyncTimer ();
324 !IsSyncTimerTimeout (Timer
) &&
325 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
336 Replace OS MTRR's with SMI MTRR's.
338 @param CpuIndex Processor Index
346 SmmCpuFeaturesDisableSmrr ();
349 // Replace all MTRRs registers
351 MtrrSetAllMtrrs (&gSmiMtrrs
);
355 Wheck whether task has been finished by all APs.
357 @param BlockMode Whether did it in block mode or non-block mode.
359 @retval TRUE Task has been finished by all APs.
360 @retval FALSE Task not has been finished by all APs.
364 WaitForAllAPsNotBusy (
370 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
372 // Ignore BSP and APs which not call in SMM.
374 if (!IsPresentAp(Index
)) {
379 AcquireSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
380 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
382 if (AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
383 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
394 Check whether it is an present AP.
396 @param CpuIndex The AP index which calls this function.
398 @retval TRUE It's a present AP.
399 @retval TRUE This is not an AP or it is not present.
407 return ((CpuIndex
!= gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) &&
408 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
));
412 Clean up the status flags used during executing the procedure.
414 @param CpuIndex The AP index which calls this function.
422 PROCEDURE_TOKEN
*Token
;
424 Token
= mSmmMpSyncData
->CpuData
[CpuIndex
].Token
;
426 if (InterlockedDecrement (&Token
->RunningApCount
) == 0) {
427 ReleaseSpinLock (Token
->SpinLock
);
430 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= NULL
;
434 Free the tokens in the maintained list.
443 // Reset the FirstFreeToken to the beginning of token list upon exiting SMI.
445 gSmmCpuPrivate
->FirstFreeToken
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
451 @param CpuIndex BSP processor Index
452 @param SyncMode SMM MP sync mode
458 IN SMM_CPU_SYNC_MODE SyncMode
464 BOOLEAN ClearTopLevelSmiResult
;
467 ASSERT (CpuIndex
== mSmmMpSyncData
->BspIndex
);
471 // Flag BSP's presence
473 *mSmmMpSyncData
->InsideSmm
= TRUE
;
476 // Initialize Debug Agent to start source level debug in BSP handler
478 InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI
, NULL
, NULL
);
481 // Mark this processor's presence
483 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
486 // Clear platform top level SMI status bit before calling SMI handlers. If
487 // we cleared it after SMI handlers are run, we would miss the SMI that
488 // occurs after SMI handlers are done and before SMI status bit is cleared.
490 ClearTopLevelSmiResult
= ClearTopLevelSmiStatus();
491 ASSERT (ClearTopLevelSmiResult
== TRUE
);
494 // Set running processor index
496 gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
= CpuIndex
;
499 // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
501 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
504 // Wait for APs to arrive
506 SmmWaitForApArrival();
509 // Lock the counter down and retrieve the number of APs
511 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
512 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
515 // Wait for all APs to get ready for programming MTRRs
517 WaitForAllAPs (ApCount
);
519 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
521 // Signal all APs it's time for backup MTRRs
526 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
527 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
528 // to a large enough value to avoid this situation.
529 // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
530 // We do the backup first and then set MTRR to avoid race condition for threads
533 MtrrGetAllMtrrs(&Mtrrs
);
536 // Wait for all APs to complete their MTRR saving
538 WaitForAllAPs (ApCount
);
541 // Let all processors program SMM MTRRs together
546 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
547 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
548 // to a large enough value to avoid this situation.
550 ReplaceOSMtrrs (CpuIndex
);
553 // Wait for all APs to complete their MTRR programming
555 WaitForAllAPs (ApCount
);
560 // The BUSY lock is initialized to Acquired state
562 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
565 // Perform the pre tasks
570 // Invoke SMM Foundation EntryPoint with the processor information context.
572 gSmmCpuPrivate
->SmmCoreEntry (&gSmmCpuPrivate
->SmmCoreEntryContext
);
575 // Make sure all APs have completed their pending none-block tasks
577 WaitForAllAPsNotBusy (TRUE
);
580 // Perform the remaining tasks
582 PerformRemainingTasks ();
585 // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
586 // make those APs to exit SMI synchronously. APs which arrive later will be excluded and
587 // will run through freely.
589 if (SyncMode
!= SmmCpuSyncModeTradition
&& !SmmCpuFeaturesNeedConfigureMtrrs()) {
592 // Lock the counter down and retrieve the number of APs
594 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
595 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
597 // Make sure all APs have their Present flag set
601 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
602 if (*(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
606 if (PresentCount
> ApCount
) {
613 // Notify all APs to exit
615 *mSmmMpSyncData
->InsideSmm
= FALSE
;
619 // Wait for all APs to complete their pending tasks
621 WaitForAllAPs (ApCount
);
623 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
625 // Signal APs to restore MTRRs
632 SmmCpuFeaturesReenableSmrr ();
633 MtrrSetAllMtrrs(&Mtrrs
);
636 // Wait for all APs to complete MTRR programming
638 WaitForAllAPs (ApCount
);
642 // Stop source level debug in BSP handler, the code below will not be
645 InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI
, NULL
, NULL
);
648 // Signal APs to Reset states/semaphore for this processor
653 // Perform pending operations for hot-plug
658 // Clear the Present flag of BSP
660 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
663 // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
664 // WaitForAllAps does not depend on the Present flag.
666 WaitForAllAPs (ApCount
);
669 // Reset the tokens buffer.
674 // Reset BspIndex to -1, meaning BSP has not been elected.
676 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
677 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
681 // Allow APs to check in from this point on
683 *mSmmMpSyncData
->Counter
= 0;
684 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
690 @param CpuIndex AP processor Index.
691 @param ValidSmi Indicates that current SMI is a valid SMI or not.
692 @param SyncMode SMM MP sync mode.
699 IN SMM_CPU_SYNC_MODE SyncMode
705 EFI_STATUS ProcedureStatus
;
710 for (Timer
= StartSyncTimer ();
711 !IsSyncTimerTimeout (Timer
) &&
712 !(*mSmmMpSyncData
->InsideSmm
);
717 if (!(*mSmmMpSyncData
->InsideSmm
)) {
719 // BSP timeout in the first round
721 if (mSmmMpSyncData
->BspIndex
!= -1) {
723 // BSP Index is known
725 BspIndex
= mSmmMpSyncData
->BspIndex
;
726 ASSERT (CpuIndex
!= BspIndex
);
729 // Send SMI IPI to bring BSP in
731 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[BspIndex
].ProcessorId
);
734 // Now clock BSP for the 2nd time
736 for (Timer
= StartSyncTimer ();
737 !IsSyncTimerTimeout (Timer
) &&
738 !(*mSmmMpSyncData
->InsideSmm
);
743 if (!(*mSmmMpSyncData
->InsideSmm
)) {
745 // Give up since BSP is unable to enter SMM
746 // and signal the completion of this AP
747 WaitForSemaphore (mSmmMpSyncData
->Counter
);
752 // Don't know BSP index. Give up without sending IPI to BSP.
754 WaitForSemaphore (mSmmMpSyncData
->Counter
);
762 BspIndex
= mSmmMpSyncData
->BspIndex
;
763 ASSERT (CpuIndex
!= BspIndex
);
766 // Mark this processor's presence
768 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
770 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
772 // Notify BSP of arrival at this point
774 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
777 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
779 // Wait for the signal from BSP to backup MTRRs
781 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
786 MtrrGetAllMtrrs(&Mtrrs
);
789 // Signal BSP the completion of this AP
791 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
794 // Wait for BSP's signal to program MTRRs
796 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
799 // Replace OS MTRRs with SMI MTRRs
801 ReplaceOSMtrrs (CpuIndex
);
804 // Signal BSP the completion of this AP
806 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
811 // Wait for something to happen
813 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
816 // Check if BSP wants to exit SMM
818 if (!(*mSmmMpSyncData
->InsideSmm
)) {
823 // BUSY should be acquired by SmmStartupThisAp()
826 !AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)
830 // Invoke the scheduled procedure
832 ProcedureStatus
= (*mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
) (
833 (VOID
*)mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
835 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
836 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= ProcedureStatus
;
839 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Token
!= NULL
) {
840 ReleaseToken (CpuIndex
);
846 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
849 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
851 // Notify BSP the readiness of this AP to program MTRRs
853 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
856 // Wait for the signal from BSP to program MTRRs
858 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
863 SmmCpuFeaturesReenableSmrr ();
864 MtrrSetAllMtrrs(&Mtrrs
);
868 // Notify BSP the readiness of this AP to Reset states/semaphore for this processor
870 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
873 // Wait for the signal from BSP to Reset states/semaphore for this processor
875 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
878 // Reset states/semaphore for this processor
880 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
883 // Notify BSP the readiness of this AP to exit SMM
885 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
890 Create 4G PageTable in SMRAM.
892 @param[in] Is32BitPageTable Whether the page table is 32-bit PAE
893 @return PageTable Address
898 IN BOOLEAN Is32BitPageTable
906 UINTN High2MBoundary
;
916 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
918 // Add one more page for known good stack, then find the lower 2MB aligned address.
920 Low2MBoundary
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
) & ~(SIZE_2MB
-1);
922 // Add two more pages for known good stack and stack guard page,
923 // then find the lower 2MB aligned address.
925 High2MBoundary
= (mSmmStackArrayEnd
- mSmmStackSize
- mSmmShadowStackSize
+ EFI_PAGE_SIZE
* 2) & ~(SIZE_2MB
-1);
926 PagesNeeded
= ((High2MBoundary
- Low2MBoundary
) / SIZE_2MB
) + 1;
929 // Allocate the page table
931 PageTable
= AllocatePageTableMemory (5 + PagesNeeded
);
932 ASSERT (PageTable
!= NULL
);
934 PageTable
= (VOID
*)((UINTN
)PageTable
);
935 Pte
= (UINT64
*)PageTable
;
938 // Zero out all page table entries first
940 ZeroMem (Pte
, EFI_PAGES_TO_SIZE (1));
943 // Set Page Directory Pointers
945 for (Index
= 0; Index
< 4; Index
++) {
946 Pte
[Index
] = ((UINTN
)PageTable
+ EFI_PAGE_SIZE
* (Index
+ 1)) | mAddressEncMask
|
947 (Is32BitPageTable
? IA32_PAE_PDPTE_ATTRIBUTE_BITS
: PAGE_ATTRIBUTE_BITS
);
949 Pte
+= EFI_PAGE_SIZE
/ sizeof (*Pte
);
952 // Fill in Page Directory Entries
954 for (Index
= 0; Index
< EFI_PAGE_SIZE
* 4 / sizeof (*Pte
); Index
++) {
955 Pte
[Index
] = (Index
<< 21) | mAddressEncMask
| IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
958 Pdpte
= (UINT64
*)PageTable
;
959 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
960 Pages
= (UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (5);
961 GuardPage
= mSmmStackArrayBase
+ EFI_PAGE_SIZE
;
962 for (PageIndex
= Low2MBoundary
; PageIndex
<= High2MBoundary
; PageIndex
+= SIZE_2MB
) {
963 Pte
= (UINT64
*)(UINTN
)(Pdpte
[BitFieldRead32 ((UINT32
)PageIndex
, 30, 31)] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
964 Pte
[BitFieldRead32 ((UINT32
)PageIndex
, 21, 29)] = (UINT64
)Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
966 // Fill in Page Table Entries
968 Pte
= (UINT64
*)Pages
;
969 PageAddress
= PageIndex
;
970 for (Index
= 0; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
971 if (PageAddress
== GuardPage
) {
973 // Mark the guard page as non-present
975 Pte
[Index
] = PageAddress
| mAddressEncMask
;
976 GuardPage
+= (mSmmStackSize
+ mSmmShadowStackSize
);
977 if (GuardPage
> mSmmStackArrayEnd
) {
981 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
983 PageAddress
+= EFI_PAGE_SIZE
;
985 Pages
+= EFI_PAGE_SIZE
;
989 if ((PcdGet8 (PcdNullPointerDetectionPropertyMask
) & BIT1
) != 0) {
990 Pte
= (UINT64
*)(UINTN
)(Pdpte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
991 if ((Pte
[0] & IA32_PG_PS
) == 0) {
992 // 4K-page entries are already mapped. Just hide the first one anyway.
993 Pte
= (UINT64
*)(UINTN
)(Pte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
994 Pte
[0] &= ~(UINT64
)IA32_PG_P
; // Hide page 0
996 // Create 4K-page entries
997 Pages
= (UINTN
)AllocatePageTableMemory (1);
1000 Pte
[0] = (UINT64
)(Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
);
1002 Pte
= (UINT64
*)Pages
;
1004 Pte
[0] = PageAddress
| mAddressEncMask
; // Hide page 0 but present left
1005 for (Index
= 1; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
1006 PageAddress
+= EFI_PAGE_SIZE
;
1007 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
1012 return (UINT32
)(UINTN
)PageTable
;
1016 Checks whether the input token is the current used token.
1018 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1021 @retval TRUE The input token is the current used token.
1022 @retval FALSE The input token is not the current used token.
1030 PROCEDURE_TOKEN
*ProcToken
;
1032 if (Token
== NULL
) {
1036 Link
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
1038 // Only search used tokens.
1040 while (Link
!= gSmmCpuPrivate
->FirstFreeToken
) {
1041 ProcToken
= PROCEDURE_TOKEN_FROM_LINK (Link
);
1043 if (ProcToken
->SpinLock
== Token
) {
1047 Link
= GetNextNode (&gSmmCpuPrivate
->TokenList
, Link
);
1054 Allocate buffer for the SPIN_LOCK and PROCEDURE_TOKEN.
1056 @return First token of the token buffer.
1059 AllocateTokenBuffer (
1064 UINT32 TokenCountPerChunk
;
1066 SPIN_LOCK
*SpinLock
;
1067 UINT8
*SpinLockBuffer
;
1068 PROCEDURE_TOKEN
*ProcTokens
;
1070 SpinLockSize
= GetSpinLockProperties ();
1072 TokenCountPerChunk
= FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk
);
1073 ASSERT (TokenCountPerChunk
!= 0);
1074 if (TokenCountPerChunk
== 0) {
1075 DEBUG ((DEBUG_ERROR
, "PcdCpuSmmMpTokenCountPerChunk should not be Zero!\n"));
1078 DEBUG ((DEBUG_INFO
, "CpuSmm: SpinLock Size = 0x%x, PcdCpuSmmMpTokenCountPerChunk = 0x%x\n", SpinLockSize
, TokenCountPerChunk
));
1081 // Separate the Spin_lock and Proc_token because the alignment requires by Spin_Lock.
1083 SpinLockBuffer
= AllocatePool (SpinLockSize
* TokenCountPerChunk
);
1084 ASSERT (SpinLockBuffer
!= NULL
);
1086 ProcTokens
= AllocatePool (sizeof (PROCEDURE_TOKEN
) * TokenCountPerChunk
);
1087 ASSERT (ProcTokens
!= NULL
);
1089 for (Index
= 0; Index
< TokenCountPerChunk
; Index
++) {
1090 SpinLock
= (SPIN_LOCK
*)(SpinLockBuffer
+ SpinLockSize
* Index
);
1091 InitializeSpinLock (SpinLock
);
1093 ProcTokens
[Index
].Signature
= PROCEDURE_TOKEN_SIGNATURE
;
1094 ProcTokens
[Index
].SpinLock
= SpinLock
;
1095 ProcTokens
[Index
].RunningApCount
= 0;
1097 InsertTailList (&gSmmCpuPrivate
->TokenList
, &ProcTokens
[Index
].Link
);
1100 return &ProcTokens
[0].Link
;
1106 If no free token, allocate new tokens then return the free one.
1108 @param RunningApsCount The Running Aps count for this token.
1110 @retval return the first free PROCEDURE_TOKEN.
1115 IN UINT32 RunningApsCount
1118 PROCEDURE_TOKEN
*NewToken
;
1121 // If FirstFreeToken meets the end of token list, enlarge the token list.
1122 // Set FirstFreeToken to the first free token.
1124 if (gSmmCpuPrivate
->FirstFreeToken
== &gSmmCpuPrivate
->TokenList
) {
1125 gSmmCpuPrivate
->FirstFreeToken
= AllocateTokenBuffer ();
1127 NewToken
= PROCEDURE_TOKEN_FROM_LINK (gSmmCpuPrivate
->FirstFreeToken
);
1128 gSmmCpuPrivate
->FirstFreeToken
= GetNextNode (&gSmmCpuPrivate
->TokenList
, gSmmCpuPrivate
->FirstFreeToken
);
1130 NewToken
->RunningApCount
= RunningApsCount
;
1131 AcquireSpinLock (NewToken
->SpinLock
);
1137 Checks status of specified AP.
1139 This function checks whether the specified AP has finished the task assigned
1140 by StartupThisAP(), and whether timeout expires.
1142 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1145 @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
1146 @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
1153 if (AcquireSpinLockOrFail (Token
)) {
1154 ReleaseSpinLock (Token
);
1158 return EFI_NOT_READY
;
1162 Schedule a procedure to run on the specified CPU.
1164 @param[in] Procedure The address of the procedure to run
1165 @param[in] CpuIndex Target CPU Index
1166 @param[in,out] ProcArguments The parameter to pass to the procedure
1167 @param[in] Token This is an optional parameter that allows the caller to execute the
1168 procedure in a blocking or non-blocking fashion. If it is NULL the
1169 call is blocking, and the call will not return until the AP has
1170 completed the procedure. If the token is not NULL, the call will
1171 return immediately. The caller can check whether the procedure has
1172 completed with CheckOnProcedure or WaitForProcedure.
1173 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for the APs to finish
1174 execution of Procedure, either for blocking or non-blocking mode.
1175 Zero means infinity. If the timeout expires before all APs return
1176 from Procedure, then Procedure on the failed APs is terminated. If
1177 the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
1178 If the timeout expires in non-blocking mode, the timeout determined
1179 can be through CheckOnProcedure or WaitForProcedure.
1180 Note that timeout support is optional. Whether an implementation
1181 supports this feature can be determined via the Attributes data
1183 @param[in,out] CpuStatus This optional pointer may be used to get the status code returned
1184 by Procedure when it completes execution on the target AP, or with
1185 EFI_TIMEOUT if the Procedure fails to complete within the optional
1186 timeout. The implementation will update this variable with
1187 EFI_NOT_READY prior to starting Procedure on the target AP.
1189 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1190 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1191 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1192 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1193 @retval EFI_SUCCESS The procedure has been successfully scheduled
1197 InternalSmmStartupThisAp (
1198 IN EFI_AP_PROCEDURE2 Procedure
,
1200 IN OUT VOID
*ProcArguments OPTIONAL
,
1201 IN MM_COMPLETION
*Token
,
1202 IN UINTN TimeoutInMicroseconds
,
1203 IN OUT EFI_STATUS
*CpuStatus
1206 PROCEDURE_TOKEN
*ProcToken
;
1208 if (CpuIndex
>= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
) {
1209 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus(%d)\n", CpuIndex
, gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
));
1210 return EFI_INVALID_PARAMETER
;
1212 if (CpuIndex
== gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) {
1213 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu\n", CpuIndex
));
1214 return EFI_INVALID_PARAMETER
;
1216 if (gSmmCpuPrivate
->ProcessorInfo
[CpuIndex
].ProcessorId
== INVALID_APIC_ID
) {
1217 return EFI_INVALID_PARAMETER
;
1219 if (!(*(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
))) {
1220 if (mSmmMpSyncData
->EffectiveSyncMode
== SmmCpuSyncModeTradition
) {
1221 DEBUG((DEBUG_ERROR
, "!mSmmMpSyncData->CpuData[%d].Present\n", CpuIndex
));
1223 return EFI_INVALID_PARAMETER
;
1225 if (gSmmCpuPrivate
->Operation
[CpuIndex
] == SmmCpuRemove
) {
1226 if (!FeaturePcdGet (PcdCpuHotPlugSupport
)) {
1227 DEBUG((DEBUG_ERROR
, "gSmmCpuPrivate->Operation[%d] == SmmCpuRemove\n", CpuIndex
));
1229 return EFI_INVALID_PARAMETER
;
1231 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1232 return EFI_INVALID_PARAMETER
;
1234 if (Procedure
== NULL
) {
1235 return EFI_INVALID_PARAMETER
;
1238 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1240 mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
= Procedure
;
1241 mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
= ProcArguments
;
1242 if (Token
!= NULL
) {
1243 ProcToken
= GetFreeToken (1);
1244 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= ProcToken
;
1245 *Token
= (MM_COMPLETION
)ProcToken
->SpinLock
;
1247 mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= CpuStatus
;
1248 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
1249 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= EFI_NOT_READY
;
1252 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
1254 if (Token
== NULL
) {
1255 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1256 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1263 Worker function to execute a caller provided function on all enabled APs.
1265 @param[in] Procedure A pointer to the function to be run on
1266 enabled APs of the system.
1267 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
1268 APs to return from Procedure, either for
1269 blocking or non-blocking mode.
1270 @param[in,out] ProcedureArguments The parameter passed into Procedure for
1272 @param[in,out] Token This is an optional parameter that allows the caller to execute the
1273 procedure in a blocking or non-blocking fashion. If it is NULL the
1274 call is blocking, and the call will not return until the AP has
1275 completed the procedure. If the token is not NULL, the call will
1276 return immediately. The caller can check whether the procedure has
1277 completed with CheckOnProcedure or WaitForProcedure.
1278 @param[in,out] CPUStatus This optional pointer may be used to get the status code returned
1279 by Procedure when it completes execution on the target AP, or with
1280 EFI_TIMEOUT if the Procedure fails to complete within the optional
1281 timeout. The implementation will update this variable with
1282 EFI_NOT_READY prior to starting Procedure on the target AP.
1285 @retval EFI_SUCCESS In blocking mode, all APs have finished before
1286 the timeout expired.
1287 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
1289 @retval others Failed to Startup all APs.
1293 InternalSmmStartupAllAPs (
1294 IN EFI_AP_PROCEDURE2 Procedure
,
1295 IN UINTN TimeoutInMicroseconds
,
1296 IN OUT VOID
*ProcedureArguments OPTIONAL
,
1297 IN OUT MM_COMPLETION
*Token
,
1298 IN OUT EFI_STATUS
*CPUStatus
1303 PROCEDURE_TOKEN
*ProcToken
;
1305 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1306 return EFI_INVALID_PARAMETER
;
1308 if (Procedure
== NULL
) {
1309 return EFI_INVALID_PARAMETER
;
1313 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1314 if (IsPresentAp (Index
)) {
1317 if (gSmmCpuPrivate
->Operation
[Index
] == SmmCpuRemove
) {
1318 return EFI_INVALID_PARAMETER
;
1321 if (!AcquireSpinLockOrFail(mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
1322 return EFI_NOT_READY
;
1324 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1327 if (CpuCount
== 0) {
1328 return EFI_NOT_STARTED
;
1331 if (Token
!= NULL
) {
1332 ProcToken
= GetFreeToken ((UINT32
)mMaxNumberOfCpus
);
1333 *Token
= (MM_COMPLETION
)ProcToken
->SpinLock
;
1339 // Make sure all BUSY should be acquired.
1341 // Because former code already check mSmmMpSyncData->CpuData[***].Busy for each AP.
1342 // Here code always use AcquireSpinLock instead of AcquireSpinLockOrFail for not
1345 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1346 if (IsPresentAp (Index
)) {
1347 AcquireSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1351 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1352 if (IsPresentAp (Index
)) {
1353 mSmmMpSyncData
->CpuData
[Index
].Procedure
= (EFI_AP_PROCEDURE2
) Procedure
;
1354 mSmmMpSyncData
->CpuData
[Index
].Parameter
= ProcedureArguments
;
1355 if (ProcToken
!= NULL
) {
1356 mSmmMpSyncData
->CpuData
[Index
].Token
= ProcToken
;
1358 if (CPUStatus
!= NULL
) {
1359 mSmmMpSyncData
->CpuData
[Index
].Status
= &CPUStatus
[Index
];
1360 if (mSmmMpSyncData
->CpuData
[Index
].Status
!= NULL
) {
1361 *mSmmMpSyncData
->CpuData
[Index
].Status
= EFI_NOT_READY
;
1366 // PI spec requirement:
1367 // For every excluded processor, the array entry must contain a value of EFI_NOT_STARTED.
1369 if (CPUStatus
!= NULL
) {
1370 CPUStatus
[Index
] = EFI_NOT_STARTED
;
1374 // Decrease the count to mark this processor(AP or BSP) as finished.
1376 if (ProcToken
!= NULL
) {
1377 WaitForSemaphore (&ProcToken
->RunningApCount
);
1384 if (Token
== NULL
) {
1386 // Make sure all APs have completed their tasks.
1388 WaitForAllAPsNotBusy (TRUE
);
1395 ISO C99 6.5.2.2 "Function calls", paragraph 9:
1396 If the function is defined with a type that is not compatible with
1397 the type (of the expression) pointed to by the expression that
1398 denotes the called function, the behavior is undefined.
1400 So add below wrapper function to convert between EFI_AP_PROCEDURE
1401 and EFI_AP_PROCEDURE2.
1403 Wrapper for Procedures.
1405 @param[in] Buffer Pointer to PROCEDURE_WRAPPER buffer.
1414 PROCEDURE_WRAPPER
*Wrapper
;
1417 Wrapper
->Procedure (Wrapper
->ProcedureArgument
);
1423 Schedule a procedure to run on the specified CPU in blocking mode.
1425 @param[in] Procedure The address of the procedure to run
1426 @param[in] CpuIndex Target CPU Index
1427 @param[in, out] ProcArguments The parameter to pass to the procedure
1429 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1430 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1431 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1432 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1433 @retval EFI_SUCCESS The procedure has been successfully scheduled
1438 SmmBlockingStartupThisAp (
1439 IN EFI_AP_PROCEDURE Procedure
,
1441 IN OUT VOID
*ProcArguments OPTIONAL
1444 PROCEDURE_WRAPPER Wrapper
;
1446 Wrapper
.Procedure
= Procedure
;
1447 Wrapper
.ProcedureArgument
= ProcArguments
;
1450 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1452 return InternalSmmStartupThisAp (ProcedureWrapper
, CpuIndex
, &Wrapper
, NULL
, 0, NULL
);
1456 Schedule a procedure to run on the specified CPU.
1458 @param Procedure The address of the procedure to run
1459 @param CpuIndex Target CPU Index
1460 @param ProcArguments The parameter to pass to the procedure
1462 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1463 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1464 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1465 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1466 @retval EFI_SUCCESS The procedure has been successfully scheduled
1472 IN EFI_AP_PROCEDURE Procedure
,
1474 IN OUT VOID
*ProcArguments OPTIONAL
1477 MM_COMPLETION Token
;
1479 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].Procedure
= Procedure
;
1480 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].ProcedureArgument
= ProcArguments
;
1483 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1485 return InternalSmmStartupThisAp (
1488 &gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
],
1489 FeaturePcdGet (PcdCpuSmmBlockStartupThisAp
) ? NULL
: &Token
,
1496 This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
1497 They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
1499 NOTE: It might not be appreciated in runtime since it might
1500 conflict with OS debugging facilities. Turn them off in RELEASE.
1502 @param CpuIndex CPU Index
1511 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1513 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1514 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1515 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1516 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1517 AsmWriteDr6 (CpuSaveState
->x86
._DR6
);
1518 AsmWriteDr7 (CpuSaveState
->x86
._DR7
);
1520 AsmWriteDr6 ((UINTN
)CpuSaveState
->x64
._DR6
);
1521 AsmWriteDr7 ((UINTN
)CpuSaveState
->x64
._DR7
);
1527 This function restores DR6 & DR7 to SMM save state.
1529 NOTE: It might not be appreciated in runtime since it might
1530 conflict with OS debugging facilities. Turn them off in RELEASE.
1532 @param CpuIndex CPU Index
1541 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1543 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1544 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1545 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1546 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1547 CpuSaveState
->x86
._DR7
= (UINT32
)AsmReadDr7 ();
1548 CpuSaveState
->x86
._DR6
= (UINT32
)AsmReadDr6 ();
1550 CpuSaveState
->x64
._DR7
= AsmReadDr7 ();
1551 CpuSaveState
->x64
._DR6
= AsmReadDr6 ();
1557 C function for SMI entry, each processor comes here upon SMI trigger.
1559 @param CpuIndex CPU Index
1571 BOOLEAN BspInProgress
;
1575 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1578 // Save Cr2 because Page Fault exception in SMM may override its value,
1579 // when using on-demand paging for above 4G memory.
1585 // Call the user register Startup function first.
1587 if (mSmmMpSyncData
->StartupProcedure
!= NULL
) {
1588 mSmmMpSyncData
->StartupProcedure (mSmmMpSyncData
->StartupProcArgs
);
1592 // Perform CPU specific entry hooks
1594 SmmCpuFeaturesRendezvousEntry (CpuIndex
);
1597 // Determine if this is a valid SMI
1599 ValidSmi
= PlatformValidSmi();
1602 // Determine if BSP has been already in progress. Note this must be checked after
1603 // ValidSmi because BSP may clear a valid SMI source after checking in.
1605 BspInProgress
= *mSmmMpSyncData
->InsideSmm
;
1607 if (!BspInProgress
&& !ValidSmi
) {
1609 // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not
1610 // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI
1611 // status had been cleared by BSP and an existing SMI run has almost ended. (Note
1612 // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there
1613 // is nothing we need to do.
1618 // Signal presence of this processor
1620 if (ReleaseSemaphore (mSmmMpSyncData
->Counter
) == 0) {
1622 // BSP has already ended the synchronization, so QUIT!!!
1626 // Wait for BSP's signal to finish SMI
1628 while (*mSmmMpSyncData
->AllCpusInSync
) {
1635 // The BUSY lock is initialized to Released state.
1636 // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.
1637 // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
1638 // after AP's present flag is detected.
1640 InitializeSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1643 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1644 ActivateSmmProfile (CpuIndex
);
1647 if (BspInProgress
) {
1649 // BSP has been elected. Follow AP path, regardless of ValidSmi flag
1650 // as BSP may have cleared the SMI status
1652 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1655 // We have a valid SMI
1662 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1663 if (!mSmmMpSyncData
->SwitchBsp
|| mSmmMpSyncData
->CandidateBsp
[CpuIndex
]) {
1665 // Call platform hook to do BSP election
1667 Status
= PlatformSmmBspElection (&IsBsp
);
1668 if (EFI_SUCCESS
== Status
) {
1670 // Platform hook determines successfully
1673 mSmmMpSyncData
->BspIndex
= (UINT32
)CpuIndex
;
1677 // Platform hook fails to determine, use default BSP election method
1679 InterlockedCompareExchange32 (
1680 (UINT32
*)&mSmmMpSyncData
->BspIndex
,
1689 // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP
1691 if (mSmmMpSyncData
->BspIndex
== CpuIndex
) {
1694 // Clear last request for SwitchBsp.
1696 if (mSmmMpSyncData
->SwitchBsp
) {
1697 mSmmMpSyncData
->SwitchBsp
= FALSE
;
1698 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1699 mSmmMpSyncData
->CandidateBsp
[Index
] = FALSE
;
1703 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1704 SmmProfileRecordSmiNum ();
1708 // BSP Handler is always called with a ValidSmi == TRUE
1710 BSPHandler (CpuIndex
, mSmmMpSyncData
->EffectiveSyncMode
);
1712 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1716 ASSERT (*mSmmMpSyncData
->CpuData
[CpuIndex
].Run
== 0);
1719 // Wait for BSP's signal to exit SMI
1721 while (*mSmmMpSyncData
->AllCpusInSync
) {
1727 SmmCpuFeaturesRendezvousExit (CpuIndex
);
1736 Allocate buffer for SpinLock and Wrapper function buffer.
1740 InitializeDataForMmMp (
1744 gSmmCpuPrivate
->ApWrapperFunc
= AllocatePool (sizeof (PROCEDURE_WRAPPER
) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1745 ASSERT (gSmmCpuPrivate
->ApWrapperFunc
!= NULL
);
1747 InitializeListHead (&gSmmCpuPrivate
->TokenList
);
1749 gSmmCpuPrivate
->FirstFreeToken
= AllocateTokenBuffer ();
1753 Allocate buffer for all semaphores and spin locks.
1757 InitializeSmmCpuSemaphores (
1761 UINTN ProcessorCount
;
1763 UINTN GlobalSemaphoresSize
;
1764 UINTN CpuSemaphoresSize
;
1765 UINTN SemaphoreSize
;
1767 UINTN
*SemaphoreBlock
;
1768 UINTN SemaphoreAddr
;
1770 SemaphoreSize
= GetSpinLockProperties ();
1771 ProcessorCount
= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1772 GlobalSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_GLOBAL
) / sizeof (VOID
*)) * SemaphoreSize
;
1773 CpuSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_CPU
) / sizeof (VOID
*)) * ProcessorCount
* SemaphoreSize
;
1774 TotalSize
= GlobalSemaphoresSize
+ CpuSemaphoresSize
;
1775 DEBUG((EFI_D_INFO
, "One Semaphore Size = 0x%x\n", SemaphoreSize
));
1776 DEBUG((EFI_D_INFO
, "Total Semaphores Size = 0x%x\n", TotalSize
));
1777 Pages
= EFI_SIZE_TO_PAGES (TotalSize
);
1778 SemaphoreBlock
= AllocatePages (Pages
);
1779 ASSERT (SemaphoreBlock
!= NULL
);
1780 ZeroMem (SemaphoreBlock
, TotalSize
);
1782 SemaphoreAddr
= (UINTN
)SemaphoreBlock
;
1783 mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
= (UINT32
*)SemaphoreAddr
;
1784 SemaphoreAddr
+= SemaphoreSize
;
1785 mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
= (BOOLEAN
*)SemaphoreAddr
;
1786 SemaphoreAddr
+= SemaphoreSize
;
1787 mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
= (BOOLEAN
*)SemaphoreAddr
;
1788 SemaphoreAddr
+= SemaphoreSize
;
1789 mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
= (SPIN_LOCK
*)SemaphoreAddr
;
1790 SemaphoreAddr
+= SemaphoreSize
;
1791 mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
1792 = (SPIN_LOCK
*)SemaphoreAddr
;
1793 SemaphoreAddr
+= SemaphoreSize
;
1795 SemaphoreAddr
= (UINTN
)SemaphoreBlock
+ GlobalSemaphoresSize
;
1796 mSmmCpuSemaphores
.SemaphoreCpu
.Busy
= (SPIN_LOCK
*)SemaphoreAddr
;
1797 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1798 mSmmCpuSemaphores
.SemaphoreCpu
.Run
= (UINT32
*)SemaphoreAddr
;
1799 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1800 mSmmCpuSemaphores
.SemaphoreCpu
.Present
= (BOOLEAN
*)SemaphoreAddr
;
1802 mPFLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
;
1803 mConfigSmmCodeAccessCheckLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
;
1805 mSemaphoreSize
= SemaphoreSize
;
1809 Initialize un-cacheable data.
1814 InitializeMpSyncData (
1820 if (mSmmMpSyncData
!= NULL
) {
1822 // mSmmMpSyncDataSize includes one structure of SMM_DISPATCHER_MP_SYNC_DATA, one
1823 // CpuData array of SMM_CPU_DATA_BLOCK and one CandidateBsp array of BOOLEAN.
1825 ZeroMem (mSmmMpSyncData
, mSmmMpSyncDataSize
);
1826 mSmmMpSyncData
->CpuData
= (SMM_CPU_DATA_BLOCK
*)((UINT8
*)mSmmMpSyncData
+ sizeof (SMM_DISPATCHER_MP_SYNC_DATA
));
1827 mSmmMpSyncData
->CandidateBsp
= (BOOLEAN
*)(mSmmMpSyncData
->CpuData
+ gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1828 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1830 // Enable BSP election by setting BspIndex to -1
1832 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
1834 mSmmMpSyncData
->EffectiveSyncMode
= mCpuSmmSyncMode
;
1836 mSmmMpSyncData
->Counter
= mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
;
1837 mSmmMpSyncData
->InsideSmm
= mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
;
1838 mSmmMpSyncData
->AllCpusInSync
= mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
;
1839 ASSERT (mSmmMpSyncData
->Counter
!= NULL
&& mSmmMpSyncData
->InsideSmm
!= NULL
&&
1840 mSmmMpSyncData
->AllCpusInSync
!= NULL
);
1841 *mSmmMpSyncData
->Counter
= 0;
1842 *mSmmMpSyncData
->InsideSmm
= FALSE
;
1843 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
1845 for (CpuIndex
= 0; CpuIndex
< gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
; CpuIndex
++) {
1846 mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
=
1847 (SPIN_LOCK
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Busy
+ mSemaphoreSize
* CpuIndex
);
1848 mSmmMpSyncData
->CpuData
[CpuIndex
].Run
=
1849 (UINT32
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Run
+ mSemaphoreSize
* CpuIndex
);
1850 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
=
1851 (BOOLEAN
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Present
+ mSemaphoreSize
* CpuIndex
);
1852 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
) = 0;
1853 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Run
) = 0;
1854 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
1860 Initialize global data for MP synchronization.
1862 @param Stacks Base address of SMI stack buffer for all processors.
1863 @param StackSize Stack size for each processor in SMM.
1864 @param ShadowStackSize Shadow Stack size for each processor in SMM.
1868 InitializeMpServiceData (
1871 IN UINTN ShadowStackSize
1876 UINT8
*GdtTssTables
;
1877 UINTN GdtTableStepSize
;
1878 CPUID_VERSION_INFO_EDX RegEdx
;
1881 // Determine if this CPU supports machine check
1883 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
1884 mMachineCheckSupported
= (BOOLEAN
)(RegEdx
.Bits
.MCA
== 1);
1887 // Allocate memory for all locks and semaphores
1889 InitializeSmmCpuSemaphores ();
1892 // Initialize mSmmMpSyncData
1894 mSmmMpSyncDataSize
= sizeof (SMM_DISPATCHER_MP_SYNC_DATA
) +
1895 (sizeof (SMM_CPU_DATA_BLOCK
) + sizeof (BOOLEAN
)) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1896 mSmmMpSyncData
= (SMM_DISPATCHER_MP_SYNC_DATA
*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize
));
1897 ASSERT (mSmmMpSyncData
!= NULL
);
1898 mCpuSmmSyncMode
= (SMM_CPU_SYNC_MODE
)PcdGet8 (PcdCpuSmmSyncMode
);
1899 InitializeMpSyncData ();
1902 // Initialize physical address mask
1903 // NOTE: Physical memory above virtual address limit is not supported !!!
1905 AsmCpuid (0x80000008, (UINT32
*)&Index
, NULL
, NULL
, NULL
);
1906 gPhyMask
= LShiftU64 (1, (UINT8
)Index
) - 1;
1907 gPhyMask
&= (1ull << 48) - EFI_PAGE_SIZE
;
1910 // Create page tables
1912 Cr3
= SmmInitPageTable ();
1914 GdtTssTables
= InitGdt (Cr3
, &GdtTableStepSize
);
1917 // Install SMI handler for each CPU
1919 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1922 (UINT32
)mCpuHotPlugData
.SmBase
[Index
],
1923 (VOID
*)((UINTN
)Stacks
+ (StackSize
+ ShadowStackSize
) * Index
),
1925 (UINTN
)(GdtTssTables
+ GdtTableStepSize
* Index
),
1926 gcSmiGdtr
.Limit
+ 1,
1928 gcSmiIdtr
.Limit
+ 1,
1934 // Record current MTRR settings
1936 ZeroMem (&gSmiMtrrs
, sizeof (gSmiMtrrs
));
1937 MtrrGetAllMtrrs (&gSmiMtrrs
);
1944 Register the SMM Foundation entry point.
1946 @param This Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance
1947 @param SmmEntryPoint SMM Foundation EntryPoint
1949 @retval EFI_SUCCESS Successfully to register SMM foundation entry point
1955 IN CONST EFI_SMM_CONFIGURATION_PROTOCOL
*This
,
1956 IN EFI_SMM_ENTRY_POINT SmmEntryPoint
1960 // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.
1962 gSmmCpuPrivate
->SmmCoreEntry
= SmmEntryPoint
;
1968 Register the SMM Foundation entry point.
1970 @param[in] Procedure A pointer to the code stream to be run on the designated target AP
1971 of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
1972 with the related definitions of
1973 EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
1974 If caller may pass a value of NULL to deregister any existing
1976 @param[in,out] ProcedureArguments Allows the caller to pass a list of parameters to the code that is
1977 run by the AP. It is an optional common mailbox between APs and
1978 the caller to share information
1980 @retval EFI_SUCCESS The Procedure has been set successfully.
1981 @retval EFI_INVALID_PARAMETER The Procedure is NULL but ProcedureArguments not NULL.
1985 RegisterStartupProcedure (
1986 IN EFI_AP_PROCEDURE Procedure
,
1987 IN OUT VOID
*ProcedureArguments OPTIONAL
1990 if (Procedure
== NULL
&& ProcedureArguments
!= NULL
) {
1991 return EFI_INVALID_PARAMETER
;
1993 if (mSmmMpSyncData
== NULL
) {
1994 return EFI_NOT_READY
;
1997 mSmmMpSyncData
->StartupProcedure
= Procedure
;
1998 mSmmMpSyncData
->StartupProcArgs
= ProcedureArguments
;