2 SMM MP service implementation
4 Copyright (c) 2009 - 2020, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "PiSmmCpuDxeSmm.h"
14 // Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
16 MTRR_SETTINGS gSmiMtrrs
;
18 SMM_DISPATCHER_MP_SYNC_DATA
*mSmmMpSyncData
= NULL
;
19 UINTN mSmmMpSyncDataSize
;
20 SMM_CPU_SEMAPHORES mSmmCpuSemaphores
;
22 SPIN_LOCK
*mPFLock
= NULL
;
23 SMM_CPU_SYNC_MODE mCpuSmmSyncMode
;
24 BOOLEAN mMachineCheckSupported
= FALSE
;
27 Performs an atomic compare exchange operation to get semaphore.
28 The compare exchange operation must be performed using
31 @param Sem IN: 32-bit unsigned integer
32 OUT: original integer - 1
33 @return Original integer - 1
38 IN OUT
volatile UINT32
*Sem
45 } while (Value
== 0 ||
46 InterlockedCompareExchange32 (
56 Performs an atomic compare exchange operation to release semaphore.
57 The compare exchange operation must be performed using
60 @param Sem IN: 32-bit unsigned integer
61 OUT: original integer + 1
62 @return Original integer + 1
67 IN OUT
volatile UINT32
*Sem
74 } while (Value
+ 1 != 0 &&
75 InterlockedCompareExchange32 (
84 Performs an atomic compare exchange operation to lock semaphore.
85 The compare exchange operation must be performed using
88 @param Sem IN: 32-bit unsigned integer
90 @return Original integer
95 IN OUT
volatile UINT32
*Sem
102 } while (InterlockedCompareExchange32 (
110 Wait all APs to performs an atomic compare exchange operation to release semaphore.
112 @param NumberOfAPs AP number
122 BspIndex
= mSmmMpSyncData
->BspIndex
;
123 while (NumberOfAPs
-- > 0) {
124 WaitForSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
129 Performs an atomic compare exchange operation to release semaphore
140 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
141 if (IsPresentAp (Index
)) {
142 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[Index
].Run
);
148 Checks if all CPUs (with certain exceptions) have checked in for this SMI run
150 @param Exceptions CPU Arrival exception flags.
152 @retval TRUE if all CPUs the have checked in.
153 @retval FALSE if at least one Normal AP hasn't checked in.
157 AllCpusInSmmWithExceptions (
158 SMM_CPU_ARRIVAL_EXCEPTIONS Exceptions
162 SMM_CPU_DATA_BLOCK
*CpuData
;
163 EFI_PROCESSOR_INFORMATION
*ProcessorInfo
;
165 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
167 if (*mSmmMpSyncData
->Counter
== mNumberOfCpus
) {
171 CpuData
= mSmmMpSyncData
->CpuData
;
172 ProcessorInfo
= gSmmCpuPrivate
->ProcessorInfo
;
173 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
174 if (!(*(CpuData
[Index
].Present
)) && ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
175 if (((Exceptions
& ARRIVAL_EXCEPTION_DELAYED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmDelayed
) != 0) {
178 if (((Exceptions
& ARRIVAL_EXCEPTION_BLOCKED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmBlocked
) != 0) {
181 if (((Exceptions
& ARRIVAL_EXCEPTION_SMI_DISABLED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmEnable
) != 0) {
193 Has OS enabled Lmce in the MSR_IA32_MCG_EXT_CTL
195 @retval TRUE Os enable lmce.
196 @retval FALSE Os not enable lmce.
204 MSR_IA32_MCG_CAP_REGISTER McgCap
;
205 MSR_IA32_FEATURE_CONTROL_REGISTER FeatureCtrl
;
206 MSR_IA32_MCG_EXT_CTL_REGISTER McgExtCtrl
;
208 McgCap
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_CAP
);
209 if (McgCap
.Bits
.MCG_LMCE_P
== 0) {
213 FeatureCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_FEATURE_CONTROL
);
214 if (FeatureCtrl
.Bits
.LmceOn
== 0) {
218 McgExtCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_EXT_CTL
);
219 return (BOOLEAN
) (McgExtCtrl
.Bits
.LMCE_EN
== 1);
223 Return if Local machine check exception signaled.
225 Indicates (when set) that a local machine check exception was generated. This indicates that the current machine-check event was
226 delivered to only the logical processor.
228 @retval TRUE LMCE was signaled.
229 @retval FALSE LMCE was not signaled.
237 MSR_IA32_MCG_STATUS_REGISTER McgStatus
;
239 McgStatus
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_STATUS
);
240 return (BOOLEAN
) (McgStatus
.Bits
.LMCE_S
== 1);
244 Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
245 entering SMM, except SMI disabled APs.
249 SmmWaitForApArrival (
258 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
262 if (mMachineCheckSupported
) {
263 LmceEn
= IsLmceOsEnabled ();
264 LmceSignal
= IsLmceSignaled();
268 // Platform implementor should choose a timeout value appropriately:
269 // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note
270 // the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.
271 // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI
272 // and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will
273 // be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the
274 // SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.
275 // - The timeout value must be longer than longest possible IO operation in the system
279 // Sync with APs 1st timeout
281 for (Timer
= StartSyncTimer ();
282 !IsSyncTimerTimeout (Timer
) && !(LmceEn
&& LmceSignal
) &&
283 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
289 // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,
291 // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running
292 // normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they
293 // enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode
294 // work while SMI handling is on-going.
295 // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.
296 // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state
297 // will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal
298 // mode work while SMI handling is on-going.
299 // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:
300 // - In traditional flow, SMI disabling is discouraged.
301 // - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
302 // In both cases, adding SMI-disabling checking code increases overhead.
304 if (*mSmmMpSyncData
->Counter
< mNumberOfCpus
) {
306 // Send SMI IPIs to bring outside processors in
308 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
309 if (!(*(mSmmMpSyncData
->CpuData
[Index
].Present
)) && gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
310 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
);
315 // Sync with APs 2nd timeout.
317 for (Timer
= StartSyncTimer ();
318 !IsSyncTimerTimeout (Timer
) &&
319 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
330 Replace OS MTRR's with SMI MTRR's.
332 @param CpuIndex Processor Index
340 SmmCpuFeaturesDisableSmrr ();
343 // Replace all MTRRs registers
345 MtrrSetAllMtrrs (&gSmiMtrrs
);
349 Wheck whether task has been finished by all APs.
351 @param BlockMode Whether did it in block mode or non-block mode.
353 @retval TRUE Task has been finished by all APs.
354 @retval FALSE Task not has been finished by all APs.
358 WaitForAllAPsNotBusy (
364 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
366 // Ignore BSP and APs which not call in SMM.
368 if (!IsPresentAp(Index
)) {
373 AcquireSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
374 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
376 if (AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
377 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
388 Check whether it is an present AP.
390 @param CpuIndex The AP index which calls this function.
392 @retval TRUE It's a present AP.
393 @retval TRUE This is not an AP or it is not present.
401 return ((CpuIndex
!= gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) &&
402 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
));
406 Clean up the status flags used during executing the procedure.
408 @param CpuIndex The AP index which calls this function.
416 PROCEDURE_TOKEN
*Token
;
418 Token
= mSmmMpSyncData
->CpuData
[CpuIndex
].Token
;
420 if (InterlockedDecrement (&Token
->RunningApCount
) == 0) {
421 ReleaseSpinLock (Token
->SpinLock
);
424 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= NULL
;
428 Free the tokens in the maintained list.
437 PROCEDURE_TOKEN
*ProcToken
;
439 Link
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
440 while (!IsNull (&gSmmCpuPrivate
->TokenList
, Link
)) {
441 ProcToken
= PROCEDURE_TOKEN_FROM_LINK (Link
);
443 ProcToken
->RunningApCount
= 0;
444 ProcToken
->Used
= FALSE
;
447 // Check the spinlock status and release it if not released yet.
449 if (!AcquireSpinLockOrFail(ProcToken
->SpinLock
)) {
450 DEBUG((DEBUG_ERROR
, "Risk::SpinLock still not released!"));
452 ReleaseSpinLock (ProcToken
->SpinLock
);
454 Link
= GetNextNode (&gSmmCpuPrivate
->TokenList
, Link
);
461 @param CpuIndex BSP processor Index
462 @param SyncMode SMM MP sync mode
468 IN SMM_CPU_SYNC_MODE SyncMode
474 BOOLEAN ClearTopLevelSmiResult
;
477 ASSERT (CpuIndex
== mSmmMpSyncData
->BspIndex
);
481 // Flag BSP's presence
483 *mSmmMpSyncData
->InsideSmm
= TRUE
;
486 // Initialize Debug Agent to start source level debug in BSP handler
488 InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI
, NULL
, NULL
);
491 // Mark this processor's presence
493 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
496 // Clear platform top level SMI status bit before calling SMI handlers. If
497 // we cleared it after SMI handlers are run, we would miss the SMI that
498 // occurs after SMI handlers are done and before SMI status bit is cleared.
500 ClearTopLevelSmiResult
= ClearTopLevelSmiStatus();
501 ASSERT (ClearTopLevelSmiResult
== TRUE
);
504 // Set running processor index
506 gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
= CpuIndex
;
509 // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
511 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
514 // Wait for APs to arrive
516 SmmWaitForApArrival();
519 // Lock the counter down and retrieve the number of APs
521 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
522 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
525 // Wait for all APs to get ready for programming MTRRs
527 WaitForAllAPs (ApCount
);
529 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
531 // Signal all APs it's time for backup MTRRs
536 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
537 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
538 // to a large enough value to avoid this situation.
539 // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
540 // We do the backup first and then set MTRR to avoid race condition for threads
543 MtrrGetAllMtrrs(&Mtrrs
);
546 // Wait for all APs to complete their MTRR saving
548 WaitForAllAPs (ApCount
);
551 // Let all processors program SMM MTRRs together
556 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
557 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
558 // to a large enough value to avoid this situation.
560 ReplaceOSMtrrs (CpuIndex
);
563 // Wait for all APs to complete their MTRR programming
565 WaitForAllAPs (ApCount
);
570 // The BUSY lock is initialized to Acquired state
572 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
575 // Perform the pre tasks
580 // Invoke SMM Foundation EntryPoint with the processor information context.
582 gSmmCpuPrivate
->SmmCoreEntry (&gSmmCpuPrivate
->SmmCoreEntryContext
);
585 // Make sure all APs have completed their pending none-block tasks
587 WaitForAllAPsNotBusy (TRUE
);
590 // Perform the remaining tasks
592 PerformRemainingTasks ();
595 // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
596 // make those APs to exit SMI synchronously. APs which arrive later will be excluded and
597 // will run through freely.
599 if (SyncMode
!= SmmCpuSyncModeTradition
&& !SmmCpuFeaturesNeedConfigureMtrrs()) {
602 // Lock the counter down and retrieve the number of APs
604 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
605 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
607 // Make sure all APs have their Present flag set
611 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
612 if (*(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
616 if (PresentCount
> ApCount
) {
623 // Notify all APs to exit
625 *mSmmMpSyncData
->InsideSmm
= FALSE
;
629 // Wait for all APs to complete their pending tasks
631 WaitForAllAPs (ApCount
);
633 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
635 // Signal APs to restore MTRRs
642 SmmCpuFeaturesReenableSmrr ();
643 MtrrSetAllMtrrs(&Mtrrs
);
646 // Wait for all APs to complete MTRR programming
648 WaitForAllAPs (ApCount
);
652 // Stop source level debug in BSP handler, the code below will not be
655 InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI
, NULL
, NULL
);
658 // Signal APs to Reset states/semaphore for this processor
663 // Perform pending operations for hot-plug
668 // Clear the Present flag of BSP
670 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
673 // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
674 // WaitForAllAps does not depend on the Present flag.
676 WaitForAllAPs (ApCount
);
679 // Reset the tokens buffer.
684 // Reset BspIndex to -1, meaning BSP has not been elected.
686 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
687 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
691 // Allow APs to check in from this point on
693 *mSmmMpSyncData
->Counter
= 0;
694 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
700 @param CpuIndex AP processor Index.
701 @param ValidSmi Indicates that current SMI is a valid SMI or not.
702 @param SyncMode SMM MP sync mode.
709 IN SMM_CPU_SYNC_MODE SyncMode
715 EFI_STATUS ProcedureStatus
;
720 for (Timer
= StartSyncTimer ();
721 !IsSyncTimerTimeout (Timer
) &&
722 !(*mSmmMpSyncData
->InsideSmm
);
727 if (!(*mSmmMpSyncData
->InsideSmm
)) {
729 // BSP timeout in the first round
731 if (mSmmMpSyncData
->BspIndex
!= -1) {
733 // BSP Index is known
735 BspIndex
= mSmmMpSyncData
->BspIndex
;
736 ASSERT (CpuIndex
!= BspIndex
);
739 // Send SMI IPI to bring BSP in
741 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[BspIndex
].ProcessorId
);
744 // Now clock BSP for the 2nd time
746 for (Timer
= StartSyncTimer ();
747 !IsSyncTimerTimeout (Timer
) &&
748 !(*mSmmMpSyncData
->InsideSmm
);
753 if (!(*mSmmMpSyncData
->InsideSmm
)) {
755 // Give up since BSP is unable to enter SMM
756 // and signal the completion of this AP
757 WaitForSemaphore (mSmmMpSyncData
->Counter
);
762 // Don't know BSP index. Give up without sending IPI to BSP.
764 WaitForSemaphore (mSmmMpSyncData
->Counter
);
772 BspIndex
= mSmmMpSyncData
->BspIndex
;
773 ASSERT (CpuIndex
!= BspIndex
);
776 // Mark this processor's presence
778 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
780 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
782 // Notify BSP of arrival at this point
784 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
787 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
789 // Wait for the signal from BSP to backup MTRRs
791 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
796 MtrrGetAllMtrrs(&Mtrrs
);
799 // Signal BSP the completion of this AP
801 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
804 // Wait for BSP's signal to program MTRRs
806 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
809 // Replace OS MTRRs with SMI MTRRs
811 ReplaceOSMtrrs (CpuIndex
);
814 // Signal BSP the completion of this AP
816 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
821 // Wait for something to happen
823 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
826 // Check if BSP wants to exit SMM
828 if (!(*mSmmMpSyncData
->InsideSmm
)) {
833 // BUSY should be acquired by SmmStartupThisAp()
836 !AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)
840 // Invoke the scheduled procedure
842 ProcedureStatus
= (*mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
) (
843 (VOID
*)mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
845 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
846 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= ProcedureStatus
;
849 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Token
!= NULL
) {
850 ReleaseToken (CpuIndex
);
856 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
859 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
861 // Notify BSP the readiness of this AP to program MTRRs
863 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
866 // Wait for the signal from BSP to program MTRRs
868 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
873 SmmCpuFeaturesReenableSmrr ();
874 MtrrSetAllMtrrs(&Mtrrs
);
878 // Notify BSP the readiness of this AP to Reset states/semaphore for this processor
880 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
883 // Wait for the signal from BSP to Reset states/semaphore for this processor
885 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
888 // Reset states/semaphore for this processor
890 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
893 // Notify BSP the readiness of this AP to exit SMM
895 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
900 Create 4G PageTable in SMRAM.
902 @param[in] Is32BitPageTable Whether the page table is 32-bit PAE
903 @return PageTable Address
908 IN BOOLEAN Is32BitPageTable
916 UINTN High2MBoundary
;
926 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
928 // Add one more page for known good stack, then find the lower 2MB aligned address.
930 Low2MBoundary
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
) & ~(SIZE_2MB
-1);
932 // Add two more pages for known good stack and stack guard page,
933 // then find the lower 2MB aligned address.
935 High2MBoundary
= (mSmmStackArrayEnd
- mSmmStackSize
+ EFI_PAGE_SIZE
* 2) & ~(SIZE_2MB
-1);
936 PagesNeeded
= ((High2MBoundary
- Low2MBoundary
) / SIZE_2MB
) + 1;
939 // Allocate the page table
941 PageTable
= AllocatePageTableMemory (5 + PagesNeeded
);
942 ASSERT (PageTable
!= NULL
);
944 PageTable
= (VOID
*)((UINTN
)PageTable
);
945 Pte
= (UINT64
*)PageTable
;
948 // Zero out all page table entries first
950 ZeroMem (Pte
, EFI_PAGES_TO_SIZE (1));
953 // Set Page Directory Pointers
955 for (Index
= 0; Index
< 4; Index
++) {
956 Pte
[Index
] = ((UINTN
)PageTable
+ EFI_PAGE_SIZE
* (Index
+ 1)) | mAddressEncMask
|
957 (Is32BitPageTable
? IA32_PAE_PDPTE_ATTRIBUTE_BITS
: PAGE_ATTRIBUTE_BITS
);
959 Pte
+= EFI_PAGE_SIZE
/ sizeof (*Pte
);
962 // Fill in Page Directory Entries
964 for (Index
= 0; Index
< EFI_PAGE_SIZE
* 4 / sizeof (*Pte
); Index
++) {
965 Pte
[Index
] = (Index
<< 21) | mAddressEncMask
| IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
968 Pdpte
= (UINT64
*)PageTable
;
969 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
970 Pages
= (UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (5);
971 GuardPage
= mSmmStackArrayBase
+ EFI_PAGE_SIZE
;
972 for (PageIndex
= Low2MBoundary
; PageIndex
<= High2MBoundary
; PageIndex
+= SIZE_2MB
) {
973 Pte
= (UINT64
*)(UINTN
)(Pdpte
[BitFieldRead32 ((UINT32
)PageIndex
, 30, 31)] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
974 Pte
[BitFieldRead32 ((UINT32
)PageIndex
, 21, 29)] = (UINT64
)Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
976 // Fill in Page Table Entries
978 Pte
= (UINT64
*)Pages
;
979 PageAddress
= PageIndex
;
980 for (Index
= 0; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
981 if (PageAddress
== GuardPage
) {
983 // Mark the guard page as non-present
985 Pte
[Index
] = PageAddress
| mAddressEncMask
;
986 GuardPage
+= mSmmStackSize
;
987 if (GuardPage
> mSmmStackArrayEnd
) {
991 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
993 PageAddress
+= EFI_PAGE_SIZE
;
995 Pages
+= EFI_PAGE_SIZE
;
999 if ((PcdGet8 (PcdNullPointerDetectionPropertyMask
) & BIT1
) != 0) {
1000 Pte
= (UINT64
*)(UINTN
)(Pdpte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
1001 if ((Pte
[0] & IA32_PG_PS
) == 0) {
1002 // 4K-page entries are already mapped. Just hide the first one anyway.
1003 Pte
= (UINT64
*)(UINTN
)(Pte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
1004 Pte
[0] &= ~(UINT64
)IA32_PG_P
; // Hide page 0
1006 // Create 4K-page entries
1007 Pages
= (UINTN
)AllocatePageTableMemory (1);
1008 ASSERT (Pages
!= 0);
1010 Pte
[0] = (UINT64
)(Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
);
1012 Pte
= (UINT64
*)Pages
;
1014 Pte
[0] = PageAddress
| mAddressEncMask
; // Hide page 0 but present left
1015 for (Index
= 1; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
1016 PageAddress
+= EFI_PAGE_SIZE
;
1017 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
1022 return (UINT32
)(UINTN
)PageTable
;
1026 Checks whether the input token is the current used token.
1028 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1031 @retval TRUE The input token is the current used token.
1032 @retval FALSE The input token is not the current used token.
1040 PROCEDURE_TOKEN
*ProcToken
;
1042 if (Token
== NULL
) {
1046 Link
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
1047 while (!IsNull (&gSmmCpuPrivate
->TokenList
, Link
)) {
1048 ProcToken
= PROCEDURE_TOKEN_FROM_LINK (Link
);
1050 if (ProcToken
->Used
&& ProcToken
->SpinLock
== Token
) {
1054 Link
= GetNextNode (&gSmmCpuPrivate
->TokenList
, Link
);
1061 Allocate buffer for the SPIN_LOCK and PROCEDURE_TOKEN.
1065 AllocateTokenBuffer (
1070 UINT32 TokenCountPerChunk
;
1071 UINTN ProcTokenSize
;
1073 PROCEDURE_TOKEN
*ProcToken
;
1074 SPIN_LOCK
*SpinLock
;
1075 UINT8
*SpinLockBuffer
;
1076 UINT8
*ProcTokenBuffer
;
1078 SpinLockSize
= GetSpinLockProperties ();
1079 ProcTokenSize
= sizeof (PROCEDURE_TOKEN
);
1081 TokenCountPerChunk
= FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk
);
1082 ASSERT (TokenCountPerChunk
!= 0);
1083 if (TokenCountPerChunk
== 0) {
1084 DEBUG ((DEBUG_ERROR
, "PcdCpuSmmMpTokenCountPerChunk should not be Zero!\n"));
1087 DEBUG ((DEBUG_INFO
, "CpuSmm: SpinLock Size = 0x%x, PcdCpuSmmMpTokenCountPerChunk = 0x%x\n", SpinLockSize
, TokenCountPerChunk
));
1090 // Separate the Spin_lock and Proc_token because the alignment requires by Spin_Lock.
1092 SpinLockBuffer
= AllocatePool (SpinLockSize
* TokenCountPerChunk
);
1093 ASSERT (SpinLockBuffer
!= NULL
);
1095 ProcTokenBuffer
= AllocatePool (ProcTokenSize
* TokenCountPerChunk
);
1096 ASSERT (ProcTokenBuffer
!= NULL
);
1098 for (Index
= 0; Index
< TokenCountPerChunk
; Index
++) {
1099 SpinLock
= (SPIN_LOCK
*)(SpinLockBuffer
+ SpinLockSize
* Index
);
1100 InitializeSpinLock (SpinLock
);
1102 ProcToken
= (PROCEDURE_TOKEN
*)(ProcTokenBuffer
+ ProcTokenSize
* Index
);
1103 ProcToken
->Signature
= PROCEDURE_TOKEN_SIGNATURE
;
1104 ProcToken
->SpinLock
= SpinLock
;
1105 ProcToken
->Used
= FALSE
;
1106 ProcToken
->RunningApCount
= 0;
1108 InsertTailList (&gSmmCpuPrivate
->TokenList
, &ProcToken
->Link
);
1113 Find first free token in the allocated token list.
1115 @retval return the first free PROCEDURE_TOKEN.
1119 FindFirstFreeToken (
1124 PROCEDURE_TOKEN
*ProcToken
;
1126 Link
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
1127 while (!IsNull (&gSmmCpuPrivate
->TokenList
, Link
)) {
1128 ProcToken
= PROCEDURE_TOKEN_FROM_LINK (Link
);
1130 if (!ProcToken
->Used
) {
1134 Link
= GetNextNode (&gSmmCpuPrivate
->TokenList
, Link
);
1143 If no free token, allocate new tokens then return the free one.
1145 @param RunningApsCount The Running Aps count for this token.
1147 @retval return the first free PROCEDURE_TOKEN.
1152 IN UINT32 RunningApsCount
1155 PROCEDURE_TOKEN
*NewToken
;
1157 NewToken
= FindFirstFreeToken ();
1158 if (NewToken
== NULL
) {
1159 AllocateTokenBuffer ();
1160 NewToken
= FindFirstFreeToken ();
1162 ASSERT (NewToken
!= NULL
);
1164 NewToken
->Used
= TRUE
;
1165 NewToken
->RunningApCount
= RunningApsCount
;
1166 AcquireSpinLock (NewToken
->SpinLock
);
1172 Checks status of specified AP.
1174 This function checks whether the specified AP has finished the task assigned
1175 by StartupThisAP(), and whether timeout expires.
1177 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1180 @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
1181 @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
1188 if (AcquireSpinLockOrFail (Token
)) {
1189 ReleaseSpinLock (Token
);
1193 return EFI_NOT_READY
;
1197 Schedule a procedure to run on the specified CPU.
1199 @param[in] Procedure The address of the procedure to run
1200 @param[in] CpuIndex Target CPU Index
1201 @param[in,out] ProcArguments The parameter to pass to the procedure
1202 @param[in] Token This is an optional parameter that allows the caller to execute the
1203 procedure in a blocking or non-blocking fashion. If it is NULL the
1204 call is blocking, and the call will not return until the AP has
1205 completed the procedure. If the token is not NULL, the call will
1206 return immediately. The caller can check whether the procedure has
1207 completed with CheckOnProcedure or WaitForProcedure.
1208 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for the APs to finish
1209 execution of Procedure, either for blocking or non-blocking mode.
1210 Zero means infinity. If the timeout expires before all APs return
1211 from Procedure, then Procedure on the failed APs is terminated. If
1212 the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
1213 If the timeout expires in non-blocking mode, the timeout determined
1214 can be through CheckOnProcedure or WaitForProcedure.
1215 Note that timeout support is optional. Whether an implementation
1216 supports this feature can be determined via the Attributes data
1218 @param[in,out] CpuStatus This optional pointer may be used to get the status code returned
1219 by Procedure when it completes execution on the target AP, or with
1220 EFI_TIMEOUT if the Procedure fails to complete within the optional
1221 timeout. The implementation will update this variable with
1222 EFI_NOT_READY prior to starting Procedure on the target AP.
1224 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1225 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1226 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1227 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1228 @retval EFI_SUCCESS The procedure has been successfully scheduled
1232 InternalSmmStartupThisAp (
1233 IN EFI_AP_PROCEDURE2 Procedure
,
1235 IN OUT VOID
*ProcArguments OPTIONAL
,
1236 IN MM_COMPLETION
*Token
,
1237 IN UINTN TimeoutInMicroseconds
,
1238 IN OUT EFI_STATUS
*CpuStatus
1241 PROCEDURE_TOKEN
*ProcToken
;
1243 if (CpuIndex
>= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
) {
1244 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus(%d)\n", CpuIndex
, gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
));
1245 return EFI_INVALID_PARAMETER
;
1247 if (CpuIndex
== gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) {
1248 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu\n", CpuIndex
));
1249 return EFI_INVALID_PARAMETER
;
1251 if (gSmmCpuPrivate
->ProcessorInfo
[CpuIndex
].ProcessorId
== INVALID_APIC_ID
) {
1252 return EFI_INVALID_PARAMETER
;
1254 if (!(*(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
))) {
1255 if (mSmmMpSyncData
->EffectiveSyncMode
== SmmCpuSyncModeTradition
) {
1256 DEBUG((DEBUG_ERROR
, "!mSmmMpSyncData->CpuData[%d].Present\n", CpuIndex
));
1258 return EFI_INVALID_PARAMETER
;
1260 if (gSmmCpuPrivate
->Operation
[CpuIndex
] == SmmCpuRemove
) {
1261 if (!FeaturePcdGet (PcdCpuHotPlugSupport
)) {
1262 DEBUG((DEBUG_ERROR
, "gSmmCpuPrivate->Operation[%d] == SmmCpuRemove\n", CpuIndex
));
1264 return EFI_INVALID_PARAMETER
;
1266 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1267 return EFI_INVALID_PARAMETER
;
1269 if (Procedure
== NULL
) {
1270 return EFI_INVALID_PARAMETER
;
1273 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1275 mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
= Procedure
;
1276 mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
= ProcArguments
;
1277 if (Token
!= NULL
) {
1278 ProcToken
= GetFreeToken (1);
1279 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= ProcToken
;
1280 *Token
= (MM_COMPLETION
)ProcToken
->SpinLock
;
1282 mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= CpuStatus
;
1283 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
1284 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= EFI_NOT_READY
;
1287 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
1289 if (Token
== NULL
) {
1290 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1291 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1298 Worker function to execute a caller provided function on all enabled APs.
1300 @param[in] Procedure A pointer to the function to be run on
1301 enabled APs of the system.
1302 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
1303 APs to return from Procedure, either for
1304 blocking or non-blocking mode.
1305 @param[in,out] ProcedureArguments The parameter passed into Procedure for
1307 @param[in,out] Token This is an optional parameter that allows the caller to execute the
1308 procedure in a blocking or non-blocking fashion. If it is NULL the
1309 call is blocking, and the call will not return until the AP has
1310 completed the procedure. If the token is not NULL, the call will
1311 return immediately. The caller can check whether the procedure has
1312 completed with CheckOnProcedure or WaitForProcedure.
1313 @param[in,out] CPUStatus This optional pointer may be used to get the status code returned
1314 by Procedure when it completes execution on the target AP, or with
1315 EFI_TIMEOUT if the Procedure fails to complete within the optional
1316 timeout. The implementation will update this variable with
1317 EFI_NOT_READY prior to starting Procedure on the target AP.
1320 @retval EFI_SUCCESS In blocking mode, all APs have finished before
1321 the timeout expired.
1322 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
1324 @retval others Failed to Startup all APs.
1328 InternalSmmStartupAllAPs (
1329 IN EFI_AP_PROCEDURE2 Procedure
,
1330 IN UINTN TimeoutInMicroseconds
,
1331 IN OUT VOID
*ProcedureArguments OPTIONAL
,
1332 IN OUT MM_COMPLETION
*Token
,
1333 IN OUT EFI_STATUS
*CPUStatus
1338 PROCEDURE_TOKEN
*ProcToken
;
1340 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1341 return EFI_INVALID_PARAMETER
;
1343 if (Procedure
== NULL
) {
1344 return EFI_INVALID_PARAMETER
;
1348 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1349 if (IsPresentAp (Index
)) {
1352 if (gSmmCpuPrivate
->Operation
[Index
] == SmmCpuRemove
) {
1353 return EFI_INVALID_PARAMETER
;
1356 if (!AcquireSpinLockOrFail(mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
1357 return EFI_NOT_READY
;
1359 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1362 if (CpuCount
== 0) {
1363 return EFI_NOT_STARTED
;
1366 if (Token
!= NULL
) {
1367 ProcToken
= GetFreeToken ((UINT32
)mMaxNumberOfCpus
);
1368 *Token
= (MM_COMPLETION
)ProcToken
->SpinLock
;
1374 // Make sure all BUSY should be acquired.
1376 // Because former code already check mSmmMpSyncData->CpuData[***].Busy for each AP.
1377 // Here code always use AcquireSpinLock instead of AcquireSpinLockOrFail for not
1380 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1381 if (IsPresentAp (Index
)) {
1382 AcquireSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1386 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1387 if (IsPresentAp (Index
)) {
1388 mSmmMpSyncData
->CpuData
[Index
].Procedure
= (EFI_AP_PROCEDURE2
) Procedure
;
1389 mSmmMpSyncData
->CpuData
[Index
].Parameter
= ProcedureArguments
;
1390 if (ProcToken
!= NULL
) {
1391 mSmmMpSyncData
->CpuData
[Index
].Token
= ProcToken
;
1393 if (CPUStatus
!= NULL
) {
1394 mSmmMpSyncData
->CpuData
[Index
].Status
= &CPUStatus
[Index
];
1395 if (mSmmMpSyncData
->CpuData
[Index
].Status
!= NULL
) {
1396 *mSmmMpSyncData
->CpuData
[Index
].Status
= EFI_NOT_READY
;
1401 // PI spec requirement:
1402 // For every excluded processor, the array entry must contain a value of EFI_NOT_STARTED.
1404 if (CPUStatus
!= NULL
) {
1405 CPUStatus
[Index
] = EFI_NOT_STARTED
;
1409 // Decrease the count to mark this processor(AP or BSP) as finished.
1411 if (ProcToken
!= NULL
) {
1412 WaitForSemaphore (&ProcToken
->RunningApCount
);
1419 if (Token
== NULL
) {
1421 // Make sure all APs have completed their tasks.
1423 WaitForAllAPsNotBusy (TRUE
);
1430 ISO C99 6.5.2.2 "Function calls", paragraph 9:
1431 If the function is defined with a type that is not compatible with
1432 the type (of the expression) pointed to by the expression that
1433 denotes the called function, the behavior is undefined.
1435 So add below wrapper function to convert between EFI_AP_PROCEDURE
1436 and EFI_AP_PROCEDURE2.
1438 Wrapper for Procedures.
1440 @param[in] Buffer Pointer to PROCEDURE_WRAPPER buffer.
1449 PROCEDURE_WRAPPER
*Wrapper
;
1452 Wrapper
->Procedure (Wrapper
->ProcedureArgument
);
1458 Schedule a procedure to run on the specified CPU in blocking mode.
1460 @param[in] Procedure The address of the procedure to run
1461 @param[in] CpuIndex Target CPU Index
1462 @param[in, out] ProcArguments The parameter to pass to the procedure
1464 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1465 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1466 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1467 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1468 @retval EFI_SUCCESS The procedure has been successfully scheduled
1473 SmmBlockingStartupThisAp (
1474 IN EFI_AP_PROCEDURE Procedure
,
1476 IN OUT VOID
*ProcArguments OPTIONAL
1479 PROCEDURE_WRAPPER Wrapper
;
1481 Wrapper
.Procedure
= Procedure
;
1482 Wrapper
.ProcedureArgument
= ProcArguments
;
1485 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1487 return InternalSmmStartupThisAp (ProcedureWrapper
, CpuIndex
, &Wrapper
, NULL
, 0, NULL
);
1491 Schedule a procedure to run on the specified CPU.
1493 @param Procedure The address of the procedure to run
1494 @param CpuIndex Target CPU Index
1495 @param ProcArguments The parameter to pass to the procedure
1497 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1498 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1499 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1500 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1501 @retval EFI_SUCCESS The procedure has been successfully scheduled
1507 IN EFI_AP_PROCEDURE Procedure
,
1509 IN OUT VOID
*ProcArguments OPTIONAL
1512 MM_COMPLETION Token
;
1514 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].Procedure
= Procedure
;
1515 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].ProcedureArgument
= ProcArguments
;
1518 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1520 return InternalSmmStartupThisAp (
1523 &gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
],
1524 FeaturePcdGet (PcdCpuSmmBlockStartupThisAp
) ? NULL
: &Token
,
1531 This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
1532 They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
1534 NOTE: It might not be appreciated in runtime since it might
1535 conflict with OS debugging facilities. Turn them off in RELEASE.
1537 @param CpuIndex CPU Index
1546 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1548 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1549 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1550 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1551 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1552 AsmWriteDr6 (CpuSaveState
->x86
._DR6
);
1553 AsmWriteDr7 (CpuSaveState
->x86
._DR7
);
1555 AsmWriteDr6 ((UINTN
)CpuSaveState
->x64
._DR6
);
1556 AsmWriteDr7 ((UINTN
)CpuSaveState
->x64
._DR7
);
1562 This function restores DR6 & DR7 to SMM save state.
1564 NOTE: It might not be appreciated in runtime since it might
1565 conflict with OS debugging facilities. Turn them off in RELEASE.
1567 @param CpuIndex CPU Index
1576 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1578 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1579 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1580 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1581 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1582 CpuSaveState
->x86
._DR7
= (UINT32
)AsmReadDr7 ();
1583 CpuSaveState
->x86
._DR6
= (UINT32
)AsmReadDr6 ();
1585 CpuSaveState
->x64
._DR7
= AsmReadDr7 ();
1586 CpuSaveState
->x64
._DR6
= AsmReadDr6 ();
1592 C function for SMI entry, each processor comes here upon SMI trigger.
1594 @param CpuIndex CPU Index
1606 BOOLEAN BspInProgress
;
1610 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1613 // Save Cr2 because Page Fault exception in SMM may override its value,
1614 // when using on-demand paging for above 4G memory.
1620 // Call the user register Startup function first.
1622 if (mSmmMpSyncData
->StartupProcedure
!= NULL
) {
1623 mSmmMpSyncData
->StartupProcedure (mSmmMpSyncData
->StartupProcArgs
);
1627 // Perform CPU specific entry hooks
1629 SmmCpuFeaturesRendezvousEntry (CpuIndex
);
1632 // Determine if this is a valid SMI
1634 ValidSmi
= PlatformValidSmi();
1637 // Determine if BSP has been already in progress. Note this must be checked after
1638 // ValidSmi because BSP may clear a valid SMI source after checking in.
1640 BspInProgress
= *mSmmMpSyncData
->InsideSmm
;
1642 if (!BspInProgress
&& !ValidSmi
) {
1644 // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not
1645 // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI
1646 // status had been cleared by BSP and an existing SMI run has almost ended. (Note
1647 // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there
1648 // is nothing we need to do.
1653 // Signal presence of this processor
1655 if (ReleaseSemaphore (mSmmMpSyncData
->Counter
) == 0) {
1657 // BSP has already ended the synchronization, so QUIT!!!
1661 // Wait for BSP's signal to finish SMI
1663 while (*mSmmMpSyncData
->AllCpusInSync
) {
1670 // The BUSY lock is initialized to Released state.
1671 // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.
1672 // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
1673 // after AP's present flag is detected.
1675 InitializeSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1678 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1679 ActivateSmmProfile (CpuIndex
);
1682 if (BspInProgress
) {
1684 // BSP has been elected. Follow AP path, regardless of ValidSmi flag
1685 // as BSP may have cleared the SMI status
1687 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1690 // We have a valid SMI
1697 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1698 if (!mSmmMpSyncData
->SwitchBsp
|| mSmmMpSyncData
->CandidateBsp
[CpuIndex
]) {
1700 // Call platform hook to do BSP election
1702 Status
= PlatformSmmBspElection (&IsBsp
);
1703 if (EFI_SUCCESS
== Status
) {
1705 // Platform hook determines successfully
1708 mSmmMpSyncData
->BspIndex
= (UINT32
)CpuIndex
;
1712 // Platform hook fails to determine, use default BSP election method
1714 InterlockedCompareExchange32 (
1715 (UINT32
*)&mSmmMpSyncData
->BspIndex
,
1724 // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP
1726 if (mSmmMpSyncData
->BspIndex
== CpuIndex
) {
1729 // Clear last request for SwitchBsp.
1731 if (mSmmMpSyncData
->SwitchBsp
) {
1732 mSmmMpSyncData
->SwitchBsp
= FALSE
;
1733 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1734 mSmmMpSyncData
->CandidateBsp
[Index
] = FALSE
;
1738 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1739 SmmProfileRecordSmiNum ();
1743 // BSP Handler is always called with a ValidSmi == TRUE
1745 BSPHandler (CpuIndex
, mSmmMpSyncData
->EffectiveSyncMode
);
1747 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1751 ASSERT (*mSmmMpSyncData
->CpuData
[CpuIndex
].Run
== 0);
1754 // Wait for BSP's signal to exit SMI
1756 while (*mSmmMpSyncData
->AllCpusInSync
) {
1762 SmmCpuFeaturesRendezvousExit (CpuIndex
);
1771 Allocate buffer for SpinLock and Wrapper function buffer.
1775 InitializeDataForMmMp (
1779 gSmmCpuPrivate
->ApWrapperFunc
= AllocatePool (sizeof (PROCEDURE_WRAPPER
) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1780 ASSERT (gSmmCpuPrivate
->ApWrapperFunc
!= NULL
);
1782 InitializeListHead (&gSmmCpuPrivate
->TokenList
);
1784 AllocateTokenBuffer ();
1788 Allocate buffer for all semaphores and spin locks.
1792 InitializeSmmCpuSemaphores (
1796 UINTN ProcessorCount
;
1798 UINTN GlobalSemaphoresSize
;
1799 UINTN CpuSemaphoresSize
;
1800 UINTN SemaphoreSize
;
1802 UINTN
*SemaphoreBlock
;
1803 UINTN SemaphoreAddr
;
1805 SemaphoreSize
= GetSpinLockProperties ();
1806 ProcessorCount
= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1807 GlobalSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_GLOBAL
) / sizeof (VOID
*)) * SemaphoreSize
;
1808 CpuSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_CPU
) / sizeof (VOID
*)) * ProcessorCount
* SemaphoreSize
;
1809 TotalSize
= GlobalSemaphoresSize
+ CpuSemaphoresSize
;
1810 DEBUG((EFI_D_INFO
, "One Semaphore Size = 0x%x\n", SemaphoreSize
));
1811 DEBUG((EFI_D_INFO
, "Total Semaphores Size = 0x%x\n", TotalSize
));
1812 Pages
= EFI_SIZE_TO_PAGES (TotalSize
);
1813 SemaphoreBlock
= AllocatePages (Pages
);
1814 ASSERT (SemaphoreBlock
!= NULL
);
1815 ZeroMem (SemaphoreBlock
, TotalSize
);
1817 SemaphoreAddr
= (UINTN
)SemaphoreBlock
;
1818 mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
= (UINT32
*)SemaphoreAddr
;
1819 SemaphoreAddr
+= SemaphoreSize
;
1820 mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
= (BOOLEAN
*)SemaphoreAddr
;
1821 SemaphoreAddr
+= SemaphoreSize
;
1822 mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
= (BOOLEAN
*)SemaphoreAddr
;
1823 SemaphoreAddr
+= SemaphoreSize
;
1824 mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
= (SPIN_LOCK
*)SemaphoreAddr
;
1825 SemaphoreAddr
+= SemaphoreSize
;
1826 mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
1827 = (SPIN_LOCK
*)SemaphoreAddr
;
1828 SemaphoreAddr
+= SemaphoreSize
;
1830 SemaphoreAddr
= (UINTN
)SemaphoreBlock
+ GlobalSemaphoresSize
;
1831 mSmmCpuSemaphores
.SemaphoreCpu
.Busy
= (SPIN_LOCK
*)SemaphoreAddr
;
1832 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1833 mSmmCpuSemaphores
.SemaphoreCpu
.Run
= (UINT32
*)SemaphoreAddr
;
1834 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1835 mSmmCpuSemaphores
.SemaphoreCpu
.Present
= (BOOLEAN
*)SemaphoreAddr
;
1837 mPFLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
;
1838 mConfigSmmCodeAccessCheckLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
;
1840 mSemaphoreSize
= SemaphoreSize
;
1844 Initialize un-cacheable data.
1849 InitializeMpSyncData (
1855 if (mSmmMpSyncData
!= NULL
) {
1857 // mSmmMpSyncDataSize includes one structure of SMM_DISPATCHER_MP_SYNC_DATA, one
1858 // CpuData array of SMM_CPU_DATA_BLOCK and one CandidateBsp array of BOOLEAN.
1860 ZeroMem (mSmmMpSyncData
, mSmmMpSyncDataSize
);
1861 mSmmMpSyncData
->CpuData
= (SMM_CPU_DATA_BLOCK
*)((UINT8
*)mSmmMpSyncData
+ sizeof (SMM_DISPATCHER_MP_SYNC_DATA
));
1862 mSmmMpSyncData
->CandidateBsp
= (BOOLEAN
*)(mSmmMpSyncData
->CpuData
+ gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1863 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1865 // Enable BSP election by setting BspIndex to -1
1867 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
1869 mSmmMpSyncData
->EffectiveSyncMode
= mCpuSmmSyncMode
;
1871 mSmmMpSyncData
->Counter
= mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
;
1872 mSmmMpSyncData
->InsideSmm
= mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
;
1873 mSmmMpSyncData
->AllCpusInSync
= mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
;
1874 ASSERT (mSmmMpSyncData
->Counter
!= NULL
&& mSmmMpSyncData
->InsideSmm
!= NULL
&&
1875 mSmmMpSyncData
->AllCpusInSync
!= NULL
);
1876 *mSmmMpSyncData
->Counter
= 0;
1877 *mSmmMpSyncData
->InsideSmm
= FALSE
;
1878 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
1880 for (CpuIndex
= 0; CpuIndex
< gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
; CpuIndex
++) {
1881 mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
=
1882 (SPIN_LOCK
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Busy
+ mSemaphoreSize
* CpuIndex
);
1883 mSmmMpSyncData
->CpuData
[CpuIndex
].Run
=
1884 (UINT32
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Run
+ mSemaphoreSize
* CpuIndex
);
1885 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
=
1886 (BOOLEAN
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Present
+ mSemaphoreSize
* CpuIndex
);
1887 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
) = 0;
1888 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Run
) = 0;
1889 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
1895 Initialize global data for MP synchronization.
1897 @param Stacks Base address of SMI stack buffer for all processors.
1898 @param StackSize Stack size for each processor in SMM.
1899 @param ShadowStackSize Shadow Stack size for each processor in SMM.
1903 InitializeMpServiceData (
1906 IN UINTN ShadowStackSize
1911 UINT8
*GdtTssTables
;
1912 UINTN GdtTableStepSize
;
1913 CPUID_VERSION_INFO_EDX RegEdx
;
1916 // Determine if this CPU supports machine check
1918 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
1919 mMachineCheckSupported
= (BOOLEAN
)(RegEdx
.Bits
.MCA
== 1);
1922 // Allocate memory for all locks and semaphores
1924 InitializeSmmCpuSemaphores ();
1927 // Initialize mSmmMpSyncData
1929 mSmmMpSyncDataSize
= sizeof (SMM_DISPATCHER_MP_SYNC_DATA
) +
1930 (sizeof (SMM_CPU_DATA_BLOCK
) + sizeof (BOOLEAN
)) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1931 mSmmMpSyncData
= (SMM_DISPATCHER_MP_SYNC_DATA
*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize
));
1932 ASSERT (mSmmMpSyncData
!= NULL
);
1933 mCpuSmmSyncMode
= (SMM_CPU_SYNC_MODE
)PcdGet8 (PcdCpuSmmSyncMode
);
1934 InitializeMpSyncData ();
1937 // Initialize physical address mask
1938 // NOTE: Physical memory above virtual address limit is not supported !!!
1940 AsmCpuid (0x80000008, (UINT32
*)&Index
, NULL
, NULL
, NULL
);
1941 gPhyMask
= LShiftU64 (1, (UINT8
)Index
) - 1;
1942 gPhyMask
&= (1ull << 48) - EFI_PAGE_SIZE
;
1945 // Create page tables
1947 Cr3
= SmmInitPageTable ();
1949 GdtTssTables
= InitGdt (Cr3
, &GdtTableStepSize
);
1952 // Install SMI handler for each CPU
1954 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1957 (UINT32
)mCpuHotPlugData
.SmBase
[Index
],
1958 (VOID
*)((UINTN
)Stacks
+ (StackSize
+ ShadowStackSize
) * Index
),
1960 (UINTN
)(GdtTssTables
+ GdtTableStepSize
* Index
),
1961 gcSmiGdtr
.Limit
+ 1,
1963 gcSmiIdtr
.Limit
+ 1,
1969 // Record current MTRR settings
1971 ZeroMem (&gSmiMtrrs
, sizeof (gSmiMtrrs
));
1972 MtrrGetAllMtrrs (&gSmiMtrrs
);
1979 Register the SMM Foundation entry point.
1981 @param This Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance
1982 @param SmmEntryPoint SMM Foundation EntryPoint
1984 @retval EFI_SUCCESS Successfully to register SMM foundation entry point
1990 IN CONST EFI_SMM_CONFIGURATION_PROTOCOL
*This
,
1991 IN EFI_SMM_ENTRY_POINT SmmEntryPoint
1995 // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.
1997 gSmmCpuPrivate
->SmmCoreEntry
= SmmEntryPoint
;
2003 Register the SMM Foundation entry point.
2005 @param[in] Procedure A pointer to the code stream to be run on the designated target AP
2006 of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
2007 with the related definitions of
2008 EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
2009 If caller may pass a value of NULL to deregister any existing
2011 @param[in,out] ProcedureArguments Allows the caller to pass a list of parameters to the code that is
2012 run by the AP. It is an optional common mailbox between APs and
2013 the caller to share information
2015 @retval EFI_SUCCESS The Procedure has been set successfully.
2016 @retval EFI_INVALID_PARAMETER The Procedure is NULL but ProcedureArguments not NULL.
2020 RegisterStartupProcedure (
2021 IN EFI_AP_PROCEDURE Procedure
,
2022 IN OUT VOID
*ProcedureArguments OPTIONAL
2025 if (Procedure
== NULL
&& ProcedureArguments
!= NULL
) {
2026 return EFI_INVALID_PARAMETER
;
2028 if (mSmmMpSyncData
== NULL
) {
2029 return EFI_NOT_READY
;
2032 mSmmMpSyncData
->StartupProcedure
= Procedure
;
2033 mSmmMpSyncData
->StartupProcArgs
= ProcedureArguments
;