2 SMM MP service implementation
4 Copyright (c) 2009 - 2020, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "PiSmmCpuDxeSmm.h"
14 // Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
16 MTRR_SETTINGS gSmiMtrrs
;
18 SMM_DISPATCHER_MP_SYNC_DATA
*mSmmMpSyncData
= NULL
;
19 UINTN mSmmMpSyncDataSize
;
20 SMM_CPU_SEMAPHORES mSmmCpuSemaphores
;
22 SPIN_LOCK
*mPFLock
= NULL
;
23 SMM_CPU_SYNC_MODE mCpuSmmSyncMode
;
24 BOOLEAN mMachineCheckSupported
= FALSE
;
27 Performs an atomic compare exchange operation to get semaphore.
28 The compare exchange operation must be performed using
31 @param Sem IN: 32-bit unsigned integer
32 OUT: original integer - 1
33 @return Original integer - 1
38 IN OUT
volatile UINT32
*Sem
45 } while (Value
== 0 ||
46 InterlockedCompareExchange32 (
56 Performs an atomic compare exchange operation to release semaphore.
57 The compare exchange operation must be performed using
60 @param Sem IN: 32-bit unsigned integer
61 OUT: original integer + 1
62 @return Original integer + 1
67 IN OUT
volatile UINT32
*Sem
74 } while (Value
+ 1 != 0 &&
75 InterlockedCompareExchange32 (
84 Performs an atomic compare exchange operation to lock semaphore.
85 The compare exchange operation must be performed using
88 @param Sem IN: 32-bit unsigned integer
90 @return Original integer
95 IN OUT
volatile UINT32
*Sem
102 } while (InterlockedCompareExchange32 (
110 Wait all APs to performs an atomic compare exchange operation to release semaphore.
112 @param NumberOfAPs AP number
122 BspIndex
= mSmmMpSyncData
->BspIndex
;
123 while (NumberOfAPs
-- > 0) {
124 WaitForSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
129 Performs an atomic compare exchange operation to release semaphore
140 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
141 if (IsPresentAp (Index
)) {
142 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[Index
].Run
);
148 Checks if all CPUs (with certain exceptions) have checked in for this SMI run
150 @param Exceptions CPU Arrival exception flags.
152 @retval TRUE if all CPUs the have checked in.
153 @retval FALSE if at least one Normal AP hasn't checked in.
157 AllCpusInSmmWithExceptions (
158 SMM_CPU_ARRIVAL_EXCEPTIONS Exceptions
162 SMM_CPU_DATA_BLOCK
*CpuData
;
163 EFI_PROCESSOR_INFORMATION
*ProcessorInfo
;
165 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
167 if (*mSmmMpSyncData
->Counter
== mNumberOfCpus
) {
171 CpuData
= mSmmMpSyncData
->CpuData
;
172 ProcessorInfo
= gSmmCpuPrivate
->ProcessorInfo
;
173 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
174 if (!(*(CpuData
[Index
].Present
)) && ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
175 if (((Exceptions
& ARRIVAL_EXCEPTION_DELAYED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmDelayed
) != 0) {
178 if (((Exceptions
& ARRIVAL_EXCEPTION_BLOCKED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmBlocked
) != 0) {
181 if (((Exceptions
& ARRIVAL_EXCEPTION_SMI_DISABLED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmEnable
) != 0) {
193 Has OS enabled Lmce in the MSR_IA32_MCG_EXT_CTL
195 @retval TRUE Os enable lmce.
196 @retval FALSE Os not enable lmce.
204 MSR_IA32_MCG_CAP_REGISTER McgCap
;
205 MSR_IA32_FEATURE_CONTROL_REGISTER FeatureCtrl
;
206 MSR_IA32_MCG_EXT_CTL_REGISTER McgExtCtrl
;
208 McgCap
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_CAP
);
209 if (McgCap
.Bits
.MCG_LMCE_P
== 0) {
213 FeatureCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_FEATURE_CONTROL
);
214 if (FeatureCtrl
.Bits
.LmceOn
== 0) {
218 McgExtCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_EXT_CTL
);
219 return (BOOLEAN
) (McgExtCtrl
.Bits
.LMCE_EN
== 1);
223 Return if Local machine check exception signaled.
225 Indicates (when set) that a local machine check exception was generated. This indicates that the current machine-check event was
226 delivered to only the logical processor.
228 @retval TRUE LMCE was signaled.
229 @retval FALSE LMCE was not signaled.
237 MSR_IA32_MCG_STATUS_REGISTER McgStatus
;
239 McgStatus
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_STATUS
);
240 return (BOOLEAN
) (McgStatus
.Bits
.LMCE_S
== 1);
244 Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
245 entering SMM, except SMI disabled APs.
249 SmmWaitForApArrival (
258 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
262 if (mMachineCheckSupported
) {
263 LmceEn
= IsLmceOsEnabled ();
264 LmceSignal
= IsLmceSignaled();
268 // Platform implementor should choose a timeout value appropriately:
269 // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note
270 // the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.
271 // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI
272 // and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will
273 // be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the
274 // SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.
275 // - The timeout value must be longer than longest possible IO operation in the system
279 // Sync with APs 1st timeout
281 for (Timer
= StartSyncTimer ();
282 !IsSyncTimerTimeout (Timer
) && !(LmceEn
&& LmceSignal
) &&
283 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
289 // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,
291 // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running
292 // normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they
293 // enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode
294 // work while SMI handling is on-going.
295 // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.
296 // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state
297 // will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal
298 // mode work while SMI handling is on-going.
299 // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:
300 // - In traditional flow, SMI disabling is discouraged.
301 // - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
302 // In both cases, adding SMI-disabling checking code increases overhead.
304 if (*mSmmMpSyncData
->Counter
< mNumberOfCpus
) {
306 // Send SMI IPIs to bring outside processors in
308 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
309 if (!(*(mSmmMpSyncData
->CpuData
[Index
].Present
)) && gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
310 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
);
315 // Sync with APs 2nd timeout.
317 for (Timer
= StartSyncTimer ();
318 !IsSyncTimerTimeout (Timer
) &&
319 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
330 Replace OS MTRR's with SMI MTRR's.
332 @param CpuIndex Processor Index
340 SmmCpuFeaturesDisableSmrr ();
343 // Replace all MTRRs registers
345 MtrrSetAllMtrrs (&gSmiMtrrs
);
349 Wheck whether task has been finished by all APs.
351 @param BlockMode Whether did it in block mode or non-block mode.
353 @retval TRUE Task has been finished by all APs.
354 @retval FALSE Task not has been finished by all APs.
358 WaitForAllAPsNotBusy (
364 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
366 // Ignore BSP and APs which not call in SMM.
368 if (!IsPresentAp(Index
)) {
373 AcquireSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
374 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
376 if (AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
377 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
388 Check whether it is an present AP.
390 @param CpuIndex The AP index which calls this function.
392 @retval TRUE It's a present AP.
393 @retval TRUE This is not an AP or it is not present.
401 return ((CpuIndex
!= gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) &&
402 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
));
406 Clean up the status flags used during executing the procedure.
408 @param CpuIndex The AP index which calls this function.
416 PROCEDURE_TOKEN
*Token
;
418 Token
= mSmmMpSyncData
->CpuData
[CpuIndex
].Token
;
420 if (InterlockedDecrement (&Token
->RunningApCount
) == 0) {
421 ReleaseSpinLock (Token
->SpinLock
);
424 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= NULL
;
428 Free the tokens in the maintained list.
437 PROCEDURE_TOKEN
*ProcToken
;
439 Link
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
440 while (!IsNull (&gSmmCpuPrivate
->TokenList
, Link
)) {
441 ProcToken
= PROCEDURE_TOKEN_FROM_LINK (Link
);
443 ProcToken
->RunningApCount
= 0;
446 // Check the spinlock status and release it if not released yet.
448 if (!AcquireSpinLockOrFail(ProcToken
->SpinLock
)) {
449 DEBUG((DEBUG_ERROR
, "Risk::SpinLock still not released!"));
451 ReleaseSpinLock (ProcToken
->SpinLock
);
453 Link
= GetNextNode (&gSmmCpuPrivate
->TokenList
, Link
);
457 // Reset the FirstFreeToken to the beginning of token list upon exiting SMI.
459 gSmmCpuPrivate
->FirstFreeToken
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
465 @param CpuIndex BSP processor Index
466 @param SyncMode SMM MP sync mode
472 IN SMM_CPU_SYNC_MODE SyncMode
478 BOOLEAN ClearTopLevelSmiResult
;
481 ASSERT (CpuIndex
== mSmmMpSyncData
->BspIndex
);
485 // Flag BSP's presence
487 *mSmmMpSyncData
->InsideSmm
= TRUE
;
490 // Initialize Debug Agent to start source level debug in BSP handler
492 InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI
, NULL
, NULL
);
495 // Mark this processor's presence
497 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
500 // Clear platform top level SMI status bit before calling SMI handlers. If
501 // we cleared it after SMI handlers are run, we would miss the SMI that
502 // occurs after SMI handlers are done and before SMI status bit is cleared.
504 ClearTopLevelSmiResult
= ClearTopLevelSmiStatus();
505 ASSERT (ClearTopLevelSmiResult
== TRUE
);
508 // Set running processor index
510 gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
= CpuIndex
;
513 // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
515 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
518 // Wait for APs to arrive
520 SmmWaitForApArrival();
523 // Lock the counter down and retrieve the number of APs
525 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
526 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
529 // Wait for all APs to get ready for programming MTRRs
531 WaitForAllAPs (ApCount
);
533 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
535 // Signal all APs it's time for backup MTRRs
540 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
541 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
542 // to a large enough value to avoid this situation.
543 // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
544 // We do the backup first and then set MTRR to avoid race condition for threads
547 MtrrGetAllMtrrs(&Mtrrs
);
550 // Wait for all APs to complete their MTRR saving
552 WaitForAllAPs (ApCount
);
555 // Let all processors program SMM MTRRs together
560 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
561 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
562 // to a large enough value to avoid this situation.
564 ReplaceOSMtrrs (CpuIndex
);
567 // Wait for all APs to complete their MTRR programming
569 WaitForAllAPs (ApCount
);
574 // The BUSY lock is initialized to Acquired state
576 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
579 // Perform the pre tasks
584 // Invoke SMM Foundation EntryPoint with the processor information context.
586 gSmmCpuPrivate
->SmmCoreEntry (&gSmmCpuPrivate
->SmmCoreEntryContext
);
589 // Make sure all APs have completed their pending none-block tasks
591 WaitForAllAPsNotBusy (TRUE
);
594 // Perform the remaining tasks
596 PerformRemainingTasks ();
599 // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
600 // make those APs to exit SMI synchronously. APs which arrive later will be excluded and
601 // will run through freely.
603 if (SyncMode
!= SmmCpuSyncModeTradition
&& !SmmCpuFeaturesNeedConfigureMtrrs()) {
606 // Lock the counter down and retrieve the number of APs
608 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
609 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
611 // Make sure all APs have their Present flag set
615 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
616 if (*(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
620 if (PresentCount
> ApCount
) {
627 // Notify all APs to exit
629 *mSmmMpSyncData
->InsideSmm
= FALSE
;
633 // Wait for all APs to complete their pending tasks
635 WaitForAllAPs (ApCount
);
637 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
639 // Signal APs to restore MTRRs
646 SmmCpuFeaturesReenableSmrr ();
647 MtrrSetAllMtrrs(&Mtrrs
);
650 // Wait for all APs to complete MTRR programming
652 WaitForAllAPs (ApCount
);
656 // Stop source level debug in BSP handler, the code below will not be
659 InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI
, NULL
, NULL
);
662 // Signal APs to Reset states/semaphore for this processor
667 // Perform pending operations for hot-plug
672 // Clear the Present flag of BSP
674 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
677 // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
678 // WaitForAllAps does not depend on the Present flag.
680 WaitForAllAPs (ApCount
);
683 // Reset the tokens buffer.
688 // Reset BspIndex to -1, meaning BSP has not been elected.
690 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
691 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
695 // Allow APs to check in from this point on
697 *mSmmMpSyncData
->Counter
= 0;
698 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
704 @param CpuIndex AP processor Index.
705 @param ValidSmi Indicates that current SMI is a valid SMI or not.
706 @param SyncMode SMM MP sync mode.
713 IN SMM_CPU_SYNC_MODE SyncMode
719 EFI_STATUS ProcedureStatus
;
724 for (Timer
= StartSyncTimer ();
725 !IsSyncTimerTimeout (Timer
) &&
726 !(*mSmmMpSyncData
->InsideSmm
);
731 if (!(*mSmmMpSyncData
->InsideSmm
)) {
733 // BSP timeout in the first round
735 if (mSmmMpSyncData
->BspIndex
!= -1) {
737 // BSP Index is known
739 BspIndex
= mSmmMpSyncData
->BspIndex
;
740 ASSERT (CpuIndex
!= BspIndex
);
743 // Send SMI IPI to bring BSP in
745 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[BspIndex
].ProcessorId
);
748 // Now clock BSP for the 2nd time
750 for (Timer
= StartSyncTimer ();
751 !IsSyncTimerTimeout (Timer
) &&
752 !(*mSmmMpSyncData
->InsideSmm
);
757 if (!(*mSmmMpSyncData
->InsideSmm
)) {
759 // Give up since BSP is unable to enter SMM
760 // and signal the completion of this AP
761 WaitForSemaphore (mSmmMpSyncData
->Counter
);
766 // Don't know BSP index. Give up without sending IPI to BSP.
768 WaitForSemaphore (mSmmMpSyncData
->Counter
);
776 BspIndex
= mSmmMpSyncData
->BspIndex
;
777 ASSERT (CpuIndex
!= BspIndex
);
780 // Mark this processor's presence
782 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
784 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
786 // Notify BSP of arrival at this point
788 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
791 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
793 // Wait for the signal from BSP to backup MTRRs
795 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
800 MtrrGetAllMtrrs(&Mtrrs
);
803 // Signal BSP the completion of this AP
805 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
808 // Wait for BSP's signal to program MTRRs
810 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
813 // Replace OS MTRRs with SMI MTRRs
815 ReplaceOSMtrrs (CpuIndex
);
818 // Signal BSP the completion of this AP
820 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
825 // Wait for something to happen
827 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
830 // Check if BSP wants to exit SMM
832 if (!(*mSmmMpSyncData
->InsideSmm
)) {
837 // BUSY should be acquired by SmmStartupThisAp()
840 !AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)
844 // Invoke the scheduled procedure
846 ProcedureStatus
= (*mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
) (
847 (VOID
*)mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
849 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
850 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= ProcedureStatus
;
853 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Token
!= NULL
) {
854 ReleaseToken (CpuIndex
);
860 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
863 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
865 // Notify BSP the readiness of this AP to program MTRRs
867 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
870 // Wait for the signal from BSP to program MTRRs
872 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
877 SmmCpuFeaturesReenableSmrr ();
878 MtrrSetAllMtrrs(&Mtrrs
);
882 // Notify BSP the readiness of this AP to Reset states/semaphore for this processor
884 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
887 // Wait for the signal from BSP to Reset states/semaphore for this processor
889 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
892 // Reset states/semaphore for this processor
894 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
897 // Notify BSP the readiness of this AP to exit SMM
899 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
904 Create 4G PageTable in SMRAM.
906 @param[in] Is32BitPageTable Whether the page table is 32-bit PAE
907 @return PageTable Address
912 IN BOOLEAN Is32BitPageTable
920 UINTN High2MBoundary
;
930 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
932 // Add one more page for known good stack, then find the lower 2MB aligned address.
934 Low2MBoundary
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
) & ~(SIZE_2MB
-1);
936 // Add two more pages for known good stack and stack guard page,
937 // then find the lower 2MB aligned address.
939 High2MBoundary
= (mSmmStackArrayEnd
- mSmmStackSize
+ EFI_PAGE_SIZE
* 2) & ~(SIZE_2MB
-1);
940 PagesNeeded
= ((High2MBoundary
- Low2MBoundary
) / SIZE_2MB
) + 1;
943 // Allocate the page table
945 PageTable
= AllocatePageTableMemory (5 + PagesNeeded
);
946 ASSERT (PageTable
!= NULL
);
948 PageTable
= (VOID
*)((UINTN
)PageTable
);
949 Pte
= (UINT64
*)PageTable
;
952 // Zero out all page table entries first
954 ZeroMem (Pte
, EFI_PAGES_TO_SIZE (1));
957 // Set Page Directory Pointers
959 for (Index
= 0; Index
< 4; Index
++) {
960 Pte
[Index
] = ((UINTN
)PageTable
+ EFI_PAGE_SIZE
* (Index
+ 1)) | mAddressEncMask
|
961 (Is32BitPageTable
? IA32_PAE_PDPTE_ATTRIBUTE_BITS
: PAGE_ATTRIBUTE_BITS
);
963 Pte
+= EFI_PAGE_SIZE
/ sizeof (*Pte
);
966 // Fill in Page Directory Entries
968 for (Index
= 0; Index
< EFI_PAGE_SIZE
* 4 / sizeof (*Pte
); Index
++) {
969 Pte
[Index
] = (Index
<< 21) | mAddressEncMask
| IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
972 Pdpte
= (UINT64
*)PageTable
;
973 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
974 Pages
= (UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (5);
975 GuardPage
= mSmmStackArrayBase
+ EFI_PAGE_SIZE
;
976 for (PageIndex
= Low2MBoundary
; PageIndex
<= High2MBoundary
; PageIndex
+= SIZE_2MB
) {
977 Pte
= (UINT64
*)(UINTN
)(Pdpte
[BitFieldRead32 ((UINT32
)PageIndex
, 30, 31)] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
978 Pte
[BitFieldRead32 ((UINT32
)PageIndex
, 21, 29)] = (UINT64
)Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
980 // Fill in Page Table Entries
982 Pte
= (UINT64
*)Pages
;
983 PageAddress
= PageIndex
;
984 for (Index
= 0; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
985 if (PageAddress
== GuardPage
) {
987 // Mark the guard page as non-present
989 Pte
[Index
] = PageAddress
| mAddressEncMask
;
990 GuardPage
+= mSmmStackSize
;
991 if (GuardPage
> mSmmStackArrayEnd
) {
995 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
997 PageAddress
+= EFI_PAGE_SIZE
;
999 Pages
+= EFI_PAGE_SIZE
;
1003 if ((PcdGet8 (PcdNullPointerDetectionPropertyMask
) & BIT1
) != 0) {
1004 Pte
= (UINT64
*)(UINTN
)(Pdpte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
1005 if ((Pte
[0] & IA32_PG_PS
) == 0) {
1006 // 4K-page entries are already mapped. Just hide the first one anyway.
1007 Pte
= (UINT64
*)(UINTN
)(Pte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
1008 Pte
[0] &= ~(UINT64
)IA32_PG_P
; // Hide page 0
1010 // Create 4K-page entries
1011 Pages
= (UINTN
)AllocatePageTableMemory (1);
1012 ASSERT (Pages
!= 0);
1014 Pte
[0] = (UINT64
)(Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
);
1016 Pte
= (UINT64
*)Pages
;
1018 Pte
[0] = PageAddress
| mAddressEncMask
; // Hide page 0 but present left
1019 for (Index
= 1; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
1020 PageAddress
+= EFI_PAGE_SIZE
;
1021 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
1026 return (UINT32
)(UINTN
)PageTable
;
1030 Checks whether the input token is the current used token.
1032 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1035 @retval TRUE The input token is the current used token.
1036 @retval FALSE The input token is not the current used token.
1044 PROCEDURE_TOKEN
*ProcToken
;
1046 if (Token
== NULL
) {
1050 Link
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
1052 // Only search used tokens.
1054 while (Link
!= gSmmCpuPrivate
->FirstFreeToken
) {
1055 ProcToken
= PROCEDURE_TOKEN_FROM_LINK (Link
);
1057 if (ProcToken
->SpinLock
== Token
) {
1061 Link
= GetNextNode (&gSmmCpuPrivate
->TokenList
, Link
);
1068 Allocate buffer for the SPIN_LOCK and PROCEDURE_TOKEN.
1070 @return First token of the token buffer.
1073 AllocateTokenBuffer (
1078 UINT32 TokenCountPerChunk
;
1080 SPIN_LOCK
*SpinLock
;
1081 UINT8
*SpinLockBuffer
;
1082 PROCEDURE_TOKEN
*ProcTokens
;
1084 SpinLockSize
= GetSpinLockProperties ();
1086 TokenCountPerChunk
= FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk
);
1087 ASSERT (TokenCountPerChunk
!= 0);
1088 if (TokenCountPerChunk
== 0) {
1089 DEBUG ((DEBUG_ERROR
, "PcdCpuSmmMpTokenCountPerChunk should not be Zero!\n"));
1092 DEBUG ((DEBUG_INFO
, "CpuSmm: SpinLock Size = 0x%x, PcdCpuSmmMpTokenCountPerChunk = 0x%x\n", SpinLockSize
, TokenCountPerChunk
));
1095 // Separate the Spin_lock and Proc_token because the alignment requires by Spin_Lock.
1097 SpinLockBuffer
= AllocatePool (SpinLockSize
* TokenCountPerChunk
);
1098 ASSERT (SpinLockBuffer
!= NULL
);
1100 ProcTokens
= AllocatePool (sizeof (PROCEDURE_TOKEN
) * TokenCountPerChunk
);
1101 ASSERT (ProcTokens
!= NULL
);
1103 for (Index
= 0; Index
< TokenCountPerChunk
; Index
++) {
1104 SpinLock
= (SPIN_LOCK
*)(SpinLockBuffer
+ SpinLockSize
* Index
);
1105 InitializeSpinLock (SpinLock
);
1107 ProcTokens
[Index
].Signature
= PROCEDURE_TOKEN_SIGNATURE
;
1108 ProcTokens
[Index
].SpinLock
= SpinLock
;
1109 ProcTokens
[Index
].RunningApCount
= 0;
1111 InsertTailList (&gSmmCpuPrivate
->TokenList
, &ProcTokens
[Index
].Link
);
1114 return &ProcTokens
[0].Link
;
1120 If no free token, allocate new tokens then return the free one.
1122 @param RunningApsCount The Running Aps count for this token.
1124 @retval return the first free PROCEDURE_TOKEN.
1129 IN UINT32 RunningApsCount
1132 PROCEDURE_TOKEN
*NewToken
;
1135 // If FirstFreeToken meets the end of token list, enlarge the token list.
1136 // Set FirstFreeToken to the first free token.
1138 if (gSmmCpuPrivate
->FirstFreeToken
== &gSmmCpuPrivate
->TokenList
) {
1139 gSmmCpuPrivate
->FirstFreeToken
= AllocateTokenBuffer ();
1141 NewToken
= PROCEDURE_TOKEN_FROM_LINK (gSmmCpuPrivate
->FirstFreeToken
);
1142 gSmmCpuPrivate
->FirstFreeToken
= GetNextNode (&gSmmCpuPrivate
->TokenList
, gSmmCpuPrivate
->FirstFreeToken
);
1144 NewToken
->RunningApCount
= RunningApsCount
;
1145 AcquireSpinLock (NewToken
->SpinLock
);
1151 Checks status of specified AP.
1153 This function checks whether the specified AP has finished the task assigned
1154 by StartupThisAP(), and whether timeout expires.
1156 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1159 @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
1160 @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
1167 if (AcquireSpinLockOrFail (Token
)) {
1168 ReleaseSpinLock (Token
);
1172 return EFI_NOT_READY
;
1176 Schedule a procedure to run on the specified CPU.
1178 @param[in] Procedure The address of the procedure to run
1179 @param[in] CpuIndex Target CPU Index
1180 @param[in,out] ProcArguments The parameter to pass to the procedure
1181 @param[in] Token This is an optional parameter that allows the caller to execute the
1182 procedure in a blocking or non-blocking fashion. If it is NULL the
1183 call is blocking, and the call will not return until the AP has
1184 completed the procedure. If the token is not NULL, the call will
1185 return immediately. The caller can check whether the procedure has
1186 completed with CheckOnProcedure or WaitForProcedure.
1187 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for the APs to finish
1188 execution of Procedure, either for blocking or non-blocking mode.
1189 Zero means infinity. If the timeout expires before all APs return
1190 from Procedure, then Procedure on the failed APs is terminated. If
1191 the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
1192 If the timeout expires in non-blocking mode, the timeout determined
1193 can be through CheckOnProcedure or WaitForProcedure.
1194 Note that timeout support is optional. Whether an implementation
1195 supports this feature can be determined via the Attributes data
1197 @param[in,out] CpuStatus This optional pointer may be used to get the status code returned
1198 by Procedure when it completes execution on the target AP, or with
1199 EFI_TIMEOUT if the Procedure fails to complete within the optional
1200 timeout. The implementation will update this variable with
1201 EFI_NOT_READY prior to starting Procedure on the target AP.
1203 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1204 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1205 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1206 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1207 @retval EFI_SUCCESS The procedure has been successfully scheduled
1211 InternalSmmStartupThisAp (
1212 IN EFI_AP_PROCEDURE2 Procedure
,
1214 IN OUT VOID
*ProcArguments OPTIONAL
,
1215 IN MM_COMPLETION
*Token
,
1216 IN UINTN TimeoutInMicroseconds
,
1217 IN OUT EFI_STATUS
*CpuStatus
1220 PROCEDURE_TOKEN
*ProcToken
;
1222 if (CpuIndex
>= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
) {
1223 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus(%d)\n", CpuIndex
, gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
));
1224 return EFI_INVALID_PARAMETER
;
1226 if (CpuIndex
== gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) {
1227 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu\n", CpuIndex
));
1228 return EFI_INVALID_PARAMETER
;
1230 if (gSmmCpuPrivate
->ProcessorInfo
[CpuIndex
].ProcessorId
== INVALID_APIC_ID
) {
1231 return EFI_INVALID_PARAMETER
;
1233 if (!(*(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
))) {
1234 if (mSmmMpSyncData
->EffectiveSyncMode
== SmmCpuSyncModeTradition
) {
1235 DEBUG((DEBUG_ERROR
, "!mSmmMpSyncData->CpuData[%d].Present\n", CpuIndex
));
1237 return EFI_INVALID_PARAMETER
;
1239 if (gSmmCpuPrivate
->Operation
[CpuIndex
] == SmmCpuRemove
) {
1240 if (!FeaturePcdGet (PcdCpuHotPlugSupport
)) {
1241 DEBUG((DEBUG_ERROR
, "gSmmCpuPrivate->Operation[%d] == SmmCpuRemove\n", CpuIndex
));
1243 return EFI_INVALID_PARAMETER
;
1245 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1246 return EFI_INVALID_PARAMETER
;
1248 if (Procedure
== NULL
) {
1249 return EFI_INVALID_PARAMETER
;
1252 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1254 mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
= Procedure
;
1255 mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
= ProcArguments
;
1256 if (Token
!= NULL
) {
1257 ProcToken
= GetFreeToken (1);
1258 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= ProcToken
;
1259 *Token
= (MM_COMPLETION
)ProcToken
->SpinLock
;
1261 mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= CpuStatus
;
1262 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
1263 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= EFI_NOT_READY
;
1266 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
1268 if (Token
== NULL
) {
1269 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1270 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1277 Worker function to execute a caller provided function on all enabled APs.
1279 @param[in] Procedure A pointer to the function to be run on
1280 enabled APs of the system.
1281 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
1282 APs to return from Procedure, either for
1283 blocking or non-blocking mode.
1284 @param[in,out] ProcedureArguments The parameter passed into Procedure for
1286 @param[in,out] Token This is an optional parameter that allows the caller to execute the
1287 procedure in a blocking or non-blocking fashion. If it is NULL the
1288 call is blocking, and the call will not return until the AP has
1289 completed the procedure. If the token is not NULL, the call will
1290 return immediately. The caller can check whether the procedure has
1291 completed with CheckOnProcedure or WaitForProcedure.
1292 @param[in,out] CPUStatus This optional pointer may be used to get the status code returned
1293 by Procedure when it completes execution on the target AP, or with
1294 EFI_TIMEOUT if the Procedure fails to complete within the optional
1295 timeout. The implementation will update this variable with
1296 EFI_NOT_READY prior to starting Procedure on the target AP.
1299 @retval EFI_SUCCESS In blocking mode, all APs have finished before
1300 the timeout expired.
1301 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
1303 @retval others Failed to Startup all APs.
1307 InternalSmmStartupAllAPs (
1308 IN EFI_AP_PROCEDURE2 Procedure
,
1309 IN UINTN TimeoutInMicroseconds
,
1310 IN OUT VOID
*ProcedureArguments OPTIONAL
,
1311 IN OUT MM_COMPLETION
*Token
,
1312 IN OUT EFI_STATUS
*CPUStatus
1317 PROCEDURE_TOKEN
*ProcToken
;
1319 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1320 return EFI_INVALID_PARAMETER
;
1322 if (Procedure
== NULL
) {
1323 return EFI_INVALID_PARAMETER
;
1327 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1328 if (IsPresentAp (Index
)) {
1331 if (gSmmCpuPrivate
->Operation
[Index
] == SmmCpuRemove
) {
1332 return EFI_INVALID_PARAMETER
;
1335 if (!AcquireSpinLockOrFail(mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
1336 return EFI_NOT_READY
;
1338 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1341 if (CpuCount
== 0) {
1342 return EFI_NOT_STARTED
;
1345 if (Token
!= NULL
) {
1346 ProcToken
= GetFreeToken ((UINT32
)mMaxNumberOfCpus
);
1347 *Token
= (MM_COMPLETION
)ProcToken
->SpinLock
;
1353 // Make sure all BUSY should be acquired.
1355 // Because former code already check mSmmMpSyncData->CpuData[***].Busy for each AP.
1356 // Here code always use AcquireSpinLock instead of AcquireSpinLockOrFail for not
1359 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1360 if (IsPresentAp (Index
)) {
1361 AcquireSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1365 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1366 if (IsPresentAp (Index
)) {
1367 mSmmMpSyncData
->CpuData
[Index
].Procedure
= (EFI_AP_PROCEDURE2
) Procedure
;
1368 mSmmMpSyncData
->CpuData
[Index
].Parameter
= ProcedureArguments
;
1369 if (ProcToken
!= NULL
) {
1370 mSmmMpSyncData
->CpuData
[Index
].Token
= ProcToken
;
1372 if (CPUStatus
!= NULL
) {
1373 mSmmMpSyncData
->CpuData
[Index
].Status
= &CPUStatus
[Index
];
1374 if (mSmmMpSyncData
->CpuData
[Index
].Status
!= NULL
) {
1375 *mSmmMpSyncData
->CpuData
[Index
].Status
= EFI_NOT_READY
;
1380 // PI spec requirement:
1381 // For every excluded processor, the array entry must contain a value of EFI_NOT_STARTED.
1383 if (CPUStatus
!= NULL
) {
1384 CPUStatus
[Index
] = EFI_NOT_STARTED
;
1388 // Decrease the count to mark this processor(AP or BSP) as finished.
1390 if (ProcToken
!= NULL
) {
1391 WaitForSemaphore (&ProcToken
->RunningApCount
);
1398 if (Token
== NULL
) {
1400 // Make sure all APs have completed their tasks.
1402 WaitForAllAPsNotBusy (TRUE
);
1409 ISO C99 6.5.2.2 "Function calls", paragraph 9:
1410 If the function is defined with a type that is not compatible with
1411 the type (of the expression) pointed to by the expression that
1412 denotes the called function, the behavior is undefined.
1414 So add below wrapper function to convert between EFI_AP_PROCEDURE
1415 and EFI_AP_PROCEDURE2.
1417 Wrapper for Procedures.
1419 @param[in] Buffer Pointer to PROCEDURE_WRAPPER buffer.
1428 PROCEDURE_WRAPPER
*Wrapper
;
1431 Wrapper
->Procedure (Wrapper
->ProcedureArgument
);
1437 Schedule a procedure to run on the specified CPU in blocking mode.
1439 @param[in] Procedure The address of the procedure to run
1440 @param[in] CpuIndex Target CPU Index
1441 @param[in, out] ProcArguments The parameter to pass to the procedure
1443 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1444 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1445 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1446 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1447 @retval EFI_SUCCESS The procedure has been successfully scheduled
1452 SmmBlockingStartupThisAp (
1453 IN EFI_AP_PROCEDURE Procedure
,
1455 IN OUT VOID
*ProcArguments OPTIONAL
1458 PROCEDURE_WRAPPER Wrapper
;
1460 Wrapper
.Procedure
= Procedure
;
1461 Wrapper
.ProcedureArgument
= ProcArguments
;
1464 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1466 return InternalSmmStartupThisAp (ProcedureWrapper
, CpuIndex
, &Wrapper
, NULL
, 0, NULL
);
1470 Schedule a procedure to run on the specified CPU.
1472 @param Procedure The address of the procedure to run
1473 @param CpuIndex Target CPU Index
1474 @param ProcArguments The parameter to pass to the procedure
1476 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1477 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1478 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1479 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1480 @retval EFI_SUCCESS The procedure has been successfully scheduled
1486 IN EFI_AP_PROCEDURE Procedure
,
1488 IN OUT VOID
*ProcArguments OPTIONAL
1491 MM_COMPLETION Token
;
1493 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].Procedure
= Procedure
;
1494 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].ProcedureArgument
= ProcArguments
;
1497 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1499 return InternalSmmStartupThisAp (
1502 &gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
],
1503 FeaturePcdGet (PcdCpuSmmBlockStartupThisAp
) ? NULL
: &Token
,
1510 This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
1511 They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
1513 NOTE: It might not be appreciated in runtime since it might
1514 conflict with OS debugging facilities. Turn them off in RELEASE.
1516 @param CpuIndex CPU Index
1525 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1527 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1528 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1529 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1530 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1531 AsmWriteDr6 (CpuSaveState
->x86
._DR6
);
1532 AsmWriteDr7 (CpuSaveState
->x86
._DR7
);
1534 AsmWriteDr6 ((UINTN
)CpuSaveState
->x64
._DR6
);
1535 AsmWriteDr7 ((UINTN
)CpuSaveState
->x64
._DR7
);
1541 This function restores DR6 & DR7 to SMM save state.
1543 NOTE: It might not be appreciated in runtime since it might
1544 conflict with OS debugging facilities. Turn them off in RELEASE.
1546 @param CpuIndex CPU Index
1555 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1557 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1558 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1559 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1560 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1561 CpuSaveState
->x86
._DR7
= (UINT32
)AsmReadDr7 ();
1562 CpuSaveState
->x86
._DR6
= (UINT32
)AsmReadDr6 ();
1564 CpuSaveState
->x64
._DR7
= AsmReadDr7 ();
1565 CpuSaveState
->x64
._DR6
= AsmReadDr6 ();
1571 C function for SMI entry, each processor comes here upon SMI trigger.
1573 @param CpuIndex CPU Index
1585 BOOLEAN BspInProgress
;
1589 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1592 // Save Cr2 because Page Fault exception in SMM may override its value,
1593 // when using on-demand paging for above 4G memory.
1599 // Call the user register Startup function first.
1601 if (mSmmMpSyncData
->StartupProcedure
!= NULL
) {
1602 mSmmMpSyncData
->StartupProcedure (mSmmMpSyncData
->StartupProcArgs
);
1606 // Perform CPU specific entry hooks
1608 SmmCpuFeaturesRendezvousEntry (CpuIndex
);
1611 // Determine if this is a valid SMI
1613 ValidSmi
= PlatformValidSmi();
1616 // Determine if BSP has been already in progress. Note this must be checked after
1617 // ValidSmi because BSP may clear a valid SMI source after checking in.
1619 BspInProgress
= *mSmmMpSyncData
->InsideSmm
;
1621 if (!BspInProgress
&& !ValidSmi
) {
1623 // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not
1624 // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI
1625 // status had been cleared by BSP and an existing SMI run has almost ended. (Note
1626 // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there
1627 // is nothing we need to do.
1632 // Signal presence of this processor
1634 if (ReleaseSemaphore (mSmmMpSyncData
->Counter
) == 0) {
1636 // BSP has already ended the synchronization, so QUIT!!!
1640 // Wait for BSP's signal to finish SMI
1642 while (*mSmmMpSyncData
->AllCpusInSync
) {
1649 // The BUSY lock is initialized to Released state.
1650 // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.
1651 // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
1652 // after AP's present flag is detected.
1654 InitializeSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1657 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1658 ActivateSmmProfile (CpuIndex
);
1661 if (BspInProgress
) {
1663 // BSP has been elected. Follow AP path, regardless of ValidSmi flag
1664 // as BSP may have cleared the SMI status
1666 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1669 // We have a valid SMI
1676 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1677 if (!mSmmMpSyncData
->SwitchBsp
|| mSmmMpSyncData
->CandidateBsp
[CpuIndex
]) {
1679 // Call platform hook to do BSP election
1681 Status
= PlatformSmmBspElection (&IsBsp
);
1682 if (EFI_SUCCESS
== Status
) {
1684 // Platform hook determines successfully
1687 mSmmMpSyncData
->BspIndex
= (UINT32
)CpuIndex
;
1691 // Platform hook fails to determine, use default BSP election method
1693 InterlockedCompareExchange32 (
1694 (UINT32
*)&mSmmMpSyncData
->BspIndex
,
1703 // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP
1705 if (mSmmMpSyncData
->BspIndex
== CpuIndex
) {
1708 // Clear last request for SwitchBsp.
1710 if (mSmmMpSyncData
->SwitchBsp
) {
1711 mSmmMpSyncData
->SwitchBsp
= FALSE
;
1712 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1713 mSmmMpSyncData
->CandidateBsp
[Index
] = FALSE
;
1717 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1718 SmmProfileRecordSmiNum ();
1722 // BSP Handler is always called with a ValidSmi == TRUE
1724 BSPHandler (CpuIndex
, mSmmMpSyncData
->EffectiveSyncMode
);
1726 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1730 ASSERT (*mSmmMpSyncData
->CpuData
[CpuIndex
].Run
== 0);
1733 // Wait for BSP's signal to exit SMI
1735 while (*mSmmMpSyncData
->AllCpusInSync
) {
1741 SmmCpuFeaturesRendezvousExit (CpuIndex
);
1750 Allocate buffer for SpinLock and Wrapper function buffer.
1754 InitializeDataForMmMp (
1758 gSmmCpuPrivate
->ApWrapperFunc
= AllocatePool (sizeof (PROCEDURE_WRAPPER
) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1759 ASSERT (gSmmCpuPrivate
->ApWrapperFunc
!= NULL
);
1761 InitializeListHead (&gSmmCpuPrivate
->TokenList
);
1763 gSmmCpuPrivate
->FirstFreeToken
= AllocateTokenBuffer ();
1767 Allocate buffer for all semaphores and spin locks.
1771 InitializeSmmCpuSemaphores (
1775 UINTN ProcessorCount
;
1777 UINTN GlobalSemaphoresSize
;
1778 UINTN CpuSemaphoresSize
;
1779 UINTN SemaphoreSize
;
1781 UINTN
*SemaphoreBlock
;
1782 UINTN SemaphoreAddr
;
1784 SemaphoreSize
= GetSpinLockProperties ();
1785 ProcessorCount
= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1786 GlobalSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_GLOBAL
) / sizeof (VOID
*)) * SemaphoreSize
;
1787 CpuSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_CPU
) / sizeof (VOID
*)) * ProcessorCount
* SemaphoreSize
;
1788 TotalSize
= GlobalSemaphoresSize
+ CpuSemaphoresSize
;
1789 DEBUG((EFI_D_INFO
, "One Semaphore Size = 0x%x\n", SemaphoreSize
));
1790 DEBUG((EFI_D_INFO
, "Total Semaphores Size = 0x%x\n", TotalSize
));
1791 Pages
= EFI_SIZE_TO_PAGES (TotalSize
);
1792 SemaphoreBlock
= AllocatePages (Pages
);
1793 ASSERT (SemaphoreBlock
!= NULL
);
1794 ZeroMem (SemaphoreBlock
, TotalSize
);
1796 SemaphoreAddr
= (UINTN
)SemaphoreBlock
;
1797 mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
= (UINT32
*)SemaphoreAddr
;
1798 SemaphoreAddr
+= SemaphoreSize
;
1799 mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
= (BOOLEAN
*)SemaphoreAddr
;
1800 SemaphoreAddr
+= SemaphoreSize
;
1801 mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
= (BOOLEAN
*)SemaphoreAddr
;
1802 SemaphoreAddr
+= SemaphoreSize
;
1803 mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
= (SPIN_LOCK
*)SemaphoreAddr
;
1804 SemaphoreAddr
+= SemaphoreSize
;
1805 mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
1806 = (SPIN_LOCK
*)SemaphoreAddr
;
1807 SemaphoreAddr
+= SemaphoreSize
;
1809 SemaphoreAddr
= (UINTN
)SemaphoreBlock
+ GlobalSemaphoresSize
;
1810 mSmmCpuSemaphores
.SemaphoreCpu
.Busy
= (SPIN_LOCK
*)SemaphoreAddr
;
1811 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1812 mSmmCpuSemaphores
.SemaphoreCpu
.Run
= (UINT32
*)SemaphoreAddr
;
1813 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1814 mSmmCpuSemaphores
.SemaphoreCpu
.Present
= (BOOLEAN
*)SemaphoreAddr
;
1816 mPFLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
;
1817 mConfigSmmCodeAccessCheckLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
;
1819 mSemaphoreSize
= SemaphoreSize
;
1823 Initialize un-cacheable data.
1828 InitializeMpSyncData (
1834 if (mSmmMpSyncData
!= NULL
) {
1836 // mSmmMpSyncDataSize includes one structure of SMM_DISPATCHER_MP_SYNC_DATA, one
1837 // CpuData array of SMM_CPU_DATA_BLOCK and one CandidateBsp array of BOOLEAN.
1839 ZeroMem (mSmmMpSyncData
, mSmmMpSyncDataSize
);
1840 mSmmMpSyncData
->CpuData
= (SMM_CPU_DATA_BLOCK
*)((UINT8
*)mSmmMpSyncData
+ sizeof (SMM_DISPATCHER_MP_SYNC_DATA
));
1841 mSmmMpSyncData
->CandidateBsp
= (BOOLEAN
*)(mSmmMpSyncData
->CpuData
+ gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1842 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1844 // Enable BSP election by setting BspIndex to -1
1846 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
1848 mSmmMpSyncData
->EffectiveSyncMode
= mCpuSmmSyncMode
;
1850 mSmmMpSyncData
->Counter
= mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
;
1851 mSmmMpSyncData
->InsideSmm
= mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
;
1852 mSmmMpSyncData
->AllCpusInSync
= mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
;
1853 ASSERT (mSmmMpSyncData
->Counter
!= NULL
&& mSmmMpSyncData
->InsideSmm
!= NULL
&&
1854 mSmmMpSyncData
->AllCpusInSync
!= NULL
);
1855 *mSmmMpSyncData
->Counter
= 0;
1856 *mSmmMpSyncData
->InsideSmm
= FALSE
;
1857 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
1859 for (CpuIndex
= 0; CpuIndex
< gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
; CpuIndex
++) {
1860 mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
=
1861 (SPIN_LOCK
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Busy
+ mSemaphoreSize
* CpuIndex
);
1862 mSmmMpSyncData
->CpuData
[CpuIndex
].Run
=
1863 (UINT32
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Run
+ mSemaphoreSize
* CpuIndex
);
1864 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
=
1865 (BOOLEAN
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Present
+ mSemaphoreSize
* CpuIndex
);
1866 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
) = 0;
1867 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Run
) = 0;
1868 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
1874 Initialize global data for MP synchronization.
1876 @param Stacks Base address of SMI stack buffer for all processors.
1877 @param StackSize Stack size for each processor in SMM.
1878 @param ShadowStackSize Shadow Stack size for each processor in SMM.
1882 InitializeMpServiceData (
1885 IN UINTN ShadowStackSize
1890 UINT8
*GdtTssTables
;
1891 UINTN GdtTableStepSize
;
1892 CPUID_VERSION_INFO_EDX RegEdx
;
1895 // Determine if this CPU supports machine check
1897 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
1898 mMachineCheckSupported
= (BOOLEAN
)(RegEdx
.Bits
.MCA
== 1);
1901 // Allocate memory for all locks and semaphores
1903 InitializeSmmCpuSemaphores ();
1906 // Initialize mSmmMpSyncData
1908 mSmmMpSyncDataSize
= sizeof (SMM_DISPATCHER_MP_SYNC_DATA
) +
1909 (sizeof (SMM_CPU_DATA_BLOCK
) + sizeof (BOOLEAN
)) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1910 mSmmMpSyncData
= (SMM_DISPATCHER_MP_SYNC_DATA
*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize
));
1911 ASSERT (mSmmMpSyncData
!= NULL
);
1912 mCpuSmmSyncMode
= (SMM_CPU_SYNC_MODE
)PcdGet8 (PcdCpuSmmSyncMode
);
1913 InitializeMpSyncData ();
1916 // Initialize physical address mask
1917 // NOTE: Physical memory above virtual address limit is not supported !!!
1919 AsmCpuid (0x80000008, (UINT32
*)&Index
, NULL
, NULL
, NULL
);
1920 gPhyMask
= LShiftU64 (1, (UINT8
)Index
) - 1;
1921 gPhyMask
&= (1ull << 48) - EFI_PAGE_SIZE
;
1924 // Create page tables
1926 Cr3
= SmmInitPageTable ();
1928 GdtTssTables
= InitGdt (Cr3
, &GdtTableStepSize
);
1931 // Install SMI handler for each CPU
1933 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1936 (UINT32
)mCpuHotPlugData
.SmBase
[Index
],
1937 (VOID
*)((UINTN
)Stacks
+ (StackSize
+ ShadowStackSize
) * Index
),
1939 (UINTN
)(GdtTssTables
+ GdtTableStepSize
* Index
),
1940 gcSmiGdtr
.Limit
+ 1,
1942 gcSmiIdtr
.Limit
+ 1,
1948 // Record current MTRR settings
1950 ZeroMem (&gSmiMtrrs
, sizeof (gSmiMtrrs
));
1951 MtrrGetAllMtrrs (&gSmiMtrrs
);
1958 Register the SMM Foundation entry point.
1960 @param This Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance
1961 @param SmmEntryPoint SMM Foundation EntryPoint
1963 @retval EFI_SUCCESS Successfully to register SMM foundation entry point
1969 IN CONST EFI_SMM_CONFIGURATION_PROTOCOL
*This
,
1970 IN EFI_SMM_ENTRY_POINT SmmEntryPoint
1974 // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.
1976 gSmmCpuPrivate
->SmmCoreEntry
= SmmEntryPoint
;
1982 Register the SMM Foundation entry point.
1984 @param[in] Procedure A pointer to the code stream to be run on the designated target AP
1985 of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
1986 with the related definitions of
1987 EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
1988 If caller may pass a value of NULL to deregister any existing
1990 @param[in,out] ProcedureArguments Allows the caller to pass a list of parameters to the code that is
1991 run by the AP. It is an optional common mailbox between APs and
1992 the caller to share information
1994 @retval EFI_SUCCESS The Procedure has been set successfully.
1995 @retval EFI_INVALID_PARAMETER The Procedure is NULL but ProcedureArguments not NULL.
1999 RegisterStartupProcedure (
2000 IN EFI_AP_PROCEDURE Procedure
,
2001 IN OUT VOID
*ProcedureArguments OPTIONAL
2004 if (Procedure
== NULL
&& ProcedureArguments
!= NULL
) {
2005 return EFI_INVALID_PARAMETER
;
2007 if (mSmmMpSyncData
== NULL
) {
2008 return EFI_NOT_READY
;
2011 mSmmMpSyncData
->StartupProcedure
= Procedure
;
2012 mSmmMpSyncData
->StartupProcArgs
= ProcedureArguments
;