2 SMM MP service implementation
4 Copyright (c) 2009 - 2020, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
11 #include "PiSmmCpuDxeSmm.h"
14 // Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
16 MTRR_SETTINGS gSmiMtrrs
;
18 SMM_DISPATCHER_MP_SYNC_DATA
*mSmmMpSyncData
= NULL
;
19 UINTN mSmmMpSyncDataSize
;
20 SMM_CPU_SEMAPHORES mSmmCpuSemaphores
;
22 SPIN_LOCK
*mPFLock
= NULL
;
23 SMM_CPU_SYNC_MODE mCpuSmmSyncMode
;
24 BOOLEAN mMachineCheckSupported
= FALSE
;
27 Performs an atomic compare exchange operation to get semaphore.
28 The compare exchange operation must be performed using
31 @param Sem IN: 32-bit unsigned integer
32 OUT: original integer - 1
33 @return Original integer - 1
38 IN OUT
volatile UINT32
*Sem
45 } while (Value
== 0 ||
46 InterlockedCompareExchange32 (
56 Performs an atomic compare exchange operation to release semaphore.
57 The compare exchange operation must be performed using
60 @param Sem IN: 32-bit unsigned integer
61 OUT: original integer + 1
62 @return Original integer + 1
67 IN OUT
volatile UINT32
*Sem
74 } while (Value
+ 1 != 0 &&
75 InterlockedCompareExchange32 (
84 Performs an atomic compare exchange operation to lock semaphore.
85 The compare exchange operation must be performed using
88 @param Sem IN: 32-bit unsigned integer
90 @return Original integer
95 IN OUT
volatile UINT32
*Sem
102 } while (InterlockedCompareExchange32 (
110 Wait all APs to performs an atomic compare exchange operation to release semaphore.
112 @param NumberOfAPs AP number
122 BspIndex
= mSmmMpSyncData
->BspIndex
;
123 while (NumberOfAPs
-- > 0) {
124 WaitForSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
129 Performs an atomic compare exchange operation to release semaphore
140 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
141 if (IsPresentAp (Index
)) {
142 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[Index
].Run
);
148 Checks if all CPUs (with certain exceptions) have checked in for this SMI run
150 @param Exceptions CPU Arrival exception flags.
152 @retval TRUE if all CPUs the have checked in.
153 @retval FALSE if at least one Normal AP hasn't checked in.
157 AllCpusInSmmWithExceptions (
158 SMM_CPU_ARRIVAL_EXCEPTIONS Exceptions
162 SMM_CPU_DATA_BLOCK
*CpuData
;
163 EFI_PROCESSOR_INFORMATION
*ProcessorInfo
;
165 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
167 if (*mSmmMpSyncData
->Counter
== mNumberOfCpus
) {
171 CpuData
= mSmmMpSyncData
->CpuData
;
172 ProcessorInfo
= gSmmCpuPrivate
->ProcessorInfo
;
173 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
174 if (!(*(CpuData
[Index
].Present
)) && ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
175 if (((Exceptions
& ARRIVAL_EXCEPTION_DELAYED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmDelayed
) != 0) {
178 if (((Exceptions
& ARRIVAL_EXCEPTION_BLOCKED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmBlocked
) != 0) {
181 if (((Exceptions
& ARRIVAL_EXCEPTION_SMI_DISABLED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmEnable
) != 0) {
193 Has OS enabled Lmce in the MSR_IA32_MCG_EXT_CTL
195 @retval TRUE Os enable lmce.
196 @retval FALSE Os not enable lmce.
204 MSR_IA32_MCG_CAP_REGISTER McgCap
;
205 MSR_IA32_FEATURE_CONTROL_REGISTER FeatureCtrl
;
206 MSR_IA32_MCG_EXT_CTL_REGISTER McgExtCtrl
;
208 McgCap
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_CAP
);
209 if (McgCap
.Bits
.MCG_LMCE_P
== 0) {
213 FeatureCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_FEATURE_CONTROL
);
214 if (FeatureCtrl
.Bits
.LmceOn
== 0) {
218 McgExtCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_EXT_CTL
);
219 return (BOOLEAN
) (McgExtCtrl
.Bits
.LMCE_EN
== 1);
223 Return if Local machine check exception signaled.
225 Indicates (when set) that a local machine check exception was generated. This indicates that the current machine-check event was
226 delivered to only the logical processor.
228 @retval TRUE LMCE was signaled.
229 @retval FALSE LMCE was not signaled.
237 MSR_IA32_MCG_STATUS_REGISTER McgStatus
;
239 McgStatus
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_STATUS
);
240 return (BOOLEAN
) (McgStatus
.Bits
.LMCE_S
== 1);
244 Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
245 entering SMM, except SMI disabled APs.
249 SmmWaitForApArrival (
258 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
262 if (mMachineCheckSupported
) {
263 LmceEn
= IsLmceOsEnabled ();
264 LmceSignal
= IsLmceSignaled();
268 // Platform implementor should choose a timeout value appropriately:
269 // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note
270 // the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.
271 // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI
272 // and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will
273 // be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the
274 // SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.
275 // - The timeout value must be longer than longest possible IO operation in the system
279 // Sync with APs 1st timeout
281 for (Timer
= StartSyncTimer ();
282 !IsSyncTimerTimeout (Timer
) && !(LmceEn
&& LmceSignal
) &&
283 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
289 // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,
291 // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running
292 // normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they
293 // enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode
294 // work while SMI handling is on-going.
295 // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.
296 // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state
297 // will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal
298 // mode work while SMI handling is on-going.
299 // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:
300 // - In traditional flow, SMI disabling is discouraged.
301 // - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
302 // In both cases, adding SMI-disabling checking code increases overhead.
304 if (*mSmmMpSyncData
->Counter
< mNumberOfCpus
) {
306 // Send SMI IPIs to bring outside processors in
308 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
309 if (!(*(mSmmMpSyncData
->CpuData
[Index
].Present
)) && gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
310 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
);
315 // Sync with APs 2nd timeout.
317 for (Timer
= StartSyncTimer ();
318 !IsSyncTimerTimeout (Timer
) &&
319 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
330 Replace OS MTRR's with SMI MTRR's.
332 @param CpuIndex Processor Index
340 SmmCpuFeaturesDisableSmrr ();
343 // Replace all MTRRs registers
345 MtrrSetAllMtrrs (&gSmiMtrrs
);
349 Wheck whether task has been finished by all APs.
351 @param BlockMode Whether did it in block mode or non-block mode.
353 @retval TRUE Task has been finished by all APs.
354 @retval FALSE Task not has been finished by all APs.
358 WaitForAllAPsNotBusy (
364 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
366 // Ignore BSP and APs which not call in SMM.
368 if (!IsPresentAp(Index
)) {
373 AcquireSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
374 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
376 if (AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
377 ReleaseSpinLock(mSmmMpSyncData
->CpuData
[Index
].Busy
);
388 Check whether it is an present AP.
390 @param CpuIndex The AP index which calls this function.
392 @retval TRUE It's a present AP.
393 @retval TRUE This is not an AP or it is not present.
401 return ((CpuIndex
!= gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) &&
402 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
));
406 Clean up the status flags used during executing the procedure.
408 @param CpuIndex The AP index which calls this function.
416 PROCEDURE_TOKEN
*Token
;
418 Token
= mSmmMpSyncData
->CpuData
[CpuIndex
].Token
;
420 if (InterlockedDecrement (&Token
->RunningApCount
) == 0) {
421 ReleaseSpinLock (Token
->SpinLock
);
424 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= NULL
;
428 Free the tokens in the maintained list.
437 // Reset the FirstFreeToken to the beginning of token list upon exiting SMI.
439 gSmmCpuPrivate
->FirstFreeToken
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
445 @param CpuIndex BSP processor Index
446 @param SyncMode SMM MP sync mode
452 IN SMM_CPU_SYNC_MODE SyncMode
458 BOOLEAN ClearTopLevelSmiResult
;
461 ASSERT (CpuIndex
== mSmmMpSyncData
->BspIndex
);
465 // Flag BSP's presence
467 *mSmmMpSyncData
->InsideSmm
= TRUE
;
470 // Initialize Debug Agent to start source level debug in BSP handler
472 InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI
, NULL
, NULL
);
475 // Mark this processor's presence
477 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
480 // Clear platform top level SMI status bit before calling SMI handlers. If
481 // we cleared it after SMI handlers are run, we would miss the SMI that
482 // occurs after SMI handlers are done and before SMI status bit is cleared.
484 ClearTopLevelSmiResult
= ClearTopLevelSmiStatus();
485 ASSERT (ClearTopLevelSmiResult
== TRUE
);
488 // Set running processor index
490 gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
= CpuIndex
;
493 // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
495 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
498 // Wait for APs to arrive
500 SmmWaitForApArrival();
503 // Lock the counter down and retrieve the number of APs
505 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
506 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
509 // Wait for all APs to get ready for programming MTRRs
511 WaitForAllAPs (ApCount
);
513 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
515 // Signal all APs it's time for backup MTRRs
520 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
521 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
522 // to a large enough value to avoid this situation.
523 // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
524 // We do the backup first and then set MTRR to avoid race condition for threads
527 MtrrGetAllMtrrs(&Mtrrs
);
530 // Wait for all APs to complete their MTRR saving
532 WaitForAllAPs (ApCount
);
535 // Let all processors program SMM MTRRs together
540 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
541 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
542 // to a large enough value to avoid this situation.
544 ReplaceOSMtrrs (CpuIndex
);
547 // Wait for all APs to complete their MTRR programming
549 WaitForAllAPs (ApCount
);
554 // The BUSY lock is initialized to Acquired state
556 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
559 // Perform the pre tasks
564 // Invoke SMM Foundation EntryPoint with the processor information context.
566 gSmmCpuPrivate
->SmmCoreEntry (&gSmmCpuPrivate
->SmmCoreEntryContext
);
569 // Make sure all APs have completed their pending none-block tasks
571 WaitForAllAPsNotBusy (TRUE
);
574 // Perform the remaining tasks
576 PerformRemainingTasks ();
579 // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
580 // make those APs to exit SMI synchronously. APs which arrive later will be excluded and
581 // will run through freely.
583 if (SyncMode
!= SmmCpuSyncModeTradition
&& !SmmCpuFeaturesNeedConfigureMtrrs()) {
586 // Lock the counter down and retrieve the number of APs
588 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
589 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
591 // Make sure all APs have their Present flag set
595 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
596 if (*(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
600 if (PresentCount
> ApCount
) {
607 // Notify all APs to exit
609 *mSmmMpSyncData
->InsideSmm
= FALSE
;
613 // Wait for all APs to complete their pending tasks
615 WaitForAllAPs (ApCount
);
617 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
619 // Signal APs to restore MTRRs
626 SmmCpuFeaturesReenableSmrr ();
627 MtrrSetAllMtrrs(&Mtrrs
);
630 // Wait for all APs to complete MTRR programming
632 WaitForAllAPs (ApCount
);
636 // Stop source level debug in BSP handler, the code below will not be
639 InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI
, NULL
, NULL
);
642 // Signal APs to Reset states/semaphore for this processor
647 // Perform pending operations for hot-plug
652 // Clear the Present flag of BSP
654 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
657 // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
658 // WaitForAllAps does not depend on the Present flag.
660 WaitForAllAPs (ApCount
);
663 // Reset the tokens buffer.
668 // Reset BspIndex to -1, meaning BSP has not been elected.
670 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
671 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
675 // Allow APs to check in from this point on
677 *mSmmMpSyncData
->Counter
= 0;
678 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
684 @param CpuIndex AP processor Index.
685 @param ValidSmi Indicates that current SMI is a valid SMI or not.
686 @param SyncMode SMM MP sync mode.
693 IN SMM_CPU_SYNC_MODE SyncMode
699 EFI_STATUS ProcedureStatus
;
704 for (Timer
= StartSyncTimer ();
705 !IsSyncTimerTimeout (Timer
) &&
706 !(*mSmmMpSyncData
->InsideSmm
);
711 if (!(*mSmmMpSyncData
->InsideSmm
)) {
713 // BSP timeout in the first round
715 if (mSmmMpSyncData
->BspIndex
!= -1) {
717 // BSP Index is known
719 BspIndex
= mSmmMpSyncData
->BspIndex
;
720 ASSERT (CpuIndex
!= BspIndex
);
723 // Send SMI IPI to bring BSP in
725 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[BspIndex
].ProcessorId
);
728 // Now clock BSP for the 2nd time
730 for (Timer
= StartSyncTimer ();
731 !IsSyncTimerTimeout (Timer
) &&
732 !(*mSmmMpSyncData
->InsideSmm
);
737 if (!(*mSmmMpSyncData
->InsideSmm
)) {
739 // Give up since BSP is unable to enter SMM
740 // and signal the completion of this AP
741 WaitForSemaphore (mSmmMpSyncData
->Counter
);
746 // Don't know BSP index. Give up without sending IPI to BSP.
748 WaitForSemaphore (mSmmMpSyncData
->Counter
);
756 BspIndex
= mSmmMpSyncData
->BspIndex
;
757 ASSERT (CpuIndex
!= BspIndex
);
760 // Mark this processor's presence
762 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
764 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
766 // Notify BSP of arrival at this point
768 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
771 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
773 // Wait for the signal from BSP to backup MTRRs
775 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
780 MtrrGetAllMtrrs(&Mtrrs
);
783 // Signal BSP the completion of this AP
785 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
788 // Wait for BSP's signal to program MTRRs
790 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
793 // Replace OS MTRRs with SMI MTRRs
795 ReplaceOSMtrrs (CpuIndex
);
798 // Signal BSP the completion of this AP
800 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
805 // Wait for something to happen
807 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
810 // Check if BSP wants to exit SMM
812 if (!(*mSmmMpSyncData
->InsideSmm
)) {
817 // BUSY should be acquired by SmmStartupThisAp()
820 !AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)
824 // Invoke the scheduled procedure
826 ProcedureStatus
= (*mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
) (
827 (VOID
*)mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
829 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
830 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= ProcedureStatus
;
833 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Token
!= NULL
) {
834 ReleaseToken (CpuIndex
);
840 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
843 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
845 // Notify BSP the readiness of this AP to program MTRRs
847 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
850 // Wait for the signal from BSP to program MTRRs
852 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
857 SmmCpuFeaturesReenableSmrr ();
858 MtrrSetAllMtrrs(&Mtrrs
);
862 // Notify BSP the readiness of this AP to Reset states/semaphore for this processor
864 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
867 // Wait for the signal from BSP to Reset states/semaphore for this processor
869 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
872 // Reset states/semaphore for this processor
874 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
877 // Notify BSP the readiness of this AP to exit SMM
879 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
884 Create 4G PageTable in SMRAM.
886 @param[in] Is32BitPageTable Whether the page table is 32-bit PAE
887 @return PageTable Address
892 IN BOOLEAN Is32BitPageTable
900 UINTN High2MBoundary
;
910 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
912 // Add one more page for known good stack, then find the lower 2MB aligned address.
914 Low2MBoundary
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
) & ~(SIZE_2MB
-1);
916 // Add two more pages for known good stack and stack guard page,
917 // then find the lower 2MB aligned address.
919 High2MBoundary
= (mSmmStackArrayEnd
- mSmmStackSize
+ EFI_PAGE_SIZE
* 2) & ~(SIZE_2MB
-1);
920 PagesNeeded
= ((High2MBoundary
- Low2MBoundary
) / SIZE_2MB
) + 1;
923 // Allocate the page table
925 PageTable
= AllocatePageTableMemory (5 + PagesNeeded
);
926 ASSERT (PageTable
!= NULL
);
928 PageTable
= (VOID
*)((UINTN
)PageTable
);
929 Pte
= (UINT64
*)PageTable
;
932 // Zero out all page table entries first
934 ZeroMem (Pte
, EFI_PAGES_TO_SIZE (1));
937 // Set Page Directory Pointers
939 for (Index
= 0; Index
< 4; Index
++) {
940 Pte
[Index
] = ((UINTN
)PageTable
+ EFI_PAGE_SIZE
* (Index
+ 1)) | mAddressEncMask
|
941 (Is32BitPageTable
? IA32_PAE_PDPTE_ATTRIBUTE_BITS
: PAGE_ATTRIBUTE_BITS
);
943 Pte
+= EFI_PAGE_SIZE
/ sizeof (*Pte
);
946 // Fill in Page Directory Entries
948 for (Index
= 0; Index
< EFI_PAGE_SIZE
* 4 / sizeof (*Pte
); Index
++) {
949 Pte
[Index
] = (Index
<< 21) | mAddressEncMask
| IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
952 Pdpte
= (UINT64
*)PageTable
;
953 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
954 Pages
= (UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (5);
955 GuardPage
= mSmmStackArrayBase
+ EFI_PAGE_SIZE
;
956 for (PageIndex
= Low2MBoundary
; PageIndex
<= High2MBoundary
; PageIndex
+= SIZE_2MB
) {
957 Pte
= (UINT64
*)(UINTN
)(Pdpte
[BitFieldRead32 ((UINT32
)PageIndex
, 30, 31)] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
958 Pte
[BitFieldRead32 ((UINT32
)PageIndex
, 21, 29)] = (UINT64
)Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
960 // Fill in Page Table Entries
962 Pte
= (UINT64
*)Pages
;
963 PageAddress
= PageIndex
;
964 for (Index
= 0; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
965 if (PageAddress
== GuardPage
) {
967 // Mark the guard page as non-present
969 Pte
[Index
] = PageAddress
| mAddressEncMask
;
970 GuardPage
+= mSmmStackSize
;
971 if (GuardPage
> mSmmStackArrayEnd
) {
975 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
977 PageAddress
+= EFI_PAGE_SIZE
;
979 Pages
+= EFI_PAGE_SIZE
;
983 if ((PcdGet8 (PcdNullPointerDetectionPropertyMask
) & BIT1
) != 0) {
984 Pte
= (UINT64
*)(UINTN
)(Pdpte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
985 if ((Pte
[0] & IA32_PG_PS
) == 0) {
986 // 4K-page entries are already mapped. Just hide the first one anyway.
987 Pte
= (UINT64
*)(UINTN
)(Pte
[0] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
988 Pte
[0] &= ~(UINT64
)IA32_PG_P
; // Hide page 0
990 // Create 4K-page entries
991 Pages
= (UINTN
)AllocatePageTableMemory (1);
994 Pte
[0] = (UINT64
)(Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
);
996 Pte
= (UINT64
*)Pages
;
998 Pte
[0] = PageAddress
| mAddressEncMask
; // Hide page 0 but present left
999 for (Index
= 1; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
1000 PageAddress
+= EFI_PAGE_SIZE
;
1001 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
1006 return (UINT32
)(UINTN
)PageTable
;
1010 Checks whether the input token is the current used token.
1012 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1015 @retval TRUE The input token is the current used token.
1016 @retval FALSE The input token is not the current used token.
1024 PROCEDURE_TOKEN
*ProcToken
;
1026 if (Token
== NULL
) {
1030 Link
= GetFirstNode (&gSmmCpuPrivate
->TokenList
);
1032 // Only search used tokens.
1034 while (Link
!= gSmmCpuPrivate
->FirstFreeToken
) {
1035 ProcToken
= PROCEDURE_TOKEN_FROM_LINK (Link
);
1037 if (ProcToken
->SpinLock
== Token
) {
1041 Link
= GetNextNode (&gSmmCpuPrivate
->TokenList
, Link
);
1048 Allocate buffer for the SPIN_LOCK and PROCEDURE_TOKEN.
1050 @return First token of the token buffer.
1053 AllocateTokenBuffer (
1058 UINT32 TokenCountPerChunk
;
1060 SPIN_LOCK
*SpinLock
;
1061 UINT8
*SpinLockBuffer
;
1062 PROCEDURE_TOKEN
*ProcTokens
;
1064 SpinLockSize
= GetSpinLockProperties ();
1066 TokenCountPerChunk
= FixedPcdGet32 (PcdCpuSmmMpTokenCountPerChunk
);
1067 ASSERT (TokenCountPerChunk
!= 0);
1068 if (TokenCountPerChunk
== 0) {
1069 DEBUG ((DEBUG_ERROR
, "PcdCpuSmmMpTokenCountPerChunk should not be Zero!\n"));
1072 DEBUG ((DEBUG_INFO
, "CpuSmm: SpinLock Size = 0x%x, PcdCpuSmmMpTokenCountPerChunk = 0x%x\n", SpinLockSize
, TokenCountPerChunk
));
1075 // Separate the Spin_lock and Proc_token because the alignment requires by Spin_Lock.
1077 SpinLockBuffer
= AllocatePool (SpinLockSize
* TokenCountPerChunk
);
1078 ASSERT (SpinLockBuffer
!= NULL
);
1080 ProcTokens
= AllocatePool (sizeof (PROCEDURE_TOKEN
) * TokenCountPerChunk
);
1081 ASSERT (ProcTokens
!= NULL
);
1083 for (Index
= 0; Index
< TokenCountPerChunk
; Index
++) {
1084 SpinLock
= (SPIN_LOCK
*)(SpinLockBuffer
+ SpinLockSize
* Index
);
1085 InitializeSpinLock (SpinLock
);
1087 ProcTokens
[Index
].Signature
= PROCEDURE_TOKEN_SIGNATURE
;
1088 ProcTokens
[Index
].SpinLock
= SpinLock
;
1089 ProcTokens
[Index
].RunningApCount
= 0;
1091 InsertTailList (&gSmmCpuPrivate
->TokenList
, &ProcTokens
[Index
].Link
);
1094 return &ProcTokens
[0].Link
;
1100 If no free token, allocate new tokens then return the free one.
1102 @param RunningApsCount The Running Aps count for this token.
1104 @retval return the first free PROCEDURE_TOKEN.
1109 IN UINT32 RunningApsCount
1112 PROCEDURE_TOKEN
*NewToken
;
1115 // If FirstFreeToken meets the end of token list, enlarge the token list.
1116 // Set FirstFreeToken to the first free token.
1118 if (gSmmCpuPrivate
->FirstFreeToken
== &gSmmCpuPrivate
->TokenList
) {
1119 gSmmCpuPrivate
->FirstFreeToken
= AllocateTokenBuffer ();
1121 NewToken
= PROCEDURE_TOKEN_FROM_LINK (gSmmCpuPrivate
->FirstFreeToken
);
1122 gSmmCpuPrivate
->FirstFreeToken
= GetNextNode (&gSmmCpuPrivate
->TokenList
, gSmmCpuPrivate
->FirstFreeToken
);
1124 NewToken
->RunningApCount
= RunningApsCount
;
1125 AcquireSpinLock (NewToken
->SpinLock
);
1131 Checks status of specified AP.
1133 This function checks whether the specified AP has finished the task assigned
1134 by StartupThisAP(), and whether timeout expires.
1136 @param[in] Token This parameter describes the token that was passed into DispatchProcedure or
1139 @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
1140 @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
1147 if (AcquireSpinLockOrFail (Token
)) {
1148 ReleaseSpinLock (Token
);
1152 return EFI_NOT_READY
;
1156 Schedule a procedure to run on the specified CPU.
1158 @param[in] Procedure The address of the procedure to run
1159 @param[in] CpuIndex Target CPU Index
1160 @param[in,out] ProcArguments The parameter to pass to the procedure
1161 @param[in] Token This is an optional parameter that allows the caller to execute the
1162 procedure in a blocking or non-blocking fashion. If it is NULL the
1163 call is blocking, and the call will not return until the AP has
1164 completed the procedure. If the token is not NULL, the call will
1165 return immediately. The caller can check whether the procedure has
1166 completed with CheckOnProcedure or WaitForProcedure.
1167 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for the APs to finish
1168 execution of Procedure, either for blocking or non-blocking mode.
1169 Zero means infinity. If the timeout expires before all APs return
1170 from Procedure, then Procedure on the failed APs is terminated. If
1171 the timeout expires in blocking mode, the call returns EFI_TIMEOUT.
1172 If the timeout expires in non-blocking mode, the timeout determined
1173 can be through CheckOnProcedure or WaitForProcedure.
1174 Note that timeout support is optional. Whether an implementation
1175 supports this feature can be determined via the Attributes data
1177 @param[in,out] CpuStatus This optional pointer may be used to get the status code returned
1178 by Procedure when it completes execution on the target AP, or with
1179 EFI_TIMEOUT if the Procedure fails to complete within the optional
1180 timeout. The implementation will update this variable with
1181 EFI_NOT_READY prior to starting Procedure on the target AP.
1183 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1184 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1185 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1186 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1187 @retval EFI_SUCCESS The procedure has been successfully scheduled
1191 InternalSmmStartupThisAp (
1192 IN EFI_AP_PROCEDURE2 Procedure
,
1194 IN OUT VOID
*ProcArguments OPTIONAL
,
1195 IN MM_COMPLETION
*Token
,
1196 IN UINTN TimeoutInMicroseconds
,
1197 IN OUT EFI_STATUS
*CpuStatus
1200 PROCEDURE_TOKEN
*ProcToken
;
1202 if (CpuIndex
>= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
) {
1203 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus(%d)\n", CpuIndex
, gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
));
1204 return EFI_INVALID_PARAMETER
;
1206 if (CpuIndex
== gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) {
1207 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu\n", CpuIndex
));
1208 return EFI_INVALID_PARAMETER
;
1210 if (gSmmCpuPrivate
->ProcessorInfo
[CpuIndex
].ProcessorId
== INVALID_APIC_ID
) {
1211 return EFI_INVALID_PARAMETER
;
1213 if (!(*(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
))) {
1214 if (mSmmMpSyncData
->EffectiveSyncMode
== SmmCpuSyncModeTradition
) {
1215 DEBUG((DEBUG_ERROR
, "!mSmmMpSyncData->CpuData[%d].Present\n", CpuIndex
));
1217 return EFI_INVALID_PARAMETER
;
1219 if (gSmmCpuPrivate
->Operation
[CpuIndex
] == SmmCpuRemove
) {
1220 if (!FeaturePcdGet (PcdCpuHotPlugSupport
)) {
1221 DEBUG((DEBUG_ERROR
, "gSmmCpuPrivate->Operation[%d] == SmmCpuRemove\n", CpuIndex
));
1223 return EFI_INVALID_PARAMETER
;
1225 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1226 return EFI_INVALID_PARAMETER
;
1228 if (Procedure
== NULL
) {
1229 return EFI_INVALID_PARAMETER
;
1232 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1234 mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
= Procedure
;
1235 mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
= ProcArguments
;
1236 if (Token
!= NULL
) {
1237 ProcToken
= GetFreeToken (1);
1238 mSmmMpSyncData
->CpuData
[CpuIndex
].Token
= ProcToken
;
1239 *Token
= (MM_COMPLETION
)ProcToken
->SpinLock
;
1241 mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= CpuStatus
;
1242 if (mSmmMpSyncData
->CpuData
[CpuIndex
].Status
!= NULL
) {
1243 *mSmmMpSyncData
->CpuData
[CpuIndex
].Status
= EFI_NOT_READY
;
1246 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
1248 if (Token
== NULL
) {
1249 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1250 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1257 Worker function to execute a caller provided function on all enabled APs.
1259 @param[in] Procedure A pointer to the function to be run on
1260 enabled APs of the system.
1261 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
1262 APs to return from Procedure, either for
1263 blocking or non-blocking mode.
1264 @param[in,out] ProcedureArguments The parameter passed into Procedure for
1266 @param[in,out] Token This is an optional parameter that allows the caller to execute the
1267 procedure in a blocking or non-blocking fashion. If it is NULL the
1268 call is blocking, and the call will not return until the AP has
1269 completed the procedure. If the token is not NULL, the call will
1270 return immediately. The caller can check whether the procedure has
1271 completed with CheckOnProcedure or WaitForProcedure.
1272 @param[in,out] CPUStatus This optional pointer may be used to get the status code returned
1273 by Procedure when it completes execution on the target AP, or with
1274 EFI_TIMEOUT if the Procedure fails to complete within the optional
1275 timeout. The implementation will update this variable with
1276 EFI_NOT_READY prior to starting Procedure on the target AP.
1279 @retval EFI_SUCCESS In blocking mode, all APs have finished before
1280 the timeout expired.
1281 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
1283 @retval others Failed to Startup all APs.
1287 InternalSmmStartupAllAPs (
1288 IN EFI_AP_PROCEDURE2 Procedure
,
1289 IN UINTN TimeoutInMicroseconds
,
1290 IN OUT VOID
*ProcedureArguments OPTIONAL
,
1291 IN OUT MM_COMPLETION
*Token
,
1292 IN OUT EFI_STATUS
*CPUStatus
1297 PROCEDURE_TOKEN
*ProcToken
;
1299 if ((TimeoutInMicroseconds
!= 0) && ((mSmmMp
.Attributes
& EFI_MM_MP_TIMEOUT_SUPPORTED
) == 0)) {
1300 return EFI_INVALID_PARAMETER
;
1302 if (Procedure
== NULL
) {
1303 return EFI_INVALID_PARAMETER
;
1307 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1308 if (IsPresentAp (Index
)) {
1311 if (gSmmCpuPrivate
->Operation
[Index
] == SmmCpuRemove
) {
1312 return EFI_INVALID_PARAMETER
;
1315 if (!AcquireSpinLockOrFail(mSmmMpSyncData
->CpuData
[Index
].Busy
)) {
1316 return EFI_NOT_READY
;
1318 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1321 if (CpuCount
== 0) {
1322 return EFI_NOT_STARTED
;
1325 if (Token
!= NULL
) {
1326 ProcToken
= GetFreeToken ((UINT32
)mMaxNumberOfCpus
);
1327 *Token
= (MM_COMPLETION
)ProcToken
->SpinLock
;
1333 // Make sure all BUSY should be acquired.
1335 // Because former code already check mSmmMpSyncData->CpuData[***].Busy for each AP.
1336 // Here code always use AcquireSpinLock instead of AcquireSpinLockOrFail for not
1339 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1340 if (IsPresentAp (Index
)) {
1341 AcquireSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
1345 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1346 if (IsPresentAp (Index
)) {
1347 mSmmMpSyncData
->CpuData
[Index
].Procedure
= (EFI_AP_PROCEDURE2
) Procedure
;
1348 mSmmMpSyncData
->CpuData
[Index
].Parameter
= ProcedureArguments
;
1349 if (ProcToken
!= NULL
) {
1350 mSmmMpSyncData
->CpuData
[Index
].Token
= ProcToken
;
1352 if (CPUStatus
!= NULL
) {
1353 mSmmMpSyncData
->CpuData
[Index
].Status
= &CPUStatus
[Index
];
1354 if (mSmmMpSyncData
->CpuData
[Index
].Status
!= NULL
) {
1355 *mSmmMpSyncData
->CpuData
[Index
].Status
= EFI_NOT_READY
;
1360 // PI spec requirement:
1361 // For every excluded processor, the array entry must contain a value of EFI_NOT_STARTED.
1363 if (CPUStatus
!= NULL
) {
1364 CPUStatus
[Index
] = EFI_NOT_STARTED
;
1368 // Decrease the count to mark this processor(AP or BSP) as finished.
1370 if (ProcToken
!= NULL
) {
1371 WaitForSemaphore (&ProcToken
->RunningApCount
);
1378 if (Token
== NULL
) {
1380 // Make sure all APs have completed their tasks.
1382 WaitForAllAPsNotBusy (TRUE
);
1389 ISO C99 6.5.2.2 "Function calls", paragraph 9:
1390 If the function is defined with a type that is not compatible with
1391 the type (of the expression) pointed to by the expression that
1392 denotes the called function, the behavior is undefined.
1394 So add below wrapper function to convert between EFI_AP_PROCEDURE
1395 and EFI_AP_PROCEDURE2.
1397 Wrapper for Procedures.
1399 @param[in] Buffer Pointer to PROCEDURE_WRAPPER buffer.
1408 PROCEDURE_WRAPPER
*Wrapper
;
1411 Wrapper
->Procedure (Wrapper
->ProcedureArgument
);
1417 Schedule a procedure to run on the specified CPU in blocking mode.
1419 @param[in] Procedure The address of the procedure to run
1420 @param[in] CpuIndex Target CPU Index
1421 @param[in, out] ProcArguments The parameter to pass to the procedure
1423 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1424 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1425 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1426 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1427 @retval EFI_SUCCESS The procedure has been successfully scheduled
1432 SmmBlockingStartupThisAp (
1433 IN EFI_AP_PROCEDURE Procedure
,
1435 IN OUT VOID
*ProcArguments OPTIONAL
1438 PROCEDURE_WRAPPER Wrapper
;
1440 Wrapper
.Procedure
= Procedure
;
1441 Wrapper
.ProcedureArgument
= ProcArguments
;
1444 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1446 return InternalSmmStartupThisAp (ProcedureWrapper
, CpuIndex
, &Wrapper
, NULL
, 0, NULL
);
1450 Schedule a procedure to run on the specified CPU.
1452 @param Procedure The address of the procedure to run
1453 @param CpuIndex Target CPU Index
1454 @param ProcArguments The parameter to pass to the procedure
1456 @retval EFI_INVALID_PARAMETER CpuNumber not valid
1457 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
1458 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
1459 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
1460 @retval EFI_SUCCESS The procedure has been successfully scheduled
1466 IN EFI_AP_PROCEDURE Procedure
,
1468 IN OUT VOID
*ProcArguments OPTIONAL
1471 MM_COMPLETION Token
;
1473 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].Procedure
= Procedure
;
1474 gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
].ProcedureArgument
= ProcArguments
;
1477 // Use wrapper function to convert EFI_AP_PROCEDURE to EFI_AP_PROCEDURE2.
1479 return InternalSmmStartupThisAp (
1482 &gSmmCpuPrivate
->ApWrapperFunc
[CpuIndex
],
1483 FeaturePcdGet (PcdCpuSmmBlockStartupThisAp
) ? NULL
: &Token
,
1490 This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
1491 They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
1493 NOTE: It might not be appreciated in runtime since it might
1494 conflict with OS debugging facilities. Turn them off in RELEASE.
1496 @param CpuIndex CPU Index
1505 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1507 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1508 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1509 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1510 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1511 AsmWriteDr6 (CpuSaveState
->x86
._DR6
);
1512 AsmWriteDr7 (CpuSaveState
->x86
._DR7
);
1514 AsmWriteDr6 ((UINTN
)CpuSaveState
->x64
._DR6
);
1515 AsmWriteDr7 ((UINTN
)CpuSaveState
->x64
._DR7
);
1521 This function restores DR6 & DR7 to SMM save state.
1523 NOTE: It might not be appreciated in runtime since it might
1524 conflict with OS debugging facilities. Turn them off in RELEASE.
1526 @param CpuIndex CPU Index
1535 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1537 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1538 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1539 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1540 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1541 CpuSaveState
->x86
._DR7
= (UINT32
)AsmReadDr7 ();
1542 CpuSaveState
->x86
._DR6
= (UINT32
)AsmReadDr6 ();
1544 CpuSaveState
->x64
._DR7
= AsmReadDr7 ();
1545 CpuSaveState
->x64
._DR6
= AsmReadDr6 ();
1551 C function for SMI entry, each processor comes here upon SMI trigger.
1553 @param CpuIndex CPU Index
1565 BOOLEAN BspInProgress
;
1569 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1572 // Save Cr2 because Page Fault exception in SMM may override its value,
1573 // when using on-demand paging for above 4G memory.
1579 // Call the user register Startup function first.
1581 if (mSmmMpSyncData
->StartupProcedure
!= NULL
) {
1582 mSmmMpSyncData
->StartupProcedure (mSmmMpSyncData
->StartupProcArgs
);
1586 // Perform CPU specific entry hooks
1588 SmmCpuFeaturesRendezvousEntry (CpuIndex
);
1591 // Determine if this is a valid SMI
1593 ValidSmi
= PlatformValidSmi();
1596 // Determine if BSP has been already in progress. Note this must be checked after
1597 // ValidSmi because BSP may clear a valid SMI source after checking in.
1599 BspInProgress
= *mSmmMpSyncData
->InsideSmm
;
1601 if (!BspInProgress
&& !ValidSmi
) {
1603 // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not
1604 // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI
1605 // status had been cleared by BSP and an existing SMI run has almost ended. (Note
1606 // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there
1607 // is nothing we need to do.
1612 // Signal presence of this processor
1614 if (ReleaseSemaphore (mSmmMpSyncData
->Counter
) == 0) {
1616 // BSP has already ended the synchronization, so QUIT!!!
1620 // Wait for BSP's signal to finish SMI
1622 while (*mSmmMpSyncData
->AllCpusInSync
) {
1629 // The BUSY lock is initialized to Released state.
1630 // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.
1631 // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
1632 // after AP's present flag is detected.
1634 InitializeSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1637 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1638 ActivateSmmProfile (CpuIndex
);
1641 if (BspInProgress
) {
1643 // BSP has been elected. Follow AP path, regardless of ValidSmi flag
1644 // as BSP may have cleared the SMI status
1646 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1649 // We have a valid SMI
1656 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1657 if (!mSmmMpSyncData
->SwitchBsp
|| mSmmMpSyncData
->CandidateBsp
[CpuIndex
]) {
1659 // Call platform hook to do BSP election
1661 Status
= PlatformSmmBspElection (&IsBsp
);
1662 if (EFI_SUCCESS
== Status
) {
1664 // Platform hook determines successfully
1667 mSmmMpSyncData
->BspIndex
= (UINT32
)CpuIndex
;
1671 // Platform hook fails to determine, use default BSP election method
1673 InterlockedCompareExchange32 (
1674 (UINT32
*)&mSmmMpSyncData
->BspIndex
,
1683 // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP
1685 if (mSmmMpSyncData
->BspIndex
== CpuIndex
) {
1688 // Clear last request for SwitchBsp.
1690 if (mSmmMpSyncData
->SwitchBsp
) {
1691 mSmmMpSyncData
->SwitchBsp
= FALSE
;
1692 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1693 mSmmMpSyncData
->CandidateBsp
[Index
] = FALSE
;
1697 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1698 SmmProfileRecordSmiNum ();
1702 // BSP Handler is always called with a ValidSmi == TRUE
1704 BSPHandler (CpuIndex
, mSmmMpSyncData
->EffectiveSyncMode
);
1706 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1710 ASSERT (*mSmmMpSyncData
->CpuData
[CpuIndex
].Run
== 0);
1713 // Wait for BSP's signal to exit SMI
1715 while (*mSmmMpSyncData
->AllCpusInSync
) {
1721 SmmCpuFeaturesRendezvousExit (CpuIndex
);
1730 Allocate buffer for SpinLock and Wrapper function buffer.
1734 InitializeDataForMmMp (
1738 gSmmCpuPrivate
->ApWrapperFunc
= AllocatePool (sizeof (PROCEDURE_WRAPPER
) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1739 ASSERT (gSmmCpuPrivate
->ApWrapperFunc
!= NULL
);
1741 InitializeListHead (&gSmmCpuPrivate
->TokenList
);
1743 gSmmCpuPrivate
->FirstFreeToken
= AllocateTokenBuffer ();
1747 Allocate buffer for all semaphores and spin locks.
1751 InitializeSmmCpuSemaphores (
1755 UINTN ProcessorCount
;
1757 UINTN GlobalSemaphoresSize
;
1758 UINTN CpuSemaphoresSize
;
1759 UINTN SemaphoreSize
;
1761 UINTN
*SemaphoreBlock
;
1762 UINTN SemaphoreAddr
;
1764 SemaphoreSize
= GetSpinLockProperties ();
1765 ProcessorCount
= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1766 GlobalSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_GLOBAL
) / sizeof (VOID
*)) * SemaphoreSize
;
1767 CpuSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_CPU
) / sizeof (VOID
*)) * ProcessorCount
* SemaphoreSize
;
1768 TotalSize
= GlobalSemaphoresSize
+ CpuSemaphoresSize
;
1769 DEBUG((EFI_D_INFO
, "One Semaphore Size = 0x%x\n", SemaphoreSize
));
1770 DEBUG((EFI_D_INFO
, "Total Semaphores Size = 0x%x\n", TotalSize
));
1771 Pages
= EFI_SIZE_TO_PAGES (TotalSize
);
1772 SemaphoreBlock
= AllocatePages (Pages
);
1773 ASSERT (SemaphoreBlock
!= NULL
);
1774 ZeroMem (SemaphoreBlock
, TotalSize
);
1776 SemaphoreAddr
= (UINTN
)SemaphoreBlock
;
1777 mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
= (UINT32
*)SemaphoreAddr
;
1778 SemaphoreAddr
+= SemaphoreSize
;
1779 mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
= (BOOLEAN
*)SemaphoreAddr
;
1780 SemaphoreAddr
+= SemaphoreSize
;
1781 mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
= (BOOLEAN
*)SemaphoreAddr
;
1782 SemaphoreAddr
+= SemaphoreSize
;
1783 mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
= (SPIN_LOCK
*)SemaphoreAddr
;
1784 SemaphoreAddr
+= SemaphoreSize
;
1785 mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
1786 = (SPIN_LOCK
*)SemaphoreAddr
;
1787 SemaphoreAddr
+= SemaphoreSize
;
1789 SemaphoreAddr
= (UINTN
)SemaphoreBlock
+ GlobalSemaphoresSize
;
1790 mSmmCpuSemaphores
.SemaphoreCpu
.Busy
= (SPIN_LOCK
*)SemaphoreAddr
;
1791 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1792 mSmmCpuSemaphores
.SemaphoreCpu
.Run
= (UINT32
*)SemaphoreAddr
;
1793 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1794 mSmmCpuSemaphores
.SemaphoreCpu
.Present
= (BOOLEAN
*)SemaphoreAddr
;
1796 mPFLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
;
1797 mConfigSmmCodeAccessCheckLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
;
1799 mSemaphoreSize
= SemaphoreSize
;
1803 Initialize un-cacheable data.
1808 InitializeMpSyncData (
1814 if (mSmmMpSyncData
!= NULL
) {
1816 // mSmmMpSyncDataSize includes one structure of SMM_DISPATCHER_MP_SYNC_DATA, one
1817 // CpuData array of SMM_CPU_DATA_BLOCK and one CandidateBsp array of BOOLEAN.
1819 ZeroMem (mSmmMpSyncData
, mSmmMpSyncDataSize
);
1820 mSmmMpSyncData
->CpuData
= (SMM_CPU_DATA_BLOCK
*)((UINT8
*)mSmmMpSyncData
+ sizeof (SMM_DISPATCHER_MP_SYNC_DATA
));
1821 mSmmMpSyncData
->CandidateBsp
= (BOOLEAN
*)(mSmmMpSyncData
->CpuData
+ gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1822 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1824 // Enable BSP election by setting BspIndex to -1
1826 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
1828 mSmmMpSyncData
->EffectiveSyncMode
= mCpuSmmSyncMode
;
1830 mSmmMpSyncData
->Counter
= mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
;
1831 mSmmMpSyncData
->InsideSmm
= mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
;
1832 mSmmMpSyncData
->AllCpusInSync
= mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
;
1833 ASSERT (mSmmMpSyncData
->Counter
!= NULL
&& mSmmMpSyncData
->InsideSmm
!= NULL
&&
1834 mSmmMpSyncData
->AllCpusInSync
!= NULL
);
1835 *mSmmMpSyncData
->Counter
= 0;
1836 *mSmmMpSyncData
->InsideSmm
= FALSE
;
1837 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
1839 for (CpuIndex
= 0; CpuIndex
< gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
; CpuIndex
++) {
1840 mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
=
1841 (SPIN_LOCK
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Busy
+ mSemaphoreSize
* CpuIndex
);
1842 mSmmMpSyncData
->CpuData
[CpuIndex
].Run
=
1843 (UINT32
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Run
+ mSemaphoreSize
* CpuIndex
);
1844 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
=
1845 (BOOLEAN
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Present
+ mSemaphoreSize
* CpuIndex
);
1846 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
) = 0;
1847 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Run
) = 0;
1848 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
1854 Initialize global data for MP synchronization.
1856 @param Stacks Base address of SMI stack buffer for all processors.
1857 @param StackSize Stack size for each processor in SMM.
1858 @param ShadowStackSize Shadow Stack size for each processor in SMM.
1862 InitializeMpServiceData (
1865 IN UINTN ShadowStackSize
1870 UINT8
*GdtTssTables
;
1871 UINTN GdtTableStepSize
;
1872 CPUID_VERSION_INFO_EDX RegEdx
;
1875 // Determine if this CPU supports machine check
1877 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
1878 mMachineCheckSupported
= (BOOLEAN
)(RegEdx
.Bits
.MCA
== 1);
1881 // Allocate memory for all locks and semaphores
1883 InitializeSmmCpuSemaphores ();
1886 // Initialize mSmmMpSyncData
1888 mSmmMpSyncDataSize
= sizeof (SMM_DISPATCHER_MP_SYNC_DATA
) +
1889 (sizeof (SMM_CPU_DATA_BLOCK
) + sizeof (BOOLEAN
)) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1890 mSmmMpSyncData
= (SMM_DISPATCHER_MP_SYNC_DATA
*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize
));
1891 ASSERT (mSmmMpSyncData
!= NULL
);
1892 mCpuSmmSyncMode
= (SMM_CPU_SYNC_MODE
)PcdGet8 (PcdCpuSmmSyncMode
);
1893 InitializeMpSyncData ();
1896 // Initialize physical address mask
1897 // NOTE: Physical memory above virtual address limit is not supported !!!
1899 AsmCpuid (0x80000008, (UINT32
*)&Index
, NULL
, NULL
, NULL
);
1900 gPhyMask
= LShiftU64 (1, (UINT8
)Index
) - 1;
1901 gPhyMask
&= (1ull << 48) - EFI_PAGE_SIZE
;
1904 // Create page tables
1906 Cr3
= SmmInitPageTable ();
1908 GdtTssTables
= InitGdt (Cr3
, &GdtTableStepSize
);
1911 // Install SMI handler for each CPU
1913 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1916 (UINT32
)mCpuHotPlugData
.SmBase
[Index
],
1917 (VOID
*)((UINTN
)Stacks
+ (StackSize
+ ShadowStackSize
) * Index
),
1919 (UINTN
)(GdtTssTables
+ GdtTableStepSize
* Index
),
1920 gcSmiGdtr
.Limit
+ 1,
1922 gcSmiIdtr
.Limit
+ 1,
1928 // Record current MTRR settings
1930 ZeroMem (&gSmiMtrrs
, sizeof (gSmiMtrrs
));
1931 MtrrGetAllMtrrs (&gSmiMtrrs
);
1938 Register the SMM Foundation entry point.
1940 @param This Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance
1941 @param SmmEntryPoint SMM Foundation EntryPoint
1943 @retval EFI_SUCCESS Successfully to register SMM foundation entry point
1949 IN CONST EFI_SMM_CONFIGURATION_PROTOCOL
*This
,
1950 IN EFI_SMM_ENTRY_POINT SmmEntryPoint
1954 // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.
1956 gSmmCpuPrivate
->SmmCoreEntry
= SmmEntryPoint
;
1962 Register the SMM Foundation entry point.
1964 @param[in] Procedure A pointer to the code stream to be run on the designated target AP
1965 of the system. Type EFI_AP_PROCEDURE is defined below in Volume 2
1966 with the related definitions of
1967 EFI_MP_SERVICES_PROTOCOL.StartupAllAPs.
1968 If caller may pass a value of NULL to deregister any existing
1970 @param[in,out] ProcedureArguments Allows the caller to pass a list of parameters to the code that is
1971 run by the AP. It is an optional common mailbox between APs and
1972 the caller to share information
1974 @retval EFI_SUCCESS The Procedure has been set successfully.
1975 @retval EFI_INVALID_PARAMETER The Procedure is NULL but ProcedureArguments not NULL.
1979 RegisterStartupProcedure (
1980 IN EFI_AP_PROCEDURE Procedure
,
1981 IN OUT VOID
*ProcedureArguments OPTIONAL
1984 if (Procedure
== NULL
&& ProcedureArguments
!= NULL
) {
1985 return EFI_INVALID_PARAMETER
;
1987 if (mSmmMpSyncData
== NULL
) {
1988 return EFI_NOT_READY
;
1991 mSmmMpSyncData
->StartupProcedure
= Procedure
;
1992 mSmmMpSyncData
->StartupProcArgs
= ProcedureArguments
;