2 SMM MP service implementation
4 Copyright (c) 2009 - 2017, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
7 This program and the accompanying materials
8 are licensed and made available under the terms and conditions of the BSD License
9 which accompanies this distribution. The full text of the license may be found at
10 http://opensource.org/licenses/bsd-license.php
12 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
13 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
17 #include "PiSmmCpuDxeSmm.h"
20 // Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
22 MTRR_SETTINGS gSmiMtrrs
;
24 SMM_DISPATCHER_MP_SYNC_DATA
*mSmmMpSyncData
= NULL
;
25 UINTN mSmmMpSyncDataSize
;
26 SMM_CPU_SEMAPHORES mSmmCpuSemaphores
;
28 SPIN_LOCK
*mPFLock
= NULL
;
29 SMM_CPU_SYNC_MODE mCpuSmmSyncMode
;
30 BOOLEAN mMachineCheckSupported
= FALSE
;
33 Performs an atomic compare exchange operation to get semaphore.
34 The compare exchange operation must be performed using
37 @param Sem IN: 32-bit unsigned integer
38 OUT: original integer - 1
39 @return Original integer - 1
44 IN OUT
volatile UINT32
*Sem
51 } while (Value
== 0 ||
52 InterlockedCompareExchange32 (
62 Performs an atomic compare exchange operation to release semaphore.
63 The compare exchange operation must be performed using
66 @param Sem IN: 32-bit unsigned integer
67 OUT: original integer + 1
68 @return Original integer + 1
73 IN OUT
volatile UINT32
*Sem
80 } while (Value
+ 1 != 0 &&
81 InterlockedCompareExchange32 (
90 Performs an atomic compare exchange operation to lock semaphore.
91 The compare exchange operation must be performed using
94 @param Sem IN: 32-bit unsigned integer
96 @return Original integer
101 IN OUT
volatile UINT32
*Sem
108 } while (InterlockedCompareExchange32 (
116 Wait all APs to performs an atomic compare exchange operation to release semaphore.
118 @param NumberOfAPs AP number
128 BspIndex
= mSmmMpSyncData
->BspIndex
;
129 while (NumberOfAPs
-- > 0) {
130 WaitForSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
135 Performs an atomic compare exchange operation to release semaphore
147 BspIndex
= mSmmMpSyncData
->BspIndex
;
148 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
149 if (Index
!= BspIndex
&& *(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
150 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[Index
].Run
);
156 Checks if all CPUs (with certain exceptions) have checked in for this SMI run
158 @param Exceptions CPU Arrival exception flags.
160 @retval TRUE if all CPUs the have checked in.
161 @retval FALSE if at least one Normal AP hasn't checked in.
165 AllCpusInSmmWithExceptions (
166 SMM_CPU_ARRIVAL_EXCEPTIONS Exceptions
170 SMM_CPU_DATA_BLOCK
*CpuData
;
171 EFI_PROCESSOR_INFORMATION
*ProcessorInfo
;
173 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
175 if (*mSmmMpSyncData
->Counter
== mNumberOfCpus
) {
179 CpuData
= mSmmMpSyncData
->CpuData
;
180 ProcessorInfo
= gSmmCpuPrivate
->ProcessorInfo
;
181 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
182 if (!(*(CpuData
[Index
].Present
)) && ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
183 if (((Exceptions
& ARRIVAL_EXCEPTION_DELAYED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmDelayed
) != 0) {
186 if (((Exceptions
& ARRIVAL_EXCEPTION_BLOCKED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmBlocked
) != 0) {
189 if (((Exceptions
& ARRIVAL_EXCEPTION_SMI_DISABLED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmEnable
) != 0) {
201 Has OS enabled Lmce in the MSR_IA32_MCG_EXT_CTL
203 @retval TRUE Os enable lmce.
204 @retval FALSE Os not enable lmce.
212 MSR_IA32_MCG_CAP_REGISTER McgCap
;
213 MSR_IA32_FEATURE_CONTROL_REGISTER FeatureCtrl
;
214 MSR_IA32_MCG_EXT_CTL_REGISTER McgExtCtrl
;
216 McgCap
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_CAP
);
217 if (McgCap
.Bits
.MCG_LMCE_P
== 0) {
221 FeatureCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_FEATURE_CONTROL
);
222 if (FeatureCtrl
.Bits
.LmceOn
== 0) {
226 McgExtCtrl
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_EXT_CTL
);
227 return (BOOLEAN
) (McgExtCtrl
.Bits
.LMCE_EN
== 1);
231 Return if Local machine check exception signaled.
233 Indicates (when set) that a local machine check exception was generated. This indicates that the current machine-check event was
234 delivered to only the logical processor.
236 @retval TRUE LMCE was signaled.
237 @retval FALSE LMCE was not signaled.
245 MSR_IA32_MCG_STATUS_REGISTER McgStatus
;
247 McgStatus
.Uint64
= AsmReadMsr64 (MSR_IA32_MCG_STATUS
);
248 return (BOOLEAN
) (McgStatus
.Bits
.LMCE_S
== 1);
252 Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
253 entering SMM, except SMI disabled APs.
257 SmmWaitForApArrival (
266 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
270 if (mMachineCheckSupported
) {
271 LmceEn
= IsLmceOsEnabled ();
272 LmceSignal
= IsLmceSignaled();
276 // Platform implementor should choose a timeout value appropriately:
277 // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note
278 // the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.
279 // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI
280 // and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will
281 // be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the
282 // SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.
283 // - The timeout value must be longer than longest possible IO operation in the system
287 // Sync with APs 1st timeout
289 for (Timer
= StartSyncTimer ();
290 !IsSyncTimerTimeout (Timer
) && !(LmceEn
&& LmceSignal
) &&
291 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
297 // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,
299 // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running
300 // normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they
301 // enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode
302 // work while SMI handling is on-going.
303 // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.
304 // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state
305 // will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal
306 // mode work while SMI handling is on-going.
307 // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:
308 // - In traditional flow, SMI disabling is discouraged.
309 // - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
310 // In both cases, adding SMI-disabling checking code increases overhead.
312 if (*mSmmMpSyncData
->Counter
< mNumberOfCpus
) {
314 // Send SMI IPIs to bring outside processors in
316 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
317 if (!(*(mSmmMpSyncData
->CpuData
[Index
].Present
)) && gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
318 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
);
323 // Sync with APs 2nd timeout.
325 for (Timer
= StartSyncTimer ();
326 !IsSyncTimerTimeout (Timer
) &&
327 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
338 Replace OS MTRR's with SMI MTRR's.
340 @param CpuIndex Processor Index
348 SmmCpuFeaturesDisableSmrr ();
351 // Replace all MTRRs registers
353 MtrrSetAllMtrrs (&gSmiMtrrs
);
359 @param CpuIndex BSP processor Index
360 @param SyncMode SMM MP sync mode
366 IN SMM_CPU_SYNC_MODE SyncMode
372 BOOLEAN ClearTopLevelSmiResult
;
375 ASSERT (CpuIndex
== mSmmMpSyncData
->BspIndex
);
379 // Flag BSP's presence
381 *mSmmMpSyncData
->InsideSmm
= TRUE
;
384 // Initialize Debug Agent to start source level debug in BSP handler
386 InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI
, NULL
, NULL
);
389 // Mark this processor's presence
391 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
394 // Clear platform top level SMI status bit before calling SMI handlers. If
395 // we cleared it after SMI handlers are run, we would miss the SMI that
396 // occurs after SMI handlers are done and before SMI status bit is cleared.
398 ClearTopLevelSmiResult
= ClearTopLevelSmiStatus();
399 ASSERT (ClearTopLevelSmiResult
== TRUE
);
402 // Set running processor index
404 gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
= CpuIndex
;
407 // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
409 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
412 // Wait for APs to arrive
414 SmmWaitForApArrival();
417 // Lock the counter down and retrieve the number of APs
419 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
420 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
423 // Wait for all APs to get ready for programming MTRRs
425 WaitForAllAPs (ApCount
);
427 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
429 // Signal all APs it's time for backup MTRRs
434 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
435 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
436 // to a large enough value to avoid this situation.
437 // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
438 // We do the backup first and then set MTRR to avoid race condition for threads
441 MtrrGetAllMtrrs(&Mtrrs
);
444 // Wait for all APs to complete their MTRR saving
446 WaitForAllAPs (ApCount
);
449 // Let all processors program SMM MTRRs together
454 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
455 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
456 // to a large enough value to avoid this situation.
458 ReplaceOSMtrrs (CpuIndex
);
461 // Wait for all APs to complete their MTRR programming
463 WaitForAllAPs (ApCount
);
468 // The BUSY lock is initialized to Acquired state
470 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
473 // Perform the pre tasks
478 // Invoke SMM Foundation EntryPoint with the processor information context.
480 gSmmCpuPrivate
->SmmCoreEntry (&gSmmCpuPrivate
->SmmCoreEntryContext
);
483 // Make sure all APs have completed their pending none-block tasks
485 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
486 if (Index
!= CpuIndex
&& *(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
487 AcquireSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
488 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
493 // Perform the remaining tasks
495 PerformRemainingTasks ();
498 // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
499 // make those APs to exit SMI synchronously. APs which arrive later will be excluded and
500 // will run through freely.
502 if (SyncMode
!= SmmCpuSyncModeTradition
&& !SmmCpuFeaturesNeedConfigureMtrrs()) {
505 // Lock the counter down and retrieve the number of APs
507 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
508 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
510 // Make sure all APs have their Present flag set
514 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
515 if (*(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
519 if (PresentCount
> ApCount
) {
526 // Notify all APs to exit
528 *mSmmMpSyncData
->InsideSmm
= FALSE
;
532 // Wait for all APs to complete their pending tasks
534 WaitForAllAPs (ApCount
);
536 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
538 // Signal APs to restore MTRRs
545 SmmCpuFeaturesReenableSmrr ();
546 MtrrSetAllMtrrs(&Mtrrs
);
549 // Wait for all APs to complete MTRR programming
551 WaitForAllAPs (ApCount
);
555 // Stop source level debug in BSP handler, the code below will not be
558 InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI
, NULL
, NULL
);
561 // Signal APs to Reset states/semaphore for this processor
566 // Perform pending operations for hot-plug
571 // Clear the Present flag of BSP
573 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
576 // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
577 // WaitForAllAps does not depend on the Present flag.
579 WaitForAllAPs (ApCount
);
582 // Reset BspIndex to -1, meaning BSP has not been elected.
584 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
585 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
589 // Allow APs to check in from this point on
591 *mSmmMpSyncData
->Counter
= 0;
592 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
598 @param CpuIndex AP processor Index.
599 @param ValidSmi Indicates that current SMI is a valid SMI or not.
600 @param SyncMode SMM MP sync mode.
607 IN SMM_CPU_SYNC_MODE SyncMode
617 for (Timer
= StartSyncTimer ();
618 !IsSyncTimerTimeout (Timer
) &&
619 !(*mSmmMpSyncData
->InsideSmm
);
624 if (!(*mSmmMpSyncData
->InsideSmm
)) {
626 // BSP timeout in the first round
628 if (mSmmMpSyncData
->BspIndex
!= -1) {
630 // BSP Index is known
632 BspIndex
= mSmmMpSyncData
->BspIndex
;
633 ASSERT (CpuIndex
!= BspIndex
);
636 // Send SMI IPI to bring BSP in
638 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[BspIndex
].ProcessorId
);
641 // Now clock BSP for the 2nd time
643 for (Timer
= StartSyncTimer ();
644 !IsSyncTimerTimeout (Timer
) &&
645 !(*mSmmMpSyncData
->InsideSmm
);
650 if (!(*mSmmMpSyncData
->InsideSmm
)) {
652 // Give up since BSP is unable to enter SMM
653 // and signal the completion of this AP
654 WaitForSemaphore (mSmmMpSyncData
->Counter
);
659 // Don't know BSP index. Give up without sending IPI to BSP.
661 WaitForSemaphore (mSmmMpSyncData
->Counter
);
669 BspIndex
= mSmmMpSyncData
->BspIndex
;
670 ASSERT (CpuIndex
!= BspIndex
);
673 // Mark this processor's presence
675 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
677 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
679 // Notify BSP of arrival at this point
681 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
684 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
686 // Wait for the signal from BSP to backup MTRRs
688 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
693 MtrrGetAllMtrrs(&Mtrrs
);
696 // Signal BSP the completion of this AP
698 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
701 // Wait for BSP's signal to program MTRRs
703 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
706 // Replace OS MTRRs with SMI MTRRs
708 ReplaceOSMtrrs (CpuIndex
);
711 // Signal BSP the completion of this AP
713 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
718 // Wait for something to happen
720 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
723 // Check if BSP wants to exit SMM
725 if (!(*mSmmMpSyncData
->InsideSmm
)) {
730 // BUSY should be acquired by SmmStartupThisAp()
733 !AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)
737 // Invoke the scheduled procedure
739 (*mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
) (
740 (VOID
*)mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
746 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
749 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
751 // Notify BSP the readiness of this AP to program MTRRs
753 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
756 // Wait for the signal from BSP to program MTRRs
758 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
763 SmmCpuFeaturesReenableSmrr ();
764 MtrrSetAllMtrrs(&Mtrrs
);
768 // Notify BSP the readiness of this AP to Reset states/semaphore for this processor
770 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
773 // Wait for the signal from BSP to Reset states/semaphore for this processor
775 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
778 // Reset states/semaphore for this processor
780 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
783 // Notify BSP the readiness of this AP to exit SMM
785 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
790 Create 4G PageTable in SMRAM.
792 @param[in] Is32BitPageTable Whether the page table is 32-bit PAE
793 @return PageTable Address
798 IN BOOLEAN Is32BitPageTable
806 UINTN High2MBoundary
;
816 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
818 // Add one more page for known good stack, then find the lower 2MB aligned address.
820 Low2MBoundary
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
) & ~(SIZE_2MB
-1);
822 // Add two more pages for known good stack and stack guard page,
823 // then find the lower 2MB aligned address.
825 High2MBoundary
= (mSmmStackArrayEnd
- mSmmStackSize
+ EFI_PAGE_SIZE
* 2) & ~(SIZE_2MB
-1);
826 PagesNeeded
= ((High2MBoundary
- Low2MBoundary
) / SIZE_2MB
) + 1;
829 // Allocate the page table
831 PageTable
= AllocatePageTableMemory (5 + PagesNeeded
);
832 ASSERT (PageTable
!= NULL
);
834 PageTable
= (VOID
*)((UINTN
)PageTable
);
835 Pte
= (UINT64
*)PageTable
;
838 // Zero out all page table entries first
840 ZeroMem (Pte
, EFI_PAGES_TO_SIZE (1));
843 // Set Page Directory Pointers
845 for (Index
= 0; Index
< 4; Index
++) {
846 Pte
[Index
] = ((UINTN
)PageTable
+ EFI_PAGE_SIZE
* (Index
+ 1)) | mAddressEncMask
|
847 (Is32BitPageTable
? IA32_PAE_PDPTE_ATTRIBUTE_BITS
: PAGE_ATTRIBUTE_BITS
);
849 Pte
+= EFI_PAGE_SIZE
/ sizeof (*Pte
);
852 // Fill in Page Directory Entries
854 for (Index
= 0; Index
< EFI_PAGE_SIZE
* 4 / sizeof (*Pte
); Index
++) {
855 Pte
[Index
] = (Index
<< 21) | mAddressEncMask
| IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
858 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
859 Pages
= (UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (5);
860 GuardPage
= mSmmStackArrayBase
+ EFI_PAGE_SIZE
;
861 Pdpte
= (UINT64
*)PageTable
;
862 for (PageIndex
= Low2MBoundary
; PageIndex
<= High2MBoundary
; PageIndex
+= SIZE_2MB
) {
863 Pte
= (UINT64
*)(UINTN
)(Pdpte
[BitFieldRead32 ((UINT32
)PageIndex
, 30, 31)] & ~mAddressEncMask
& ~(EFI_PAGE_SIZE
- 1));
864 Pte
[BitFieldRead32 ((UINT32
)PageIndex
, 21, 29)] = (UINT64
)Pages
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
866 // Fill in Page Table Entries
868 Pte
= (UINT64
*)Pages
;
869 PageAddress
= PageIndex
;
870 for (Index
= 0; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
871 if (PageAddress
== GuardPage
) {
873 // Mark the guard page as non-present
875 Pte
[Index
] = PageAddress
| mAddressEncMask
;
876 GuardPage
+= mSmmStackSize
;
877 if (GuardPage
> mSmmStackArrayEnd
) {
881 Pte
[Index
] = PageAddress
| mAddressEncMask
| PAGE_ATTRIBUTE_BITS
;
883 PageAddress
+= EFI_PAGE_SIZE
;
885 Pages
+= EFI_PAGE_SIZE
;
889 return (UINT32
)(UINTN
)PageTable
;
893 Schedule a procedure to run on the specified CPU.
895 @param[in] Procedure The address of the procedure to run
896 @param[in] CpuIndex Target CPU Index
897 @param[in, out] ProcArguments The parameter to pass to the procedure
898 @param[in] BlockingMode Startup AP in blocking mode or not
900 @retval EFI_INVALID_PARAMETER CpuNumber not valid
901 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
902 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
903 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
904 @retval EFI_SUCCESS The procedure has been successfully scheduled
908 InternalSmmStartupThisAp (
909 IN EFI_AP_PROCEDURE Procedure
,
911 IN OUT VOID
*ProcArguments OPTIONAL
,
912 IN BOOLEAN BlockingMode
915 if (CpuIndex
>= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
) {
916 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus(%d)\n", CpuIndex
, gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
));
917 return EFI_INVALID_PARAMETER
;
919 if (CpuIndex
== gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) {
920 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu\n", CpuIndex
));
921 return EFI_INVALID_PARAMETER
;
923 if (gSmmCpuPrivate
->ProcessorInfo
[CpuIndex
].ProcessorId
== INVALID_APIC_ID
) {
924 return EFI_INVALID_PARAMETER
;
926 if (!(*(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
))) {
927 if (mSmmMpSyncData
->EffectiveSyncMode
== SmmCpuSyncModeTradition
) {
928 DEBUG((DEBUG_ERROR
, "!mSmmMpSyncData->CpuData[%d].Present\n", CpuIndex
));
930 return EFI_INVALID_PARAMETER
;
932 if (gSmmCpuPrivate
->Operation
[CpuIndex
] == SmmCpuRemove
) {
933 if (!FeaturePcdGet (PcdCpuHotPlugSupport
)) {
934 DEBUG((DEBUG_ERROR
, "gSmmCpuPrivate->Operation[%d] == SmmCpuRemove\n", CpuIndex
));
936 return EFI_INVALID_PARAMETER
;
940 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
942 if (!AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)) {
943 DEBUG((DEBUG_ERROR
, "mSmmMpSyncData->CpuData[%d].Busy\n", CpuIndex
));
944 return EFI_INVALID_PARAMETER
;
948 mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
= Procedure
;
949 mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
= ProcArguments
;
950 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
953 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
954 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
960 Schedule a procedure to run on the specified CPU in blocking mode.
962 @param[in] Procedure The address of the procedure to run
963 @param[in] CpuIndex Target CPU Index
964 @param[in, out] ProcArguments The parameter to pass to the procedure
966 @retval EFI_INVALID_PARAMETER CpuNumber not valid
967 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
968 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
969 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
970 @retval EFI_SUCCESS The procedure has been successfully scheduled
975 SmmBlockingStartupThisAp (
976 IN EFI_AP_PROCEDURE Procedure
,
978 IN OUT VOID
*ProcArguments OPTIONAL
981 return InternalSmmStartupThisAp(Procedure
, CpuIndex
, ProcArguments
, TRUE
);
985 Schedule a procedure to run on the specified CPU.
987 @param Procedure The address of the procedure to run
988 @param CpuIndex Target CPU Index
989 @param ProcArguments The parameter to pass to the procedure
991 @retval EFI_INVALID_PARAMETER CpuNumber not valid
992 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
993 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
994 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
995 @retval EFI_SUCCESS The procedure has been successfully scheduled
1001 IN EFI_AP_PROCEDURE Procedure
,
1003 IN OUT VOID
*ProcArguments OPTIONAL
1006 return InternalSmmStartupThisAp(Procedure
, CpuIndex
, ProcArguments
, FeaturePcdGet (PcdCpuSmmBlockStartupThisAp
));
1010 This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
1011 They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
1013 NOTE: It might not be appreciated in runtime since it might
1014 conflict with OS debugging facilities. Turn them off in RELEASE.
1016 @param CpuIndex CPU Index
1025 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1027 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1028 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1029 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1030 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1031 AsmWriteDr6 (CpuSaveState
->x86
._DR6
);
1032 AsmWriteDr7 (CpuSaveState
->x86
._DR7
);
1034 AsmWriteDr6 ((UINTN
)CpuSaveState
->x64
._DR6
);
1035 AsmWriteDr7 ((UINTN
)CpuSaveState
->x64
._DR7
);
1041 This function restores DR6 & DR7 to SMM save state.
1043 NOTE: It might not be appreciated in runtime since it might
1044 conflict with OS debugging facilities. Turn them off in RELEASE.
1046 @param CpuIndex CPU Index
1055 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1057 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1058 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1059 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1060 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1061 CpuSaveState
->x86
._DR7
= (UINT32
)AsmReadDr7 ();
1062 CpuSaveState
->x86
._DR6
= (UINT32
)AsmReadDr6 ();
1064 CpuSaveState
->x64
._DR7
= AsmReadDr7 ();
1065 CpuSaveState
->x64
._DR6
= AsmReadDr6 ();
1071 C function for SMI entry, each processor comes here upon SMI trigger.
1073 @param CpuIndex CPU Index
1085 BOOLEAN BspInProgress
;
1089 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1092 // Save Cr2 because Page Fault exception in SMM may override its value
1094 Cr2
= AsmReadCr2 ();
1097 // Perform CPU specific entry hooks
1099 SmmCpuFeaturesRendezvousEntry (CpuIndex
);
1102 // Determine if this is a valid SMI
1104 ValidSmi
= PlatformValidSmi();
1107 // Determine if BSP has been already in progress. Note this must be checked after
1108 // ValidSmi because BSP may clear a valid SMI source after checking in.
1110 BspInProgress
= *mSmmMpSyncData
->InsideSmm
;
1112 if (!BspInProgress
&& !ValidSmi
) {
1114 // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not
1115 // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI
1116 // status had been cleared by BSP and an existing SMI run has almost ended. (Note
1117 // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there
1118 // is nothing we need to do.
1123 // Signal presence of this processor
1125 if (ReleaseSemaphore (mSmmMpSyncData
->Counter
) == 0) {
1127 // BSP has already ended the synchronization, so QUIT!!!
1131 // Wait for BSP's signal to finish SMI
1133 while (*mSmmMpSyncData
->AllCpusInSync
) {
1140 // The BUSY lock is initialized to Released state.
1141 // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.
1142 // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
1143 // after AP's present flag is detected.
1145 InitializeSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1148 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1149 ActivateSmmProfile (CpuIndex
);
1152 if (BspInProgress
) {
1154 // BSP has been elected. Follow AP path, regardless of ValidSmi flag
1155 // as BSP may have cleared the SMI status
1157 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1160 // We have a valid SMI
1167 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1168 if (!mSmmMpSyncData
->SwitchBsp
|| mSmmMpSyncData
->CandidateBsp
[CpuIndex
]) {
1170 // Call platform hook to do BSP election
1172 Status
= PlatformSmmBspElection (&IsBsp
);
1173 if (EFI_SUCCESS
== Status
) {
1175 // Platform hook determines successfully
1178 mSmmMpSyncData
->BspIndex
= (UINT32
)CpuIndex
;
1182 // Platform hook fails to determine, use default BSP election method
1184 InterlockedCompareExchange32 (
1185 (UINT32
*)&mSmmMpSyncData
->BspIndex
,
1194 // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP
1196 if (mSmmMpSyncData
->BspIndex
== CpuIndex
) {
1199 // Clear last request for SwitchBsp.
1201 if (mSmmMpSyncData
->SwitchBsp
) {
1202 mSmmMpSyncData
->SwitchBsp
= FALSE
;
1203 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1204 mSmmMpSyncData
->CandidateBsp
[Index
] = FALSE
;
1208 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1209 SmmProfileRecordSmiNum ();
1213 // BSP Handler is always called with a ValidSmi == TRUE
1215 BSPHandler (CpuIndex
, mSmmMpSyncData
->EffectiveSyncMode
);
1217 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1221 ASSERT (*mSmmMpSyncData
->CpuData
[CpuIndex
].Run
== 0);
1224 // Wait for BSP's signal to exit SMI
1226 while (*mSmmMpSyncData
->AllCpusInSync
) {
1232 SmmCpuFeaturesRendezvousExit (CpuIndex
);
1240 Allocate buffer for all semaphores and spin locks.
1244 InitializeSmmCpuSemaphores (
1248 UINTN ProcessorCount
;
1250 UINTN GlobalSemaphoresSize
;
1251 UINTN CpuSemaphoresSize
;
1252 UINTN MsrSemahporeSize
;
1253 UINTN SemaphoreSize
;
1255 UINTN
*SemaphoreBlock
;
1256 UINTN SemaphoreAddr
;
1258 SemaphoreSize
= GetSpinLockProperties ();
1259 ProcessorCount
= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1260 GlobalSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_GLOBAL
) / sizeof (VOID
*)) * SemaphoreSize
;
1261 CpuSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_CPU
) / sizeof (VOID
*)) * ProcessorCount
* SemaphoreSize
;
1262 MsrSemahporeSize
= MSR_SPIN_LOCK_INIT_NUM
* SemaphoreSize
;
1263 TotalSize
= GlobalSemaphoresSize
+ CpuSemaphoresSize
+ MsrSemahporeSize
;
1264 DEBUG((EFI_D_INFO
, "One Semaphore Size = 0x%x\n", SemaphoreSize
));
1265 DEBUG((EFI_D_INFO
, "Total Semaphores Size = 0x%x\n", TotalSize
));
1266 Pages
= EFI_SIZE_TO_PAGES (TotalSize
);
1267 SemaphoreBlock
= AllocatePages (Pages
);
1268 ASSERT (SemaphoreBlock
!= NULL
);
1269 ZeroMem (SemaphoreBlock
, TotalSize
);
1271 SemaphoreAddr
= (UINTN
)SemaphoreBlock
;
1272 mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
= (UINT32
*)SemaphoreAddr
;
1273 SemaphoreAddr
+= SemaphoreSize
;
1274 mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
= (BOOLEAN
*)SemaphoreAddr
;
1275 SemaphoreAddr
+= SemaphoreSize
;
1276 mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
= (BOOLEAN
*)SemaphoreAddr
;
1277 SemaphoreAddr
+= SemaphoreSize
;
1278 mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
= (SPIN_LOCK
*)SemaphoreAddr
;
1279 SemaphoreAddr
+= SemaphoreSize
;
1280 mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
1281 = (SPIN_LOCK
*)SemaphoreAddr
;
1282 SemaphoreAddr
+= SemaphoreSize
;
1283 mSmmCpuSemaphores
.SemaphoreGlobal
.MemoryMappedLock
1284 = (SPIN_LOCK
*)SemaphoreAddr
;
1286 SemaphoreAddr
= (UINTN
)SemaphoreBlock
+ GlobalSemaphoresSize
;
1287 mSmmCpuSemaphores
.SemaphoreCpu
.Busy
= (SPIN_LOCK
*)SemaphoreAddr
;
1288 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1289 mSmmCpuSemaphores
.SemaphoreCpu
.Run
= (UINT32
*)SemaphoreAddr
;
1290 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1291 mSmmCpuSemaphores
.SemaphoreCpu
.Present
= (BOOLEAN
*)SemaphoreAddr
;
1293 SemaphoreAddr
= (UINTN
)SemaphoreBlock
+ GlobalSemaphoresSize
+ CpuSemaphoresSize
;
1294 mSmmCpuSemaphores
.SemaphoreMsr
.Msr
= (SPIN_LOCK
*)SemaphoreAddr
;
1295 mSmmCpuSemaphores
.SemaphoreMsr
.AvailableCounter
=
1296 ((UINTN
)SemaphoreBlock
+ Pages
* SIZE_4KB
- SemaphoreAddr
) / SemaphoreSize
;
1297 ASSERT (mSmmCpuSemaphores
.SemaphoreMsr
.AvailableCounter
>= MSR_SPIN_LOCK_INIT_NUM
);
1299 mPFLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
;
1300 mConfigSmmCodeAccessCheckLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
;
1301 mMemoryMappedLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.MemoryMappedLock
;
1303 mSemaphoreSize
= SemaphoreSize
;
1307 Initialize un-cacheable data.
1312 InitializeMpSyncData (
1318 if (mSmmMpSyncData
!= NULL
) {
1320 // mSmmMpSyncDataSize includes one structure of SMM_DISPATCHER_MP_SYNC_DATA, one
1321 // CpuData array of SMM_CPU_DATA_BLOCK and one CandidateBsp array of BOOLEAN.
1323 ZeroMem (mSmmMpSyncData
, mSmmMpSyncDataSize
);
1324 mSmmMpSyncData
->CpuData
= (SMM_CPU_DATA_BLOCK
*)((UINT8
*)mSmmMpSyncData
+ sizeof (SMM_DISPATCHER_MP_SYNC_DATA
));
1325 mSmmMpSyncData
->CandidateBsp
= (BOOLEAN
*)(mSmmMpSyncData
->CpuData
+ gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1326 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1328 // Enable BSP election by setting BspIndex to -1
1330 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
1332 mSmmMpSyncData
->EffectiveSyncMode
= mCpuSmmSyncMode
;
1334 mSmmMpSyncData
->Counter
= mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
;
1335 mSmmMpSyncData
->InsideSmm
= mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
;
1336 mSmmMpSyncData
->AllCpusInSync
= mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
;
1337 ASSERT (mSmmMpSyncData
->Counter
!= NULL
&& mSmmMpSyncData
->InsideSmm
!= NULL
&&
1338 mSmmMpSyncData
->AllCpusInSync
!= NULL
);
1339 *mSmmMpSyncData
->Counter
= 0;
1340 *mSmmMpSyncData
->InsideSmm
= FALSE
;
1341 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
1343 for (CpuIndex
= 0; CpuIndex
< gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
; CpuIndex
++) {
1344 mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
=
1345 (SPIN_LOCK
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Busy
+ mSemaphoreSize
* CpuIndex
);
1346 mSmmMpSyncData
->CpuData
[CpuIndex
].Run
=
1347 (UINT32
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Run
+ mSemaphoreSize
* CpuIndex
);
1348 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
=
1349 (BOOLEAN
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Present
+ mSemaphoreSize
* CpuIndex
);
1350 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
) = 0;
1351 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Run
) = 0;
1352 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
1358 Initialize global data for MP synchronization.
1360 @param Stacks Base address of SMI stack buffer for all processors.
1361 @param StackSize Stack size for each processor in SMM.
1365 InitializeMpServiceData (
1372 UINT8
*GdtTssTables
;
1373 UINTN GdtTableStepSize
;
1374 CPUID_VERSION_INFO_EDX RegEdx
;
1377 // Determine if this CPU supports machine check
1379 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &RegEdx
.Uint32
);
1380 mMachineCheckSupported
= (BOOLEAN
)(RegEdx
.Bits
.MCA
== 1);
1383 // Allocate memory for all locks and semaphores
1385 InitializeSmmCpuSemaphores ();
1388 // Initialize mSmmMpSyncData
1390 mSmmMpSyncDataSize
= sizeof (SMM_DISPATCHER_MP_SYNC_DATA
) +
1391 (sizeof (SMM_CPU_DATA_BLOCK
) + sizeof (BOOLEAN
)) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1392 mSmmMpSyncData
= (SMM_DISPATCHER_MP_SYNC_DATA
*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize
));
1393 ASSERT (mSmmMpSyncData
!= NULL
);
1394 mCpuSmmSyncMode
= (SMM_CPU_SYNC_MODE
)PcdGet8 (PcdCpuSmmSyncMode
);
1395 InitializeMpSyncData ();
1398 // Initialize physical address mask
1399 // NOTE: Physical memory above virtual address limit is not supported !!!
1401 AsmCpuid (0x80000008, (UINT32
*)&Index
, NULL
, NULL
, NULL
);
1402 gPhyMask
= LShiftU64 (1, (UINT8
)Index
) - 1;
1403 gPhyMask
&= (1ull << 48) - EFI_PAGE_SIZE
;
1406 // Create page tables
1408 Cr3
= SmmInitPageTable ();
1410 GdtTssTables
= InitGdt (Cr3
, &GdtTableStepSize
);
1413 // Install SMI handler for each CPU
1415 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1418 (UINT32
)mCpuHotPlugData
.SmBase
[Index
],
1419 (VOID
*)((UINTN
)Stacks
+ (StackSize
* Index
)),
1421 (UINTN
)(GdtTssTables
+ GdtTableStepSize
* Index
),
1422 gcSmiGdtr
.Limit
+ 1,
1424 gcSmiIdtr
.Limit
+ 1,
1430 // Record current MTRR settings
1432 ZeroMem (&gSmiMtrrs
, sizeof (gSmiMtrrs
));
1433 MtrrGetAllMtrrs (&gSmiMtrrs
);
1440 Register the SMM Foundation entry point.
1442 @param This Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance
1443 @param SmmEntryPoint SMM Foundation EntryPoint
1445 @retval EFI_SUCCESS Successfully to register SMM foundation entry point
1451 IN CONST EFI_SMM_CONFIGURATION_PROTOCOL
*This
,
1452 IN EFI_SMM_ENTRY_POINT SmmEntryPoint
1456 // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.
1458 gSmmCpuPrivate
->SmmCoreEntry
= SmmEntryPoint
;