2 SMM MP service implementation
4 Copyright (c) 2009 - 2016, Intel Corporation. All rights reserved.<BR>
5 This program and the accompanying materials
6 are licensed and made available under the terms and conditions of the BSD License
7 which accompanies this distribution. The full text of the license may be found at
8 http://opensource.org/licenses/bsd-license.php
10 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
15 #include "PiSmmCpuDxeSmm.h"
18 // Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
20 MTRR_SETTINGS gSmiMtrrs
;
22 SMM_DISPATCHER_MP_SYNC_DATA
*mSmmMpSyncData
= NULL
;
23 UINTN mSmmMpSyncDataSize
;
24 SMM_CPU_SEMAPHORES mSmmCpuSemaphores
;
26 SPIN_LOCK
*mPFLock
= NULL
;
27 SMM_CPU_SYNC_MODE mCpuSmmSyncMode
;
30 Performs an atomic compare exchange operation to get semaphore.
31 The compare exchange operation must be performed using
34 @param Sem IN: 32-bit unsigned integer
35 OUT: original integer - 1
36 @return Original integer - 1
41 IN OUT
volatile UINT32
*Sem
48 } while (Value
== 0 ||
49 InterlockedCompareExchange32 (
59 Performs an atomic compare exchange operation to release semaphore.
60 The compare exchange operation must be performed using
63 @param Sem IN: 32-bit unsigned integer
64 OUT: original integer + 1
65 @return Original integer + 1
70 IN OUT
volatile UINT32
*Sem
77 } while (Value
+ 1 != 0 &&
78 InterlockedCompareExchange32 (
87 Performs an atomic compare exchange operation to lock semaphore.
88 The compare exchange operation must be performed using
91 @param Sem IN: 32-bit unsigned integer
93 @return Original integer
98 IN OUT
volatile UINT32
*Sem
105 } while (InterlockedCompareExchange32 (
113 Wait all APs to performs an atomic compare exchange operation to release semaphore.
115 @param NumberOfAPs AP number
125 BspIndex
= mSmmMpSyncData
->BspIndex
;
126 while (NumberOfAPs
-- > 0) {
127 WaitForSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
132 Performs an atomic compare exchange operation to release semaphore
144 BspIndex
= mSmmMpSyncData
->BspIndex
;
145 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
146 if (Index
!= BspIndex
&& *(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
147 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[Index
].Run
);
153 Checks if all CPUs (with certain exceptions) have checked in for this SMI run
155 @param Exceptions CPU Arrival exception flags.
157 @retval TRUE if all CPUs the have checked in.
158 @retval FALSE if at least one Normal AP hasn't checked in.
162 AllCpusInSmmWithExceptions (
163 SMM_CPU_ARRIVAL_EXCEPTIONS Exceptions
167 SMM_CPU_DATA_BLOCK
*CpuData
;
168 EFI_PROCESSOR_INFORMATION
*ProcessorInfo
;
170 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
172 if (*mSmmMpSyncData
->Counter
== mNumberOfCpus
) {
176 CpuData
= mSmmMpSyncData
->CpuData
;
177 ProcessorInfo
= gSmmCpuPrivate
->ProcessorInfo
;
178 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
179 if (!(*(CpuData
[Index
].Present
)) && ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
180 if (((Exceptions
& ARRIVAL_EXCEPTION_DELAYED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmDelayed
) != 0) {
183 if (((Exceptions
& ARRIVAL_EXCEPTION_BLOCKED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmBlocked
) != 0) {
186 if (((Exceptions
& ARRIVAL_EXCEPTION_SMI_DISABLED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmEnable
) != 0) {
199 Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
200 entering SMM, except SMI disabled APs.
204 SmmWaitForApArrival (
211 ASSERT (*mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
214 // Platform implementor should choose a timeout value appropriately:
215 // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note
216 // the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.
217 // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI
218 // and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will
219 // be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the
220 // SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.
221 // - The timeout value must be longer than longest possible IO operation in the system
225 // Sync with APs 1st timeout
227 for (Timer
= StartSyncTimer ();
228 !IsSyncTimerTimeout (Timer
) &&
229 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
235 // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,
237 // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running
238 // normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they
239 // enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode
240 // work while SMI handling is on-going.
241 // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.
242 // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state
243 // will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal
244 // mode work while SMI handling is on-going.
245 // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:
246 // - In traditional flow, SMI disabling is discouraged.
247 // - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
248 // In both cases, adding SMI-disabling checking code increases overhead.
250 if (*mSmmMpSyncData
->Counter
< mNumberOfCpus
) {
252 // Send SMI IPIs to bring outside processors in
254 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
255 if (!(*(mSmmMpSyncData
->CpuData
[Index
].Present
)) && gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
256 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
);
261 // Sync with APs 2nd timeout.
263 for (Timer
= StartSyncTimer ();
264 !IsSyncTimerTimeout (Timer
) &&
265 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
276 Replace OS MTRR's with SMI MTRR's.
278 @param CpuIndex Processor Index
286 SmmCpuFeaturesDisableSmrr ();
289 // Replace all MTRRs registers
291 MtrrSetAllMtrrs (&gSmiMtrrs
);
297 @param CpuIndex BSP processor Index
298 @param SyncMode SMM MP sync mode
304 IN SMM_CPU_SYNC_MODE SyncMode
310 BOOLEAN ClearTopLevelSmiResult
;
313 ASSERT (CpuIndex
== mSmmMpSyncData
->BspIndex
);
317 // Flag BSP's presence
319 *mSmmMpSyncData
->InsideSmm
= TRUE
;
322 // Initialize Debug Agent to start source level debug in BSP handler
324 InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI
, NULL
, NULL
);
327 // Mark this processor's presence
329 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
332 // Clear platform top level SMI status bit before calling SMI handlers. If
333 // we cleared it after SMI handlers are run, we would miss the SMI that
334 // occurs after SMI handlers are done and before SMI status bit is cleared.
336 ClearTopLevelSmiResult
= ClearTopLevelSmiStatus();
337 ASSERT (ClearTopLevelSmiResult
== TRUE
);
340 // Set running processor index
342 gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
= CpuIndex
;
345 // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
347 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
350 // Wait for APs to arrive
352 SmmWaitForApArrival();
355 // Lock the counter down and retrieve the number of APs
357 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
358 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
361 // Wait for all APs to get ready for programming MTRRs
363 WaitForAllAPs (ApCount
);
365 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
367 // Signal all APs it's time for backup MTRRs
372 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
373 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
374 // to a large enough value to avoid this situation.
375 // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
376 // We do the backup first and then set MTRR to avoid race condition for threads
379 MtrrGetAllMtrrs(&Mtrrs
);
382 // Wait for all APs to complete their MTRR saving
384 WaitForAllAPs (ApCount
);
387 // Let all processors program SMM MTRRs together
392 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
393 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
394 // to a large enough value to avoid this situation.
396 ReplaceOSMtrrs (CpuIndex
);
399 // Wait for all APs to complete their MTRR programming
401 WaitForAllAPs (ApCount
);
406 // The BUSY lock is initialized to Acquired state
408 AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
411 // Perform the pre tasks
416 // Invoke SMM Foundation EntryPoint with the processor information context.
418 gSmmCpuPrivate
->SmmCoreEntry (&gSmmCpuPrivate
->SmmCoreEntryContext
);
421 // Make sure all APs have completed their pending none-block tasks
423 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
424 if (Index
!= CpuIndex
&& *(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
425 AcquireSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
426 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[Index
].Busy
);
431 // Perform the remaining tasks
433 PerformRemainingTasks ();
436 // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
437 // make those APs to exit SMI synchronously. APs which arrive later will be excluded and
438 // will run through freely.
440 if (SyncMode
!= SmmCpuSyncModeTradition
&& !SmmCpuFeaturesNeedConfigureMtrrs()) {
443 // Lock the counter down and retrieve the number of APs
445 *mSmmMpSyncData
->AllCpusInSync
= TRUE
;
446 ApCount
= LockdownSemaphore (mSmmMpSyncData
->Counter
) - 1;
448 // Make sure all APs have their Present flag set
452 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
453 if (*(mSmmMpSyncData
->CpuData
[Index
].Present
)) {
457 if (PresentCount
> ApCount
) {
464 // Notify all APs to exit
466 *mSmmMpSyncData
->InsideSmm
= FALSE
;
470 // Wait for all APs to complete their pending tasks
472 WaitForAllAPs (ApCount
);
474 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
476 // Signal APs to restore MTRRs
483 SmmCpuFeaturesReenableSmrr ();
484 MtrrSetAllMtrrs(&Mtrrs
);
487 // Wait for all APs to complete MTRR programming
489 WaitForAllAPs (ApCount
);
493 // Stop source level debug in BSP handler, the code below will not be
496 InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI
, NULL
, NULL
);
499 // Signal APs to Reset states/semaphore for this processor
504 // Perform pending operations for hot-plug
509 // Clear the Present flag of BSP
511 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
514 // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
515 // WaitForAllAps does not depend on the Present flag.
517 WaitForAllAPs (ApCount
);
520 // Reset BspIndex to -1, meaning BSP has not been elected.
522 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
523 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
527 // Allow APs to check in from this point on
529 *mSmmMpSyncData
->Counter
= 0;
530 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
536 @param CpuIndex AP processor Index.
537 @param ValidSmi Indicates that current SMI is a valid SMI or not.
538 @param SyncMode SMM MP sync mode.
545 IN SMM_CPU_SYNC_MODE SyncMode
555 for (Timer
= StartSyncTimer ();
556 !IsSyncTimerTimeout (Timer
) &&
557 !(*mSmmMpSyncData
->InsideSmm
);
562 if (!(*mSmmMpSyncData
->InsideSmm
)) {
564 // BSP timeout in the first round
566 if (mSmmMpSyncData
->BspIndex
!= -1) {
568 // BSP Index is known
570 BspIndex
= mSmmMpSyncData
->BspIndex
;
571 ASSERT (CpuIndex
!= BspIndex
);
574 // Send SMI IPI to bring BSP in
576 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[BspIndex
].ProcessorId
);
579 // Now clock BSP for the 2nd time
581 for (Timer
= StartSyncTimer ();
582 !IsSyncTimerTimeout (Timer
) &&
583 !(*mSmmMpSyncData
->InsideSmm
);
588 if (!(*mSmmMpSyncData
->InsideSmm
)) {
590 // Give up since BSP is unable to enter SMM
591 // and signal the completion of this AP
592 WaitForSemaphore (mSmmMpSyncData
->Counter
);
597 // Don't know BSP index. Give up without sending IPI to BSP.
599 WaitForSemaphore (mSmmMpSyncData
->Counter
);
607 BspIndex
= mSmmMpSyncData
->BspIndex
;
608 ASSERT (CpuIndex
!= BspIndex
);
611 // Mark this processor's presence
613 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = TRUE
;
615 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
617 // Notify BSP of arrival at this point
619 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
622 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
624 // Wait for the signal from BSP to backup MTRRs
626 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
631 MtrrGetAllMtrrs(&Mtrrs
);
634 // Signal BSP the completion of this AP
636 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
639 // Wait for BSP's signal to program MTRRs
641 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
644 // Replace OS MTRRs with SMI MTRRs
646 ReplaceOSMtrrs (CpuIndex
);
649 // Signal BSP the completion of this AP
651 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
656 // Wait for something to happen
658 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
661 // Check if BSP wants to exit SMM
663 if (!(*mSmmMpSyncData
->InsideSmm
)) {
668 // BUSY should be acquired by SmmStartupThisAp()
671 !AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)
675 // Invoke the scheduled procedure
677 (*mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
) (
678 (VOID
*)mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
684 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
687 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
689 // Notify BSP the readiness of this AP to program MTRRs
691 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
694 // Wait for the signal from BSP to program MTRRs
696 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
701 SmmCpuFeaturesReenableSmrr ();
702 MtrrSetAllMtrrs(&Mtrrs
);
706 // Notify BSP the readiness of this AP to Reset states/semaphore for this processor
708 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
711 // Wait for the signal from BSP to Reset states/semaphore for this processor
713 WaitForSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
716 // Reset states/semaphore for this processor
718 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
721 // Notify BSP the readiness of this AP to exit SMM
723 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
728 Create 4G PageTable in SMRAM.
730 @param[in] Is32BitPageTable Whether the page table is 32-bit PAE
731 @return PageTable Address
736 IN BOOLEAN Is32BitPageTable
744 UINTN High2MBoundary
;
754 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
756 // Add one more page for known good stack, then find the lower 2MB aligned address.
758 Low2MBoundary
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
) & ~(SIZE_2MB
-1);
760 // Add two more pages for known good stack and stack guard page,
761 // then find the lower 2MB aligned address.
763 High2MBoundary
= (mSmmStackArrayEnd
- mSmmStackSize
+ EFI_PAGE_SIZE
* 2) & ~(SIZE_2MB
-1);
764 PagesNeeded
= ((High2MBoundary
- Low2MBoundary
) / SIZE_2MB
) + 1;
767 // Allocate the page table
769 PageTable
= AllocatePageTableMemory (5 + PagesNeeded
);
770 ASSERT (PageTable
!= NULL
);
772 PageTable
= (VOID
*)((UINTN
)PageTable
);
773 Pte
= (UINT64
*)PageTable
;
776 // Zero out all page table entries first
778 ZeroMem (Pte
, EFI_PAGES_TO_SIZE (1));
781 // Set Page Directory Pointers
783 for (Index
= 0; Index
< 4; Index
++) {
784 Pte
[Index
] = (UINTN
)PageTable
+ EFI_PAGE_SIZE
* (Index
+ 1) + (Is32BitPageTable
? IA32_PAE_PDPTE_ATTRIBUTE_BITS
: PAGE_ATTRIBUTE_BITS
);
786 Pte
+= EFI_PAGE_SIZE
/ sizeof (*Pte
);
789 // Fill in Page Directory Entries
791 for (Index
= 0; Index
< EFI_PAGE_SIZE
* 4 / sizeof (*Pte
); Index
++) {
792 Pte
[Index
] = (Index
<< 21) | IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
795 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
796 Pages
= (UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (5);
797 GuardPage
= mSmmStackArrayBase
+ EFI_PAGE_SIZE
;
798 Pdpte
= (UINT64
*)PageTable
;
799 for (PageIndex
= Low2MBoundary
; PageIndex
<= High2MBoundary
; PageIndex
+= SIZE_2MB
) {
800 Pte
= (UINT64
*)(UINTN
)(Pdpte
[BitFieldRead32 ((UINT32
)PageIndex
, 30, 31)] & ~(EFI_PAGE_SIZE
- 1));
801 Pte
[BitFieldRead32 ((UINT32
)PageIndex
, 21, 29)] = (UINT64
)Pages
| PAGE_ATTRIBUTE_BITS
;
803 // Fill in Page Table Entries
805 Pte
= (UINT64
*)Pages
;
806 PageAddress
= PageIndex
;
807 for (Index
= 0; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
808 if (PageAddress
== GuardPage
) {
810 // Mark the guard page as non-present
812 Pte
[Index
] = PageAddress
;
813 GuardPage
+= mSmmStackSize
;
814 if (GuardPage
> mSmmStackArrayEnd
) {
818 Pte
[Index
] = PageAddress
| PAGE_ATTRIBUTE_BITS
;
820 PageAddress
+= EFI_PAGE_SIZE
;
822 Pages
+= EFI_PAGE_SIZE
;
826 return (UINT32
)(UINTN
)PageTable
;
830 Set memory cache ability.
832 @param PageTable PageTable Address
833 @param Address Memory Address to change cache ability
834 @param Cacheability Cache ability to set
839 IN UINT64
*PageTable
,
841 IN UINT8 Cacheability
845 VOID
*NewPageTableAddress
;
846 UINT64
*NewPageTable
;
849 ASSERT ((Address
& EFI_PAGE_MASK
) == 0);
851 if (sizeof (UINTN
) == sizeof (UINT64
)) {
852 PTIndex
= (UINTN
)RShiftU64 (Address
, 39) & 0x1ff;
853 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
854 PageTable
= (UINT64
*)(UINTN
)(PageTable
[PTIndex
] & gPhyMask
);
857 PTIndex
= (UINTN
)RShiftU64 (Address
, 30) & 0x1ff;
858 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
859 PageTable
= (UINT64
*)(UINTN
)(PageTable
[PTIndex
] & gPhyMask
);
862 // A perfect implementation should check the original cacheability with the
863 // one being set, and break a 2M page entry into pieces only when they
866 PTIndex
= (UINTN
)RShiftU64 (Address
, 21) & 0x1ff;
867 if ((PageTable
[PTIndex
] & IA32_PG_PS
) != 0) {
869 // Allocate a page from SMRAM
871 NewPageTableAddress
= AllocatePageTableMemory (1);
872 ASSERT (NewPageTableAddress
!= NULL
);
874 NewPageTable
= (UINT64
*)NewPageTableAddress
;
876 for (Index
= 0; Index
< 0x200; Index
++) {
877 NewPageTable
[Index
] = PageTable
[PTIndex
];
878 if ((NewPageTable
[Index
] & IA32_PG_PAT_2M
) != 0) {
879 NewPageTable
[Index
] &= ~((UINT64
)IA32_PG_PAT_2M
);
880 NewPageTable
[Index
] |= (UINT64
)IA32_PG_PAT_4K
;
882 NewPageTable
[Index
] |= (UINT64
)(Index
<< EFI_PAGE_SHIFT
);
885 PageTable
[PTIndex
] = ((UINTN
)NewPageTableAddress
& gPhyMask
) | PAGE_ATTRIBUTE_BITS
;
888 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
889 PageTable
= (UINT64
*)(UINTN
)(PageTable
[PTIndex
] & gPhyMask
);
891 PTIndex
= (UINTN
)RShiftU64 (Address
, 12) & 0x1ff;
892 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
893 PageTable
[PTIndex
] &= ~((UINT64
)((IA32_PG_PAT_4K
| IA32_PG_CD
| IA32_PG_WT
)));
894 PageTable
[PTIndex
] |= (UINT64
)Cacheability
;
898 Schedule a procedure to run on the specified CPU.
900 @param[in] Procedure The address of the procedure to run
901 @param[in] CpuIndex Target CPU Index
902 @param[in, OUT] ProcArguments The parameter to pass to the procedure
903 @param[in] BlockingMode Startup AP in blocking mode or not
905 @retval EFI_INVALID_PARAMETER CpuNumber not valid
906 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
907 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
908 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
909 @retval EFI_SUCCESS The procedure has been successfully scheduled
913 InternalSmmStartupThisAp (
914 IN EFI_AP_PROCEDURE Procedure
,
916 IN OUT VOID
*ProcArguments OPTIONAL
,
917 IN BOOLEAN BlockingMode
920 if (CpuIndex
>= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
) {
921 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) >= gSmmCpuPrivate->SmmCoreEntryContext.NumberOfCpus(%d)\n", CpuIndex
, gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
));
922 return EFI_INVALID_PARAMETER
;
924 if (CpuIndex
== gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
) {
925 DEBUG((DEBUG_ERROR
, "CpuIndex(%d) == gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu\n", CpuIndex
));
926 return EFI_INVALID_PARAMETER
;
928 if (!(*(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
))) {
929 if (mSmmMpSyncData
->EffectiveSyncMode
== SmmCpuSyncModeTradition
) {
930 DEBUG((DEBUG_ERROR
, "!mSmmMpSyncData->CpuData[%d].Present\n", CpuIndex
));
932 return EFI_INVALID_PARAMETER
;
934 if (gSmmCpuPrivate
->Operation
[CpuIndex
] == SmmCpuRemove
) {
935 if (!FeaturePcdGet (PcdCpuHotPlugSupport
)) {
936 DEBUG((DEBUG_ERROR
, "gSmmCpuPrivate->Operation[%d] == SmmCpuRemove\n", CpuIndex
));
938 return EFI_INVALID_PARAMETER
;
942 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
944 if (!AcquireSpinLockOrFail (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)) {
945 DEBUG((DEBUG_ERROR
, "mSmmMpSyncData->CpuData[%d].Busy\n", CpuIndex
));
946 return EFI_INVALID_PARAMETER
;
950 mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
= Procedure
;
951 mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
= ProcArguments
;
952 ReleaseSemaphore (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
955 AcquireSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
956 ReleaseSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
962 Schedule a procedure to run on the specified CPU in blocking mode.
964 @param[in] Procedure The address of the procedure to run
965 @param[in] CpuIndex Target CPU Index
966 @param[in, out] ProcArguments The parameter to pass to the procedure
968 @retval EFI_INVALID_PARAMETER CpuNumber not valid
969 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
970 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
971 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
972 @retval EFI_SUCCESS The procedure has been successfully scheduled
977 SmmBlockingStartupThisAp (
978 IN EFI_AP_PROCEDURE Procedure
,
980 IN OUT VOID
*ProcArguments OPTIONAL
983 return InternalSmmStartupThisAp(Procedure
, CpuIndex
, ProcArguments
, TRUE
);
987 Schedule a procedure to run on the specified CPU.
989 @param Procedure The address of the procedure to run
990 @param CpuIndex Target CPU Index
991 @param ProcArguments The parameter to pass to the procedure
993 @retval EFI_INVALID_PARAMETER CpuNumber not valid
994 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
995 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
996 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
997 @retval EFI_SUCCESS The procedure has been successfully scheduled
1003 IN EFI_AP_PROCEDURE Procedure
,
1005 IN OUT VOID
*ProcArguments OPTIONAL
1008 return InternalSmmStartupThisAp(Procedure
, CpuIndex
, ProcArguments
, FeaturePcdGet (PcdCpuSmmBlockStartupThisAp
));
1012 This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
1013 They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
1015 NOTE: It might not be appreciated in runtime since it might
1016 conflict with OS debugging facilities. Turn them off in RELEASE.
1018 @param CpuIndex CPU Index
1027 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1029 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1030 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1031 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1032 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1033 AsmWriteDr6 (CpuSaveState
->x86
._DR6
);
1034 AsmWriteDr7 (CpuSaveState
->x86
._DR7
);
1036 AsmWriteDr6 ((UINTN
)CpuSaveState
->x64
._DR6
);
1037 AsmWriteDr7 ((UINTN
)CpuSaveState
->x64
._DR7
);
1043 This function restores DR6 & DR7 to SMM save state.
1045 NOTE: It might not be appreciated in runtime since it might
1046 conflict with OS debugging facilities. Turn them off in RELEASE.
1048 @param CpuIndex CPU Index
1057 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
1059 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
1060 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1061 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
1062 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
1063 CpuSaveState
->x86
._DR7
= (UINT32
)AsmReadDr7 ();
1064 CpuSaveState
->x86
._DR6
= (UINT32
)AsmReadDr6 ();
1066 CpuSaveState
->x64
._DR7
= AsmReadDr7 ();
1067 CpuSaveState
->x64
._DR6
= AsmReadDr6 ();
1073 C function for SMI entry, each processor comes here upon SMI trigger.
1075 @param CpuIndex CPU Index
1087 BOOLEAN BspInProgress
;
1091 ASSERT(CpuIndex
< mMaxNumberOfCpus
);
1094 // Save Cr2 because Page Fault exception in SMM may override its value
1096 Cr2
= AsmReadCr2 ();
1099 // Perform CPU specific entry hooks
1101 SmmCpuFeaturesRendezvousEntry (CpuIndex
);
1104 // Determine if this is a valid SMI
1106 ValidSmi
= PlatformValidSmi();
1109 // Determine if BSP has been already in progress. Note this must be checked after
1110 // ValidSmi because BSP may clear a valid SMI source after checking in.
1112 BspInProgress
= *mSmmMpSyncData
->InsideSmm
;
1114 if (!BspInProgress
&& !ValidSmi
) {
1116 // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not
1117 // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI
1118 // status had been cleared by BSP and an existing SMI run has almost ended. (Note
1119 // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there
1120 // is nothing we need to do.
1125 // Signal presence of this processor
1127 if (ReleaseSemaphore (mSmmMpSyncData
->Counter
) == 0) {
1129 // BSP has already ended the synchronization, so QUIT!!!
1133 // Wait for BSP's signal to finish SMI
1135 while (*mSmmMpSyncData
->AllCpusInSync
) {
1142 // The BUSY lock is initialized to Released state.
1143 // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.
1144 // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
1145 // after AP's present flag is detected.
1147 InitializeSpinLock (mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1150 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1151 ActivateSmmProfile (CpuIndex
);
1154 if (BspInProgress
) {
1156 // BSP has been elected. Follow AP path, regardless of ValidSmi flag
1157 // as BSP may have cleared the SMI status
1159 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1162 // We have a valid SMI
1169 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1170 if (!mSmmMpSyncData
->SwitchBsp
|| mSmmMpSyncData
->CandidateBsp
[CpuIndex
]) {
1172 // Call platform hook to do BSP election
1174 Status
= PlatformSmmBspElection (&IsBsp
);
1175 if (EFI_SUCCESS
== Status
) {
1177 // Platform hook determines successfully
1180 mSmmMpSyncData
->BspIndex
= (UINT32
)CpuIndex
;
1184 // Platform hook fails to determine, use default BSP election method
1186 InterlockedCompareExchange32 (
1187 (UINT32
*)&mSmmMpSyncData
->BspIndex
,
1196 // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP
1198 if (mSmmMpSyncData
->BspIndex
== CpuIndex
) {
1201 // Clear last request for SwitchBsp.
1203 if (mSmmMpSyncData
->SwitchBsp
) {
1204 mSmmMpSyncData
->SwitchBsp
= FALSE
;
1205 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1206 mSmmMpSyncData
->CandidateBsp
[Index
] = FALSE
;
1210 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1211 SmmProfileRecordSmiNum ();
1215 // BSP Handler is always called with a ValidSmi == TRUE
1217 BSPHandler (CpuIndex
, mSmmMpSyncData
->EffectiveSyncMode
);
1219 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1223 ASSERT (*mSmmMpSyncData
->CpuData
[CpuIndex
].Run
== 0);
1226 // Wait for BSP's signal to exit SMI
1228 while (*mSmmMpSyncData
->AllCpusInSync
) {
1234 SmmCpuFeaturesRendezvousExit (CpuIndex
);
1242 Allocate buffer for all semaphores and spin locks.
1246 InitializeSmmCpuSemaphores (
1250 UINTN ProcessorCount
;
1252 UINTN GlobalSemaphoresSize
;
1253 UINTN CpuSemaphoresSize
;
1254 UINTN MsrSemahporeSize
;
1255 UINTN SemaphoreSize
;
1257 UINTN
*SemaphoreBlock
;
1258 UINTN SemaphoreAddr
;
1260 SemaphoreSize
= GetSpinLockProperties ();
1261 ProcessorCount
= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1262 GlobalSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_GLOBAL
) / sizeof (VOID
*)) * SemaphoreSize
;
1263 CpuSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_CPU
) / sizeof (VOID
*)) * ProcessorCount
* SemaphoreSize
;
1264 MsrSemahporeSize
= MSR_SPIN_LOCK_INIT_NUM
* SemaphoreSize
;
1265 TotalSize
= GlobalSemaphoresSize
+ CpuSemaphoresSize
+ MsrSemahporeSize
;
1266 DEBUG((EFI_D_INFO
, "One Semaphore Size = 0x%x\n", SemaphoreSize
));
1267 DEBUG((EFI_D_INFO
, "Total Semaphores Size = 0x%x\n", TotalSize
));
1268 Pages
= EFI_SIZE_TO_PAGES (TotalSize
);
1269 SemaphoreBlock
= AllocatePages (Pages
);
1270 ASSERT (SemaphoreBlock
!= NULL
);
1271 ZeroMem (SemaphoreBlock
, TotalSize
);
1273 SemaphoreAddr
= (UINTN
)SemaphoreBlock
;
1274 mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
= (UINT32
*)SemaphoreAddr
;
1275 SemaphoreAddr
+= SemaphoreSize
;
1276 mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
= (BOOLEAN
*)SemaphoreAddr
;
1277 SemaphoreAddr
+= SemaphoreSize
;
1278 mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
= (BOOLEAN
*)SemaphoreAddr
;
1279 SemaphoreAddr
+= SemaphoreSize
;
1280 mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
= (SPIN_LOCK
*)SemaphoreAddr
;
1281 SemaphoreAddr
+= SemaphoreSize
;
1282 mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
1283 = (SPIN_LOCK
*)SemaphoreAddr
;
1284 SemaphoreAddr
+= SemaphoreSize
;
1285 mSmmCpuSemaphores
.SemaphoreGlobal
.MemoryMappedLock
1286 = (SPIN_LOCK
*)SemaphoreAddr
;
1288 SemaphoreAddr
= (UINTN
)SemaphoreBlock
+ GlobalSemaphoresSize
;
1289 mSmmCpuSemaphores
.SemaphoreCpu
.Busy
= (SPIN_LOCK
*)SemaphoreAddr
;
1290 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1291 mSmmCpuSemaphores
.SemaphoreCpu
.Run
= (UINT32
*)SemaphoreAddr
;
1292 SemaphoreAddr
+= ProcessorCount
* SemaphoreSize
;
1293 mSmmCpuSemaphores
.SemaphoreCpu
.Present
= (BOOLEAN
*)SemaphoreAddr
;
1295 SemaphoreAddr
= (UINTN
)SemaphoreBlock
+ GlobalSemaphoresSize
+ CpuSemaphoresSize
;
1296 mSmmCpuSemaphores
.SemaphoreMsr
.Msr
= (SPIN_LOCK
*)SemaphoreAddr
;
1297 mSmmCpuSemaphores
.SemaphoreMsr
.AvailableCounter
=
1298 ((UINTN
)SemaphoreBlock
+ Pages
* SIZE_4KB
- SemaphoreAddr
) / SemaphoreSize
;
1299 ASSERT (mSmmCpuSemaphores
.SemaphoreMsr
.AvailableCounter
>= MSR_SPIN_LOCK_INIT_NUM
);
1301 mPFLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
;
1302 mConfigSmmCodeAccessCheckLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
;
1303 mMemoryMappedLock
= mSmmCpuSemaphores
.SemaphoreGlobal
.MemoryMappedLock
;
1305 mSemaphoreSize
= SemaphoreSize
;
1309 Initialize un-cacheable data.
1314 InitializeMpSyncData (
1320 if (mSmmMpSyncData
!= NULL
) {
1322 // mSmmMpSyncDataSize includes one structure of SMM_DISPATCHER_MP_SYNC_DATA, one
1323 // CpuData array of SMM_CPU_DATA_BLOCK and one CandidateBsp array of BOOLEAN.
1325 ZeroMem (mSmmMpSyncData
, mSmmMpSyncDataSize
);
1326 mSmmMpSyncData
->CpuData
= (SMM_CPU_DATA_BLOCK
*)((UINT8
*)mSmmMpSyncData
+ sizeof (SMM_DISPATCHER_MP_SYNC_DATA
));
1327 mSmmMpSyncData
->CandidateBsp
= (BOOLEAN
*)(mSmmMpSyncData
->CpuData
+ gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1328 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1330 // Enable BSP election by setting BspIndex to -1
1332 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
1334 mSmmMpSyncData
->EffectiveSyncMode
= mCpuSmmSyncMode
;
1336 mSmmMpSyncData
->Counter
= mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
;
1337 mSmmMpSyncData
->InsideSmm
= mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
;
1338 mSmmMpSyncData
->AllCpusInSync
= mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
;
1339 ASSERT (mSmmMpSyncData
->Counter
!= NULL
&& mSmmMpSyncData
->InsideSmm
!= NULL
&&
1340 mSmmMpSyncData
->AllCpusInSync
!= NULL
);
1341 *mSmmMpSyncData
->Counter
= 0;
1342 *mSmmMpSyncData
->InsideSmm
= FALSE
;
1343 *mSmmMpSyncData
->AllCpusInSync
= FALSE
;
1345 for (CpuIndex
= 0; CpuIndex
< gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
; CpuIndex
++) {
1346 mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
=
1347 (SPIN_LOCK
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Busy
+ mSemaphoreSize
* CpuIndex
);
1348 mSmmMpSyncData
->CpuData
[CpuIndex
].Run
=
1349 (UINT32
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Run
+ mSemaphoreSize
* CpuIndex
);
1350 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
=
1351 (BOOLEAN
*)((UINTN
)mSmmCpuSemaphores
.SemaphoreCpu
.Present
+ mSemaphoreSize
* CpuIndex
);
1352 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
) = 0;
1353 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Run
) = 0;
1354 *(mSmmMpSyncData
->CpuData
[CpuIndex
].Present
) = FALSE
;
1360 Initialize global data for MP synchronization.
1362 @param Stacks Base address of SMI stack buffer for all processors.
1363 @param StackSize Stack size for each processor in SMM.
1367 InitializeMpServiceData (
1374 UINT8
*GdtTssTables
;
1375 UINTN GdtTableStepSize
;
1378 // Allocate memory for all locks and semaphores
1380 InitializeSmmCpuSemaphores ();
1383 // Initialize mSmmMpSyncData
1385 mSmmMpSyncDataSize
= sizeof (SMM_DISPATCHER_MP_SYNC_DATA
) +
1386 (sizeof (SMM_CPU_DATA_BLOCK
) + sizeof (BOOLEAN
)) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1387 mSmmMpSyncData
= (SMM_DISPATCHER_MP_SYNC_DATA
*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize
));
1388 ASSERT (mSmmMpSyncData
!= NULL
);
1389 mCpuSmmSyncMode
= (SMM_CPU_SYNC_MODE
)PcdGet8 (PcdCpuSmmSyncMode
);
1390 InitializeMpSyncData ();
1393 // Initialize physical address mask
1394 // NOTE: Physical memory above virtual address limit is not supported !!!
1396 AsmCpuid (0x80000008, (UINT32
*)&Index
, NULL
, NULL
, NULL
);
1397 gPhyMask
= LShiftU64 (1, (UINT8
)Index
) - 1;
1398 gPhyMask
&= (1ull << 48) - EFI_PAGE_SIZE
;
1401 // Create page tables
1403 Cr3
= SmmInitPageTable ();
1405 GdtTssTables
= InitGdt (Cr3
, &GdtTableStepSize
);
1408 // Install SMI handler for each CPU
1410 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1413 (UINT32
)mCpuHotPlugData
.SmBase
[Index
],
1414 (VOID
*)((UINTN
)Stacks
+ (StackSize
* Index
)),
1416 (UINTN
)(GdtTssTables
+ GdtTableStepSize
* Index
),
1417 gcSmiGdtr
.Limit
+ 1,
1419 gcSmiIdtr
.Limit
+ 1,
1425 // Record current MTRR settings
1427 ZeroMem (&gSmiMtrrs
, sizeof (gSmiMtrrs
));
1428 MtrrGetAllMtrrs (&gSmiMtrrs
);
1435 Register the SMM Foundation entry point.
1437 @param This Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance
1438 @param SmmEntryPoint SMM Foundation EntryPoint
1440 @retval EFI_SUCCESS Successfully to register SMM foundation entry point
1446 IN CONST EFI_SMM_CONFIGURATION_PROTOCOL
*This
,
1447 IN EFI_SMM_ENTRY_POINT SmmEntryPoint
1451 // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.
1453 gSmmCpuPrivate
->SmmCoreEntry
= SmmEntryPoint
;