2 SMM MP service implementation
4 Copyright (c) 2009 - 2016, Intel Corporation. All rights reserved.<BR>
5 This program and the accompanying materials
6 are licensed and made available under the terms and conditions of the BSD License
7 which accompanies this distribution. The full text of the license may be found at
8 http://opensource.org/licenses/bsd-license.php
10 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
15 #include "PiSmmCpuDxeSmm.h"
18 // Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
20 UINT64 gSmiMtrrs
[MTRR_NUMBER_OF_FIXED_MTRR
+ 2 * MTRR_NUMBER_OF_VARIABLE_MTRR
+ 1];
22 SMM_DISPATCHER_MP_SYNC_DATA
*mSmmMpSyncData
= NULL
;
23 UINTN mSmmMpSyncDataSize
;
26 Performs an atomic compare exchange operation to get semaphore.
27 The compare exchange operation must be performed using
30 @param Sem IN: 32-bit unsigned integer
31 OUT: original integer - 1
32 @return Original integer - 1
37 IN OUT
volatile UINT32
*Sem
44 } while (Value
== 0 ||
45 InterlockedCompareExchange32 (
55 Performs an atomic compare exchange operation to release semaphore.
56 The compare exchange operation must be performed using
59 @param Sem IN: 32-bit unsigned integer
60 OUT: original integer + 1
61 @return Original integer + 1
66 IN OUT
volatile UINT32
*Sem
73 } while (Value
+ 1 != 0 &&
74 InterlockedCompareExchange32 (
83 Performs an atomic compare exchange operation to lock semaphore.
84 The compare exchange operation must be performed using
87 @param Sem IN: 32-bit unsigned integer
89 @return Original integer
94 IN OUT
volatile UINT32
*Sem
101 } while (InterlockedCompareExchange32 (
109 Wait all APs to performs an atomic compare exchange operation to release semaphore.
111 @param NumberOfAPs AP number
121 BspIndex
= mSmmMpSyncData
->BspIndex
;
122 while (NumberOfAPs
-- > 0) {
123 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
128 Performs an atomic compare exchange operation to release semaphore
140 BspIndex
= mSmmMpSyncData
->BspIndex
;
141 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
142 if (Index
!= BspIndex
&& mSmmMpSyncData
->CpuData
[Index
].Present
) {
143 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[Index
].Run
);
149 Checks if all CPUs (with certain exceptions) have checked in for this SMI run
151 @param Exceptions CPU Arrival exception flags.
153 @retval TRUE if all CPUs the have checked in.
154 @retval FALSE if at least one Normal AP hasn't checked in.
158 AllCpusInSmmWithExceptions (
159 SMM_CPU_ARRIVAL_EXCEPTIONS Exceptions
163 SMM_CPU_DATA_BLOCK
*CpuData
;
164 EFI_PROCESSOR_INFORMATION
*ProcessorInfo
;
166 ASSERT (mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
168 if (mSmmMpSyncData
->Counter
== mNumberOfCpus
) {
172 CpuData
= mSmmMpSyncData
->CpuData
;
173 ProcessorInfo
= gSmmCpuPrivate
->ProcessorInfo
;
174 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
175 if (!CpuData
[Index
].Present
&& ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
176 if (((Exceptions
& ARRIVAL_EXCEPTION_DELAYED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmDelayed
) != 0) {
179 if (((Exceptions
& ARRIVAL_EXCEPTION_BLOCKED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmBlocked
) != 0) {
182 if (((Exceptions
& ARRIVAL_EXCEPTION_SMI_DISABLED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmEnable
) != 0) {
195 Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
196 entering SMM, except SMI disabled APs.
200 SmmWaitForApArrival (
207 ASSERT (mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
210 // Platform implementor should choose a timeout value appropriately:
211 // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note
212 // the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.
213 // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI
214 // and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will
215 // be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the
216 // SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.
217 // - The timeout value must be longer than longest possible IO operation in the system
221 // Sync with APs 1st timeout
223 for (Timer
= StartSyncTimer ();
224 !IsSyncTimerTimeout (Timer
) &&
225 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
231 // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,
233 // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running
234 // normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they
235 // enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode
236 // work while SMI handling is on-going.
237 // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.
238 // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state
239 // will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal
240 // mode work while SMI handling is on-going.
241 // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:
242 // - In traditional flow, SMI disabling is discouraged.
243 // - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
244 // In both cases, adding SMI-disabling checking code increases overhead.
246 if (mSmmMpSyncData
->Counter
< mNumberOfCpus
) {
248 // Send SMI IPIs to bring outside processors in
250 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
251 if (!mSmmMpSyncData
->CpuData
[Index
].Present
&& gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
252 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
);
257 // Sync with APs 2nd timeout.
259 for (Timer
= StartSyncTimer ();
260 !IsSyncTimerTimeout (Timer
) &&
261 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
272 Replace OS MTRR's with SMI MTRR's.
274 @param CpuIndex Processor Index
282 PROCESSOR_SMM_DESCRIPTOR
*Psd
;
284 MTRR_SETTINGS
*BiosMtrr
;
286 Psd
= (PROCESSOR_SMM_DESCRIPTOR
*)(mCpuHotPlugData
.SmBase
[CpuIndex
] + SMM_PSD_OFFSET
);
287 SmiMtrrs
= (UINT64
*)(UINTN
)Psd
->MtrrBaseMaskPtr
;
289 SmmCpuFeaturesDisableSmrr ();
292 // Replace all MTRRs registers
294 BiosMtrr
= (MTRR_SETTINGS
*)SmiMtrrs
;
295 MtrrSetAllMtrrs(BiosMtrr
);
301 @param CpuIndex BSP processor Index
302 @param SyncMode SMM MP sync mode
308 IN SMM_CPU_SYNC_MODE SyncMode
314 BOOLEAN ClearTopLevelSmiResult
;
317 ASSERT (CpuIndex
== mSmmMpSyncData
->BspIndex
);
321 // Flag BSP's presence
323 mSmmMpSyncData
->InsideSmm
= TRUE
;
326 // Initialize Debug Agent to start source level debug in BSP handler
328 InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI
, NULL
, NULL
);
331 // Mark this processor's presence
333 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
= TRUE
;
336 // Clear platform top level SMI status bit before calling SMI handlers. If
337 // we cleared it after SMI handlers are run, we would miss the SMI that
338 // occurs after SMI handlers are done and before SMI status bit is cleared.
340 ClearTopLevelSmiResult
= ClearTopLevelSmiStatus();
341 ASSERT (ClearTopLevelSmiResult
== TRUE
);
344 // Set running processor index
346 gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
= CpuIndex
;
349 // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
351 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
354 // Wait for APs to arrive
356 SmmWaitForApArrival();
359 // Lock the counter down and retrieve the number of APs
361 mSmmMpSyncData
->AllCpusInSync
= TRUE
;
362 ApCount
= LockdownSemaphore (&mSmmMpSyncData
->Counter
) - 1;
365 // Wait for all APs to get ready for programming MTRRs
367 WaitForAllAPs (ApCount
);
369 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
371 // Signal all APs it's time for backup MTRRs
376 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
377 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
378 // to a large enough value to avoid this situation.
379 // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
380 // We do the backup first and then set MTRR to avoid race condition for threads
383 MtrrGetAllMtrrs(&Mtrrs
);
386 // Wait for all APs to complete their MTRR saving
388 WaitForAllAPs (ApCount
);
391 // Let all processors program SMM MTRRs together
396 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
397 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
398 // to a large enough value to avoid this situation.
400 ReplaceOSMtrrs (CpuIndex
);
403 // Wait for all APs to complete their MTRR programming
405 WaitForAllAPs (ApCount
);
410 // The BUSY lock is initialized to Acquired state
412 AcquireSpinLockOrFail (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
415 // Perform the pre tasks
420 // Invoke SMM Foundation EntryPoint with the processor information context.
422 gSmmCpuPrivate
->SmmCoreEntry (&gSmmCpuPrivate
->SmmCoreEntryContext
);
425 // Make sure all APs have completed their pending none-block tasks
427 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
428 if (Index
!= CpuIndex
&& mSmmMpSyncData
->CpuData
[Index
].Present
) {
429 AcquireSpinLock (&mSmmMpSyncData
->CpuData
[Index
].Busy
);
430 ReleaseSpinLock (&mSmmMpSyncData
->CpuData
[Index
].Busy
);;
435 // Perform the remaining tasks
437 PerformRemainingTasks ();
440 // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
441 // make those APs to exit SMI synchronously. APs which arrive later will be excluded and
442 // will run through freely.
444 if (SyncMode
!= SmmCpuSyncModeTradition
&& !SmmCpuFeaturesNeedConfigureMtrrs()) {
447 // Lock the counter down and retrieve the number of APs
449 mSmmMpSyncData
->AllCpusInSync
= TRUE
;
450 ApCount
= LockdownSemaphore (&mSmmMpSyncData
->Counter
) - 1;
452 // Make sure all APs have their Present flag set
456 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
457 if (mSmmMpSyncData
->CpuData
[Index
].Present
) {
461 if (PresentCount
> ApCount
) {
468 // Notify all APs to exit
470 mSmmMpSyncData
->InsideSmm
= FALSE
;
474 // Wait for all APs to complete their pending tasks
476 WaitForAllAPs (ApCount
);
478 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
480 // Signal APs to restore MTRRs
487 SmmCpuFeaturesReenableSmrr ();
488 MtrrSetAllMtrrs(&Mtrrs
);
491 // Wait for all APs to complete MTRR programming
493 WaitForAllAPs (ApCount
);
497 // Stop source level debug in BSP handler, the code below will not be
500 InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI
, NULL
, NULL
);
503 // Signal APs to Reset states/semaphore for this processor
508 // Perform pending operations for hot-plug
513 // Clear the Present flag of BSP
515 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
= FALSE
;
518 // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
519 // WaitForAllAps does not depend on the Present flag.
521 WaitForAllAPs (ApCount
);
524 // Reset BspIndex to -1, meaning BSP has not been elected.
526 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
527 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
531 // Allow APs to check in from this point on
533 mSmmMpSyncData
->Counter
= 0;
534 mSmmMpSyncData
->AllCpusInSync
= FALSE
;
540 @param CpuIndex AP processor Index.
541 @param ValidSmi Indicates that current SMI is a valid SMI or not.
542 @param SyncMode SMM MP sync mode.
549 IN SMM_CPU_SYNC_MODE SyncMode
559 for (Timer
= StartSyncTimer ();
560 !IsSyncTimerTimeout (Timer
) &&
561 !mSmmMpSyncData
->InsideSmm
;
566 if (!mSmmMpSyncData
->InsideSmm
) {
568 // BSP timeout in the first round
570 if (mSmmMpSyncData
->BspIndex
!= -1) {
572 // BSP Index is known
574 BspIndex
= mSmmMpSyncData
->BspIndex
;
575 ASSERT (CpuIndex
!= BspIndex
);
578 // Send SMI IPI to bring BSP in
580 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[BspIndex
].ProcessorId
);
583 // Now clock BSP for the 2nd time
585 for (Timer
= StartSyncTimer ();
586 !IsSyncTimerTimeout (Timer
) &&
587 !mSmmMpSyncData
->InsideSmm
;
592 if (!mSmmMpSyncData
->InsideSmm
) {
594 // Give up since BSP is unable to enter SMM
595 // and signal the completion of this AP
596 WaitForSemaphore (&mSmmMpSyncData
->Counter
);
601 // Don't know BSP index. Give up without sending IPI to BSP.
603 WaitForSemaphore (&mSmmMpSyncData
->Counter
);
611 BspIndex
= mSmmMpSyncData
->BspIndex
;
612 ASSERT (CpuIndex
!= BspIndex
);
615 // Mark this processor's presence
617 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
= TRUE
;
619 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
621 // Notify BSP of arrival at this point
623 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
626 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
628 // Wait for the signal from BSP to backup MTRRs
630 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
635 MtrrGetAllMtrrs(&Mtrrs
);
638 // Signal BSP the completion of this AP
640 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
643 // Wait for BSP's signal to program MTRRs
645 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
648 // Replace OS MTRRs with SMI MTRRs
650 ReplaceOSMtrrs (CpuIndex
);
653 // Signal BSP the completion of this AP
655 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
660 // Wait for something to happen
662 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
665 // Check if BSP wants to exit SMM
667 if (!mSmmMpSyncData
->InsideSmm
) {
672 // BUSY should be acquired by SmmStartupThisAp()
675 !AcquireSpinLockOrFail (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)
679 // Invoke the scheduled procedure
681 (*mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
) (
682 (VOID
*)mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
688 ReleaseSpinLock (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
691 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
693 // Notify BSP the readiness of this AP to program MTRRs
695 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
698 // Wait for the signal from BSP to program MTRRs
700 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
705 SmmCpuFeaturesReenableSmrr ();
706 MtrrSetAllMtrrs(&Mtrrs
);
710 // Notify BSP the readiness of this AP to Reset states/semaphore for this processor
712 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
715 // Wait for the signal from BSP to Reset states/semaphore for this processor
717 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
720 // Reset states/semaphore for this processor
722 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
= FALSE
;
725 // Notify BSP the readiness of this AP to exit SMM
727 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
732 Create 4G PageTable in SMRAM.
734 @param ExtraPages Additional page numbers besides for 4G memory
735 @param Is32BitPageTable Whether the page table is 32-bit PAE
736 @return PageTable Address
742 IN BOOLEAN Is32BitPageTable
750 UINTN High2MBoundary
;
760 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
762 // Add one more page for known good stack, then find the lower 2MB aligned address.
764 Low2MBoundary
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
) & ~(SIZE_2MB
-1);
766 // Add two more pages for known good stack and stack guard page,
767 // then find the lower 2MB aligned address.
769 High2MBoundary
= (mSmmStackArrayEnd
- mSmmStackSize
+ EFI_PAGE_SIZE
* 2) & ~(SIZE_2MB
-1);
770 PagesNeeded
= ((High2MBoundary
- Low2MBoundary
) / SIZE_2MB
) + 1;
773 // Allocate the page table
775 PageTable
= AllocatePageTableMemory (ExtraPages
+ 5 + PagesNeeded
);
776 ASSERT (PageTable
!= NULL
);
778 PageTable
= (VOID
*)((UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (ExtraPages
));
779 Pte
= (UINT64
*)PageTable
;
782 // Zero out all page table entries first
784 ZeroMem (Pte
, EFI_PAGES_TO_SIZE (1));
787 // Set Page Directory Pointers
789 for (Index
= 0; Index
< 4; Index
++) {
790 Pte
[Index
] = (UINTN
)PageTable
+ EFI_PAGE_SIZE
* (Index
+ 1) + (Is32BitPageTable
? IA32_PAE_PDPTE_ATTRIBUTE_BITS
: PAGE_ATTRIBUTE_BITS
);
792 Pte
+= EFI_PAGE_SIZE
/ sizeof (*Pte
);
795 // Fill in Page Directory Entries
797 for (Index
= 0; Index
< EFI_PAGE_SIZE
* 4 / sizeof (*Pte
); Index
++) {
798 Pte
[Index
] = (Index
<< 21) | IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
801 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
802 Pages
= (UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (5);
803 GuardPage
= mSmmStackArrayBase
+ EFI_PAGE_SIZE
;
804 Pdpte
= (UINT64
*)PageTable
;
805 for (PageIndex
= Low2MBoundary
; PageIndex
<= High2MBoundary
; PageIndex
+= SIZE_2MB
) {
806 Pte
= (UINT64
*)(UINTN
)(Pdpte
[BitFieldRead32 ((UINT32
)PageIndex
, 30, 31)] & ~(EFI_PAGE_SIZE
- 1));
807 Pte
[BitFieldRead32 ((UINT32
)PageIndex
, 21, 29)] = (UINT64
)Pages
| PAGE_ATTRIBUTE_BITS
;
809 // Fill in Page Table Entries
811 Pte
= (UINT64
*)Pages
;
812 PageAddress
= PageIndex
;
813 for (Index
= 0; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
814 if (PageAddress
== GuardPage
) {
816 // Mark the guard page as non-present
818 Pte
[Index
] = PageAddress
;
819 GuardPage
+= mSmmStackSize
;
820 if (GuardPage
> mSmmStackArrayEnd
) {
824 Pte
[Index
] = PageAddress
| PAGE_ATTRIBUTE_BITS
;
826 PageAddress
+= EFI_PAGE_SIZE
;
828 Pages
+= EFI_PAGE_SIZE
;
832 return (UINT32
)(UINTN
)PageTable
;
836 Set memory cache ability.
838 @param PageTable PageTable Address
839 @param Address Memory Address to change cache ability
840 @param Cacheability Cache ability to set
845 IN UINT64
*PageTable
,
847 IN UINT8 Cacheability
851 VOID
*NewPageTableAddress
;
852 UINT64
*NewPageTable
;
855 ASSERT ((Address
& EFI_PAGE_MASK
) == 0);
857 if (sizeof (UINTN
) == sizeof (UINT64
)) {
858 PTIndex
= (UINTN
)RShiftU64 (Address
, 39) & 0x1ff;
859 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
860 PageTable
= (UINT64
*)(UINTN
)(PageTable
[PTIndex
] & gPhyMask
);
863 PTIndex
= (UINTN
)RShiftU64 (Address
, 30) & 0x1ff;
864 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
865 PageTable
= (UINT64
*)(UINTN
)(PageTable
[PTIndex
] & gPhyMask
);
868 // A perfect implementation should check the original cacheability with the
869 // one being set, and break a 2M page entry into pieces only when they
872 PTIndex
= (UINTN
)RShiftU64 (Address
, 21) & 0x1ff;
873 if ((PageTable
[PTIndex
] & IA32_PG_PS
) != 0) {
875 // Allocate a page from SMRAM
877 NewPageTableAddress
= AllocatePageTableMemory (1);
878 ASSERT (NewPageTableAddress
!= NULL
);
880 NewPageTable
= (UINT64
*)NewPageTableAddress
;
882 for (Index
= 0; Index
< 0x200; Index
++) {
883 NewPageTable
[Index
] = PageTable
[PTIndex
];
884 if ((NewPageTable
[Index
] & IA32_PG_PAT_2M
) != 0) {
885 NewPageTable
[Index
] &= ~((UINT64
)IA32_PG_PAT_2M
);
886 NewPageTable
[Index
] |= (UINT64
)IA32_PG_PAT_4K
;
888 NewPageTable
[Index
] |= (UINT64
)(Index
<< EFI_PAGE_SHIFT
);
891 PageTable
[PTIndex
] = ((UINTN
)NewPageTableAddress
& gPhyMask
) | PAGE_ATTRIBUTE_BITS
;
894 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
895 PageTable
= (UINT64
*)(UINTN
)(PageTable
[PTIndex
] & gPhyMask
);
897 PTIndex
= (UINTN
)RShiftU64 (Address
, 12) & 0x1ff;
898 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
899 PageTable
[PTIndex
] &= ~((UINT64
)((IA32_PG_PAT_4K
| IA32_PG_CD
| IA32_PG_WT
)));
900 PageTable
[PTIndex
] |= (UINT64
)Cacheability
;
905 Schedule a procedure to run on the specified CPU.
907 @param Procedure The address of the procedure to run
908 @param CpuIndex Target CPU Index
909 @param ProcArguments The parameter to pass to the procedure
911 @retval EFI_INVALID_PARAMETER CpuNumber not valid
912 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
913 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
914 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
915 @retval EFI_SUCCESS The procedure has been successfully scheduled
921 IN EFI_AP_PROCEDURE Procedure
,
923 IN OUT VOID
*ProcArguments OPTIONAL
926 if (CpuIndex
>= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
||
927 CpuIndex
== gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
||
928 !mSmmMpSyncData
->CpuData
[CpuIndex
].Present
||
929 gSmmCpuPrivate
->Operation
[CpuIndex
] == SmmCpuRemove
||
930 !AcquireSpinLockOrFail (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)) {
931 return EFI_INVALID_PARAMETER
;
934 mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
= Procedure
;
935 mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
= ProcArguments
;
936 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
938 if (FeaturePcdGet (PcdCpuSmmBlockStartupThisAp
)) {
939 AcquireSpinLock (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
940 ReleaseSpinLock (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
946 This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
947 They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
949 NOTE: It might not be appreciated in runtime since it might
950 conflict with OS debugging facilities. Turn them off in RELEASE.
952 @param CpuIndex CPU Index
961 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
963 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
964 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
965 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
966 AsmWriteDr6 (CpuSaveState
->x86
._DR6
);
967 AsmWriteDr7 (CpuSaveState
->x86
._DR7
);
969 AsmWriteDr6 ((UINTN
)CpuSaveState
->x64
._DR6
);
970 AsmWriteDr7 ((UINTN
)CpuSaveState
->x64
._DR7
);
976 This function restores DR6 & DR7 to SMM save state.
978 NOTE: It might not be appreciated in runtime since it might
979 conflict with OS debugging facilities. Turn them off in RELEASE.
981 @param CpuIndex CPU Index
990 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
992 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
993 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
994 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
995 CpuSaveState
->x86
._DR7
= (UINT32
)AsmReadDr7 ();
996 CpuSaveState
->x86
._DR6
= (UINT32
)AsmReadDr6 ();
998 CpuSaveState
->x64
._DR7
= AsmReadDr7 ();
999 CpuSaveState
->x64
._DR6
= AsmReadDr6 ();
1005 C function for SMI entry, each processor comes here upon SMI trigger.
1007 @param CpuIndex CPU Index
1019 BOOLEAN BspInProgress
;
1022 BOOLEAN XdDisableFlag
;
1023 MSR_IA32_MISC_ENABLE_REGISTER MiscEnableMsr
;
1026 // Save Cr2 because Page Fault exception in SMM may override its value
1028 Cr2
= AsmReadCr2 ();
1031 // Perform CPU specific entry hooks
1033 SmmCpuFeaturesRendezvousEntry (CpuIndex
);
1036 // Determine if this is a valid SMI
1038 ValidSmi
= PlatformValidSmi();
1041 // Determine if BSP has been already in progress. Note this must be checked after
1042 // ValidSmi because BSP may clear a valid SMI source after checking in.
1044 BspInProgress
= mSmmMpSyncData
->InsideSmm
;
1046 if (!BspInProgress
&& !ValidSmi
) {
1048 // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not
1049 // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI
1050 // status had been cleared by BSP and an existing SMI run has almost ended. (Note
1051 // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there
1052 // is nothing we need to do.
1057 // Signal presence of this processor
1059 if (ReleaseSemaphore (&mSmmMpSyncData
->Counter
) == 0) {
1061 // BSP has already ended the synchronization, so QUIT!!!
1065 // Wait for BSP's signal to finish SMI
1067 while (mSmmMpSyncData
->AllCpusInSync
) {
1074 // The BUSY lock is initialized to Released state.
1075 // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.
1076 // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
1077 // after AP's present flag is detected.
1079 InitializeSpinLock (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1085 XdDisableFlag
= FALSE
;
1087 MiscEnableMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_MISC_ENABLE
);
1088 if (MiscEnableMsr
.Bits
.XD
== 1) {
1089 XdDisableFlag
= TRUE
;
1090 MiscEnableMsr
.Bits
.XD
= 0;
1091 AsmWriteMsr64 (MSR_IA32_MISC_ENABLE
, MiscEnableMsr
.Uint64
);
1096 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1097 ActivateSmmProfile (CpuIndex
);
1100 if (BspInProgress
) {
1102 // BSP has been elected. Follow AP path, regardless of ValidSmi flag
1103 // as BSP may have cleared the SMI status
1105 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1108 // We have a valid SMI
1115 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1116 if (!mSmmMpSyncData
->SwitchBsp
|| mSmmMpSyncData
->CandidateBsp
[CpuIndex
]) {
1118 // Call platform hook to do BSP election
1120 Status
= PlatformSmmBspElection (&IsBsp
);
1121 if (EFI_SUCCESS
== Status
) {
1123 // Platform hook determines successfully
1126 mSmmMpSyncData
->BspIndex
= (UINT32
)CpuIndex
;
1130 // Platform hook fails to determine, use default BSP election method
1132 InterlockedCompareExchange32 (
1133 (UINT32
*)&mSmmMpSyncData
->BspIndex
,
1142 // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP
1144 if (mSmmMpSyncData
->BspIndex
== CpuIndex
) {
1147 // Clear last request for SwitchBsp.
1149 if (mSmmMpSyncData
->SwitchBsp
) {
1150 mSmmMpSyncData
->SwitchBsp
= FALSE
;
1151 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1152 mSmmMpSyncData
->CandidateBsp
[Index
] = FALSE
;
1156 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1157 SmmProfileRecordSmiNum ();
1161 // BSP Handler is always called with a ValidSmi == TRUE
1163 BSPHandler (CpuIndex
, mSmmMpSyncData
->EffectiveSyncMode
);
1165 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1169 ASSERT (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
== 0);
1172 // Wait for BSP's signal to exit SMI
1174 while (mSmmMpSyncData
->AllCpusInSync
) {
1181 if (XdDisableFlag
) {
1182 MiscEnableMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_MISC_ENABLE
);
1183 MiscEnableMsr
.Bits
.XD
= 1;
1184 AsmWriteMsr64 (MSR_IA32_MISC_ENABLE
, MiscEnableMsr
.Uint64
);
1189 SmmCpuFeaturesRendezvousExit (CpuIndex
);
1198 Initialize un-cacheable data.
1203 InitializeMpSyncData (
1207 if (mSmmMpSyncData
!= NULL
) {
1208 ZeroMem (mSmmMpSyncData
, mSmmMpSyncDataSize
);
1209 mSmmMpSyncData
->CpuData
= (SMM_CPU_DATA_BLOCK
*)((UINT8
*)mSmmMpSyncData
+ sizeof (SMM_DISPATCHER_MP_SYNC_DATA
));
1210 mSmmMpSyncData
->CandidateBsp
= (BOOLEAN
*)(mSmmMpSyncData
->CpuData
+ gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1211 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1213 // Enable BSP election by setting BspIndex to -1
1215 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
1217 mSmmMpSyncData
->EffectiveSyncMode
= (SMM_CPU_SYNC_MODE
) PcdGet8 (PcdCpuSmmSyncMode
);
1222 Initialize global data for MP synchronization.
1224 @param Stacks Base address of SMI stack buffer for all processors.
1225 @param StackSize Stack size for each processor in SMM.
1229 InitializeMpServiceData (
1236 MTRR_SETTINGS
*Mtrr
;
1237 PROCESSOR_SMM_DESCRIPTOR
*Psd
;
1238 UINT8
*GdtTssTables
;
1239 UINTN GdtTableStepSize
;
1242 // Initialize physical address mask
1243 // NOTE: Physical memory above virtual address limit is not supported !!!
1245 AsmCpuid (0x80000008, (UINT32
*)&Index
, NULL
, NULL
, NULL
);
1246 gPhyMask
= LShiftU64 (1, (UINT8
)Index
) - 1;
1247 gPhyMask
&= (1ull << 48) - EFI_PAGE_SIZE
;
1250 // Create page tables
1252 Cr3
= SmmInitPageTable ();
1254 GdtTssTables
= InitGdt (Cr3
, &GdtTableStepSize
);
1257 // Initialize PROCESSOR_SMM_DESCRIPTOR for each CPU
1259 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1260 Psd
= (PROCESSOR_SMM_DESCRIPTOR
*)(VOID
*)(UINTN
)(mCpuHotPlugData
.SmBase
[Index
] + SMM_PSD_OFFSET
);
1261 CopyMem (Psd
, &gcPsd
, sizeof (gcPsd
));
1262 Psd
->SmmGdtPtr
= (UINT64
)(UINTN
)(GdtTssTables
+ GdtTableStepSize
* Index
);
1263 Psd
->SmmGdtSize
= gcSmiGdtr
.Limit
+ 1;
1266 // Install SMI handler
1270 (UINT32
)mCpuHotPlugData
.SmBase
[Index
],
1271 (VOID
*)((UINTN
)Stacks
+ (StackSize
* Index
)),
1273 (UINTN
)Psd
->SmmGdtPtr
,
1276 gcSmiIdtr
.Limit
+ 1,
1282 // Initialize mSmmMpSyncData
1284 mSmmMpSyncDataSize
= sizeof (SMM_DISPATCHER_MP_SYNC_DATA
) +
1285 (sizeof (SMM_CPU_DATA_BLOCK
) + sizeof (BOOLEAN
)) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1286 mSmmMpSyncData
= (SMM_DISPATCHER_MP_SYNC_DATA
*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize
));
1287 ASSERT (mSmmMpSyncData
!= NULL
);
1288 InitializeMpSyncData ();
1291 // Record current MTRR settings
1293 ZeroMem(gSmiMtrrs
, sizeof (gSmiMtrrs
));
1294 Mtrr
= (MTRR_SETTINGS
*)gSmiMtrrs
;
1295 MtrrGetAllMtrrs (Mtrr
);
1302 Register the SMM Foundation entry point.
1304 @param This Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance
1305 @param SmmEntryPoint SMM Foundation EntryPoint
1307 @retval EFI_SUCCESS Successfully to register SMM foundation entry point
1313 IN CONST EFI_SMM_CONFIGURATION_PROTOCOL
*This
,
1314 IN EFI_SMM_ENTRY_POINT SmmEntryPoint
1318 // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.
1320 gSmmCpuPrivate
->SmmCoreEntry
= SmmEntryPoint
;