2 SMM MP service implementation
4 Copyright (c) 2009 - 2016, Intel Corporation. All rights reserved.<BR>
5 This program and the accompanying materials
6 are licensed and made available under the terms and conditions of the BSD License
7 which accompanies this distribution. The full text of the license may be found at
8 http://opensource.org/licenses/bsd-license.php
10 THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11 WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
15 #include "PiSmmCpuDxeSmm.h"
18 // Slots for all MTRR( FIXED MTRR + VARIABLE MTRR + MTRR_LIB_IA32_MTRR_DEF_TYPE)
20 UINT64 gSmiMtrrs
[MTRR_NUMBER_OF_FIXED_MTRR
+ 2 * MTRR_NUMBER_OF_VARIABLE_MTRR
+ 1];
22 SMM_DISPATCHER_MP_SYNC_DATA
*mSmmMpSyncData
= NULL
;
23 UINTN mSmmMpSyncDataSize
;
24 SMM_CPU_SEMAPHORES mSmmCpuSemaphores
;
28 Performs an atomic compare exchange operation to get semaphore.
29 The compare exchange operation must be performed using
32 @param Sem IN: 32-bit unsigned integer
33 OUT: original integer - 1
34 @return Original integer - 1
39 IN OUT
volatile UINT32
*Sem
46 } while (Value
== 0 ||
47 InterlockedCompareExchange32 (
57 Performs an atomic compare exchange operation to release semaphore.
58 The compare exchange operation must be performed using
61 @param Sem IN: 32-bit unsigned integer
62 OUT: original integer + 1
63 @return Original integer + 1
68 IN OUT
volatile UINT32
*Sem
75 } while (Value
+ 1 != 0 &&
76 InterlockedCompareExchange32 (
85 Performs an atomic compare exchange operation to lock semaphore.
86 The compare exchange operation must be performed using
89 @param Sem IN: 32-bit unsigned integer
91 @return Original integer
96 IN OUT
volatile UINT32
*Sem
103 } while (InterlockedCompareExchange32 (
111 Wait all APs to performs an atomic compare exchange operation to release semaphore.
113 @param NumberOfAPs AP number
123 BspIndex
= mSmmMpSyncData
->BspIndex
;
124 while (NumberOfAPs
-- > 0) {
125 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
130 Performs an atomic compare exchange operation to release semaphore
142 BspIndex
= mSmmMpSyncData
->BspIndex
;
143 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
144 if (Index
!= BspIndex
&& mSmmMpSyncData
->CpuData
[Index
].Present
) {
145 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[Index
].Run
);
151 Checks if all CPUs (with certain exceptions) have checked in for this SMI run
153 @param Exceptions CPU Arrival exception flags.
155 @retval TRUE if all CPUs the have checked in.
156 @retval FALSE if at least one Normal AP hasn't checked in.
160 AllCpusInSmmWithExceptions (
161 SMM_CPU_ARRIVAL_EXCEPTIONS Exceptions
165 SMM_CPU_DATA_BLOCK
*CpuData
;
166 EFI_PROCESSOR_INFORMATION
*ProcessorInfo
;
168 ASSERT (mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
170 if (mSmmMpSyncData
->Counter
== mNumberOfCpus
) {
174 CpuData
= mSmmMpSyncData
->CpuData
;
175 ProcessorInfo
= gSmmCpuPrivate
->ProcessorInfo
;
176 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
177 if (!CpuData
[Index
].Present
&& ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
178 if (((Exceptions
& ARRIVAL_EXCEPTION_DELAYED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmDelayed
) != 0) {
181 if (((Exceptions
& ARRIVAL_EXCEPTION_BLOCKED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmBlocked
) != 0) {
184 if (((Exceptions
& ARRIVAL_EXCEPTION_SMI_DISABLED
) != 0) && SmmCpuFeaturesGetSmmRegister (Index
, SmmRegSmmEnable
) != 0) {
197 Given timeout constraint, wait for all APs to arrive, and insure when this function returns, no AP will execute normal mode code before
198 entering SMM, except SMI disabled APs.
202 SmmWaitForApArrival (
209 ASSERT (mSmmMpSyncData
->Counter
<= mNumberOfCpus
);
212 // Platform implementor should choose a timeout value appropriately:
213 // - The timeout value should balance the SMM time constrains and the likelihood that delayed CPUs are excluded in the SMM run. Note
214 // the SMI Handlers must ALWAYS take into account the cases that not all APs are available in an SMI run.
215 // - The timeout value must, in the case of 2nd timeout, be at least long enough to give time for all APs to receive the SMI IPI
216 // and either enter SMM or buffer the SMI, to insure there is no CPU running normal mode code when SMI handling starts. This will
217 // be TRUE even if a blocked CPU is brought out of the blocked state by a normal mode CPU (before the normal mode CPU received the
218 // SMI IPI), because with a buffered SMI, and CPU will enter SMM immediately after it is brought out of the blocked state.
219 // - The timeout value must be longer than longest possible IO operation in the system
223 // Sync with APs 1st timeout
225 for (Timer
= StartSyncTimer ();
226 !IsSyncTimerTimeout (Timer
) &&
227 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
233 // Not all APs have arrived, so we need 2nd round of timeout. IPIs should be sent to ALL none present APs,
235 // a) Delayed AP may have just come out of the delayed state. Blocked AP may have just been brought out of blocked state by some AP running
236 // normal mode code. These APs need to be guaranteed to have an SMI pending to insure that once they are out of delayed / blocked state, they
237 // enter SMI immediately without executing instructions in normal mode. Note traditional flow requires there are no APs doing normal mode
238 // work while SMI handling is on-going.
239 // b) As a consequence of SMI IPI sending, (spurious) SMI may occur after this SMM run.
240 // c) ** NOTE **: Use SMI disabling feature VERY CAREFULLY (if at all) for traditional flow, because a processor in SMI-disabled state
241 // will execute normal mode code, which breaks the traditional SMI handlers' assumption that no APs are doing normal
242 // mode work while SMI handling is on-going.
243 // d) We don't add code to check SMI disabling status to skip sending IPI to SMI disabled APs, because:
244 // - In traditional flow, SMI disabling is discouraged.
245 // - In relaxed flow, CheckApArrival() will check SMI disabling status before calling this function.
246 // In both cases, adding SMI-disabling checking code increases overhead.
248 if (mSmmMpSyncData
->Counter
< mNumberOfCpus
) {
250 // Send SMI IPIs to bring outside processors in
252 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
253 if (!mSmmMpSyncData
->CpuData
[Index
].Present
&& gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
!= INVALID_APIC_ID
) {
254 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[Index
].ProcessorId
);
259 // Sync with APs 2nd timeout.
261 for (Timer
= StartSyncTimer ();
262 !IsSyncTimerTimeout (Timer
) &&
263 !AllCpusInSmmWithExceptions (ARRIVAL_EXCEPTION_BLOCKED
| ARRIVAL_EXCEPTION_SMI_DISABLED
);
274 Replace OS MTRR's with SMI MTRR's.
276 @param CpuIndex Processor Index
284 PROCESSOR_SMM_DESCRIPTOR
*Psd
;
286 MTRR_SETTINGS
*BiosMtrr
;
288 Psd
= (PROCESSOR_SMM_DESCRIPTOR
*)(mCpuHotPlugData
.SmBase
[CpuIndex
] + SMM_PSD_OFFSET
);
289 SmiMtrrs
= (UINT64
*)(UINTN
)Psd
->MtrrBaseMaskPtr
;
291 SmmCpuFeaturesDisableSmrr ();
294 // Replace all MTRRs registers
296 BiosMtrr
= (MTRR_SETTINGS
*)SmiMtrrs
;
297 MtrrSetAllMtrrs(BiosMtrr
);
303 @param CpuIndex BSP processor Index
304 @param SyncMode SMM MP sync mode
310 IN SMM_CPU_SYNC_MODE SyncMode
316 BOOLEAN ClearTopLevelSmiResult
;
319 ASSERT (CpuIndex
== mSmmMpSyncData
->BspIndex
);
323 // Flag BSP's presence
325 mSmmMpSyncData
->InsideSmm
= TRUE
;
328 // Initialize Debug Agent to start source level debug in BSP handler
330 InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI
, NULL
, NULL
);
333 // Mark this processor's presence
335 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
= TRUE
;
338 // Clear platform top level SMI status bit before calling SMI handlers. If
339 // we cleared it after SMI handlers are run, we would miss the SMI that
340 // occurs after SMI handlers are done and before SMI status bit is cleared.
342 ClearTopLevelSmiResult
= ClearTopLevelSmiStatus();
343 ASSERT (ClearTopLevelSmiResult
== TRUE
);
346 // Set running processor index
348 gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
= CpuIndex
;
351 // If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
353 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
356 // Wait for APs to arrive
358 SmmWaitForApArrival();
361 // Lock the counter down and retrieve the number of APs
363 mSmmMpSyncData
->AllCpusInSync
= TRUE
;
364 ApCount
= LockdownSemaphore (&mSmmMpSyncData
->Counter
) - 1;
367 // Wait for all APs to get ready for programming MTRRs
369 WaitForAllAPs (ApCount
);
371 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
373 // Signal all APs it's time for backup MTRRs
378 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
379 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
380 // to a large enough value to avoid this situation.
381 // Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
382 // We do the backup first and then set MTRR to avoid race condition for threads
385 MtrrGetAllMtrrs(&Mtrrs
);
388 // Wait for all APs to complete their MTRR saving
390 WaitForAllAPs (ApCount
);
393 // Let all processors program SMM MTRRs together
398 // WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
399 // exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
400 // to a large enough value to avoid this situation.
402 ReplaceOSMtrrs (CpuIndex
);
405 // Wait for all APs to complete their MTRR programming
407 WaitForAllAPs (ApCount
);
412 // The BUSY lock is initialized to Acquired state
414 AcquireSpinLockOrFail (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
417 // Perform the pre tasks
422 // Invoke SMM Foundation EntryPoint with the processor information context.
424 gSmmCpuPrivate
->SmmCoreEntry (&gSmmCpuPrivate
->SmmCoreEntryContext
);
427 // Make sure all APs have completed their pending none-block tasks
429 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
430 if (Index
!= CpuIndex
&& mSmmMpSyncData
->CpuData
[Index
].Present
) {
431 AcquireSpinLock (&mSmmMpSyncData
->CpuData
[Index
].Busy
);
432 ReleaseSpinLock (&mSmmMpSyncData
->CpuData
[Index
].Busy
);;
437 // Perform the remaining tasks
439 PerformRemainingTasks ();
442 // If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
443 // make those APs to exit SMI synchronously. APs which arrive later will be excluded and
444 // will run through freely.
446 if (SyncMode
!= SmmCpuSyncModeTradition
&& !SmmCpuFeaturesNeedConfigureMtrrs()) {
449 // Lock the counter down and retrieve the number of APs
451 mSmmMpSyncData
->AllCpusInSync
= TRUE
;
452 ApCount
= LockdownSemaphore (&mSmmMpSyncData
->Counter
) - 1;
454 // Make sure all APs have their Present flag set
458 for (Index
= mMaxNumberOfCpus
; Index
-- > 0;) {
459 if (mSmmMpSyncData
->CpuData
[Index
].Present
) {
463 if (PresentCount
> ApCount
) {
470 // Notify all APs to exit
472 mSmmMpSyncData
->InsideSmm
= FALSE
;
476 // Wait for all APs to complete their pending tasks
478 WaitForAllAPs (ApCount
);
480 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
482 // Signal APs to restore MTRRs
489 SmmCpuFeaturesReenableSmrr ();
490 MtrrSetAllMtrrs(&Mtrrs
);
493 // Wait for all APs to complete MTRR programming
495 WaitForAllAPs (ApCount
);
499 // Stop source level debug in BSP handler, the code below will not be
502 InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI
, NULL
, NULL
);
505 // Signal APs to Reset states/semaphore for this processor
510 // Perform pending operations for hot-plug
515 // Clear the Present flag of BSP
517 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
= FALSE
;
520 // Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
521 // WaitForAllAps does not depend on the Present flag.
523 WaitForAllAPs (ApCount
);
526 // Reset BspIndex to -1, meaning BSP has not been elected.
528 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
529 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
533 // Allow APs to check in from this point on
535 mSmmMpSyncData
->Counter
= 0;
536 mSmmMpSyncData
->AllCpusInSync
= FALSE
;
542 @param CpuIndex AP processor Index.
543 @param ValidSmi Indicates that current SMI is a valid SMI or not.
544 @param SyncMode SMM MP sync mode.
551 IN SMM_CPU_SYNC_MODE SyncMode
561 for (Timer
= StartSyncTimer ();
562 !IsSyncTimerTimeout (Timer
) &&
563 !mSmmMpSyncData
->InsideSmm
;
568 if (!mSmmMpSyncData
->InsideSmm
) {
570 // BSP timeout in the first round
572 if (mSmmMpSyncData
->BspIndex
!= -1) {
574 // BSP Index is known
576 BspIndex
= mSmmMpSyncData
->BspIndex
;
577 ASSERT (CpuIndex
!= BspIndex
);
580 // Send SMI IPI to bring BSP in
582 SendSmiIpi ((UINT32
)gSmmCpuPrivate
->ProcessorInfo
[BspIndex
].ProcessorId
);
585 // Now clock BSP for the 2nd time
587 for (Timer
= StartSyncTimer ();
588 !IsSyncTimerTimeout (Timer
) &&
589 !mSmmMpSyncData
->InsideSmm
;
594 if (!mSmmMpSyncData
->InsideSmm
) {
596 // Give up since BSP is unable to enter SMM
597 // and signal the completion of this AP
598 WaitForSemaphore (&mSmmMpSyncData
->Counter
);
603 // Don't know BSP index. Give up without sending IPI to BSP.
605 WaitForSemaphore (&mSmmMpSyncData
->Counter
);
613 BspIndex
= mSmmMpSyncData
->BspIndex
;
614 ASSERT (CpuIndex
!= BspIndex
);
617 // Mark this processor's presence
619 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
= TRUE
;
621 if (SyncMode
== SmmCpuSyncModeTradition
|| SmmCpuFeaturesNeedConfigureMtrrs()) {
623 // Notify BSP of arrival at this point
625 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
628 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
630 // Wait for the signal from BSP to backup MTRRs
632 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
637 MtrrGetAllMtrrs(&Mtrrs
);
640 // Signal BSP the completion of this AP
642 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
645 // Wait for BSP's signal to program MTRRs
647 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
650 // Replace OS MTRRs with SMI MTRRs
652 ReplaceOSMtrrs (CpuIndex
);
655 // Signal BSP the completion of this AP
657 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
662 // Wait for something to happen
664 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
667 // Check if BSP wants to exit SMM
669 if (!mSmmMpSyncData
->InsideSmm
) {
674 // BUSY should be acquired by SmmStartupThisAp()
677 !AcquireSpinLockOrFail (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)
681 // Invoke the scheduled procedure
683 (*mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
) (
684 (VOID
*)mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
690 ReleaseSpinLock (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
693 if (SmmCpuFeaturesNeedConfigureMtrrs()) {
695 // Notify BSP the readiness of this AP to program MTRRs
697 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
700 // Wait for the signal from BSP to program MTRRs
702 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
707 SmmCpuFeaturesReenableSmrr ();
708 MtrrSetAllMtrrs(&Mtrrs
);
712 // Notify BSP the readiness of this AP to Reset states/semaphore for this processor
714 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
717 // Wait for the signal from BSP to Reset states/semaphore for this processor
719 WaitForSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
722 // Reset states/semaphore for this processor
724 mSmmMpSyncData
->CpuData
[CpuIndex
].Present
= FALSE
;
727 // Notify BSP the readiness of this AP to exit SMM
729 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[BspIndex
].Run
);
734 Create 4G PageTable in SMRAM.
736 @param ExtraPages Additional page numbers besides for 4G memory
737 @param Is32BitPageTable Whether the page table is 32-bit PAE
738 @return PageTable Address
744 IN BOOLEAN Is32BitPageTable
752 UINTN High2MBoundary
;
762 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
764 // Add one more page for known good stack, then find the lower 2MB aligned address.
766 Low2MBoundary
= (mSmmStackArrayBase
+ EFI_PAGE_SIZE
) & ~(SIZE_2MB
-1);
768 // Add two more pages for known good stack and stack guard page,
769 // then find the lower 2MB aligned address.
771 High2MBoundary
= (mSmmStackArrayEnd
- mSmmStackSize
+ EFI_PAGE_SIZE
* 2) & ~(SIZE_2MB
-1);
772 PagesNeeded
= ((High2MBoundary
- Low2MBoundary
) / SIZE_2MB
) + 1;
775 // Allocate the page table
777 PageTable
= AllocatePageTableMemory (ExtraPages
+ 5 + PagesNeeded
);
778 ASSERT (PageTable
!= NULL
);
780 PageTable
= (VOID
*)((UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (ExtraPages
));
781 Pte
= (UINT64
*)PageTable
;
784 // Zero out all page table entries first
786 ZeroMem (Pte
, EFI_PAGES_TO_SIZE (1));
789 // Set Page Directory Pointers
791 for (Index
= 0; Index
< 4; Index
++) {
792 Pte
[Index
] = (UINTN
)PageTable
+ EFI_PAGE_SIZE
* (Index
+ 1) + (Is32BitPageTable
? IA32_PAE_PDPTE_ATTRIBUTE_BITS
: PAGE_ATTRIBUTE_BITS
);
794 Pte
+= EFI_PAGE_SIZE
/ sizeof (*Pte
);
797 // Fill in Page Directory Entries
799 for (Index
= 0; Index
< EFI_PAGE_SIZE
* 4 / sizeof (*Pte
); Index
++) {
800 Pte
[Index
] = (Index
<< 21) | IA32_PG_PS
| PAGE_ATTRIBUTE_BITS
;
803 if (FeaturePcdGet (PcdCpuSmmStackGuard
)) {
804 Pages
= (UINTN
)PageTable
+ EFI_PAGES_TO_SIZE (5);
805 GuardPage
= mSmmStackArrayBase
+ EFI_PAGE_SIZE
;
806 Pdpte
= (UINT64
*)PageTable
;
807 for (PageIndex
= Low2MBoundary
; PageIndex
<= High2MBoundary
; PageIndex
+= SIZE_2MB
) {
808 Pte
= (UINT64
*)(UINTN
)(Pdpte
[BitFieldRead32 ((UINT32
)PageIndex
, 30, 31)] & ~(EFI_PAGE_SIZE
- 1));
809 Pte
[BitFieldRead32 ((UINT32
)PageIndex
, 21, 29)] = (UINT64
)Pages
| PAGE_ATTRIBUTE_BITS
;
811 // Fill in Page Table Entries
813 Pte
= (UINT64
*)Pages
;
814 PageAddress
= PageIndex
;
815 for (Index
= 0; Index
< EFI_PAGE_SIZE
/ sizeof (*Pte
); Index
++) {
816 if (PageAddress
== GuardPage
) {
818 // Mark the guard page as non-present
820 Pte
[Index
] = PageAddress
;
821 GuardPage
+= mSmmStackSize
;
822 if (GuardPage
> mSmmStackArrayEnd
) {
826 Pte
[Index
] = PageAddress
| PAGE_ATTRIBUTE_BITS
;
828 PageAddress
+= EFI_PAGE_SIZE
;
830 Pages
+= EFI_PAGE_SIZE
;
834 return (UINT32
)(UINTN
)PageTable
;
838 Set memory cache ability.
840 @param PageTable PageTable Address
841 @param Address Memory Address to change cache ability
842 @param Cacheability Cache ability to set
847 IN UINT64
*PageTable
,
849 IN UINT8 Cacheability
853 VOID
*NewPageTableAddress
;
854 UINT64
*NewPageTable
;
857 ASSERT ((Address
& EFI_PAGE_MASK
) == 0);
859 if (sizeof (UINTN
) == sizeof (UINT64
)) {
860 PTIndex
= (UINTN
)RShiftU64 (Address
, 39) & 0x1ff;
861 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
862 PageTable
= (UINT64
*)(UINTN
)(PageTable
[PTIndex
] & gPhyMask
);
865 PTIndex
= (UINTN
)RShiftU64 (Address
, 30) & 0x1ff;
866 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
867 PageTable
= (UINT64
*)(UINTN
)(PageTable
[PTIndex
] & gPhyMask
);
870 // A perfect implementation should check the original cacheability with the
871 // one being set, and break a 2M page entry into pieces only when they
874 PTIndex
= (UINTN
)RShiftU64 (Address
, 21) & 0x1ff;
875 if ((PageTable
[PTIndex
] & IA32_PG_PS
) != 0) {
877 // Allocate a page from SMRAM
879 NewPageTableAddress
= AllocatePageTableMemory (1);
880 ASSERT (NewPageTableAddress
!= NULL
);
882 NewPageTable
= (UINT64
*)NewPageTableAddress
;
884 for (Index
= 0; Index
< 0x200; Index
++) {
885 NewPageTable
[Index
] = PageTable
[PTIndex
];
886 if ((NewPageTable
[Index
] & IA32_PG_PAT_2M
) != 0) {
887 NewPageTable
[Index
] &= ~((UINT64
)IA32_PG_PAT_2M
);
888 NewPageTable
[Index
] |= (UINT64
)IA32_PG_PAT_4K
;
890 NewPageTable
[Index
] |= (UINT64
)(Index
<< EFI_PAGE_SHIFT
);
893 PageTable
[PTIndex
] = ((UINTN
)NewPageTableAddress
& gPhyMask
) | PAGE_ATTRIBUTE_BITS
;
896 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
897 PageTable
= (UINT64
*)(UINTN
)(PageTable
[PTIndex
] & gPhyMask
);
899 PTIndex
= (UINTN
)RShiftU64 (Address
, 12) & 0x1ff;
900 ASSERT (PageTable
[PTIndex
] & IA32_PG_P
);
901 PageTable
[PTIndex
] &= ~((UINT64
)((IA32_PG_PAT_4K
| IA32_PG_CD
| IA32_PG_WT
)));
902 PageTable
[PTIndex
] |= (UINT64
)Cacheability
;
907 Schedule a procedure to run on the specified CPU.
909 @param Procedure The address of the procedure to run
910 @param CpuIndex Target CPU Index
911 @param ProcArguments The parameter to pass to the procedure
913 @retval EFI_INVALID_PARAMETER CpuNumber not valid
914 @retval EFI_INVALID_PARAMETER CpuNumber specifying BSP
915 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber did not enter SMM
916 @retval EFI_INVALID_PARAMETER The AP specified by CpuNumber is busy
917 @retval EFI_SUCCESS The procedure has been successfully scheduled
923 IN EFI_AP_PROCEDURE Procedure
,
925 IN OUT VOID
*ProcArguments OPTIONAL
928 if (CpuIndex
>= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
||
929 CpuIndex
== gSmmCpuPrivate
->SmmCoreEntryContext
.CurrentlyExecutingCpu
||
930 !mSmmMpSyncData
->CpuData
[CpuIndex
].Present
||
931 gSmmCpuPrivate
->Operation
[CpuIndex
] == SmmCpuRemove
||
932 !AcquireSpinLockOrFail (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
)) {
933 return EFI_INVALID_PARAMETER
;
936 mSmmMpSyncData
->CpuData
[CpuIndex
].Procedure
= Procedure
;
937 mSmmMpSyncData
->CpuData
[CpuIndex
].Parameter
= ProcArguments
;
938 ReleaseSemaphore (&mSmmMpSyncData
->CpuData
[CpuIndex
].Run
);
940 if (FeaturePcdGet (PcdCpuSmmBlockStartupThisAp
)) {
941 AcquireSpinLock (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
942 ReleaseSpinLock (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
948 This function sets DR6 & DR7 according to SMM save state, before running SMM C code.
949 They are useful when you want to enable hardware breakpoints in SMM without entry SMM mode.
951 NOTE: It might not be appreciated in runtime since it might
952 conflict with OS debugging facilities. Turn them off in RELEASE.
954 @param CpuIndex CPU Index
963 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
965 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
966 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
967 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
968 AsmWriteDr6 (CpuSaveState
->x86
._DR6
);
969 AsmWriteDr7 (CpuSaveState
->x86
._DR7
);
971 AsmWriteDr6 ((UINTN
)CpuSaveState
->x64
._DR6
);
972 AsmWriteDr7 ((UINTN
)CpuSaveState
->x64
._DR7
);
978 This function restores DR6 & DR7 to SMM save state.
980 NOTE: It might not be appreciated in runtime since it might
981 conflict with OS debugging facilities. Turn them off in RELEASE.
983 @param CpuIndex CPU Index
992 SMRAM_SAVE_STATE_MAP
*CpuSaveState
;
994 if (FeaturePcdGet (PcdCpuSmmDebug
)) {
995 CpuSaveState
= (SMRAM_SAVE_STATE_MAP
*)gSmmCpuPrivate
->CpuSaveState
[CpuIndex
];
996 if (mSmmSaveStateRegisterLma
== EFI_SMM_SAVE_STATE_REGISTER_LMA_32BIT
) {
997 CpuSaveState
->x86
._DR7
= (UINT32
)AsmReadDr7 ();
998 CpuSaveState
->x86
._DR6
= (UINT32
)AsmReadDr6 ();
1000 CpuSaveState
->x64
._DR7
= AsmReadDr7 ();
1001 CpuSaveState
->x64
._DR6
= AsmReadDr6 ();
1007 C function for SMI entry, each processor comes here upon SMI trigger.
1009 @param CpuIndex CPU Index
1021 BOOLEAN BspInProgress
;
1024 BOOLEAN XdDisableFlag
;
1025 MSR_IA32_MISC_ENABLE_REGISTER MiscEnableMsr
;
1028 // Save Cr2 because Page Fault exception in SMM may override its value
1030 Cr2
= AsmReadCr2 ();
1033 // Perform CPU specific entry hooks
1035 SmmCpuFeaturesRendezvousEntry (CpuIndex
);
1038 // Determine if this is a valid SMI
1040 ValidSmi
= PlatformValidSmi();
1043 // Determine if BSP has been already in progress. Note this must be checked after
1044 // ValidSmi because BSP may clear a valid SMI source after checking in.
1046 BspInProgress
= mSmmMpSyncData
->InsideSmm
;
1048 if (!BspInProgress
&& !ValidSmi
) {
1050 // If we reach here, it means when we sampled the ValidSmi flag, SMI status had not
1051 // been cleared by BSP in a new SMI run (so we have a truly invalid SMI), or SMI
1052 // status had been cleared by BSP and an existing SMI run has almost ended. (Note
1053 // we sampled ValidSmi flag BEFORE judging BSP-in-progress status.) In both cases, there
1054 // is nothing we need to do.
1059 // Signal presence of this processor
1061 if (ReleaseSemaphore (&mSmmMpSyncData
->Counter
) == 0) {
1063 // BSP has already ended the synchronization, so QUIT!!!
1067 // Wait for BSP's signal to finish SMI
1069 while (mSmmMpSyncData
->AllCpusInSync
) {
1076 // The BUSY lock is initialized to Released state.
1077 // This needs to be done early enough to be ready for BSP's SmmStartupThisAp() call.
1078 // E.g., with Relaxed AP flow, SmmStartupThisAp() may be called immediately
1079 // after AP's present flag is detected.
1081 InitializeSpinLock (&mSmmMpSyncData
->CpuData
[CpuIndex
].Busy
);
1087 XdDisableFlag
= FALSE
;
1089 MiscEnableMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_MISC_ENABLE
);
1090 if (MiscEnableMsr
.Bits
.XD
== 1) {
1091 XdDisableFlag
= TRUE
;
1092 MiscEnableMsr
.Bits
.XD
= 0;
1093 AsmWriteMsr64 (MSR_IA32_MISC_ENABLE
, MiscEnableMsr
.Uint64
);
1098 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1099 ActivateSmmProfile (CpuIndex
);
1102 if (BspInProgress
) {
1104 // BSP has been elected. Follow AP path, regardless of ValidSmi flag
1105 // as BSP may have cleared the SMI status
1107 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1110 // We have a valid SMI
1117 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1118 if (!mSmmMpSyncData
->SwitchBsp
|| mSmmMpSyncData
->CandidateBsp
[CpuIndex
]) {
1120 // Call platform hook to do BSP election
1122 Status
= PlatformSmmBspElection (&IsBsp
);
1123 if (EFI_SUCCESS
== Status
) {
1125 // Platform hook determines successfully
1128 mSmmMpSyncData
->BspIndex
= (UINT32
)CpuIndex
;
1132 // Platform hook fails to determine, use default BSP election method
1134 InterlockedCompareExchange32 (
1135 (UINT32
*)&mSmmMpSyncData
->BspIndex
,
1144 // "mSmmMpSyncData->BspIndex == CpuIndex" means this is the BSP
1146 if (mSmmMpSyncData
->BspIndex
== CpuIndex
) {
1149 // Clear last request for SwitchBsp.
1151 if (mSmmMpSyncData
->SwitchBsp
) {
1152 mSmmMpSyncData
->SwitchBsp
= FALSE
;
1153 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1154 mSmmMpSyncData
->CandidateBsp
[Index
] = FALSE
;
1158 if (FeaturePcdGet (PcdCpuSmmProfileEnable
)) {
1159 SmmProfileRecordSmiNum ();
1163 // BSP Handler is always called with a ValidSmi == TRUE
1165 BSPHandler (CpuIndex
, mSmmMpSyncData
->EffectiveSyncMode
);
1167 APHandler (CpuIndex
, ValidSmi
, mSmmMpSyncData
->EffectiveSyncMode
);
1171 ASSERT (mSmmMpSyncData
->CpuData
[CpuIndex
].Run
== 0);
1174 // Wait for BSP's signal to exit SMI
1176 while (mSmmMpSyncData
->AllCpusInSync
) {
1183 if (XdDisableFlag
) {
1184 MiscEnableMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_MISC_ENABLE
);
1185 MiscEnableMsr
.Bits
.XD
= 1;
1186 AsmWriteMsr64 (MSR_IA32_MISC_ENABLE
, MiscEnableMsr
.Uint64
);
1191 SmmCpuFeaturesRendezvousExit (CpuIndex
);
1199 Allocate buffer for all semaphores and spin locks.
1203 InitializeSmmCpuSemaphores (
1207 UINTN ProcessorCount
;
1209 UINTN GlobalSemaphoresSize
;
1210 UINTN SemaphoreSize
;
1212 UINTN
*SemaphoreBlock
;
1213 UINTN SemaphoreAddr
;
1215 SemaphoreSize
= GetSpinLockProperties ();
1216 ProcessorCount
= gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1217 GlobalSemaphoresSize
= (sizeof (SMM_CPU_SEMAPHORE_GLOBAL
) / sizeof (VOID
*)) * SemaphoreSize
;
1218 TotalSize
= GlobalSemaphoresSize
;
1219 DEBUG((EFI_D_INFO
, "One Semaphore Size = 0x%x\n", SemaphoreSize
));
1220 DEBUG((EFI_D_INFO
, "Total Semaphores Size = 0x%x\n", TotalSize
));
1221 Pages
= EFI_SIZE_TO_PAGES (TotalSize
);
1222 SemaphoreBlock
= AllocatePages (Pages
);
1223 ASSERT (SemaphoreBlock
!= NULL
);
1224 ZeroMem (SemaphoreBlock
, TotalSize
);
1226 SemaphoreAddr
= (UINTN
)SemaphoreBlock
;
1227 mSmmCpuSemaphores
.SemaphoreGlobal
.Counter
= (UINT32
*)SemaphoreAddr
;
1228 SemaphoreAddr
+= SemaphoreSize
;
1229 mSmmCpuSemaphores
.SemaphoreGlobal
.InsideSmm
= (BOOLEAN
*)SemaphoreAddr
;
1230 SemaphoreAddr
+= SemaphoreSize
;
1231 mSmmCpuSemaphores
.SemaphoreGlobal
.AllCpusInSync
= (BOOLEAN
*)SemaphoreAddr
;
1232 SemaphoreAddr
+= SemaphoreSize
;
1233 mSmmCpuSemaphores
.SemaphoreGlobal
.PFLock
= (SPIN_LOCK
*)SemaphoreAddr
;
1234 SemaphoreAddr
+= SemaphoreSize
;
1235 mSmmCpuSemaphores
.SemaphoreGlobal
.CodeAccessCheckLock
1236 = (SPIN_LOCK
*)SemaphoreAddr
;
1238 mSemaphoreSize
= SemaphoreSize
;
1242 Initialize un-cacheable data.
1247 InitializeMpSyncData (
1251 if (mSmmMpSyncData
!= NULL
) {
1252 ZeroMem (mSmmMpSyncData
, mSmmMpSyncDataSize
);
1253 mSmmMpSyncData
->CpuData
= (SMM_CPU_DATA_BLOCK
*)((UINT8
*)mSmmMpSyncData
+ sizeof (SMM_DISPATCHER_MP_SYNC_DATA
));
1254 mSmmMpSyncData
->CandidateBsp
= (BOOLEAN
*)(mSmmMpSyncData
->CpuData
+ gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
);
1255 if (FeaturePcdGet (PcdCpuSmmEnableBspElection
)) {
1257 // Enable BSP election by setting BspIndex to -1
1259 mSmmMpSyncData
->BspIndex
= (UINT32
)-1;
1261 mSmmMpSyncData
->EffectiveSyncMode
= (SMM_CPU_SYNC_MODE
) PcdGet8 (PcdCpuSmmSyncMode
);
1263 InitializeSmmCpuSemaphores ();
1268 Initialize global data for MP synchronization.
1270 @param Stacks Base address of SMI stack buffer for all processors.
1271 @param StackSize Stack size for each processor in SMM.
1275 InitializeMpServiceData (
1282 MTRR_SETTINGS
*Mtrr
;
1283 PROCESSOR_SMM_DESCRIPTOR
*Psd
;
1284 UINT8
*GdtTssTables
;
1285 UINTN GdtTableStepSize
;
1288 // Initialize physical address mask
1289 // NOTE: Physical memory above virtual address limit is not supported !!!
1291 AsmCpuid (0x80000008, (UINT32
*)&Index
, NULL
, NULL
, NULL
);
1292 gPhyMask
= LShiftU64 (1, (UINT8
)Index
) - 1;
1293 gPhyMask
&= (1ull << 48) - EFI_PAGE_SIZE
;
1296 // Create page tables
1298 Cr3
= SmmInitPageTable ();
1300 GdtTssTables
= InitGdt (Cr3
, &GdtTableStepSize
);
1303 // Initialize PROCESSOR_SMM_DESCRIPTOR for each CPU
1305 for (Index
= 0; Index
< mMaxNumberOfCpus
; Index
++) {
1306 Psd
= (PROCESSOR_SMM_DESCRIPTOR
*)(VOID
*)(UINTN
)(mCpuHotPlugData
.SmBase
[Index
] + SMM_PSD_OFFSET
);
1307 CopyMem (Psd
, &gcPsd
, sizeof (gcPsd
));
1308 Psd
->SmmGdtPtr
= (UINT64
)(UINTN
)(GdtTssTables
+ GdtTableStepSize
* Index
);
1309 Psd
->SmmGdtSize
= gcSmiGdtr
.Limit
+ 1;
1312 // Install SMI handler
1316 (UINT32
)mCpuHotPlugData
.SmBase
[Index
],
1317 (VOID
*)((UINTN
)Stacks
+ (StackSize
* Index
)),
1319 (UINTN
)Psd
->SmmGdtPtr
,
1322 gcSmiIdtr
.Limit
+ 1,
1328 // Initialize mSmmMpSyncData
1330 mSmmMpSyncDataSize
= sizeof (SMM_DISPATCHER_MP_SYNC_DATA
) +
1331 (sizeof (SMM_CPU_DATA_BLOCK
) + sizeof (BOOLEAN
)) * gSmmCpuPrivate
->SmmCoreEntryContext
.NumberOfCpus
;
1332 mSmmMpSyncData
= (SMM_DISPATCHER_MP_SYNC_DATA
*) AllocatePages (EFI_SIZE_TO_PAGES (mSmmMpSyncDataSize
));
1333 ASSERT (mSmmMpSyncData
!= NULL
);
1334 InitializeMpSyncData ();
1337 // Record current MTRR settings
1339 ZeroMem(gSmiMtrrs
, sizeof (gSmiMtrrs
));
1340 Mtrr
= (MTRR_SETTINGS
*)gSmiMtrrs
;
1341 MtrrGetAllMtrrs (Mtrr
);
1348 Register the SMM Foundation entry point.
1350 @param This Pointer to EFI_SMM_CONFIGURATION_PROTOCOL instance
1351 @param SmmEntryPoint SMM Foundation EntryPoint
1353 @retval EFI_SUCCESS Successfully to register SMM foundation entry point
1359 IN CONST EFI_SMM_CONFIGURATION_PROTOCOL
*This
,
1360 IN EFI_SMM_ENTRY_POINT SmmEntryPoint
1364 // Record SMM Foundation EntryPoint, later invoke it on SMI entry vector.
1366 gSmmCpuPrivate
->SmmCoreEntry
= SmmEntryPoint
;