2 CPU MP Initialize Library common functions.
4 Copyright (c) 2016 - 2021, Intel Corporation. All rights reserved.<BR>
5 Copyright (c) 2020, AMD Inc. All rights reserved.<BR>
7 SPDX-License-Identifier: BSD-2-Clause-Patent
12 #include "MpIntelTdx.h"
13 #include <Library/VmgExitLib.h>
14 #include <Register/Amd/Fam17Msr.h>
15 #include <Register/Amd/Ghcb.h>
16 #include <ConfidentialComputingGuestAttr.h>
18 EFI_GUID mCpuInitMpLibHobGuid
= CPU_INIT_MP_LIB_HOB_GUID
;
21 The function will check if BSP Execute Disable is enabled.
23 DxeIpl may have enabled Execute Disable for BSP, APs need to
24 get the status and sync up the settings.
25 If BSP's CR0.Paging is not set, BSP execute Disble feature is
28 @retval TRUE BSP Execute Disable is enabled.
29 @retval FALSE BSP Execute Disable is not enabled.
32 IsBspExecuteDisableEnabled (
37 CPUID_EXTENDED_CPU_SIG_EDX Edx
;
38 MSR_IA32_EFER_REGISTER EferMsr
;
43 Cr0
.UintN
= AsmReadCr0 ();
44 if (Cr0
.Bits
.PG
!= 0) {
46 // If CR0 Paging bit is set
48 AsmCpuid (CPUID_EXTENDED_FUNCTION
, &Eax
, NULL
, NULL
, NULL
);
49 if (Eax
>= CPUID_EXTENDED_CPU_SIG
) {
50 AsmCpuid (CPUID_EXTENDED_CPU_SIG
, NULL
, NULL
, NULL
, &Edx
.Uint32
);
53 // Bit 20: Execute Disable Bit available.
55 if (Edx
.Bits
.NX
!= 0) {
56 EferMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_EFER
);
59 // Bit 11: Execute Disable Bit enable.
61 if (EferMsr
.Bits
.NXE
!= 0) {
72 Worker function for SwitchBSP().
74 Worker function for SwitchBSP(), assigned to the AP which is intended
77 @param[in] Buffer Pointer to CPU MP Data
85 CPU_MP_DATA
*DataInHob
;
87 DataInHob
= (CPU_MP_DATA
*)Buffer
;
88 AsmExchangeRole (&DataInHob
->APInfo
, &DataInHob
->BSPInfo
);
92 Get the Application Processors state.
94 @param[in] CpuData The pointer to CPU_AP_DATA of specified AP
100 IN CPU_AP_DATA
*CpuData
103 return CpuData
->State
;
107 Set the Application Processors state.
109 @param[in] CpuData The pointer to CPU_AP_DATA of specified AP
110 @param[in] State The AP status
114 IN CPU_AP_DATA
*CpuData
,
118 AcquireSpinLock (&CpuData
->ApLock
);
119 CpuData
->State
= State
;
120 ReleaseSpinLock (&CpuData
->ApLock
);
124 Save BSP's local APIC timer setting.
126 @param[in] CpuMpData Pointer to CPU MP Data
129 SaveLocalApicTimerSetting (
130 IN CPU_MP_DATA
*CpuMpData
134 // Record the current local APIC timer setting of BSP
137 &CpuMpData
->DivideValue
,
138 &CpuMpData
->PeriodicMode
,
141 CpuMpData
->CurrentTimerCount
= GetApicTimerCurrentCount ();
142 CpuMpData
->TimerInterruptState
= GetApicTimerInterruptState ();
146 Sync local APIC timer setting from BSP to AP.
148 @param[in] CpuMpData Pointer to CPU MP Data
151 SyncLocalApicTimerSetting (
152 IN CPU_MP_DATA
*CpuMpData
156 // Sync local APIC timer setting from BSP to AP
158 InitializeApicTimer (
159 CpuMpData
->DivideValue
,
160 CpuMpData
->CurrentTimerCount
,
161 CpuMpData
->PeriodicMode
,
165 // Disable AP's local APIC timer interrupt
167 DisableApicTimerInterrupt ();
171 Save the volatile registers required to be restored following INIT IPI.
173 @param[out] VolatileRegisters Returns buffer saved the volatile resisters
176 SaveVolatileRegisters (
177 OUT CPU_VOLATILE_REGISTERS
*VolatileRegisters
180 CPUID_VERSION_INFO_EDX VersionInfoEdx
;
182 VolatileRegisters
->Cr0
= AsmReadCr0 ();
183 VolatileRegisters
->Cr3
= AsmReadCr3 ();
184 VolatileRegisters
->Cr4
= AsmReadCr4 ();
186 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &VersionInfoEdx
.Uint32
);
187 if (VersionInfoEdx
.Bits
.DE
!= 0) {
189 // If processor supports Debugging Extensions feature
190 // by CPUID.[EAX=01H]:EDX.BIT2
192 VolatileRegisters
->Dr0
= AsmReadDr0 ();
193 VolatileRegisters
->Dr1
= AsmReadDr1 ();
194 VolatileRegisters
->Dr2
= AsmReadDr2 ();
195 VolatileRegisters
->Dr3
= AsmReadDr3 ();
196 VolatileRegisters
->Dr6
= AsmReadDr6 ();
197 VolatileRegisters
->Dr7
= AsmReadDr7 ();
200 AsmReadGdtr (&VolatileRegisters
->Gdtr
);
201 AsmReadIdtr (&VolatileRegisters
->Idtr
);
202 VolatileRegisters
->Tr
= AsmReadTr ();
206 Restore the volatile registers following INIT IPI.
208 @param[in] VolatileRegisters Pointer to volatile resisters
209 @param[in] IsRestoreDr TRUE: Restore DRx if supported
210 FALSE: Do not restore DRx
213 RestoreVolatileRegisters (
214 IN CPU_VOLATILE_REGISTERS
*VolatileRegisters
,
215 IN BOOLEAN IsRestoreDr
218 CPUID_VERSION_INFO_EDX VersionInfoEdx
;
219 IA32_TSS_DESCRIPTOR
*Tss
;
221 AsmWriteCr3 (VolatileRegisters
->Cr3
);
222 AsmWriteCr4 (VolatileRegisters
->Cr4
);
223 AsmWriteCr0 (VolatileRegisters
->Cr0
);
226 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, NULL
, &VersionInfoEdx
.Uint32
);
227 if (VersionInfoEdx
.Bits
.DE
!= 0) {
229 // If processor supports Debugging Extensions feature
230 // by CPUID.[EAX=01H]:EDX.BIT2
232 AsmWriteDr0 (VolatileRegisters
->Dr0
);
233 AsmWriteDr1 (VolatileRegisters
->Dr1
);
234 AsmWriteDr2 (VolatileRegisters
->Dr2
);
235 AsmWriteDr3 (VolatileRegisters
->Dr3
);
236 AsmWriteDr6 (VolatileRegisters
->Dr6
);
237 AsmWriteDr7 (VolatileRegisters
->Dr7
);
241 AsmWriteGdtr (&VolatileRegisters
->Gdtr
);
242 AsmWriteIdtr (&VolatileRegisters
->Idtr
);
243 if ((VolatileRegisters
->Tr
!= 0) &&
244 (VolatileRegisters
->Tr
< VolatileRegisters
->Gdtr
.Limit
))
246 Tss
= (IA32_TSS_DESCRIPTOR
*)(VolatileRegisters
->Gdtr
.Base
+
247 VolatileRegisters
->Tr
);
248 if (Tss
->Bits
.P
== 1) {
249 Tss
->Bits
.Type
&= 0xD; // 1101 - Clear busy bit just in case
250 AsmWriteTr (VolatileRegisters
->Tr
);
256 Detect whether Mwait-monitor feature is supported.
258 @retval TRUE Mwait-monitor feature is supported.
259 @retval FALSE Mwait-monitor feature is not supported.
266 CPUID_VERSION_INFO_ECX VersionInfoEcx
;
268 AsmCpuid (CPUID_VERSION_INFO
, NULL
, NULL
, &VersionInfoEcx
.Uint32
, NULL
);
269 return (VersionInfoEcx
.Bits
.MONITOR
== 1) ? TRUE
: FALSE
;
275 @param[out] MonitorFilterSize Returns the largest monitor-line size in bytes.
277 @return The AP loop mode.
281 OUT UINT32
*MonitorFilterSize
285 CPUID_MONITOR_MWAIT_EBX MonitorMwaitEbx
;
287 ASSERT (MonitorFilterSize
!= NULL
);
289 ApLoopMode
= PcdGet8 (PcdCpuApLoopMode
);
290 ASSERT (ApLoopMode
>= ApInHltLoop
&& ApLoopMode
<= ApInRunLoop
);
291 if (ApLoopMode
== ApInMwaitLoop
) {
292 if (!IsMwaitSupport ()) {
294 // If processor does not support MONITOR/MWAIT feature,
295 // force AP in Hlt-loop mode
297 ApLoopMode
= ApInHltLoop
;
300 if (ConfidentialComputingGuestHas (CCAttrAmdSevEs
) &&
301 !ConfidentialComputingGuestHas (CCAttrAmdSevSnp
))
304 // For SEV-ES (SEV-SNP is also considered SEV-ES), force AP in Hlt-loop
305 // mode in order to use the GHCB protocol for starting APs
307 ApLoopMode
= ApInHltLoop
;
311 if (ApLoopMode
!= ApInMwaitLoop
) {
312 *MonitorFilterSize
= sizeof (UINT32
);
315 // CPUID.[EAX=05H]:EBX.BIT0-15: Largest monitor-line size in bytes
316 // CPUID.[EAX=05H].EDX: C-states supported using MWAIT
318 AsmCpuid (CPUID_MONITOR_MWAIT
, NULL
, &MonitorMwaitEbx
.Uint32
, NULL
, NULL
);
319 *MonitorFilterSize
= MonitorMwaitEbx
.Bits
.LargestMonitorLineSize
;
326 Sort the APIC ID of all processors.
328 This function sorts the APIC ID of all processors so that processor number is
329 assigned in the ascending order of APIC ID which eases MP debugging.
331 @param[in] CpuMpData Pointer to PEI CPU MP Data
335 IN CPU_MP_DATA
*CpuMpData
342 CPU_INFO_IN_HOB CpuInfo
;
344 CPU_INFO_IN_HOB
*CpuInfoInHob
;
345 volatile UINT32
*StartupApSignal
;
347 ApCount
= CpuMpData
->CpuCount
- 1;
348 CpuInfoInHob
= (CPU_INFO_IN_HOB
*)(UINTN
)CpuMpData
->CpuInfoInHob
;
350 for (Index1
= 0; Index1
< ApCount
; Index1
++) {
353 // Sort key is the hardware default APIC ID
355 ApicId
= CpuInfoInHob
[Index1
].ApicId
;
356 for (Index2
= Index1
+ 1; Index2
<= ApCount
; Index2
++) {
357 if (ApicId
> CpuInfoInHob
[Index2
].ApicId
) {
359 ApicId
= CpuInfoInHob
[Index2
].ApicId
;
363 if (Index3
!= Index1
) {
364 CopyMem (&CpuInfo
, &CpuInfoInHob
[Index3
], sizeof (CPU_INFO_IN_HOB
));
366 &CpuInfoInHob
[Index3
],
367 &CpuInfoInHob
[Index1
],
368 sizeof (CPU_INFO_IN_HOB
)
370 CopyMem (&CpuInfoInHob
[Index1
], &CpuInfo
, sizeof (CPU_INFO_IN_HOB
));
373 // Also exchange the StartupApSignal.
375 StartupApSignal
= CpuMpData
->CpuData
[Index3
].StartupApSignal
;
376 CpuMpData
->CpuData
[Index3
].StartupApSignal
=
377 CpuMpData
->CpuData
[Index1
].StartupApSignal
;
378 CpuMpData
->CpuData
[Index1
].StartupApSignal
= StartupApSignal
;
383 // Get the processor number for the BSP
385 ApicId
= GetInitialApicId ();
386 for (Index1
= 0; Index1
< CpuMpData
->CpuCount
; Index1
++) {
387 if (CpuInfoInHob
[Index1
].ApicId
== ApicId
) {
388 CpuMpData
->BspNumber
= (UINT32
)Index1
;
396 Enable x2APIC mode on APs.
398 @param[in, out] Buffer Pointer to private data buffer.
406 SetApicMode (LOCAL_APIC_MODE_X2APIC
);
412 @param[in, out] Buffer Pointer to private data buffer.
420 CPU_MP_DATA
*CpuMpData
;
421 UINTN ProcessorNumber
;
424 CpuMpData
= (CPU_MP_DATA
*)Buffer
;
425 Status
= GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
426 ASSERT_EFI_ERROR (Status
);
428 // Load microcode on AP
430 MicrocodeDetect (CpuMpData
, ProcessorNumber
);
432 // Sync BSP's MTRR table to AP
434 MtrrSetAllMtrrs (&CpuMpData
->MtrrTable
);
438 Find the current Processor number by APIC ID.
440 @param[in] CpuMpData Pointer to PEI CPU MP Data
441 @param[out] ProcessorNumber Return the pocessor number found
443 @retval EFI_SUCCESS ProcessorNumber is found and returned.
444 @retval EFI_NOT_FOUND ProcessorNumber is not found.
448 IN CPU_MP_DATA
*CpuMpData
,
449 OUT UINTN
*ProcessorNumber
452 UINTN TotalProcessorNumber
;
454 CPU_INFO_IN_HOB
*CpuInfoInHob
;
455 UINT32 CurrentApicId
;
457 CpuInfoInHob
= (CPU_INFO_IN_HOB
*)(UINTN
)CpuMpData
->CpuInfoInHob
;
459 TotalProcessorNumber
= CpuMpData
->CpuCount
;
460 CurrentApicId
= GetApicId ();
461 for (Index
= 0; Index
< TotalProcessorNumber
; Index
++) {
462 if (CpuInfoInHob
[Index
].ApicId
== CurrentApicId
) {
463 *ProcessorNumber
= Index
;
468 return EFI_NOT_FOUND
;
472 This function will get CPU count in the system.
474 @param[in] CpuMpData Pointer to PEI CPU MP Data
476 @return CPU count detected
479 CollectProcessorCount (
480 IN CPU_MP_DATA
*CpuMpData
484 CPU_INFO_IN_HOB
*CpuInfoInHob
;
488 // Send 1st broadcast IPI to APs to wakeup APs
490 CpuMpData
->InitFlag
= ApInitConfig
;
491 WakeUpAP (CpuMpData
, TRUE
, 0, NULL
, NULL
, TRUE
);
492 CpuMpData
->InitFlag
= ApInitDone
;
494 // When InitFlag == ApInitConfig, WakeUpAP () guarantees all APs are checked in.
495 // FinishedCount is the number of check-in APs.
497 CpuMpData
->CpuCount
= CpuMpData
->FinishedCount
+ 1;
498 ASSERT (CpuMpData
->CpuCount
<= PcdGet32 (PcdCpuMaxLogicalProcessorNumber
));
501 // Enable x2APIC mode if
502 // 1. Number of CPU is greater than 255; or
503 // 2. There are any logical processors reporting an Initial APIC ID of 255 or greater.
506 if (CpuMpData
->CpuCount
> 255) {
508 // If there are more than 255 processor found, force to enable X2APIC
512 CpuInfoInHob
= (CPU_INFO_IN_HOB
*)(UINTN
)CpuMpData
->CpuInfoInHob
;
513 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
514 if (CpuInfoInHob
[Index
].InitialApicId
>= 0xFF) {
522 DEBUG ((DEBUG_INFO
, "Force x2APIC mode!\n"));
524 // Wakeup all APs to enable x2APIC mode
526 WakeUpAP (CpuMpData
, TRUE
, 0, ApFuncEnableX2Apic
, NULL
, TRUE
);
528 // Wait for all known APs finished
530 while (CpuMpData
->FinishedCount
< (CpuMpData
->CpuCount
- 1)) {
535 // Enable x2APIC on BSP
537 SetApicMode (LOCAL_APIC_MODE_X2APIC
);
539 // Set BSP/Aps state to IDLE
541 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
542 SetApState (&CpuMpData
->CpuData
[Index
], CpuStateIdle
);
546 DEBUG ((DEBUG_INFO
, "APIC MODE is %d\n", GetApicMode ()));
548 // Sort BSP/Aps by CPU APIC ID in ascending order
550 SortApicId (CpuMpData
);
552 DEBUG ((DEBUG_INFO
, "MpInitLib: Find %d processors in system.\n", CpuMpData
->CpuCount
));
554 return CpuMpData
->CpuCount
;
558 Initialize CPU AP Data when AP is wakeup at the first time.
560 @param[in, out] CpuMpData Pointer to PEI CPU MP Data
561 @param[in] ProcessorNumber The handle number of processor
562 @param[in] BistData Processor BIST data
563 @param[in] ApTopOfStack Top of AP stack
568 IN OUT CPU_MP_DATA
*CpuMpData
,
569 IN UINTN ProcessorNumber
,
571 IN UINT64 ApTopOfStack
574 CPU_INFO_IN_HOB
*CpuInfoInHob
;
575 MSR_IA32_PLATFORM_ID_REGISTER PlatformIdMsr
;
577 CpuInfoInHob
= (CPU_INFO_IN_HOB
*)(UINTN
)CpuMpData
->CpuInfoInHob
;
578 CpuInfoInHob
[ProcessorNumber
].InitialApicId
= GetInitialApicId ();
579 CpuInfoInHob
[ProcessorNumber
].ApicId
= GetApicId ();
580 CpuInfoInHob
[ProcessorNumber
].Health
= BistData
;
581 CpuInfoInHob
[ProcessorNumber
].ApTopOfStack
= ApTopOfStack
;
583 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
584 CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
= (BistData
== 0) ? TRUE
: FALSE
;
587 // NOTE: PlatformId is not relevant on AMD platforms.
589 if (!StandardSignatureIsAuthenticAMD ()) {
590 PlatformIdMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_PLATFORM_ID
);
591 CpuMpData
->CpuData
[ProcessorNumber
].PlatformId
= (UINT8
)PlatformIdMsr
.Bits
.PlatformId
;
596 &CpuMpData
->CpuData
[ProcessorNumber
].ProcessorSignature
,
602 InitializeSpinLock (&CpuMpData
->CpuData
[ProcessorNumber
].ApLock
);
603 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateIdle
);
607 This function will be called from AP reset code if BSP uses WakeUpAP.
609 @param[in] ExchangeInfo Pointer to the MP exchange info buffer
610 @param[in] ApIndex Number of current executing AP
615 IN MP_CPU_EXCHANGE_INFO
*ExchangeInfo
,
619 CPU_MP_DATA
*CpuMpData
;
620 UINTN ProcessorNumber
;
621 EFI_AP_PROCEDURE Procedure
;
624 volatile UINT32
*ApStartupSignalBuffer
;
625 CPU_INFO_IN_HOB
*CpuInfoInHob
;
627 UINTN CurrentApicMode
;
630 // AP finished assembly code and begin to execute C code
632 CpuMpData
= ExchangeInfo
->CpuMpData
;
635 // AP's local APIC settings will be lost after received INIT IPI
636 // We need to re-initialize them at here
638 ProgramVirtualWireMode ();
640 // Mask the LINT0 and LINT1 so that AP doesn't enter the system timer interrupt handler.
642 DisableLvtInterrupts ();
643 SyncLocalApicTimerSetting (CpuMpData
);
645 CurrentApicMode
= GetApicMode ();
647 if (CpuMpData
->InitFlag
== ApInitConfig
) {
648 ProcessorNumber
= ApIndex
;
650 // This is first time AP wakeup, get BIST information from AP stack
652 ApTopOfStack
= CpuMpData
->Buffer
+ (ProcessorNumber
+ 1) * CpuMpData
->CpuApStackSize
;
653 BistData
= *(UINT32
*)((UINTN
)ApTopOfStack
- sizeof (UINTN
));
655 // CpuMpData->CpuData[0].VolatileRegisters is initialized based on BSP environment,
656 // to initialize AP in InitConfig path.
657 // NOTE: IDTR.BASE stored in CpuMpData->CpuData[0].VolatileRegisters points to a different IDT shared by all APs.
659 RestoreVolatileRegisters (&CpuMpData
->CpuData
[0].VolatileRegisters
, FALSE
);
660 InitializeApData (CpuMpData
, ProcessorNumber
, BistData
, ApTopOfStack
);
661 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
664 // Execute AP function if AP is ready
666 GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
668 // Clear AP start-up signal when AP waken up
670 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
671 InterlockedCompareExchange32 (
672 (UINT32
*)ApStartupSignalBuffer
,
677 if (CpuMpData
->InitFlag
== ApInitReconfig
) {
679 // ApInitReconfig happens when:
680 // 1. AP is re-enabled after it's disabled, in either PEI or DXE phase.
681 // 2. AP is initialized in DXE phase.
682 // In either case, use the volatile registers value derived from BSP.
683 // NOTE: IDTR.BASE stored in CpuMpData->CpuData[0].VolatileRegisters points to a
684 // different IDT shared by all APs.
686 RestoreVolatileRegisters (&CpuMpData
->CpuData
[0].VolatileRegisters
, FALSE
);
688 if (CpuMpData
->ApLoopMode
== ApInHltLoop
) {
690 // Restore AP's volatile registers saved before AP is halted
692 RestoreVolatileRegisters (&CpuMpData
->CpuData
[ProcessorNumber
].VolatileRegisters
, TRUE
);
695 // The CPU driver might not flush TLB for APs on spot after updating
696 // page attributes. AP in mwait loop mode needs to take care of it when
703 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateReady
) {
704 Procedure
= (EFI_AP_PROCEDURE
)CpuMpData
->CpuData
[ProcessorNumber
].ApFunction
;
705 Parameter
= (VOID
*)CpuMpData
->CpuData
[ProcessorNumber
].ApFunctionArgument
;
706 if (Procedure
!= NULL
) {
707 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateBusy
);
709 // Enable source debugging on AP function
713 // Invoke AP function here
715 Procedure (Parameter
);
716 CpuInfoInHob
= (CPU_INFO_IN_HOB
*)(UINTN
)CpuMpData
->CpuInfoInHob
;
717 if (CpuMpData
->SwitchBspFlag
) {
719 // Re-get the processor number due to BSP/AP maybe exchange in AP function
721 GetProcessorNumber (CpuMpData
, &ProcessorNumber
);
722 CpuMpData
->CpuData
[ProcessorNumber
].ApFunction
= 0;
723 CpuMpData
->CpuData
[ProcessorNumber
].ApFunctionArgument
= 0;
724 ApStartupSignalBuffer
= CpuMpData
->CpuData
[ProcessorNumber
].StartupApSignal
;
725 CpuInfoInHob
[ProcessorNumber
].ApTopOfStack
= CpuInfoInHob
[CpuMpData
->NewBspNumber
].ApTopOfStack
;
727 if ((CpuInfoInHob
[ProcessorNumber
].ApicId
!= GetApicId ()) ||
728 (CpuInfoInHob
[ProcessorNumber
].InitialApicId
!= GetInitialApicId ()))
730 if (CurrentApicMode
!= GetApicMode ()) {
732 // If APIC mode change happened during AP function execution,
733 // we do not support APIC ID value changed.
739 // Re-get the CPU APICID and Initial APICID if they are changed
741 CpuInfoInHob
[ProcessorNumber
].ApicId
= GetApicId ();
742 CpuInfoInHob
[ProcessorNumber
].InitialApicId
= GetInitialApicId ();
748 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateFinished
);
752 if (CpuMpData
->ApLoopMode
== ApInHltLoop
) {
754 // Save AP volatile registers
756 SaveVolatileRegisters (&CpuMpData
->CpuData
[ProcessorNumber
].VolatileRegisters
);
760 // AP finished executing C code
762 InterlockedIncrement ((UINT32
*)&CpuMpData
->FinishedCount
);
764 if (CpuMpData
->InitFlag
== ApInitConfig
) {
766 // Delay decrementing the APs executing count when SEV-ES is enabled
767 // to allow the APs to issue an AP_RESET_HOLD before the BSP possibly
768 // performs another INIT-SIPI-SIPI sequence.
770 if (!CpuMpData
->UseSevEsAPMethod
) {
771 InterlockedDecrement ((UINT32
*)&CpuMpData
->MpCpuExchangeInfo
->NumApsExecuting
);
776 // Place AP is specified loop mode
778 if (CpuMpData
->ApLoopMode
== ApInHltLoop
) {
780 // Place AP in HLT-loop
783 DisableInterrupts ();
784 if (CpuMpData
->UseSevEsAPMethod
) {
785 SevEsPlaceApHlt (CpuMpData
);
795 DisableInterrupts ();
796 if (CpuMpData
->ApLoopMode
== ApInMwaitLoop
) {
798 // Place AP in MWAIT-loop
800 AsmMonitor ((UINTN
)ApStartupSignalBuffer
, 0, 0);
801 if (*ApStartupSignalBuffer
!= WAKEUP_AP_SIGNAL
) {
803 // Check AP start-up signal again.
804 // If AP start-up signal is not set, place AP into
805 // the specified C-state
807 AsmMwait (CpuMpData
->ApTargetCState
<< 4, 0);
809 } else if (CpuMpData
->ApLoopMode
== ApInRunLoop
) {
811 // Place AP in Run-loop
819 // If AP start-up signal is written, AP is waken up
820 // otherwise place AP in loop again
822 if (*ApStartupSignalBuffer
== WAKEUP_AP_SIGNAL
) {
830 Wait for AP wakeup and write AP start-up signal till AP is waken up.
832 @param[in] ApStartupSignalBuffer Pointer to AP wakeup signal
836 IN
volatile UINT32
*ApStartupSignalBuffer
840 // If AP is waken up, StartupApSignal should be cleared.
841 // Otherwise, write StartupApSignal again till AP waken up.
843 while (InterlockedCompareExchange32 (
844 (UINT32
*)ApStartupSignalBuffer
,
854 This function will fill the exchange info structure.
856 @param[in] CpuMpData Pointer to CPU MP Data
860 FillExchangeInfoData (
861 IN CPU_MP_DATA
*CpuMpData
864 volatile MP_CPU_EXCHANGE_INFO
*ExchangeInfo
;
866 IA32_SEGMENT_DESCRIPTOR
*Selector
;
869 ExchangeInfo
= CpuMpData
->MpCpuExchangeInfo
;
870 ExchangeInfo
->StackStart
= CpuMpData
->Buffer
;
871 ExchangeInfo
->StackSize
= CpuMpData
->CpuApStackSize
;
872 ExchangeInfo
->BufferStart
= CpuMpData
->WakeupBuffer
;
873 ExchangeInfo
->ModeOffset
= CpuMpData
->AddressMap
.ModeEntryOffset
;
875 ExchangeInfo
->CodeSegment
= AsmReadCs ();
876 ExchangeInfo
->DataSegment
= AsmReadDs ();
878 ExchangeInfo
->Cr3
= AsmReadCr3 ();
880 ExchangeInfo
->CFunction
= (UINTN
)ApWakeupFunction
;
881 ExchangeInfo
->ApIndex
= 0;
882 ExchangeInfo
->NumApsExecuting
= 0;
883 ExchangeInfo
->InitFlag
= (UINTN
)CpuMpData
->InitFlag
;
884 ExchangeInfo
->CpuInfo
= (CPU_INFO_IN_HOB
*)(UINTN
)CpuMpData
->CpuInfoInHob
;
885 ExchangeInfo
->CpuMpData
= CpuMpData
;
887 ExchangeInfo
->EnableExecuteDisable
= IsBspExecuteDisableEnabled ();
889 ExchangeInfo
->InitializeFloatingPointUnitsAddress
= (UINTN
)InitializeFloatingPointUnits
;
892 // We can check either CPUID(7).ECX[bit16] or check CR4.LA57[bit12]
893 // to determin whether 5-Level Paging is enabled.
894 // CPUID(7).ECX[bit16] shows CPU's capability, CR4.LA57[bit12] shows
895 // current system setting.
896 // Using latter way is simpler because it also eliminates the needs to
897 // check whether platform wants to enable it.
899 Cr4
.UintN
= AsmReadCr4 ();
900 ExchangeInfo
->Enable5LevelPaging
= (BOOLEAN
)(Cr4
.Bits
.LA57
== 1);
901 DEBUG ((DEBUG_INFO
, "%a: 5-Level Paging = %d\n", gEfiCallerBaseName
, ExchangeInfo
->Enable5LevelPaging
));
903 ExchangeInfo
->SevEsIsEnabled
= CpuMpData
->SevEsIsEnabled
;
904 ExchangeInfo
->SevSnpIsEnabled
= CpuMpData
->SevSnpIsEnabled
;
905 ExchangeInfo
->GhcbBase
= (UINTN
)CpuMpData
->GhcbBase
;
908 // Populate SEV-ES specific exchange data.
910 if (ExchangeInfo
->SevSnpIsEnabled
) {
911 FillExchangeInfoDataSevEs (ExchangeInfo
);
915 // Get the BSP's data of GDT and IDT
917 AsmReadGdtr ((IA32_DESCRIPTOR
*)&ExchangeInfo
->GdtrProfile
);
918 AsmReadIdtr ((IA32_DESCRIPTOR
*)&ExchangeInfo
->IdtrProfile
);
921 // Find a 32-bit code segment
923 Selector
= (IA32_SEGMENT_DESCRIPTOR
*)ExchangeInfo
->GdtrProfile
.Base
;
924 Size
= ExchangeInfo
->GdtrProfile
.Limit
+ 1;
926 if ((Selector
->Bits
.L
== 0) && (Selector
->Bits
.Type
>= 8)) {
927 ExchangeInfo
->ModeTransitionSegment
=
928 (UINT16
)((UINTN
)Selector
- ExchangeInfo
->GdtrProfile
.Base
);
933 Size
-= sizeof (IA32_SEGMENT_DESCRIPTOR
);
937 // Copy all 32-bit code and 64-bit code into memory with type of
938 // EfiBootServicesCode to avoid page fault if NX memory protection is enabled.
940 if (CpuMpData
->WakeupBufferHigh
!= 0) {
941 Size
= CpuMpData
->AddressMap
.RendezvousFunnelSize
+
942 CpuMpData
->AddressMap
.SwitchToRealSize
-
943 CpuMpData
->AddressMap
.ModeTransitionOffset
;
945 (VOID
*)CpuMpData
->WakeupBufferHigh
,
946 CpuMpData
->AddressMap
.RendezvousFunnelAddress
+
947 CpuMpData
->AddressMap
.ModeTransitionOffset
,
951 ExchangeInfo
->ModeTransitionMemory
= (UINT32
)CpuMpData
->WakeupBufferHigh
;
953 ExchangeInfo
->ModeTransitionMemory
= (UINT32
)
954 (ExchangeInfo
->BufferStart
+ CpuMpData
->AddressMap
.ModeTransitionOffset
);
957 ExchangeInfo
->ModeHighMemory
= ExchangeInfo
->ModeTransitionMemory
+
958 (UINT32
)ExchangeInfo
->ModeOffset
-
959 (UINT32
)CpuMpData
->AddressMap
.ModeTransitionOffset
;
960 ExchangeInfo
->ModeHighSegment
= (UINT16
)ExchangeInfo
->CodeSegment
;
964 Helper function that waits until the finished AP count reaches the specified
965 limit, or the specified timeout elapses (whichever comes first).
967 @param[in] CpuMpData Pointer to CPU MP Data.
968 @param[in] FinishedApLimit The number of finished APs to wait for.
969 @param[in] TimeLimit The number of microseconds to wait for.
972 TimedWaitForApFinish (
973 IN CPU_MP_DATA
*CpuMpData
,
974 IN UINT32 FinishedApLimit
,
979 Get available system memory below 1MB by specified size.
981 @param[in] CpuMpData The pointer to CPU MP Data structure.
984 BackupAndPrepareWakeupBuffer (
985 IN CPU_MP_DATA
*CpuMpData
989 (VOID
*)CpuMpData
->BackupBuffer
,
990 (VOID
*)CpuMpData
->WakeupBuffer
,
991 CpuMpData
->BackupBufferSize
994 (VOID
*)CpuMpData
->WakeupBuffer
,
995 (VOID
*)CpuMpData
->AddressMap
.RendezvousFunnelAddress
,
996 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
997 CpuMpData
->AddressMap
.SwitchToRealSize
1002 Restore wakeup buffer data.
1004 @param[in] CpuMpData The pointer to CPU MP Data structure.
1007 RestoreWakeupBuffer (
1008 IN CPU_MP_DATA
*CpuMpData
1012 (VOID
*)CpuMpData
->WakeupBuffer
,
1013 (VOID
*)CpuMpData
->BackupBuffer
,
1014 CpuMpData
->BackupBufferSize
1019 Calculate the size of the reset vector.
1021 @param[in] AddressMap The pointer to Address Map structure.
1023 @return Total amount of memory required for the AP reset area
1027 GetApResetVectorSize (
1028 IN MP_ASSEMBLY_ADDRESS_MAP
*AddressMap
1033 Size
= AddressMap
->RendezvousFunnelSize
+
1034 AddressMap
->SwitchToRealSize
+
1035 sizeof (MP_CPU_EXCHANGE_INFO
);
1041 Allocate reset vector buffer.
1043 @param[in, out] CpuMpData The pointer to CPU MP Data structure.
1046 AllocateResetVector (
1047 IN OUT CPU_MP_DATA
*CpuMpData
1050 UINTN ApResetVectorSize
;
1051 UINTN ApResetStackSize
;
1053 if (CpuMpData
->WakeupBuffer
== (UINTN
)-1) {
1054 ApResetVectorSize
= GetApResetVectorSize (&CpuMpData
->AddressMap
);
1056 CpuMpData
->WakeupBuffer
= GetWakeupBuffer (ApResetVectorSize
);
1057 CpuMpData
->MpCpuExchangeInfo
= (MP_CPU_EXCHANGE_INFO
*)(UINTN
)
1058 (CpuMpData
->WakeupBuffer
+
1059 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1060 CpuMpData
->AddressMap
.SwitchToRealSize
);
1061 CpuMpData
->WakeupBufferHigh
= GetModeTransitionBuffer (
1062 CpuMpData
->AddressMap
.RendezvousFunnelSize
+
1063 CpuMpData
->AddressMap
.SwitchToRealSize
-
1064 CpuMpData
->AddressMap
.ModeTransitionOffset
1067 // The AP reset stack is only used by SEV-ES guests. Do not allocate it
1068 // if SEV-ES is not enabled. An SEV-SNP guest is also considered
1069 // an SEV-ES guest, but uses a different method of AP startup, eliminating
1070 // the need for the allocation.
1072 if (ConfidentialComputingGuestHas (CCAttrAmdSevEs
) &&
1073 !ConfidentialComputingGuestHas (CCAttrAmdSevSnp
))
1076 // Stack location is based on ProcessorNumber, so use the total number
1077 // of processors for calculating the total stack area.
1079 ApResetStackSize
= (AP_RESET_STACK_SIZE
*
1080 PcdGet32 (PcdCpuMaxLogicalProcessorNumber
));
1083 // Invoke GetWakeupBuffer a second time to allocate the stack area
1084 // below 1MB. The returned buffer will be page aligned and sized and
1085 // below the previously allocated buffer.
1087 CpuMpData
->SevEsAPResetStackStart
= GetWakeupBuffer (ApResetStackSize
);
1090 // Check to be sure that the "allocate below" behavior hasn't changed.
1091 // This will also catch a failed allocation, as "-1" is returned on
1094 if (CpuMpData
->SevEsAPResetStackStart
>= CpuMpData
->WakeupBuffer
) {
1097 "SEV-ES AP reset stack is not below wakeup buffer\n"
1106 BackupAndPrepareWakeupBuffer (CpuMpData
);
1110 Free AP reset vector buffer.
1112 @param[in] CpuMpData The pointer to CPU MP Data structure.
1116 IN CPU_MP_DATA
*CpuMpData
1120 // If SEV-ES is enabled, the reset area is needed for AP parking and
1121 // and AP startup in the OS, so the reset area is reserved. Do not
1122 // perform the restore as this will overwrite memory which has data
1123 // needed by SEV-ES.
1125 if (!CpuMpData
->UseSevEsAPMethod
) {
1126 RestoreWakeupBuffer (CpuMpData
);
1131 This function will be called by BSP to wakeup AP.
1133 @param[in] CpuMpData Pointer to CPU MP Data
1134 @param[in] Broadcast TRUE: Send broadcast IPI to all APs
1135 FALSE: Send IPI to AP by ApicId
1136 @param[in] ProcessorNumber The handle number of specified processor
1137 @param[in] Procedure The function to be invoked by AP
1138 @param[in] ProcedureArgument The argument to be passed into AP function
1139 @param[in] WakeUpDisabledAps Whether need to wake up disabled APs in broadcast mode.
1143 IN CPU_MP_DATA
*CpuMpData
,
1144 IN BOOLEAN Broadcast
,
1145 IN UINTN ProcessorNumber
,
1146 IN EFI_AP_PROCEDURE Procedure OPTIONAL
,
1147 IN VOID
*ProcedureArgument OPTIONAL
,
1148 IN BOOLEAN WakeUpDisabledAps
1151 volatile MP_CPU_EXCHANGE_INFO
*ExchangeInfo
;
1153 CPU_AP_DATA
*CpuData
;
1154 BOOLEAN ResetVectorRequired
;
1155 CPU_INFO_IN_HOB
*CpuInfoInHob
;
1157 CpuMpData
->FinishedCount
= 0;
1158 ResetVectorRequired
= FALSE
;
1160 if (CpuMpData
->WakeUpByInitSipiSipi
||
1161 (CpuMpData
->InitFlag
!= ApInitDone
))
1163 ResetVectorRequired
= TRUE
;
1164 AllocateResetVector (CpuMpData
);
1165 AllocateSevEsAPMemory (CpuMpData
);
1166 FillExchangeInfoData (CpuMpData
);
1167 SaveLocalApicTimerSetting (CpuMpData
);
1170 if (CpuMpData
->ApLoopMode
== ApInMwaitLoop
) {
1172 // Get AP target C-state each time when waking up AP,
1173 // for it maybe updated by platform again
1175 CpuMpData
->ApTargetCState
= PcdGet8 (PcdCpuApTargetCstate
);
1178 ExchangeInfo
= CpuMpData
->MpCpuExchangeInfo
;
1181 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
1182 if (Index
!= CpuMpData
->BspNumber
) {
1183 CpuData
= &CpuMpData
->CpuData
[Index
];
1185 // All AP(include disabled AP) will be woke up by INIT-SIPI-SIPI, but
1186 // the AP procedure will be skipped for disabled AP because AP state
1187 // is not CpuStateReady.
1189 if ((GetApState (CpuData
) == CpuStateDisabled
) && !WakeUpDisabledAps
) {
1193 CpuData
->ApFunction
= (UINTN
)Procedure
;
1194 CpuData
->ApFunctionArgument
= (UINTN
)ProcedureArgument
;
1195 SetApState (CpuData
, CpuStateReady
);
1196 if (CpuMpData
->InitFlag
!= ApInitConfig
) {
1197 *(UINT32
*)CpuData
->StartupApSignal
= WAKEUP_AP_SIGNAL
;
1202 if (ResetVectorRequired
) {
1204 // For SEV-ES and SEV-SNP, the initial AP boot address will be defined by
1205 // PcdSevEsWorkAreaBase. The Segment/Rip must be the jump address
1206 // from the original INIT-SIPI-SIPI.
1208 if (CpuMpData
->SevEsIsEnabled
) {
1209 SetSevEsJumpTable (ExchangeInfo
->BufferStart
);
1214 // Must use the INIT-SIPI-SIPI method for initial configuration in
1215 // order to obtain the APIC ID.
1217 if (CpuMpData
->SevSnpIsEnabled
&& (CpuMpData
->InitFlag
!= ApInitConfig
)) {
1218 SevSnpCreateAP (CpuMpData
, -1);
1220 SendInitSipiSipiAllExcludingSelf ((UINT32
)ExchangeInfo
->BufferStart
);
1224 if (CpuMpData
->InitFlag
== ApInitConfig
) {
1225 if (PcdGet32 (PcdCpuBootLogicalProcessorNumber
) > 0) {
1227 // The AP enumeration algorithm below is suitable only when the
1228 // platform can tell us the *exact* boot CPU count in advance.
1230 // The wait below finishes only when the detected AP count reaches
1231 // (PcdCpuBootLogicalProcessorNumber - 1), regardless of how long that
1232 // takes. If at least one AP fails to check in (meaning a platform
1233 // hardware bug), the detection hangs forever, by design. If the actual
1234 // boot CPU count in the system is higher than
1235 // PcdCpuBootLogicalProcessorNumber (meaning a platform
1236 // misconfiguration), then some APs may complete initialization after
1237 // the wait finishes, and cause undefined behavior.
1239 TimedWaitForApFinish (
1241 PcdGet32 (PcdCpuBootLogicalProcessorNumber
) - 1,
1242 MAX_UINT32
// approx. 71 minutes
1246 // The AP enumeration algorithm below is suitable for two use cases.
1248 // (1) The check-in time for an individual AP is bounded, and APs run
1249 // through their initialization routines strongly concurrently. In
1250 // particular, the number of concurrently running APs
1251 // ("NumApsExecuting") is never expected to fall to zero
1252 // *temporarily* -- it is expected to fall to zero only when all
1253 // APs have checked-in.
1255 // In this case, the platform is supposed to set
1256 // PcdCpuApInitTimeOutInMicroSeconds to a low-ish value (just long
1257 // enough for one AP to start initialization). The timeout will be
1258 // reached soon, and remaining APs are collected by watching
1259 // NumApsExecuting fall to zero. If NumApsExecuting falls to zero
1260 // mid-process, while some APs have not completed initialization,
1261 // the behavior is undefined.
1263 // (2) The check-in time for an individual AP is unbounded, and/or APs
1264 // may complete their initializations widely spread out. In
1265 // particular, some APs may finish initialization before some APs
1268 // In this case, the platform is supposed to set
1269 // PcdCpuApInitTimeOutInMicroSeconds to a high-ish value. The AP
1270 // enumeration will always take that long (except when the boot CPU
1271 // count happens to be maximal, that is,
1272 // PcdCpuMaxLogicalProcessorNumber). All APs are expected to
1273 // check-in before the timeout, and NumApsExecuting is assumed zero
1274 // at timeout. APs that miss the time-out may cause undefined
1277 TimedWaitForApFinish (
1279 PcdGet32 (PcdCpuMaxLogicalProcessorNumber
) - 1,
1280 PcdGet32 (PcdCpuApInitTimeOutInMicroSeconds
)
1283 while (CpuMpData
->MpCpuExchangeInfo
->NumApsExecuting
!= 0) {
1289 // Wait all APs waken up if this is not the 1st broadcast of SIPI
1291 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
1292 CpuData
= &CpuMpData
->CpuData
[Index
];
1293 if (Index
!= CpuMpData
->BspNumber
) {
1294 WaitApWakeup (CpuData
->StartupApSignal
);
1299 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1300 CpuData
->ApFunction
= (UINTN
)Procedure
;
1301 CpuData
->ApFunctionArgument
= (UINTN
)ProcedureArgument
;
1302 SetApState (CpuData
, CpuStateReady
);
1304 // Wakeup specified AP
1306 ASSERT (CpuMpData
->InitFlag
!= ApInitConfig
);
1307 *(UINT32
*)CpuData
->StartupApSignal
= WAKEUP_AP_SIGNAL
;
1308 if (ResetVectorRequired
) {
1309 CpuInfoInHob
= (CPU_INFO_IN_HOB
*)(UINTN
)CpuMpData
->CpuInfoInHob
;
1312 // For SEV-ES and SEV-SNP, the initial AP boot address will be defined by
1313 // PcdSevEsWorkAreaBase. The Segment/Rip must be the jump address
1314 // from the original INIT-SIPI-SIPI.
1316 if (CpuMpData
->SevEsIsEnabled
) {
1317 SetSevEsJumpTable (ExchangeInfo
->BufferStart
);
1320 if (CpuMpData
->SevSnpIsEnabled
&& (CpuMpData
->InitFlag
!= ApInitConfig
)) {
1321 SevSnpCreateAP (CpuMpData
, (INTN
)ProcessorNumber
);
1324 CpuInfoInHob
[ProcessorNumber
].ApicId
,
1325 (UINT32
)ExchangeInfo
->BufferStart
1331 // Wait specified AP waken up
1333 WaitApWakeup (CpuData
->StartupApSignal
);
1336 if (ResetVectorRequired
) {
1337 FreeResetVector (CpuMpData
);
1341 // After one round of Wakeup Ap actions, need to re-sync ApLoopMode with
1342 // WakeUpByInitSipiSipi flag. WakeUpByInitSipiSipi flag maybe changed by
1343 // S3SmmInitDone Ppi.
1345 CpuMpData
->WakeUpByInitSipiSipi
= (CpuMpData
->ApLoopMode
== ApInHltLoop
);
1349 Calculate timeout value and return the current performance counter value.
1351 Calculate the number of performance counter ticks required for a timeout.
1352 If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1355 @param[in] TimeoutInMicroseconds Timeout value in microseconds.
1356 @param[out] CurrentTime Returns the current value of the performance counter.
1358 @return Expected time stamp counter for timeout.
1359 If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1365 IN UINTN TimeoutInMicroseconds
,
1366 OUT UINT64
*CurrentTime
1369 UINT64 TimeoutInSeconds
;
1370 UINT64 TimestampCounterFreq
;
1373 // Read the current value of the performance counter
1375 *CurrentTime
= GetPerformanceCounter ();
1378 // If TimeoutInMicroseconds is 0, return value is also 0, which is recognized
1381 if (TimeoutInMicroseconds
== 0) {
1386 // GetPerformanceCounterProperties () returns the timestamp counter's frequency
1389 TimestampCounterFreq
= GetPerformanceCounterProperties (NULL
, NULL
);
1392 // Check the potential overflow before calculate the number of ticks for the timeout value.
1394 if (DivU64x64Remainder (MAX_UINT64
, TimeoutInMicroseconds
, NULL
) < TimestampCounterFreq
) {
1396 // Convert microseconds into seconds if direct multiplication overflows
1398 TimeoutInSeconds
= DivU64x32 (TimeoutInMicroseconds
, 1000000);
1400 // Assertion if the final tick count exceeds MAX_UINT64
1402 ASSERT (DivU64x64Remainder (MAX_UINT64
, TimeoutInSeconds
, NULL
) >= TimestampCounterFreq
);
1403 return MultU64x64 (TimestampCounterFreq
, TimeoutInSeconds
);
1406 // No overflow case, multiply the return value with TimeoutInMicroseconds and then divide
1407 // it by 1,000,000, to get the number of ticks for the timeout value.
1411 TimestampCounterFreq
,
1412 TimeoutInMicroseconds
1420 Checks whether timeout expires.
1422 Check whether the number of elapsed performance counter ticks required for
1423 a timeout condition has been reached.
1424 If Timeout is zero, which means infinity, return value is always FALSE.
1426 @param[in, out] PreviousTime On input, the value of the performance counter
1427 when it was last read.
1428 On output, the current value of the performance
1430 @param[in] TotalTime The total amount of elapsed time in performance
1432 @param[in] Timeout The number of performance counter ticks required
1433 to reach a timeout condition.
1435 @retval TRUE A timeout condition has been reached.
1436 @retval FALSE A timeout condition has not been reached.
1441 IN OUT UINT64
*PreviousTime
,
1442 IN UINT64
*TotalTime
,
1456 GetPerformanceCounterProperties (&Start
, &End
);
1457 Cycle
= End
- Start
;
1463 CurrentTime
= GetPerformanceCounter ();
1464 Delta
= (INT64
)(CurrentTime
- *PreviousTime
);
1473 *TotalTime
+= Delta
;
1474 *PreviousTime
= CurrentTime
;
1475 if (*TotalTime
> Timeout
) {
1483 Helper function that waits until the finished AP count reaches the specified
1484 limit, or the specified timeout elapses (whichever comes first).
1486 @param[in] CpuMpData Pointer to CPU MP Data.
1487 @param[in] FinishedApLimit The number of finished APs to wait for.
1488 @param[in] TimeLimit The number of microseconds to wait for.
1491 TimedWaitForApFinish (
1492 IN CPU_MP_DATA
*CpuMpData
,
1493 IN UINT32 FinishedApLimit
,
1498 // CalculateTimeout() and CheckTimeout() consider a TimeLimit of 0
1499 // "infinity", so check for (TimeLimit == 0) explicitly.
1501 if (TimeLimit
== 0) {
1505 CpuMpData
->TotalTime
= 0;
1506 CpuMpData
->ExpectedTime
= CalculateTimeout (
1508 &CpuMpData
->CurrentTime
1510 while (CpuMpData
->FinishedCount
< FinishedApLimit
&&
1512 &CpuMpData
->CurrentTime
,
1513 &CpuMpData
->TotalTime
,
1514 CpuMpData
->ExpectedTime
1520 if (CpuMpData
->FinishedCount
>= FinishedApLimit
) {
1523 "%a: reached FinishedApLimit=%u in %Lu microseconds\n",
1526 DivU64x64Remainder (
1527 MultU64x32 (CpuMpData
->TotalTime
, 1000000),
1528 GetPerformanceCounterProperties (NULL
, NULL
),
1536 Reset an AP to Idle state.
1538 Any task being executed by the AP will be aborted and the AP
1539 will be waiting for a new task in Wait-For-SIPI state.
1541 @param[in] ProcessorNumber The handle number of processor.
1544 ResetProcessorToIdleState (
1545 IN UINTN ProcessorNumber
1548 CPU_MP_DATA
*CpuMpData
;
1550 CpuMpData
= GetCpuMpData ();
1552 CpuMpData
->InitFlag
= ApInitReconfig
;
1553 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, NULL
, NULL
, TRUE
);
1554 while (CpuMpData
->FinishedCount
< 1) {
1558 CpuMpData
->InitFlag
= ApInitDone
;
1560 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateIdle
);
1564 Searches for the next waiting AP.
1566 Search for the next AP that is put in waiting state by single-threaded StartupAllAPs().
1568 @param[out] NextProcessorNumber Pointer to the processor number of the next waiting AP.
1570 @retval EFI_SUCCESS The next waiting AP has been found.
1571 @retval EFI_NOT_FOUND No waiting AP exists.
1575 GetNextWaitingProcessorNumber (
1576 OUT UINTN
*NextProcessorNumber
1579 UINTN ProcessorNumber
;
1580 CPU_MP_DATA
*CpuMpData
;
1582 CpuMpData
= GetCpuMpData ();
1584 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1585 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1586 *NextProcessorNumber
= ProcessorNumber
;
1591 return EFI_NOT_FOUND
;
1594 /** Checks status of specified AP.
1596 This function checks whether the specified AP has finished the task assigned
1597 by StartupThisAP(), and whether timeout expires.
1599 @param[in] ProcessorNumber The handle number of processor.
1601 @retval EFI_SUCCESS Specified AP has finished task assigned by StartupThisAPs().
1602 @retval EFI_TIMEOUT The timeout expires.
1603 @retval EFI_NOT_READY Specified AP has not finished task and timeout has not expired.
1607 IN UINTN ProcessorNumber
1610 CPU_MP_DATA
*CpuMpData
;
1611 CPU_AP_DATA
*CpuData
;
1613 CpuMpData
= GetCpuMpData ();
1614 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1617 // Check the CPU state of AP. If it is CpuStateIdle, then the AP has finished its task.
1618 // Only BSP and corresponding AP access this unit of CPU Data. This means the AP will not modify the
1619 // value of state after setting the it to CpuStateIdle, so BSP can safely make use of its value.
1622 // If the AP finishes for StartupThisAP(), return EFI_SUCCESS.
1624 if (GetApState (CpuData
) == CpuStateFinished
) {
1625 if (CpuData
->Finished
!= NULL
) {
1626 *(CpuData
->Finished
) = TRUE
;
1629 SetApState (CpuData
, CpuStateIdle
);
1633 // If timeout expires for StartupThisAP(), report timeout.
1635 if (CheckTimeout (&CpuData
->CurrentTime
, &CpuData
->TotalTime
, CpuData
->ExpectedTime
)) {
1636 if (CpuData
->Finished
!= NULL
) {
1637 *(CpuData
->Finished
) = FALSE
;
1641 // Reset failed AP to idle state
1643 ResetProcessorToIdleState (ProcessorNumber
);
1649 return EFI_NOT_READY
;
1653 Checks status of all APs.
1655 This function checks whether all APs have finished task assigned by StartupAllAPs(),
1656 and whether timeout expires.
1658 @retval EFI_SUCCESS All APs have finished task assigned by StartupAllAPs().
1659 @retval EFI_TIMEOUT The timeout expires.
1660 @retval EFI_NOT_READY APs have not finished task and timeout has not expired.
1667 UINTN ProcessorNumber
;
1668 UINTN NextProcessorNumber
;
1671 CPU_MP_DATA
*CpuMpData
;
1672 CPU_AP_DATA
*CpuData
;
1674 CpuMpData
= GetCpuMpData ();
1676 NextProcessorNumber
= 0;
1679 // Go through all APs that are responsible for the StartupAllAPs().
1681 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1682 if (!CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1686 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
1688 // Check the CPU state of AP. If it is CpuStateIdle, then the AP has finished its task.
1689 // Only BSP and corresponding AP access this unit of CPU Data. This means the AP will not modify the
1690 // value of state after setting the it to CpuStateIdle, so BSP can safely make use of its value.
1692 if (GetApState (CpuData
) == CpuStateFinished
) {
1693 CpuMpData
->RunningCount
--;
1694 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
1695 SetApState (CpuData
, CpuStateIdle
);
1698 // If in Single Thread mode, then search for the next waiting AP for execution.
1700 if (CpuMpData
->SingleThread
) {
1701 Status
= GetNextWaitingProcessorNumber (&NextProcessorNumber
);
1703 if (!EFI_ERROR (Status
)) {
1707 (UINT32
)NextProcessorNumber
,
1708 CpuMpData
->Procedure
,
1709 CpuMpData
->ProcArguments
,
1718 // If all APs finish, return EFI_SUCCESS.
1720 if (CpuMpData
->RunningCount
== 0) {
1725 // If timeout expires, report timeout.
1728 &CpuMpData
->CurrentTime
,
1729 &CpuMpData
->TotalTime
,
1730 CpuMpData
->ExpectedTime
1735 // If FailedCpuList is not NULL, record all failed APs in it.
1737 if (CpuMpData
->FailedCpuList
!= NULL
) {
1738 *CpuMpData
->FailedCpuList
=
1739 AllocatePool ((CpuMpData
->RunningCount
+ 1) * sizeof (UINTN
));
1740 ASSERT (*CpuMpData
->FailedCpuList
!= NULL
);
1745 for (ProcessorNumber
= 0; ProcessorNumber
< CpuMpData
->CpuCount
; ProcessorNumber
++) {
1747 // Check whether this processor is responsible for StartupAllAPs().
1749 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
1751 // Reset failed APs to idle state
1753 ResetProcessorToIdleState (ProcessorNumber
);
1754 CpuMpData
->CpuData
[ProcessorNumber
].Waiting
= FALSE
;
1755 if (CpuMpData
->FailedCpuList
!= NULL
) {
1756 (*CpuMpData
->FailedCpuList
)[ListIndex
++] = ProcessorNumber
;
1761 if (CpuMpData
->FailedCpuList
!= NULL
) {
1762 (*CpuMpData
->FailedCpuList
)[ListIndex
] = END_OF_CPU_LIST
;
1768 return EFI_NOT_READY
;
1772 MP Initialize Library initialization.
1774 This service will allocate AP reset vector and wakeup all APs to do APs
1777 This service must be invoked before all other MP Initialize Library
1778 service are invoked.
1780 @retval EFI_SUCCESS MP initialization succeeds.
1781 @retval Others MP initialization fails.
1786 MpInitLibInitialize (
1790 CPU_MP_DATA
*OldCpuMpData
;
1791 CPU_INFO_IN_HOB
*CpuInfoInHob
;
1792 UINT32 MaxLogicalProcessorNumber
;
1794 MP_ASSEMBLY_ADDRESS_MAP AddressMap
;
1795 CPU_VOLATILE_REGISTERS VolatileRegisters
;
1797 UINT32 MonitorFilterSize
;
1800 CPU_MP_DATA
*CpuMpData
;
1802 UINT8
*MonitorBuffer
;
1804 UINTN ApResetVectorSize
;
1805 UINTN BackupBufferAddr
;
1808 if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr
))) {
1812 OldCpuMpData
= GetCpuMpDataFromGuidedHob ();
1813 if (OldCpuMpData
== NULL
) {
1814 MaxLogicalProcessorNumber
= PcdGet32 (PcdCpuMaxLogicalProcessorNumber
);
1816 MaxLogicalProcessorNumber
= OldCpuMpData
->CpuCount
;
1819 ASSERT (MaxLogicalProcessorNumber
!= 0);
1821 AsmGetAddressMap (&AddressMap
);
1822 ApResetVectorSize
= GetApResetVectorSize (&AddressMap
);
1823 ApStackSize
= PcdGet32 (PcdCpuApStackSize
);
1824 ApLoopMode
= GetApLoopMode (&MonitorFilterSize
);
1827 // Save BSP's Control registers for APs.
1829 SaveVolatileRegisters (&VolatileRegisters
);
1831 BufferSize
= ApStackSize
* MaxLogicalProcessorNumber
;
1832 BufferSize
+= MonitorFilterSize
* MaxLogicalProcessorNumber
;
1833 BufferSize
+= ApResetVectorSize
;
1834 BufferSize
= ALIGN_VALUE (BufferSize
, 8);
1835 BufferSize
+= VolatileRegisters
.Idtr
.Limit
+ 1;
1836 BufferSize
+= sizeof (CPU_MP_DATA
);
1837 BufferSize
+= (sizeof (CPU_AP_DATA
) + sizeof (CPU_INFO_IN_HOB
))* MaxLogicalProcessorNumber
;
1838 MpBuffer
= AllocatePages (EFI_SIZE_TO_PAGES (BufferSize
));
1839 ASSERT (MpBuffer
!= NULL
);
1840 ZeroMem (MpBuffer
, BufferSize
);
1841 Buffer
= (UINTN
)MpBuffer
;
1844 // The layout of the Buffer is as below:
1846 // +--------------------+ <-- Buffer
1848 // +--------------------+ <-- MonitorBuffer
1849 // AP Monitor Filters (N)
1850 // +--------------------+ <-- BackupBufferAddr (CpuMpData->BackupBuffer)
1852 // +--------------------+
1854 // +--------------------+ <-- ApIdtBase (8-byte boundary)
1855 // AP IDT All APs share one separate IDT. So AP can get address of CPU_MP_DATA from IDT Base.
1856 // +--------------------+ <-- CpuMpData
1858 // +--------------------+ <-- CpuMpData->CpuData
1860 // +--------------------+ <-- CpuMpData->CpuInfoInHob
1861 // CPU_INFO_IN_HOB (N)
1862 // +--------------------+
1864 MonitorBuffer
= (UINT8
*)(Buffer
+ ApStackSize
* MaxLogicalProcessorNumber
);
1865 BackupBufferAddr
= (UINTN
)MonitorBuffer
+ MonitorFilterSize
* MaxLogicalProcessorNumber
;
1866 ApIdtBase
= ALIGN_VALUE (BackupBufferAddr
+ ApResetVectorSize
, 8);
1867 CpuMpData
= (CPU_MP_DATA
*)(ApIdtBase
+ VolatileRegisters
.Idtr
.Limit
+ 1);
1868 CpuMpData
->Buffer
= Buffer
;
1869 CpuMpData
->CpuApStackSize
= ApStackSize
;
1870 CpuMpData
->BackupBuffer
= BackupBufferAddr
;
1871 CpuMpData
->BackupBufferSize
= ApResetVectorSize
;
1872 CpuMpData
->WakeupBuffer
= (UINTN
)-1;
1873 CpuMpData
->CpuCount
= 1;
1874 CpuMpData
->BspNumber
= 0;
1875 CpuMpData
->WaitEvent
= NULL
;
1876 CpuMpData
->SwitchBspFlag
= FALSE
;
1877 CpuMpData
->CpuData
= (CPU_AP_DATA
*)(CpuMpData
+ 1);
1878 CpuMpData
->CpuInfoInHob
= (UINT64
)(UINTN
)(CpuMpData
->CpuData
+ MaxLogicalProcessorNumber
);
1879 InitializeSpinLock (&CpuMpData
->MpLock
);
1880 CpuMpData
->SevEsIsEnabled
= ConfidentialComputingGuestHas (CCAttrAmdSevEs
);
1881 CpuMpData
->SevSnpIsEnabled
= ConfidentialComputingGuestHas (CCAttrAmdSevSnp
);
1882 CpuMpData
->SevEsAPBuffer
= (UINTN
)-1;
1883 CpuMpData
->GhcbBase
= PcdGet64 (PcdGhcbBase
);
1884 CpuMpData
->UseSevEsAPMethod
= CpuMpData
->SevEsIsEnabled
&& !CpuMpData
->SevSnpIsEnabled
;
1886 if (CpuMpData
->SevSnpIsEnabled
) {
1887 ASSERT ((PcdGet64 (PcdGhcbHypervisorFeatures
) & GHCB_HV_FEATURES_SNP_AP_CREATE
) == GHCB_HV_FEATURES_SNP_AP_CREATE
);
1891 // Make sure no memory usage outside of the allocated buffer.
1894 (CpuMpData
->CpuInfoInHob
+ sizeof (CPU_INFO_IN_HOB
) * MaxLogicalProcessorNumber
) ==
1899 // Duplicate BSP's IDT to APs.
1900 // All APs share one separate IDT. So AP can get the address of CpuMpData by using IDTR.BASE + IDTR.LIMIT + 1
1902 CopyMem ((VOID
*)ApIdtBase
, (VOID
*)VolatileRegisters
.Idtr
.Base
, VolatileRegisters
.Idtr
.Limit
+ 1);
1903 VolatileRegisters
.Idtr
.Base
= ApIdtBase
;
1905 // Don't pass BSP's TR to APs to avoid AP init failure.
1907 VolatileRegisters
.Tr
= 0;
1908 CopyMem (&CpuMpData
->CpuData
[0].VolatileRegisters
, &VolatileRegisters
, sizeof (VolatileRegisters
));
1910 // Set BSP basic information
1912 InitializeApData (CpuMpData
, 0, 0, CpuMpData
->Buffer
+ ApStackSize
);
1914 // Save assembly code information
1916 CopyMem (&CpuMpData
->AddressMap
, &AddressMap
, sizeof (MP_ASSEMBLY_ADDRESS_MAP
));
1918 // Finally set AP loop mode
1920 CpuMpData
->ApLoopMode
= ApLoopMode
;
1921 DEBUG ((DEBUG_INFO
, "AP Loop Mode is %d\n", CpuMpData
->ApLoopMode
));
1923 CpuMpData
->WakeUpByInitSipiSipi
= (CpuMpData
->ApLoopMode
== ApInHltLoop
);
1926 // Set up APs wakeup signal buffer
1928 for (Index
= 0; Index
< MaxLogicalProcessorNumber
; Index
++) {
1929 CpuMpData
->CpuData
[Index
].StartupApSignal
=
1930 (UINT32
*)(MonitorBuffer
+ MonitorFilterSize
* Index
);
1934 // Enable the local APIC for Virtual Wire Mode.
1936 ProgramVirtualWireMode ();
1938 if (OldCpuMpData
== NULL
) {
1939 if (MaxLogicalProcessorNumber
> 1) {
1941 // Wakeup all APs and calculate the processor count in system
1943 CollectProcessorCount (CpuMpData
);
1947 // APs have been wakeup before, just get the CPU Information
1950 OldCpuMpData
->NewCpuMpData
= CpuMpData
;
1951 CpuMpData
->CpuCount
= OldCpuMpData
->CpuCount
;
1952 CpuMpData
->BspNumber
= OldCpuMpData
->BspNumber
;
1953 CpuMpData
->CpuInfoInHob
= OldCpuMpData
->CpuInfoInHob
;
1954 CpuInfoInHob
= (CPU_INFO_IN_HOB
*)(UINTN
)CpuMpData
->CpuInfoInHob
;
1955 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
1956 InitializeSpinLock (&CpuMpData
->CpuData
[Index
].ApLock
);
1957 CpuMpData
->CpuData
[Index
].CpuHealthy
= (CpuInfoInHob
[Index
].Health
== 0) ? TRUE
: FALSE
;
1958 CpuMpData
->CpuData
[Index
].ApFunction
= 0;
1962 if (!GetMicrocodePatchInfoFromHob (
1963 &CpuMpData
->MicrocodePatchAddress
,
1964 &CpuMpData
->MicrocodePatchRegionSize
1968 // The microcode patch information cache HOB does not exist, which means
1969 // the microcode patches data has not been loaded into memory yet
1971 ShadowMicrocodeUpdatePatch (CpuMpData
);
1975 // Detect and apply Microcode on BSP
1977 MicrocodeDetect (CpuMpData
, CpuMpData
->BspNumber
);
1979 // Store BSP's MTRR setting
1981 MtrrGetAllMtrrs (&CpuMpData
->MtrrTable
);
1984 // Wakeup APs to do some AP initialize sync (Microcode & MTRR)
1986 if (CpuMpData
->CpuCount
> 1) {
1987 if (OldCpuMpData
!= NULL
) {
1989 // Only needs to use this flag for DXE phase to update the wake up
1990 // buffer. Wakeup buffer allocated in PEI phase is no longer valid
1993 CpuMpData
->InitFlag
= ApInitReconfig
;
1996 WakeUpAP (CpuMpData
, TRUE
, 0, ApInitializeSync
, CpuMpData
, TRUE
);
1998 // Wait for all APs finished initialization
2000 while (CpuMpData
->FinishedCount
< (CpuMpData
->CpuCount
- 1)) {
2004 if (OldCpuMpData
!= NULL
) {
2005 CpuMpData
->InitFlag
= ApInitDone
;
2008 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
2009 SetApState (&CpuMpData
->CpuData
[Index
], CpuStateIdle
);
2014 // Dump the microcode revision for each core.
2016 DEBUG_CODE_BEGIN ();
2018 UINT32 ExpectedMicrocodeRevision
;
2020 CpuInfoInHob
= (CPU_INFO_IN_HOB
*)(UINTN
)CpuMpData
->CpuInfoInHob
;
2021 for (Index
= 0; Index
< CpuMpData
->CpuCount
; Index
++) {
2022 GetProcessorLocationByApicId (CpuInfoInHob
[Index
].InitialApicId
, NULL
, NULL
, &ThreadId
);
2023 if (ThreadId
== 0) {
2025 // MicrocodeDetect() loads microcode in first thread of each core, so,
2026 // CpuMpData->CpuData[Index].MicrocodeEntryAddr is initialized only for first thread of each core.
2028 ExpectedMicrocodeRevision
= 0;
2029 if (CpuMpData
->CpuData
[Index
].MicrocodeEntryAddr
!= 0) {
2030 ExpectedMicrocodeRevision
= ((CPU_MICROCODE_HEADER
*)(UINTN
)CpuMpData
->CpuData
[Index
].MicrocodeEntryAddr
)->UpdateRevision
;
2035 "CPU[%04d]: Microcode revision = %08x, expected = %08x\n",
2037 CpuMpData
->CpuData
[Index
].MicrocodeRevision
,
2038 ExpectedMicrocodeRevision
2045 // Initialize global data for MP support
2047 InitMpGlobalData (CpuMpData
);
2053 Gets detailed MP-related information on the requested processor at the
2054 instant this call is made. This service may only be called from the BSP.
2056 @param[in] ProcessorNumber The handle number of processor.
2057 @param[out] ProcessorInfoBuffer A pointer to the buffer where information for
2058 the requested processor is deposited.
2059 @param[out] HealthData Return processor health data.
2061 @retval EFI_SUCCESS Processor information was returned.
2062 @retval EFI_DEVICE_ERROR The calling processor is an AP.
2063 @retval EFI_INVALID_PARAMETER ProcessorInfoBuffer is NULL.
2064 @retval EFI_NOT_FOUND The processor with the handle specified by
2065 ProcessorNumber does not exist in the platform.
2066 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2071 MpInitLibGetProcessorInfo (
2072 IN UINTN ProcessorNumber
,
2073 OUT EFI_PROCESSOR_INFORMATION
*ProcessorInfoBuffer
,
2074 OUT EFI_HEALTH_FLAGS
*HealthData OPTIONAL
2077 CPU_MP_DATA
*CpuMpData
;
2079 CPU_INFO_IN_HOB
*CpuInfoInHob
;
2080 UINTN OriginalProcessorNumber
;
2082 if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr
))) {
2083 return TdxMpInitLibGetProcessorInfo (ProcessorNumber
, ProcessorInfoBuffer
, HealthData
);
2086 CpuMpData
= GetCpuMpData ();
2087 CpuInfoInHob
= (CPU_INFO_IN_HOB
*)(UINTN
)CpuMpData
->CpuInfoInHob
;
2090 // Lower 24 bits contains the actual processor number.
2092 OriginalProcessorNumber
= ProcessorNumber
;
2093 ProcessorNumber
&= BIT24
- 1;
2096 // Check whether caller processor is BSP
2098 MpInitLibWhoAmI (&CallerNumber
);
2099 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2100 return EFI_DEVICE_ERROR
;
2103 if (ProcessorInfoBuffer
== NULL
) {
2104 return EFI_INVALID_PARAMETER
;
2107 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2108 return EFI_NOT_FOUND
;
2111 ProcessorInfoBuffer
->ProcessorId
= (UINT64
)CpuInfoInHob
[ProcessorNumber
].ApicId
;
2112 ProcessorInfoBuffer
->StatusFlag
= 0;
2113 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2114 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_AS_BSP_BIT
;
2117 if (CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
) {
2118 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_HEALTH_STATUS_BIT
;
2121 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateDisabled
) {
2122 ProcessorInfoBuffer
->StatusFlag
&= ~PROCESSOR_ENABLED_BIT
;
2124 ProcessorInfoBuffer
->StatusFlag
|= PROCESSOR_ENABLED_BIT
;
2128 // Get processor location information
2130 GetProcessorLocationByApicId (
2131 CpuInfoInHob
[ProcessorNumber
].ApicId
,
2132 &ProcessorInfoBuffer
->Location
.Package
,
2133 &ProcessorInfoBuffer
->Location
.Core
,
2134 &ProcessorInfoBuffer
->Location
.Thread
2137 if ((OriginalProcessorNumber
& CPU_V2_EXTENDED_TOPOLOGY
) != 0) {
2138 GetProcessorLocation2ByApicId (
2139 CpuInfoInHob
[ProcessorNumber
].ApicId
,
2140 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Package
,
2141 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Die
,
2142 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Tile
,
2143 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Module
,
2144 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Core
,
2145 &ProcessorInfoBuffer
->ExtendedInformation
.Location2
.Thread
2149 if (HealthData
!= NULL
) {
2150 HealthData
->Uint32
= CpuInfoInHob
[ProcessorNumber
].Health
;
2157 Worker function to switch the requested AP to be the BSP from that point onward.
2159 @param[in] ProcessorNumber The handle number of AP that is to become the new BSP.
2160 @param[in] EnableOldBSP If TRUE, then the old BSP will be listed as an
2161 enabled AP. Otherwise, it will be disabled.
2163 @retval EFI_SUCCESS BSP successfully switched.
2164 @retval others Failed to switch BSP.
2169 IN UINTN ProcessorNumber
,
2170 IN BOOLEAN EnableOldBSP
2173 CPU_MP_DATA
*CpuMpData
;
2176 MSR_IA32_APIC_BASE_REGISTER ApicBaseMsr
;
2177 BOOLEAN OldInterruptState
;
2178 BOOLEAN OldTimerInterruptState
;
2180 if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr
))) {
2181 return EFI_UNSUPPORTED
;
2185 // Save and Disable Local APIC timer interrupt
2187 OldTimerInterruptState
= GetApicTimerInterruptState ();
2188 DisableApicTimerInterrupt ();
2190 // Before send both BSP and AP to a procedure to exchange their roles,
2191 // interrupt must be disabled. This is because during the exchange role
2192 // process, 2 CPU may use 1 stack. If interrupt happens, the stack will
2193 // be corrupted, since interrupt return address will be pushed to stack
2196 OldInterruptState
= SaveAndDisableInterrupts ();
2199 // Mask LINT0 & LINT1 for the old BSP
2201 DisableLvtInterrupts ();
2203 CpuMpData
= GetCpuMpData ();
2206 // Check whether caller processor is BSP
2208 MpInitLibWhoAmI (&CallerNumber
);
2209 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2210 return EFI_DEVICE_ERROR
;
2213 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2214 return EFI_NOT_FOUND
;
2218 // Check whether specified AP is disabled
2220 State
= GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]);
2221 if (State
== CpuStateDisabled
) {
2222 return EFI_INVALID_PARAMETER
;
2226 // Check whether ProcessorNumber specifies the current BSP
2228 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2229 return EFI_INVALID_PARAMETER
;
2233 // Check whether specified AP is busy
2235 if (State
== CpuStateBusy
) {
2236 return EFI_NOT_READY
;
2239 CpuMpData
->BSPInfo
.State
= CPU_SWITCH_STATE_IDLE
;
2240 CpuMpData
->APInfo
.State
= CPU_SWITCH_STATE_IDLE
;
2241 CpuMpData
->SwitchBspFlag
= TRUE
;
2242 CpuMpData
->NewBspNumber
= ProcessorNumber
;
2245 // Clear the BSP bit of MSR_IA32_APIC_BASE
2247 ApicBaseMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_APIC_BASE
);
2248 ApicBaseMsr
.Bits
.BSP
= 0;
2249 AsmWriteMsr64 (MSR_IA32_APIC_BASE
, ApicBaseMsr
.Uint64
);
2252 // Need to wakeUp AP (future BSP).
2254 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, FutureBSPProc
, CpuMpData
, TRUE
);
2256 AsmExchangeRole (&CpuMpData
->BSPInfo
, &CpuMpData
->APInfo
);
2259 // Set the BSP bit of MSR_IA32_APIC_BASE on new BSP
2261 ApicBaseMsr
.Uint64
= AsmReadMsr64 (MSR_IA32_APIC_BASE
);
2262 ApicBaseMsr
.Bits
.BSP
= 1;
2263 AsmWriteMsr64 (MSR_IA32_APIC_BASE
, ApicBaseMsr
.Uint64
);
2264 ProgramVirtualWireMode ();
2267 // Wait for old BSP finished AP task
2269 while (GetApState (&CpuMpData
->CpuData
[CallerNumber
]) != CpuStateFinished
) {
2273 CpuMpData
->SwitchBspFlag
= FALSE
;
2275 // Set old BSP enable state
2277 if (!EnableOldBSP
) {
2278 SetApState (&CpuMpData
->CpuData
[CallerNumber
], CpuStateDisabled
);
2280 SetApState (&CpuMpData
->CpuData
[CallerNumber
], CpuStateIdle
);
2284 // Save new BSP number
2286 CpuMpData
->BspNumber
= (UINT32
)ProcessorNumber
;
2289 // Restore interrupt state.
2291 SetInterruptState (OldInterruptState
);
2293 if (OldTimerInterruptState
) {
2294 EnableApicTimerInterrupt ();
2301 Worker function to let the caller enable or disable an AP from this point onward.
2302 This service may only be called from the BSP.
2304 @param[in] ProcessorNumber The handle number of AP.
2305 @param[in] EnableAP Specifies the new state for the processor for
2306 enabled, FALSE for disabled.
2307 @param[in] HealthFlag If not NULL, a pointer to a value that specifies
2308 the new health status of the AP.
2310 @retval EFI_SUCCESS The specified AP was enabled or disabled successfully.
2311 @retval others Failed to Enable/Disable AP.
2315 EnableDisableApWorker (
2316 IN UINTN ProcessorNumber
,
2317 IN BOOLEAN EnableAP
,
2318 IN UINT32
*HealthFlag OPTIONAL
2321 CPU_MP_DATA
*CpuMpData
;
2324 if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr
))) {
2325 return EFI_UNSUPPORTED
;
2328 CpuMpData
= GetCpuMpData ();
2331 // Check whether caller processor is BSP
2333 MpInitLibWhoAmI (&CallerNumber
);
2334 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2335 return EFI_DEVICE_ERROR
;
2338 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2339 return EFI_INVALID_PARAMETER
;
2342 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2343 return EFI_NOT_FOUND
;
2347 SetApState (&CpuMpData
->CpuData
[ProcessorNumber
], CpuStateDisabled
);
2349 ResetProcessorToIdleState (ProcessorNumber
);
2352 if (HealthFlag
!= NULL
) {
2353 CpuMpData
->CpuData
[ProcessorNumber
].CpuHealthy
=
2354 (BOOLEAN
)((*HealthFlag
& PROCESSOR_HEALTH_STATUS_BIT
) != 0);
2361 This return the handle number for the calling processor. This service may be
2362 called from the BSP and APs.
2364 @param[out] ProcessorNumber Pointer to the handle number of AP.
2365 The range is from 0 to the total number of
2366 logical processors minus 1. The total number of
2367 logical processors can be retrieved by
2368 MpInitLibGetNumberOfProcessors().
2370 @retval EFI_SUCCESS The current processor handle number was returned
2372 @retval EFI_INVALID_PARAMETER ProcessorNumber is NULL.
2373 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2379 OUT UINTN
*ProcessorNumber
2382 CPU_MP_DATA
*CpuMpData
;
2384 if (ProcessorNumber
== NULL
) {
2385 return EFI_INVALID_PARAMETER
;
2388 if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr
))) {
2389 *ProcessorNumber
= 0;
2393 CpuMpData
= GetCpuMpData ();
2395 return GetProcessorNumber (CpuMpData
, ProcessorNumber
);
2399 Retrieves the number of logical processor in the platform and the number of
2400 those logical processors that are enabled on this boot. This service may only
2401 be called from the BSP.
2403 @param[out] NumberOfProcessors Pointer to the total number of logical
2404 processors in the system, including the BSP
2406 @param[out] NumberOfEnabledProcessors Pointer to the number of enabled logical
2407 processors that exist in system, including
2410 @retval EFI_SUCCESS The number of logical processors and enabled
2411 logical processors was retrieved.
2412 @retval EFI_DEVICE_ERROR The calling processor is an AP.
2413 @retval EFI_INVALID_PARAMETER NumberOfProcessors is NULL and NumberOfEnabledProcessors
2415 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2420 MpInitLibGetNumberOfProcessors (
2421 OUT UINTN
*NumberOfProcessors OPTIONAL
,
2422 OUT UINTN
*NumberOfEnabledProcessors OPTIONAL
2425 CPU_MP_DATA
*CpuMpData
;
2427 UINTN ProcessorNumber
;
2428 UINTN EnabledProcessorNumber
;
2431 if ((NumberOfProcessors
== NULL
) && (NumberOfEnabledProcessors
== NULL
)) {
2432 return EFI_INVALID_PARAMETER
;
2435 if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr
))) {
2436 return TdxMpInitLibGetNumberOfProcessors (NumberOfProcessors
, NumberOfEnabledProcessors
);
2439 CpuMpData
= GetCpuMpData ();
2442 // Check whether caller processor is BSP
2444 MpInitLibWhoAmI (&CallerNumber
);
2445 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2446 return EFI_DEVICE_ERROR
;
2449 ProcessorNumber
= CpuMpData
->CpuCount
;
2450 EnabledProcessorNumber
= 0;
2451 for (Index
= 0; Index
< ProcessorNumber
; Index
++) {
2452 if (GetApState (&CpuMpData
->CpuData
[Index
]) != CpuStateDisabled
) {
2453 EnabledProcessorNumber
++;
2457 if (NumberOfProcessors
!= NULL
) {
2458 *NumberOfProcessors
= ProcessorNumber
;
2461 if (NumberOfEnabledProcessors
!= NULL
) {
2462 *NumberOfEnabledProcessors
= EnabledProcessorNumber
;
2469 Worker function to execute a caller provided function on all enabled APs.
2471 @param[in] Procedure A pointer to the function to be run on
2472 enabled APs of the system.
2473 @param[in] SingleThread If TRUE, then all the enabled APs execute
2474 the function specified by Procedure one by
2475 one, in ascending order of processor handle
2476 number. If FALSE, then all the enabled APs
2477 execute the function specified by Procedure
2479 @param[in] ExcludeBsp Whether let BSP also trig this task.
2480 @param[in] WaitEvent The event created by the caller with CreateEvent()
2482 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2483 APs to return from Procedure, either for
2484 blocking or non-blocking mode.
2485 @param[in] ProcedureArgument The parameter passed into Procedure for
2487 @param[out] FailedCpuList If all APs finish successfully, then its
2488 content is set to NULL. If not all APs
2489 finish before timeout expires, then its
2490 content is set to address of the buffer
2491 holding handle numbers of the failed APs.
2493 @retval EFI_SUCCESS In blocking mode, all APs have finished before
2494 the timeout expired.
2495 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
2497 @retval others Failed to Startup all APs.
2501 StartupAllCPUsWorker (
2502 IN EFI_AP_PROCEDURE Procedure
,
2503 IN BOOLEAN SingleThread
,
2504 IN BOOLEAN ExcludeBsp
,
2505 IN EFI_EVENT WaitEvent OPTIONAL
,
2506 IN UINTN TimeoutInMicroseconds
,
2507 IN VOID
*ProcedureArgument OPTIONAL
,
2508 OUT UINTN
**FailedCpuList OPTIONAL
2512 CPU_MP_DATA
*CpuMpData
;
2513 UINTN ProcessorCount
;
2514 UINTN ProcessorNumber
;
2516 CPU_AP_DATA
*CpuData
;
2517 BOOLEAN HasEnabledAp
;
2520 if (FailedCpuList
!= NULL
) {
2521 *FailedCpuList
= NULL
;
2524 Status
= MpInitLibGetNumberOfProcessors (&ProcessorCount
, NULL
);
2525 if (EFI_ERROR (Status
)) {
2529 if ((ProcessorCount
== 1) && ExcludeBsp
) {
2530 return EFI_NOT_STARTED
;
2533 if (Procedure
== NULL
) {
2534 return EFI_INVALID_PARAMETER
;
2537 if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr
))) {
2539 // For Td guest ExcludeBsp must be FALSE. Otherwise it will return in above checks.
2541 ASSERT (!ExcludeBsp
);
2546 Procedure (ProcedureArgument
);
2551 CpuMpData
= GetCpuMpData ();
2554 // Check whether caller processor is BSP
2556 MpInitLibWhoAmI (&CallerNumber
);
2557 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2558 return EFI_DEVICE_ERROR
;
2564 CheckAndUpdateApsStatus ();
2566 ProcessorCount
= CpuMpData
->CpuCount
;
2567 HasEnabledAp
= FALSE
;
2569 // Check whether all enabled APs are idle.
2570 // If any enabled AP is not idle, return EFI_NOT_READY.
2572 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2573 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2574 if (ProcessorNumber
!= CpuMpData
->BspNumber
) {
2575 ApState
= GetApState (CpuData
);
2576 if (ApState
!= CpuStateDisabled
) {
2577 HasEnabledAp
= TRUE
;
2578 if (ApState
!= CpuStateIdle
) {
2580 // If any enabled APs are busy, return EFI_NOT_READY.
2582 return EFI_NOT_READY
;
2588 if (!HasEnabledAp
&& ExcludeBsp
) {
2590 // If no enabled AP exists and not include Bsp to do the procedure, return EFI_NOT_STARTED.
2592 return EFI_NOT_STARTED
;
2595 CpuMpData
->RunningCount
= 0;
2596 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2597 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2598 CpuData
->Waiting
= FALSE
;
2599 if (ProcessorNumber
!= CpuMpData
->BspNumber
) {
2600 if (CpuData
->State
== CpuStateIdle
) {
2602 // Mark this processor as responsible for current calling.
2604 CpuData
->Waiting
= TRUE
;
2605 CpuMpData
->RunningCount
++;
2610 CpuMpData
->Procedure
= Procedure
;
2611 CpuMpData
->ProcArguments
= ProcedureArgument
;
2612 CpuMpData
->SingleThread
= SingleThread
;
2613 CpuMpData
->FinishedCount
= 0;
2614 CpuMpData
->FailedCpuList
= FailedCpuList
;
2615 CpuMpData
->ExpectedTime
= CalculateTimeout (
2616 TimeoutInMicroseconds
,
2617 &CpuMpData
->CurrentTime
2619 CpuMpData
->TotalTime
= 0;
2620 CpuMpData
->WaitEvent
= WaitEvent
;
2622 if (!SingleThread
) {
2623 WakeUpAP (CpuMpData
, TRUE
, 0, Procedure
, ProcedureArgument
, FALSE
);
2625 for (ProcessorNumber
= 0; ProcessorNumber
< ProcessorCount
; ProcessorNumber
++) {
2626 if (ProcessorNumber
== CallerNumber
) {
2630 if (CpuMpData
->CpuData
[ProcessorNumber
].Waiting
) {
2631 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, Procedure
, ProcedureArgument
, TRUE
);
2641 Procedure (ProcedureArgument
);
2644 Status
= EFI_SUCCESS
;
2645 if (WaitEvent
== NULL
) {
2647 Status
= CheckAllAPs ();
2648 } while (Status
== EFI_NOT_READY
);
2655 Worker function to let the caller get one enabled AP to execute a caller-provided
2658 @param[in] Procedure A pointer to the function to be run on
2659 enabled APs of the system.
2660 @param[in] ProcessorNumber The handle number of the AP.
2661 @param[in] WaitEvent The event created by the caller with CreateEvent()
2663 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2664 APs to return from Procedure, either for
2665 blocking or non-blocking mode.
2666 @param[in] ProcedureArgument The parameter passed into Procedure for
2668 @param[out] Finished If AP returns from Procedure before the
2669 timeout expires, its content is set to TRUE.
2670 Otherwise, the value is set to FALSE.
2672 @retval EFI_SUCCESS In blocking mode, specified AP finished before
2673 the timeout expires.
2674 @retval others Failed to Startup AP.
2678 StartupThisAPWorker (
2679 IN EFI_AP_PROCEDURE Procedure
,
2680 IN UINTN ProcessorNumber
,
2681 IN EFI_EVENT WaitEvent OPTIONAL
,
2682 IN UINTN TimeoutInMicroseconds
,
2683 IN VOID
*ProcedureArgument OPTIONAL
,
2684 OUT BOOLEAN
*Finished OPTIONAL
2688 CPU_MP_DATA
*CpuMpData
;
2689 CPU_AP_DATA
*CpuData
;
2693 // In Td guest, startup of AP is not supported in current stage.
2695 if (CC_GUEST_IS_TDX (PcdGet64 (PcdConfidentialComputingGuestAttr
))) {
2696 return EFI_UNSUPPORTED
;
2699 CpuMpData
= GetCpuMpData ();
2701 if (Finished
!= NULL
) {
2706 // Check whether caller processor is BSP
2708 MpInitLibWhoAmI (&CallerNumber
);
2709 if (CallerNumber
!= CpuMpData
->BspNumber
) {
2710 return EFI_DEVICE_ERROR
;
2714 // Check whether processor with the handle specified by ProcessorNumber exists
2716 if (ProcessorNumber
>= CpuMpData
->CpuCount
) {
2717 return EFI_NOT_FOUND
;
2721 // Check whether specified processor is BSP
2723 if (ProcessorNumber
== CpuMpData
->BspNumber
) {
2724 return EFI_INVALID_PARAMETER
;
2728 // Check parameter Procedure
2730 if (Procedure
== NULL
) {
2731 return EFI_INVALID_PARAMETER
;
2737 CheckAndUpdateApsStatus ();
2740 // Check whether specified AP is disabled
2742 if (GetApState (&CpuMpData
->CpuData
[ProcessorNumber
]) == CpuStateDisabled
) {
2743 return EFI_INVALID_PARAMETER
;
2747 // If WaitEvent is not NULL, execute in non-blocking mode.
2748 // BSP saves data for CheckAPsStatus(), and returns EFI_SUCCESS.
2749 // CheckAPsStatus() will check completion and timeout periodically.
2751 CpuData
= &CpuMpData
->CpuData
[ProcessorNumber
];
2752 CpuData
->WaitEvent
= WaitEvent
;
2753 CpuData
->Finished
= Finished
;
2754 CpuData
->ExpectedTime
= CalculateTimeout (TimeoutInMicroseconds
, &CpuData
->CurrentTime
);
2755 CpuData
->TotalTime
= 0;
2757 WakeUpAP (CpuMpData
, FALSE
, ProcessorNumber
, Procedure
, ProcedureArgument
, TRUE
);
2760 // If WaitEvent is NULL, execute in blocking mode.
2761 // BSP checks AP's state until it finishes or TimeoutInMicrosecsond expires.
2763 Status
= EFI_SUCCESS
;
2764 if (WaitEvent
== NULL
) {
2766 Status
= CheckThisAP (ProcessorNumber
);
2767 } while (Status
== EFI_NOT_READY
);
2774 Get pointer to CPU MP Data structure from GUIDed HOB.
2776 @return The pointer to CPU MP Data structure.
2779 GetCpuMpDataFromGuidedHob (
2783 EFI_HOB_GUID_TYPE
*GuidHob
;
2785 CPU_MP_DATA
*CpuMpData
;
2788 GuidHob
= GetFirstGuidHob (&mCpuInitMpLibHobGuid
);
2789 if (GuidHob
!= NULL
) {
2790 DataInHob
= GET_GUID_HOB_DATA (GuidHob
);
2791 CpuMpData
= (CPU_MP_DATA
*)(*(UINTN
*)DataInHob
);
2798 This service executes a caller provided function on all enabled CPUs.
2800 @param[in] Procedure A pointer to the function to be run on
2801 enabled APs of the system. See type
2803 @param[in] TimeoutInMicroseconds Indicates the time limit in microseconds for
2804 APs to return from Procedure, either for
2805 blocking or non-blocking mode. Zero means
2806 infinity. TimeoutInMicroseconds is ignored
2808 @param[in] ProcedureArgument The parameter passed into Procedure for
2811 @retval EFI_SUCCESS In blocking mode, all CPUs have finished before
2812 the timeout expired.
2813 @retval EFI_SUCCESS In non-blocking mode, function has been dispatched
2814 to all enabled CPUs.
2815 @retval EFI_DEVICE_ERROR Caller processor is AP.
2816 @retval EFI_NOT_READY Any enabled APs are busy.
2817 @retval EFI_NOT_READY MP Initialize Library is not initialized.
2818 @retval EFI_TIMEOUT In blocking mode, the timeout expired before
2819 all enabled APs have finished.
2820 @retval EFI_INVALID_PARAMETER Procedure is NULL.
2825 MpInitLibStartupAllCPUs (
2826 IN EFI_AP_PROCEDURE Procedure
,
2827 IN UINTN TimeoutInMicroseconds
,
2828 IN VOID
*ProcedureArgument OPTIONAL
2831 return StartupAllCPUsWorker (
2836 TimeoutInMicroseconds
,
2843 The function check if the specified Attr is set.
2845 @param[in] CurrentAttr The current attribute.
2846 @param[in] Attr The attribute to check.
2848 @retval TRUE The specified Attr is set.
2849 @retval FALSE The specified Attr is not set.
2854 AmdMemEncryptionAttrCheck (
2855 IN UINT64 CurrentAttr
,
2856 IN CONFIDENTIAL_COMPUTING_GUEST_ATTR Attr
2862 // SEV is automatically enabled if SEV-ES or SEV-SNP is active.
2864 return CurrentAttr
>= CCAttrAmdSev
;
2865 case CCAttrAmdSevEs
:
2867 // SEV-ES is automatically enabled if SEV-SNP is active.
2869 return CurrentAttr
>= CCAttrAmdSevEs
;
2870 case CCAttrAmdSevSnp
:
2871 return CurrentAttr
== CCAttrAmdSevSnp
;
2878 Check if the specified confidential computing attribute is active.
2880 @param[in] Attr The attribute to check.
2882 @retval TRUE The specified Attr is active.
2883 @retval FALSE The specified Attr is not active.
2888 ConfidentialComputingGuestHas (
2889 IN CONFIDENTIAL_COMPUTING_GUEST_ATTR Attr
2895 // Get the current CC attribute.
2897 CurrentAttr
= PcdGet64 (PcdConfidentialComputingGuestAttr
);
2900 // If attr is for the AMD group then call AMD specific checks.
2902 if (((RShiftU64 (CurrentAttr
, 8)) & 0xff) == 1) {
2903 return AmdMemEncryptionAttrCheck (CurrentAttr
, Attr
);
2906 return (CurrentAttr
== Attr
);