1 ;------------------------------------------------------------------------------ ;
2 ; Copyright (c) 2015 - 2022, Intel Corporation. All rights reserved.<BR>
3 ; SPDX-License-Identifier: BSD-2-Clause-Patent
11 ; This is the assembly code for MP support
13 ;-------------------------------------------------------------------------------
16 extern ASM_PFX(InitializeFloatingPointUnits)
20 %1 %+ OneTimerCallReturn:
23 %macro OneTimeCallRet 1
24 jmp %1 %+ OneTimerCallReturn
31 ;-------------------------------------------------------------------------------------
32 ;RendezvousFunnelProc procedure follows. All APs execute their procedure. This
33 ;procedure serializes all the AP processors through an Init sequence. It must be
34 ;noted that APs arrive here very raw...ie: real mode, no stack.
35 ;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC
37 ;-------------------------------------------------------------------------------------
38 RendezvousFunnelProcStart:
39 ; At this point CS = 0x(vv00) and ip= 0x0.
40 ; Save BIST information to ebp firstly
43 mov ebp, eax ; Save BIST information
53 mov si, MP_CPU_EXCHANGE_INFO_FIELD (BufferStart)
56 mov si, MP_CPU_EXCHANGE_INFO_FIELD (DataSegment)
60 ; Get start address of 32-bit code in low memory (<1MB)
62 mov edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeTransitionMemory)
64 mov si, MP_CPU_EXCHANGE_INFO_FIELD (GdtrProfile)
67 mov si, MP_CPU_EXCHANGE_INFO_FIELD (IdtrProfile)
71 ; Switch to protected mode
73 mov eax, cr0 ; Get control register 0
74 or eax, 000000003h ; Set PE bit (bit #0) & MP
77 ; Switch to 32-bit code (>1MB)
81 ; Following code must be copied to memory with type of EfiBootServicesCode.
82 ; This is required if NX is enabled for EfiBootServicesCode of memory.
85 Flat32Start: ; protected mode entry point
93 ; Enable execute disable bit
95 mov esi, MP_CPU_EXCHANGE_INFO_FIELD (EnableExecuteDisable)
96 cmp byte [ebx + esi], 0
97 jz SkipEnableExecuteDisableBit
99 mov ecx, 0c0000080h ; EFER MSR number
101 bts eax, 11 ; Enable Execute Disable Bit
104 SkipEnableExecuteDisableBit:
111 mov esi, MP_CPU_EXCHANGE_INFO_FIELD (Enable5LevelPaging)
112 cmp byte [ebx + esi], 0
113 jz SkipEnable5LevelPaging
116 ; Enable 5 Level Paging
118 bts eax, 12 ; Set LA57=1.
120 SkipEnable5LevelPaging:
127 mov esi, MP_CPU_EXCHANGE_INFO_FIELD (Cr3) ; Save CR3 in ecx
129 mov cr3, ecx ; Load CR3
134 mov ecx, 0c0000080h ; EFER MSR number
136 bts eax, 8 ; Set LME=1
142 mov eax, cr0 ; Read CR0
143 bts eax, 31 ; Set PG=1
144 mov cr0, eax ; Write CR0
147 ; Far jump to 64-bit code
149 mov edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeHighMemory)
156 ; Required for the AMD SEV helper functions
158 %include "AmdSev.nasm"
162 lea edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitFlag)]
163 cmp qword [edi], 1 ; ApInitConfig
166 ; Increment the number of APs executing here as early as possible
167 ; This is decremented in C code when AP is finished executing
169 add edi, MP_CPU_EXCHANGE_INFO_FIELD (NumApsExecuting)
174 add edi, MP_CPU_EXCHANGE_INFO_FIELD (ApIndex)
176 lock xadd dword [edi], ebx ; EBX = ApIndex++
177 inc ebx ; EBX is CpuNumber
181 add edi, MP_CPU_EXCHANGE_INFO_FIELD (StackSize)
185 mul ecx ; EAX = StackSize * (CpuNumber + 1)
187 add edi, MP_CPU_EXCHANGE_INFO_FIELD (StackStart)
192 ; Setup the GHCB when AMD SEV-ES active.
194 OneTimeCall SevEsSetupGhcb
199 ; Use the GHCB protocol to get the ApicId when SEV-ES is active.
201 OneTimeCall SevEsGetApicId
207 jb NoX2Apic ; CPUID level below CPUID_EXTENDED_TOPOLOGY
213 jz NoX2Apic ; CPUID.0BH:EBX[15:0] is zero
215 ; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX
216 jmp GetProcessorNumber
219 ; Processor is not x2APIC capable, so get 8-bit APIC ID
227 ; Get processor number for this AP
228 ; Note that BSP may become an AP due to SwitchBsp()
231 lea eax, [esi + MP_CPU_EXCHANGE_INFO_FIELD (CpuInfo)]
235 cmp dword [rdi + CPU_INFO_IN_HOB.InitialApicId], edx ; APIC ID match?
237 add rdi, CPU_INFO_IN_HOB_size
239 jmp GetNextProcNumber
242 mov rsp, qword [rdi + CPU_INFO_IN_HOB.ApTopOfStack]
245 push rbp ; Push BIST data at top of AP stack
246 xor rbp, rbp ; Clear ebp for call stack trace
250 mov rax, qword [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitializeFloatingPointUnits)]
252 call rax ; Call assembly function to initialize FPU per UEFI spec
255 mov edx, ebx ; edx is ApIndex
257 add ecx, MP_CPU_EXCHANGE_INFO_OFFSET ; rcx is address of exchange info data buffer
260 add edi, MP_CPU_EXCHANGE_INFO_FIELD (CFunction)
264 call rax ; Invoke C function
266 jmp $ ; Should never reach here
268 RendezvousFunnelProcEnd:
270 ;-------------------------------------------------------------------------------------
271 ;SwitchToRealProc procedure follows.
272 ;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC
274 ; SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart)
276 ; rdx - Code16 Selector Offset
277 ; r8 - Code32 Selector Offset
279 ;-------------------------------------------------------------------------------------
280 SwitchToRealProcStart:
285 ; Get RDX reset value before changing stacks since the
286 ; new stack won't be able to accomodate a #VC exception.
295 mov rsi, rax ; Save off the reset value for RDX
303 ; Establish stack below 1MB
308 ; Push ultimate Reset Vector onto the stack
312 push word 0x0002 ; RFLAGS
314 push word 0x0000 ; RIP
315 push word 0x0000 ; For alignment, will be discarded
318 ; Get address of "16-bit operand size" label
323 ; Push addresses used to change to compatibility mode
325 lea rax, [CompatMode]
330 ; Clear R8 - R15, for reset, before going into 32-bit mode
342 ; Far return into 32-bit mode
349 ; Set up stack to prepare for exiting protected mode
352 push ebx ; PM16Mode label address
357 mov eax, cr0 ; Read CR0
358 btr eax, 31 ; Set PG=0
359 mov cr0, eax ; Write CR0
364 mov ecx, 0c0000080h ; EFER MSR number
366 btr eax, 8 ; Set LME=0
372 mov eax, cr4 ; Read CR4
373 btr eax, 5 ; Set PAE=0
374 mov cr4, eax ; Write CR4
376 mov edx, esi ; Restore RDX reset value
379 ; Switch to 16-bit operand size
385 ; At entry to this label
386 ; - RDX will have its reset value
387 ; - On the top of the stack
388 ; - Alignment data (two bytes) to be discarded
389 ; - IP for Real Mode (two bytes)
390 ; - CS for Real Mode (two bytes)
392 ; This label is also used with AsmRelocateApLoop. During MP finalization,
393 ; the code from PM16Mode to SwitchToRealProcEnd is copied to the start of
394 ; the WakeupBuffer, allowing a parked AP to be booted by an OS.
397 mov eax, cr0 ; Read CR0
398 btr eax, 0 ; Set PE=0
399 mov cr0, eax ; Write CR0
401 pop ax ; Discard alignment data
404 ; Clear registers (except RDX and RSP) before going into 16-bit mode
417 ;-------------------------------------------------------------------------------------
418 ; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);
419 ;-------------------------------------------------------------------------------------
420 AsmRelocateApLoopStart:
422 cmp qword [rsp + 56], 0 ; SevEsAPJumpTable
426 ; Perform some SEV-ES related setup before leaving 64-bit mode
432 ; Get the RDX reset value using CPUID
436 mov rsi, rax ; Save off the reset value for RDX
439 ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call
440 ; - Must be done while in 64-bit long mode so that writes to
441 ; the GHCB memory will be unencrypted.
442 ; - No NAE events can be generated once this is set otherwise
443 ; the AP_RESET_HOLD SW_EXITCODE will be overwritten.
446 rdmsr ; Retrieve current GHCB address
454 rep stosq ; Clear the GHCB
456 mov rax, 0x80000004 ; VMGEXIT AP_RESET_HOLD
457 mov [rdx + 0x390], rax
458 mov rax, 114 ; Set SwExitCode valid bit
459 bts [rdx + 0x3f0], rax
460 inc rax ; Set SwExitInfo1 valid bit
461 bts [rdx + 0x3f0], rax
462 inc rax ; Set SwExitInfo2 valid bit
463 bts [rdx + 0x3f0], rax
469 cli ; Disable interrupt before switching to 32-bit mode
470 mov rax, [rsp + 40] ; CountTofinish
471 lock dec dword [rax] ; (*CountTofinish)--
473 mov r10, [rsp + 48] ; Pm16CodeSegment
474 mov rax, [rsp + 56] ; SevEsAPJumpTable
475 mov rbx, [rsp + 64] ; WakeupBuffer
476 mov rsp, r9 ; TopOfApStack
478 push rax ; Save SevEsAPJumpTable
479 push rbx ; Save WakeupBuffer
480 push r10 ; Save Pm16CodeSegment
481 push rcx ; Save MwaitSupport
482 push rdx ; Save ApTargetCState
484 lea rax, [PmEntry] ; rax <- The start address of transition code
490 ; Clear R8 - R15, for reset, before going into 32-bit mode
502 ; Far return into 32-bit mode
509 btr eax, 31 ; Clear CR0.PG
510 mov cr0, eax ; Disable paging and caches
514 and ah, ~ 1 ; Clear LME
517 and al, ~ (1 << 5) ; Clear PAE
526 cmp cl, 1 ; Check mwait-monitor support
528 mov ebx, edx ; Save C-State to ebx
531 mov eax, esp ; Set Monitor Address
532 xor ecx, ecx ; ecx = 0
533 xor edx, edx ; edx = 0
535 mov eax, ebx ; Mwait Cx, Target C-State per eax[7:4]
541 pop edx ; PM16CodeSegment
543 pop ebx ; WakeupBuffer
545 pop eax ; SevEsAPJumpTable
547 cmp eax, 0 ; Check for SEV-ES
552 ; SEV-ES is enabled, use VMGEXIT (GHCB information already
560 ; Back from VMGEXIT AP_HLT_LOOP
561 ; Push the FLAGS/CS/IP values to use
563 push word 0x0002 ; EFLAGS
565 mov cx, [eax + 2] ; CS
569 push word 0x0000 ; For alignment, will be discarded
574 mov edx, esi ; Restore RDX reset value
584 AsmRelocateApLoopEnd:
586 ;-------------------------------------------------------------------------------------
587 ; AsmGetAddressMap (&AddressMap);
588 ;-------------------------------------------------------------------------------------
589 global ASM_PFX(AsmGetAddressMap)
590 ASM_PFX(AsmGetAddressMap):
591 lea rax, [RendezvousFunnelProcStart]
592 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelAddress], rax
593 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeEntryOffset], LongModeStart - RendezvousFunnelProcStart
594 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelSize], RendezvousFunnelProcEnd - RendezvousFunnelProcStart
595 lea rax, [AsmRelocateApLoopStart]
596 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncAddress], rax
597 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncSize], AsmRelocateApLoopEnd - AsmRelocateApLoopStart
598 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeTransitionOffset], Flat32Start - RendezvousFunnelProcStart
599 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealSize], SwitchToRealProcEnd - SwitchToRealProcStart
600 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealOffset], SwitchToRealProcStart - RendezvousFunnelProcStart
601 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealNoNxOffset], SwitchToRealProcStart - Flat32Start
602 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeOffset], PM16Mode - RendezvousFunnelProcStart
603 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeSize], SwitchToRealProcEnd - PM16Mode
606 ;-------------------------------------------------------------------------------------
607 ;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is
608 ;about to become an AP. It switches its stack with the current AP.
609 ;AsmExchangeRole (IN CPU_EXCHANGE_INFO *MyInfo, IN CPU_EXCHANGE_INFO *OthersInfo);
610 ;-------------------------------------------------------------------------------------
611 global ASM_PFX(AsmExchangeRole)
612 ASM_PFX(AsmExchangeRole):
613 ; DO NOT call other functions in this function, since 2 CPU may use 1 stack
614 ; at the same time. If 1 CPU try to call a function, stack will be corrupted.
638 ; rsi contains MyInfo pointer
641 ; rdi contains OthersInfo pointer
644 ;Store EFLAGS, GDTR and IDTR regiter to stack
646 sgdt [rsi + CPU_EXCHANGE_ROLE_INFO.Gdtr]
647 sidt [rsi + CPU_EXCHANGE_ROLE_INFO.Idtr]
649 ; Store the its StackPointer
650 mov [rsi + CPU_EXCHANGE_ROLE_INFO.StackPointer], rsp
652 ; update its switch state to STORED
653 mov byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
656 ; wait until the other CPU finish storing its state
657 cmp byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
660 jmp WaitForOtherStored
663 ; Since another CPU already stored its state, load them
665 lgdt [rdi + CPU_EXCHANGE_ROLE_INFO.Gdtr]
668 lidt [rdi + CPU_EXCHANGE_ROLE_INFO.Idtr]
670 ; load its future StackPointer
671 mov rsp, [rdi + CPU_EXCHANGE_ROLE_INFO.StackPointer]
673 ; update the other CPU's switch state to LOADED
674 mov byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
677 ; wait until the other CPU finish loading new state,
678 ; otherwise the data in stack may corrupt
679 cmp byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
682 jmp WaitForOtherLoaded
685 ; since the other CPU already get the data it want, leave this procedure