1 ;------------------------------------------------------------------------------ ;
2 ; Copyright (c) 2015 - 2021, Intel Corporation. All rights reserved.<BR>
3 ; SPDX-License-Identifier: BSD-2-Clause-Patent
11 ; This is the assembly code for MP support
13 ;-------------------------------------------------------------------------------
16 extern ASM_PFX(InitializeFloatingPointUnits)
22 ;-------------------------------------------------------------------------------------
23 ;RendezvousFunnelProc procedure follows. All APs execute their procedure. This
24 ;procedure serializes all the AP processors through an Init sequence. It must be
25 ;noted that APs arrive here very raw...ie: real mode, no stack.
26 ;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC
28 ;-------------------------------------------------------------------------------------
29 global ASM_PFX(RendezvousFunnelProc)
30 ASM_PFX(RendezvousFunnelProc):
31 RendezvousFunnelProcStart:
32 ; At this point CS = 0x(vv00) and ip= 0x0.
33 ; Save BIST information to ebp firstly
36 mov ebp, eax ; Save BIST information
46 mov si, BufferStartLocation
49 mov si, DataSegmentLocation
53 ; Get start address of 32-bit code in low memory (<1MB)
55 mov edi, ModeTransitionMemoryLocation
64 ; Switch to protected mode
66 mov eax, cr0 ; Get control register 0
67 or eax, 000000003h ; Set PE bit (bit #0) & MP
70 ; Switch to 32-bit code (>1MB)
74 ; Following code must be copied to memory with type of EfiBootServicesCode.
75 ; This is required if NX is enabled for EfiBootServicesCode of memory.
78 Flat32Start: ; protected mode entry point
86 ; Enable execute disable bit
88 mov esi, EnableExecuteDisableLocation
89 cmp byte [ebx + esi], 0
90 jz SkipEnableExecuteDisableBit
92 mov ecx, 0c0000080h ; EFER MSR number
94 bts eax, 11 ; Enable Execute Disable Bit
97 SkipEnableExecuteDisableBit:
104 mov esi, Enable5LevelPagingLocation
105 cmp byte [ebx + esi], 0
106 jz SkipEnable5LevelPaging
109 ; Enable 5 Level Paging
111 bts eax, 12 ; Set LA57=1.
113 SkipEnable5LevelPaging:
120 mov esi, Cr3Location ; Save CR3 in ecx
122 mov cr3, ecx ; Load CR3
127 mov ecx, 0c0000080h ; EFER MSR number
129 bts eax, 8 ; Set LME=1
135 mov eax, cr0 ; Read CR0
136 bts eax, 31 ; Set PG=1
137 mov cr0, eax ; Write CR0
140 ; Far jump to 64-bit code
142 mov edi, ModeHighMemoryLocation
149 lea edi, [esi + InitFlagLocation]
150 cmp qword [edi], 1 ; ApInitConfig
153 ; Increment the number of APs executing here as early as possible
154 ; This is decremented in C code when AP is finished executing
156 add edi, NumApsExecutingLocation
161 add edi, LockLocation
162 mov rax, NotVacantFlag
165 add edi, ApIndexLocation
167 lock xadd dword [edi], ebx ; EBX = ApIndex++
168 inc ebx ; EBX is CpuNumber
172 add edi, StackSizeLocation
176 mul ecx ; EAX = StackSize * (CpuNumber + 1)
178 add edi, StackStartAddressLocation
182 lea edi, [esi + SevEsIsEnabledLocation]
183 cmp byte [edi], 1 ; SevEsIsEnabled
188 ; Each page after the GHCB is a per-CPU page, so the calculation programs
189 ; a GHCB to be every 8KB.
192 shl eax, 1 ; EAX = SIZE_4K * 2
194 mul ecx ; EAX = SIZE_4K * 2 * CpuNumber
196 add edi, GhcbBaseLocation
205 lea edi, [esi + SevEsIsEnabledLocation]
206 cmp byte [edi], 1 ; SevEsIsEnabled
210 ; Since we don't have a stack yet, we can't take a #VC
211 ; exception. Use the GHCB protocol to perform the CPUID
218 mov rdi, rax ; RDI now holds the original GHCB GPA
220 mov rdx, 0 ; CPUID function 0
221 mov rax, 0 ; RAX register requested
227 jb NoX2ApicSevEs ; CPUID level below CPUID_EXTENDED_TOPOLOGY
229 mov rdx, 0bh ; CPUID function 0x0b
230 mov rax, 040000000h ; RBX register requested
236 jz NoX2ApicSevEs ; CPUID.0BH:EBX[15:0] is zero
238 mov rdx, 0bh ; CPUID function 0x0b
239 mov rax, 0c0000000h ; RDX register requested
245 ; Processor is x2APIC capable; 32-bit x2APIC ID is now in EDX
249 ; Processor is not x2APIC capable, so get 8-bit APIC ID
250 mov rdx, 1 ; CPUID function 1
251 mov rax, 040000000h ; RBX register requested
259 mov rbx, rdx ; Save x2APIC/APIC ID
261 mov rdx, rdi ; RDI holds the saved GHCB GPA
268 ; x2APIC ID or APIC ID is in EDX
269 jmp GetProcessorNumber
275 jb NoX2Apic ; CPUID level below CPUID_EXTENDED_TOPOLOGY
281 jz NoX2Apic ; CPUID.0BH:EBX[15:0] is zero
283 ; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX
284 jmp GetProcessorNumber
287 ; Processor is not x2APIC capable, so get 8-bit APIC ID
295 ; Get processor number for this AP
296 ; Note that BSP may become an AP due to SwitchBsp()
299 lea eax, [esi + CpuInfoLocation]
303 cmp dword [rdi], edx ; APIC ID match?
307 jmp GetNextProcNumber
310 mov rsp, qword [rdi + 12]
313 push rbp ; Push BIST data at top of AP stack
314 xor rbp, rbp ; Clear ebp for call stack trace
318 mov rax, qword [esi + InitializeFloatingPointUnitsAddress]
320 call rax ; Call assembly function to initialize FPU per UEFI spec
323 mov edx, ebx ; edx is ApIndex
325 add ecx, LockLocation ; rcx is address of exchange info data buffer
328 add edi, ApProcedureLocation
332 call rax ; Invoke C function
334 jmp $ ; Should never reach here
336 RendezvousFunnelProcEnd:
338 ;-------------------------------------------------------------------------------------
339 ;SwitchToRealProc procedure follows.
340 ;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC
342 ; SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart)
344 ; rdx - Code16 Selector Offset
345 ; r8 - Code32 Selector Offset
347 ;-------------------------------------------------------------------------------------
348 global ASM_PFX(SwitchToRealProc)
349 ASM_PFX(SwitchToRealProc):
350 SwitchToRealProcStart:
355 ; Get RDX reset value before changing stacks since the
356 ; new stack won't be able to accomodate a #VC exception.
365 mov rsi, rax ; Save off the reset value for RDX
373 ; Establish stack below 1MB
378 ; Push ultimate Reset Vector onto the stack
382 push word 0x0002 ; RFLAGS
384 push word 0x0000 ; RIP
385 push word 0x0000 ; For alignment, will be discarded
388 ; Get address of "16-bit operand size" label
393 ; Push addresses used to change to compatibility mode
395 lea rax, [CompatMode]
400 ; Clear R8 - R15, for reset, before going into 32-bit mode
412 ; Far return into 32-bit mode
419 ; Set up stack to prepare for exiting protected mode
422 push ebx ; PM16Mode label address
427 mov eax, cr0 ; Read CR0
428 btr eax, 31 ; Set PG=0
429 mov cr0, eax ; Write CR0
434 mov ecx, 0c0000080h ; EFER MSR number
436 btr eax, 8 ; Set LME=0
442 mov eax, cr4 ; Read CR4
443 btr eax, 5 ; Set PAE=0
444 mov cr4, eax ; Write CR4
446 mov edx, esi ; Restore RDX reset value
449 ; Switch to 16-bit operand size
455 ; At entry to this label
456 ; - RDX will have its reset value
457 ; - On the top of the stack
458 ; - Alignment data (two bytes) to be discarded
459 ; - IP for Real Mode (two bytes)
460 ; - CS for Real Mode (two bytes)
462 ; This label is also used with AsmRelocateApLoop. During MP finalization,
463 ; the code from PM16Mode to SwitchToRealProcEnd is copied to the start of
464 ; the WakeupBuffer, allowing a parked AP to be booted by an OS.
467 mov eax, cr0 ; Read CR0
468 btr eax, 0 ; Set PE=0
469 mov cr0, eax ; Write CR0
471 pop ax ; Discard alignment data
474 ; Clear registers (except RDX and RSP) before going into 16-bit mode
487 ;-------------------------------------------------------------------------------------
488 ; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);
489 ;-------------------------------------------------------------------------------------
490 global ASM_PFX(AsmRelocateApLoop)
491 ASM_PFX(AsmRelocateApLoop):
492 AsmRelocateApLoopStart:
494 cmp qword [rsp + 56], 0 ; SevEsAPJumpTable
498 ; Perform some SEV-ES related setup before leaving 64-bit mode
504 ; Get the RDX reset value using CPUID
508 mov rsi, rax ; Save off the reset value for RDX
511 ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call
512 ; - Must be done while in 64-bit long mode so that writes to
513 ; the GHCB memory will be unencrypted.
514 ; - No NAE events can be generated once this is set otherwise
515 ; the AP_RESET_HOLD SW_EXITCODE will be overwritten.
518 rdmsr ; Retrieve current GHCB address
526 rep stosq ; Clear the GHCB
528 mov rax, 0x80000004 ; VMGEXIT AP_RESET_HOLD
529 mov [rdx + 0x390], rax
530 mov rax, 114 ; Set SwExitCode valid bit
531 bts [rdx + 0x3f0], rax
532 inc rax ; Set SwExitInfo1 valid bit
533 bts [rdx + 0x3f0], rax
534 inc rax ; Set SwExitInfo2 valid bit
535 bts [rdx + 0x3f0], rax
541 cli ; Disable interrupt before switching to 32-bit mode
542 mov rax, [rsp + 40] ; CountTofinish
543 lock dec dword [rax] ; (*CountTofinish)--
545 mov r10, [rsp + 48] ; Pm16CodeSegment
546 mov rax, [rsp + 56] ; SevEsAPJumpTable
547 mov rbx, [rsp + 64] ; WakeupBuffer
548 mov rsp, r9 ; TopOfApStack
550 push rax ; Save SevEsAPJumpTable
551 push rbx ; Save WakeupBuffer
552 push r10 ; Save Pm16CodeSegment
553 push rcx ; Save MwaitSupport
554 push rdx ; Save ApTargetCState
556 lea rax, [PmEntry] ; rax <- The start address of transition code
562 ; Clear R8 - R15, for reset, before going into 32-bit mode
574 ; Far return into 32-bit mode
581 btr eax, 31 ; Clear CR0.PG
582 mov cr0, eax ; Disable paging and caches
586 and ah, ~ 1 ; Clear LME
589 and al, ~ (1 << 5) ; Clear PAE
598 cmp cl, 1 ; Check mwait-monitor support
600 mov ebx, edx ; Save C-State to ebx
603 mov eax, esp ; Set Monitor Address
604 xor ecx, ecx ; ecx = 0
605 xor edx, edx ; edx = 0
607 mov eax, ebx ; Mwait Cx, Target C-State per eax[7:4]
613 pop edx ; PM16CodeSegment
615 pop ebx ; WakeupBuffer
617 pop eax ; SevEsAPJumpTable
619 cmp eax, 0 ; Check for SEV-ES
624 ; SEV-ES is enabled, use VMGEXIT (GHCB information already
632 ; Back from VMGEXIT AP_HLT_LOOP
633 ; Push the FLAGS/CS/IP values to use
635 push word 0x0002 ; EFLAGS
637 mov cx, [eax + 2] ; CS
641 push word 0x0000 ; For alignment, will be discarded
646 mov edx, esi ; Restore RDX reset value
656 AsmRelocateApLoopEnd:
658 ;-------------------------------------------------------------------------------------
659 ; AsmGetAddressMap (&AddressMap);
660 ;-------------------------------------------------------------------------------------
661 global ASM_PFX(AsmGetAddressMap)
662 ASM_PFX(AsmGetAddressMap):
663 lea rax, [ASM_PFX(RendezvousFunnelProc)]
665 mov qword [rcx + 8h], LongModeStart - RendezvousFunnelProcStart
666 mov qword [rcx + 10h], RendezvousFunnelProcEnd - RendezvousFunnelProcStart
667 lea rax, [ASM_PFX(AsmRelocateApLoop)]
668 mov qword [rcx + 18h], rax
669 mov qword [rcx + 20h], AsmRelocateApLoopEnd - AsmRelocateApLoopStart
670 mov qword [rcx + 28h], Flat32Start - RendezvousFunnelProcStart
671 mov qword [rcx + 30h], SwitchToRealProcEnd - SwitchToRealProcStart ; SwitchToRealSize
672 mov qword [rcx + 38h], SwitchToRealProcStart - RendezvousFunnelProcStart ; SwitchToRealOffset
673 mov qword [rcx + 40h], SwitchToRealProcStart - Flat32Start ; SwitchToRealNoNxOffset
674 mov qword [rcx + 48h], PM16Mode - RendezvousFunnelProcStart ; SwitchToRealPM16ModeOffset
675 mov qword [rcx + 50h], SwitchToRealProcEnd - PM16Mode ; SwitchToRealPM16ModeSize
678 ;-------------------------------------------------------------------------------------
679 ;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is
680 ;about to become an AP. It switches its stack with the current AP.
681 ;AsmExchangeRole (IN CPU_EXCHANGE_INFO *MyInfo, IN CPU_EXCHANGE_INFO *OthersInfo);
682 ;-------------------------------------------------------------------------------------
683 global ASM_PFX(AsmExchangeRole)
684 ASM_PFX(AsmExchangeRole):
685 ; DO NOT call other functions in this function, since 2 CPU may use 1 stack
686 ; at the same time. If 1 CPU try to call a function, stack will be corrupted.
710 ; rsi contains MyInfo pointer
713 ; rdi contains OthersInfo pointer
716 ;Store EFLAGS, GDTR and IDTR regiter to stack
721 ; Store the its StackPointer
724 ; update its switch state to STORED
725 mov byte [rsi], CPU_SWITCH_STATE_STORED
728 ; wait until the other CPU finish storing its state
729 cmp byte [rdi], CPU_SWITCH_STATE_STORED
732 jmp WaitForOtherStored
735 ; Since another CPU already stored its state, load them
742 ; load its future StackPointer
745 ; update the other CPU's switch state to LOADED
746 mov byte [rdi], CPU_SWITCH_STATE_LOADED
749 ; wait until the other CPU finish loading new state,
750 ; otherwise the data in stack may corrupt
751 cmp byte [rsi], CPU_SWITCH_STATE_LOADED
754 jmp WaitForOtherLoaded
757 ; since the other CPU already get the data it want, leave this procedure