1 ;------------------------------------------------------------------------------ ;
2 ; Copyright (c) 2015 - 2019, Intel Corporation. All rights reserved.<BR>
3 ; SPDX-License-Identifier: BSD-2-Clause-Patent
11 ; This is the assembly code for MP support
13 ;-------------------------------------------------------------------------------
16 extern ASM_PFX(InitializeFloatingPointUnits)
22 ;-------------------------------------------------------------------------------------
23 ;RendezvousFunnelProc procedure follows. All APs execute their procedure. This
24 ;procedure serializes all the AP processors through an Init sequence. It must be
25 ;noted that APs arrive here very raw...ie: real mode, no stack.
26 ;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC
28 ;-------------------------------------------------------------------------------------
29 global ASM_PFX(RendezvousFunnelProc)
30 ASM_PFX(RendezvousFunnelProc):
31 RendezvousFunnelProcStart:
32 ; At this point CS = 0x(vv00) and ip= 0x0.
33 ; Save BIST information to ebp firstly
36 mov ebp, eax ; Save BIST information
46 mov si, BufferStartLocation
49 mov si, DataSegmentLocation
53 ; Get start address of 32-bit code in low memory (<1MB)
55 mov edi, ModeTransitionMemoryLocation
64 ; Switch to protected mode
66 mov eax, cr0 ; Get control register 0
67 or eax, 000000003h ; Set PE bit (bit #0) & MP
70 ; Switch to 32-bit code (>1MB)
74 ; Following code must be copied to memory with type of EfiBootServicesCode.
75 ; This is required if NX is enabled for EfiBootServicesCode of memory.
78 Flat32Start: ; protected mode entry point
86 ; Enable execute disable bit
88 mov esi, EnableExecuteDisableLocation
89 cmp byte [ebx + esi], 0
90 jz SkipEnableExecuteDisableBit
92 mov ecx, 0c0000080h ; EFER MSR number
94 bts eax, 11 ; Enable Execute Disable Bit
97 SkipEnableExecuteDisableBit:
104 mov esi, Enable5LevelPagingLocation
105 cmp byte [ebx + esi], 0
106 jz SkipEnable5LevelPaging
109 ; Enable 5 Level Paging
111 bts eax, 12 ; Set LA57=1.
113 SkipEnable5LevelPaging:
120 mov esi, Cr3Location ; Save CR3 in ecx
122 mov cr3, ecx ; Load CR3
127 mov ecx, 0c0000080h ; EFER MSR number
129 bts eax, 8 ; Set LME=1
135 mov eax, cr0 ; Read CR0
136 bts eax, 31 ; Set PG=1
137 mov cr0, eax ; Write CR0
140 ; Far jump to 64-bit code
142 mov edi, ModeHighMemoryLocation
149 lea edi, [esi + InitFlagLocation]
150 cmp qword [edi], 1 ; ApInitConfig
153 ; Increment the number of APs executing here as early as possible
154 ; This is decremented in C code when AP is finished executing
156 add edi, NumApsExecutingLocation
161 add edi, LockLocation
162 mov rax, NotVacantFlag
165 xchg qword [edi], rax
166 cmp rax, NotVacantFlag
169 lea ecx, [esi + ApIndexLocation]
175 xchg qword [edi], rax
178 add edi, StackSizeLocation
182 mul ecx ; EAX = StackSize * (CpuNumber + 1)
184 add edi, StackStartAddressLocation
188 lea edi, [esi + SevEsIsEnabledLocation]
189 cmp byte [edi], 1 ; SevEsIsEnabled
194 ; Each page after the GHCB is a per-CPU page, so the calculation programs
195 ; a GHCB to be every 8KB.
198 shl eax, 1 ; EAX = SIZE_4K * 2
200 mul ecx ; EAX = SIZE_4K * 2 * CpuNumber
202 add edi, GhcbBaseLocation
211 lea edi, [esi + SevEsIsEnabledLocation]
212 cmp byte [edi], 1 ; SevEsIsEnabled
216 ; Since we don't have a stack yet, we can't take a #VC
217 ; exception. Use the GHCB protocol to perform the CPUID
224 mov rdi, rax ; RDI now holds the original GHCB GPA
226 mov rdx, 0 ; CPUID function 0
227 mov rax, 0 ; RAX register requested
233 jb NoX2ApicSevEs ; CPUID level below CPUID_EXTENDED_TOPOLOGY
235 mov rdx, 0bh ; CPUID function 0x0b
236 mov rax, 040000000h ; RBX register requested
242 jz NoX2ApicSevEs ; CPUID.0BH:EBX[15:0] is zero
244 mov rdx, 0bh ; CPUID function 0x0b
245 mov rax, 0c0000000h ; RDX register requested
251 ; Processor is x2APIC capable; 32-bit x2APIC ID is now in EDX
255 ; Processor is not x2APIC capable, so get 8-bit APIC ID
256 mov rdx, 1 ; CPUID function 1
257 mov rax, 040000000h ; RBX register requested
265 mov rbx, rdx ; Save x2APIC/APIC ID
267 mov rdx, rdi ; RDI holds the saved GHCB GPA
274 ; x2APIC ID or APIC ID is in EDX
275 jmp GetProcessorNumber
281 jb NoX2Apic ; CPUID level below CPUID_EXTENDED_TOPOLOGY
287 jz NoX2Apic ; CPUID.0BH:EBX[15:0] is zero
289 ; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX
290 jmp GetProcessorNumber
293 ; Processor is not x2APIC capable, so get 8-bit APIC ID
301 ; Get processor number for this AP
302 ; Note that BSP may become an AP due to SwitchBsp()
305 lea eax, [esi + CpuInfoLocation]
309 cmp dword [rdi], edx ; APIC ID match?
313 jmp GetNextProcNumber
316 mov rsp, qword [rdi + 12]
319 push rbp ; Push BIST data at top of AP stack
320 xor rbp, rbp ; Clear ebp for call stack trace
324 mov rax, qword [esi + InitializeFloatingPointUnitsAddress]
326 call rax ; Call assembly function to initialize FPU per UEFI spec
329 mov edx, ebx ; edx is ApIndex
331 add ecx, LockLocation ; rcx is address of exchange info data buffer
334 add edi, ApProcedureLocation
338 call rax ; Invoke C function
340 jmp $ ; Should never reach here
342 RendezvousFunnelProcEnd:
344 ;-------------------------------------------------------------------------------------
345 ;SwitchToRealProc procedure follows.
346 ;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC
348 ; SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart)
350 ; rdx - Code16 Selector Offset
351 ; r8 - Code32 Selector Offset
353 ;-------------------------------------------------------------------------------------
354 global ASM_PFX(SwitchToRealProc)
355 ASM_PFX(SwitchToRealProc):
356 SwitchToRealProcStart:
361 ; Get RDX reset value before changing stacks since the
362 ; new stack won't be able to accomodate a #VC exception.
371 mov rsi, rax ; Save off the reset value for RDX
379 ; Establish stack below 1MB
384 ; Push ultimate Reset Vector onto the stack
388 push word 0x0002 ; RFLAGS
390 push word 0x0000 ; RIP
391 push word 0x0000 ; For alignment, will be discarded
394 ; Get address of "16-bit operand size" label
399 ; Push addresses used to change to compatibility mode
401 lea rax, [CompatMode]
406 ; Clear R8 - R15, for reset, before going into 32-bit mode
418 ; Far return into 32-bit mode
425 ; Set up stack to prepare for exiting protected mode
428 push ebx ; PM16Mode label address
433 mov eax, cr0 ; Read CR0
434 btr eax, 31 ; Set PG=0
435 mov cr0, eax ; Write CR0
440 mov ecx, 0c0000080h ; EFER MSR number
442 btr eax, 8 ; Set LME=0
448 mov eax, cr4 ; Read CR4
449 btr eax, 5 ; Set PAE=0
450 mov cr4, eax ; Write CR4
452 mov edx, esi ; Restore RDX reset value
455 ; Switch to 16-bit operand size
461 ; At entry to this label
462 ; - RDX will have its reset value
463 ; - On the top of the stack
464 ; - Alignment data (two bytes) to be discarded
465 ; - IP for Real Mode (two bytes)
466 ; - CS for Real Mode (two bytes)
468 ; This label is also used with AsmRelocateApLoop. During MP finalization,
469 ; the code from PM16Mode to SwitchToRealProcEnd is copied to the start of
470 ; the WakeupBuffer, allowing a parked AP to be booted by an OS.
473 mov eax, cr0 ; Read CR0
474 btr eax, 0 ; Set PE=0
475 mov cr0, eax ; Write CR0
477 pop ax ; Discard alignment data
480 ; Clear registers (except RDX and RSP) before going into 16-bit mode
493 ;-------------------------------------------------------------------------------------
494 ; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);
495 ;-------------------------------------------------------------------------------------
496 global ASM_PFX(AsmRelocateApLoop)
497 ASM_PFX(AsmRelocateApLoop):
498 AsmRelocateApLoopStart:
500 cmp qword [rsp + 56], 0 ; SevEsAPJumpTable
504 ; Perform some SEV-ES related setup before leaving 64-bit mode
510 ; Get the RDX reset value using CPUID
514 mov rsi, rax ; Save off the reset value for RDX
517 ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call
518 ; - Must be done while in 64-bit long mode so that writes to
519 ; the GHCB memory will be unencrypted.
520 ; - No NAE events can be generated once this is set otherwise
521 ; the AP_RESET_HOLD SW_EXITCODE will be overwritten.
524 rdmsr ; Retrieve current GHCB address
532 rep stosq ; Clear the GHCB
534 mov rax, 0x80000004 ; VMGEXIT AP_RESET_HOLD
535 mov [rdx + 0x390], rax
536 mov rax, 114 ; Set SwExitCode valid bit
537 bts [rdx + 0x3f0], rax
538 inc rax ; Set SwExitInfo1 valid bit
539 bts [rdx + 0x3f0], rax
540 inc rax ; Set SwExitInfo2 valid bit
541 bts [rdx + 0x3f0], rax
547 cli ; Disable interrupt before switching to 32-bit mode
548 mov rax, [rsp + 40] ; CountTofinish
549 lock dec dword [rax] ; (*CountTofinish)--
551 mov r10, [rsp + 48] ; Pm16CodeSegment
552 mov rax, [rsp + 56] ; SevEsAPJumpTable
553 mov rbx, [rsp + 64] ; WakeupBuffer
554 mov rsp, r9 ; TopOfApStack
556 push rax ; Save SevEsAPJumpTable
557 push rbx ; Save WakeupBuffer
558 push r10 ; Save Pm16CodeSegment
559 push rcx ; Save MwaitSupport
560 push rdx ; Save ApTargetCState
562 lea rax, [PmEntry] ; rax <- The start address of transition code
568 ; Clear R8 - R15, for reset, before going into 32-bit mode
580 ; Far return into 32-bit mode
587 btr eax, 31 ; Clear CR0.PG
588 mov cr0, eax ; Disable paging and caches
592 and ah, ~ 1 ; Clear LME
595 and al, ~ (1 << 5) ; Clear PAE
604 cmp cl, 1 ; Check mwait-monitor support
606 mov ebx, edx ; Save C-State to ebx
609 mov eax, esp ; Set Monitor Address
610 xor ecx, ecx ; ecx = 0
611 xor edx, edx ; edx = 0
613 mov eax, ebx ; Mwait Cx, Target C-State per eax[7:4]
619 pop edx ; PM16CodeSegment
621 pop ebx ; WakeupBuffer
623 pop eax ; SevEsAPJumpTable
625 cmp eax, 0 ; Check for SEV-ES
630 ; SEV-ES is enabled, use VMGEXIT (GHCB information already
638 ; Back from VMGEXIT AP_HLT_LOOP
639 ; Push the FLAGS/CS/IP values to use
641 push word 0x0002 ; EFLAGS
643 mov cx, [eax + 2] ; CS
647 push word 0x0000 ; For alignment, will be discarded
652 mov edx, esi ; Restore RDX reset value
662 AsmRelocateApLoopEnd:
664 ;-------------------------------------------------------------------------------------
665 ; AsmGetAddressMap (&AddressMap);
666 ;-------------------------------------------------------------------------------------
667 global ASM_PFX(AsmGetAddressMap)
668 ASM_PFX(AsmGetAddressMap):
669 lea rax, [ASM_PFX(RendezvousFunnelProc)]
671 mov qword [rcx + 8h], LongModeStart - RendezvousFunnelProcStart
672 mov qword [rcx + 10h], RendezvousFunnelProcEnd - RendezvousFunnelProcStart
673 lea rax, [ASM_PFX(AsmRelocateApLoop)]
674 mov qword [rcx + 18h], rax
675 mov qword [rcx + 20h], AsmRelocateApLoopEnd - AsmRelocateApLoopStart
676 mov qword [rcx + 28h], Flat32Start - RendezvousFunnelProcStart
677 mov qword [rcx + 30h], SwitchToRealProcEnd - SwitchToRealProcStart ; SwitchToRealSize
678 mov qword [rcx + 38h], SwitchToRealProcStart - RendezvousFunnelProcStart ; SwitchToRealOffset
679 mov qword [rcx + 40h], SwitchToRealProcStart - Flat32Start ; SwitchToRealNoNxOffset
680 mov qword [rcx + 48h], PM16Mode - RendezvousFunnelProcStart ; SwitchToRealPM16ModeOffset
681 mov qword [rcx + 50h], SwitchToRealProcEnd - PM16Mode ; SwitchToRealPM16ModeSize
684 ;-------------------------------------------------------------------------------------
685 ;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is
686 ;about to become an AP. It switches its stack with the current AP.
687 ;AsmExchangeRole (IN CPU_EXCHANGE_INFO *MyInfo, IN CPU_EXCHANGE_INFO *OthersInfo);
688 ;-------------------------------------------------------------------------------------
689 global ASM_PFX(AsmExchangeRole)
690 ASM_PFX(AsmExchangeRole):
691 ; DO NOT call other functions in this function, since 2 CPU may use 1 stack
692 ; at the same time. If 1 CPU try to call a function, stack will be corrupted.
716 ; rsi contains MyInfo pointer
719 ; rdi contains OthersInfo pointer
722 ;Store EFLAGS, GDTR and IDTR regiter to stack
727 ; Store the its StackPointer
730 ; update its switch state to STORED
731 mov byte [rsi], CPU_SWITCH_STATE_STORED
734 ; wait until the other CPU finish storing its state
735 cmp byte [rdi], CPU_SWITCH_STATE_STORED
738 jmp WaitForOtherStored
741 ; Since another CPU already stored its state, load them
748 ; load its future StackPointer
751 ; update the other CPU's switch state to LOADED
752 mov byte [rdi], CPU_SWITCH_STATE_LOADED
755 ; wait until the other CPU finish loading new state,
756 ; otherwise the data in stack may corrupt
757 cmp byte [rsi], CPU_SWITCH_STATE_LOADED
760 jmp WaitForOtherLoaded
763 ; since the other CPU already get the data it want, leave this procedure