1 ;------------------------------------------------------------------------------ ;
2 ; Copyright (c) 2015 - 2023, Intel Corporation. All rights reserved.<BR>
3 ; SPDX-License-Identifier: BSD-2-Clause-Patent
11 ; This is the assembly code for MP support
13 ;-------------------------------------------------------------------------------
16 extern ASM_PFX(InitializeFloatingPointUnits)
20 %1 %+ OneTimerCallReturn:
23 %macro OneTimeCallRet 1
24 jmp %1 %+ OneTimerCallReturn
31 ;-------------------------------------------------------------------------------------
32 ;RendezvousFunnelProc procedure follows. All APs execute their procedure. This
33 ;procedure serializes all the AP processors through an Init sequence. It must be
34 ;noted that APs arrive here very raw...ie: real mode, no stack.
35 ;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC
37 ;-------------------------------------------------------------------------------------
38 RendezvousFunnelProcStart:
39 ; At this point CS = 0x(vv00) and ip= 0x0.
40 ; Save BIST information to ebp firstly
43 mov ebp, eax ; Save BIST information
53 mov si, MP_CPU_EXCHANGE_INFO_FIELD (BufferStart)
56 mov si, MP_CPU_EXCHANGE_INFO_FIELD (DataSegment)
60 ; Get start address of 32-bit code in low memory (<1MB)
62 mov edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeTransitionMemory)
64 mov si, MP_CPU_EXCHANGE_INFO_FIELD (GdtrProfile)
68 ; Switch to protected mode
70 mov eax, cr0 ; Get control register 0
71 or eax, 000000003h ; Set PE bit (bit #0) & MP
74 ; Switch to 32-bit code (>1MB)
78 ; Following code must be copied to memory with type of EfiBootServicesCode.
79 ; This is required if NX is enabled for EfiBootServicesCode of memory.
82 Flat32Start: ; protected mode entry point
90 ; Enable execute disable bit
92 mov esi, MP_CPU_EXCHANGE_INFO_FIELD (EnableExecuteDisable)
93 cmp byte [ebx + esi], 0
94 jz SkipEnableExecuteDisableBit
96 mov ecx, 0c0000080h ; EFER MSR number
98 bts eax, 11 ; Enable Execute Disable Bit
101 SkipEnableExecuteDisableBit:
108 mov esi, MP_CPU_EXCHANGE_INFO_FIELD (Enable5LevelPaging)
109 cmp byte [ebx + esi], 0
110 jz SkipEnable5LevelPaging
113 ; Enable 5 Level Paging
115 bts eax, 12 ; Set LA57=1.
117 SkipEnable5LevelPaging:
124 mov esi, MP_CPU_EXCHANGE_INFO_FIELD (Cr3) ; Save CR3 in ecx
126 mov cr3, ecx ; Load CR3
131 mov ecx, 0c0000080h ; EFER MSR number
133 bts eax, 8 ; Set LME=1
139 mov eax, cr0 ; Read CR0
140 bts eax, 31 ; Set PG=1
141 mov cr0, eax ; Write CR0
144 ; Far jump to 64-bit code
146 mov edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeHighMemory)
155 ; Set IDT table at the start of 64 bit code
156 lea edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (IdtrProfile)]
159 lea edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitFlag)]
160 cmp qword [edi], 1 ; ApInitConfig
163 ; Increment the number of APs executing here as early as possible
164 ; This is decremented in C code when AP is finished executing
166 add edi, MP_CPU_EXCHANGE_INFO_FIELD (NumApsExecuting)
171 add edi, MP_CPU_EXCHANGE_INFO_FIELD (ApIndex)
173 lock xadd dword [edi], ebx ; EBX = ApIndex++
174 inc ebx ; EBX is CpuNumber
178 add edi, MP_CPU_EXCHANGE_INFO_FIELD (StackSize)
182 mul ecx ; EAX = StackSize * (CpuNumber + 1)
184 add edi, MP_CPU_EXCHANGE_INFO_FIELD (StackStart)
189 ; Setup the GHCB when AMD SEV-ES active.
191 OneTimeCall SevEsSetupGhcb
196 ; Use the GHCB protocol to get the ApicId when SEV-ES is active.
198 OneTimeCall SevEsGetApicId
204 jb NoX2Apic ; CPUID level below CPUID_EXTENDED_TOPOLOGY
210 jz NoX2Apic ; CPUID.0BH:EBX[15:0] is zero
212 ; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX
213 jmp GetProcessorNumber
216 ; Processor is not x2APIC capable, so get 8-bit APIC ID
224 ; Get processor number for this AP
225 ; Note that BSP may become an AP due to SwitchBsp()
228 lea eax, [esi + MP_CPU_EXCHANGE_INFO_FIELD (CpuInfo)]
232 cmp dword [rdi + CPU_INFO_IN_HOB.InitialApicId], edx ; APIC ID match?
234 add rdi, CPU_INFO_IN_HOB_size
236 jmp GetNextProcNumber
239 mov rsp, qword [rdi + CPU_INFO_IN_HOB.ApTopOfStack]
243 ; Reserve 8 bytes for CpuMpData.
244 ; When the AP wakes up again via INIT-SIPI-SIPI, push 0 will cause the existing CpuMpData to be overwritten with 0.
245 ; CpuMpData is filled in via InitializeApData() during the first time INIT-SIPI-SIPI,
246 ; while overwirrten may occurs when under ApInHltLoop but InitFlag is not set to ApInitConfig.
247 ; Therefore reservation is implemented by sub rsp instead of push 0.
250 push rbp ; Push BIST data at top of AP stack
251 xor rbp, rbp ; Clear ebp for call stack trace
255 push qword 0 ; Push 8 bytes for alignment
256 mov rax, qword [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitializeFloatingPointUnits)]
258 call rax ; Call assembly function to initialize FPU per UEFI spec
261 mov edx, ebx ; edx is ApIndex
263 add ecx, MP_CPU_EXCHANGE_INFO_OFFSET ; rcx is address of exchange info data buffer
266 add edi, MP_CPU_EXCHANGE_INFO_FIELD (CFunction)
270 call rax ; Invoke C function
272 jmp $ ; Should never reach here
275 ; Required for the AMD SEV helper functions
277 %include "AmdSev.nasm"
279 RendezvousFunnelProcEnd:
280 ;-------------------------------------------------------------------------------------
281 ; AsmRelocateApLoopAmdSev (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);
282 ;-------------------------------------------------------------------------------------
284 AsmRelocateApLoopAmdSevStart:
286 cmp qword [rsp + 56], 0 ; SevEsAPJumpTable
290 ; Perform some SEV-ES related setup before leaving 64-bit mode
296 ; Get the RDX reset value using CPUID
300 mov rsi, rax ; Save off the reset value for RDX
303 ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call
304 ; - Must be done while in 64-bit long mode so that writes to
305 ; the GHCB memory will be unencrypted.
306 ; - No NAE events can be generated once this is set otherwise
307 ; the AP_RESET_HOLD SW_EXITCODE will be overwritten.
310 rdmsr ; Retrieve current GHCB address
318 rep stosq ; Clear the GHCB
320 mov rax, 0x80000004 ; VMGEXIT AP_RESET_HOLD
321 mov [rdx + 0x390], rax
322 mov rax, 114 ; Set SwExitCode valid bit
323 bts [rdx + 0x3f0], rax
324 inc rax ; Set SwExitInfo1 valid bit
325 bts [rdx + 0x3f0], rax
326 inc rax ; Set SwExitInfo2 valid bit
327 bts [rdx + 0x3f0], rax
333 cli ; Disable interrupt before switching to 32-bit mode
334 mov rax, [rsp + 40] ; CountTofinish
335 lock dec dword [rax] ; (*CountTofinish)--
337 mov r10, [rsp + 48] ; Pm16CodeSegment
338 mov rax, [rsp + 56] ; SevEsAPJumpTable
339 mov rbx, [rsp + 64] ; WakeupBuffer
340 mov rsp, r9 ; TopOfApStack
342 push rax ; Save SevEsAPJumpTable
343 push rbx ; Save WakeupBuffer
344 push r10 ; Save Pm16CodeSegment
345 push rcx ; Save MwaitSupport
346 push rdx ; Save ApTargetCState
348 lea rax, [PmEntryAmdSev] ; rax <- The start address of transition code
354 ; Clear R8 - R15, for reset, before going into 32-bit mode
366 ; Far return into 32-bit mode
373 btr eax, 31 ; Clear CR0.PG
374 mov cr0, eax ; Disable paging and caches
378 and ah, ~ 1 ; Clear LME
381 and al, ~ (1 << 5) ; Clear PAE
390 cmp cl, 1 ; Check mwait-monitor support
392 mov ebx, edx ; Save C-State to ebx
395 mov eax, esp ; Set Monitor Address
396 xor ecx, ecx ; ecx = 0
397 xor edx, edx ; edx = 0
399 mov eax, ebx ; Mwait Cx, Target C-State per eax[7:4]
405 pop edx ; PM16CodeSegment
407 pop ebx ; WakeupBuffer
409 pop eax ; SevEsAPJumpTable
411 cmp eax, 0 ; Check for SEV-ES
416 ; SEV-ES is enabled, use VMGEXIT (GHCB information already
424 ; Back from VMGEXIT AP_HLT_LOOP
425 ; Push the FLAGS/CS/IP values to use
427 push word 0x0002 ; EFLAGS
429 mov cx, [eax + 2] ; CS
433 push word 0x0000 ; For alignment, will be discarded
438 mov edx, esi ; Restore RDX reset value
448 AsmRelocateApLoopAmdSevEnd:
450 ;-------------------------------------------------------------------------------------
451 ; AsmRelocateApLoop (MwaitSupport, ApTargetCState, TopOfApStack, CountTofinish, Cr3);
452 ; This function is called during the finalizaiton of Mp initialization before booting
453 ; to OS, and aim to put Aps either in Mwait or HLT.
454 ;-------------------------------------------------------------------------------------
458 ; | CountTofinish | r9
460 ; | TopOfApStack | r8
462 ; | ApTargetCState | rdx
464 ; | MwaitSupport | rcx
467 ; +----------------+ low address
469 AsmRelocateApLoopGenericStart:
470 mov rax, r9 ; CountTofinish
471 lock dec dword [rax] ; (*CountTofinish)--
473 mov rax, [rsp + 40] ; Cr3
474 ; Do not push on old stack, since old stack is not mapped
475 ; in the page table pointed by cr3
477 mov rsp, r8 ; TopOfApStack
480 cmp cl, 1 ; Check mwait-monitor support
482 mov rbx, rdx ; Save C-State to ebx
486 mov rax, rsp ; Set Monitor Address
487 xor ecx, ecx ; ecx = 0
488 xor edx, edx ; edx = 0
490 mov rax, rbx ; Mwait Cx, Target C-State per eax[7:4]
500 AsmRelocateApLoopGenericEnd:
502 ;-------------------------------------------------------------------------------------
503 ; AsmGetAddressMap (&AddressMap);
504 ;-------------------------------------------------------------------------------------
505 global ASM_PFX(AsmGetAddressMap)
506 ASM_PFX(AsmGetAddressMap):
507 lea rax, [RendezvousFunnelProcStart]
508 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelAddress], rax
509 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeEntryOffset], LongModeStart - RendezvousFunnelProcStart
510 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelSize], RendezvousFunnelProcEnd - RendezvousFunnelProcStart
511 lea rax, [AsmRelocateApLoopGenericStart]
512 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncAddressGeneric], rax
513 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncSizeGeneric], AsmRelocateApLoopGenericEnd - AsmRelocateApLoopGenericStart
514 lea rax, [AsmRelocateApLoopAmdSevStart]
515 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncAddressAmdSev], rax
516 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncSizeAmdSev], AsmRelocateApLoopAmdSevEnd - AsmRelocateApLoopAmdSevStart
517 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeTransitionOffset], Flat32Start - RendezvousFunnelProcStart
518 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealNoNxOffset], SwitchToRealProcStart - Flat32Start
519 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeOffset], PM16Mode - RendezvousFunnelProcStart
520 mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeSize], SwitchToRealProcEnd - PM16Mode
523 ;-------------------------------------------------------------------------------------
524 ;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is
525 ;about to become an AP. It switches its stack with the current AP.
526 ;AsmExchangeRole (IN CPU_EXCHANGE_INFO *MyInfo, IN CPU_EXCHANGE_INFO *OthersInfo);
527 ;-------------------------------------------------------------------------------------
528 global ASM_PFX(AsmExchangeRole)
529 ASM_PFX(AsmExchangeRole):
530 ; DO NOT call other functions in this function, since 2 CPU may use 1 stack
531 ; at the same time. If 1 CPU try to call a function, stack will be corrupted.
549 ; rsi contains MyInfo pointer
552 ; rdi contains OthersInfo pointer
557 ; Store the its StackPointer
558 mov [rsi + CPU_EXCHANGE_ROLE_INFO.StackPointer], rsp
560 ; update its switch state to STORED
561 mov byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
564 ; wait until the other CPU finish storing its state
565 cmp byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
568 jmp WaitForOtherStored
572 ; load its future StackPointer
573 mov rsp, [rdi + CPU_EXCHANGE_ROLE_INFO.StackPointer]
575 ; update the other CPU's switch state to LOADED
576 mov byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
579 ; wait until the other CPU finish loading new state,
580 ; otherwise the data in stack may corrupt
581 cmp byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
584 jmp WaitForOtherLoaded
587 ; since the other CPU already get the data it want, leave this procedure