;------------------------------------------------------------------------------ ;\r
-; Copyright (c) 2015 - 2019, Intel Corporation. All rights reserved.<BR>\r
+; Copyright (c) 2015 - 2022, Intel Corporation. All rights reserved.<BR>\r
; SPDX-License-Identifier: BSD-2-Clause-Patent\r
;\r
; Module Name:\r
%include "MpEqu.inc"\r
extern ASM_PFX(InitializeFloatingPointUnits)\r
\r
+%macro OneTimeCall 1\r
+ jmp %1\r
+%1 %+ OneTimerCallReturn:\r
+%endmacro\r
+\r
+%macro OneTimeCallRet 1\r
+ jmp %1 %+ OneTimerCallReturn\r
+%endmacro\r
+\r
DEFAULT REL\r
\r
SECTION .text\r
;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC\r
;IS IN MACHINE CODE.\r
;-------------------------------------------------------------------------------------\r
-global ASM_PFX(RendezvousFunnelProc)\r
-ASM_PFX(RendezvousFunnelProc):\r
RendezvousFunnelProcStart:\r
; At this point CS = 0x(vv00) and ip= 0x0.\r
; Save BIST information to ebp firstly\r
mov fs, ax\r
mov gs, ax\r
\r
- mov si, BufferStartLocation\r
+ mov si, MP_CPU_EXCHANGE_INFO_FIELD (BufferStart)\r
mov ebx, [si]\r
\r
- mov si, DataSegmentLocation\r
+ mov si, MP_CPU_EXCHANGE_INFO_FIELD (DataSegment)\r
mov edx, [si]\r
\r
;\r
; Get start address of 32-bit code in low memory (<1MB)\r
;\r
- mov edi, ModeTransitionMemoryLocation\r
+ mov edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeTransitionMemory)\r
\r
- mov si, GdtrLocation\r
+ mov si, MP_CPU_EXCHANGE_INFO_FIELD (GdtrProfile)\r
o32 lgdt [cs:si]\r
\r
- mov si, IdtrLocation\r
+ mov si, MP_CPU_EXCHANGE_INFO_FIELD (IdtrProfile)\r
o32 lidt [cs:si]\r
\r
;\r
;\r
; Enable execute disable bit\r
;\r
- mov esi, EnableExecuteDisableLocation\r
+ mov esi, MP_CPU_EXCHANGE_INFO_FIELD (EnableExecuteDisable)\r
cmp byte [ebx + esi], 0\r
jz SkipEnableExecuteDisableBit\r
\r
mov eax, cr4\r
bts eax, 5\r
\r
- mov esi, Enable5LevelPagingLocation\r
+ mov esi, MP_CPU_EXCHANGE_INFO_FIELD (Enable5LevelPaging)\r
cmp byte [ebx + esi], 0\r
jz SkipEnable5LevelPaging\r
\r
;\r
; Load page table\r
;\r
- mov esi, Cr3Location ; Save CR3 in ecx\r
+ mov esi, MP_CPU_EXCHANGE_INFO_FIELD (Cr3) ; Save CR3 in ecx\r
mov ecx, [ebx + esi]\r
mov cr3, ecx ; Load CR3\r
\r
;\r
; Far jump to 64-bit code\r
;\r
- mov edi, ModeHighMemoryLocation\r
+ mov edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeHighMemory)\r
add edi, ebx\r
jmp far [edi]\r
\r
BITS 64\r
+\r
LongModeStart:\r
mov esi, ebx\r
- lea edi, [esi + InitFlagLocation]\r
+ lea edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitFlag)]\r
cmp qword [edi], 1 ; ApInitConfig\r
jnz GetApicId\r
\r
; Increment the number of APs executing here as early as possible\r
; This is decremented in C code when AP is finished executing\r
mov edi, esi\r
- add edi, NumApsExecutingLocation\r
+ add edi, MP_CPU_EXCHANGE_INFO_FIELD (NumApsExecuting)\r
lock inc dword [edi]\r
\r
; AP init\r
mov edi, esi\r
- add edi, LockLocation\r
- mov rax, NotVacantFlag\r
-\r
-TestLock:\r
- xchg qword [edi], rax\r
- cmp rax, NotVacantFlag\r
- jz TestLock\r
+ add edi, MP_CPU_EXCHANGE_INFO_FIELD (ApIndex)\r
+ mov ebx, 1\r
+ lock xadd dword [edi], ebx ; EBX = ApIndex++\r
+ inc ebx ; EBX is CpuNumber\r
\r
- lea ecx, [esi + ApIndexLocation]\r
- inc dword [ecx]\r
- mov ebx, [ecx]\r
-\r
-Releaselock:\r
- mov rax, VacantFlag\r
- xchg qword [edi], rax\r
; program stack\r
mov edi, esi\r
- add edi, StackSizeLocation\r
+ add edi, MP_CPU_EXCHANGE_INFO_FIELD (StackSize)\r
mov eax, dword [edi]\r
mov ecx, ebx\r
inc ecx\r
mul ecx ; EAX = StackSize * (CpuNumber + 1)\r
mov edi, esi\r
- add edi, StackStartAddressLocation\r
+ add edi, MP_CPU_EXCHANGE_INFO_FIELD (StackStart)\r
add rax, qword [edi]\r
mov rsp, rax\r
\r
- lea edi, [esi + SevEsIsEnabledLocation]\r
- cmp byte [edi], 1 ; SevEsIsEnabled\r
- jne CProcedureInvoke\r
-\r
;\r
- ; program GHCB\r
- ; Each page after the GHCB is a per-CPU page, so the calculation programs\r
- ; a GHCB to be every 8KB.\r
+ ; Setup the GHCB when AMD SEV-ES active.\r
;\r
- mov eax, SIZE_4KB\r
- shl eax, 1 ; EAX = SIZE_4K * 2\r
- mov ecx, ebx\r
- mul ecx ; EAX = SIZE_4K * 2 * CpuNumber\r
- mov edi, esi\r
- add edi, GhcbBaseLocation\r
- add rax, qword [edi]\r
- mov rdx, rax\r
- shr rdx, 32\r
- mov rcx, 0xc0010130\r
- wrmsr\r
+ OneTimeCall SevEsSetupGhcb\r
jmp CProcedureInvoke\r
\r
GetApicId:\r
- lea edi, [esi + SevEsIsEnabledLocation]\r
- cmp byte [edi], 1 ; SevEsIsEnabled\r
- jne DoCpuid\r
-\r
;\r
- ; Since we don't have a stack yet, we can't take a #VC\r
- ; exception. Use the GHCB protocol to perform the CPUID\r
- ; calls.\r
+ ; Use the GHCB protocol to get the ApicId when SEV-ES is active.\r
;\r
- mov rcx, 0xc0010130\r
- rdmsr\r
- shl rdx, 32\r
- or rax, rdx\r
- mov rdi, rax ; RDI now holds the original GHCB GPA\r
-\r
- mov rdx, 0 ; CPUID function 0\r
- mov rax, 0 ; RAX register requested\r
- or rax, 4\r
- wrmsr\r
- rep vmmcall\r
- rdmsr\r
- cmp edx, 0bh\r
- jb NoX2ApicSevEs ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r
-\r
- mov rdx, 0bh ; CPUID function 0x0b\r
- mov rax, 040000000h ; RBX register requested\r
- or rax, 4\r
- wrmsr\r
- rep vmmcall\r
- rdmsr\r
- test edx, 0ffffh\r
- jz NoX2ApicSevEs ; CPUID.0BH:EBX[15:0] is zero\r
-\r
- mov rdx, 0bh ; CPUID function 0x0b\r
- mov rax, 0c0000000h ; RDX register requested\r
- or rax, 4\r
- wrmsr\r
- rep vmmcall\r
- rdmsr\r
-\r
- ; Processor is x2APIC capable; 32-bit x2APIC ID is now in EDX\r
- jmp RestoreGhcb\r
-\r
-NoX2ApicSevEs:\r
- ; Processor is not x2APIC capable, so get 8-bit APIC ID\r
- mov rdx, 1 ; CPUID function 1\r
- mov rax, 040000000h ; RBX register requested\r
- or rax, 4\r
- wrmsr\r
- rep vmmcall\r
- rdmsr\r
- shr edx, 24\r
-\r
-RestoreGhcb:\r
- mov rbx, rdx ; Save x2APIC/APIC ID\r
-\r
- mov rdx, rdi ; RDI holds the saved GHCB GPA\r
- shr rdx, 32\r
- mov eax, edi\r
- wrmsr\r
-\r
- mov rdx, rbx\r
-\r
- ; x2APIC ID or APIC ID is in EDX\r
- jmp GetProcessorNumber\r
+ OneTimeCall SevEsGetApicId\r
\r
DoCpuid:\r
mov eax, 0\r
; Note that BSP may become an AP due to SwitchBsp()\r
;\r
xor ebx, ebx\r
- lea eax, [esi + CpuInfoLocation]\r
+ lea eax, [esi + MP_CPU_EXCHANGE_INFO_FIELD (CpuInfo)]\r
mov rdi, [eax]\r
\r
GetNextProcNumber:\r
- cmp dword [rdi], edx ; APIC ID match?\r
+ cmp dword [rdi + CPU_INFO_IN_HOB.InitialApicId], edx ; APIC ID match?\r
jz ProgramStack\r
- add rdi, 20\r
+ add rdi, CPU_INFO_IN_HOB_size\r
inc ebx\r
jmp GetNextProcNumber\r
\r
ProgramStack:\r
- mov rsp, qword [rdi + 12]\r
+ mov rsp, qword [rdi + CPU_INFO_IN_HOB.ApTopOfStack]\r
\r
CProcedureInvoke:\r
push rbp ; Push BIST data at top of AP stack\r
push rbp\r
mov rbp, rsp\r
\r
- mov rax, qword [esi + InitializeFloatingPointUnitsAddress]\r
+ mov rax, qword [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitializeFloatingPointUnits)]\r
sub rsp, 20h\r
call rax ; Call assembly function to initialize FPU per UEFI spec\r
add rsp, 20h\r
\r
mov edx, ebx ; edx is ApIndex\r
mov ecx, esi\r
- add ecx, LockLocation ; rcx is address of exchange info data buffer\r
+ add ecx, MP_CPU_EXCHANGE_INFO_OFFSET ; rcx is address of exchange info data buffer\r
\r
mov edi, esi\r
- add edi, ApProcedureLocation\r
+ add edi, MP_CPU_EXCHANGE_INFO_FIELD (CFunction)\r
mov rax, qword [edi]\r
\r
sub rsp, 20h\r
add rsp, 20h\r
jmp $ ; Should never reach here\r
\r
-RendezvousFunnelProcEnd:\r
-\r
-;-------------------------------------------------------------------------------------\r
-;SwitchToRealProc procedure follows.\r
-;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC\r
-;IS IN MACHINE CODE.\r
-; SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart)\r
-; rcx - Buffer Start\r
-; rdx - Code16 Selector Offset\r
-; r8 - Code32 Selector Offset\r
-; r9 - Stack Start\r
-;-------------------------------------------------------------------------------------\r
-global ASM_PFX(SwitchToRealProc)\r
-ASM_PFX(SwitchToRealProc):\r
-SwitchToRealProcStart:\r
-BITS 64\r
- cli\r
-\r
- ;\r
- ; Get RDX reset value before changing stacks since the\r
- ; new stack won't be able to accomodate a #VC exception.\r
- ;\r
- push rax\r
- push rbx\r
- push rcx\r
- push rdx\r
-\r
- mov rax, 1\r
- cpuid\r
- mov rsi, rax ; Save off the reset value for RDX\r
-\r
- pop rdx\r
- pop rcx\r
- pop rbx\r
- pop rax\r
-\r
- ;\r
- ; Establish stack below 1MB\r
- ;\r
- mov rsp, r9\r
-\r
- ;\r
- ; Push ultimate Reset Vector onto the stack\r
- ;\r
- mov rax, rcx\r
- shr rax, 4\r
- push word 0x0002 ; RFLAGS\r
- push ax ; CS\r
- push word 0x0000 ; RIP\r
- push word 0x0000 ; For alignment, will be discarded\r
-\r
- ;\r
- ; Get address of "16-bit operand size" label\r
- ;\r
- lea rbx, [PM16Mode]\r
-\r
- ;\r
- ; Push addresses used to change to compatibility mode\r
- ;\r
- lea rax, [CompatMode]\r
- push r8\r
- push rax\r
-\r
- ;\r
- ; Clear R8 - R15, for reset, before going into 32-bit mode\r
- ;\r
- xor r8, r8\r
- xor r9, r9\r
- xor r10, r10\r
- xor r11, r11\r
- xor r12, r12\r
- xor r13, r13\r
- xor r14, r14\r
- xor r15, r15\r
-\r
- ;\r
- ; Far return into 32-bit mode\r
- ;\r
-o64 retf\r
-\r
-BITS 32\r
-CompatMode:\r
- ;\r
- ; Set up stack to prepare for exiting protected mode\r
- ;\r
- push edx ; Code16 CS\r
- push ebx ; PM16Mode label address\r
-\r
- ;\r
- ; Disable paging\r
- ;\r
- mov eax, cr0 ; Read CR0\r
- btr eax, 31 ; Set PG=0\r
- mov cr0, eax ; Write CR0\r
-\r
- ;\r
- ; Disable long mode\r
- ;\r
- mov ecx, 0c0000080h ; EFER MSR number\r
- rdmsr ; Read EFER\r
- btr eax, 8 ; Set LME=0\r
- wrmsr ; Write EFER\r
-\r
- ;\r
- ; Disable PAE\r
- ;\r
- mov eax, cr4 ; Read CR4\r
- btr eax, 5 ; Set PAE=0\r
- mov cr4, eax ; Write CR4\r
-\r
- mov edx, esi ; Restore RDX reset value\r
-\r
- ;\r
- ; Switch to 16-bit operand size\r
- ;\r
- retf\r
-\r
-BITS 16\r
- ;\r
- ; At entry to this label\r
- ; - RDX will have its reset value\r
- ; - On the top of the stack\r
- ; - Alignment data (two bytes) to be discarded\r
- ; - IP for Real Mode (two bytes)\r
- ; - CS for Real Mode (two bytes)\r
- ;\r
- ; This label is also used with AsmRelocateApLoop. During MP finalization,\r
- ; the code from PM16Mode to SwitchToRealProcEnd is copied to the start of\r
- ; the WakeupBuffer, allowing a parked AP to be booted by an OS.\r
- ;\r
-PM16Mode:\r
- mov eax, cr0 ; Read CR0\r
- btr eax, 0 ; Set PE=0\r
- mov cr0, eax ; Write CR0\r
-\r
- pop ax ; Discard alignment data\r
-\r
- ;\r
- ; Clear registers (except RDX and RSP) before going into 16-bit mode\r
- ;\r
- xor eax, eax\r
- xor ebx, ebx\r
- xor ecx, ecx\r
- xor esi, esi\r
- xor edi, edi\r
- xor ebp, ebp\r
-\r
- iret\r
+;\r
+; Required for the AMD SEV helper functions\r
+;\r
+%include "AmdSev.nasm"\r
\r
-SwitchToRealProcEnd:\r
+RendezvousFunnelProcEnd:\r
\r
;-------------------------------------------------------------------------------------\r
; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);\r
;-------------------------------------------------------------------------------------\r
-global ASM_PFX(AsmRelocateApLoop)\r
-ASM_PFX(AsmRelocateApLoop):\r
AsmRelocateApLoopStart:\r
BITS 64\r
cmp qword [rsp + 56], 0 ; SevEsAPJumpTable\r
;\r
; Far return into 32-bit mode\r
;\r
-o64 retf\r
+ retfq\r
\r
BITS 32\r
PmEntry:\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(AsmGetAddressMap)\r
ASM_PFX(AsmGetAddressMap):\r
- lea rax, [ASM_PFX(RendezvousFunnelProc)]\r
- mov qword [rcx], rax\r
- mov qword [rcx + 8h], LongModeStart - RendezvousFunnelProcStart\r
- mov qword [rcx + 10h], RendezvousFunnelProcEnd - RendezvousFunnelProcStart\r
- lea rax, [ASM_PFX(AsmRelocateApLoop)]\r
- mov qword [rcx + 18h], rax\r
- mov qword [rcx + 20h], AsmRelocateApLoopEnd - AsmRelocateApLoopStart\r
- mov qword [rcx + 28h], Flat32Start - RendezvousFunnelProcStart\r
- mov qword [rcx + 30h], SwitchToRealProcEnd - SwitchToRealProcStart ; SwitchToRealSize\r
- mov qword [rcx + 38h], SwitchToRealProcStart - RendezvousFunnelProcStart ; SwitchToRealOffset\r
- mov qword [rcx + 40h], SwitchToRealProcStart - Flat32Start ; SwitchToRealNoNxOffset\r
- mov qword [rcx + 48h], PM16Mode - RendezvousFunnelProcStart ; SwitchToRealPM16ModeOffset\r
- mov qword [rcx + 50h], SwitchToRealProcEnd - PM16Mode ; SwitchToRealPM16ModeSize\r
+ lea rax, [RendezvousFunnelProcStart]\r
+ mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelAddress], rax\r
+ mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeEntryOffset], LongModeStart - RendezvousFunnelProcStart\r
+ mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelSize], RendezvousFunnelProcEnd - RendezvousFunnelProcStart\r
+ lea rax, [AsmRelocateApLoopStart]\r
+ mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncAddress], rax\r
+ mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncSize], AsmRelocateApLoopEnd - AsmRelocateApLoopStart\r
+ mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeTransitionOffset], Flat32Start - RendezvousFunnelProcStart\r
+ mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealNoNxOffset], SwitchToRealProcStart - Flat32Start\r
+ mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeOffset], PM16Mode - RendezvousFunnelProcStart\r
+ mov qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeSize], SwitchToRealProcEnd - PM16Mode\r
ret\r
\r
;-------------------------------------------------------------------------------------\r
\r
;Store EFLAGS, GDTR and IDTR regiter to stack\r
pushfq\r
- sgdt [rsi + 16]\r
- sidt [rsi + 26]\r
+ sgdt [rsi + CPU_EXCHANGE_ROLE_INFO.Gdtr]\r
+ sidt [rsi + CPU_EXCHANGE_ROLE_INFO.Idtr]\r
\r
; Store the its StackPointer\r
- mov [rsi + 8], rsp\r
+ mov [rsi + CPU_EXCHANGE_ROLE_INFO.StackPointer], rsp\r
\r
; update its switch state to STORED\r
- mov byte [rsi], CPU_SWITCH_STATE_STORED\r
+ mov byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED\r
\r
WaitForOtherStored:\r
; wait until the other CPU finish storing its state\r
- cmp byte [rdi], CPU_SWITCH_STATE_STORED\r
+ cmp byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED\r
jz OtherStored\r
pause\r
jmp WaitForOtherStored\r
OtherStored:\r
; Since another CPU already stored its state, load them\r
; load GDTR value\r
- lgdt [rdi + 16]\r
+ lgdt [rdi + CPU_EXCHANGE_ROLE_INFO.Gdtr]\r
\r
; load IDTR value\r
- lidt [rdi + 26]\r
+ lidt [rdi + CPU_EXCHANGE_ROLE_INFO.Idtr]\r
\r
; load its future StackPointer\r
- mov rsp, [rdi + 8]\r
+ mov rsp, [rdi + CPU_EXCHANGE_ROLE_INFO.StackPointer]\r
\r
; update the other CPU's switch state to LOADED\r
- mov byte [rdi], CPU_SWITCH_STATE_LOADED\r
+ mov byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED\r
\r
WaitForOtherLoaded:\r
; wait until the other CPU finish loading new state,\r
; otherwise the data in stack may corrupt\r
- cmp byte [rsi], CPU_SWITCH_STATE_LOADED\r
+ cmp byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED\r
jz OtherLoaded\r
pause\r
jmp WaitForOtherLoaded\r
projects / mirror_edk2.git / blobdiff