[mirror_edk2.git] / UefiCpuPkg / Library / MpInitLib / X64 / AmdSev.nasm

;------------------------------------------------------------------------------ ;\r
; Copyright (c) 2021, AMD Inc. All rights reserved.<BR>\r
; SPDX-License-Identifier: BSD-2-Clause-Patent\r
;\r
; Module Name:\r
;\r
;   AmdSev.nasm\r
;\r
; Abstract:\r
;\r
;   This provides helper used by the MpFunc.nasm. If AMD SEV-ES is active\r
;   then helpers perform the additional setups (such as GHCB).\r
;\r
;-------------------------------------------------------------------------------\r
\r
%define SIZE_4KB    0x1000\r
\r
RegisterGhcbGpa:\r
    ;\r
    ; Register GHCB GPA when SEV-SNP is enabled\r
    ;\r
    lea        edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (SevSnpIsEnabled)]\r
    cmp        byte [edi], 1        ; SevSnpIsEnabled\r
    jne        RegisterGhcbGpaDone\r
\r
    ; Save the rdi and rsi to used for later comparison\r
    push       rdi\r
    push       rsi\r
    mov        edi, eax\r
    mov        esi, edx\r
    or         eax, 18              ; Ghcb registration request\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
    mov        r12, rax\r
    and        r12, 0fffh\r
    cmp        r12, 19              ; Ghcb registration response\r
    jne        GhcbGpaRegisterFailure\r
\r
    ; Verify that GPA is not changed\r
    and        eax, 0fffff000h\r
    cmp        edi, eax\r
    jne        GhcbGpaRegisterFailure\r
    cmp        esi, edx\r
    jne        GhcbGpaRegisterFailure\r
    pop        rsi\r
    pop        rdi\r
    jmp        RegisterGhcbGpaDone\r
\r
    ;\r
    ; Request the guest termination\r
    ;\r
GhcbGpaRegisterFailure:\r
    xor        edx, edx\r
    mov        eax, 256             ; GHCB terminate\r
    wrmsr\r
    rep vmmcall\r
\r
    ; We should not return from the above terminate request, but if we do\r
    ; then enter into the hlt loop.\r
DoHltLoop:\r
    cli\r
    hlt\r
    jmp        DoHltLoop\r
\r
RegisterGhcbGpaDone:\r
    OneTimeCallRet    RegisterGhcbGpa\r
\r
;\r
; The function checks whether SEV-ES is enabled, if enabled\r
; then setup the GHCB page.\r
;\r
SevEsSetupGhcb:\r
    lea        edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (SevEsIsEnabled)]\r
    cmp        byte [edi], 1        ; SevEsIsEnabled\r
    jne        SevEsSetupGhcbExit\r
\r
    ;\r
    ; program GHCB\r
    ;   Each page after the GHCB is a per-CPU page, so the calculation programs\r
    ;   a GHCB to be every 8KB.\r
    ;\r
    mov        eax, SIZE_4KB\r
    shl        eax, 1                            ; EAX = SIZE_4K * 2\r
    mov        ecx, ebx\r
    mul        ecx                               ; EAX = SIZE_4K * 2 * CpuNumber\r
    mov        edi, esi\r
    add        edi, MP_CPU_EXCHANGE_INFO_FIELD (GhcbBase)\r
    add        rax, qword [edi]\r
    mov        rdx, rax\r
    shr        rdx, 32\r
    mov        rcx, 0xc0010130\r
\r
    OneTimeCall RegisterGhcbGpa\r
\r
    wrmsr\r
\r
SevEsSetupGhcbExit:\r
    OneTimeCallRet    SevEsSetupGhcb\r
\r
;\r
; The function checks whether SEV-ES is enabled, if enabled, use\r
; the GHCB\r
;\r
SevEsGetApicId:\r
    lea        edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (SevEsIsEnabled)]\r
    cmp        byte [edi], 1        ; SevEsIsEnabled\r
    jne        SevEsGetApicIdExit\r
\r
    ;\r
    ; Since we don't have a stack yet, we can't take a #VC\r
    ; exception. Use the GHCB protocol to perform the CPUID\r
    ; calls.\r
    ;\r
    mov        rcx, 0xc0010130\r
    rdmsr\r
    shl        rdx, 32\r
    or         rax, rdx\r
    mov        rdi, rax             ; RDI now holds the original GHCB GPA\r
\r
    ;\r
    ; For SEV-SNP, the recommended handling for getting the x2APIC ID\r
    ; would be to use the SNP CPUID table to fetch CPUID.00H:EAX and\r
    ; CPUID:0BH:EBX[15:0] instead of the GHCB MSR protocol vmgexits\r
    ; below.\r
    ;\r
    ; To avoid the unecessary ugliness to accomplish that here, the BSP\r
    ; has performed these checks in advance (where #VC handler handles\r
    ; the CPUID table lookups automatically) and cached them in a flag\r
    ; so those checks can be skipped here.\r
    ;\r
    mov        eax, [esi + MP_CPU_EXCHANGE_INFO_FIELD (SevSnpIsEnabled)]\r
    cmp        al, 1\r
    jne        CheckExtTopoAvail\r
\r
    ;\r
    ; Even with SEV-SNP, the actual x2APIC ID in CPUID.0BH:EDX\r
    ; fetched from the hypervisor the same way SEV-ES does it.\r
    ;\r
    mov        eax, [esi + MP_CPU_EXCHANGE_INFO_FIELD (ExtTopoAvail)]\r
    cmp        al, 1\r
    je         GetApicIdSevEs\r
    ; The 8-bit APIC ID fallback is also the same as with SEV-ES\r
    jmp        NoX2ApicSevEs\r
\r
CheckExtTopoAvail:\r
    mov        rdx, 0               ; CPUID function 0\r
    mov        rax, 0               ; RAX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
    cmp        edx, 0bh\r
    jb         NoX2ApicSevEs        ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r
\r
    mov        rdx, 0bh             ; CPUID function 0x0b\r
    mov        rax, 040000000h      ; RBX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
    test       edx, 0ffffh\r
    jz         NoX2ApicSevEs        ; CPUID.0BH:EBX[15:0] is zero\r
\r
GetApicIdSevEs:\r
    mov        rdx, 0bh             ; CPUID function 0x0b\r
    mov        rax, 0c0000000h      ; RDX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
\r
    ; Processor is x2APIC capable; 32-bit x2APIC ID is now in EDX\r
    jmp        RestoreGhcb\r
\r
NoX2ApicSevEs:\r
    ; Processor is not x2APIC capable, so get 8-bit APIC ID\r
    mov        rdx, 1               ; CPUID function 1\r
    mov        rax, 040000000h      ; RBX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
    shr        edx, 24\r
\r
RestoreGhcb:\r
    mov        rbx, rdx             ; Save x2APIC/APIC ID\r
\r
    mov        rdx, rdi             ; RDI holds the saved GHCB GPA\r
    shr        rdx, 32\r
    mov        eax, edi\r
    wrmsr\r
\r
    mov        rdx, rbx\r
\r
    ; x2APIC ID or APIC ID is in EDX\r
    jmp        GetProcessorNumber\r
\r
SevEsGetApicIdExit:\r
    OneTimeCallRet    SevEsGetApicId\r
\r
\r
;-------------------------------------------------------------------------------------\r
;SwitchToRealProc procedure follows.\r
;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC\r
;IS IN MACHINE CODE.\r
;  SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart)\r
;  rcx - Buffer Start\r
;  rdx - Code16 Selector Offset\r
;  r8  - Code32 Selector Offset\r
;  r9  - Stack Start\r
;-------------------------------------------------------------------------------------\r
SwitchToRealProcStart:\r
BITS 64\r
    cli\r
\r
    ;\r
    ; Get RDX reset value before changing stacks since the\r
    ; new stack won't be able to accomodate a #VC exception.\r
    ;\r
    push       rax\r
    push       rbx\r
    push       rcx\r
    push       rdx\r
\r
    mov        rax, 1\r
    cpuid\r
    mov        rsi, rax                    ; Save off the reset value for RDX\r
\r
    pop        rdx\r
    pop        rcx\r
    pop        rbx\r
    pop        rax\r
\r
    ;\r
    ; Establish stack below 1MB\r
    ;\r
    mov        rsp, r9\r
\r
    ;\r
    ; Push ultimate Reset Vector onto the stack\r
    ;\r
    mov        rax, rcx\r
    shr        rax, 4\r
    push       word 0x0002                 ; RFLAGS\r
    push       ax                          ; CS\r
    push       word 0x0000                 ; RIP\r
    push       word 0x0000                 ; For alignment, will be discarded\r
\r
    ;\r
    ; Get address of "16-bit operand size" label\r
    ;\r
    lea        rbx, [PM16Mode]\r
\r
    ;\r
    ; Push addresses used to change to compatibility mode\r
    ;\r
    lea        rax, [CompatMode]\r
    push       r8\r
    push       rax\r
\r
    ;\r
    ; Clear R8 - R15, for reset, before going into 32-bit mode\r
    ;\r
    xor        r8, r8\r
    xor        r9, r9\r
    xor        r10, r10\r
    xor        r11, r11\r
    xor        r12, r12\r
    xor        r13, r13\r
    xor        r14, r14\r
    xor        r15, r15\r
\r
    ;\r
    ; Far return into 32-bit mode\r
    ;\r
    retfq\r
\r
BITS 32\r
CompatMode:\r
    ;\r
    ; Set up stack to prepare for exiting protected mode\r
    ;\r
    push       edx                         ; Code16 CS\r
    push       ebx                         ; PM16Mode label address\r
\r
    ;\r
    ; Disable paging\r
    ;\r
    mov        eax, cr0                    ; Read CR0\r
    btr        eax, 31                     ; Set PG=0\r
    mov        cr0, eax                    ; Write CR0\r
\r
    ;\r
    ; Disable long mode\r
    ;\r
    mov        ecx, 0c0000080h             ; EFER MSR number\r
    rdmsr                                  ; Read EFER\r
    btr        eax, 8                      ; Set LME=0\r
    wrmsr                                  ; Write EFER\r
\r
    ;\r
    ; Disable PAE\r
    ;\r
    mov        eax, cr4                    ; Read CR4\r
    btr        eax, 5                      ; Set PAE=0\r
    mov        cr4, eax                    ; Write CR4\r
\r
    mov        edx, esi                    ; Restore RDX reset value\r
\r
    ;\r
    ; Switch to 16-bit operand size\r
    ;\r
    retf\r
\r
BITS 16\r
    ;\r
    ; At entry to this label\r
    ;   - RDX will have its reset value\r
    ;   - On the top of the stack\r
    ;     - Alignment data (two bytes) to be discarded\r
    ;     - IP for Real Mode (two bytes)\r
    ;     - CS for Real Mode (two bytes)\r
    ;\r
    ; This label is also used with AsmRelocateApLoop. During MP finalization,\r
    ; the code from PM16Mode to SwitchToRealProcEnd is copied to the start of\r
    ; the WakeupBuffer, allowing a parked AP to be booted by an OS.\r
    ;\r
PM16Mode:\r
    mov        eax, cr0                    ; Read CR0\r
    btr        eax, 0                      ; Set PE=0\r
    mov        cr0, eax                    ; Write CR0\r
\r
    pop        ax                          ; Discard alignment data\r
\r
    ;\r
    ; Clear registers (except RDX and RSP) before going into 16-bit mode\r
    ;\r
    xor        eax, eax\r
    xor        ebx, ebx\r
    xor        ecx, ecx\r
    xor        esi, esi\r
    xor        edi, edi\r
    xor        ebp, ebp\r
\r
    iret\r
\r
SwitchToRealProcEnd:\r
\r
;-------------------------------------------------------------------------------------\r
;  AsmRelocateApLoopAmdSev (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);\r
;-------------------------------------------------------------------------------------\r
\r
AsmRelocateApLoopAmdSevStart:\r
BITS 64\r
    cmp        qword [rsp + 56], 0  ; SevEsAPJumpTable\r
    je         NoSevEsAmdSev\r
\r
    ;\r
    ; Perform some SEV-ES related setup before leaving 64-bit mode\r
    ;\r
    push       rcx\r
    push       rdx\r
\r
    ;\r
    ; Get the RDX reset value using CPUID\r
    ;\r
    mov        rax, 1\r
    cpuid\r
    mov        rsi, rax          ; Save off the reset value for RDX\r
\r
    ;\r
    ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call\r
    ;   - Must be done while in 64-bit long mode so that writes to\r
    ;     the GHCB memory will be unencrypted.\r
    ;   - No NAE events can be generated once this is set otherwise\r
    ;     the AP_RESET_HOLD SW_EXITCODE will be overwritten.\r
    ;\r
    mov        rcx, 0xc0010130\r
    rdmsr                        ; Retrieve current GHCB address\r
    shl        rdx, 32\r
    or         rdx, rax\r
\r
    mov        rdi, rdx\r
    xor        rax, rax\r
    mov        rcx, 0x800\r
    shr        rcx, 3\r
    rep stosq                    ; Clear the GHCB\r
\r
    mov        rax, 0x80000004   ; VMGEXIT AP_RESET_HOLD\r
    mov        [rdx + 0x390], rax\r
    mov        rax, 114          ; Set SwExitCode valid bit\r
    bts        [rdx + 0x3f0], rax\r
    inc        rax               ; Set SwExitInfo1 valid bit\r
    bts        [rdx + 0x3f0], rax\r
    inc        rax               ; Set SwExitInfo2 valid bit\r
    bts        [rdx + 0x3f0], rax\r
\r
    pop        rdx\r
    pop        rcx\r
\r
NoSevEsAmdSev:\r
    cli                          ; Disable interrupt before switching to 32-bit mode\r
    mov        rax, [rsp + 40]   ; CountTofinish\r
    lock dec   dword [rax]       ; (*CountTofinish)--\r
\r
    mov        r10, [rsp + 48]   ; Pm16CodeSegment\r
    mov        rax, [rsp + 56]   ; SevEsAPJumpTable\r
    mov        rbx, [rsp + 64]   ; WakeupBuffer\r
    mov        rsp, r9           ; TopOfApStack\r
\r
    push       rax               ; Save SevEsAPJumpTable\r
    push       rbx               ; Save WakeupBuffer\r
    push       r10               ; Save Pm16CodeSegment\r
    push       rcx               ; Save MwaitSupport\r
    push       rdx               ; Save ApTargetCState\r
\r
    lea        rax, [PmEntryAmdSev]    ; rax <- The start address of transition code\r
\r
    push       r8\r
    push       rax\r
\r
    ;\r
    ; Clear R8 - R15, for reset, before going into 32-bit mode\r
    ;\r
    xor        r8, r8\r
    xor        r9, r9\r
    xor        r10, r10\r
    xor        r11, r11\r
    xor        r12, r12\r
    xor        r13, r13\r
    xor        r14, r14\r
    xor        r15, r15\r
\r
    ;\r
    ; Far return into 32-bit mode\r
    ;\r
o64 retf\r
\r
BITS 32\r
PmEntryAmdSev:\r
    mov        eax, cr0\r
    btr        eax, 31           ; Clear CR0.PG\r
    mov        cr0, eax          ; Disable paging and caches\r
\r
    mov        ecx, 0xc0000080\r
    rdmsr\r
    and        ah, ~ 1           ; Clear LME\r
    wrmsr\r
    mov        eax, cr4\r
    and        al, ~ (1 << 5)    ; Clear PAE\r
    mov        cr4, eax\r
\r
    pop        edx\r
    add        esp, 4\r
    pop        ecx,\r
    add        esp, 4\r
\r
MwaitCheckAmdSev:\r
    cmp        cl, 1              ; Check mwait-monitor support\r
    jnz        HltLoopAmdSev\r
    mov        ebx, edx           ; Save C-State to ebx\r
MwaitLoopAmdSev:\r
    cli\r
    mov        eax, esp           ; Set Monitor Address\r
    xor        ecx, ecx           ; ecx = 0\r
    xor        edx, edx           ; edx = 0\r
    monitor\r
    mov        eax, ebx           ; Mwait Cx, Target C-State per eax[7:4]\r
    shl        eax, 4\r
    mwait\r
    jmp        MwaitLoopAmdSev\r
\r
HltLoopAmdSev:\r
    pop        edx                ; PM16CodeSegment\r
    add        esp, 4\r
    pop        ebx                ; WakeupBuffer\r
    add        esp, 4\r
    pop        eax                ; SevEsAPJumpTable\r
    add        esp, 4\r
    cmp        eax, 0             ; Check for SEV-ES\r
    je         DoHltAmdSev\r
\r
    cli\r
    ;\r
    ; SEV-ES is enabled, use VMGEXIT (GHCB information already\r
    ; set by caller)\r
    ;\r
BITS 64\r
    rep        vmmcall\r
BITS 32\r
\r
    ;\r
    ; Back from VMGEXIT AP_HLT_LOOP\r
    ;   Push the FLAGS/CS/IP values to use\r
    ;\r
    push       word 0x0002        ; EFLAGS\r
    xor        ecx, ecx\r
    mov        cx, [eax + 2]      ; CS\r
    push       cx\r
    mov        cx, [eax]          ; IP\r
    push       cx\r
    push       word 0x0000        ; For alignment, will be discarded\r
\r
    push       edx\r
    push       ebx\r
\r
    mov        edx, esi           ; Restore RDX reset value\r
\r
    retf\r
\r
DoHltAmdSev:\r
    cli\r
    hlt\r
    jmp        DoHltAmdSev\r
\r
BITS 64\r
AsmRelocateApLoopAmdSevEnd:\r
Commit	Line	Data
e2289d19 BS	1	;------------------------------------------------------------------------------ ;\r
	2	; Copyright (c) 2021, AMD Inc. All rights reserved.<BR>\r
	3	; SPDX-License-Identifier: BSD-2-Clause-Patent\r
	4	;\r
	5	; Module Name:\r
	6	;\r
	7	; AmdSev.nasm\r
	8	;\r
	9	; Abstract:\r
	10	;\r
	11	; This provides helper used by the MpFunc.nasm. If AMD SEV-ES is active\r
	12	; then helpers perform the additional setups (such as GHCB).\r
	13	;\r
	14	;-------------------------------------------------------------------------------\r
	15	\r
	16	%define SIZE_4KB 0x1000\r
	17	\r
9c703bc0 BS	18	RegisterGhcbGpa:\r
	19	;\r
	20	; Register GHCB GPA when SEV-SNP is enabled\r
	21	;\r
	22	lea edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (SevSnpIsEnabled)]\r
	23	cmp byte [edi], 1 ; SevSnpIsEnabled\r
	24	jne RegisterGhcbGpaDone\r
	25	\r
	26	; Save the rdi and rsi to used for later comparison\r
	27	push rdi\r
	28	push rsi\r
	29	mov edi, eax\r
	30	mov esi, edx\r
	31	or eax, 18 ; Ghcb registration request\r
	32	wrmsr\r
	33	rep vmmcall\r
	34	rdmsr\r
	35	mov r12, rax\r
	36	and r12, 0fffh\r
	37	cmp r12, 19 ; Ghcb registration response\r
	38	jne GhcbGpaRegisterFailure\r
	39	\r
	40	; Verify that GPA is not changed\r
	41	and eax, 0fffff000h\r
	42	cmp edi, eax\r
	43	jne GhcbGpaRegisterFailure\r
	44	cmp esi, edx\r
	45	jne GhcbGpaRegisterFailure\r
	46	pop rsi\r
	47	pop rdi\r
	48	jmp RegisterGhcbGpaDone\r
	49	\r
	50	;\r
	51	; Request the guest termination\r
	52	;\r
	53	GhcbGpaRegisterFailure:\r
	54	xor edx, edx\r
	55	mov eax, 256 ; GHCB terminate\r
	56	wrmsr\r
	57	rep vmmcall\r
	58	\r
	59	; We should not return from the above terminate request, but if we do\r
	60	; then enter into the hlt loop.\r
	61	DoHltLoop:\r
	62	cli\r
	63	hlt\r
	64	jmp DoHltLoop\r
	65	\r
	66	RegisterGhcbGpaDone:\r
	67	OneTimeCallRet RegisterGhcbGpa\r
	68	\r
e2289d19 BS	69	;\r
	70	; The function checks whether SEV-ES is enabled, if enabled\r
	71	; then setup the GHCB page.\r
	72	;\r
	73	SevEsSetupGhcb:\r
	74	lea edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (SevEsIsEnabled)]\r
	75	cmp byte [edi], 1 ; SevEsIsEnabled\r
	76	jne SevEsSetupGhcbExit\r
	77	\r
	78	;\r
	79	; program GHCB\r
	80	; Each page after the GHCB is a per-CPU page, so the calculation programs\r
	81	; a GHCB to be every 8KB.\r
	82	;\r
	83	mov eax, SIZE_4KB\r
	84	shl eax, 1 ; EAX = SIZE_4K * 2\r
	85	mov ecx, ebx\r
	86	mul ecx ; EAX = SIZE_4K * 2 * CpuNumber\r
	87	mov edi, esi\r
	88	add edi, MP_CPU_EXCHANGE_INFO_FIELD (GhcbBase)\r
	89	add rax, qword [edi]\r
	90	mov rdx, rax\r
	91	shr rdx, 32\r
	92	mov rcx, 0xc0010130\r
9c703bc0 BS	93	\r
	94	OneTimeCall RegisterGhcbGpa\r
	95	\r
e2289d19 BS	96	wrmsr\r
	97	\r
	98	SevEsSetupGhcbExit:\r
	99	OneTimeCallRet SevEsSetupGhcb\r
	100	\r
	101	;\r
	102	; The function checks whether SEV-ES is enabled, if enabled, use\r
	103	; the GHCB\r
	104	;\r
	105	SevEsGetApicId:\r
	106	lea edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (SevEsIsEnabled)]\r
	107	cmp byte [edi], 1 ; SevEsIsEnabled\r
	108	jne SevEsGetApicIdExit\r
	109	\r
	110	;\r
	111	; Since we don't have a stack yet, we can't take a #VC\r
	112	; exception. Use the GHCB protocol to perform the CPUID\r
	113	; calls.\r
	114	;\r
	115	mov rcx, 0xc0010130\r
	116	rdmsr\r
	117	shl rdx, 32\r
	118	or rax, rdx\r
	119	mov rdi, rax ; RDI now holds the original GHCB GPA\r
	120	\r
d4d7c9ad MR	121	;\r
	122	; For SEV-SNP, the recommended handling for getting the x2APIC ID\r
	123	; would be to use the SNP CPUID table to fetch CPUID.00H:EAX and\r
	124	; CPUID:0BH:EBX[15:0] instead of the GHCB MSR protocol vmgexits\r
	125	; below.\r
	126	;\r
	127	; To avoid the unecessary ugliness to accomplish that here, the BSP\r
	128	; has performed these checks in advance (where #VC handler handles\r
	129	; the CPUID table lookups automatically) and cached them in a flag\r
	130	; so those checks can be skipped here.\r
	131	;\r
	132	mov eax, [esi + MP_CPU_EXCHANGE_INFO_FIELD (SevSnpIsEnabled)]\r
	133	cmp al, 1\r
	134	jne CheckExtTopoAvail\r
	135	\r
	136	;\r
	137	; Even with SEV-SNP, the actual x2APIC ID in CPUID.0BH:EDX\r
	138	; fetched from the hypervisor the same way SEV-ES does it.\r
	139	;\r
	140	mov eax, [esi + MP_CPU_EXCHANGE_INFO_FIELD (ExtTopoAvail)]\r
	141	cmp al, 1\r
	142	je GetApicIdSevEs\r
	143	; The 8-bit APIC ID fallback is also the same as with SEV-ES\r
	144	jmp NoX2ApicSevEs\r
	145	\r
	146	CheckExtTopoAvail:\r
e2289d19 BS	147	mov rdx, 0 ; CPUID function 0\r
	148	mov rax, 0 ; RAX register requested\r
	149	or rax, 4\r
	150	wrmsr\r
	151	rep vmmcall\r
	152	rdmsr\r
	153	cmp edx, 0bh\r
	154	jb NoX2ApicSevEs ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r
	155	\r
	156	mov rdx, 0bh ; CPUID function 0x0b\r
	157	mov rax, 040000000h ; RBX register requested\r
	158	or rax, 4\r
	159	wrmsr\r
	160	rep vmmcall\r
	161	rdmsr\r
	162	test edx, 0ffffh\r
	163	jz NoX2ApicSevEs ; CPUID.0BH:EBX[15:0] is zero\r
	164	\r
d4d7c9ad	165	GetApicIdSevEs:\r
e2289d19 BS	166	mov rdx, 0bh ; CPUID function 0x0b\r
	167	mov rax, 0c0000000h ; RDX register requested\r
	168	or rax, 4\r
	169	wrmsr\r
	170	rep vmmcall\r
	171	rdmsr\r
	172	\r
	173	; Processor is x2APIC capable; 32-bit x2APIC ID is now in EDX\r
	174	jmp RestoreGhcb\r
	175	\r
	176	NoX2ApicSevEs:\r
	177	; Processor is not x2APIC capable, so get 8-bit APIC ID\r
	178	mov rdx, 1 ; CPUID function 1\r
	179	mov rax, 040000000h ; RBX register requested\r
	180	or rax, 4\r
	181	wrmsr\r
	182	rep vmmcall\r
	183	rdmsr\r
	184	shr edx, 24\r
	185	\r
	186	RestoreGhcb:\r
	187	mov rbx, rdx ; Save x2APIC/APIC ID\r
	188	\r
	189	mov rdx, rdi ; RDI holds the saved GHCB GPA\r
	190	shr rdx, 32\r
	191	mov eax, edi\r
	192	wrmsr\r
	193	\r
	194	mov rdx, rbx\r
	195	\r
	196	; x2APIC ID or APIC ID is in EDX\r
	197	jmp GetProcessorNumber\r
	198	\r
	199	SevEsGetApicIdExit:\r
	200	OneTimeCallRet SevEsGetApicId\r
b4d7b9d2 RN	201	\r
	202	\r
	203	;-------------------------------------------------------------------------------------\r
	204	;SwitchToRealProc procedure follows.\r
	205	;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC\r
	206	;IS IN MACHINE CODE.\r
	207	; SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart)\r
	208	; rcx - Buffer Start\r
	209	; rdx - Code16 Selector Offset\r
	210	; r8 - Code32 Selector Offset\r
	211	; r9 - Stack Start\r
	212	;-------------------------------------------------------------------------------------\r
	213	SwitchToRealProcStart:\r
	214	BITS 64\r
	215	cli\r
	216	\r
	217	;\r
	218	; Get RDX reset value before changing stacks since the\r
	219	; new stack won't be able to accomodate a #VC exception.\r
	220	;\r
	221	push rax\r
	222	push rbx\r
	223	push rcx\r
	224	push rdx\r
	225	\r
	226	mov rax, 1\r
	227	cpuid\r
	228	mov rsi, rax ; Save off the reset value for RDX\r
	229	\r
	230	pop rdx\r
	231	pop rcx\r
	232	pop rbx\r
	233	pop rax\r
	234	\r
	235	;\r
	236	; Establish stack below 1MB\r
	237	;\r
	238	mov rsp, r9\r
	239	\r
	240	;\r
	241	; Push ultimate Reset Vector onto the stack\r
	242	;\r
	243	mov rax, rcx\r
	244	shr rax, 4\r
	245	push word 0x0002 ; RFLAGS\r
	246	push ax ; CS\r
	247	push word 0x0000 ; RIP\r
	248	push word 0x0000 ; For alignment, will be discarded\r
	249	\r
	250	;\r
	251	; Get address of "16-bit operand size" label\r
	252	;\r
	253	lea rbx, [PM16Mode]\r
	254	\r
	255	;\r
	256	; Push addresses used to change to compatibility mode\r
	257	;\r
	258	lea rax, [CompatMode]\r
	259	push r8\r
	260	push rax\r
	261	\r
	262	;\r
	263	; Clear R8 - R15, for reset, before going into 32-bit mode\r
	264	;\r
265	xor r8, r8\r
266	xor r9, r9\r
267	xor r10, r10\r
268	xor r11, r11\r
269	xor r12, r12\r
270	xor r13, r13\r
271	xor r14, r14\r
272	xor r15, r15\r
273	\r
274	;\r
275	; Far return into 32-bit mode\r
276	;\r
277	retfq\r
278	\r
279	BITS 32\r
280	CompatMode:\r
281	;\r
282	; Set up stack to prepare for exiting protected mode\r
283	;\r
284	push edx ; Code16 CS\r
285	push ebx ; PM16Mode label address\r
286	\r
287	;\r
288	; Disable paging\r
289	;\r
290	mov eax, cr0 ; Read CR0\r
291	btr eax, 31 ; Set PG=0\r
292	mov cr0, eax ; Write CR0\r
293	\r
294	;\r
295	; Disable long mode\r
296	;\r
297	mov ecx, 0c0000080h ; EFER MSR number\r
298	rdmsr ; Read EFER\r
299	btr eax, 8 ; Set LME=0\r
300	wrmsr ; Write EFER\r
301	\r
302	;\r
303	; Disable PAE\r
304	;\r
305	mov eax, cr4 ; Read CR4\r
306	btr eax, 5 ; Set PAE=0\r
307	mov cr4, eax ; Write CR4\r
308	\r
309	mov edx, esi ; Restore RDX reset value\r
310	\r
311	;\r
312	; Switch to 16-bit operand size\r
313	;\r
314	retf\r
315	\r
316	BITS 16\r
317	;\r
318	; At entry to this label\r
319	; - RDX will have its reset value\r
320	; - On the top of the stack\r
321	; - Alignment data (two bytes) to be discarded\r
322	; - IP for Real Mode (two bytes)\r
323	; - CS for Real Mode (two bytes)\r
324	;\r
325	; This label is also used with AsmRelocateApLoop. During MP finalization,\r
326	; the code from PM16Mode to SwitchToRealProcEnd is copied to the start of\r
327	; the WakeupBuffer, allowing a parked AP to be booted by an OS.\r
328	;\r
329	PM16Mode:\r
330	mov eax, cr0 ; Read CR0\r
331	btr eax, 0 ; Set PE=0\r
332	mov cr0, eax ; Write CR0\r
333	\r
334	pop ax ; Discard alignment data\r
335	\r
336	;\r
337	; Clear registers (except RDX and RSP) before going into 16-bit mode\r
338	;\r
339	xor eax, eax\r
340	xor ebx, ebx\r
341	xor ecx, ecx\r
342	xor esi, esi\r
343	xor edi, edi\r
344	xor ebp, ebp\r
345	\r
346	iret\r
347	\r
348	SwitchToRealProcEnd:\r
c7c25997 XY	349	\r
	350	;-------------------------------------------------------------------------------------\r
	351	; AsmRelocateApLoopAmdSev (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);\r
	352	;-------------------------------------------------------------------------------------\r
	353	\r
	354	AsmRelocateApLoopAmdSevStart:\r
	355	BITS 64\r
	356	cmp qword [rsp + 56], 0 ; SevEsAPJumpTable\r
	357	je NoSevEsAmdSev\r
	358	\r
	359	;\r
	360	; Perform some SEV-ES related setup before leaving 64-bit mode\r
	361	;\r
	362	push rcx\r
	363	push rdx\r
	364	\r
	365	;\r
	366	; Get the RDX reset value using CPUID\r
	367	;\r
	368	mov rax, 1\r
	369	cpuid\r
	370	mov rsi, rax ; Save off the reset value for RDX\r
	371	\r
	372	;\r
	373	; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call\r
	374	; - Must be done while in 64-bit long mode so that writes to\r
	375	; the GHCB memory will be unencrypted.\r
	376	; - No NAE events can be generated once this is set otherwise\r
	377	; the AP_RESET_HOLD SW_EXITCODE will be overwritten.\r
	378	;\r
	379	mov rcx, 0xc0010130\r
	380	rdmsr ; Retrieve current GHCB address\r
	381	shl rdx, 32\r
	382	or rdx, rax\r
	383	\r
	384	mov rdi, rdx\r
	385	xor rax, rax\r
	386	mov rcx, 0x800\r
	387	shr rcx, 3\r
	388	rep stosq ; Clear the GHCB\r
	389	\r
	390	mov rax, 0x80000004 ; VMGEXIT AP_RESET_HOLD\r
	391	mov [rdx + 0x390], rax\r
	392	mov rax, 114 ; Set SwExitCode valid bit\r
	393	bts [rdx + 0x3f0], rax\r
	394	inc rax ; Set SwExitInfo1 valid bit\r
	395	bts [rdx + 0x3f0], rax\r
	396	inc rax ; Set SwExitInfo2 valid bit\r
	397	bts [rdx + 0x3f0], rax\r
	398	\r
	399	pop rdx\r
	400	pop rcx\r
	401	\r
	402	NoSevEsAmdSev:\r
	403	cli ; Disable interrupt before switching to 32-bit mode\r
	404	mov rax, [rsp + 40] ; CountTofinish\r
	405	lock dec dword [rax] ; (*CountTofinish)--\r
	406	\r
	407	mov r10, [rsp + 48] ; Pm16CodeSegment\r
	408	mov rax, [rsp + 56] ; SevEsAPJumpTable\r
	409	mov rbx, [rsp + 64] ; WakeupBuffer\r
	410	mov rsp, r9 ; TopOfApStack\r
	411	\r
	412	push rax ; Save SevEsAPJumpTable\r
413	push rbx ; Save WakeupBuffer\r
414	push r10 ; Save Pm16CodeSegment\r
415	push rcx ; Save MwaitSupport\r
416	push rdx ; Save ApTargetCState\r
417	\r
418	lea rax, [PmEntryAmdSev] ; rax <- The start address of transition code\r
419	\r
420	push r8\r
421	push rax\r
422	\r
423	;\r
424	; Clear R8 - R15, for reset, before going into 32-bit mode\r
425	;\r
426	xor r8, r8\r
427	xor r9, r9\r
428	xor r10, r10\r
429	xor r11, r11\r
430	xor r12, r12\r
431	xor r13, r13\r
432	xor r14, r14\r
433	xor r15, r15\r
434	\r
435	;\r
436	; Far return into 32-bit mode\r
437	;\r
438	o64 retf\r
439	\r
440	BITS 32\r
441	PmEntryAmdSev:\r
442	mov eax, cr0\r
443	btr eax, 31 ; Clear CR0.PG\r
444	mov cr0, eax ; Disable paging and caches\r
445	\r
446	mov ecx, 0xc0000080\r
447	rdmsr\r
448	and ah, ~ 1 ; Clear LME\r
449	wrmsr\r
450	mov eax, cr4\r
451	and al, ~ (1 << 5) ; Clear PAE\r
452	mov cr4, eax\r
453	\r
454	pop edx\r
455	add esp, 4\r
456	pop ecx,\r
457	add esp, 4\r
458	\r
459	MwaitCheckAmdSev:\r
460	cmp cl, 1 ; Check mwait-monitor support\r
461	jnz HltLoopAmdSev\r
462	mov ebx, edx ; Save C-State to ebx\r
463	MwaitLoopAmdSev:\r
464	cli\r
465	mov eax, esp ; Set Monitor Address\r
466	xor ecx, ecx ; ecx = 0\r
467	xor edx, edx ; edx = 0\r
468	monitor\r
469	mov eax, ebx ; Mwait Cx, Target C-State per eax[7:4]\r
470	shl eax, 4\r
471	mwait\r
472	jmp MwaitLoopAmdSev\r
473	\r
474	HltLoopAmdSev:\r
475	pop edx ; PM16CodeSegment\r
476	add esp, 4\r
477	pop ebx ; WakeupBuffer\r
478	add esp, 4\r
479	pop eax ; SevEsAPJumpTable\r
480	add esp, 4\r
481	cmp eax, 0 ; Check for SEV-ES\r
482	je DoHltAmdSev\r
483	\r
484	cli\r
485	;\r
486	; SEV-ES is enabled, use VMGEXIT (GHCB information already\r
487	; set by caller)\r
488	;\r
489	BITS 64\r
490	rep vmmcall\r
491	BITS 32\r
492	\r
493	;\r
494	; Back from VMGEXIT AP_HLT_LOOP\r
495	; Push the FLAGS/CS/IP values to use\r
496	;\r
497	push word 0x0002 ; EFLAGS\r
498	xor ecx, ecx\r
499	mov cx, [eax + 2] ; CS\r
500	push cx\r
501	mov cx, [eax] ; IP\r
502	push cx\r
503	push word 0x0000 ; For alignment, will be discarded\r
504	\r
505	push edx\r
506	push ebx\r
507	\r
508	mov edx, esi ; Restore RDX reset value\r
509	\r
510	retf\r
511	\r
512	DoHltAmdSev:\r
513	cli\r
514	hlt\r
515	jmp DoHltAmdSev\r
516	\r
517	BITS 64\r
518	AsmRelocateApLoopAmdSevEnd:\r