[mirror_edk2.git] / UefiCpuPkg / Library / MpInitLib / X64 / MpFuncs.nasm

;------------------------------------------------------------------------------ ;\r
; Copyright (c) 2015 - 2019, Intel Corporation. All rights reserved.<BR>\r
; SPDX-License-Identifier: BSD-2-Clause-Patent\r
;\r
; Module Name:\r
;\r
;   MpFuncs.nasm\r
;\r
; Abstract:\r
;\r
;   This is the assembly code for MP support\r
;\r
;-------------------------------------------------------------------------------\r
\r
%include "MpEqu.inc"\r
extern ASM_PFX(InitializeFloatingPointUnits)\r
\r
DEFAULT REL\r
\r
SECTION .text\r
\r
;-------------------------------------------------------------------------------------\r
;RendezvousFunnelProc  procedure follows. All APs execute their procedure. This\r
;procedure serializes all the AP processors through an Init sequence. It must be\r
;noted that APs arrive here very raw...ie: real mode, no stack.\r
;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC\r
;IS IN MACHINE CODE.\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(RendezvousFunnelProc)\r
ASM_PFX(RendezvousFunnelProc):\r
RendezvousFunnelProcStart:\r
; At this point CS = 0x(vv00) and ip= 0x0.\r
; Save BIST information to ebp firstly\r
\r
BITS 16\r
    mov        ebp, eax                        ; Save BIST information\r
\r
    mov        ax, cs\r
    mov        ds, ax\r
    mov        es, ax\r
    mov        ss, ax\r
    xor        ax, ax\r
    mov        fs, ax\r
    mov        gs, ax\r
\r
    mov        si,  BufferStartLocation\r
    mov        ebx, [si]\r
\r
    mov        si,  DataSegmentLocation\r
    mov        edx, [si]\r
\r
    ;\r
    ; Get start address of 32-bit code in low memory (<1MB)\r
    ;\r
    mov        edi, ModeTransitionMemoryLocation\r
\r
    mov        si, GdtrLocation\r
o32 lgdt       [cs:si]\r
\r
    mov        si, IdtrLocation\r
o32 lidt       [cs:si]\r
\r
    ;\r
    ; Switch to protected mode\r
    ;\r
    mov        eax, cr0                    ; Get control register 0\r
    or         eax, 000000003h             ; Set PE bit (bit #0) & MP\r
    mov        cr0, eax\r
\r
    ; Switch to 32-bit code (>1MB)\r
o32 jmp far    [cs:di]\r
\r
;\r
; Following code must be copied to memory with type of EfiBootServicesCode.\r
; This is required if NX is enabled for EfiBootServicesCode of memory.\r
;\r
BITS 32\r
Flat32Start:                                   ; protected mode entry point\r
    mov        ds, dx\r
    mov        es, dx\r
    mov        fs, dx\r
    mov        gs, dx\r
    mov        ss, dx\r
\r
    ;\r
    ; Enable execute disable bit\r
    ;\r
    mov        esi, EnableExecuteDisableLocation\r
    cmp        byte [ebx + esi], 0\r
    jz         SkipEnableExecuteDisableBit\r
\r
    mov        ecx, 0c0000080h             ; EFER MSR number\r
    rdmsr                                  ; Read EFER\r
    bts        eax, 11                     ; Enable Execute Disable Bit\r
    wrmsr                                  ; Write EFER\r
\r
SkipEnableExecuteDisableBit:\r
    ;\r
    ; Enable PAE\r
    ;\r
    mov        eax, cr4\r
    bts        eax, 5\r
\r
    mov        esi, Enable5LevelPagingLocation\r
    cmp        byte [ebx + esi], 0\r
    jz         SkipEnable5LevelPaging\r
\r
    ;\r
    ; Enable 5 Level Paging\r
    ;\r
    bts        eax, 12                     ; Set LA57=1.\r
\r
SkipEnable5LevelPaging:\r
\r
    mov        cr4, eax\r
\r
    ;\r
    ; Load page table\r
    ;\r
    mov        esi, Cr3Location             ; Save CR3 in ecx\r
    mov        ecx, [ebx + esi]\r
    mov        cr3, ecx                    ; Load CR3\r
\r
    ;\r
    ; Enable long mode\r
    ;\r
    mov        ecx, 0c0000080h             ; EFER MSR number\r
    rdmsr                                  ; Read EFER\r
    bts        eax, 8                      ; Set LME=1\r
    wrmsr                                  ; Write EFER\r
\r
    ;\r
    ; Enable paging\r
    ;\r
    mov        eax, cr0                    ; Read CR0\r
    bts        eax, 31                     ; Set PG=1\r
    mov        cr0, eax                    ; Write CR0\r
\r
    ;\r
    ; Far jump to 64-bit code\r
    ;\r
    mov        edi, ModeHighMemoryLocation\r
    add        edi, ebx\r
    jmp far    [edi]\r
\r
BITS 64\r
LongModeStart:\r
    mov        esi, ebx\r
    lea        edi, [esi + InitFlagLocation]\r
    cmp        qword [edi], 1       ; ApInitConfig\r
    jnz        GetApicId\r
\r
    ; Increment the number of APs executing here as early as possible\r
    ; This is decremented in C code when AP is finished executing\r
    mov        edi, esi\r
    add        edi, NumApsExecutingLocation\r
    lock inc   dword [edi]\r
\r
    ; AP init\r
    mov        edi, esi\r
    add        edi, LockLocation\r
    mov        rax, NotVacantFlag\r
\r
TestLock:\r
    xchg       qword [edi], rax\r
    cmp        rax, NotVacantFlag\r
    jz         TestLock\r
\r
    lea        ecx, [esi + ApIndexLocation]\r
    inc        dword [ecx]\r
    mov        ebx, [ecx]\r
\r
Releaselock:\r
    mov        rax, VacantFlag\r
    xchg       qword [edi], rax\r
    ; program stack\r
    mov        edi, esi\r
    add        edi, StackSizeLocation\r
    mov        eax, dword [edi]\r
    mov        ecx, ebx\r
    inc        ecx\r
    mul        ecx                               ; EAX = StackSize * (CpuNumber + 1)\r
    mov        edi, esi\r
    add        edi, StackStartAddressLocation\r
    add        rax, qword [edi]\r
    mov        rsp, rax\r
\r
    lea        edi, [esi + SevEsIsEnabledLocation]\r
    cmp        byte [edi], 1        ; SevEsIsEnabled\r
    jne        CProcedureInvoke\r
\r
    ;\r
    ; program GHCB\r
    ;   Each page after the GHCB is a per-CPU page, so the calculation programs\r
    ;   a GHCB to be every 8KB.\r
    ;\r
    mov        eax, SIZE_4KB\r
    shl        eax, 1                            ; EAX = SIZE_4K * 2\r
    mov        ecx, ebx\r
    mul        ecx                               ; EAX = SIZE_4K * 2 * CpuNumber\r
    mov        edi, esi\r
    add        edi, GhcbBaseLocation\r
    add        rax, qword [edi]\r
    mov        rdx, rax\r
    shr        rdx, 32\r
    mov        rcx, 0xc0010130\r
    wrmsr\r
    jmp        CProcedureInvoke\r
\r
GetApicId:\r
    lea        edi, [esi + SevEsIsEnabledLocation]\r
    cmp        byte [edi], 1        ; SevEsIsEnabled\r
    jne        DoCpuid\r
\r
    ;\r
    ; Since we don't have a stack yet, we can't take a #VC\r
    ; exception. Use the GHCB protocol to perform the CPUID\r
    ; calls.\r
    ;\r
    mov        rcx, 0xc0010130\r
    rdmsr\r
    shl        rdx, 32\r
    or         rax, rdx\r
    mov        rdi, rax             ; RDI now holds the original GHCB GPA\r
\r
    mov        rdx, 0               ; CPUID function 0\r
    mov        rax, 0               ; RAX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
    cmp        edx, 0bh\r
    jb         NoX2ApicSevEs        ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r
\r
    mov        rdx, 0bh             ; CPUID function 0x0b\r
    mov        rax, 040000000h      ; RBX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
    test       edx, 0ffffh\r
    jz         NoX2ApicSevEs        ; CPUID.0BH:EBX[15:0] is zero\r
\r
    mov        rdx, 0bh             ; CPUID function 0x0b\r
    mov        rax, 0c0000000h      ; RDX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
\r
    ; Processor is x2APIC capable; 32-bit x2APIC ID is now in EDX\r
    jmp        RestoreGhcb\r
\r
NoX2ApicSevEs:\r
    ; Processor is not x2APIC capable, so get 8-bit APIC ID\r
    mov        rdx, 1               ; CPUID function 1\r
    mov        rax, 040000000h      ; RBX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
    shr        edx, 24\r
\r
RestoreGhcb:\r
    mov        rbx, rdx             ; Save x2APIC/APIC ID\r
\r
    mov        rdx, rdi             ; RDI holds the saved GHCB GPA\r
    shr        rdx, 32\r
    mov        eax, edi\r
    wrmsr\r
\r
    mov        rdx, rbx\r
\r
    ; x2APIC ID or APIC ID is in EDX\r
    jmp        GetProcessorNumber\r
\r
DoCpuid:\r
    mov        eax, 0\r
    cpuid\r
    cmp        eax, 0bh\r
    jb         NoX2Apic             ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r
\r
    mov        eax, 0bh\r
    xor        ecx, ecx\r
    cpuid\r
    test       ebx, 0ffffh\r
    jz         NoX2Apic             ; CPUID.0BH:EBX[15:0] is zero\r
\r
    ; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX\r
    jmp        GetProcessorNumber\r
\r
NoX2Apic:\r
    ; Processor is not x2APIC capable, so get 8-bit APIC ID\r
    mov        eax, 1\r
    cpuid\r
    shr        ebx, 24\r
    mov        edx, ebx\r
\r
GetProcessorNumber:\r
    ;\r
    ; Get processor number for this AP\r
    ; Note that BSP may become an AP due to SwitchBsp()\r
    ;\r
    xor         ebx, ebx\r
    lea         eax, [esi + CpuInfoLocation]\r
    mov         rdi, [eax]\r
\r
GetNextProcNumber:\r
    cmp         dword [rdi], edx                      ; APIC ID match?\r
    jz          ProgramStack\r
    add         rdi, 20\r
    inc         ebx\r
    jmp         GetNextProcNumber\r
\r
ProgramStack:\r
    mov         rsp, qword [rdi + 12]\r
\r
CProcedureInvoke:\r
    push       rbp               ; Push BIST data at top of AP stack\r
    xor        rbp, rbp          ; Clear ebp for call stack trace\r
    push       rbp\r
    mov        rbp, rsp\r
\r
    mov        rax, qword [esi + InitializeFloatingPointUnitsAddress]\r
    sub        rsp, 20h\r
    call       rax               ; Call assembly function to initialize FPU per UEFI spec\r
    add        rsp, 20h\r
\r
    mov        edx, ebx          ; edx is ApIndex\r
    mov        ecx, esi\r
    add        ecx, LockLocation ; rcx is address of exchange info data buffer\r
\r
    mov        edi, esi\r
    add        edi, ApProcedureLocation\r
    mov        rax, qword [edi]\r
\r
    sub        rsp, 20h\r
    call       rax               ; Invoke C function\r
    add        rsp, 20h\r
    jmp        $                 ; Should never reach here\r
\r
RendezvousFunnelProcEnd:\r
\r
;-------------------------------------------------------------------------------------\r
;SwitchToRealProc procedure follows.\r
;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC\r
;IS IN MACHINE CODE.\r
;  SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart)\r
;  rcx - Buffer Start\r
;  rdx - Code16 Selector Offset\r
;  r8  - Code32 Selector Offset\r
;  r9  - Stack Start\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(SwitchToRealProc)\r
ASM_PFX(SwitchToRealProc):\r
SwitchToRealProcStart:\r
BITS 64\r
    cli\r
\r
    ;\r
    ; Get RDX reset value before changing stacks since the\r
    ; new stack won't be able to accomodate a #VC exception.\r
    ;\r
    push       rax\r
    push       rbx\r
    push       rcx\r
    push       rdx\r
\r
    mov        rax, 1\r
    cpuid\r
    mov        rsi, rax                    ; Save off the reset value for RDX\r
\r
    pop        rdx\r
    pop        rcx\r
    pop        rbx\r
    pop        rax\r
\r
    ;\r
    ; Establish stack below 1MB\r
    ;\r
    mov        rsp, r9\r
\r
    ;\r
    ; Push ultimate Reset Vector onto the stack\r
    ;\r
    mov        rax, rcx\r
    shr        rax, 4\r
    push       word 0x0002                 ; RFLAGS\r
    push       ax                          ; CS\r
    push       word 0x0000                 ; RIP\r
    push       word 0x0000                 ; For alignment, will be discarded\r
\r
    ;\r
    ; Get address of "16-bit operand size" label\r
    ;\r
    lea        rbx, [PM16Mode]\r
\r
    ;\r
    ; Push addresses used to change to compatibility mode\r
    ;\r
    lea        rax, [CompatMode]\r
    push       r8\r
    push       rax\r
\r
    ;\r
    ; Clear R8 - R15, for reset, before going into 32-bit mode\r
    ;\r
    xor        r8, r8\r
    xor        r9, r9\r
    xor        r10, r10\r
    xor        r11, r11\r
    xor        r12, r12\r
    xor        r13, r13\r
    xor        r14, r14\r
    xor        r15, r15\r
\r
    ;\r
    ; Far return into 32-bit mode\r
    ;\r
o64 retf\r
\r
BITS 32\r
CompatMode:\r
    ;\r
    ; Set up stack to prepare for exiting protected mode\r
    ;\r
    push       edx                         ; Code16 CS\r
    push       ebx                         ; PM16Mode label address\r
\r
    ;\r
    ; Disable paging\r
    ;\r
    mov        eax, cr0                    ; Read CR0\r
    btr        eax, 31                     ; Set PG=0\r
    mov        cr0, eax                    ; Write CR0\r
\r
    ;\r
    ; Disable long mode\r
    ;\r
    mov        ecx, 0c0000080h             ; EFER MSR number\r
    rdmsr                                  ; Read EFER\r
    btr        eax, 8                      ; Set LME=0\r
    wrmsr                                  ; Write EFER\r
\r
    ;\r
    ; Disable PAE\r
    ;\r
    mov        eax, cr4                    ; Read CR4\r
    btr        eax, 5                      ; Set PAE=0\r
    mov        cr4, eax                    ; Write CR4\r
\r
    mov        edx, esi                    ; Restore RDX reset value\r
\r
    ;\r
    ; Switch to 16-bit operand size\r
    ;\r
    retf\r
\r
BITS 16\r
    ;\r
    ; At entry to this label\r
    ;   - RDX will have its reset value\r
    ;   - On the top of the stack\r
    ;     - Alignment data (two bytes) to be discarded\r
    ;     - IP for Real Mode (two bytes)\r
    ;     - CS for Real Mode (two bytes)\r
    ;\r
    ; This label is also used with AsmRelocateApLoop. During MP finalization,\r
    ; the code from PM16Mode to SwitchToRealProcEnd is copied to the start of\r
    ; the WakeupBuffer, allowing a parked AP to be booted by an OS.\r
    ;\r
PM16Mode:\r
    mov        eax, cr0                    ; Read CR0\r
    btr        eax, 0                      ; Set PE=0\r
    mov        cr0, eax                    ; Write CR0\r
\r
    pop        ax                          ; Discard alignment data\r
\r
    ;\r
    ; Clear registers (except RDX and RSP) before going into 16-bit mode\r
    ;\r
    xor        eax, eax\r
    xor        ebx, ebx\r
    xor        ecx, ecx\r
    xor        esi, esi\r
    xor        edi, edi\r
    xor        ebp, ebp\r
\r
    iret\r
\r
SwitchToRealProcEnd:\r
\r
;-------------------------------------------------------------------------------------\r
;  AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(AsmRelocateApLoop)\r
ASM_PFX(AsmRelocateApLoop):\r
AsmRelocateApLoopStart:\r
BITS 64\r
    cmp        qword [rsp + 56], 0  ; SevEsAPJumpTable\r
    je         NoSevEs\r
\r
    ;\r
    ; Perform some SEV-ES related setup before leaving 64-bit mode\r
    ;\r
    push       rcx\r
    push       rdx\r
\r
    ;\r
    ; Get the RDX reset value using CPUID\r
    ;\r
    mov        rax, 1\r
    cpuid\r
    mov        rsi, rax          ; Save off the reset value for RDX\r
\r
    ;\r
    ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call\r
    ;   - Must be done while in 64-bit long mode so that writes to\r
    ;     the GHCB memory will be unencrypted.\r
    ;   - No NAE events can be generated once this is set otherwise\r
    ;     the AP_RESET_HOLD SW_EXITCODE will be overwritten.\r
    ;\r
    mov        rcx, 0xc0010130\r
    rdmsr                        ; Retrieve current GHCB address\r
    shl        rdx, 32\r
    or         rdx, rax\r
\r
    mov        rdi, rdx\r
    xor        rax, rax\r
    mov        rcx, 0x800\r
    shr        rcx, 3\r
    rep stosq                    ; Clear the GHCB\r
\r
    mov        rax, 0x80000004   ; VMGEXIT AP_RESET_HOLD\r
    mov        [rdx + 0x390], rax\r
    mov        rax, 114          ; Set SwExitCode valid bit\r
    bts        [rdx + 0x3f0], rax\r
    inc        rax               ; Set SwExitInfo1 valid bit\r
    bts        [rdx + 0x3f0], rax\r
    inc        rax               ; Set SwExitInfo2 valid bit\r
    bts        [rdx + 0x3f0], rax\r
\r
    pop        rdx\r
    pop        rcx\r
\r
NoSevEs:\r
    cli                          ; Disable interrupt before switching to 32-bit mode\r
    mov        rax, [rsp + 40]   ; CountTofinish\r
    lock dec   dword [rax]       ; (*CountTofinish)--\r
\r
    mov        r10, [rsp + 48]   ; Pm16CodeSegment\r
    mov        rax, [rsp + 56]   ; SevEsAPJumpTable\r
    mov        rbx, [rsp + 64]   ; WakeupBuffer\r
    mov        rsp, r9           ; TopOfApStack\r
\r
    push       rax               ; Save SevEsAPJumpTable\r
    push       rbx               ; Save WakeupBuffer\r
    push       r10               ; Save Pm16CodeSegment\r
    push       rcx               ; Save MwaitSupport\r
    push       rdx               ; Save ApTargetCState\r
\r
    lea        rax, [PmEntry]    ; rax <- The start address of transition code\r
\r
    push       r8\r
    push       rax\r
\r
    ;\r
    ; Clear R8 - R15, for reset, before going into 32-bit mode\r
    ;\r
    xor        r8, r8\r
    xor        r9, r9\r
    xor        r10, r10\r
    xor        r11, r11\r
    xor        r12, r12\r
    xor        r13, r13\r
    xor        r14, r14\r
    xor        r15, r15\r
\r
    ;\r
    ; Far return into 32-bit mode\r
    ;\r
o64 retf\r
\r
BITS 32\r
PmEntry:\r
    mov        eax, cr0\r
    btr        eax, 31           ; Clear CR0.PG\r
    mov        cr0, eax          ; Disable paging and caches\r
\r
    mov        ecx, 0xc0000080\r
    rdmsr\r
    and        ah, ~ 1           ; Clear LME\r
    wrmsr\r
    mov        eax, cr4\r
    and        al, ~ (1 << 5)    ; Clear PAE\r
    mov        cr4, eax\r
\r
    pop        edx\r
    add        esp, 4\r
    pop        ecx,\r
    add        esp, 4\r
\r
MwaitCheck:\r
    cmp        cl, 1              ; Check mwait-monitor support\r
    jnz        HltLoop\r
    mov        ebx, edx           ; Save C-State to ebx\r
MwaitLoop:\r
    cli\r
    mov        eax, esp           ; Set Monitor Address\r
    xor        ecx, ecx           ; ecx = 0\r
    xor        edx, edx           ; edx = 0\r
    monitor\r
    mov        eax, ebx           ; Mwait Cx, Target C-State per eax[7:4]\r
    shl        eax, 4\r
    mwait\r
    jmp        MwaitLoop\r
\r
HltLoop:\r
    pop        edx                ; PM16CodeSegment\r
    add        esp, 4\r
    pop        ebx                ; WakeupBuffer\r
    add        esp, 4\r
    pop        eax                ; SevEsAPJumpTable\r
    add        esp, 4\r
    cmp        eax, 0             ; Check for SEV-ES\r
    je         DoHlt\r
\r
    cli\r
    ;\r
    ; SEV-ES is enabled, use VMGEXIT (GHCB information already\r
    ; set by caller)\r
    ;\r
BITS 64\r
    rep        vmmcall\r
BITS 32\r
\r
    ;\r
    ; Back from VMGEXIT AP_HLT_LOOP\r
    ;   Push the FLAGS/CS/IP values to use\r
    ;\r
    push       word 0x0002        ; EFLAGS\r
    xor        ecx, ecx\r
    mov        cx, [eax + 2]      ; CS\r
    push       cx\r
    mov        cx, [eax]          ; IP\r
    push       cx\r
    push       word 0x0000        ; For alignment, will be discarded\r
\r
    push       edx\r
    push       ebx\r
\r
    mov        edx, esi           ; Restore RDX reset value\r
\r
    retf\r
\r
DoHlt:\r
    cli\r
    hlt\r
    jmp        DoHlt\r
\r
BITS 64\r
AsmRelocateApLoopEnd:\r
\r
;-------------------------------------------------------------------------------------\r
;  AsmGetAddressMap (&AddressMap);\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(AsmGetAddressMap)\r
ASM_PFX(AsmGetAddressMap):\r
    lea        rax, [ASM_PFX(RendezvousFunnelProc)]\r
    mov        qword [rcx], rax\r
    mov        qword [rcx +  8h], LongModeStart - RendezvousFunnelProcStart\r
    mov        qword [rcx + 10h], RendezvousFunnelProcEnd - RendezvousFunnelProcStart\r
    lea        rax, [ASM_PFX(AsmRelocateApLoop)]\r
    mov        qword [rcx + 18h], rax\r
    mov        qword [rcx + 20h], AsmRelocateApLoopEnd - AsmRelocateApLoopStart\r
    mov        qword [rcx + 28h], Flat32Start - RendezvousFunnelProcStart\r
    mov        qword [rcx + 30h], SwitchToRealProcEnd - SwitchToRealProcStart          ; SwitchToRealSize\r
    mov        qword [rcx + 38h], SwitchToRealProcStart - RendezvousFunnelProcStart    ; SwitchToRealOffset\r
    mov        qword [rcx + 40h], SwitchToRealProcStart - Flat32Start                  ; SwitchToRealNoNxOffset\r
    mov        qword [rcx + 48h], PM16Mode - RendezvousFunnelProcStart                 ; SwitchToRealPM16ModeOffset\r
    mov        qword [rcx + 50h], SwitchToRealProcEnd - PM16Mode                       ; SwitchToRealPM16ModeSize\r
    ret\r
\r
;-------------------------------------------------------------------------------------\r
;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is\r
;about to become an AP. It switches its stack with the current AP.\r
;AsmExchangeRole (IN   CPU_EXCHANGE_INFO    *MyInfo, IN   CPU_EXCHANGE_INFO    *OthersInfo);\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(AsmExchangeRole)\r
ASM_PFX(AsmExchangeRole):\r
    ; DO NOT call other functions in this function, since 2 CPU may use 1 stack\r
    ; at the same time. If 1 CPU try to call a function, stack will be corrupted.\r
\r
    push       rax\r
    push       rbx\r
    push       rcx\r
    push       rdx\r
    push       rsi\r
    push       rdi\r
    push       rbp\r
    push       r8\r
    push       r9\r
    push       r10\r
    push       r11\r
    push       r12\r
    push       r13\r
    push       r14\r
    push       r15\r
\r
    mov        rax, cr0\r
    push       rax\r
\r
    mov        rax, cr4\r
    push       rax\r
\r
    ; rsi contains MyInfo pointer\r
    mov        rsi, rcx\r
\r
    ; rdi contains OthersInfo pointer\r
    mov        rdi, rdx\r
\r
    ;Store EFLAGS, GDTR and IDTR regiter to stack\r
    pushfq\r
    sgdt       [rsi + 16]\r
    sidt       [rsi + 26]\r
\r
    ; Store the its StackPointer\r
    mov        [rsi + 8], rsp\r
\r
    ; update its switch state to STORED\r
    mov        byte [rsi], CPU_SWITCH_STATE_STORED\r
\r
WaitForOtherStored:\r
    ; wait until the other CPU finish storing its state\r
    cmp        byte [rdi], CPU_SWITCH_STATE_STORED\r
    jz         OtherStored\r
    pause\r
    jmp        WaitForOtherStored\r
\r
OtherStored:\r
    ; Since another CPU already stored its state, load them\r
    ; load GDTR value\r
    lgdt       [rdi + 16]\r
\r
    ; load IDTR value\r
    lidt       [rdi + 26]\r
\r
    ; load its future StackPointer\r
    mov        rsp, [rdi + 8]\r
\r
    ; update the other CPU's switch state to LOADED\r
    mov        byte [rdi], CPU_SWITCH_STATE_LOADED\r
\r
WaitForOtherLoaded:\r
    ; wait until the other CPU finish loading new state,\r
    ; otherwise the data in stack may corrupt\r
    cmp        byte [rsi], CPU_SWITCH_STATE_LOADED\r
    jz         OtherLoaded\r
    pause\r
    jmp        WaitForOtherLoaded\r
\r
OtherLoaded:\r
    ; since the other CPU already get the data it want, leave this procedure\r
    popfq\r
\r
    pop        rax\r
    mov        cr4, rax\r
\r
    pop        rax\r
    mov        cr0, rax\r
\r
    pop        r15\r
    pop        r14\r
    pop        r13\r
    pop        r12\r
    pop        r11\r
    pop        r10\r
    pop        r9\r
    pop        r8\r
    pop        rbp\r
    pop        rdi\r
    pop        rsi\r
    pop        rdx\r
    pop        rcx\r
    pop        rbx\r
    pop        rax\r
\r
    ret\r
Commit	Line	Data
d94e5f67	1	;------------------------------------------------------------------------------ ;\r
09f69a87	2	; Copyright (c) 2015 - 2019, Intel Corporation. All rights reserved.<BR>\r
0acd8697	3	; SPDX-License-Identifier: BSD-2-Clause-Patent\r
d94e5f67 JF	4	;\r
	5	; Module Name:\r
	6	;\r
	7	; MpFuncs.nasm\r
	8	;\r
	9	; Abstract:\r
	10	;\r
	11	; This is the assembly code for MP support\r
	12	;\r
	13	;-------------------------------------------------------------------------------\r
	14	\r
	15	%include "MpEqu.inc"\r
	16	extern ASM_PFX(InitializeFloatingPointUnits)\r
	17	\r
	18	DEFAULT REL\r
	19	\r
	20	SECTION .text\r
	21	\r
	22	;-------------------------------------------------------------------------------------\r
	23	;RendezvousFunnelProc procedure follows. All APs execute their procedure. This\r
	24	;procedure serializes all the AP processors through an Init sequence. It must be\r
	25	;noted that APs arrive here very raw...ie: real mode, no stack.\r
	26	;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC\r
	27	;IS IN MACHINE CODE.\r
	28	;-------------------------------------------------------------------------------------\r
	29	global ASM_PFX(RendezvousFunnelProc)\r
	30	ASM_PFX(RendezvousFunnelProc):\r
	31	RendezvousFunnelProcStart:\r
	32	; At this point CS = 0x(vv00) and ip= 0x0.\r
	33	; Save BIST information to ebp firstly\r
	34	\r
	35	BITS 16\r
	36	mov ebp, eax ; Save BIST information\r
	37	\r
	38	mov ax, cs\r
	39	mov ds, ax\r
	40	mov es, ax\r
	41	mov ss, ax\r
	42	xor ax, ax\r
	43	mov fs, ax\r
	44	mov gs, ax\r
	45	\r
	46	mov si, BufferStartLocation\r
	47	mov ebx, [si]\r
	48	\r
f32bfe6d JW	49	mov si, DataSegmentLocation\r
	50	mov edx, [si]\r
	51	\r
	52	;\r
	53	; Get start address of 32-bit code in low memory (<1MB)\r
	54	;\r
	55	mov edi, ModeTransitionMemoryLocation\r
d94e5f67 JF	56	\r
	57	mov si, GdtrLocation\r
	58	o32 lgdt [cs:si]\r
	59	\r
	60	mov si, IdtrLocation\r
	61	o32 lidt [cs:si]\r
	62	\r
f32bfe6d JW	63	;\r
	64	; Switch to protected mode\r
	65	;\r
	66	mov eax, cr0 ; Get control register 0\r
	67	or eax, 000000003h ; Set PE bit (bit #0) & MP\r
	68	mov cr0, eax\r
	69	\r
	70	; Switch to 32-bit code (>1MB)\r
	71	o32 jmp far [cs:di]\r
	72	\r
	73	;\r
	74	; Following code must be copied to memory with type of EfiBootServicesCode.\r
	75	; This is required if NX is enabled for EfiBootServicesCode of memory.\r
	76	;\r
	77	BITS 32\r
	78	Flat32Start: ; protected mode entry point\r
	79	mov ds, dx\r
	80	mov es, dx\r
	81	mov fs, dx\r
	82	mov gs, dx\r
	83	mov ss, dx\r
5c66d125 JF	84	\r
	85	;\r
	86	; Enable execute disable bit\r
	87	;\r
f32bfe6d JW	88	mov esi, EnableExecuteDisableLocation\r
	89	cmp byte [ebx + esi], 0\r
	90	jz SkipEnableExecuteDisableBit\r
	91	\r
5c66d125 JF	92	mov ecx, 0c0000080h ; EFER MSR number\r
	93	rdmsr ; Read EFER\r
	94	bts eax, 11 ; Enable Execute Disable Bit\r
	95	wrmsr ; Write EFER\r
	96	\r
	97	SkipEnableExecuteDisableBit:\r
f32bfe6d JW	98	;\r
	99	; Enable PAE\r
	100	;\r
d94e5f67 JF	101	mov eax, cr4\r
d94e5f67 JF	102	bts eax, 5\r
09f69a87 RN	103	\r
	104	mov esi, Enable5LevelPagingLocation\r
	105	cmp byte [ebx + esi], 0\r
	106	jz SkipEnable5LevelPaging\r
	107	\r
	108	;\r
	109	; Enable 5 Level Paging\r
	110	;\r
	111	bts eax, 12 ; Set LA57=1.\r
	112	\r
	113	SkipEnable5LevelPaging:\r
	114	\r
d94e5f67 JF	115	mov cr4, eax\r
d94e5f67 JF	116	\r
f32bfe6d JW	117	;\r
	118	; Load page table\r
	119	;\r
	120	mov esi, Cr3Location ; Save CR3 in ecx\r
	121	mov ecx, [ebx + esi]\r
d94e5f67 JF	122	mov cr3, ecx ; Load CR3\r
d94e5f67 JF	123	\r
f32bfe6d JW	124	;\r
	125	; Enable long mode\r
	126	;\r
d94e5f67 JF	127	mov ecx, 0c0000080h ; EFER MSR number\r
	128	rdmsr ; Read EFER\r
	129	bts eax, 8 ; Set LME=1\r
	130	wrmsr ; Write EFER\r
	131	\r
f32bfe6d JW	132	;\r
	133	; Enable paging\r
	134	;\r
d94e5f67 JF	135	mov eax, cr0 ; Read CR0\r
	136	bts eax, 31 ; Set PG=1\r
	137	mov cr0, eax ; Write CR0\r
	138	\r
f32bfe6d JW	139	;\r
	140	; Far jump to 64-bit code\r
	141	;\r
	142	mov edi, ModeHighMemoryLocation\r
	143	add edi, ebx\r
	144	jmp far [edi]\r
	145	\r
d94e5f67 JF	146	BITS 64\r
d94e5f67 JF	147	LongModeStart:\r
845c5be1 JF	148	mov esi, ebx\r
	149	lea edi, [esi + InitFlagLocation]\r
	150	cmp qword [edi], 1 ; ApInitConfig\r
	151	jnz GetApicId\r
	152	\r
0594ec41 ED	153	; Increment the number of APs executing here as early as possible\r
	154	; This is decremented in C code when AP is finished executing\r
	155	mov edi, esi\r
	156	add edi, NumApsExecutingLocation\r
	157	lock inc dword [edi]\r
	158	\r
845c5be1	159	; AP init\r
d94e5f67 JF	160	mov edi, esi\r
	161	add edi, LockLocation\r
	162	mov rax, NotVacantFlag\r
	163	\r
	164	TestLock:\r
	165	xchg qword [edi], rax\r
	166	cmp rax, NotVacantFlag\r
	167	jz TestLock\r
	168	\r
37676b9f	169	lea ecx, [esi + ApIndexLocation]\r
845c5be1 JF	170	inc dword [ecx]\r
845c5be1 JF	171	mov ebx, [ecx]\r
d94e5f67	172	\r
845c5be1 JF	173	Releaselock:\r
	174	mov rax, VacantFlag\r
	175	xchg qword [edi], rax\r
	176	; program stack\r
d94e5f67 JF	177	mov edi, esi\r
d94e5f67 JF	178	add edi, StackSizeLocation\r
845c5be1 JF	179	mov eax, dword [edi]\r
	180	mov ecx, ebx\r
	181	inc ecx\r
	182	mul ecx ; EAX = StackSize * (CpuNumber + 1)\r
d94e5f67 JF	183	mov edi, esi\r
	184	add edi, StackStartAddressLocation\r
	185	add rax, qword [edi]\r
	186	mov rsp, rax\r
7b7508ad TL	187	\r
	188	lea edi, [esi + SevEsIsEnabledLocation]\r
	189	cmp byte [edi], 1 ; SevEsIsEnabled\r
	190	jne CProcedureInvoke\r
	191	\r
	192	;\r
	193	; program GHCB\r
	194	; Each page after the GHCB is a per-CPU page, so the calculation programs\r
	195	; a GHCB to be every 8KB.\r
	196	;\r
	197	mov eax, SIZE_4KB\r
	198	shl eax, 1 ; EAX = SIZE_4K * 2\r
	199	mov ecx, ebx\r
	200	mul ecx ; EAX = SIZE_4K * 2 * CpuNumber\r
	201	mov edi, esi\r
	202	add edi, GhcbBaseLocation\r
	203	add rax, qword [edi]\r
	204	mov rdx, rax\r
	205	shr rdx, 32\r
	206	mov rcx, 0xc0010130\r
	207	wrmsr\r
845c5be1 JF	208	jmp CProcedureInvoke\r
	209	\r
	210	GetApicId:\r
7b7508ad TL	211	lea edi, [esi + SevEsIsEnabledLocation]\r
	212	cmp byte [edi], 1 ; SevEsIsEnabled\r
	213	jne DoCpuid\r
	214	\r
	215	;\r
	216	; Since we don't have a stack yet, we can't take a #VC\r
	217	; exception. Use the GHCB protocol to perform the CPUID\r
	218	; calls.\r
	219	;\r
	220	mov rcx, 0xc0010130\r
	221	rdmsr\r
	222	shl rdx, 32\r
	223	or rax, rdx\r
	224	mov rdi, rax ; RDI now holds the original GHCB GPA\r
	225	\r
	226	mov rdx, 0 ; CPUID function 0\r
	227	mov rax, 0 ; RAX register requested\r
	228	or rax, 4\r
	229	wrmsr\r
	230	rep vmmcall\r
	231	rdmsr\r
	232	cmp edx, 0bh\r
	233	jb NoX2ApicSevEs ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r
	234	\r
	235	mov rdx, 0bh ; CPUID function 0x0b\r
	236	mov rax, 040000000h ; RBX register requested\r
	237	or rax, 4\r
	238	wrmsr\r
	239	rep vmmcall\r
	240	rdmsr\r
	241	test edx, 0ffffh\r
	242	jz NoX2ApicSevEs ; CPUID.0BH:EBX[15:0] is zero\r
	243	\r
	244	mov rdx, 0bh ; CPUID function 0x0b\r
	245	mov rax, 0c0000000h ; RDX register requested\r
	246	or rax, 4\r
	247	wrmsr\r
	248	rep vmmcall\r
	249	rdmsr\r
	250	\r
	251	; Processor is x2APIC capable; 32-bit x2APIC ID is now in EDX\r
	252	jmp RestoreGhcb\r
	253	\r
	254	NoX2ApicSevEs:\r
	255	; Processor is not x2APIC capable, so get 8-bit APIC ID\r
	256	mov rdx, 1 ; CPUID function 1\r
	257	mov rax, 040000000h ; RBX register requested\r
	258	or rax, 4\r
	259	wrmsr\r
	260	rep vmmcall\r
	261	rdmsr\r
	262	shr edx, 24\r
	263	\r
	264	RestoreGhcb:\r
	265	mov rbx, rdx ; Save x2APIC/APIC ID\r
	266	\r
	267	mov rdx, rdi ; RDI holds the saved GHCB GPA\r
	268	shr rdx, 32\r
	269	mov eax, edi\r
	270	wrmsr\r
	271	\r
	272	mov rdx, rbx\r
	273	\r
	274	; x2APIC ID or APIC ID is in EDX\r
275	jmp GetProcessorNumber\r
276	\r
277	DoCpuid:\r
845c5be1 JF	278	mov eax, 0\r
	279	cpuid\r
	280	cmp eax, 0bh\r
1cbd8330 LE	281	jb NoX2Apic ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r
	282	\r
	283	mov eax, 0bh\r
	284	xor ecx, ecx\r
	285	cpuid\r
	286	test ebx, 0ffffh\r
	287	jz NoX2Apic ; CPUID.0BH:EBX[15:0] is zero\r
	288	\r
	289	; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX\r
	290	jmp GetProcessorNumber\r
	291	\r
	292	NoX2Apic:\r
845c5be1 JF	293	; Processor is not x2APIC capable, so get 8-bit APIC ID\r
	294	mov eax, 1\r
	295	cpuid\r
	296	shr ebx, 24\r
	297	mov edx, ebx\r
845c5be1	298	\r
845c5be1 JF	299	GetProcessorNumber:\r
	300	;\r
	301	; Get processor number for this AP\r
	302	; Note that BSP may become an AP due to SwitchBsp()\r
	303	;\r
	304	xor ebx, ebx\r
	305	lea eax, [esi + CpuInfoLocation]\r
edd74ad3	306	mov rdi, [eax]\r
d94e5f67	307	\r
845c5be1	308	GetNextProcNumber:\r
edd74ad3	309	cmp dword [rdi], edx ; APIC ID match?\r
845c5be1	310	jz ProgramStack\r
edd74ad3	311	add rdi, 20\r
845c5be1	312	inc ebx\r
7367cc6c	313	jmp GetNextProcNumber\r
845c5be1 JF	314	\r
845c5be1 JF	315	ProgramStack:\r
edd74ad3	316	mov rsp, qword [rdi + 12]\r
d94e5f67 JF	317	\r
d94e5f67 JF	318	CProcedureInvoke:\r
8396e2dd JF	319	push rbp ; Push BIST data at top of AP stack\r
8396e2dd JF	320	xor rbp, rbp ; Clear ebp for call stack trace\r
d94e5f67 JF	321	push rbp\r
	322	mov rbp, rsp\r
	323	\r
3b2928b4	324	mov rax, qword [esi + InitializeFloatingPointUnitsAddress]\r
d94e5f67 JF	325	sub rsp, 20h\r
	326	call rax ; Call assembly function to initialize FPU per UEFI spec\r
	327	add rsp, 20h\r
	328	\r
37676b9f	329	mov edx, ebx ; edx is ApIndex\r
d94e5f67 JF	330	mov ecx, esi\r
	331	add ecx, LockLocation ; rcx is address of exchange info data buffer\r
	332	\r
	333	mov edi, esi\r
	334	add edi, ApProcedureLocation\r
	335	mov rax, qword [edi]\r
	336	\r
	337	sub rsp, 20h\r
8396e2dd	338	call rax ; Invoke C function\r
d94e5f67	339	add rsp, 20h\r
8396e2dd	340	jmp $ ; Should never reach here\r
d94e5f67 JF	341	\r
	342	RendezvousFunnelProcEnd:\r
	343	\r
7b7508ad TL	344	;-------------------------------------------------------------------------------------\r
	345	;SwitchToRealProc procedure follows.\r
	346	;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC\r
	347	;IS IN MACHINE CODE.\r
	348	; SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart)\r
	349	; rcx - Buffer Start\r
	350	; rdx - Code16 Selector Offset\r
	351	; r8 - Code32 Selector Offset\r
	352	; r9 - Stack Start\r
	353	;-------------------------------------------------------------------------------------\r
	354	global ASM_PFX(SwitchToRealProc)\r
	355	ASM_PFX(SwitchToRealProc):\r
	356	SwitchToRealProcStart:\r
	357	BITS 64\r
	358	cli\r
	359	\r
	360	;\r
	361	; Get RDX reset value before changing stacks since the\r
	362	; new stack won't be able to accomodate a #VC exception.\r
	363	;\r
	364	push rax\r
	365	push rbx\r
	366	push rcx\r
	367	push rdx\r
	368	\r
	369	mov rax, 1\r
	370	cpuid\r
	371	mov rsi, rax ; Save off the reset value for RDX\r
	372	\r
	373	pop rdx\r
	374	pop rcx\r
	375	pop rbx\r
	376	pop rax\r
	377	\r
	378	;\r
	379	; Establish stack below 1MB\r
	380	;\r
	381	mov rsp, r9\r
	382	\r
	383	;\r
	384	; Push ultimate Reset Vector onto the stack\r
	385	;\r
	386	mov rax, rcx\r
	387	shr rax, 4\r
	388	push word 0x0002 ; RFLAGS\r
	389	push ax ; CS\r
	390	push word 0x0000 ; RIP\r
	391	push word 0x0000 ; For alignment, will be discarded\r
	392	\r
	393	;\r
	394	; Get address of "16-bit operand size" label\r
	395	;\r
	396	lea rbx, [PM16Mode]\r
	397	\r
	398	;\r
	399	; Push addresses used to change to compatibility mode\r
	400	;\r
	401	lea rax, [CompatMode]\r
	402	push r8\r
	403	push rax\r
	404	\r
	405	;\r
	406	; Clear R8 - R15, for reset, before going into 32-bit mode\r
	407	;\r
408	xor r8, r8\r
409	xor r9, r9\r
410	xor r10, r10\r
411	xor r11, r11\r
412	xor r12, r12\r
413	xor r13, r13\r
414	xor r14, r14\r
415	xor r15, r15\r
416	\r
417	;\r
418	; Far return into 32-bit mode\r
419	;\r
420	o64 retf\r
421	\r
422	BITS 32\r
423	CompatMode:\r
424	;\r
425	; Set up stack to prepare for exiting protected mode\r
426	;\r
427	push edx ; Code16 CS\r
428	push ebx ; PM16Mode label address\r
429	\r
430	;\r
431	; Disable paging\r
432	;\r
433	mov eax, cr0 ; Read CR0\r
434	btr eax, 31 ; Set PG=0\r
435	mov cr0, eax ; Write CR0\r
436	\r
437	;\r
438	; Disable long mode\r
439	;\r
440	mov ecx, 0c0000080h ; EFER MSR number\r
441	rdmsr ; Read EFER\r
442	btr eax, 8 ; Set LME=0\r
443	wrmsr ; Write EFER\r
444	\r
445	;\r
446	; Disable PAE\r
447	;\r
448	mov eax, cr4 ; Read CR4\r
449	btr eax, 5 ; Set PAE=0\r
450	mov cr4, eax ; Write CR4\r
451	\r
452	mov edx, esi ; Restore RDX reset value\r
453	\r
454	;\r
455	; Switch to 16-bit operand size\r
456	;\r
457	retf\r
458	\r
459	BITS 16\r
460	;\r
461	; At entry to this label\r
462	; - RDX will have its reset value\r
463	; - On the top of the stack\r
464	; - Alignment data (two bytes) to be discarded\r
465	; - IP for Real Mode (two bytes)\r
466	; - CS for Real Mode (two bytes)\r
467	;\r
20da7ca4 TL	468	; This label is also used with AsmRelocateApLoop. During MP finalization,\r
	469	; the code from PM16Mode to SwitchToRealProcEnd is copied to the start of\r
	470	; the WakeupBuffer, allowing a parked AP to be booted by an OS.\r
	471	;\r
7b7508ad TL	472	PM16Mode:\r
	473	mov eax, cr0 ; Read CR0\r
	474	btr eax, 0 ; Set PE=0\r
	475	mov cr0, eax ; Write CR0\r
	476	\r
	477	pop ax ; Discard alignment data\r
	478	\r
	479	;\r
	480	; Clear registers (except RDX and RSP) before going into 16-bit mode\r
	481	;\r
	482	xor eax, eax\r
	483	xor ebx, ebx\r
	484	xor ecx, ecx\r
	485	xor esi, esi\r
	486	xor edi, edi\r
	487	xor ebp, ebp\r
	488	\r
	489	iret\r
	490	\r
	491	SwitchToRealProcEnd:\r
	492	\r
76157021	493	;-------------------------------------------------------------------------------------\r
20da7ca4	494	; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);\r
76157021 JF	495	;-------------------------------------------------------------------------------------\r
	496	global ASM_PFX(AsmRelocateApLoop)\r
	497	ASM_PFX(AsmRelocateApLoop):\r
	498	AsmRelocateApLoopStart:\r
7b7508ad	499	BITS 64\r
20da7ca4 TL	500	cmp qword [rsp + 56], 0 ; SevEsAPJumpTable\r
	501	je NoSevEs\r
	502	\r
	503	;\r
	504	; Perform some SEV-ES related setup before leaving 64-bit mode\r
	505	;\r
	506	push rcx\r
	507	push rdx\r
	508	\r
	509	;\r
	510	; Get the RDX reset value using CPUID\r
	511	;\r
	512	mov rax, 1\r
	513	cpuid\r
	514	mov rsi, rax ; Save off the reset value for RDX\r
	515	\r
	516	;\r
	517	; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call\r
	518	; - Must be done while in 64-bit long mode so that writes to\r
	519	; the GHCB memory will be unencrypted.\r
	520	; - No NAE events can be generated once this is set otherwise\r
	521	; the AP_RESET_HOLD SW_EXITCODE will be overwritten.\r
	522	;\r
	523	mov rcx, 0xc0010130\r
	524	rdmsr ; Retrieve current GHCB address\r
	525	shl rdx, 32\r
	526	or rdx, rax\r
	527	\r
	528	mov rdi, rdx\r
	529	xor rax, rax\r
	530	mov rcx, 0x800\r
	531	shr rcx, 3\r
	532	rep stosq ; Clear the GHCB\r
	533	\r
	534	mov rax, 0x80000004 ; VMGEXIT AP_RESET_HOLD\r
	535	mov [rdx + 0x390], rax\r
fb2a1a36 TL	536	mov rax, 114 ; Set SwExitCode valid bit\r
	537	bts [rdx + 0x3f0], rax\r
	538	inc rax ; Set SwExitInfo1 valid bit\r
	539	bts [rdx + 0x3f0], rax\r
	540	inc rax ; Set SwExitInfo2 valid bit\r
	541	bts [rdx + 0x3f0], rax\r
20da7ca4 TL	542	\r
	543	pop rdx\r
	544	pop rcx\r
	545	\r
	546	NoSevEs:\r
a7bbe9d2	547	cli ; Disable interrupt before switching to 32-bit mode\r
9f91cb01 JF	548	mov rax, [rsp + 40] ; CountTofinish\r
9f91cb01 JF	549	lock dec dword [rax] ; (*CountTofinish)--\r
76157021	550	\r
20da7ca4 TL	551	mov r10, [rsp + 48] ; Pm16CodeSegment\r
	552	mov rax, [rsp + 56] ; SevEsAPJumpTable\r
	553	mov rbx, [rsp + 64] ; WakeupBuffer\r
	554	mov rsp, r9 ; TopOfApStack\r
	555	\r
	556	push rax ; Save SevEsAPJumpTable\r
	557	push rbx ; Save WakeupBuffer\r
	558	push r10 ; Save Pm16CodeSegment\r
	559	push rcx ; Save MwaitSupport\r
	560	push rdx ; Save ApTargetCState\r
	561	\r
	562	lea rax, [PmEntry] ; rax <- The start address of transition code\r
76157021 JF	563	\r
76157021 JF	564	push r8\r
20da7ca4 TL	565	push rax\r
	566	\r
	567	;\r
	568	; Clear R8 - R15, for reset, before going into 32-bit mode\r
	569	;\r
	570	xor r8, r8\r
	571	xor r9, r9\r
	572	xor r10, r10\r
	573	xor r11, r11\r
	574	xor r12, r12\r
	575	xor r13, r13\r
	576	xor r14, r14\r
	577	xor r15, r15\r
	578	\r
	579	;\r
	580	; Far return into 32-bit mode\r
	581	;\r
	582	o64 retf\r
	583	\r
76157021 JF	584	BITS 32\r
	585	PmEntry:\r
	586	mov eax, cr0\r
	587	btr eax, 31 ; Clear CR0.PG\r
	588	mov cr0, eax ; Disable paging and caches\r
	589	\r
76157021 JF	590	mov ecx, 0xc0000080\r
	591	rdmsr\r
	592	and ah, ~ 1 ; Clear LME\r
	593	wrmsr\r
	594	mov eax, cr4\r
	595	and al, ~ (1 << 5) ; Clear PAE\r
	596	mov cr4, eax\r
	597	\r
	598	pop edx\r
	599	add esp, 4\r
	600	pop ecx,\r
	601	add esp, 4\r
20da7ca4 TL	602	\r
20da7ca4 TL	603	MwaitCheck:\r
76157021 JF	604	cmp cl, 1 ; Check mwait-monitor support\r
	605	jnz HltLoop\r
	606	mov ebx, edx ; Save C-State to ebx\r
	607	MwaitLoop:\r
a7bbe9d2	608	cli\r
76157021 JF	609	mov eax, esp ; Set Monitor Address\r
	610	xor ecx, ecx ; ecx = 0\r
	611	xor edx, edx ; edx = 0\r
	612	monitor\r
76157021	613	mov eax, ebx ; Mwait Cx, Target C-State per eax[7:4]\r
f56379f3	614	shl eax, 4\r
76157021 JF	615	mwait\r
76157021 JF	616	jmp MwaitLoop\r
20da7ca4	617	\r
76157021	618	HltLoop:\r
20da7ca4 TL	619	pop edx ; PM16CodeSegment\r
	620	add esp, 4\r
	621	pop ebx ; WakeupBuffer\r
	622	add esp, 4\r
	623	pop eax ; SevEsAPJumpTable\r
	624	add esp, 4\r
	625	cmp eax, 0 ; Check for SEV-ES\r
	626	je DoHlt\r
	627	\r
	628	cli\r
	629	;\r
	630	; SEV-ES is enabled, use VMGEXIT (GHCB information already\r
	631	; set by caller)\r
	632	;\r
	633	BITS 64\r
	634	rep vmmcall\r
	635	BITS 32\r
	636	\r
	637	;\r
	638	; Back from VMGEXIT AP_HLT_LOOP\r
	639	; Push the FLAGS/CS/IP values to use\r
	640	;\r
	641	push word 0x0002 ; EFLAGS\r
	642	xor ecx, ecx\r
	643	mov cx, [eax + 2] ; CS\r
	644	push cx\r
	645	mov cx, [eax] ; IP\r
	646	push cx\r
	647	push word 0x0000 ; For alignment, will be discarded\r
	648	\r
	649	push edx\r
	650	push ebx\r
	651	\r
	652	mov edx, esi ; Restore RDX reset value\r
	653	\r
	654	retf\r
	655	\r
	656	DoHlt:\r
76157021 JF	657	cli\r
76157021 JF	658	hlt\r
20da7ca4 TL	659	jmp DoHlt\r
20da7ca4 TL	660	\r
76157021 JF	661	BITS 64\r
	662	AsmRelocateApLoopEnd:\r
	663	\r
d94e5f67 JF	664	;-------------------------------------------------------------------------------------\r
	665	; AsmGetAddressMap (&AddressMap);\r
	666	;-------------------------------------------------------------------------------------\r
	667	global ASM_PFX(AsmGetAddressMap)\r
	668	ASM_PFX(AsmGetAddressMap):\r
3b2928b4	669	lea rax, [ASM_PFX(RendezvousFunnelProc)]\r
d94e5f67 JF	670	mov qword [rcx], rax\r
	671	mov qword [rcx + 8h], LongModeStart - RendezvousFunnelProcStart\r
	672	mov qword [rcx + 10h], RendezvousFunnelProcEnd - RendezvousFunnelProcStart\r
3b2928b4	673	lea rax, [ASM_PFX(AsmRelocateApLoop)]\r
f7f85d83 JF	674	mov qword [rcx + 18h], rax\r
f7f85d83 JF	675	mov qword [rcx + 20h], AsmRelocateApLoopEnd - AsmRelocateApLoopStart\r
f32bfe6d	676	mov qword [rcx + 28h], Flat32Start - RendezvousFunnelProcStart\r
7b7508ad TL	677	mov qword [rcx + 30h], SwitchToRealProcEnd - SwitchToRealProcStart ; SwitchToRealSize\r
	678	mov qword [rcx + 38h], SwitchToRealProcStart - RendezvousFunnelProcStart ; SwitchToRealOffset\r
	679	mov qword [rcx + 40h], SwitchToRealProcStart - Flat32Start ; SwitchToRealNoNxOffset\r
	680	mov qword [rcx + 48h], PM16Mode - RendezvousFunnelProcStart ; SwitchToRealPM16ModeOffset\r
	681	mov qword [rcx + 50h], SwitchToRealProcEnd - PM16Mode ; SwitchToRealPM16ModeSize\r
d94e5f67 JF	682	ret\r
	683	\r
	684	;-------------------------------------------------------------------------------------\r
	685	;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is\r
8396e2dd	686	;about to become an AP. It switches its stack with the current AP.\r
d94e5f67 JF	687	;AsmExchangeRole (IN CPU_EXCHANGE_INFO MyInfo, IN CPU_EXCHANGE_INFO OthersInfo);\r
	688	;-------------------------------------------------------------------------------------\r
	689	global ASM_PFX(AsmExchangeRole)\r
	690	ASM_PFX(AsmExchangeRole):\r
	691	; DO NOT call other functions in this function, since 2 CPU may use 1 stack\r
	692	; at the same time. If 1 CPU try to call a function, stack will be corrupted.\r
	693	\r
	694	push rax\r
	695	push rbx\r
	696	push rcx\r
	697	push rdx\r
	698	push rsi\r
	699	push rdi\r
	700	push rbp\r
	701	push r8\r
	702	push r9\r
	703	push r10\r
	704	push r11\r
	705	push r12\r
	706	push r13\r
	707	push r14\r
	708	push r15\r
	709	\r
	710	mov rax, cr0\r
	711	push rax\r
	712	\r
	713	mov rax, cr4\r
	714	push rax\r
	715	\r
	716	; rsi contains MyInfo pointer\r
	717	mov rsi, rcx\r
	718	\r
	719	; rdi contains OthersInfo pointer\r
	720	mov rdi, rdx\r
	721	\r
	722	;Store EFLAGS, GDTR and IDTR regiter to stack\r
	723	pushfq\r
	724	sgdt [rsi + 16]\r
	725	sidt [rsi + 26]\r
	726	\r
	727	; Store the its StackPointer\r
	728	mov [rsi + 8], rsp\r
	729	\r
	730	; update its switch state to STORED\r
	731	mov byte [rsi], CPU_SWITCH_STATE_STORED\r
	732	\r
	733	WaitForOtherStored:\r
	734	; wait until the other CPU finish storing its state\r
	735	cmp byte [rdi], CPU_SWITCH_STATE_STORED\r
	736	jz OtherStored\r
	737	pause\r
	738	jmp WaitForOtherStored\r
	739	\r
	740	OtherStored:\r
	741	; Since another CPU already stored its state, load them\r
	742	; load GDTR value\r
	743	lgdt [rdi + 16]\r
	744	\r
	745	; load IDTR value\r
	746	lidt [rdi + 26]\r
	747	\r
	748	; load its future StackPointer\r
	749	mov rsp, [rdi + 8]\r
	750	\r
751	; update the other CPU's switch state to LOADED\r
752	mov byte [rdi], CPU_SWITCH_STATE_LOADED\r
753	\r
754	WaitForOtherLoaded:\r
755	; wait until the other CPU finish loading new state,\r
756	; otherwise the data in stack may corrupt\r
757	cmp byte [rsi], CPU_SWITCH_STATE_LOADED\r
758	jz OtherLoaded\r
759	pause\r
760	jmp WaitForOtherLoaded\r
761	\r
762	OtherLoaded:\r
763	; since the other CPU already get the data it want, leave this procedure\r
764	popfq\r
765	\r
766	pop rax\r
767	mov cr4, rax\r
768	\r
769	pop rax\r
770	mov cr0, rax\r
771	\r
772	pop r15\r
773	pop r14\r
774	pop r13\r
775	pop r12\r
776	pop r11\r
777	pop r10\r
778	pop r9\r
779	pop r8\r
780	pop rbp\r
781	pop rdi\r
782	pop rsi\r
783	pop rdx\r
784	pop rcx\r
785	pop rbx\r
786	pop rax\r
787	\r
788	ret\r