[mirror_edk2.git] / UefiCpuPkg / Library / MpInitLib / X64 / MpFuncs.nasm

;------------------------------------------------------------------------------ ;\r
; Copyright (c) 2015 - 2021, Intel Corporation. All rights reserved.<BR>\r
; SPDX-License-Identifier: BSD-2-Clause-Patent\r
;\r
; Module Name:\r
;\r
;   MpFuncs.nasm\r
;\r
; Abstract:\r
;\r
;   This is the assembly code for MP support\r
;\r
;-------------------------------------------------------------------------------\r
\r
%include "MpEqu.inc"\r
extern ASM_PFX(InitializeFloatingPointUnits)\r
\r
DEFAULT REL\r
\r
SECTION .text\r
\r
;-------------------------------------------------------------------------------------\r
;RendezvousFunnelProc  procedure follows. All APs execute their procedure. This\r
;procedure serializes all the AP processors through an Init sequence. It must be\r
;noted that APs arrive here very raw...ie: real mode, no stack.\r
;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC\r
;IS IN MACHINE CODE.\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(RendezvousFunnelProc)\r
ASM_PFX(RendezvousFunnelProc):\r
RendezvousFunnelProcStart:\r
; At this point CS = 0x(vv00) and ip= 0x0.\r
; Save BIST information to ebp firstly\r
\r
BITS 16\r
    mov        ebp, eax                        ; Save BIST information\r
\r
    mov        ax, cs\r
    mov        ds, ax\r
    mov        es, ax\r
    mov        ss, ax\r
    xor        ax, ax\r
    mov        fs, ax\r
    mov        gs, ax\r
\r
    mov        si,  BufferStartLocation\r
    mov        ebx, [si]\r
\r
    mov        si,  DataSegmentLocation\r
    mov        edx, [si]\r
\r
    ;\r
    ; Get start address of 32-bit code in low memory (<1MB)\r
    ;\r
    mov        edi, ModeTransitionMemoryLocation\r
\r
    mov        si, GdtrLocation\r
o32 lgdt       [cs:si]\r
\r
    mov        si, IdtrLocation\r
o32 lidt       [cs:si]\r
\r
    ;\r
    ; Switch to protected mode\r
    ;\r
    mov        eax, cr0                    ; Get control register 0\r
    or         eax, 000000003h             ; Set PE bit (bit #0) & MP\r
    mov        cr0, eax\r
\r
    ; Switch to 32-bit code (>1MB)\r
o32 jmp far    [cs:di]\r
\r
;\r
; Following code must be copied to memory with type of EfiBootServicesCode.\r
; This is required if NX is enabled for EfiBootServicesCode of memory.\r
;\r
BITS 32\r
Flat32Start:                                   ; protected mode entry point\r
    mov        ds, dx\r
    mov        es, dx\r
    mov        fs, dx\r
    mov        gs, dx\r
    mov        ss, dx\r
\r
    ;\r
    ; Enable execute disable bit\r
    ;\r
    mov        esi, EnableExecuteDisableLocation\r
    cmp        byte [ebx + esi], 0\r
    jz         SkipEnableExecuteDisableBit\r
\r
    mov        ecx, 0c0000080h             ; EFER MSR number\r
    rdmsr                                  ; Read EFER\r
    bts        eax, 11                     ; Enable Execute Disable Bit\r
    wrmsr                                  ; Write EFER\r
\r
SkipEnableExecuteDisableBit:\r
    ;\r
    ; Enable PAE\r
    ;\r
    mov        eax, cr4\r
    bts        eax, 5\r
\r
    mov        esi, Enable5LevelPagingLocation\r
    cmp        byte [ebx + esi], 0\r
    jz         SkipEnable5LevelPaging\r
\r
    ;\r
    ; Enable 5 Level Paging\r
    ;\r
    bts        eax, 12                     ; Set LA57=1.\r
\r
SkipEnable5LevelPaging:\r
\r
    mov        cr4, eax\r
\r
    ;\r
    ; Load page table\r
    ;\r
    mov        esi, Cr3Location             ; Save CR3 in ecx\r
    mov        ecx, [ebx + esi]\r
    mov        cr3, ecx                    ; Load CR3\r
\r
    ;\r
    ; Enable long mode\r
    ;\r
    mov        ecx, 0c0000080h             ; EFER MSR number\r
    rdmsr                                  ; Read EFER\r
    bts        eax, 8                      ; Set LME=1\r
    wrmsr                                  ; Write EFER\r
\r
    ;\r
    ; Enable paging\r
    ;\r
    mov        eax, cr0                    ; Read CR0\r
    bts        eax, 31                     ; Set PG=1\r
    mov        cr0, eax                    ; Write CR0\r
\r
    ;\r
    ; Far jump to 64-bit code\r
    ;\r
    mov        edi, ModeHighMemoryLocation\r
    add        edi, ebx\r
    jmp far    [edi]\r
\r
BITS 64\r
LongModeStart:\r
    mov        esi, ebx\r
    lea        edi, [esi + InitFlagLocation]\r
    cmp        qword [edi], 1       ; ApInitConfig\r
    jnz        GetApicId\r
\r
    ; Increment the number of APs executing here as early as possible\r
    ; This is decremented in C code when AP is finished executing\r
    mov        edi, esi\r
    add        edi, NumApsExecutingLocation\r
    lock inc   dword [edi]\r
\r
    ; AP init\r
    mov        edi, esi\r
    add        edi, LockLocation\r
    mov        rax, NotVacantFlag\r
\r
    mov        edi, esi\r
    add        edi, ApIndexLocation\r
    mov        ebx, 1\r
    lock xadd  dword [edi], ebx                 ; EBX = ApIndex++\r
    inc        ebx                              ; EBX is CpuNumber\r
\r
    ; program stack\r
    mov        edi, esi\r
    add        edi, StackSizeLocation\r
    mov        eax, dword [edi]\r
    mov        ecx, ebx\r
    inc        ecx\r
    mul        ecx                               ; EAX = StackSize * (CpuNumber + 1)\r
    mov        edi, esi\r
    add        edi, StackStartAddressLocation\r
    add        rax, qword [edi]\r
    mov        rsp, rax\r
\r
    lea        edi, [esi + SevEsIsEnabledLocation]\r
    cmp        byte [edi], 1        ; SevEsIsEnabled\r
    jne        CProcedureInvoke\r
\r
    ;\r
    ; program GHCB\r
    ;   Each page after the GHCB is a per-CPU page, so the calculation programs\r
    ;   a GHCB to be every 8KB.\r
    ;\r
    mov        eax, SIZE_4KB\r
    shl        eax, 1                            ; EAX = SIZE_4K * 2\r
    mov        ecx, ebx\r
    mul        ecx                               ; EAX = SIZE_4K * 2 * CpuNumber\r
    mov        edi, esi\r
    add        edi, GhcbBaseLocation\r
    add        rax, qword [edi]\r
    mov        rdx, rax\r
    shr        rdx, 32\r
    mov        rcx, 0xc0010130\r
    wrmsr\r
    jmp        CProcedureInvoke\r
\r
GetApicId:\r
    lea        edi, [esi + SevEsIsEnabledLocation]\r
    cmp        byte [edi], 1        ; SevEsIsEnabled\r
    jne        DoCpuid\r
\r
    ;\r
    ; Since we don't have a stack yet, we can't take a #VC\r
    ; exception. Use the GHCB protocol to perform the CPUID\r
    ; calls.\r
    ;\r
    mov        rcx, 0xc0010130\r
    rdmsr\r
    shl        rdx, 32\r
    or         rax, rdx\r
    mov        rdi, rax             ; RDI now holds the original GHCB GPA\r
\r
    mov        rdx, 0               ; CPUID function 0\r
    mov        rax, 0               ; RAX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
    cmp        edx, 0bh\r
    jb         NoX2ApicSevEs        ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r
\r
    mov        rdx, 0bh             ; CPUID function 0x0b\r
    mov        rax, 040000000h      ; RBX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
    test       edx, 0ffffh\r
    jz         NoX2ApicSevEs        ; CPUID.0BH:EBX[15:0] is zero\r
\r
    mov        rdx, 0bh             ; CPUID function 0x0b\r
    mov        rax, 0c0000000h      ; RDX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
\r
    ; Processor is x2APIC capable; 32-bit x2APIC ID is now in EDX\r
    jmp        RestoreGhcb\r
\r
NoX2ApicSevEs:\r
    ; Processor is not x2APIC capable, so get 8-bit APIC ID\r
    mov        rdx, 1               ; CPUID function 1\r
    mov        rax, 040000000h      ; RBX register requested\r
    or         rax, 4\r
    wrmsr\r
    rep vmmcall\r
    rdmsr\r
    shr        edx, 24\r
\r
RestoreGhcb:\r
    mov        rbx, rdx             ; Save x2APIC/APIC ID\r
\r
    mov        rdx, rdi             ; RDI holds the saved GHCB GPA\r
    shr        rdx, 32\r
    mov        eax, edi\r
    wrmsr\r
\r
    mov        rdx, rbx\r
\r
    ; x2APIC ID or APIC ID is in EDX\r
    jmp        GetProcessorNumber\r
\r
DoCpuid:\r
    mov        eax, 0\r
    cpuid\r
    cmp        eax, 0bh\r
    jb         NoX2Apic             ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r
\r
    mov        eax, 0bh\r
    xor        ecx, ecx\r
    cpuid\r
    test       ebx, 0ffffh\r
    jz         NoX2Apic             ; CPUID.0BH:EBX[15:0] is zero\r
\r
    ; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX\r
    jmp        GetProcessorNumber\r
\r
NoX2Apic:\r
    ; Processor is not x2APIC capable, so get 8-bit APIC ID\r
    mov        eax, 1\r
    cpuid\r
    shr        ebx, 24\r
    mov        edx, ebx\r
\r
GetProcessorNumber:\r
    ;\r
    ; Get processor number for this AP\r
    ; Note that BSP may become an AP due to SwitchBsp()\r
    ;\r
    xor         ebx, ebx\r
    lea         eax, [esi + CpuInfoLocation]\r
    mov         rdi, [eax]\r
\r
GetNextProcNumber:\r
    cmp         dword [rdi], edx                      ; APIC ID match?\r
    jz          ProgramStack\r
    add         rdi, 20\r
    inc         ebx\r
    jmp         GetNextProcNumber\r
\r
ProgramStack:\r
    mov         rsp, qword [rdi + 12]\r
\r
CProcedureInvoke:\r
    push       rbp               ; Push BIST data at top of AP stack\r
    xor        rbp, rbp          ; Clear ebp for call stack trace\r
    push       rbp\r
    mov        rbp, rsp\r
\r
    mov        rax, qword [esi + InitializeFloatingPointUnitsAddress]\r
    sub        rsp, 20h\r
    call       rax               ; Call assembly function to initialize FPU per UEFI spec\r
    add        rsp, 20h\r
\r
    mov        edx, ebx          ; edx is ApIndex\r
    mov        ecx, esi\r
    add        ecx, LockLocation ; rcx is address of exchange info data buffer\r
\r
    mov        edi, esi\r
    add        edi, ApProcedureLocation\r
    mov        rax, qword [edi]\r
\r
    sub        rsp, 20h\r
    call       rax               ; Invoke C function\r
    add        rsp, 20h\r
    jmp        $                 ; Should never reach here\r
\r
RendezvousFunnelProcEnd:\r
\r
;-------------------------------------------------------------------------------------\r
;SwitchToRealProc procedure follows.\r
;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC\r
;IS IN MACHINE CODE.\r
;  SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart)\r
;  rcx - Buffer Start\r
;  rdx - Code16 Selector Offset\r
;  r8  - Code32 Selector Offset\r
;  r9  - Stack Start\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(SwitchToRealProc)\r
ASM_PFX(SwitchToRealProc):\r
SwitchToRealProcStart:\r
BITS 64\r
    cli\r
\r
    ;\r
    ; Get RDX reset value before changing stacks since the\r
    ; new stack won't be able to accomodate a #VC exception.\r
    ;\r
    push       rax\r
    push       rbx\r
    push       rcx\r
    push       rdx\r
\r
    mov        rax, 1\r
    cpuid\r
    mov        rsi, rax                    ; Save off the reset value for RDX\r
\r
    pop        rdx\r
    pop        rcx\r
    pop        rbx\r
    pop        rax\r
\r
    ;\r
    ; Establish stack below 1MB\r
    ;\r
    mov        rsp, r9\r
\r
    ;\r
    ; Push ultimate Reset Vector onto the stack\r
    ;\r
    mov        rax, rcx\r
    shr        rax, 4\r
    push       word 0x0002                 ; RFLAGS\r
    push       ax                          ; CS\r
    push       word 0x0000                 ; RIP\r
    push       word 0x0000                 ; For alignment, will be discarded\r
\r
    ;\r
    ; Get address of "16-bit operand size" label\r
    ;\r
    lea        rbx, [PM16Mode]\r
\r
    ;\r
    ; Push addresses used to change to compatibility mode\r
    ;\r
    lea        rax, [CompatMode]\r
    push       r8\r
    push       rax\r
\r
    ;\r
    ; Clear R8 - R15, for reset, before going into 32-bit mode\r
    ;\r
    xor        r8, r8\r
    xor        r9, r9\r
    xor        r10, r10\r
    xor        r11, r11\r
    xor        r12, r12\r
    xor        r13, r13\r
    xor        r14, r14\r
    xor        r15, r15\r
\r
    ;\r
    ; Far return into 32-bit mode\r
    ;\r
o64 retf\r
\r
BITS 32\r
CompatMode:\r
    ;\r
    ; Set up stack to prepare for exiting protected mode\r
    ;\r
    push       edx                         ; Code16 CS\r
    push       ebx                         ; PM16Mode label address\r
\r
    ;\r
    ; Disable paging\r
    ;\r
    mov        eax, cr0                    ; Read CR0\r
    btr        eax, 31                     ; Set PG=0\r
    mov        cr0, eax                    ; Write CR0\r
\r
    ;\r
    ; Disable long mode\r
    ;\r
    mov        ecx, 0c0000080h             ; EFER MSR number\r
    rdmsr                                  ; Read EFER\r
    btr        eax, 8                      ; Set LME=0\r
    wrmsr                                  ; Write EFER\r
\r
    ;\r
    ; Disable PAE\r
    ;\r
    mov        eax, cr4                    ; Read CR4\r
    btr        eax, 5                      ; Set PAE=0\r
    mov        cr4, eax                    ; Write CR4\r
\r
    mov        edx, esi                    ; Restore RDX reset value\r
\r
    ;\r
    ; Switch to 16-bit operand size\r
    ;\r
    retf\r
\r
BITS 16\r
    ;\r
    ; At entry to this label\r
    ;   - RDX will have its reset value\r
    ;   - On the top of the stack\r
    ;     - Alignment data (two bytes) to be discarded\r
    ;     - IP for Real Mode (two bytes)\r
    ;     - CS for Real Mode (two bytes)\r
    ;\r
    ; This label is also used with AsmRelocateApLoop. During MP finalization,\r
    ; the code from PM16Mode to SwitchToRealProcEnd is copied to the start of\r
    ; the WakeupBuffer, allowing a parked AP to be booted by an OS.\r
    ;\r
PM16Mode:\r
    mov        eax, cr0                    ; Read CR0\r
    btr        eax, 0                      ; Set PE=0\r
    mov        cr0, eax                    ; Write CR0\r
\r
    pop        ax                          ; Discard alignment data\r
\r
    ;\r
    ; Clear registers (except RDX and RSP) before going into 16-bit mode\r
    ;\r
    xor        eax, eax\r
    xor        ebx, ebx\r
    xor        ecx, ecx\r
    xor        esi, esi\r
    xor        edi, edi\r
    xor        ebp, ebp\r
\r
    iret\r
\r
SwitchToRealProcEnd:\r
\r
;-------------------------------------------------------------------------------------\r
;  AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(AsmRelocateApLoop)\r
ASM_PFX(AsmRelocateApLoop):\r
AsmRelocateApLoopStart:\r
BITS 64\r
    cmp        qword [rsp + 56], 0  ; SevEsAPJumpTable\r
    je         NoSevEs\r
\r
    ;\r
    ; Perform some SEV-ES related setup before leaving 64-bit mode\r
    ;\r
    push       rcx\r
    push       rdx\r
\r
    ;\r
    ; Get the RDX reset value using CPUID\r
    ;\r
    mov        rax, 1\r
    cpuid\r
    mov        rsi, rax          ; Save off the reset value for RDX\r
\r
    ;\r
    ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call\r
    ;   - Must be done while in 64-bit long mode so that writes to\r
    ;     the GHCB memory will be unencrypted.\r
    ;   - No NAE events can be generated once this is set otherwise\r
    ;     the AP_RESET_HOLD SW_EXITCODE will be overwritten.\r
    ;\r
    mov        rcx, 0xc0010130\r
    rdmsr                        ; Retrieve current GHCB address\r
    shl        rdx, 32\r
    or         rdx, rax\r
\r
    mov        rdi, rdx\r
    xor        rax, rax\r
    mov        rcx, 0x800\r
    shr        rcx, 3\r
    rep stosq                    ; Clear the GHCB\r
\r
    mov        rax, 0x80000004   ; VMGEXIT AP_RESET_HOLD\r
    mov        [rdx + 0x390], rax\r
    mov        rax, 114          ; Set SwExitCode valid bit\r
    bts        [rdx + 0x3f0], rax\r
    inc        rax               ; Set SwExitInfo1 valid bit\r
    bts        [rdx + 0x3f0], rax\r
    inc        rax               ; Set SwExitInfo2 valid bit\r
    bts        [rdx + 0x3f0], rax\r
\r
    pop        rdx\r
    pop        rcx\r
\r
NoSevEs:\r
    cli                          ; Disable interrupt before switching to 32-bit mode\r
    mov        rax, [rsp + 40]   ; CountTofinish\r
    lock dec   dword [rax]       ; (*CountTofinish)--\r
\r
    mov        r10, [rsp + 48]   ; Pm16CodeSegment\r
    mov        rax, [rsp + 56]   ; SevEsAPJumpTable\r
    mov        rbx, [rsp + 64]   ; WakeupBuffer\r
    mov        rsp, r9           ; TopOfApStack\r
\r
    push       rax               ; Save SevEsAPJumpTable\r
    push       rbx               ; Save WakeupBuffer\r
    push       r10               ; Save Pm16CodeSegment\r
    push       rcx               ; Save MwaitSupport\r
    push       rdx               ; Save ApTargetCState\r
\r
    lea        rax, [PmEntry]    ; rax <- The start address of transition code\r
\r
    push       r8\r
    push       rax\r
\r
    ;\r
    ; Clear R8 - R15, for reset, before going into 32-bit mode\r
    ;\r
    xor        r8, r8\r
    xor        r9, r9\r
    xor        r10, r10\r
    xor        r11, r11\r
    xor        r12, r12\r
    xor        r13, r13\r
    xor        r14, r14\r
    xor        r15, r15\r
\r
    ;\r
    ; Far return into 32-bit mode\r
    ;\r
o64 retf\r
\r
BITS 32\r
PmEntry:\r
    mov        eax, cr0\r
    btr        eax, 31           ; Clear CR0.PG\r
    mov        cr0, eax          ; Disable paging and caches\r
\r
    mov        ecx, 0xc0000080\r
    rdmsr\r
    and        ah, ~ 1           ; Clear LME\r
    wrmsr\r
    mov        eax, cr4\r
    and        al, ~ (1 << 5)    ; Clear PAE\r
    mov        cr4, eax\r
\r
    pop        edx\r
    add        esp, 4\r
    pop        ecx,\r
    add        esp, 4\r
\r
MwaitCheck:\r
    cmp        cl, 1              ; Check mwait-monitor support\r
    jnz        HltLoop\r
    mov        ebx, edx           ; Save C-State to ebx\r
MwaitLoop:\r
    cli\r
    mov        eax, esp           ; Set Monitor Address\r
    xor        ecx, ecx           ; ecx = 0\r
    xor        edx, edx           ; edx = 0\r
    monitor\r
    mov        eax, ebx           ; Mwait Cx, Target C-State per eax[7:4]\r
    shl        eax, 4\r
    mwait\r
    jmp        MwaitLoop\r
\r
HltLoop:\r
    pop        edx                ; PM16CodeSegment\r
    add        esp, 4\r
    pop        ebx                ; WakeupBuffer\r
    add        esp, 4\r
    pop        eax                ; SevEsAPJumpTable\r
    add        esp, 4\r
    cmp        eax, 0             ; Check for SEV-ES\r
    je         DoHlt\r
\r
    cli\r
    ;\r
    ; SEV-ES is enabled, use VMGEXIT (GHCB information already\r
    ; set by caller)\r
    ;\r
BITS 64\r
    rep        vmmcall\r
BITS 32\r
\r
    ;\r
    ; Back from VMGEXIT AP_HLT_LOOP\r
    ;   Push the FLAGS/CS/IP values to use\r
    ;\r
    push       word 0x0002        ; EFLAGS\r
    xor        ecx, ecx\r
    mov        cx, [eax + 2]      ; CS\r
    push       cx\r
    mov        cx, [eax]          ; IP\r
    push       cx\r
    push       word 0x0000        ; For alignment, will be discarded\r
\r
    push       edx\r
    push       ebx\r
\r
    mov        edx, esi           ; Restore RDX reset value\r
\r
    retf\r
\r
DoHlt:\r
    cli\r
    hlt\r
    jmp        DoHlt\r
\r
BITS 64\r
AsmRelocateApLoopEnd:\r
\r
;-------------------------------------------------------------------------------------\r
;  AsmGetAddressMap (&AddressMap);\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(AsmGetAddressMap)\r
ASM_PFX(AsmGetAddressMap):\r
    lea        rax, [ASM_PFX(RendezvousFunnelProc)]\r
    mov        qword [rcx], rax\r
    mov        qword [rcx +  8h], LongModeStart - RendezvousFunnelProcStart\r
    mov        qword [rcx + 10h], RendezvousFunnelProcEnd - RendezvousFunnelProcStart\r
    lea        rax, [ASM_PFX(AsmRelocateApLoop)]\r
    mov        qword [rcx + 18h], rax\r
    mov        qword [rcx + 20h], AsmRelocateApLoopEnd - AsmRelocateApLoopStart\r
    mov        qword [rcx + 28h], Flat32Start - RendezvousFunnelProcStart\r
    mov        qword [rcx + 30h], SwitchToRealProcEnd - SwitchToRealProcStart          ; SwitchToRealSize\r
    mov        qword [rcx + 38h], SwitchToRealProcStart - RendezvousFunnelProcStart    ; SwitchToRealOffset\r
    mov        qword [rcx + 40h], SwitchToRealProcStart - Flat32Start                  ; SwitchToRealNoNxOffset\r
    mov        qword [rcx + 48h], PM16Mode - RendezvousFunnelProcStart                 ; SwitchToRealPM16ModeOffset\r
    mov        qword [rcx + 50h], SwitchToRealProcEnd - PM16Mode                       ; SwitchToRealPM16ModeSize\r
    ret\r
\r
;-------------------------------------------------------------------------------------\r
;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is\r
;about to become an AP. It switches its stack with the current AP.\r
;AsmExchangeRole (IN   CPU_EXCHANGE_INFO    *MyInfo, IN   CPU_EXCHANGE_INFO    *OthersInfo);\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(AsmExchangeRole)\r
ASM_PFX(AsmExchangeRole):\r
    ; DO NOT call other functions in this function, since 2 CPU may use 1 stack\r
    ; at the same time. If 1 CPU try to call a function, stack will be corrupted.\r
\r
    push       rax\r
    push       rbx\r
    push       rcx\r
    push       rdx\r
    push       rsi\r
    push       rdi\r
    push       rbp\r
    push       r8\r
    push       r9\r
    push       r10\r
    push       r11\r
    push       r12\r
    push       r13\r
    push       r14\r
    push       r15\r
\r
    mov        rax, cr0\r
    push       rax\r
\r
    mov        rax, cr4\r
    push       rax\r
\r
    ; rsi contains MyInfo pointer\r
    mov        rsi, rcx\r
\r
    ; rdi contains OthersInfo pointer\r
    mov        rdi, rdx\r
\r
    ;Store EFLAGS, GDTR and IDTR regiter to stack\r
    pushfq\r
    sgdt       [rsi + 16]\r
    sidt       [rsi + 26]\r
\r
    ; Store the its StackPointer\r
    mov        [rsi + 8], rsp\r
\r
    ; update its switch state to STORED\r
    mov        byte [rsi], CPU_SWITCH_STATE_STORED\r
\r
WaitForOtherStored:\r
    ; wait until the other CPU finish storing its state\r
    cmp        byte [rdi], CPU_SWITCH_STATE_STORED\r
    jz         OtherStored\r
    pause\r
    jmp        WaitForOtherStored\r
\r
OtherStored:\r
    ; Since another CPU already stored its state, load them\r
    ; load GDTR value\r
    lgdt       [rdi + 16]\r
\r
    ; load IDTR value\r
    lidt       [rdi + 26]\r
\r
    ; load its future StackPointer\r
    mov        rsp, [rdi + 8]\r
\r
    ; update the other CPU's switch state to LOADED\r
    mov        byte [rdi], CPU_SWITCH_STATE_LOADED\r
\r
WaitForOtherLoaded:\r
    ; wait until the other CPU finish loading new state,\r
    ; otherwise the data in stack may corrupt\r
    cmp        byte [rsi], CPU_SWITCH_STATE_LOADED\r
    jz         OtherLoaded\r
    pause\r
    jmp        WaitForOtherLoaded\r
\r
OtherLoaded:\r
    ; since the other CPU already get the data it want, leave this procedure\r
    popfq\r
\r
    pop        rax\r
    mov        cr4, rax\r
\r
    pop        rax\r
    mov        cr0, rax\r
\r
    pop        r15\r
    pop        r14\r
    pop        r13\r
    pop        r12\r
    pop        r11\r
    pop        r10\r
    pop        r9\r
    pop        r8\r
    pop        rbp\r
    pop        rdi\r
    pop        rsi\r
    pop        rdx\r
    pop        rcx\r
    pop        rbx\r
    pop        rax\r
\r
    ret\r
Commit	Line	Data
d94e5f67	1	;------------------------------------------------------------------------------ ;\r
62f2cf57	2	; Copyright (c) 2015 - 2021, Intel Corporation. All rights reserved.<BR>\r
0acd8697	3	; SPDX-License-Identifier: BSD-2-Clause-Patent\r
d94e5f67 JF	4	;\r
	5	; Module Name:\r
	6	;\r
	7	; MpFuncs.nasm\r
	8	;\r
	9	; Abstract:\r
	10	;\r
	11	; This is the assembly code for MP support\r
	12	;\r
	13	;-------------------------------------------------------------------------------\r
	14	\r
	15	%include "MpEqu.inc"\r
	16	extern ASM_PFX(InitializeFloatingPointUnits)\r
	17	\r
	18	DEFAULT REL\r
	19	\r
	20	SECTION .text\r
	21	\r
	22	;-------------------------------------------------------------------------------------\r
	23	;RendezvousFunnelProc procedure follows. All APs execute their procedure. This\r
	24	;procedure serializes all the AP processors through an Init sequence. It must be\r
	25	;noted that APs arrive here very raw...ie: real mode, no stack.\r
	26	;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC\r
	27	;IS IN MACHINE CODE.\r
	28	;-------------------------------------------------------------------------------------\r
	29	global ASM_PFX(RendezvousFunnelProc)\r
	30	ASM_PFX(RendezvousFunnelProc):\r
	31	RendezvousFunnelProcStart:\r
	32	; At this point CS = 0x(vv00) and ip= 0x0.\r
	33	; Save BIST information to ebp firstly\r
	34	\r
	35	BITS 16\r
	36	mov ebp, eax ; Save BIST information\r
	37	\r
	38	mov ax, cs\r
	39	mov ds, ax\r
	40	mov es, ax\r
	41	mov ss, ax\r
	42	xor ax, ax\r
	43	mov fs, ax\r
	44	mov gs, ax\r
	45	\r
	46	mov si, BufferStartLocation\r
	47	mov ebx, [si]\r
	48	\r
f32bfe6d JW	49	mov si, DataSegmentLocation\r
	50	mov edx, [si]\r
	51	\r
	52	;\r
	53	; Get start address of 32-bit code in low memory (<1MB)\r
	54	;\r
	55	mov edi, ModeTransitionMemoryLocation\r
d94e5f67 JF	56	\r
	57	mov si, GdtrLocation\r
	58	o32 lgdt [cs:si]\r
	59	\r
	60	mov si, IdtrLocation\r
	61	o32 lidt [cs:si]\r
	62	\r
f32bfe6d JW	63	;\r
	64	; Switch to protected mode\r
	65	;\r
	66	mov eax, cr0 ; Get control register 0\r
	67	or eax, 000000003h ; Set PE bit (bit #0) & MP\r
	68	mov cr0, eax\r
	69	\r
	70	; Switch to 32-bit code (>1MB)\r
	71	o32 jmp far [cs:di]\r
	72	\r
	73	;\r
	74	; Following code must be copied to memory with type of EfiBootServicesCode.\r
	75	; This is required if NX is enabled for EfiBootServicesCode of memory.\r
	76	;\r
	77	BITS 32\r
	78	Flat32Start: ; protected mode entry point\r
	79	mov ds, dx\r
	80	mov es, dx\r
	81	mov fs, dx\r
	82	mov gs, dx\r
	83	mov ss, dx\r
5c66d125 JF	84	\r
	85	;\r
	86	; Enable execute disable bit\r
	87	;\r
f32bfe6d JW	88	mov esi, EnableExecuteDisableLocation\r
	89	cmp byte [ebx + esi], 0\r
	90	jz SkipEnableExecuteDisableBit\r
	91	\r
5c66d125 JF	92	mov ecx, 0c0000080h ; EFER MSR number\r
	93	rdmsr ; Read EFER\r
	94	bts eax, 11 ; Enable Execute Disable Bit\r
	95	wrmsr ; Write EFER\r
	96	\r
	97	SkipEnableExecuteDisableBit:\r
f32bfe6d JW	98	;\r
	99	; Enable PAE\r
	100	;\r
d94e5f67 JF	101	mov eax, cr4\r
d94e5f67 JF	102	bts eax, 5\r
09f69a87 RN	103	\r
	104	mov esi, Enable5LevelPagingLocation\r
	105	cmp byte [ebx + esi], 0\r
	106	jz SkipEnable5LevelPaging\r
	107	\r
	108	;\r
	109	; Enable 5 Level Paging\r
	110	;\r
	111	bts eax, 12 ; Set LA57=1.\r
	112	\r
	113	SkipEnable5LevelPaging:\r
	114	\r
d94e5f67 JF	115	mov cr4, eax\r
d94e5f67 JF	116	\r
f32bfe6d JW	117	;\r
	118	; Load page table\r
	119	;\r
	120	mov esi, Cr3Location ; Save CR3 in ecx\r
	121	mov ecx, [ebx + esi]\r
d94e5f67 JF	122	mov cr3, ecx ; Load CR3\r
d94e5f67 JF	123	\r
f32bfe6d JW	124	;\r
	125	; Enable long mode\r
	126	;\r
d94e5f67 JF	127	mov ecx, 0c0000080h ; EFER MSR number\r
	128	rdmsr ; Read EFER\r
	129	bts eax, 8 ; Set LME=1\r
	130	wrmsr ; Write EFER\r
	131	\r
f32bfe6d JW	132	;\r
	133	; Enable paging\r
	134	;\r
d94e5f67 JF	135	mov eax, cr0 ; Read CR0\r
	136	bts eax, 31 ; Set PG=1\r
	137	mov cr0, eax ; Write CR0\r
	138	\r
f32bfe6d JW	139	;\r
	140	; Far jump to 64-bit code\r
	141	;\r
	142	mov edi, ModeHighMemoryLocation\r
	143	add edi, ebx\r
	144	jmp far [edi]\r
	145	\r
d94e5f67 JF	146	BITS 64\r
d94e5f67 JF	147	LongModeStart:\r
845c5be1 JF	148	mov esi, ebx\r
	149	lea edi, [esi + InitFlagLocation]\r
	150	cmp qword [edi], 1 ; ApInitConfig\r
	151	jnz GetApicId\r
	152	\r
0594ec41 ED	153	; Increment the number of APs executing here as early as possible\r
	154	; This is decremented in C code when AP is finished executing\r
	155	mov edi, esi\r
	156	add edi, NumApsExecutingLocation\r
	157	lock inc dword [edi]\r
	158	\r
845c5be1	159	; AP init\r
d94e5f67 JF	160	mov edi, esi\r
	161	add edi, LockLocation\r
	162	mov rax, NotVacantFlag\r
	163	\r
62f2cf57 RN	164	mov edi, esi\r
	165	add edi, ApIndexLocation\r
	166	mov ebx, 1\r
	167	lock xadd dword [edi], ebx ; EBX = ApIndex++\r
	168	inc ebx ; EBX is CpuNumber\r
d94e5f67	169	\r
845c5be1	170	; program stack\r
d94e5f67 JF	171	mov edi, esi\r
d94e5f67 JF	172	add edi, StackSizeLocation\r
845c5be1 JF	173	mov eax, dword [edi]\r
	174	mov ecx, ebx\r
	175	inc ecx\r
	176	mul ecx ; EAX = StackSize * (CpuNumber + 1)\r
d94e5f67 JF	177	mov edi, esi\r
	178	add edi, StackStartAddressLocation\r
	179	add rax, qword [edi]\r
	180	mov rsp, rax\r
7b7508ad TL	181	\r
	182	lea edi, [esi + SevEsIsEnabledLocation]\r
	183	cmp byte [edi], 1 ; SevEsIsEnabled\r
	184	jne CProcedureInvoke\r
	185	\r
	186	;\r
	187	; program GHCB\r
	188	; Each page after the GHCB is a per-CPU page, so the calculation programs\r
	189	; a GHCB to be every 8KB.\r
	190	;\r
	191	mov eax, SIZE_4KB\r
	192	shl eax, 1 ; EAX = SIZE_4K * 2\r
	193	mov ecx, ebx\r
	194	mul ecx ; EAX = SIZE_4K * 2 * CpuNumber\r
	195	mov edi, esi\r
	196	add edi, GhcbBaseLocation\r
	197	add rax, qword [edi]\r
	198	mov rdx, rax\r
	199	shr rdx, 32\r
	200	mov rcx, 0xc0010130\r
	201	wrmsr\r
845c5be1 JF	202	jmp CProcedureInvoke\r
	203	\r
	204	GetApicId:\r
7b7508ad TL	205	lea edi, [esi + SevEsIsEnabledLocation]\r
	206	cmp byte [edi], 1 ; SevEsIsEnabled\r
	207	jne DoCpuid\r
	208	\r
	209	;\r
	210	; Since we don't have a stack yet, we can't take a #VC\r
	211	; exception. Use the GHCB protocol to perform the CPUID\r
	212	; calls.\r
	213	;\r
	214	mov rcx, 0xc0010130\r
	215	rdmsr\r
	216	shl rdx, 32\r
	217	or rax, rdx\r
	218	mov rdi, rax ; RDI now holds the original GHCB GPA\r
	219	\r
	220	mov rdx, 0 ; CPUID function 0\r
	221	mov rax, 0 ; RAX register requested\r
	222	or rax, 4\r
	223	wrmsr\r
	224	rep vmmcall\r
	225	rdmsr\r
	226	cmp edx, 0bh\r
	227	jb NoX2ApicSevEs ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r
	228	\r
	229	mov rdx, 0bh ; CPUID function 0x0b\r
	230	mov rax, 040000000h ; RBX register requested\r
	231	or rax, 4\r
	232	wrmsr\r
	233	rep vmmcall\r
	234	rdmsr\r
	235	test edx, 0ffffh\r
	236	jz NoX2ApicSevEs ; CPUID.0BH:EBX[15:0] is zero\r
	237	\r
	238	mov rdx, 0bh ; CPUID function 0x0b\r
	239	mov rax, 0c0000000h ; RDX register requested\r
	240	or rax, 4\r
	241	wrmsr\r
	242	rep vmmcall\r
	243	rdmsr\r
	244	\r
	245	; Processor is x2APIC capable; 32-bit x2APIC ID is now in EDX\r
	246	jmp RestoreGhcb\r
	247	\r
	248	NoX2ApicSevEs:\r
	249	; Processor is not x2APIC capable, so get 8-bit APIC ID\r
	250	mov rdx, 1 ; CPUID function 1\r
	251	mov rax, 040000000h ; RBX register requested\r
	252	or rax, 4\r
	253	wrmsr\r
	254	rep vmmcall\r
	255	rdmsr\r
	256	shr edx, 24\r
	257	\r
	258	RestoreGhcb:\r
	259	mov rbx, rdx ; Save x2APIC/APIC ID\r
	260	\r
	261	mov rdx, rdi ; RDI holds the saved GHCB GPA\r
	262	shr rdx, 32\r
	263	mov eax, edi\r
	264	wrmsr\r
	265	\r
	266	mov rdx, rbx\r
	267	\r
	268	; x2APIC ID or APIC ID is in EDX\r
269	jmp GetProcessorNumber\r
270	\r
271	DoCpuid:\r
845c5be1 JF	272	mov eax, 0\r
	273	cpuid\r
	274	cmp eax, 0bh\r
1cbd8330 LE	275	jb NoX2Apic ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r
	276	\r
	277	mov eax, 0bh\r
	278	xor ecx, ecx\r
	279	cpuid\r
	280	test ebx, 0ffffh\r
	281	jz NoX2Apic ; CPUID.0BH:EBX[15:0] is zero\r
	282	\r
	283	; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX\r
	284	jmp GetProcessorNumber\r
	285	\r
	286	NoX2Apic:\r
845c5be1 JF	287	; Processor is not x2APIC capable, so get 8-bit APIC ID\r
	288	mov eax, 1\r
	289	cpuid\r
	290	shr ebx, 24\r
	291	mov edx, ebx\r
845c5be1	292	\r
845c5be1 JF	293	GetProcessorNumber:\r
	294	;\r
	295	; Get processor number for this AP\r
	296	; Note that BSP may become an AP due to SwitchBsp()\r
	297	;\r
	298	xor ebx, ebx\r
	299	lea eax, [esi + CpuInfoLocation]\r
edd74ad3	300	mov rdi, [eax]\r
d94e5f67	301	\r
845c5be1	302	GetNextProcNumber:\r
edd74ad3	303	cmp dword [rdi], edx ; APIC ID match?\r
845c5be1	304	jz ProgramStack\r
edd74ad3	305	add rdi, 20\r
845c5be1	306	inc ebx\r
7367cc6c	307	jmp GetNextProcNumber\r
845c5be1 JF	308	\r
845c5be1 JF	309	ProgramStack:\r
edd74ad3	310	mov rsp, qword [rdi + 12]\r
d94e5f67 JF	311	\r
d94e5f67 JF	312	CProcedureInvoke:\r
8396e2dd JF	313	push rbp ; Push BIST data at top of AP stack\r
8396e2dd JF	314	xor rbp, rbp ; Clear ebp for call stack trace\r
d94e5f67 JF	315	push rbp\r
	316	mov rbp, rsp\r
	317	\r
3b2928b4	318	mov rax, qword [esi + InitializeFloatingPointUnitsAddress]\r
d94e5f67 JF	319	sub rsp, 20h\r
	320	call rax ; Call assembly function to initialize FPU per UEFI spec\r
	321	add rsp, 20h\r
	322	\r
37676b9f	323	mov edx, ebx ; edx is ApIndex\r
d94e5f67 JF	324	mov ecx, esi\r
	325	add ecx, LockLocation ; rcx is address of exchange info data buffer\r
	326	\r
	327	mov edi, esi\r
	328	add edi, ApProcedureLocation\r
	329	mov rax, qword [edi]\r
	330	\r
	331	sub rsp, 20h\r
8396e2dd	332	call rax ; Invoke C function\r
d94e5f67	333	add rsp, 20h\r
8396e2dd	334	jmp $ ; Should never reach here\r
d94e5f67 JF	335	\r
	336	RendezvousFunnelProcEnd:\r
	337	\r
7b7508ad TL	338	;-------------------------------------------------------------------------------------\r
	339	;SwitchToRealProc procedure follows.\r
	340	;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC\r
	341	;IS IN MACHINE CODE.\r
	342	; SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart)\r
	343	; rcx - Buffer Start\r
	344	; rdx - Code16 Selector Offset\r
	345	; r8 - Code32 Selector Offset\r
	346	; r9 - Stack Start\r
	347	;-------------------------------------------------------------------------------------\r
	348	global ASM_PFX(SwitchToRealProc)\r
	349	ASM_PFX(SwitchToRealProc):\r
	350	SwitchToRealProcStart:\r
	351	BITS 64\r
	352	cli\r
	353	\r
	354	;\r
	355	; Get RDX reset value before changing stacks since the\r
	356	; new stack won't be able to accomodate a #VC exception.\r
	357	;\r
	358	push rax\r
	359	push rbx\r
	360	push rcx\r
	361	push rdx\r
	362	\r
	363	mov rax, 1\r
	364	cpuid\r
	365	mov rsi, rax ; Save off the reset value for RDX\r
	366	\r
	367	pop rdx\r
	368	pop rcx\r
	369	pop rbx\r
	370	pop rax\r
	371	\r
	372	;\r
	373	; Establish stack below 1MB\r
	374	;\r
	375	mov rsp, r9\r
	376	\r
	377	;\r
	378	; Push ultimate Reset Vector onto the stack\r
	379	;\r
	380	mov rax, rcx\r
	381	shr rax, 4\r
	382	push word 0x0002 ; RFLAGS\r
	383	push ax ; CS\r
	384	push word 0x0000 ; RIP\r
	385	push word 0x0000 ; For alignment, will be discarded\r
	386	\r
	387	;\r
	388	; Get address of "16-bit operand size" label\r
	389	;\r
	390	lea rbx, [PM16Mode]\r
	391	\r
	392	;\r
	393	; Push addresses used to change to compatibility mode\r
	394	;\r
	395	lea rax, [CompatMode]\r
	396	push r8\r
	397	push rax\r
	398	\r
	399	;\r
	400	; Clear R8 - R15, for reset, before going into 32-bit mode\r
	401	;\r
402	xor r8, r8\r
403	xor r9, r9\r
404	xor r10, r10\r
405	xor r11, r11\r
406	xor r12, r12\r
407	xor r13, r13\r
408	xor r14, r14\r
409	xor r15, r15\r
410	\r
411	;\r
412	; Far return into 32-bit mode\r
413	;\r
414	o64 retf\r
415	\r
416	BITS 32\r
417	CompatMode:\r
418	;\r
419	; Set up stack to prepare for exiting protected mode\r
420	;\r
421	push edx ; Code16 CS\r
422	push ebx ; PM16Mode label address\r
423	\r
424	;\r
425	; Disable paging\r
426	;\r
427	mov eax, cr0 ; Read CR0\r
428	btr eax, 31 ; Set PG=0\r
429	mov cr0, eax ; Write CR0\r
430	\r
431	;\r
432	; Disable long mode\r
433	;\r
434	mov ecx, 0c0000080h ; EFER MSR number\r
435	rdmsr ; Read EFER\r
436	btr eax, 8 ; Set LME=0\r
437	wrmsr ; Write EFER\r
438	\r
439	;\r
440	; Disable PAE\r
441	;\r
442	mov eax, cr4 ; Read CR4\r
443	btr eax, 5 ; Set PAE=0\r
444	mov cr4, eax ; Write CR4\r
445	\r
446	mov edx, esi ; Restore RDX reset value\r
447	\r
448	;\r
449	; Switch to 16-bit operand size\r
450	;\r
451	retf\r
452	\r
453	BITS 16\r
454	;\r
455	; At entry to this label\r
456	; - RDX will have its reset value\r
457	; - On the top of the stack\r
458	; - Alignment data (two bytes) to be discarded\r
459	; - IP for Real Mode (two bytes)\r
460	; - CS for Real Mode (two bytes)\r
461	;\r
20da7ca4 TL	462	; This label is also used with AsmRelocateApLoop. During MP finalization,\r
	463	; the code from PM16Mode to SwitchToRealProcEnd is copied to the start of\r
	464	; the WakeupBuffer, allowing a parked AP to be booted by an OS.\r
	465	;\r
7b7508ad TL	466	PM16Mode:\r
	467	mov eax, cr0 ; Read CR0\r
	468	btr eax, 0 ; Set PE=0\r
	469	mov cr0, eax ; Write CR0\r
	470	\r
	471	pop ax ; Discard alignment data\r
	472	\r
	473	;\r
	474	; Clear registers (except RDX and RSP) before going into 16-bit mode\r
	475	;\r
	476	xor eax, eax\r
	477	xor ebx, ebx\r
	478	xor ecx, ecx\r
	479	xor esi, esi\r
	480	xor edi, edi\r
	481	xor ebp, ebp\r
	482	\r
	483	iret\r
	484	\r
	485	SwitchToRealProcEnd:\r
	486	\r
76157021	487	;-------------------------------------------------------------------------------------\r
20da7ca4	488	; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);\r
76157021 JF	489	;-------------------------------------------------------------------------------------\r
	490	global ASM_PFX(AsmRelocateApLoop)\r
	491	ASM_PFX(AsmRelocateApLoop):\r
	492	AsmRelocateApLoopStart:\r
7b7508ad	493	BITS 64\r
20da7ca4 TL	494	cmp qword [rsp + 56], 0 ; SevEsAPJumpTable\r
	495	je NoSevEs\r
	496	\r
	497	;\r
	498	; Perform some SEV-ES related setup before leaving 64-bit mode\r
	499	;\r
	500	push rcx\r
	501	push rdx\r
	502	\r
	503	;\r
	504	; Get the RDX reset value using CPUID\r
	505	;\r
	506	mov rax, 1\r
	507	cpuid\r
	508	mov rsi, rax ; Save off the reset value for RDX\r
	509	\r
	510	;\r
	511	; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call\r
	512	; - Must be done while in 64-bit long mode so that writes to\r
	513	; the GHCB memory will be unencrypted.\r
	514	; - No NAE events can be generated once this is set otherwise\r
	515	; the AP_RESET_HOLD SW_EXITCODE will be overwritten.\r
	516	;\r
	517	mov rcx, 0xc0010130\r
	518	rdmsr ; Retrieve current GHCB address\r
	519	shl rdx, 32\r
	520	or rdx, rax\r
	521	\r
	522	mov rdi, rdx\r
	523	xor rax, rax\r
	524	mov rcx, 0x800\r
	525	shr rcx, 3\r
	526	rep stosq ; Clear the GHCB\r
	527	\r
	528	mov rax, 0x80000004 ; VMGEXIT AP_RESET_HOLD\r
	529	mov [rdx + 0x390], rax\r
fb2a1a36 TL	530	mov rax, 114 ; Set SwExitCode valid bit\r
	531	bts [rdx + 0x3f0], rax\r
	532	inc rax ; Set SwExitInfo1 valid bit\r
	533	bts [rdx + 0x3f0], rax\r
	534	inc rax ; Set SwExitInfo2 valid bit\r
	535	bts [rdx + 0x3f0], rax\r
20da7ca4 TL	536	\r
	537	pop rdx\r
	538	pop rcx\r
	539	\r
	540	NoSevEs:\r
a7bbe9d2	541	cli ; Disable interrupt before switching to 32-bit mode\r
9f91cb01 JF	542	mov rax, [rsp + 40] ; CountTofinish\r
9f91cb01 JF	543	lock dec dword [rax] ; (*CountTofinish)--\r
76157021	544	\r
20da7ca4 TL	545	mov r10, [rsp + 48] ; Pm16CodeSegment\r
	546	mov rax, [rsp + 56] ; SevEsAPJumpTable\r
	547	mov rbx, [rsp + 64] ; WakeupBuffer\r
	548	mov rsp, r9 ; TopOfApStack\r
	549	\r
	550	push rax ; Save SevEsAPJumpTable\r
	551	push rbx ; Save WakeupBuffer\r
	552	push r10 ; Save Pm16CodeSegment\r
	553	push rcx ; Save MwaitSupport\r
	554	push rdx ; Save ApTargetCState\r
	555	\r
	556	lea rax, [PmEntry] ; rax <- The start address of transition code\r
76157021 JF	557	\r
76157021 JF	558	push r8\r
20da7ca4 TL	559	push rax\r
	560	\r
	561	;\r
	562	; Clear R8 - R15, for reset, before going into 32-bit mode\r
	563	;\r
	564	xor r8, r8\r
	565	xor r9, r9\r
	566	xor r10, r10\r
	567	xor r11, r11\r
	568	xor r12, r12\r
	569	xor r13, r13\r
	570	xor r14, r14\r
	571	xor r15, r15\r
	572	\r
	573	;\r
	574	; Far return into 32-bit mode\r
	575	;\r
	576	o64 retf\r
	577	\r
76157021 JF	578	BITS 32\r
	579	PmEntry:\r
	580	mov eax, cr0\r
	581	btr eax, 31 ; Clear CR0.PG\r
	582	mov cr0, eax ; Disable paging and caches\r
	583	\r
76157021 JF	584	mov ecx, 0xc0000080\r
	585	rdmsr\r
	586	and ah, ~ 1 ; Clear LME\r
	587	wrmsr\r
	588	mov eax, cr4\r
	589	and al, ~ (1 << 5) ; Clear PAE\r
	590	mov cr4, eax\r
	591	\r
	592	pop edx\r
	593	add esp, 4\r
	594	pop ecx,\r
	595	add esp, 4\r
20da7ca4 TL	596	\r
20da7ca4 TL	597	MwaitCheck:\r
76157021 JF	598	cmp cl, 1 ; Check mwait-monitor support\r
	599	jnz HltLoop\r
	600	mov ebx, edx ; Save C-State to ebx\r
	601	MwaitLoop:\r
a7bbe9d2	602	cli\r
76157021 JF	603	mov eax, esp ; Set Monitor Address\r
	604	xor ecx, ecx ; ecx = 0\r
	605	xor edx, edx ; edx = 0\r
	606	monitor\r
76157021	607	mov eax, ebx ; Mwait Cx, Target C-State per eax[7:4]\r
f56379f3	608	shl eax, 4\r
76157021 JF	609	mwait\r
76157021 JF	610	jmp MwaitLoop\r
20da7ca4	611	\r
76157021	612	HltLoop:\r
20da7ca4 TL	613	pop edx ; PM16CodeSegment\r
	614	add esp, 4\r
	615	pop ebx ; WakeupBuffer\r
	616	add esp, 4\r
	617	pop eax ; SevEsAPJumpTable\r
	618	add esp, 4\r
	619	cmp eax, 0 ; Check for SEV-ES\r
	620	je DoHlt\r
	621	\r
	622	cli\r
	623	;\r
	624	; SEV-ES is enabled, use VMGEXIT (GHCB information already\r
	625	; set by caller)\r
	626	;\r
	627	BITS 64\r
	628	rep vmmcall\r
	629	BITS 32\r
	630	\r
	631	;\r
	632	; Back from VMGEXIT AP_HLT_LOOP\r
	633	; Push the FLAGS/CS/IP values to use\r
	634	;\r
	635	push word 0x0002 ; EFLAGS\r
	636	xor ecx, ecx\r
	637	mov cx, [eax + 2] ; CS\r
	638	push cx\r
	639	mov cx, [eax] ; IP\r
	640	push cx\r
	641	push word 0x0000 ; For alignment, will be discarded\r
	642	\r
	643	push edx\r
	644	push ebx\r
	645	\r
	646	mov edx, esi ; Restore RDX reset value\r
	647	\r
	648	retf\r
	649	\r
	650	DoHlt:\r
76157021 JF	651	cli\r
76157021 JF	652	hlt\r
20da7ca4 TL	653	jmp DoHlt\r
20da7ca4 TL	654	\r
76157021 JF	655	BITS 64\r
	656	AsmRelocateApLoopEnd:\r
	657	\r
d94e5f67 JF	658	;-------------------------------------------------------------------------------------\r
	659	; AsmGetAddressMap (&AddressMap);\r
	660	;-------------------------------------------------------------------------------------\r
	661	global ASM_PFX(AsmGetAddressMap)\r
	662	ASM_PFX(AsmGetAddressMap):\r
3b2928b4	663	lea rax, [ASM_PFX(RendezvousFunnelProc)]\r
d94e5f67 JF	664	mov qword [rcx], rax\r
	665	mov qword [rcx + 8h], LongModeStart - RendezvousFunnelProcStart\r
	666	mov qword [rcx + 10h], RendezvousFunnelProcEnd - RendezvousFunnelProcStart\r
3b2928b4	667	lea rax, [ASM_PFX(AsmRelocateApLoop)]\r
f7f85d83 JF	668	mov qword [rcx + 18h], rax\r
f7f85d83 JF	669	mov qword [rcx + 20h], AsmRelocateApLoopEnd - AsmRelocateApLoopStart\r
f32bfe6d	670	mov qword [rcx + 28h], Flat32Start - RendezvousFunnelProcStart\r
7b7508ad TL	671	mov qword [rcx + 30h], SwitchToRealProcEnd - SwitchToRealProcStart ; SwitchToRealSize\r
	672	mov qword [rcx + 38h], SwitchToRealProcStart - RendezvousFunnelProcStart ; SwitchToRealOffset\r
	673	mov qword [rcx + 40h], SwitchToRealProcStart - Flat32Start ; SwitchToRealNoNxOffset\r
	674	mov qword [rcx + 48h], PM16Mode - RendezvousFunnelProcStart ; SwitchToRealPM16ModeOffset\r
	675	mov qword [rcx + 50h], SwitchToRealProcEnd - PM16Mode ; SwitchToRealPM16ModeSize\r
d94e5f67 JF	676	ret\r
	677	\r
	678	;-------------------------------------------------------------------------------------\r
	679	;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is\r
8396e2dd	680	;about to become an AP. It switches its stack with the current AP.\r
d94e5f67 JF	681	;AsmExchangeRole (IN CPU_EXCHANGE_INFO MyInfo, IN CPU_EXCHANGE_INFO OthersInfo);\r
	682	;-------------------------------------------------------------------------------------\r
	683	global ASM_PFX(AsmExchangeRole)\r
	684	ASM_PFX(AsmExchangeRole):\r
	685	; DO NOT call other functions in this function, since 2 CPU may use 1 stack\r
	686	; at the same time. If 1 CPU try to call a function, stack will be corrupted.\r
	687	\r
	688	push rax\r
	689	push rbx\r
	690	push rcx\r
	691	push rdx\r
	692	push rsi\r
	693	push rdi\r
	694	push rbp\r
	695	push r8\r
	696	push r9\r
	697	push r10\r
	698	push r11\r
	699	push r12\r
	700	push r13\r
	701	push r14\r
	702	push r15\r
	703	\r
	704	mov rax, cr0\r
	705	push rax\r
	706	\r
	707	mov rax, cr4\r
	708	push rax\r
	709	\r
	710	; rsi contains MyInfo pointer\r
	711	mov rsi, rcx\r
	712	\r
	713	; rdi contains OthersInfo pointer\r
	714	mov rdi, rdx\r
	715	\r
	716	;Store EFLAGS, GDTR and IDTR regiter to stack\r
	717	pushfq\r
	718	sgdt [rsi + 16]\r
	719	sidt [rsi + 26]\r
	720	\r
	721	; Store the its StackPointer\r
	722	mov [rsi + 8], rsp\r
	723	\r
	724	; update its switch state to STORED\r
	725	mov byte [rsi], CPU_SWITCH_STATE_STORED\r
	726	\r
	727	WaitForOtherStored:\r
	728	; wait until the other CPU finish storing its state\r
	729	cmp byte [rdi], CPU_SWITCH_STATE_STORED\r
	730	jz OtherStored\r
	731	pause\r
	732	jmp WaitForOtherStored\r
	733	\r
	734	OtherStored:\r
	735	; Since another CPU already stored its state, load them\r
	736	; load GDTR value\r
	737	lgdt [rdi + 16]\r
	738	\r
	739	; load IDTR value\r
	740	lidt [rdi + 26]\r
	741	\r
	742	; load its future StackPointer\r
	743	mov rsp, [rdi + 8]\r
	744	\r
745	; update the other CPU's switch state to LOADED\r
746	mov byte [rdi], CPU_SWITCH_STATE_LOADED\r
747	\r
748	WaitForOtherLoaded:\r
749	; wait until the other CPU finish loading new state,\r
750	; otherwise the data in stack may corrupt\r
751	cmp byte [rsi], CPU_SWITCH_STATE_LOADED\r
752	jz OtherLoaded\r
753	pause\r
754	jmp WaitForOtherLoaded\r
755	\r
756	OtherLoaded:\r
757	; since the other CPU already get the data it want, leave this procedure\r
758	popfq\r
759	\r
760	pop rax\r
761	mov cr4, rax\r
762	\r
763	pop rax\r
764	mov cr0, rax\r
765	\r
766	pop r15\r
767	pop r14\r
768	pop r13\r
769	pop r12\r
770	pop r11\r
771	pop r10\r
772	pop r9\r
773	pop r8\r
774	pop rbp\r
775	pop rdi\r
776	pop rsi\r
777	pop rdx\r
778	pop rcx\r
779	pop rbx\r
780	pop rax\r
781	\r
782	ret\r