UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm

   1 ;------------------------------------------------------------------------------ ;
   2 ; Copyright (c) 2015 - 2022, Intel Corporation. All rights reserved.<BR>
   3 ; SPDX-License-Identifier: BSD-2-Clause-Patent
   4 ;
   5 ; Module Name:
   6 ;
   7 ;   MpFuncs.nasm
   8 ;
   9 ; Abstract:
  10 ;
  11 ;   This is the assembly code for MP support
  12 ;
  13 ;-------------------------------------------------------------------------------
  14
  15 %include "MpEqu.inc"
  16 extern ASM_PFX(InitializeFloatingPointUnits)
  17
  18 %macro  OneTimeCall 1
  19     jmp     %1
  20 %1 %+ OneTimerCallReturn:
  21 %endmacro
  22
  23 %macro  OneTimeCallRet 1
  24     jmp     %1 %+ OneTimerCallReturn
  25 %endmacro
  26
  27 DEFAULT REL
  28
  29 SECTION .text
  30
  31 ;-------------------------------------------------------------------------------------
  32 ;RendezvousFunnelProc  procedure follows. All APs execute their procedure. This
  33 ;procedure serializes all the AP processors through an Init sequence. It must be
  34 ;noted that APs arrive here very raw...ie: real mode, no stack.
  35 ;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC
  36 ;IS IN MACHINE CODE.
  37 ;-------------------------------------------------------------------------------------
  38 RendezvousFunnelProcStart:
  39 ; At this point CS = 0x(vv00) and ip= 0x0.
  40 ; Save BIST information to ebp firstly
  41
  42 BITS 16
  43     mov        ebp, eax                        ; Save BIST information
  44
  45     mov        ax, cs
  46     mov        ds, ax
  47     mov        es, ax
  48     mov        ss, ax
  49     xor        ax, ax
  50     mov        fs, ax
  51     mov        gs, ax
  52
  53     mov        si,  MP_CPU_EXCHANGE_INFO_FIELD (BufferStart)
  54     mov        ebx, [si]
  55
  56     mov        si,  MP_CPU_EXCHANGE_INFO_FIELD (DataSegment)
  57     mov        edx, [si]
  58
  59     ;
  60     ; Get start address of 32-bit code in low memory (<1MB)
  61     ;
  62     mov        edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeTransitionMemory)
  63
  64     mov        si, MP_CPU_EXCHANGE_INFO_FIELD (GdtrProfile)
  65 o32 lgdt       [cs:si]
  66
  67     ;
  68     ; Switch to protected mode
  69     ;
  70     mov        eax, cr0                    ; Get control register 0
  71     or         eax, 000000003h             ; Set PE bit (bit #0) & MP
  72     mov        cr0, eax
  73
  74     ; Switch to 32-bit code (>1MB)
  75 o32 jmp far    [cs:di]
  76
  77 ;
  78 ; Following code must be copied to memory with type of EfiBootServicesCode.
  79 ; This is required if NX is enabled for EfiBootServicesCode of memory.
  80 ;
  81 BITS 32
  82 Flat32Start:                                   ; protected mode entry point
  83     mov        ds, dx
  84     mov        es, dx
  85     mov        fs, dx
  86     mov        gs, dx
  87     mov        ss, dx
  88
  89     ;
  90     ; Enable execute disable bit
  91     ;
  92     mov        esi, MP_CPU_EXCHANGE_INFO_FIELD (EnableExecuteDisable)
  93     cmp        byte [ebx + esi], 0
  94     jz         SkipEnableExecuteDisableBit
  95
  96     mov        ecx, 0c0000080h             ; EFER MSR number
  97     rdmsr                                  ; Read EFER
  98     bts        eax, 11                     ; Enable Execute Disable Bit
  99     wrmsr                                  ; Write EFER
 100
 101 SkipEnableExecuteDisableBit:
 102     ;
 103     ; Enable PAE
 104     ;
 105     mov        eax, cr4
 106     bts        eax, 5
 107
 108     mov        esi, MP_CPU_EXCHANGE_INFO_FIELD (Enable5LevelPaging)
 109     cmp        byte [ebx + esi], 0
 110     jz         SkipEnable5LevelPaging
 111
 112     ;
 113     ; Enable 5 Level Paging
 114     ;
 115     bts        eax, 12                     ; Set LA57=1.
 116
 117 SkipEnable5LevelPaging:
 118
 119     mov        cr4, eax
 120
 121     ;
 122     ; Load page table
 123     ;
 124     mov        esi, MP_CPU_EXCHANGE_INFO_FIELD (Cr3)             ; Save CR3 in ecx
 125     mov        ecx, [ebx + esi]
 126     mov        cr3, ecx                    ; Load CR3
 127
 128     ;
 129     ; Enable long mode
 130     ;
 131     mov        ecx, 0c0000080h             ; EFER MSR number
 132     rdmsr                                  ; Read EFER
 133     bts        eax, 8                      ; Set LME=1
 134     wrmsr                                  ; Write EFER
 135
 136     ;
 137     ; Enable paging
 138     ;
 139     mov        eax, cr0                    ; Read CR0
 140     bts        eax, 31                     ; Set PG=1
 141     mov        cr0, eax                    ; Write CR0
 142
 143     ;
 144     ; Far jump to 64-bit code
 145     ;
 146     mov        edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeHighMemory)
 147     add        edi, ebx
 148     jmp far    [edi]
 149
 150 BITS 64
 151
 152 LongModeStart:
 153     mov        esi, ebx
 154
 155     ; Set IDT table at the start of 64 bit code
 156     lea        edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (IdtrProfile)]
 157     lidt       [edi]
 158
 159     lea        edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitFlag)]
 160     cmp        qword [edi], 1       ; ApInitConfig
 161     jnz        GetApicId
 162
 163     ; Increment the number of APs executing here as early as possible
 164     ; This is decremented in C code when AP is finished executing
 165     mov        edi, esi
 166     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (NumApsExecuting)
 167     lock inc   dword [edi]
 168
 169     ; AP init
 170     mov        edi, esi
 171     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (ApIndex)
 172     mov        ebx, 1
 173     lock xadd  dword [edi], ebx                 ; EBX = ApIndex++
 174     inc        ebx                              ; EBX is CpuNumber
 175
 176     ; program stack
 177     mov        edi, esi
 178     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (StackSize)
 179     mov        eax, dword [edi]
 180     mov        ecx, ebx
 181     inc        ecx
 182     mul        ecx                               ; EAX = StackSize * (CpuNumber + 1)
 183     mov        edi, esi
 184     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (StackStart)
 185     add        rax, qword [edi]
 186     mov        rsp, rax
 187
 188     ;
 189     ;  Setup the GHCB when AMD SEV-ES active.
 190     ;
 191     OneTimeCall SevEsSetupGhcb
 192     jmp        CProcedureInvoke
 193
 194 GetApicId:
 195     ;
 196     ; Use the GHCB protocol to get the ApicId when SEV-ES is active.
 197     ;
 198     OneTimeCall SevEsGetApicId
 199
 200 DoCpuid:
 201     mov        eax, 0
 202     cpuid
 203     cmp        eax, 0bh
 204     jb         NoX2Apic             ; CPUID level below CPUID_EXTENDED_TOPOLOGY
 205
 206     mov        eax, 0bh
 207     xor        ecx, ecx
 208     cpuid
 209     test       ebx, 0ffffh
 210     jz         NoX2Apic             ; CPUID.0BH:EBX[15:0] is zero
 211
 212     ; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX
 213     jmp        GetProcessorNumber
 214
 215 NoX2Apic:
 216     ; Processor is not x2APIC capable, so get 8-bit APIC ID
 217     mov        eax, 1
 218     cpuid
 219     shr        ebx, 24
 220     mov        edx, ebx
 221
 222 GetProcessorNumber:
 223     ;
 224     ; Get processor number for this AP
 225     ; Note that BSP may become an AP due to SwitchBsp()
 226     ;
 227     xor         ebx, ebx
 228     lea         eax, [esi + MP_CPU_EXCHANGE_INFO_FIELD (CpuInfo)]
 229     mov         rdi, [eax]
 230
 231 GetNextProcNumber:
 232     cmp         dword [rdi + CPU_INFO_IN_HOB.InitialApicId], edx                      ; APIC ID match?
 233     jz          ProgramStack
 234     add         rdi, CPU_INFO_IN_HOB_size
 235     inc         ebx
 236     jmp         GetNextProcNumber
 237
 238 ProgramStack:
 239     mov         rsp, qword [rdi + CPU_INFO_IN_HOB.ApTopOfStack]
 240
 241 CProcedureInvoke:
 242     push       rbp               ; Push BIST data at top of AP stack
 243     xor        rbp, rbp          ; Clear ebp for call stack trace
 244     push       rbp
 245     mov        rbp, rsp
 246
 247     mov        rax, qword [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitializeFloatingPointUnits)]
 248     sub        rsp, 20h
 249     call       rax               ; Call assembly function to initialize FPU per UEFI spec
 250     add        rsp, 20h
 251
 252     mov        edx, ebx          ; edx is ApIndex
 253     mov        ecx, esi
 254     add        ecx, MP_CPU_EXCHANGE_INFO_OFFSET ; rcx is address of exchange info data buffer
 255
 256     mov        edi, esi
 257     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (CFunction)
 258     mov        rax, qword [edi]
 259
 260     sub        rsp, 20h
 261     call       rax               ; Invoke C function
 262     add        rsp, 20h
 263     jmp        $                 ; Should never reach here
 264
 265 ;
 266 ; Required for the AMD SEV helper functions
 267 ;
 268 %include "AmdSev.nasm"
 269
 270 RendezvousFunnelProcEnd:
 271
 272 ;-------------------------------------------------------------------------------------
 273 ;  AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);
 274 ;-------------------------------------------------------------------------------------
 275 AsmRelocateApLoopStart:
 276 BITS 64
 277     cmp        qword [rsp + 56], 0  ; SevEsAPJumpTable
 278     je         NoSevEs
 279
 280     ;
 281     ; Perform some SEV-ES related setup before leaving 64-bit mode
 282     ;
 283     push       rcx
 284     push       rdx
 285
 286     ;
 287     ; Get the RDX reset value using CPUID
 288     ;
 289     mov        rax, 1
 290     cpuid
 291     mov        rsi, rax          ; Save off the reset value for RDX
 292
 293     ;
 294     ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call
 295     ;   - Must be done while in 64-bit long mode so that writes to
 296     ;     the GHCB memory will be unencrypted.
 297     ;   - No NAE events can be generated once this is set otherwise
 298     ;     the AP_RESET_HOLD SW_EXITCODE will be overwritten.
 299     ;
 300     mov        rcx, 0xc0010130
 301     rdmsr                        ; Retrieve current GHCB address
 302     shl        rdx, 32
 303     or         rdx, rax
 304
 305     mov        rdi, rdx
 306     xor        rax, rax
 307     mov        rcx, 0x800
 308     shr        rcx, 3
 309     rep stosq                    ; Clear the GHCB
 310
 311     mov        rax, 0x80000004   ; VMGEXIT AP_RESET_HOLD
 312     mov        [rdx + 0x390], rax
 313     mov        rax, 114          ; Set SwExitCode valid bit
 314     bts        [rdx + 0x3f0], rax
 315     inc        rax               ; Set SwExitInfo1 valid bit
 316     bts        [rdx + 0x3f0], rax
 317     inc        rax               ; Set SwExitInfo2 valid bit
 318     bts        [rdx + 0x3f0], rax
 319
 320     pop        rdx
 321     pop        rcx
 322
 323 NoSevEs:
 324     cli                          ; Disable interrupt before switching to 32-bit mode
 325     mov        rax, [rsp + 40]   ; CountTofinish
 326     lock dec   dword [rax]       ; (*CountTofinish)--
 327
 328     mov        r10, [rsp + 48]   ; Pm16CodeSegment
 329     mov        rax, [rsp + 56]   ; SevEsAPJumpTable
 330     mov        rbx, [rsp + 64]   ; WakeupBuffer
 331     mov        rsp, r9           ; TopOfApStack
 332
 333     push       rax               ; Save SevEsAPJumpTable
 334     push       rbx               ; Save WakeupBuffer
 335     push       r10               ; Save Pm16CodeSegment
 336     push       rcx               ; Save MwaitSupport
 337     push       rdx               ; Save ApTargetCState
 338
 339     lea        rax, [PmEntry]    ; rax <- The start address of transition code
 340
 341     push       r8
 342     push       rax
 343
 344     ;
 345     ; Clear R8 - R15, for reset, before going into 32-bit mode
 346     ;
 347     xor        r8, r8
 348     xor        r9, r9
 349     xor        r10, r10
 350     xor        r11, r11
 351     xor        r12, r12
 352     xor        r13, r13
 353     xor        r14, r14
 354     xor        r15, r15
 355
 356     ;
 357     ; Far return into 32-bit mode
 358     ;
 359     retfq
 360
 361 BITS 32
 362 PmEntry:
 363     mov        eax, cr0
 364     btr        eax, 31           ; Clear CR0.PG
 365     mov        cr0, eax          ; Disable paging and caches
 366
 367     mov        ecx, 0xc0000080
 368     rdmsr
 369     and        ah, ~ 1           ; Clear LME
 370     wrmsr
 371     mov        eax, cr4
 372     and        al, ~ (1 << 5)    ; Clear PAE
 373     mov        cr4, eax
 374
 375     pop        edx
 376     add        esp, 4
 377     pop        ecx,
 378     add        esp, 4
 379
 380 MwaitCheck:
 381     cmp        cl, 1              ; Check mwait-monitor support
 382     jnz        HltLoop
 383     mov        ebx, edx           ; Save C-State to ebx
 384 MwaitLoop:
 385     cli
 386     mov        eax, esp           ; Set Monitor Address
 387     xor        ecx, ecx           ; ecx = 0
 388     xor        edx, edx           ; edx = 0
 389     monitor
 390     mov        eax, ebx           ; Mwait Cx, Target C-State per eax[7:4]
 391     shl        eax, 4
 392     mwait
 393     jmp        MwaitLoop
 394
 395 HltLoop:
 396     pop        edx                ; PM16CodeSegment
 397     add        esp, 4
 398     pop        ebx                ; WakeupBuffer
 399     add        esp, 4
 400     pop        eax                ; SevEsAPJumpTable
 401     add        esp, 4
 402     cmp        eax, 0             ; Check for SEV-ES
 403     je         DoHlt
 404
 405     cli
 406     ;
 407     ; SEV-ES is enabled, use VMGEXIT (GHCB information already
 408     ; set by caller)
 409     ;
 410 BITS 64
 411     rep        vmmcall
 412 BITS 32
 413
 414     ;
 415     ; Back from VMGEXIT AP_HLT_LOOP
 416     ;   Push the FLAGS/CS/IP values to use
 417     ;
 418     push       word 0x0002        ; EFLAGS
 419     xor        ecx, ecx
 420     mov        cx, [eax + 2]      ; CS
 421     push       cx
 422     mov        cx, [eax]          ; IP
 423     push       cx
 424     push       word 0x0000        ; For alignment, will be discarded
 425
 426     push       edx
 427     push       ebx
 428
 429     mov        edx, esi           ; Restore RDX reset value
 430
 431     retf
 432
 433 DoHlt:
 434     cli
 435     hlt
 436     jmp        DoHlt
 437
 438 BITS 64
 439 AsmRelocateApLoopEnd:
 440
 441 ;-------------------------------------------------------------------------------------
 442 ;  AsmGetAddressMap (&AddressMap);
 443 ;-------------------------------------------------------------------------------------
 444 global ASM_PFX(AsmGetAddressMap)
 445 ASM_PFX(AsmGetAddressMap):
 446     lea        rax, [RendezvousFunnelProcStart]
 447     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelAddress], rax
 448     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeEntryOffset], LongModeStart - RendezvousFunnelProcStart
 449     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelSize], RendezvousFunnelProcEnd - RendezvousFunnelProcStart
 450     lea        rax, [AsmRelocateApLoopStart]
 451     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncAddress], rax
 452     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncSize], AsmRelocateApLoopEnd - AsmRelocateApLoopStart
 453     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeTransitionOffset], Flat32Start - RendezvousFunnelProcStart
 454     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealNoNxOffset], SwitchToRealProcStart - Flat32Start
 455     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeOffset], PM16Mode - RendezvousFunnelProcStart
 456     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeSize], SwitchToRealProcEnd - PM16Mode
 457     ret
 458
 459 ;-------------------------------------------------------------------------------------
 460 ;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is
 461 ;about to become an AP. It switches its stack with the current AP.
 462 ;AsmExchangeRole (IN   CPU_EXCHANGE_INFO    *MyInfo, IN   CPU_EXCHANGE_INFO    *OthersInfo);
 463 ;-------------------------------------------------------------------------------------
 464 global ASM_PFX(AsmExchangeRole)
 465 ASM_PFX(AsmExchangeRole):
 466     ; DO NOT call other functions in this function, since 2 CPU may use 1 stack
 467     ; at the same time. If 1 CPU try to call a function, stack will be corrupted.
 468
 469     push       rax
 470     push       rbx
 471     push       rcx
 472     push       rdx
 473     push       rsi
 474     push       rdi
 475     push       rbp
 476     push       r8
 477     push       r9
 478     push       r10
 479     push       r11
 480     push       r12
 481     push       r13
 482     push       r14
 483     push       r15
 484
 485     ; rsi contains MyInfo pointer
 486     mov        rsi, rcx
 487
 488     ; rdi contains OthersInfo pointer
 489     mov        rdi, rdx
 490
 491     pushfq
 492
 493     ; Store the its StackPointer
 494     mov        [rsi + CPU_EXCHANGE_ROLE_INFO.StackPointer], rsp
 495
 496     ; update its switch state to STORED
 497     mov        byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
 498
 499 WaitForOtherStored:
 500     ; wait until the other CPU finish storing its state
 501     cmp        byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
 502     jz         OtherStored
 503     pause
 504     jmp        WaitForOtherStored
 505
 506 OtherStored:
 507
 508     ; load its future StackPointer
 509     mov        rsp, [rdi + CPU_EXCHANGE_ROLE_INFO.StackPointer]
 510
 511     ; update the other CPU's switch state to LOADED
 512     mov        byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
 513
 514 WaitForOtherLoaded:
 515     ; wait until the other CPU finish loading new state,
 516     ; otherwise the data in stack may corrupt
 517     cmp        byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
 518     jz         OtherLoaded
 519     pause
 520     jmp        WaitForOtherLoaded
 521
 522 OtherLoaded:
 523     ; since the other CPU already get the data it want, leave this procedure
 524     popfq
 525
 526     pop        r15
 527     pop        r14
 528     pop        r13
 529     pop        r12
 530     pop        r11
 531     pop        r10
 532     pop        r9
 533     pop        r8
 534     pop        rbp
 535     pop        rdi
 536     pop        rsi
 537     pop        rdx
 538     pop        rcx
 539     pop        rbx
 540     pop        rax
 541
 542     ret