UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm

   1 ;------------------------------------------------------------------------------ ;
   2 ; Copyright (c) 2015 - 2022, Intel Corporation. All rights reserved.<BR>
   3 ; SPDX-License-Identifier: BSD-2-Clause-Patent
   4 ;
   5 ; Module Name:
   6 ;
   7 ;   MpFuncs.nasm
   8 ;
   9 ; Abstract:
  10 ;
  11 ;   This is the assembly code for MP support
  12 ;
  13 ;-------------------------------------------------------------------------------
  14
  15 %include "MpEqu.inc"
  16 extern ASM_PFX(InitializeFloatingPointUnits)
  17
  18 %macro  OneTimeCall 1
  19     jmp     %1
  20 %1 %+ OneTimerCallReturn:
  21 %endmacro
  22
  23 %macro  OneTimeCallRet 1
  24     jmp     %1 %+ OneTimerCallReturn
  25 %endmacro
  26
  27 DEFAULT REL
  28
  29 SECTION .text
  30
  31 ;-------------------------------------------------------------------------------------
  32 ;RendezvousFunnelProc  procedure follows. All APs execute their procedure. This
  33 ;procedure serializes all the AP processors through an Init sequence. It must be
  34 ;noted that APs arrive here very raw...ie: real mode, no stack.
  35 ;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC
  36 ;IS IN MACHINE CODE.
  37 ;-------------------------------------------------------------------------------------
  38 RendezvousFunnelProcStart:
  39 ; At this point CS = 0x(vv00) and ip= 0x0.
  40 ; Save BIST information to ebp firstly
  41
  42 BITS 16
  43     mov        ebp, eax                        ; Save BIST information
  44
  45     mov        ax, cs
  46     mov        ds, ax
  47     mov        es, ax
  48     mov        ss, ax
  49     xor        ax, ax
  50     mov        fs, ax
  51     mov        gs, ax
  52
  53     mov        si,  MP_CPU_EXCHANGE_INFO_FIELD (BufferStart)
  54     mov        ebx, [si]
  55
  56     mov        si,  MP_CPU_EXCHANGE_INFO_FIELD (DataSegment)
  57     mov        edx, [si]
  58
  59     ;
  60     ; Get start address of 32-bit code in low memory (<1MB)
  61     ;
  62     mov        edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeTransitionMemory)
  63
  64     mov        si, MP_CPU_EXCHANGE_INFO_FIELD (GdtrProfile)
  65 o32 lgdt       [cs:si]
  66
  67     ;
  68     ; Switch to protected mode
  69     ;
  70     mov        eax, cr0                    ; Get control register 0
  71     or         eax, 000000003h             ; Set PE bit (bit #0) & MP
  72     mov        cr0, eax
  73
  74     ; Switch to 32-bit code (>1MB)
  75 o32 jmp far    [cs:di]
  76
  77 ;
  78 ; Following code must be copied to memory with type of EfiBootServicesCode.
  79 ; This is required if NX is enabled for EfiBootServicesCode of memory.
  80 ;
  81 BITS 32
  82 Flat32Start:                                   ; protected mode entry point
  83     mov        ds, dx
  84     mov        es, dx
  85     mov        fs, dx
  86     mov        gs, dx
  87     mov        ss, dx
  88
  89     ;
  90     ; Enable execute disable bit
  91     ;
  92     mov        esi, MP_CPU_EXCHANGE_INFO_FIELD (EnableExecuteDisable)
  93     cmp        byte [ebx + esi], 0
  94     jz         SkipEnableExecuteDisableBit
  95
  96     mov        ecx, 0c0000080h             ; EFER MSR number
  97     rdmsr                                  ; Read EFER
  98     bts        eax, 11                     ; Enable Execute Disable Bit
  99     wrmsr                                  ; Write EFER
 100
 101 SkipEnableExecuteDisableBit:
 102     ;
 103     ; Enable PAE
 104     ;
 105     mov        eax, cr4
 106     bts        eax, 5
 107
 108     mov        esi, MP_CPU_EXCHANGE_INFO_FIELD (Enable5LevelPaging)
 109     cmp        byte [ebx + esi], 0
 110     jz         SkipEnable5LevelPaging
 111
 112     ;
 113     ; Enable 5 Level Paging
 114     ;
 115     bts        eax, 12                     ; Set LA57=1.
 116
 117 SkipEnable5LevelPaging:
 118
 119     mov        cr4, eax
 120
 121     ;
 122     ; Load page table
 123     ;
 124     mov        esi, MP_CPU_EXCHANGE_INFO_FIELD (Cr3)             ; Save CR3 in ecx
 125     mov        ecx, [ebx + esi]
 126     mov        cr3, ecx                    ; Load CR3
 127
 128     ;
 129     ; Enable long mode
 130     ;
 131     mov        ecx, 0c0000080h             ; EFER MSR number
 132     rdmsr                                  ; Read EFER
 133     bts        eax, 8                      ; Set LME=1
 134     wrmsr                                  ; Write EFER
 135
 136     ;
 137     ; Enable paging
 138     ;
 139     mov        eax, cr0                    ; Read CR0
 140     bts        eax, 31                     ; Set PG=1
 141     mov        cr0, eax                    ; Write CR0
 142
 143     ;
 144     ; Far jump to 64-bit code
 145     ;
 146     mov        edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeHighMemory)
 147     add        edi, ebx
 148     jmp far    [edi]
 149
 150 BITS 64
 151
 152 LongModeStart:
 153     mov        esi, ebx
 154
 155     ; Set IDT table at the start of 64 bit code
 156     lea        edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (IdtrProfile)]
 157     lidt       [edi]
 158
 159     lea        edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitFlag)]
 160     cmp        qword [edi], 1       ; ApInitConfig
 161     jnz        GetApicId
 162
 163     ; Increment the number of APs executing here as early as possible
 164     ; This is decremented in C code when AP is finished executing
 165     mov        edi, esi
 166     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (NumApsExecuting)
 167     lock inc   dword [edi]
 168
 169     ; AP init
 170     mov        edi, esi
 171     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (ApIndex)
 172     mov        ebx, 1
 173     lock xadd  dword [edi], ebx                 ; EBX = ApIndex++
 174     inc        ebx                              ; EBX is CpuNumber
 175
 176     ; program stack
 177     mov        edi, esi
 178     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (StackSize)
 179     mov        eax, dword [edi]
 180     mov        ecx, ebx
 181     inc        ecx
 182     mul        ecx                               ; EAX = StackSize * (CpuNumber + 1)
 183     mov        edi, esi
 184     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (StackStart)
 185     add        rax, qword [edi]
 186     mov        rsp, rax
 187
 188     ;
 189     ;  Setup the GHCB when AMD SEV-ES active.
 190     ;
 191     OneTimeCall SevEsSetupGhcb
 192     jmp        CProcedureInvoke
 193
 194 GetApicId:
 195     ;
 196     ; Use the GHCB protocol to get the ApicId when SEV-ES is active.
 197     ;
 198     OneTimeCall SevEsGetApicId
 199
 200 DoCpuid:
 201     mov        eax, 0
 202     cpuid
 203     cmp        eax, 0bh
 204     jb         NoX2Apic             ; CPUID level below CPUID_EXTENDED_TOPOLOGY
 205
 206     mov        eax, 0bh
 207     xor        ecx, ecx
 208     cpuid
 209     test       ebx, 0ffffh
 210     jz         NoX2Apic             ; CPUID.0BH:EBX[15:0] is zero
 211
 212     ; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX
 213     jmp        GetProcessorNumber
 214
 215 NoX2Apic:
 216     ; Processor is not x2APIC capable, so get 8-bit APIC ID
 217     mov        eax, 1
 218     cpuid
 219     shr        ebx, 24
 220     mov        edx, ebx
 221
 222 GetProcessorNumber:
 223     ;
 224     ; Get processor number for this AP
 225     ; Note that BSP may become an AP due to SwitchBsp()
 226     ;
 227     xor         ebx, ebx
 228     lea         eax, [esi + MP_CPU_EXCHANGE_INFO_FIELD (CpuInfo)]
 229     mov         rdi, [eax]
 230
 231 GetNextProcNumber:
 232     cmp         dword [rdi + CPU_INFO_IN_HOB.InitialApicId], edx                      ; APIC ID match?
 233     jz          ProgramStack
 234     add         rdi, CPU_INFO_IN_HOB_size
 235     inc         ebx
 236     jmp         GetNextProcNumber
 237
 238 ProgramStack:
 239     mov         rsp, qword [rdi + CPU_INFO_IN_HOB.ApTopOfStack]
 240
 241 CProcedureInvoke:
 242     ;
 243     ; Reserve 8 bytes for CpuMpData.
 244     ; When the AP wakes up again via INIT-SIPI-SIPI, push 0 will cause the existing CpuMpData to be overwritten with 0.
 245     ; CpuMpData is filled in via InitializeApData() during the first time INIT-SIPI-SIPI,
 246     ; while overwirrten may occurs when under ApInHltLoop but InitFlag is not set to ApInitConfig.
 247     ; Therefore reservation is implemented by sub rsp instead of push 0.
 248     ;
 249     sub        rsp, 8
 250     push       rbp               ; Push BIST data at top of AP stack
 251     xor        rbp, rbp          ; Clear ebp for call stack trace
 252     push       rbp
 253     mov        rbp, rsp
 254
 255     push       qword 0          ; Push 8 bytes for alignment
 256     mov        rax, qword [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitializeFloatingPointUnits)]
 257     sub        rsp, 20h
 258     call       rax               ; Call assembly function to initialize FPU per UEFI spec
 259     add        rsp, 20h
 260
 261     mov        edx, ebx          ; edx is ApIndex
 262     mov        ecx, esi
 263     add        ecx, MP_CPU_EXCHANGE_INFO_OFFSET ; rcx is address of exchange info data buffer
 264
 265     mov        edi, esi
 266     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (CFunction)
 267     mov        rax, qword [edi]
 268
 269     sub        rsp, 20h
 270     call       rax               ; Invoke C function
 271     add        rsp, 20h
 272     jmp        $                 ; Should never reach here
 273
 274 ;
 275 ; Required for the AMD SEV helper functions
 276 ;
 277 %include "AmdSev.nasm"
 278
 279 RendezvousFunnelProcEnd:
 280
 281 ;-------------------------------------------------------------------------------------
 282 ;  AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);
 283 ;-------------------------------------------------------------------------------------
 284 AsmRelocateApLoopStart:
 285 BITS 64
 286     cmp        qword [rsp + 56], 0  ; SevEsAPJumpTable
 287     je         NoSevEs
 288
 289     ;
 290     ; Perform some SEV-ES related setup before leaving 64-bit mode
 291     ;
 292     push       rcx
 293     push       rdx
 294
 295     ;
 296     ; Get the RDX reset value using CPUID
 297     ;
 298     mov        rax, 1
 299     cpuid
 300     mov        rsi, rax          ; Save off the reset value for RDX
 301
 302     ;
 303     ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call
 304     ;   - Must be done while in 64-bit long mode so that writes to
 305     ;     the GHCB memory will be unencrypted.
 306     ;   - No NAE events can be generated once this is set otherwise
 307     ;     the AP_RESET_HOLD SW_EXITCODE will be overwritten.
 308     ;
 309     mov        rcx, 0xc0010130
 310     rdmsr                        ; Retrieve current GHCB address
 311     shl        rdx, 32
 312     or         rdx, rax
 313
 314     mov        rdi, rdx
 315     xor        rax, rax
 316     mov        rcx, 0x800
 317     shr        rcx, 3
 318     rep stosq                    ; Clear the GHCB
 319
 320     mov        rax, 0x80000004   ; VMGEXIT AP_RESET_HOLD
 321     mov        [rdx + 0x390], rax
 322     mov        rax, 114          ; Set SwExitCode valid bit
 323     bts        [rdx + 0x3f0], rax
 324     inc        rax               ; Set SwExitInfo1 valid bit
 325     bts        [rdx + 0x3f0], rax
 326     inc        rax               ; Set SwExitInfo2 valid bit
 327     bts        [rdx + 0x3f0], rax
 328
 329     pop        rdx
 330     pop        rcx
 331
 332 NoSevEs:
 333     cli                          ; Disable interrupt before switching to 32-bit mode
 334     mov        rax, [rsp + 40]   ; CountTofinish
 335     lock dec   dword [rax]       ; (*CountTofinish)--
 336
 337     mov        r10, [rsp + 48]   ; Pm16CodeSegment
 338     mov        rax, [rsp + 56]   ; SevEsAPJumpTable
 339     mov        rbx, [rsp + 64]   ; WakeupBuffer
 340     mov        rsp, r9           ; TopOfApStack
 341
 342     push       rax               ; Save SevEsAPJumpTable
 343     push       rbx               ; Save WakeupBuffer
 344     push       r10               ; Save Pm16CodeSegment
 345     push       rcx               ; Save MwaitSupport
 346     push       rdx               ; Save ApTargetCState
 347
 348     lea        rax, [PmEntry]    ; rax <- The start address of transition code
 349
 350     push       r8
 351     push       rax
 352
 353     ;
 354     ; Clear R8 - R15, for reset, before going into 32-bit mode
 355     ;
 356     xor        r8, r8
 357     xor        r9, r9
 358     xor        r10, r10
 359     xor        r11, r11
 360     xor        r12, r12
 361     xor        r13, r13
 362     xor        r14, r14
 363     xor        r15, r15
 364
 365     ;
 366     ; Far return into 32-bit mode
 367     ;
 368     retfq
 369
 370 BITS 32
 371 PmEntry:
 372     mov        eax, cr0
 373     btr        eax, 31           ; Clear CR0.PG
 374     mov        cr0, eax          ; Disable paging and caches
 375
 376     mov        ecx, 0xc0000080
 377     rdmsr
 378     and        ah, ~ 1           ; Clear LME
 379     wrmsr
 380     mov        eax, cr4
 381     and        al, ~ (1 << 5)    ; Clear PAE
 382     mov        cr4, eax
 383
 384     pop        edx
 385     add        esp, 4
 386     pop        ecx,
 387     add        esp, 4
 388
 389 MwaitCheck:
 390     cmp        cl, 1              ; Check mwait-monitor support
 391     jnz        HltLoop
 392     mov        ebx, edx           ; Save C-State to ebx
 393 MwaitLoop:
 394     cli
 395     mov        eax, esp           ; Set Monitor Address
 396     xor        ecx, ecx           ; ecx = 0
 397     xor        edx, edx           ; edx = 0
 398     monitor
 399     mov        eax, ebx           ; Mwait Cx, Target C-State per eax[7:4]
 400     shl        eax, 4
 401     mwait
 402     jmp        MwaitLoop
 403
 404 HltLoop:
 405     pop        edx                ; PM16CodeSegment
 406     add        esp, 4
 407     pop        ebx                ; WakeupBuffer
 408     add        esp, 4
 409     pop        eax                ; SevEsAPJumpTable
 410     add        esp, 4
 411     cmp        eax, 0             ; Check for SEV-ES
 412     je         DoHlt
 413
 414     cli
 415     ;
 416     ; SEV-ES is enabled, use VMGEXIT (GHCB information already
 417     ; set by caller)
 418     ;
 419 BITS 64
 420     rep        vmmcall
 421 BITS 32
 422
 423     ;
 424     ; Back from VMGEXIT AP_HLT_LOOP
 425     ;   Push the FLAGS/CS/IP values to use
 426     ;
 427     push       word 0x0002        ; EFLAGS
 428     xor        ecx, ecx
 429     mov        cx, [eax + 2]      ; CS
 430     push       cx
 431     mov        cx, [eax]          ; IP
 432     push       cx
 433     push       word 0x0000        ; For alignment, will be discarded
 434
 435     push       edx
 436     push       ebx
 437
 438     mov        edx, esi           ; Restore RDX reset value
 439
 440     retf
 441
 442 DoHlt:
 443     cli
 444     hlt
 445     jmp        DoHlt
 446
 447 BITS 64
 448 AsmRelocateApLoopEnd:
 449
 450 ;-------------------------------------------------------------------------------------
 451 ;  AsmGetAddressMap (&AddressMap);
 452 ;-------------------------------------------------------------------------------------
 453 global ASM_PFX(AsmGetAddressMap)
 454 ASM_PFX(AsmGetAddressMap):
 455     lea        rax, [RendezvousFunnelProcStart]
 456     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelAddress], rax
 457     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeEntryOffset], LongModeStart - RendezvousFunnelProcStart
 458     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelSize], RendezvousFunnelProcEnd - RendezvousFunnelProcStart
 459     lea        rax, [AsmRelocateApLoopStart]
 460     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncAddress], rax
 461     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncSize], AsmRelocateApLoopEnd - AsmRelocateApLoopStart
 462     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeTransitionOffset], Flat32Start - RendezvousFunnelProcStart
 463
 464     lea        rax, [AsmRelocateApLoopStartAmd]
 465     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncAddressAmd], rax
 466     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncSizeAmd], AsmRelocateApLoopEndAmd - AsmRelocateApLoopStartAmd
 467
 468     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealNoNxOffset], SwitchToRealProcStart - Flat32Start
 469     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeOffset], PM16Mode - RendezvousFunnelProcStart
 470     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeSize], SwitchToRealProcEnd - PM16Mode
 471     ret
 472
 473 ;-------------------------------------------------------------------------------------
 474 ;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is
 475 ;about to become an AP. It switches its stack with the current AP.
 476 ;AsmExchangeRole (IN   CPU_EXCHANGE_INFO    *MyInfo, IN   CPU_EXCHANGE_INFO    *OthersInfo);
 477 ;-------------------------------------------------------------------------------------
 478 global ASM_PFX(AsmExchangeRole)
 479 ASM_PFX(AsmExchangeRole):
 480     ; DO NOT call other functions in this function, since 2 CPU may use 1 stack
 481     ; at the same time. If 1 CPU try to call a function, stack will be corrupted.
 482
 483     push       rax
 484     push       rbx
 485     push       rcx
 486     push       rdx
 487     push       rsi
 488     push       rdi
 489     push       rbp
 490     push       r8
 491     push       r9
 492     push       r10
 493     push       r11
 494     push       r12
 495     push       r13
 496     push       r14
 497     push       r15
 498
 499     ; rsi contains MyInfo pointer
 500     mov        rsi, rcx
 501
 502     ; rdi contains OthersInfo pointer
 503     mov        rdi, rdx
 504
 505     pushfq
 506
 507     ; Store the its StackPointer
 508     mov        [rsi + CPU_EXCHANGE_ROLE_INFO.StackPointer], rsp
 509
 510     ; update its switch state to STORED
 511     mov        byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
 512
 513 WaitForOtherStored:
 514     ; wait until the other CPU finish storing its state
 515     cmp        byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
 516     jz         OtherStored
 517     pause
 518     jmp        WaitForOtherStored
 519
 520 OtherStored:
 521
 522     ; load its future StackPointer
 523     mov        rsp, [rdi + CPU_EXCHANGE_ROLE_INFO.StackPointer]
 524
 525     ; update the other CPU's switch state to LOADED
 526     mov        byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
 527
 528 WaitForOtherLoaded:
 529     ; wait until the other CPU finish loading new state,
 530     ; otherwise the data in stack may corrupt
 531     cmp        byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
 532     jz         OtherLoaded
 533     pause
 534     jmp        WaitForOtherLoaded
 535
 536 OtherLoaded:
 537     ; since the other CPU already get the data it want, leave this procedure
 538     popfq
 539
 540     pop        r15
 541     pop        r14
 542     pop        r13
 543     pop        r12
 544     pop        r11
 545     pop        r10
 546     pop        r9
 547     pop        r8
 548     pop        rbp
 549     pop        rdi
 550     pop        rsi
 551     pop        rdx
 552     pop        rcx
 553     pop        rbx
 554     pop        rax
 555
 556     ret