UefiCpuPkg/Library/MpInitLib/X64/MpFuncs.nasm

   1 ;------------------------------------------------------------------------------ ;
   2 ; Copyright (c) 2015 - 2023, Intel Corporation. All rights reserved.<BR>
   3 ; SPDX-License-Identifier: BSD-2-Clause-Patent
   4 ;
   5 ; Module Name:
   6 ;
   7 ;   MpFuncs.nasm
   8 ;
   9 ; Abstract:
  10 ;
  11 ;   This is the assembly code for MP support
  12 ;
  13 ;-------------------------------------------------------------------------------
  14
  15 %include "MpEqu.inc"
  16 extern ASM_PFX(InitializeFloatingPointUnits)
  17
  18 %macro  OneTimeCall 1
  19     jmp     %1
  20 %1 %+ OneTimerCallReturn:
  21 %endmacro
  22
  23 %macro  OneTimeCallRet 1
  24     jmp     %1 %+ OneTimerCallReturn
  25 %endmacro
  26
  27 DEFAULT REL
  28
  29 SECTION .text
  30
  31 ;-------------------------------------------------------------------------------------
  32 ;RendezvousFunnelProc  procedure follows. All APs execute their procedure. This
  33 ;procedure serializes all the AP processors through an Init sequence. It must be
  34 ;noted that APs arrive here very raw...ie: real mode, no stack.
  35 ;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC
  36 ;IS IN MACHINE CODE.
  37 ;-------------------------------------------------------------------------------------
  38 RendezvousFunnelProcStart:
  39 ; At this point CS = 0x(vv00) and ip= 0x0.
  40 ; Save BIST information to ebp firstly
  41
  42 BITS 16
  43     mov        ebp, eax                        ; Save BIST information
  44
  45     mov        ax, cs
  46     mov        ds, ax
  47     mov        es, ax
  48     mov        ss, ax
  49     xor        ax, ax
  50     mov        fs, ax
  51     mov        gs, ax
  52
  53     mov        si,  MP_CPU_EXCHANGE_INFO_FIELD (BufferStart)
  54     mov        ebx, [si]
  55
  56     mov        si,  MP_CPU_EXCHANGE_INFO_FIELD (DataSegment)
  57     mov        edx, [si]
  58
  59     ;
  60     ; Get start address of 32-bit code in low memory (<1MB)
  61     ;
  62     mov        edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeTransitionMemory)
  63
  64     mov        si, MP_CPU_EXCHANGE_INFO_FIELD (GdtrProfile)
  65 o32 lgdt       [cs:si]
  66
  67     ;
  68     ; Switch to protected mode
  69     ;
  70     mov        eax, cr0                    ; Get control register 0
  71     or         eax, 000000003h             ; Set PE bit (bit #0) & MP
  72     mov        cr0, eax
  73
  74     ; Switch to 32-bit code (>1MB)
  75 o32 jmp far    [cs:di]
  76
  77 ;
  78 ; Following code must be copied to memory with type of EfiBootServicesCode.
  79 ; This is required if NX is enabled for EfiBootServicesCode of memory.
  80 ;
  81 BITS 32
  82 Flat32Start:                                   ; protected mode entry point
  83     mov        ds, dx
  84     mov        es, dx
  85     mov        fs, dx
  86     mov        gs, dx
  87     mov        ss, dx
  88
  89     ;
  90     ; Enable execute disable bit
  91     ;
  92     mov        esi, MP_CPU_EXCHANGE_INFO_FIELD (EnableExecuteDisable)
  93     cmp        byte [ebx + esi], 0
  94     jz         SkipEnableExecuteDisableBit
  95
  96     mov        ecx, 0c0000080h             ; EFER MSR number
  97     rdmsr                                  ; Read EFER
  98     bts        eax, 11                     ; Enable Execute Disable Bit
  99     wrmsr                                  ; Write EFER
 100
 101 SkipEnableExecuteDisableBit:
 102     ;
 103     ; Enable PAE
 104     ;
 105     mov        eax, cr4
 106     bts        eax, 5
 107
 108     mov        esi, MP_CPU_EXCHANGE_INFO_FIELD (Enable5LevelPaging)
 109     cmp        byte [ebx + esi], 0
 110     jz         SkipEnable5LevelPaging
 111
 112     ;
 113     ; Enable 5 Level Paging
 114     ;
 115     bts        eax, 12                     ; Set LA57=1.
 116
 117 SkipEnable5LevelPaging:
 118
 119     mov        cr4, eax
 120
 121     ;
 122     ; Load page table
 123     ;
 124     mov        esi, MP_CPU_EXCHANGE_INFO_FIELD (Cr3)             ; Save CR3 in ecx
 125     mov        ecx, [ebx + esi]
 126     mov        cr3, ecx                    ; Load CR3
 127
 128     ;
 129     ; Enable long mode
 130     ;
 131     mov        ecx, 0c0000080h             ; EFER MSR number
 132     rdmsr                                  ; Read EFER
 133     bts        eax, 8                      ; Set LME=1
 134     wrmsr                                  ; Write EFER
 135
 136     ;
 137     ; Enable paging
 138     ;
 139     mov        eax, cr0                    ; Read CR0
 140     bts        eax, 31                     ; Set PG=1
 141     mov        cr0, eax                    ; Write CR0
 142
 143     ;
 144     ; Far jump to 64-bit code
 145     ;
 146     mov        edi, MP_CPU_EXCHANGE_INFO_FIELD (ModeHighMemory)
 147     add        edi, ebx
 148     jmp far    [edi]
 149
 150 BITS 64
 151
 152 LongModeStart:
 153     mov        esi, ebx
 154
 155     ; Set IDT table at the start of 64 bit code
 156     lea        edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (IdtrProfile)]
 157     lidt       [edi]
 158
 159     lea        edi, [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitFlag)]
 160     cmp        qword [edi], 1       ; ApInitConfig
 161     jnz        GetApicId
 162
 163     ; Increment the number of APs executing here as early as possible
 164     ; This is decremented in C code when AP is finished executing
 165     mov        edi, esi
 166     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (NumApsExecuting)
 167     lock inc   dword [edi]
 168
 169     ; AP init
 170     mov        edi, esi
 171     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (ApIndex)
 172     mov        ebx, 1
 173     lock xadd  dword [edi], ebx                 ; EBX = ApIndex++
 174     inc        ebx                              ; EBX is CpuNumber
 175
 176     ; program stack
 177     mov        edi, esi
 178     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (StackSize)
 179     mov        eax, dword [edi]
 180     mov        ecx, ebx
 181     inc        ecx
 182     mul        ecx                               ; EAX = StackSize * (CpuNumber + 1)
 183     mov        edi, esi
 184     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (StackStart)
 185     add        rax, qword [edi]
 186     mov        rsp, rax
 187
 188     ;
 189     ;  Setup the GHCB when AMD SEV-ES active.
 190     ;
 191     OneTimeCall SevEsSetupGhcb
 192     jmp        CProcedureInvoke
 193
 194 GetApicId:
 195     ;
 196     ; Use the GHCB protocol to get the ApicId when SEV-ES is active.
 197     ;
 198     OneTimeCall SevEsGetApicId
 199
 200 DoCpuid:
 201     mov        eax, 0
 202     cpuid
 203     cmp        eax, 0bh
 204     jb         NoX2Apic             ; CPUID level below CPUID_EXTENDED_TOPOLOGY
 205
 206     mov        eax, 0bh
 207     xor        ecx, ecx
 208     cpuid
 209     test       ebx, 0ffffh
 210     jz         NoX2Apic             ; CPUID.0BH:EBX[15:0] is zero
 211
 212     ; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX
 213     jmp        GetProcessorNumber
 214
 215 NoX2Apic:
 216     ; Processor is not x2APIC capable, so get 8-bit APIC ID
 217     mov        eax, 1
 218     cpuid
 219     shr        ebx, 24
 220     mov        edx, ebx
 221
 222 GetProcessorNumber:
 223     ;
 224     ; Get processor number for this AP
 225     ; Note that BSP may become an AP due to SwitchBsp()
 226     ;
 227     xor         ebx, ebx
 228     lea         eax, [esi + MP_CPU_EXCHANGE_INFO_FIELD (CpuInfo)]
 229     mov         rdi, [eax]
 230
 231 GetNextProcNumber:
 232     cmp         dword [rdi + CPU_INFO_IN_HOB.InitialApicId], edx                      ; APIC ID match?
 233     jz          ProgramStack
 234     add         rdi, CPU_INFO_IN_HOB_size
 235     inc         ebx
 236     jmp         GetNextProcNumber
 237
 238 ProgramStack:
 239     mov         rsp, qword [rdi + CPU_INFO_IN_HOB.ApTopOfStack]
 240
 241 CProcedureInvoke:
 242     ;
 243     ; Reserve 8 bytes for CpuMpData.
 244     ; When the AP wakes up again via INIT-SIPI-SIPI, push 0 will cause the existing CpuMpData to be overwritten with 0.
 245     ; CpuMpData is filled in via InitializeApData() during the first time INIT-SIPI-SIPI,
 246     ; while overwirrten may occurs when under ApInHltLoop but InitFlag is not set to ApInitConfig.
 247     ; Therefore reservation is implemented by sub rsp instead of push 0.
 248     ;
 249     sub        rsp, 8
 250     push       rbp               ; Push BIST data at top of AP stack
 251     xor        rbp, rbp          ; Clear ebp for call stack trace
 252     push       rbp
 253     mov        rbp, rsp
 254
 255     push       qword 0          ; Push 8 bytes for alignment
 256     mov        rax, qword [esi + MP_CPU_EXCHANGE_INFO_FIELD (InitializeFloatingPointUnits)]
 257     sub        rsp, 20h
 258     call       rax               ; Call assembly function to initialize FPU per UEFI spec
 259     add        rsp, 20h
 260
 261     mov        edx, ebx          ; edx is ApIndex
 262     mov        ecx, esi
 263     add        ecx, MP_CPU_EXCHANGE_INFO_OFFSET ; rcx is address of exchange info data buffer
 264
 265     mov        edi, esi
 266     add        edi, MP_CPU_EXCHANGE_INFO_FIELD (CFunction)
 267     mov        rax, qword [edi]
 268
 269     sub        rsp, 20h
 270     call       rax               ; Invoke C function
 271     add        rsp, 20h
 272     jmp        $                 ; Should never reach here
 273
 274 ;
 275 ; Required for the AMD SEV helper functions
 276 ;
 277 %include "AmdSev.nasm"
 278
 279 RendezvousFunnelProcEnd:
 280
 281 ;-------------------------------------------------------------------------------------
 282 ;  AsmRelocateApLoop (MwaitSupport, ApTargetCState, TopOfApStack, CountTofinish, Cr3);
 283 ;  This function is called during the finalizaiton of Mp initialization before booting
 284 ;  to OS, and aim to put Aps either in Mwait or HLT.
 285 ;-------------------------------------------------------------------------------------
 286 ; +----------------+
 287 ; | Cr3            |  rsp+40
 288 ; +----------------+
 289 ; | CountTofinish  |  r9
 290 ; +----------------+
 291 ; | TopOfApStack   |  r8
 292 ; +----------------+
 293 ; | ApTargetCState |  rdx
 294 ; +----------------+
 295 ; | MwaitSupport   |  rcx
 296 ; +----------------+
 297 ; | the return     |
 298 ; +----------------+ low address
 299
 300 AsmRelocateApLoopGenericStart:
 301     mov        rax, r9           ; CountTofinish
 302     lock dec   dword [rax]       ; (*CountTofinish)--
 303
 304     mov        rax, [rsp + 40]    ; Cr3
 305     ; Do not push on old stack, since old stack is not mapped
 306     ; in the page table pointed by cr3
 307     mov        cr3, rax
 308     mov        rsp, r8            ; TopOfApStack
 309
 310 MwaitCheckGeneric:
 311     cmp        cl, 1              ; Check mwait-monitor support
 312     jnz        HltLoopGeneric
 313     mov        rbx, rdx           ; Save C-State to ebx
 314
 315 MwaitLoopGeneric:
 316     cli
 317     mov        rax, rsp           ; Set Monitor Address
 318     xor        ecx, ecx           ; ecx = 0
 319     xor        edx, edx           ; edx = 0
 320     monitor
 321     mov        rax, rbx           ; Mwait Cx, Target C-State per eax[7:4]
 322     shl        eax, 4
 323     mwait
 324     jmp        MwaitLoopGeneric
 325
 326 HltLoopGeneric:
 327     cli
 328     hlt
 329     jmp        HltLoopGeneric
 330
 331 AsmRelocateApLoopGenericEnd:
 332
 333 ;-------------------------------------------------------------------------------------
 334 ;  AsmGetAddressMap (&AddressMap);
 335 ;-------------------------------------------------------------------------------------
 336 global ASM_PFX(AsmGetAddressMap)
 337 ASM_PFX(AsmGetAddressMap):
 338     lea        rax, [RendezvousFunnelProcStart]
 339     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelAddress], rax
 340     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeEntryOffset], LongModeStart - RendezvousFunnelProcStart
 341     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RendezvousFunnelSize], RendezvousFunnelProcEnd - RendezvousFunnelProcStart
 342 lea        rax, [AsmRelocateApLoopGenericStart]
 343     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncAddressGeneric], rax
 344     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncSizeGeneric], AsmRelocateApLoopGenericEnd - AsmRelocateApLoopGenericStart
 345     lea        rax, [AsmRelocateApLoopAmdSevStart]
 346     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncAddressAmdSev], rax
 347     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.RelocateApLoopFuncSizeAmdSev], AsmRelocateApLoopAmdSevEnd - AsmRelocateApLoopAmdSevStart
 348     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.ModeTransitionOffset], Flat32Start - RendezvousFunnelProcStart
 349     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealNoNxOffset], SwitchToRealProcStart - Flat32Start
 350     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeOffset], PM16Mode - RendezvousFunnelProcStart
 351     mov        qword [rcx + MP_ASSEMBLY_ADDRESS_MAP.SwitchToRealPM16ModeSize], SwitchToRealProcEnd - PM16Mode
 352     ret
 353
 354 ;-------------------------------------------------------------------------------------
 355 ;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is
 356 ;about to become an AP. It switches its stack with the current AP.
 357 ;AsmExchangeRole (IN   CPU_EXCHANGE_INFO    *MyInfo, IN   CPU_EXCHANGE_INFO    *OthersInfo);
 358 ;-------------------------------------------------------------------------------------
 359 global ASM_PFX(AsmExchangeRole)
 360 ASM_PFX(AsmExchangeRole):
 361     ; DO NOT call other functions in this function, since 2 CPU may use 1 stack
 362     ; at the same time. If 1 CPU try to call a function, stack will be corrupted.
 363
 364     push       rax
 365     push       rbx
 366     push       rcx
 367     push       rdx
 368     push       rsi
 369     push       rdi
 370     push       rbp
 371     push       r8
 372     push       r9
 373     push       r10
 374     push       r11
 375     push       r12
 376     push       r13
 377     push       r14
 378     push       r15
 379
 380     ; rsi contains MyInfo pointer
 381     mov        rsi, rcx
 382
 383     ; rdi contains OthersInfo pointer
 384     mov        rdi, rdx
 385
 386     pushfq
 387
 388     ; Store the its StackPointer
 389     mov        [rsi + CPU_EXCHANGE_ROLE_INFO.StackPointer], rsp
 390
 391     ; update its switch state to STORED
 392     mov        byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
 393
 394 WaitForOtherStored:
 395     ; wait until the other CPU finish storing its state
 396     cmp        byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_STORED
 397     jz         OtherStored
 398     pause
 399     jmp        WaitForOtherStored
 400
 401 OtherStored:
 402
 403     ; load its future StackPointer
 404     mov        rsp, [rdi + CPU_EXCHANGE_ROLE_INFO.StackPointer]
 405
 406     ; update the other CPU's switch state to LOADED
 407     mov        byte [rdi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
 408
 409 WaitForOtherLoaded:
 410     ; wait until the other CPU finish loading new state,
 411     ; otherwise the data in stack may corrupt
 412     cmp        byte [rsi + CPU_EXCHANGE_ROLE_INFO.State], CPU_SWITCH_STATE_LOADED
 413     jz         OtherLoaded
 414     pause
 415     jmp        WaitForOtherLoaded
 416
 417 OtherLoaded:
 418     ; since the other CPU already get the data it want, leave this procedure
 419     popfq
 420
 421     pop        r15
 422     pop        r14
 423     pop        r13
 424     pop        r12
 425     pop        r11
 426     pop        r10
 427     pop        r9
 428     pop        r8
 429     pop        rbp
 430     pop        rdi
 431     pop        rsi
 432     pop        rdx
 433     pop        rcx
 434     pop        rbx
 435     pop        rax
 436
 437     ret