[mirror_edk2.git] / UefiCpuPkg / Library / CpuExceptionHandlerLib / X64 / Xcode5ExceptionHandlerAsm.nasm

;------------------------------------------------------------------------------ ;\r
; Copyright (c) 2012 - 2018, Intel Corporation. All rights reserved.<BR>\r
; SPDX-License-Identifier: BSD-2-Clause-Patent\r
;\r
; Module Name:\r
;\r
;   ExceptionHandlerAsm.Asm\r
;\r
; Abstract:\r
;\r
;   x64 CPU Exception Handler\r
;\r
; Notes:\r
;\r
;------------------------------------------------------------------------------\r
%include "Nasm.inc"\r
\r
;\r
; Equivalent NASM structure of IA32_DESCRIPTOR\r
;\r
struc IA32_DESCRIPTOR\r
  .Limit                         CTYPE_UINT16 1\r
  .Base                          CTYPE_UINTN  1\r
endstruc\r
\r
;\r
; Equivalent NASM structure of IA32_IDT_GATE_DESCRIPTOR\r
;\r
struc IA32_IDT_GATE_DESCRIPTOR\r
  .OffsetLow                     CTYPE_UINT16 1\r
  .Selector                      CTYPE_UINT16 1\r
  .Reserved_0                    CTYPE_UINT8 1\r
  .GateType                      CTYPE_UINT8 1\r
  .OffsetHigh                    CTYPE_UINT16 1\r
  .OffsetUpper                   CTYPE_UINT32 1\r
  .Reserved_1                    CTYPE_UINT32 1\r
endstruc\r
\r
;\r
; CommonExceptionHandler()\r
;\r
\r
%define VC_EXCEPTION 29\r
\r
extern ASM_PFX(mErrorCodeFlag)    ; Error code flags for exceptions\r
extern ASM_PFX(mDoFarReturnFlag)  ; Do far return flag\r
extern ASM_PFX(CommonExceptionHandler)\r
\r
SECTION .data\r
\r
DEFAULT REL\r
SECTION .text\r
\r
ALIGN   8\r
\r
AsmIdtVectorBegin:\r
%rep  32\r
    db      0x6a        ; push  #VectorNum\r
    db      ($ - AsmIdtVectorBegin) / ((AsmIdtVectorEnd - AsmIdtVectorBegin) / 32) ; VectorNum\r
    push    rax\r
    mov     rax, strict qword 0 ;    mov     rax, ASM_PFX(CommonInterruptEntry)\r
    jmp     rax\r
%endrep\r
AsmIdtVectorEnd:\r
\r
HookAfterStubHeaderBegin:\r
    db      0x6a        ; push\r
@VectorNum:\r
    db      0          ; 0 will be fixed\r
    push    rax\r
    mov     rax, strict qword 0 ;     mov     rax, HookAfterStubHeaderEnd\r
JmpAbsoluteAddress:\r
    jmp     rax\r
HookAfterStubHeaderEnd:\r
    mov     rax, rsp\r
    and     sp,  0xfff0        ; make sure 16-byte aligned for exception context\r
    sub     rsp, 0x18           ; reserve room for filling exception data later\r
    push    rcx\r
    mov     rcx, [rax + 8]\r
    bt      [ASM_PFX(mErrorCodeFlag)], ecx\r
    jnc     .0\r
    push    qword [rsp]             ; push additional rcx to make stack alignment\r
.0:\r
    xchg    rcx, [rsp]        ; restore rcx, save Exception Number in stack\r
    push    qword [rax]             ; push rax into stack to keep code consistence\r
\r
;---------------------------------------;\r
; CommonInterruptEntry                  ;\r
;---------------------------------------;\r
; The follow algorithm is used for the common interrupt routine.\r
; Entry from each interrupt with a push eax and eax=interrupt number\r
; Stack frame would be as follows as specified in IA32 manuals:\r
;\r
; +---------------------+ <-- 16-byte aligned ensured by processor\r
; +    Old SS           +\r
; +---------------------+\r
; +    Old RSP          +\r
; +---------------------+\r
; +    RFlags           +\r
; +---------------------+\r
; +    CS               +\r
; +---------------------+\r
; +    RIP              +\r
; +---------------------+\r
; +    Error Code       +\r
; +---------------------+\r
; +   Vector Number     +\r
; +---------------------+\r
; +    RBP              +\r
; +---------------------+ <-- RBP, 16-byte aligned\r
; The follow algorithm is used for the common interrupt routine.\r
global ASM_PFX(CommonInterruptEntry)\r
ASM_PFX(CommonInterruptEntry):\r
    cli\r
    pop     rax\r
    ;\r
    ; All interrupt handlers are invoked through interrupt gates, so\r
    ; IF flag automatically cleared at the entry point\r
    ;\r
    xchg    rcx, [rsp]      ; Save rcx into stack and save vector number into rcx\r
    and     rcx, 0xFF\r
    cmp     ecx, 32         ; Intel reserved vector for exceptions?\r
    jae     NoErrorCode\r
    bt      [ASM_PFX(mErrorCodeFlag)], ecx\r
    jc      HasErrorCode\r
\r
NoErrorCode:\r
\r
    ;\r
    ; Push a dummy error code on the stack\r
    ; to maintain coherent stack map\r
    ;\r
    push    qword [rsp]\r
    mov     qword [rsp + 8], 0\r
HasErrorCode:\r
    push    rbp\r
    mov     rbp, rsp\r
    push    0             ; clear EXCEPTION_HANDLER_CONTEXT.OldIdtHandler\r
    push    0             ; clear EXCEPTION_HANDLER_CONTEXT.ExceptionDataFlag\r
\r
    ;\r
    ; Stack:\r
    ; +---------------------+ <-- 16-byte aligned ensured by processor\r
    ; +    Old SS           +\r
    ; +---------------------+\r
    ; +    Old RSP          +\r
    ; +---------------------+\r
    ; +    RFlags           +\r
    ; +---------------------+\r
    ; +    CS               +\r
    ; +---------------------+\r
    ; +    RIP              +\r
    ; +---------------------+\r
    ; +    Error Code       +\r
    ; +---------------------+\r
    ; + RCX / Vector Number +\r
    ; +---------------------+\r
    ; +    RBP              +\r
    ; +---------------------+ <-- RBP, 16-byte aligned\r
    ;\r
\r
    ;\r
    ; Since here the stack pointer is 16-byte aligned, so\r
    ; EFI_FX_SAVE_STATE_X64 of EFI_SYSTEM_CONTEXT_x64\r
    ; is 16-byte aligned\r
    ;\r
\r
;; UINT64  Rdi, Rsi, Rbp, Rsp, Rbx, Rdx, Rcx, Rax;\r
;; UINT64  R8, R9, R10, R11, R12, R13, R14, R15;\r
    push r15\r
    push r14\r
    push r13\r
    push r12\r
    push r11\r
    push r10\r
    push r9\r
    push r8\r
    push rax\r
    push qword [rbp + 8]   ; RCX\r
    push rdx\r
    push rbx\r
    push qword [rbp + 48]  ; RSP\r
    push qword [rbp]       ; RBP\r
    push rsi\r
    push rdi\r
\r
;; UINT64  Gs, Fs, Es, Ds, Cs, Ss;  insure high 16 bits of each is zero\r
    movzx   rax, word [rbp + 56]\r
    push    rax                      ; for ss\r
    movzx   rax, word [rbp + 32]\r
    push    rax                      ; for cs\r
    mov     rax, ds\r
    push    rax\r
    mov     rax, es\r
    push    rax\r
    mov     rax, fs\r
    push    rax\r
    mov     rax, gs\r
    push    rax\r
\r
    mov     [rbp + 8], rcx               ; save vector number\r
\r
;; UINT64  Rip;\r
    push    qword [rbp + 24]\r
\r
;; UINT64  Gdtr[2], Idtr[2];\r
    xor     rax, rax\r
    push    rax\r
    push    rax\r
    sidt    [rsp]\r
    mov     bx, word [rsp]\r
    mov     rax, qword [rsp + 2]\r
    mov     qword [rsp], rax\r
    mov     word [rsp + 8], bx\r
\r
    xor     rax, rax\r
    push    rax\r
    push    rax\r
    sgdt    [rsp]\r
    mov     bx, word [rsp]\r
    mov     rax, qword [rsp + 2]\r
    mov     qword [rsp], rax\r
    mov     word [rsp + 8], bx\r
\r
;; UINT64  Ldtr, Tr;\r
    xor     rax, rax\r
    str     ax\r
    push    rax\r
    sldt    ax\r
    push    rax\r
\r
;; UINT64  RFlags;\r
    push    qword [rbp + 40]\r
\r
;; UINT64  Cr0, Cr1, Cr2, Cr3, Cr4, Cr8;\r
    mov     rax, cr8\r
    push    rax\r
    mov     rax, cr4\r
    or      rax, 0x208\r
    mov     cr4, rax\r
    push    rax\r
    mov     rax, cr3\r
    push    rax\r
    mov     rax, cr2\r
    push    rax\r
    xor     rax, rax\r
    push    rax\r
    mov     rax, cr0\r
    push    rax\r
\r
;; UINT64  Dr0, Dr1, Dr2, Dr3, Dr6, Dr7;\r
    cmp     qword [rbp + 8], VC_EXCEPTION\r
    je      VcDebugRegs          ; For SEV-ES (#VC) Debug registers ignored\r
\r
    mov     rax, dr7\r
    push    rax\r
    mov     rax, dr6\r
    push    rax\r
    mov     rax, dr3\r
    push    rax\r
    mov     rax, dr2\r
    push    rax\r
    mov     rax, dr1\r
    push    rax\r
    mov     rax, dr0\r
    push    rax\r
    jmp     DrFinish\r
\r
VcDebugRegs:\r
;; UINT64  Dr0, Dr1, Dr2, Dr3, Dr6, Dr7 are skipped for #VC to avoid exception recursion\r
    xor     rax, rax\r
    push    rax\r
    push    rax\r
    push    rax\r
    push    rax\r
    push    rax\r
    push    rax\r
\r
DrFinish:\r
;; FX_SAVE_STATE_X64 FxSaveState;\r
    sub rsp, 512\r
    mov rdi, rsp\r
    db 0xf, 0xae, 0x7 ;fxsave [rdi]\r
\r
;; UEFI calling convention for x64 requires that Direction flag in EFLAGs is clear\r
    cld\r
\r
;; UINT32  ExceptionData;\r
    push    qword [rbp + 16]\r
\r
;; Prepare parameter and call\r
    mov     rcx, [rbp + 8]\r
    mov     rdx, rsp\r
    ;\r
    ; Per X64 calling convention, allocate maximum parameter stack space\r
    ; and make sure RSP is 16-byte aligned\r
    ;\r
    sub     rsp, 4 * 8 + 8\r
    call    ASM_PFX(CommonExceptionHandler)\r
    add     rsp, 4 * 8 + 8\r
\r
    ; The follow algorithm is used for clear shadow stack token busy bit.\r
    ; The comment is based on the sample shadow stack.\r
    ; Shadow stack is 32 bytes aligned.\r
    ; The sample shadow stack layout :\r
    ; Address | Context\r
    ;         +-------------------------+\r
    ;  0xFB8  |   FREE                  | It is 0xFC0|0x02|(LMA & CS.L), after SAVEPREVSSP.\r
    ;         +-------------------------+\r
    ;  0xFC0  |  Prev SSP               |\r
    ;         +-------------------------+\r
    ;  0xFC8  |   RIP                   |\r
    ;         +-------------------------+\r
    ;  0xFD0  |   CS                    |\r
    ;         +-------------------------+\r
    ;  0xFD8  |  0xFD8 | BUSY           | BUSY flag cleared after CLRSSBSY\r
    ;         +-------------------------+\r
    ;  0xFE0  | 0xFC0|0x02|(LMA & CS.L) |\r
    ;         +-------------------------+\r
    ; Instructions for Intel Control Flow Enforcement Technology (CET) are supported since NASM version 2.15.01.\r
    cmp     qword [ASM_PFX(mDoFarReturnFlag)], 0\r
    jz      CetDone\r
    mov     rax, cr4\r
    and     rax, 0x800000       ; Check if CET is enabled\r
    jz      CetDone\r
    sub     rsp, 0x10\r
    sidt    [rsp]\r
    mov     rcx, qword [rsp + IA32_DESCRIPTOR.Base]; Get IDT base address\r
    add     rsp, 0x10\r
    mov     rax, qword [rbp + 8]; Get exception number\r
    sal     rax, 0x04           ; Get IDT offset\r
    add     rax, rcx            ; Get IDT gate descriptor address\r
    mov     al, byte [rax + IA32_IDT_GATE_DESCRIPTOR.Reserved_0]\r
    and     rax, 0x01           ; Check IST field\r
    jz      CetDone\r
                                ; SSP should be 0xFC0 at this point\r
    mov     rax, 0x04           ; advance past cs:lip:prevssp;supervisor shadow stack token\r
    INCSSP_RAX                  ; After this SSP should be 0xFE0\r
    SAVEPREVSSP                 ; now the shadow stack restore token will be created at 0xFB8\r
    READSSP_RAX                 ; Read new SSP, SSP should be 0xFE8\r
    sub     rax, 0x10\r
    CLRSSBSY_RAX                ; Clear token at 0xFD8, SSP should be 0 after this\r
    sub     rax, 0x20\r
    RSTORSSP_RAX                ; Restore to token at 0xFB8, new SSP will be 0xFB8\r
    mov     rax, 0x01           ; Pop off the new save token created\r
    INCSSP_RAX                  ; SSP should be 0xFC0 now\r
CetDone:\r
\r
    cli\r
;; UINT64  ExceptionData;\r
    add     rsp, 8\r
\r
;; FX_SAVE_STATE_X64 FxSaveState;\r
\r
    mov rsi, rsp\r
    db 0xf, 0xae, 0xE ; fxrstor [rsi]\r
    add rsp, 512\r
\r
;; UINT64  Dr0, Dr1, Dr2, Dr3, Dr6, Dr7;\r
;; Skip restoration of DRx registers to support in-circuit emualators\r
;; or debuggers set breakpoint in interrupt/exception context\r
    add     rsp, 8 * 6\r
\r
;; UINT64  Cr0, Cr1, Cr2, Cr3, Cr4, Cr8;\r
    pop     rax\r
    mov     cr0, rax\r
    add     rsp, 8   ; not for Cr1\r
    pop     rax\r
    mov     cr2, rax\r
    pop     rax\r
    mov     cr3, rax\r
    pop     rax\r
    mov     cr4, rax\r
    pop     rax\r
    mov     cr8, rax\r
\r
;; UINT64  RFlags;\r
    pop     qword [rbp + 40]\r
\r
;; UINT64  Ldtr, Tr;\r
;; UINT64  Gdtr[2], Idtr[2];\r
;; Best not let anyone mess with these particular registers...\r
    add     rsp, 48\r
\r
;; UINT64  Rip;\r
    pop     qword [rbp + 24]\r
\r
;; UINT64  Gs, Fs, Es, Ds, Cs, Ss;\r
    pop     rax\r
    ; mov     gs, rax ; not for gs\r
    pop     rax\r
    ; mov     fs, rax ; not for fs\r
    ; (X64 will not use fs and gs, so we do not restore it)\r
    pop     rax\r
    mov     es, rax\r
    pop     rax\r
    mov     ds, rax\r
    pop     qword [rbp + 32]  ; for cs\r
    pop     qword [rbp + 56]  ; for ss\r
\r
;; UINT64  Rdi, Rsi, Rbp, Rsp, Rbx, Rdx, Rcx, Rax;\r
;; UINT64  R8, R9, R10, R11, R12, R13, R14, R15;\r
    pop     rdi\r
    pop     rsi\r
    add     rsp, 8               ; not for rbp\r
    pop     qword [rbp + 48] ; for rsp\r
    pop     rbx\r
    pop     rdx\r
    pop     rcx\r
    pop     rax\r
    pop     r8\r
    pop     r9\r
    pop     r10\r
    pop     r11\r
    pop     r12\r
    pop     r13\r
    pop     r14\r
    pop     r15\r
\r
    mov     rsp, rbp\r
    pop     rbp\r
    add     rsp, 16\r
    cmp     qword [rsp - 32], 0  ; check EXCEPTION_HANDLER_CONTEXT.OldIdtHandler\r
    jz      DoReturn\r
    cmp     qword [rsp - 40], 1  ; check EXCEPTION_HANDLER_CONTEXT.ExceptionDataFlag\r
    jz      ErrorCode\r
    jmp     qword [rsp - 32]\r
ErrorCode:\r
    sub     rsp, 8\r
    jmp     qword [rsp - 24]\r
\r
DoReturn:\r
    cmp     qword [ASM_PFX(mDoFarReturnFlag)], 0   ; Check if need to do far return instead of IRET\r
    jz      DoIret\r
    push    rax\r
    mov     rax, rsp          ; save old RSP to rax\r
    mov     rsp, [rsp + 0x20]\r
    push    qword [rax + 0x10]       ; save CS in new location\r
    push    qword [rax + 0x8]        ; save EIP in new location\r
    push    qword [rax + 0x18]       ; save EFLAGS in new location\r
    mov     rax, [rax]        ; restore rax\r
    popfq                     ; restore EFLAGS\r
    DB      0x48                ; prefix to composite "retq" with next "retf"\r
    retf                        ; far return\r
DoIret:\r
    iretq\r
\r
;-------------------------------------------------------------------------------------\r
;  GetTemplateAddressMap (&AddressMap);\r
;-------------------------------------------------------------------------------------\r
; comments here for definition of address map\r
global ASM_PFX(AsmGetTemplateAddressMap)\r
ASM_PFX(AsmGetTemplateAddressMap):\r
    lea     rax, [AsmIdtVectorBegin]\r
    mov     qword [rcx], rax\r
    mov     qword [rcx + 0x8],  (AsmIdtVectorEnd - AsmIdtVectorBegin) / 32\r
    lea     rax, [HookAfterStubHeaderBegin]\r
    mov     qword [rcx + 0x10], rax\r
\r
; Fix up CommonInterruptEntry address\r
    lea    rax, [ASM_PFX(CommonInterruptEntry)]\r
    lea    rcx, [AsmIdtVectorBegin]\r
%rep  32\r
    mov    qword [rcx + (JmpAbsoluteAddress - 8 - HookAfterStubHeaderBegin)], rax\r
    add    rcx, (AsmIdtVectorEnd - AsmIdtVectorBegin) / 32\r
%endrep\r
; Fix up HookAfterStubHeaderEnd\r
    lea    rax, [HookAfterStubHeaderEnd]\r
    lea    rcx, [JmpAbsoluteAddress]\r
    mov    qword [rcx - 8], rax\r
\r
    ret\r
\r
;-------------------------------------------------------------------------------------\r
;  AsmVectorNumFixup (*NewVectorAddr, VectorNum, *OldVectorAddr);\r
;-------------------------------------------------------------------------------------\r
global ASM_PFX(AsmVectorNumFixup)\r
ASM_PFX(AsmVectorNumFixup):\r
    mov     rax, rdx\r
    mov     [rcx + (@VectorNum - HookAfterStubHeaderBegin)], al\r
    ret\r
\r
Commit	Line	Data
ec94e97a TL	1	;------------------------------------------------------------------------------ ;\r
	2	; Copyright (c) 2012 - 2018, Intel Corporation. All rights reserved.<BR>\r
	3	; SPDX-License-Identifier: BSD-2-Clause-Patent\r
	4	;\r
	5	; Module Name:\r
	6	;\r
	7	; ExceptionHandlerAsm.Asm\r
	8	;\r
	9	; Abstract:\r
	10	;\r
	11	; x64 CPU Exception Handler\r
	12	;\r
	13	; Notes:\r
	14	;\r
	15	;------------------------------------------------------------------------------\r
0930e7ff	16	%include "Nasm.inc"\r
ec94e97a	17	\r
455b0347 S	18	;\r
	19	; Equivalent NASM structure of IA32_DESCRIPTOR\r
	20	;\r
	21	struc IA32_DESCRIPTOR\r
	22	.Limit CTYPE_UINT16 1\r
	23	.Base CTYPE_UINTN 1\r
	24	endstruc\r
	25	\r
	26	;\r
	27	; Equivalent NASM structure of IA32_IDT_GATE_DESCRIPTOR\r
	28	;\r
	29	struc IA32_IDT_GATE_DESCRIPTOR\r
	30	.OffsetLow CTYPE_UINT16 1\r
	31	.Selector CTYPE_UINT16 1\r
	32	.Reserved_0 CTYPE_UINT8 1\r
	33	.GateType CTYPE_UINT8 1\r
	34	.OffsetHigh CTYPE_UINT16 1\r
	35	.OffsetUpper CTYPE_UINT32 1\r
	36	.Reserved_1 CTYPE_UINT32 1\r
	37	endstruc\r
	38	\r
ec94e97a TL	39	;\r
	40	; CommonExceptionHandler()\r
	41	;\r
	42	\r
5277540e TL	43	%define VC_EXCEPTION 29\r
5277540e TL	44	\r
ec94e97a TL	45	extern ASM_PFX(mErrorCodeFlag) ; Error code flags for exceptions\r
	46	extern ASM_PFX(mDoFarReturnFlag) ; Do far return flag\r
	47	extern ASM_PFX(CommonExceptionHandler)\r
	48	\r
	49	SECTION .data\r
	50	\r
	51	DEFAULT REL\r
	52	SECTION .text\r
	53	\r
	54	ALIGN 8\r
	55	\r
	56	AsmIdtVectorBegin:\r
	57	%rep 32\r
	58	db 0x6a ; push #VectorNum\r
	59	db ($ - AsmIdtVectorBegin) / ((AsmIdtVectorEnd - AsmIdtVectorBegin) / 32) ; VectorNum\r
	60	push rax\r
	61	mov rax, strict qword 0 ; mov rax, ASM_PFX(CommonInterruptEntry)\r
	62	jmp rax\r
	63	%endrep\r
	64	AsmIdtVectorEnd:\r
	65	\r
	66	HookAfterStubHeaderBegin:\r
	67	db 0x6a ; push\r
	68	@VectorNum:\r
	69	db 0 ; 0 will be fixed\r
	70	push rax\r
	71	mov rax, strict qword 0 ; mov rax, HookAfterStubHeaderEnd\r
	72	JmpAbsoluteAddress:\r
	73	jmp rax\r
	74	HookAfterStubHeaderEnd:\r
	75	mov rax, rsp\r
	76	and sp, 0xfff0 ; make sure 16-byte aligned for exception context\r
	77	sub rsp, 0x18 ; reserve room for filling exception data later\r
	78	push rcx\r
	79	mov rcx, [rax + 8]\r
	80	bt [ASM_PFX(mErrorCodeFlag)], ecx\r
	81	jnc .0\r
	82	push qword [rsp] ; push additional rcx to make stack alignment\r
	83	.0:\r
	84	xchg rcx, [rsp] ; restore rcx, save Exception Number in stack\r
	85	push qword [rax] ; push rax into stack to keep code consistence\r
	86	\r
	87	;---------------------------------------;\r
	88	; CommonInterruptEntry ;\r
	89	;---------------------------------------;\r
	90	; The follow algorithm is used for the common interrupt routine.\r
	91	; Entry from each interrupt with a push eax and eax=interrupt number\r
	92	; Stack frame would be as follows as specified in IA32 manuals:\r
	93	;\r
	94	; +---------------------+ <-- 16-byte aligned ensured by processor\r
	95	; + Old SS +\r
	96	; +---------------------+\r
	97	; + Old RSP +\r
	98	; +---------------------+\r
	99	; + RFlags +\r
	100	; +---------------------+\r
	101	; + CS +\r
	102	; +---------------------+\r
	103	; + RIP +\r
	104	; +---------------------+\r
	105	; + Error Code +\r
	106	; +---------------------+\r
	107	; + Vector Number +\r
	108	; +---------------------+\r
109	; + RBP +\r
110	; +---------------------+ <-- RBP, 16-byte aligned\r
111	; The follow algorithm is used for the common interrupt routine.\r
112	global ASM_PFX(CommonInterruptEntry)\r
113	ASM_PFX(CommonInterruptEntry):\r
114	cli\r
115	pop rax\r
116	;\r
117	; All interrupt handlers are invoked through interrupt gates, so\r
118	; IF flag automatically cleared at the entry point\r
119	;\r
120	xchg rcx, [rsp] ; Save rcx into stack and save vector number into rcx\r
121	and rcx, 0xFF\r
122	cmp ecx, 32 ; Intel reserved vector for exceptions?\r
123	jae NoErrorCode\r
124	bt [ASM_PFX(mErrorCodeFlag)], ecx\r
125	jc HasErrorCode\r
126	\r
127	NoErrorCode:\r
128	\r
129	;\r
130	; Push a dummy error code on the stack\r
131	; to maintain coherent stack map\r
132	;\r
133	push qword [rsp]\r
134	mov qword [rsp + 8], 0\r
135	HasErrorCode:\r
136	push rbp\r
137	mov rbp, rsp\r
138	push 0 ; clear EXCEPTION_HANDLER_CONTEXT.OldIdtHandler\r
139	push 0 ; clear EXCEPTION_HANDLER_CONTEXT.ExceptionDataFlag\r
140	\r
141	;\r
142	; Stack:\r
143	; +---------------------+ <-- 16-byte aligned ensured by processor\r
144	; + Old SS +\r
145	; +---------------------+\r
146	; + Old RSP +\r
147	; +---------------------+\r
148	; + RFlags +\r
149	; +---------------------+\r
150	; + CS +\r
151	; +---------------------+\r
152	; + RIP +\r
153	; +---------------------+\r
154	; + Error Code +\r
155	; +---------------------+\r
156	; + RCX / Vector Number +\r
157	; +---------------------+\r
158	; + RBP +\r
159	; +---------------------+ <-- RBP, 16-byte aligned\r
160	;\r
161	\r
162	;\r
163	; Since here the stack pointer is 16-byte aligned, so\r
164	; EFI_FX_SAVE_STATE_X64 of EFI_SYSTEM_CONTEXT_x64\r
165	; is 16-byte aligned\r
166	;\r
167	\r
168	;; UINT64 Rdi, Rsi, Rbp, Rsp, Rbx, Rdx, Rcx, Rax;\r
169	;; UINT64 R8, R9, R10, R11, R12, R13, R14, R15;\r
170	push r15\r
171	push r14\r
172	push r13\r
173	push r12\r
174	push r11\r
175	push r10\r
176	push r9\r
177	push r8\r
178	push rax\r
179	push qword [rbp + 8] ; RCX\r
180	push rdx\r
181	push rbx\r
182	push qword [rbp + 48] ; RSP\r
183	push qword [rbp] ; RBP\r
184	push rsi\r
185	push rdi\r
186	\r
187	;; UINT64 Gs, Fs, Es, Ds, Cs, Ss; insure high 16 bits of each is zero\r
188	movzx rax, word [rbp + 56]\r
189	push rax ; for ss\r
190	movzx rax, word [rbp + 32]\r
191	push rax ; for cs\r
192	mov rax, ds\r
193	push rax\r
194	mov rax, es\r
195	push rax\r
196	mov rax, fs\r
197	push rax\r
198	mov rax, gs\r
199	push rax\r
200	\r
201	mov [rbp + 8], rcx ; save vector number\r
202	\r
203	;; UINT64 Rip;\r
204	push qword [rbp + 24]\r
205	\r
206	;; UINT64 Gdtr[2], Idtr[2];\r
207	xor rax, rax\r
208	push rax\r
209	push rax\r
210	sidt [rsp]\r
211	mov bx, word [rsp]\r
212	mov rax, qword [rsp + 2]\r
213	mov qword [rsp], rax\r
214	mov word [rsp + 8], bx\r
215	\r
216	xor rax, rax\r
217	push rax\r
218	push rax\r
219	sgdt [rsp]\r
220	mov bx, word [rsp]\r
221	mov rax, qword [rsp + 2]\r
222	mov qword [rsp], rax\r
223	mov word [rsp + 8], bx\r
224	\r
225	;; UINT64 Ldtr, Tr;\r
226	xor rax, rax\r
227	str ax\r
228	push rax\r
229	sldt ax\r
230	push rax\r
231	\r
232	;; UINT64 RFlags;\r
233	push qword [rbp + 40]\r
234	\r
235	;; UINT64 Cr0, Cr1, Cr2, Cr3, Cr4, Cr8;\r
236	mov rax, cr8\r
237	push rax\r
238	mov rax, cr4\r
239	or rax, 0x208\r
240	mov cr4, rax\r
241	push rax\r
242	mov rax, cr3\r
243	push rax\r
244	mov rax, cr2\r
245	push rax\r
246	xor rax, rax\r
247	push rax\r
248	mov rax, cr0\r
249	push rax\r
250	\r
251	;; UINT64 Dr0, Dr1, Dr2, Dr3, Dr6, Dr7;\r
5277540e TL	252	cmp qword [rbp + 8], VC_EXCEPTION\r
	253	je VcDebugRegs ; For SEV-ES (#VC) Debug registers ignored\r
	254	\r
ec94e97a TL	255	mov rax, dr7\r
	256	push rax\r
	257	mov rax, dr6\r
	258	push rax\r
	259	mov rax, dr3\r
	260	push rax\r
	261	mov rax, dr2\r
	262	push rax\r
	263	mov rax, dr1\r
	264	push rax\r
	265	mov rax, dr0\r
	266	push rax\r
5277540e TL	267	jmp DrFinish\r
	268	\r
	269	VcDebugRegs:\r
	270	;; UINT64 Dr0, Dr1, Dr2, Dr3, Dr6, Dr7 are skipped for #VC to avoid exception recursion\r
	271	xor rax, rax\r
	272	push rax\r
	273	push rax\r
	274	push rax\r
	275	push rax\r
	276	push rax\r
	277	push rax\r
ec94e97a	278	\r
5277540e	279	DrFinish:\r
ec94e97a TL	280	;; FX_SAVE_STATE_X64 FxSaveState;\r
	281	sub rsp, 512\r
	282	mov rdi, rsp\r
	283	db 0xf, 0xae, 0x7 ;fxsave [rdi]\r
	284	\r
	285	;; UEFI calling convention for x64 requires that Direction flag in EFLAGs is clear\r
	286	cld\r
	287	\r
	288	;; UINT32 ExceptionData;\r
	289	push qword [rbp + 16]\r
	290	\r
	291	;; Prepare parameter and call\r
	292	mov rcx, [rbp + 8]\r
	293	mov rdx, rsp\r
	294	;\r
	295	; Per X64 calling convention, allocate maximum parameter stack space\r
	296	; and make sure RSP is 16-byte aligned\r
	297	;\r
	298	sub rsp, 4 * 8 + 8\r
	299	call ASM_PFX(CommonExceptionHandler)\r
	300	add rsp, 4 * 8 + 8\r
	301	\r
0a6b303d SW	302	; The follow algorithm is used for clear shadow stack token busy bit.\r
0a6b303d SW	303	; The comment is based on the sample shadow stack.\r
455b0347	304	; Shadow stack is 32 bytes aligned.\r
0a6b303d SW	305	; The sample shadow stack layout :\r
	306	; Address \| Context\r
	307	; +-------------------------+\r
455b0347	308	; 0xFB8 \| FREE \| It is 0xFC0\|0x02\|(LMA & CS.L), after SAVEPREVSSP.\r
0a6b303d	309	; +-------------------------+\r
455b0347	310	; 0xFC0 \| Prev SSP \|\r
0a6b303d	311	; +-------------------------+\r
455b0347	312	; 0xFC8 \| RIP \|\r
0a6b303d	313	; +-------------------------+\r
455b0347	314	; 0xFD0 \| CS \|\r
0a6b303d	315	; +-------------------------+\r
455b0347	316	; 0xFD8 \| 0xFD8 \| BUSY \| BUSY flag cleared after CLRSSBSY\r
0a6b303d	317	; +-------------------------+\r
455b0347	318	; 0xFE0 \| 0xFC0\|0x02\|(LMA & CS.L) \|\r
0a6b303d SW	319	; +-------------------------+\r
	320	; Instructions for Intel Control Flow Enforcement Technology (CET) are supported since NASM version 2.15.01.\r
	321	cmp qword [ASM_PFX(mDoFarReturnFlag)], 0\r
	322	jz CetDone\r
0a6b303d	323	mov rax, cr4\r
455b0347 S	324	and rax, 0x800000 ; Check if CET is enabled\r
	325	jz CetDone\r
	326	sub rsp, 0x10\r
	327	sidt [rsp]\r
	328	mov rcx, qword [rsp + IA32_DESCRIPTOR.Base]; Get IDT base address\r
	329	add rsp, 0x10\r
	330	mov rax, qword [rbp + 8]; Get exception number\r
	331	sal rax, 0x04 ; Get IDT offset\r
	332	add rax, rcx ; Get IDT gate descriptor address\r
	333	mov al, byte [rax + IA32_IDT_GATE_DESCRIPTOR.Reserved_0]\r
	334	and rax, 0x01 ; Check IST field\r
0a6b303d	335	jz CetDone\r
455b0347	336	; SSP should be 0xFC0 at this point\r
0a6b303d	337	mov rax, 0x04 ; advance past cs:lip:prevssp;supervisor shadow stack token\r
455b0347 S	338	INCSSP_RAX ; After this SSP should be 0xFE0\r
	339	SAVEPREVSSP ; now the shadow stack restore token will be created at 0xFB8\r
	340	READSSP_RAX ; Read new SSP, SSP should be 0xFE8\r
0a6b303d	341	sub rax, 0x10\r
455b0347	342	CLRSSBSY_RAX ; Clear token at 0xFD8, SSP should be 0 after this\r
0a6b303d	343	sub rax, 0x20\r
455b0347	344	RSTORSSP_RAX ; Restore to token at 0xFB8, new SSP will be 0xFB8\r
0a6b303d	345	mov rax, 0x01 ; Pop off the new save token created\r
455b0347	346	INCSSP_RAX ; SSP should be 0xFC0 now\r
0a6b303d SW	347	CetDone:\r
0a6b303d SW	348	\r
ec94e97a TL	349	cli\r
	350	;; UINT64 ExceptionData;\r
	351	add rsp, 8\r
	352	\r
	353	;; FX_SAVE_STATE_X64 FxSaveState;\r
	354	\r
	355	mov rsi, rsp\r
	356	db 0xf, 0xae, 0xE ; fxrstor [rsi]\r
	357	add rsp, 512\r
	358	\r
	359	;; UINT64 Dr0, Dr1, Dr2, Dr3, Dr6, Dr7;\r
	360	;; Skip restoration of DRx registers to support in-circuit emualators\r
	361	;; or debuggers set breakpoint in interrupt/exception context\r
	362	add rsp, 8 * 6\r
	363	\r
	364	;; UINT64 Cr0, Cr1, Cr2, Cr3, Cr4, Cr8;\r
	365	pop rax\r
	366	mov cr0, rax\r
	367	add rsp, 8 ; not for Cr1\r
	368	pop rax\r
	369	mov cr2, rax\r
	370	pop rax\r
	371	mov cr3, rax\r
	372	pop rax\r
	373	mov cr4, rax\r
	374	pop rax\r
	375	mov cr8, rax\r
	376	\r
	377	;; UINT64 RFlags;\r
	378	pop qword [rbp + 40]\r
	379	\r
	380	;; UINT64 Ldtr, Tr;\r
	381	;; UINT64 Gdtr[2], Idtr[2];\r
	382	;; Best not let anyone mess with these particular registers...\r
	383	add rsp, 48\r
	384	\r
	385	;; UINT64 Rip;\r
	386	pop qword [rbp + 24]\r
	387	\r
	388	;; UINT64 Gs, Fs, Es, Ds, Cs, Ss;\r
	389	pop rax\r
	390	; mov gs, rax ; not for gs\r
	391	pop rax\r
	392	; mov fs, rax ; not for fs\r
	393	; (X64 will not use fs and gs, so we do not restore it)\r
	394	pop rax\r
	395	mov es, rax\r
	396	pop rax\r
	397	mov ds, rax\r
	398	pop qword [rbp + 32] ; for cs\r
	399	pop qword [rbp + 56] ; for ss\r
	400	\r
	401	;; UINT64 Rdi, Rsi, Rbp, Rsp, Rbx, Rdx, Rcx, Rax;\r
	402	;; UINT64 R8, R9, R10, R11, R12, R13, R14, R15;\r
	403	pop rdi\r
	404	pop rsi\r
	405	add rsp, 8 ; not for rbp\r
	406	pop qword [rbp + 48] ; for rsp\r
	407	pop rbx\r
	408	pop rdx\r
	409	pop rcx\r
	410	pop rax\r
	411	pop r8\r
	412	pop r9\r
413	pop r10\r
414	pop r11\r
415	pop r12\r
416	pop r13\r
417	pop r14\r
418	pop r15\r
419	\r
420	mov rsp, rbp\r
421	pop rbp\r
422	add rsp, 16\r
423	cmp qword [rsp - 32], 0 ; check EXCEPTION_HANDLER_CONTEXT.OldIdtHandler\r
424	jz DoReturn\r
425	cmp qword [rsp - 40], 1 ; check EXCEPTION_HANDLER_CONTEXT.ExceptionDataFlag\r
426	jz ErrorCode\r
427	jmp qword [rsp - 32]\r
428	ErrorCode:\r
429	sub rsp, 8\r
430	jmp qword [rsp - 24]\r
431	\r
432	DoReturn:\r
433	cmp qword [ASM_PFX(mDoFarReturnFlag)], 0 ; Check if need to do far return instead of IRET\r
434	jz DoIret\r
435	push rax\r
436	mov rax, rsp ; save old RSP to rax\r
437	mov rsp, [rsp + 0x20]\r
438	push qword [rax + 0x10] ; save CS in new location\r
439	push qword [rax + 0x8] ; save EIP in new location\r
440	push qword [rax + 0x18] ; save EFLAGS in new location\r
441	mov rax, [rax] ; restore rax\r
442	popfq ; restore EFLAGS\r
0a6b303d	443	DB 0x48 ; prefix to composite "retq" with next "retf"\r
0930e7ff	444	retf ; far return\r
ec94e97a TL	445	DoIret:\r
	446	iretq\r
	447	\r
	448	;-------------------------------------------------------------------------------------\r
	449	; GetTemplateAddressMap (&AddressMap);\r
	450	;-------------------------------------------------------------------------------------\r
	451	; comments here for definition of address map\r
	452	global ASM_PFX(AsmGetTemplateAddressMap)\r
	453	ASM_PFX(AsmGetTemplateAddressMap):\r
	454	lea rax, [AsmIdtVectorBegin]\r
	455	mov qword [rcx], rax\r
	456	mov qword [rcx + 0x8], (AsmIdtVectorEnd - AsmIdtVectorBegin) / 32\r
	457	lea rax, [HookAfterStubHeaderBegin]\r
	458	mov qword [rcx + 0x10], rax\r
	459	\r
	460	; Fix up CommonInterruptEntry address\r
	461	lea rax, [ASM_PFX(CommonInterruptEntry)]\r
	462	lea rcx, [AsmIdtVectorBegin]\r
	463	%rep 32\r
	464	mov qword [rcx + (JmpAbsoluteAddress - 8 - HookAfterStubHeaderBegin)], rax\r
	465	add rcx, (AsmIdtVectorEnd - AsmIdtVectorBegin) / 32\r
	466	%endrep\r
	467	; Fix up HookAfterStubHeaderEnd\r
	468	lea rax, [HookAfterStubHeaderEnd]\r
	469	lea rcx, [JmpAbsoluteAddress]\r
	470	mov qword [rcx - 8], rax\r
	471	\r
	472	ret\r
	473	\r
	474	;-------------------------------------------------------------------------------------\r
	475	; AsmVectorNumFixup (NewVectorAddr, VectorNum, OldVectorAddr);\r
	476	;-------------------------------------------------------------------------------------\r
	477	global ASM_PFX(AsmVectorNumFixup)\r
	478	ASM_PFX(AsmVectorNumFixup):\r
	479	mov rax, rdx\r
	480	mov [rcx + (@VectorNum - HookAfterStubHeaderBegin)], al\r
	481	ret\r
	482	\r