]>
Commit | Line | Data |
---|---|---|
d94e5f67 | 1 | ;------------------------------------------------------------------------------ ;\r |
09f69a87 | 2 | ; Copyright (c) 2015 - 2019, Intel Corporation. All rights reserved.<BR>\r |
0acd8697 | 3 | ; SPDX-License-Identifier: BSD-2-Clause-Patent\r |
d94e5f67 JF |
4 | ;\r |
5 | ; Module Name:\r | |
6 | ;\r | |
7 | ; MpFuncs.nasm\r | |
8 | ;\r | |
9 | ; Abstract:\r | |
10 | ;\r | |
11 | ; This is the assembly code for MP support\r | |
12 | ;\r | |
13 | ;-------------------------------------------------------------------------------\r | |
14 | \r | |
15 | %include "MpEqu.inc"\r | |
16 | extern ASM_PFX(InitializeFloatingPointUnits)\r | |
17 | \r | |
18 | DEFAULT REL\r | |
19 | \r | |
20 | SECTION .text\r | |
21 | \r | |
22 | ;-------------------------------------------------------------------------------------\r | |
23 | ;RendezvousFunnelProc procedure follows. All APs execute their procedure. This\r | |
24 | ;procedure serializes all the AP processors through an Init sequence. It must be\r | |
25 | ;noted that APs arrive here very raw...ie: real mode, no stack.\r | |
26 | ;ALSO THIS PROCEDURE IS EXECUTED BY APs ONLY ON 16 BIT MODE. HENCE THIS PROC\r | |
27 | ;IS IN MACHINE CODE.\r | |
28 | ;-------------------------------------------------------------------------------------\r | |
29 | global ASM_PFX(RendezvousFunnelProc)\r | |
30 | ASM_PFX(RendezvousFunnelProc):\r | |
31 | RendezvousFunnelProcStart:\r | |
32 | ; At this point CS = 0x(vv00) and ip= 0x0.\r | |
33 | ; Save BIST information to ebp firstly\r | |
34 | \r | |
35 | BITS 16\r | |
36 | mov ebp, eax ; Save BIST information\r | |
37 | \r | |
38 | mov ax, cs\r | |
39 | mov ds, ax\r | |
40 | mov es, ax\r | |
41 | mov ss, ax\r | |
42 | xor ax, ax\r | |
43 | mov fs, ax\r | |
44 | mov gs, ax\r | |
45 | \r | |
46 | mov si, BufferStartLocation\r | |
47 | mov ebx, [si]\r | |
48 | \r | |
f32bfe6d JW |
49 | mov si, DataSegmentLocation\r |
50 | mov edx, [si]\r | |
51 | \r | |
52 | ;\r | |
53 | ; Get start address of 32-bit code in low memory (<1MB)\r | |
54 | ;\r | |
55 | mov edi, ModeTransitionMemoryLocation\r | |
d94e5f67 JF |
56 | \r |
57 | mov si, GdtrLocation\r | |
58 | o32 lgdt [cs:si]\r | |
59 | \r | |
60 | mov si, IdtrLocation\r | |
61 | o32 lidt [cs:si]\r | |
62 | \r | |
f32bfe6d JW |
63 | ;\r |
64 | ; Switch to protected mode\r | |
65 | ;\r | |
66 | mov eax, cr0 ; Get control register 0\r | |
67 | or eax, 000000003h ; Set PE bit (bit #0) & MP\r | |
68 | mov cr0, eax\r | |
69 | \r | |
70 | ; Switch to 32-bit code (>1MB)\r | |
71 | o32 jmp far [cs:di]\r | |
72 | \r | |
73 | ;\r | |
74 | ; Following code must be copied to memory with type of EfiBootServicesCode.\r | |
75 | ; This is required if NX is enabled for EfiBootServicesCode of memory.\r | |
76 | ;\r | |
77 | BITS 32\r | |
78 | Flat32Start: ; protected mode entry point\r | |
79 | mov ds, dx\r | |
80 | mov es, dx\r | |
81 | mov fs, dx\r | |
82 | mov gs, dx\r | |
83 | mov ss, dx\r | |
5c66d125 JF |
84 | \r |
85 | ;\r | |
86 | ; Enable execute disable bit\r | |
87 | ;\r | |
f32bfe6d JW |
88 | mov esi, EnableExecuteDisableLocation\r |
89 | cmp byte [ebx + esi], 0\r | |
90 | jz SkipEnableExecuteDisableBit\r | |
91 | \r | |
5c66d125 JF |
92 | mov ecx, 0c0000080h ; EFER MSR number\r |
93 | rdmsr ; Read EFER\r | |
94 | bts eax, 11 ; Enable Execute Disable Bit\r | |
95 | wrmsr ; Write EFER\r | |
96 | \r | |
97 | SkipEnableExecuteDisableBit:\r | |
f32bfe6d JW |
98 | ;\r |
99 | ; Enable PAE\r | |
100 | ;\r | |
d94e5f67 JF |
101 | mov eax, cr4\r |
102 | bts eax, 5\r | |
09f69a87 RN |
103 | \r |
104 | mov esi, Enable5LevelPagingLocation\r | |
105 | cmp byte [ebx + esi], 0\r | |
106 | jz SkipEnable5LevelPaging\r | |
107 | \r | |
108 | ;\r | |
109 | ; Enable 5 Level Paging\r | |
110 | ;\r | |
111 | bts eax, 12 ; Set LA57=1.\r | |
112 | \r | |
113 | SkipEnable5LevelPaging:\r | |
114 | \r | |
d94e5f67 JF |
115 | mov cr4, eax\r |
116 | \r | |
f32bfe6d JW |
117 | ;\r |
118 | ; Load page table\r | |
119 | ;\r | |
120 | mov esi, Cr3Location ; Save CR3 in ecx\r | |
121 | mov ecx, [ebx + esi]\r | |
d94e5f67 JF |
122 | mov cr3, ecx ; Load CR3\r |
123 | \r | |
f32bfe6d JW |
124 | ;\r |
125 | ; Enable long mode\r | |
126 | ;\r | |
d94e5f67 JF |
127 | mov ecx, 0c0000080h ; EFER MSR number\r |
128 | rdmsr ; Read EFER\r | |
129 | bts eax, 8 ; Set LME=1\r | |
130 | wrmsr ; Write EFER\r | |
131 | \r | |
f32bfe6d JW |
132 | ;\r |
133 | ; Enable paging\r | |
134 | ;\r | |
d94e5f67 JF |
135 | mov eax, cr0 ; Read CR0\r |
136 | bts eax, 31 ; Set PG=1\r | |
137 | mov cr0, eax ; Write CR0\r | |
138 | \r | |
f32bfe6d JW |
139 | ;\r |
140 | ; Far jump to 64-bit code\r | |
141 | ;\r | |
142 | mov edi, ModeHighMemoryLocation\r | |
143 | add edi, ebx\r | |
144 | jmp far [edi]\r | |
145 | \r | |
d94e5f67 JF |
146 | BITS 64\r |
147 | LongModeStart:\r | |
845c5be1 JF |
148 | mov esi, ebx\r |
149 | lea edi, [esi + InitFlagLocation]\r | |
150 | cmp qword [edi], 1 ; ApInitConfig\r | |
151 | jnz GetApicId\r | |
152 | \r | |
0594ec41 ED |
153 | ; Increment the number of APs executing here as early as possible\r |
154 | ; This is decremented in C code when AP is finished executing\r | |
155 | mov edi, esi\r | |
156 | add edi, NumApsExecutingLocation\r | |
157 | lock inc dword [edi]\r | |
158 | \r | |
845c5be1 | 159 | ; AP init\r |
d94e5f67 JF |
160 | mov edi, esi\r |
161 | add edi, LockLocation\r | |
162 | mov rax, NotVacantFlag\r | |
163 | \r | |
164 | TestLock:\r | |
165 | xchg qword [edi], rax\r | |
166 | cmp rax, NotVacantFlag\r | |
167 | jz TestLock\r | |
168 | \r | |
37676b9f | 169 | lea ecx, [esi + ApIndexLocation]\r |
845c5be1 JF |
170 | inc dword [ecx]\r |
171 | mov ebx, [ecx]\r | |
d94e5f67 | 172 | \r |
845c5be1 JF |
173 | Releaselock:\r |
174 | mov rax, VacantFlag\r | |
175 | xchg qword [edi], rax\r | |
176 | ; program stack\r | |
d94e5f67 JF |
177 | mov edi, esi\r |
178 | add edi, StackSizeLocation\r | |
845c5be1 JF |
179 | mov eax, dword [edi]\r |
180 | mov ecx, ebx\r | |
181 | inc ecx\r | |
182 | mul ecx ; EAX = StackSize * (CpuNumber + 1)\r | |
d94e5f67 JF |
183 | mov edi, esi\r |
184 | add edi, StackStartAddressLocation\r | |
185 | add rax, qword [edi]\r | |
186 | mov rsp, rax\r | |
7b7508ad TL |
187 | \r |
188 | lea edi, [esi + SevEsIsEnabledLocation]\r | |
189 | cmp byte [edi], 1 ; SevEsIsEnabled\r | |
190 | jne CProcedureInvoke\r | |
191 | \r | |
192 | ;\r | |
193 | ; program GHCB\r | |
194 | ; Each page after the GHCB is a per-CPU page, so the calculation programs\r | |
195 | ; a GHCB to be every 8KB.\r | |
196 | ;\r | |
197 | mov eax, SIZE_4KB\r | |
198 | shl eax, 1 ; EAX = SIZE_4K * 2\r | |
199 | mov ecx, ebx\r | |
200 | mul ecx ; EAX = SIZE_4K * 2 * CpuNumber\r | |
201 | mov edi, esi\r | |
202 | add edi, GhcbBaseLocation\r | |
203 | add rax, qword [edi]\r | |
204 | mov rdx, rax\r | |
205 | shr rdx, 32\r | |
206 | mov rcx, 0xc0010130\r | |
207 | wrmsr\r | |
845c5be1 JF |
208 | jmp CProcedureInvoke\r |
209 | \r | |
210 | GetApicId:\r | |
7b7508ad TL |
211 | lea edi, [esi + SevEsIsEnabledLocation]\r |
212 | cmp byte [edi], 1 ; SevEsIsEnabled\r | |
213 | jne DoCpuid\r | |
214 | \r | |
215 | ;\r | |
216 | ; Since we don't have a stack yet, we can't take a #VC\r | |
217 | ; exception. Use the GHCB protocol to perform the CPUID\r | |
218 | ; calls.\r | |
219 | ;\r | |
220 | mov rcx, 0xc0010130\r | |
221 | rdmsr\r | |
222 | shl rdx, 32\r | |
223 | or rax, rdx\r | |
224 | mov rdi, rax ; RDI now holds the original GHCB GPA\r | |
225 | \r | |
226 | mov rdx, 0 ; CPUID function 0\r | |
227 | mov rax, 0 ; RAX register requested\r | |
228 | or rax, 4\r | |
229 | wrmsr\r | |
230 | rep vmmcall\r | |
231 | rdmsr\r | |
232 | cmp edx, 0bh\r | |
233 | jb NoX2ApicSevEs ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r | |
234 | \r | |
235 | mov rdx, 0bh ; CPUID function 0x0b\r | |
236 | mov rax, 040000000h ; RBX register requested\r | |
237 | or rax, 4\r | |
238 | wrmsr\r | |
239 | rep vmmcall\r | |
240 | rdmsr\r | |
241 | test edx, 0ffffh\r | |
242 | jz NoX2ApicSevEs ; CPUID.0BH:EBX[15:0] is zero\r | |
243 | \r | |
244 | mov rdx, 0bh ; CPUID function 0x0b\r | |
245 | mov rax, 0c0000000h ; RDX register requested\r | |
246 | or rax, 4\r | |
247 | wrmsr\r | |
248 | rep vmmcall\r | |
249 | rdmsr\r | |
250 | \r | |
251 | ; Processor is x2APIC capable; 32-bit x2APIC ID is now in EDX\r | |
252 | jmp RestoreGhcb\r | |
253 | \r | |
254 | NoX2ApicSevEs:\r | |
255 | ; Processor is not x2APIC capable, so get 8-bit APIC ID\r | |
256 | mov rdx, 1 ; CPUID function 1\r | |
257 | mov rax, 040000000h ; RBX register requested\r | |
258 | or rax, 4\r | |
259 | wrmsr\r | |
260 | rep vmmcall\r | |
261 | rdmsr\r | |
262 | shr edx, 24\r | |
263 | \r | |
264 | RestoreGhcb:\r | |
265 | mov rbx, rdx ; Save x2APIC/APIC ID\r | |
266 | \r | |
267 | mov rdx, rdi ; RDI holds the saved GHCB GPA\r | |
268 | shr rdx, 32\r | |
269 | mov eax, edi\r | |
270 | wrmsr\r | |
271 | \r | |
272 | mov rdx, rbx\r | |
273 | \r | |
274 | ; x2APIC ID or APIC ID is in EDX\r | |
275 | jmp GetProcessorNumber\r | |
276 | \r | |
277 | DoCpuid:\r | |
845c5be1 JF |
278 | mov eax, 0\r |
279 | cpuid\r | |
280 | cmp eax, 0bh\r | |
1cbd8330 LE |
281 | jb NoX2Apic ; CPUID level below CPUID_EXTENDED_TOPOLOGY\r |
282 | \r | |
283 | mov eax, 0bh\r | |
284 | xor ecx, ecx\r | |
285 | cpuid\r | |
286 | test ebx, 0ffffh\r | |
287 | jz NoX2Apic ; CPUID.0BH:EBX[15:0] is zero\r | |
288 | \r | |
289 | ; Processor is x2APIC capable; 32-bit x2APIC ID is already in EDX\r | |
290 | jmp GetProcessorNumber\r | |
291 | \r | |
292 | NoX2Apic:\r | |
845c5be1 JF |
293 | ; Processor is not x2APIC capable, so get 8-bit APIC ID\r |
294 | mov eax, 1\r | |
295 | cpuid\r | |
296 | shr ebx, 24\r | |
297 | mov edx, ebx\r | |
845c5be1 | 298 | \r |
845c5be1 JF |
299 | GetProcessorNumber:\r |
300 | ;\r | |
301 | ; Get processor number for this AP\r | |
302 | ; Note that BSP may become an AP due to SwitchBsp()\r | |
303 | ;\r | |
304 | xor ebx, ebx\r | |
305 | lea eax, [esi + CpuInfoLocation]\r | |
306 | mov edi, [eax]\r | |
d94e5f67 | 307 | \r |
845c5be1 JF |
308 | GetNextProcNumber:\r |
309 | cmp dword [edi], edx ; APIC ID match?\r | |
310 | jz ProgramStack\r | |
dd3fa0cd | 311 | add edi, 20\r |
845c5be1 | 312 | inc ebx\r |
7367cc6c | 313 | jmp GetNextProcNumber\r |
845c5be1 JF |
314 | \r |
315 | ProgramStack:\r | |
dd3fa0cd | 316 | mov rsp, qword [edi + 12]\r |
d94e5f67 JF |
317 | \r |
318 | CProcedureInvoke:\r | |
8396e2dd JF |
319 | push rbp ; Push BIST data at top of AP stack\r |
320 | xor rbp, rbp ; Clear ebp for call stack trace\r | |
d94e5f67 JF |
321 | push rbp\r |
322 | mov rbp, rsp\r | |
323 | \r | |
3b2928b4 | 324 | mov rax, qword [esi + InitializeFloatingPointUnitsAddress]\r |
d94e5f67 JF |
325 | sub rsp, 20h\r |
326 | call rax ; Call assembly function to initialize FPU per UEFI spec\r | |
327 | add rsp, 20h\r | |
328 | \r | |
37676b9f | 329 | mov edx, ebx ; edx is ApIndex\r |
d94e5f67 JF |
330 | mov ecx, esi\r |
331 | add ecx, LockLocation ; rcx is address of exchange info data buffer\r | |
332 | \r | |
333 | mov edi, esi\r | |
334 | add edi, ApProcedureLocation\r | |
335 | mov rax, qword [edi]\r | |
336 | \r | |
337 | sub rsp, 20h\r | |
8396e2dd | 338 | call rax ; Invoke C function\r |
d94e5f67 | 339 | add rsp, 20h\r |
8396e2dd | 340 | jmp $ ; Should never reach here\r |
d94e5f67 JF |
341 | \r |
342 | RendezvousFunnelProcEnd:\r | |
343 | \r | |
7b7508ad TL |
344 | ;-------------------------------------------------------------------------------------\r |
345 | ;SwitchToRealProc procedure follows.\r | |
346 | ;ALSO THIS PROCEDURE IS EXECUTED BY APs TRANSITIONING TO 16 BIT MODE. HENCE THIS PROC\r | |
347 | ;IS IN MACHINE CODE.\r | |
348 | ; SwitchToRealProc (UINTN BufferStart, UINT16 Code16, UINT16 Code32, UINTN StackStart)\r | |
349 | ; rcx - Buffer Start\r | |
350 | ; rdx - Code16 Selector Offset\r | |
351 | ; r8 - Code32 Selector Offset\r | |
352 | ; r9 - Stack Start\r | |
353 | ;-------------------------------------------------------------------------------------\r | |
354 | global ASM_PFX(SwitchToRealProc)\r | |
355 | ASM_PFX(SwitchToRealProc):\r | |
356 | SwitchToRealProcStart:\r | |
357 | BITS 64\r | |
358 | cli\r | |
359 | \r | |
360 | ;\r | |
361 | ; Get RDX reset value before changing stacks since the\r | |
362 | ; new stack won't be able to accomodate a #VC exception.\r | |
363 | ;\r | |
364 | push rax\r | |
365 | push rbx\r | |
366 | push rcx\r | |
367 | push rdx\r | |
368 | \r | |
369 | mov rax, 1\r | |
370 | cpuid\r | |
371 | mov rsi, rax ; Save off the reset value for RDX\r | |
372 | \r | |
373 | pop rdx\r | |
374 | pop rcx\r | |
375 | pop rbx\r | |
376 | pop rax\r | |
377 | \r | |
378 | ;\r | |
379 | ; Establish stack below 1MB\r | |
380 | ;\r | |
381 | mov rsp, r9\r | |
382 | \r | |
383 | ;\r | |
384 | ; Push ultimate Reset Vector onto the stack\r | |
385 | ;\r | |
386 | mov rax, rcx\r | |
387 | shr rax, 4\r | |
388 | push word 0x0002 ; RFLAGS\r | |
389 | push ax ; CS\r | |
390 | push word 0x0000 ; RIP\r | |
391 | push word 0x0000 ; For alignment, will be discarded\r | |
392 | \r | |
393 | ;\r | |
394 | ; Get address of "16-bit operand size" label\r | |
395 | ;\r | |
396 | lea rbx, [PM16Mode]\r | |
397 | \r | |
398 | ;\r | |
399 | ; Push addresses used to change to compatibility mode\r | |
400 | ;\r | |
401 | lea rax, [CompatMode]\r | |
402 | push r8\r | |
403 | push rax\r | |
404 | \r | |
405 | ;\r | |
406 | ; Clear R8 - R15, for reset, before going into 32-bit mode\r | |
407 | ;\r | |
408 | xor r8, r8\r | |
409 | xor r9, r9\r | |
410 | xor r10, r10\r | |
411 | xor r11, r11\r | |
412 | xor r12, r12\r | |
413 | xor r13, r13\r | |
414 | xor r14, r14\r | |
415 | xor r15, r15\r | |
416 | \r | |
417 | ;\r | |
418 | ; Far return into 32-bit mode\r | |
419 | ;\r | |
420 | o64 retf\r | |
421 | \r | |
422 | BITS 32\r | |
423 | CompatMode:\r | |
424 | ;\r | |
425 | ; Set up stack to prepare for exiting protected mode\r | |
426 | ;\r | |
427 | push edx ; Code16 CS\r | |
428 | push ebx ; PM16Mode label address\r | |
429 | \r | |
430 | ;\r | |
431 | ; Disable paging\r | |
432 | ;\r | |
433 | mov eax, cr0 ; Read CR0\r | |
434 | btr eax, 31 ; Set PG=0\r | |
435 | mov cr0, eax ; Write CR0\r | |
436 | \r | |
437 | ;\r | |
438 | ; Disable long mode\r | |
439 | ;\r | |
440 | mov ecx, 0c0000080h ; EFER MSR number\r | |
441 | rdmsr ; Read EFER\r | |
442 | btr eax, 8 ; Set LME=0\r | |
443 | wrmsr ; Write EFER\r | |
444 | \r | |
445 | ;\r | |
446 | ; Disable PAE\r | |
447 | ;\r | |
448 | mov eax, cr4 ; Read CR4\r | |
449 | btr eax, 5 ; Set PAE=0\r | |
450 | mov cr4, eax ; Write CR4\r | |
451 | \r | |
452 | mov edx, esi ; Restore RDX reset value\r | |
453 | \r | |
454 | ;\r | |
455 | ; Switch to 16-bit operand size\r | |
456 | ;\r | |
457 | retf\r | |
458 | \r | |
459 | BITS 16\r | |
460 | ;\r | |
461 | ; At entry to this label\r | |
462 | ; - RDX will have its reset value\r | |
463 | ; - On the top of the stack\r | |
464 | ; - Alignment data (two bytes) to be discarded\r | |
465 | ; - IP for Real Mode (two bytes)\r | |
466 | ; - CS for Real Mode (two bytes)\r | |
467 | ;\r | |
20da7ca4 TL |
468 | ; This label is also used with AsmRelocateApLoop. During MP finalization,\r |
469 | ; the code from PM16Mode to SwitchToRealProcEnd is copied to the start of\r | |
470 | ; the WakeupBuffer, allowing a parked AP to be booted by an OS.\r | |
471 | ;\r | |
7b7508ad TL |
472 | PM16Mode:\r |
473 | mov eax, cr0 ; Read CR0\r | |
474 | btr eax, 0 ; Set PE=0\r | |
475 | mov cr0, eax ; Write CR0\r | |
476 | \r | |
477 | pop ax ; Discard alignment data\r | |
478 | \r | |
479 | ;\r | |
480 | ; Clear registers (except RDX and RSP) before going into 16-bit mode\r | |
481 | ;\r | |
482 | xor eax, eax\r | |
483 | xor ebx, ebx\r | |
484 | xor ecx, ecx\r | |
485 | xor esi, esi\r | |
486 | xor edi, edi\r | |
487 | xor ebp, ebp\r | |
488 | \r | |
489 | iret\r | |
490 | \r | |
491 | SwitchToRealProcEnd:\r | |
492 | \r | |
76157021 | 493 | ;-------------------------------------------------------------------------------------\r |
20da7ca4 | 494 | ; AsmRelocateApLoop (MwaitSupport, ApTargetCState, PmCodeSegment, TopOfApStack, CountTofinish, Pm16CodeSegment, SevEsAPJumpTable, WakeupBuffer);\r |
76157021 JF |
495 | ;-------------------------------------------------------------------------------------\r |
496 | global ASM_PFX(AsmRelocateApLoop)\r | |
497 | ASM_PFX(AsmRelocateApLoop):\r | |
498 | AsmRelocateApLoopStart:\r | |
7b7508ad | 499 | BITS 64\r |
20da7ca4 TL |
500 | cmp qword [rsp + 56], 0 ; SevEsAPJumpTable\r |
501 | je NoSevEs\r | |
502 | \r | |
503 | ;\r | |
504 | ; Perform some SEV-ES related setup before leaving 64-bit mode\r | |
505 | ;\r | |
506 | push rcx\r | |
507 | push rdx\r | |
508 | \r | |
509 | ;\r | |
510 | ; Get the RDX reset value using CPUID\r | |
511 | ;\r | |
512 | mov rax, 1\r | |
513 | cpuid\r | |
514 | mov rsi, rax ; Save off the reset value for RDX\r | |
515 | \r | |
516 | ;\r | |
517 | ; Prepare the GHCB for the AP_HLT_LOOP VMGEXIT call\r | |
518 | ; - Must be done while in 64-bit long mode so that writes to\r | |
519 | ; the GHCB memory will be unencrypted.\r | |
520 | ; - No NAE events can be generated once this is set otherwise\r | |
521 | ; the AP_RESET_HOLD SW_EXITCODE will be overwritten.\r | |
522 | ;\r | |
523 | mov rcx, 0xc0010130\r | |
524 | rdmsr ; Retrieve current GHCB address\r | |
525 | shl rdx, 32\r | |
526 | or rdx, rax\r | |
527 | \r | |
528 | mov rdi, rdx\r | |
529 | xor rax, rax\r | |
530 | mov rcx, 0x800\r | |
531 | shr rcx, 3\r | |
532 | rep stosq ; Clear the GHCB\r | |
533 | \r | |
534 | mov rax, 0x80000004 ; VMGEXIT AP_RESET_HOLD\r | |
535 | mov [rdx + 0x390], rax\r | |
fb2a1a36 TL |
536 | mov rax, 114 ; Set SwExitCode valid bit\r |
537 | bts [rdx + 0x3f0], rax\r | |
538 | inc rax ; Set SwExitInfo1 valid bit\r | |
539 | bts [rdx + 0x3f0], rax\r | |
540 | inc rax ; Set SwExitInfo2 valid bit\r | |
541 | bts [rdx + 0x3f0], rax\r | |
20da7ca4 TL |
542 | \r |
543 | pop rdx\r | |
544 | pop rcx\r | |
545 | \r | |
546 | NoSevEs:\r | |
a7bbe9d2 | 547 | cli ; Disable interrupt before switching to 32-bit mode\r |
9f91cb01 JF |
548 | mov rax, [rsp + 40] ; CountTofinish\r |
549 | lock dec dword [rax] ; (*CountTofinish)--\r | |
76157021 | 550 | \r |
20da7ca4 TL |
551 | mov r10, [rsp + 48] ; Pm16CodeSegment\r |
552 | mov rax, [rsp + 56] ; SevEsAPJumpTable\r | |
553 | mov rbx, [rsp + 64] ; WakeupBuffer\r | |
554 | mov rsp, r9 ; TopOfApStack\r | |
555 | \r | |
556 | push rax ; Save SevEsAPJumpTable\r | |
557 | push rbx ; Save WakeupBuffer\r | |
558 | push r10 ; Save Pm16CodeSegment\r | |
559 | push rcx ; Save MwaitSupport\r | |
560 | push rdx ; Save ApTargetCState\r | |
561 | \r | |
562 | lea rax, [PmEntry] ; rax <- The start address of transition code\r | |
76157021 JF |
563 | \r |
564 | push r8\r | |
20da7ca4 TL |
565 | push rax\r |
566 | \r | |
567 | ;\r | |
568 | ; Clear R8 - R15, for reset, before going into 32-bit mode\r | |
569 | ;\r | |
570 | xor r8, r8\r | |
571 | xor r9, r9\r | |
572 | xor r10, r10\r | |
573 | xor r11, r11\r | |
574 | xor r12, r12\r | |
575 | xor r13, r13\r | |
576 | xor r14, r14\r | |
577 | xor r15, r15\r | |
578 | \r | |
579 | ;\r | |
580 | ; Far return into 32-bit mode\r | |
581 | ;\r | |
582 | o64 retf\r | |
583 | \r | |
76157021 JF |
584 | BITS 32\r |
585 | PmEntry:\r | |
586 | mov eax, cr0\r | |
587 | btr eax, 31 ; Clear CR0.PG\r | |
588 | mov cr0, eax ; Disable paging and caches\r | |
589 | \r | |
76157021 JF |
590 | mov ecx, 0xc0000080\r |
591 | rdmsr\r | |
592 | and ah, ~ 1 ; Clear LME\r | |
593 | wrmsr\r | |
594 | mov eax, cr4\r | |
595 | and al, ~ (1 << 5) ; Clear PAE\r | |
596 | mov cr4, eax\r | |
597 | \r | |
598 | pop edx\r | |
599 | add esp, 4\r | |
600 | pop ecx,\r | |
601 | add esp, 4\r | |
20da7ca4 TL |
602 | \r |
603 | MwaitCheck:\r | |
76157021 JF |
604 | cmp cl, 1 ; Check mwait-monitor support\r |
605 | jnz HltLoop\r | |
606 | mov ebx, edx ; Save C-State to ebx\r | |
607 | MwaitLoop:\r | |
a7bbe9d2 | 608 | cli\r |
76157021 JF |
609 | mov eax, esp ; Set Monitor Address\r |
610 | xor ecx, ecx ; ecx = 0\r | |
611 | xor edx, edx ; edx = 0\r | |
612 | monitor\r | |
76157021 | 613 | mov eax, ebx ; Mwait Cx, Target C-State per eax[7:4]\r |
f56379f3 | 614 | shl eax, 4\r |
76157021 JF |
615 | mwait\r |
616 | jmp MwaitLoop\r | |
20da7ca4 | 617 | \r |
76157021 | 618 | HltLoop:\r |
20da7ca4 TL |
619 | pop edx ; PM16CodeSegment\r |
620 | add esp, 4\r | |
621 | pop ebx ; WakeupBuffer\r | |
622 | add esp, 4\r | |
623 | pop eax ; SevEsAPJumpTable\r | |
624 | add esp, 4\r | |
625 | cmp eax, 0 ; Check for SEV-ES\r | |
626 | je DoHlt\r | |
627 | \r | |
628 | cli\r | |
629 | ;\r | |
630 | ; SEV-ES is enabled, use VMGEXIT (GHCB information already\r | |
631 | ; set by caller)\r | |
632 | ;\r | |
633 | BITS 64\r | |
634 | rep vmmcall\r | |
635 | BITS 32\r | |
636 | \r | |
637 | ;\r | |
638 | ; Back from VMGEXIT AP_HLT_LOOP\r | |
639 | ; Push the FLAGS/CS/IP values to use\r | |
640 | ;\r | |
641 | push word 0x0002 ; EFLAGS\r | |
642 | xor ecx, ecx\r | |
643 | mov cx, [eax + 2] ; CS\r | |
644 | push cx\r | |
645 | mov cx, [eax] ; IP\r | |
646 | push cx\r | |
647 | push word 0x0000 ; For alignment, will be discarded\r | |
648 | \r | |
649 | push edx\r | |
650 | push ebx\r | |
651 | \r | |
652 | mov edx, esi ; Restore RDX reset value\r | |
653 | \r | |
654 | retf\r | |
655 | \r | |
656 | DoHlt:\r | |
76157021 JF |
657 | cli\r |
658 | hlt\r | |
20da7ca4 TL |
659 | jmp DoHlt\r |
660 | \r | |
76157021 JF |
661 | BITS 64\r |
662 | AsmRelocateApLoopEnd:\r | |
663 | \r | |
d94e5f67 JF |
664 | ;-------------------------------------------------------------------------------------\r |
665 | ; AsmGetAddressMap (&AddressMap);\r | |
666 | ;-------------------------------------------------------------------------------------\r | |
667 | global ASM_PFX(AsmGetAddressMap)\r | |
668 | ASM_PFX(AsmGetAddressMap):\r | |
3b2928b4 | 669 | lea rax, [ASM_PFX(RendezvousFunnelProc)]\r |
d94e5f67 JF |
670 | mov qword [rcx], rax\r |
671 | mov qword [rcx + 8h], LongModeStart - RendezvousFunnelProcStart\r | |
672 | mov qword [rcx + 10h], RendezvousFunnelProcEnd - RendezvousFunnelProcStart\r | |
3b2928b4 | 673 | lea rax, [ASM_PFX(AsmRelocateApLoop)]\r |
f7f85d83 JF |
674 | mov qword [rcx + 18h], rax\r |
675 | mov qword [rcx + 20h], AsmRelocateApLoopEnd - AsmRelocateApLoopStart\r | |
f32bfe6d | 676 | mov qword [rcx + 28h], Flat32Start - RendezvousFunnelProcStart\r |
7b7508ad TL |
677 | mov qword [rcx + 30h], SwitchToRealProcEnd - SwitchToRealProcStart ; SwitchToRealSize\r |
678 | mov qword [rcx + 38h], SwitchToRealProcStart - RendezvousFunnelProcStart ; SwitchToRealOffset\r | |
679 | mov qword [rcx + 40h], SwitchToRealProcStart - Flat32Start ; SwitchToRealNoNxOffset\r | |
680 | mov qword [rcx + 48h], PM16Mode - RendezvousFunnelProcStart ; SwitchToRealPM16ModeOffset\r | |
681 | mov qword [rcx + 50h], SwitchToRealProcEnd - PM16Mode ; SwitchToRealPM16ModeSize\r | |
d94e5f67 JF |
682 | ret\r |
683 | \r | |
684 | ;-------------------------------------------------------------------------------------\r | |
685 | ;AsmExchangeRole procedure follows. This procedure executed by current BSP, that is\r | |
8396e2dd | 686 | ;about to become an AP. It switches its stack with the current AP.\r |
d94e5f67 JF |
687 | ;AsmExchangeRole (IN CPU_EXCHANGE_INFO *MyInfo, IN CPU_EXCHANGE_INFO *OthersInfo);\r |
688 | ;-------------------------------------------------------------------------------------\r | |
689 | global ASM_PFX(AsmExchangeRole)\r | |
690 | ASM_PFX(AsmExchangeRole):\r | |
691 | ; DO NOT call other functions in this function, since 2 CPU may use 1 stack\r | |
692 | ; at the same time. If 1 CPU try to call a function, stack will be corrupted.\r | |
693 | \r | |
694 | push rax\r | |
695 | push rbx\r | |
696 | push rcx\r | |
697 | push rdx\r | |
698 | push rsi\r | |
699 | push rdi\r | |
700 | push rbp\r | |
701 | push r8\r | |
702 | push r9\r | |
703 | push r10\r | |
704 | push r11\r | |
705 | push r12\r | |
706 | push r13\r | |
707 | push r14\r | |
708 | push r15\r | |
709 | \r | |
710 | mov rax, cr0\r | |
711 | push rax\r | |
712 | \r | |
713 | mov rax, cr4\r | |
714 | push rax\r | |
715 | \r | |
716 | ; rsi contains MyInfo pointer\r | |
717 | mov rsi, rcx\r | |
718 | \r | |
719 | ; rdi contains OthersInfo pointer\r | |
720 | mov rdi, rdx\r | |
721 | \r | |
722 | ;Store EFLAGS, GDTR and IDTR regiter to stack\r | |
723 | pushfq\r | |
724 | sgdt [rsi + 16]\r | |
725 | sidt [rsi + 26]\r | |
726 | \r | |
727 | ; Store the its StackPointer\r | |
728 | mov [rsi + 8], rsp\r | |
729 | \r | |
730 | ; update its switch state to STORED\r | |
731 | mov byte [rsi], CPU_SWITCH_STATE_STORED\r | |
732 | \r | |
733 | WaitForOtherStored:\r | |
734 | ; wait until the other CPU finish storing its state\r | |
735 | cmp byte [rdi], CPU_SWITCH_STATE_STORED\r | |
736 | jz OtherStored\r | |
737 | pause\r | |
738 | jmp WaitForOtherStored\r | |
739 | \r | |
740 | OtherStored:\r | |
741 | ; Since another CPU already stored its state, load them\r | |
742 | ; load GDTR value\r | |
743 | lgdt [rdi + 16]\r | |
744 | \r | |
745 | ; load IDTR value\r | |
746 | lidt [rdi + 26]\r | |
747 | \r | |
748 | ; load its future StackPointer\r | |
749 | mov rsp, [rdi + 8]\r | |
750 | \r | |
751 | ; update the other CPU's switch state to LOADED\r | |
752 | mov byte [rdi], CPU_SWITCH_STATE_LOADED\r | |
753 | \r | |
754 | WaitForOtherLoaded:\r | |
755 | ; wait until the other CPU finish loading new state,\r | |
756 | ; otherwise the data in stack may corrupt\r | |
757 | cmp byte [rsi], CPU_SWITCH_STATE_LOADED\r | |
758 | jz OtherLoaded\r | |
759 | pause\r | |
760 | jmp WaitForOtherLoaded\r | |
761 | \r | |
762 | OtherLoaded:\r | |
763 | ; since the other CPU already get the data it want, leave this procedure\r | |
764 | popfq\r | |
765 | \r | |
766 | pop rax\r | |
767 | mov cr4, rax\r | |
768 | \r | |
769 | pop rax\r | |
770 | mov cr0, rax\r | |
771 | \r | |
772 | pop r15\r | |
773 | pop r14\r | |
774 | pop r13\r | |
775 | pop r12\r | |
776 | pop r11\r | |
777 | pop r10\r | |
778 | pop r9\r | |
779 | pop r8\r | |
780 | pop rbp\r | |
781 | pop rdi\r | |
782 | pop rsi\r | |
783 | pop rdx\r | |
784 | pop rcx\r | |
785 | pop rbx\r | |
786 | pop rax\r | |
787 | \r | |
788 | ret\r |