OvmfPkg/Library/CcExitLib/CcExitVcHandler.c

   1 /** @file
   2   X64 #VC Exception Handler functon.
   3
   4   Copyright (C) 2020, Advanced Micro Devices, Inc. All rights reserved.<BR>
   5   SPDX-License-Identifier: BSD-2-Clause-Patent
   6
   7 **/
   8
   9 #include <Base.h>
  10 #include <Uefi.h>
  11 #include <Library/BaseMemoryLib.h>
  12 #include <Library/LocalApicLib.h>
  13 #include <Library/MemEncryptSevLib.h>
  14 #include <Library/CcExitLib.h>
  15 #include <Register/Amd/Msr.h>
  16 #include <Register/Intel/Cpuid.h>
  17 #include <IndustryStandard/InstructionParsing.h>
  18
  19 #include "CcExitVcHandler.h"
  20 #include "CcInstruction.h"
  21
  22 //
  23 // Non-automatic Exit function prototype
  24 //
  25 typedef
  26 UINT64
  27 (*NAE_EXIT) (
  28   GHCB                    *Ghcb,
  29   EFI_SYSTEM_CONTEXT_X64  *Regs,
  30   CC_INSTRUCTION_DATA     *InstructionData
  31   );
  32
  33 //
  34 // SEV-SNP Cpuid table entry/function
  35 //
  36 typedef PACKED struct {
  37   UINT32    EaxIn;
  38   UINT32    EcxIn;
  39   UINT64    Unused;
  40   UINT64    Unused2;
  41   UINT32    Eax;
  42   UINT32    Ebx;
  43   UINT32    Ecx;
  44   UINT32    Edx;
  45   UINT64    Reserved;
  46 } SEV_SNP_CPUID_FUNCTION;
  47
  48 //
  49 // SEV-SNP Cpuid page format
  50 //
  51 typedef PACKED struct {
  52   UINT32                    Count;
  53   UINT32                    Reserved1;
  54   UINT64                    Reserved2;
  55   SEV_SNP_CPUID_FUNCTION    function[0];
  56 } SEV_SNP_CPUID_INFO;
  57
  58 /**
  59   Report an unsupported event to the hypervisor
  60
  61   Use the VMGEXIT support to report an unsupported event to the hypervisor.
  62
  63   @param[in] Ghcb             Pointer to the Guest-Hypervisor Communication
  64                               Block
  65   @param[in] Regs             x64 processor context
  66   @param[in] InstructionData  Instruction parsing context
  67
  68   @return                     New exception value to propagate
  69
  70 **/
  71 STATIC
  72 UINT64
  73 UnsupportedExit (
  74   IN GHCB                    *Ghcb,
  75   IN EFI_SYSTEM_CONTEXT_X64  *Regs,
  76   IN CC_INSTRUCTION_DATA     *InstructionData
  77   )
  78 {
  79   UINT64  Status;
  80
  81   Status = CcExitVmgExit (Ghcb, SVM_EXIT_UNSUPPORTED, Regs->ExceptionData, 0);
  82   if (Status == 0) {
  83     GHCB_EVENT_INJECTION  Event;
  84
  85     Event.Uint64          = 0;
  86     Event.Elements.Vector = GP_EXCEPTION;
  87     Event.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
  88     Event.Elements.Valid  = 1;
  89
  90     Status = Event.Uint64;
  91   }
  92
  93   return Status;
  94 }
  95
  96 /**
  97   Validate that the MMIO memory access is not to encrypted memory.
  98
  99   Examine the pagetable entry for the memory specified. MMIO should not be
 100   performed against encrypted memory. MMIO to the APIC page is always allowed.
 101
 102   @param[in] Ghcb           Pointer to the Guest-Hypervisor Communication Block
 103   @param[in] MemoryAddress  Memory address to validate
 104   @param[in] MemoryLength   Memory length to validate
 105
 106   @retval 0          Memory is not encrypted
 107   @return            New exception value to propogate
 108
 109 **/
 110 STATIC
 111 UINT64
 112 ValidateMmioMemory (
 113   IN GHCB   *Ghcb,
 114   IN UINTN  MemoryAddress,
 115   IN UINTN  MemoryLength
 116   )
 117 {
 118   MEM_ENCRYPT_SEV_ADDRESS_RANGE_STATE  State;
 119   GHCB_EVENT_INJECTION                 GpEvent;
 120   UINTN                                Address;
 121
 122   //
 123   // Allow APIC accesses (which will have the encryption bit set during
 124   // SEC and PEI phases).
 125   //
 126   Address = MemoryAddress & ~(SIZE_4KB - 1);
 127   if (Address == GetLocalApicBaseAddress ()) {
 128     return 0;
 129   }
 130
 131   State = MemEncryptSevGetAddressRangeState (
 132             0,
 133             MemoryAddress,
 134             MemoryLength
 135             );
 136   if (State == MemEncryptSevAddressRangeUnencrypted) {
 137     return 0;
 138   }
 139
 140   //
 141   // Any state other than unencrypted is an error, issue a #GP.
 142   //
 143   DEBUG ((
 144     DEBUG_ERROR,
 145     "MMIO using encrypted memory: %lx\n",
 146     (UINT64)MemoryAddress
 147     ));
 148   GpEvent.Uint64          = 0;
 149   GpEvent.Elements.Vector = GP_EXCEPTION;
 150   GpEvent.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
 151   GpEvent.Elements.Valid  = 1;
 152
 153   return GpEvent.Uint64;
 154 }
 155
 156 /**
 157   Handle an MMIO event.
 158
 159   Use the VMGEXIT instruction to handle either an MMIO read or an MMIO write.
 160
 161   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 162                                    Block
 163   @param[in, out] Regs             x64 processor context
 164   @param[in, out] InstructionData  Instruction parsing context
 165
 166   @retval 0                        Event handled successfully
 167   @return                          New exception value to propagate
 168
 169 **/
 170 STATIC
 171 UINT64
 172 MmioExit (
 173   IN OUT GHCB                    *Ghcb,
 174   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
 175   IN OUT CC_INSTRUCTION_DATA     *InstructionData
 176   )
 177 {
 178   UINT64  ExitInfo1, ExitInfo2, Status;
 179   UINTN   Bytes;
 180   UINT64  *Register;
 181   UINT8   OpCode, SignByte;
 182   UINTN   Address;
 183
 184   Bytes = 0;
 185
 186   OpCode = *(InstructionData->OpCodes);
 187   if (OpCode == TWO_BYTE_OPCODE_ESCAPE) {
 188     OpCode = *(InstructionData->OpCodes + 1);
 189   }
 190
 191   switch (OpCode) {
 192     //
 193     // MMIO write (MOV reg/memX, regX)
 194     //
 195     case 0x88:
 196       Bytes = 1;
 197     //
 198     // fall through
 199     //
 200     case 0x89:
 201       CcDecodeModRm (Regs, InstructionData);
 202       Bytes = ((Bytes != 0) ? Bytes :
 203                (InstructionData->DataSize == Size16Bits) ? 2 :
 204                (InstructionData->DataSize == Size32Bits) ? 4 :
 205                (InstructionData->DataSize == Size64Bits) ? 8 :
 206                0);
 207
 208       if (InstructionData->Ext.ModRm.Mod == 3) {
 209         //
 210         // NPF on two register operands???
 211         //
 212         return UnsupportedExit (Ghcb, Regs, InstructionData);
 213       }
 214
 215       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 216       if (Status != 0) {
 217         return Status;
 218       }
 219
 220       ExitInfo1 = InstructionData->Ext.RmData;
 221       ExitInfo2 = Bytes;
 222       CopyMem (Ghcb->SharedBuffer, &InstructionData->Ext.RegData, Bytes);
 223
 224       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 225       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 226       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 227       if (Status != 0) {
 228         return Status;
 229       }
 230
 231       break;
 232
 233     //
 234     // MMIO write (MOV moffsetX, aX)
 235     //
 236     case 0xA2:
 237       Bytes = 1;
 238     //
 239     // fall through
 240     //
 241     case 0xA3:
 242       Bytes = ((Bytes != 0) ? Bytes :
 243                (InstructionData->DataSize == Size16Bits) ? 2 :
 244                (InstructionData->DataSize == Size32Bits) ? 4 :
 245                (InstructionData->DataSize == Size64Bits) ? 8 :
 246                0);
 247
 248       InstructionData->ImmediateSize = (UINTN)(1 << InstructionData->AddrSize);
 249       InstructionData->End          += InstructionData->ImmediateSize;
 250
 251       //
 252       // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
 253       // Use a STATIC_ASSERT to be certain the code is being built as X64.
 254       //
 255       STATIC_ASSERT (
 256         sizeof (UINTN) == sizeof (UINT64),
 257         "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
 258         );
 259
 260       Address = 0;
 261       CopyMem (
 262         &Address,
 263         InstructionData->Immediate,
 264         InstructionData->ImmediateSize
 265         );
 266
 267       Status = ValidateMmioMemory (Ghcb, Address, Bytes);
 268       if (Status != 0) {
 269         return Status;
 270       }
 271
 272       ExitInfo1 = Address;
 273       ExitInfo2 = Bytes;
 274       CopyMem (Ghcb->SharedBuffer, &Regs->Rax, Bytes);
 275
 276       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 277       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 278       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 279       if (Status != 0) {
 280         return Status;
 281       }
 282
 283       break;
 284
 285     //
 286     // MMIO write (MOV reg/memX, immX)
 287     //
 288     case 0xC6:
 289       Bytes = 1;
 290     //
 291     // fall through
 292     //
 293     case 0xC7:
 294       CcDecodeModRm (Regs, InstructionData);
 295       Bytes = ((Bytes != 0) ? Bytes :
 296                (InstructionData->DataSize == Size16Bits) ? 2 :
 297                (InstructionData->DataSize == Size32Bits) ? 4 :
 298                0);
 299
 300       InstructionData->ImmediateSize = Bytes;
 301       InstructionData->End          += Bytes;
 302
 303       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 304       if (Status != 0) {
 305         return Status;
 306       }
 307
 308       ExitInfo1 = InstructionData->Ext.RmData;
 309       ExitInfo2 = Bytes;
 310       CopyMem (Ghcb->SharedBuffer, InstructionData->Immediate, Bytes);
 311
 312       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 313       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 314       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 315       if (Status != 0) {
 316         return Status;
 317       }
 318
 319       break;
 320
 321     //
 322     // MMIO read (MOV regX, reg/memX)
 323     //
 324     case 0x8A:
 325       Bytes = 1;
 326     //
 327     // fall through
 328     //
 329     case 0x8B:
 330       CcDecodeModRm (Regs, InstructionData);
 331       Bytes = ((Bytes != 0) ? Bytes :
 332                (InstructionData->DataSize == Size16Bits) ? 2 :
 333                (InstructionData->DataSize == Size32Bits) ? 4 :
 334                (InstructionData->DataSize == Size64Bits) ? 8 :
 335                0);
 336       if (InstructionData->Ext.ModRm.Mod == 3) {
 337         //
 338         // NPF on two register operands???
 339         //
 340         return UnsupportedExit (Ghcb, Regs, InstructionData);
 341       }
 342
 343       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 344       if (Status != 0) {
 345         return Status;
 346       }
 347
 348       ExitInfo1 = InstructionData->Ext.RmData;
 349       ExitInfo2 = Bytes;
 350
 351       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 352       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 353       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 354       if (Status != 0) {
 355         return Status;
 356       }
 357
 358       Register = CcGetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 359       if (Bytes == 4) {
 360         //
 361         // Zero-extend for 32-bit operation
 362         //
 363         *Register = 0;
 364       }
 365
 366       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 367       break;
 368
 369     //
 370     // MMIO read (MOV aX, moffsetX)
 371     //
 372     case 0xA0:
 373       Bytes = 1;
 374     //
 375     // fall through
 376     //
 377     case 0xA1:
 378       Bytes = ((Bytes != 0) ? Bytes :
 379                (InstructionData->DataSize == Size16Bits) ? 2 :
 380                (InstructionData->DataSize == Size32Bits) ? 4 :
 381                (InstructionData->DataSize == Size64Bits) ? 8 :
 382                0);
 383
 384       InstructionData->ImmediateSize = (UINTN)(1 << InstructionData->AddrSize);
 385       InstructionData->End          += InstructionData->ImmediateSize;
 386
 387       //
 388       // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
 389       // Use a STATIC_ASSERT to be certain the code is being built as X64.
 390       //
 391       STATIC_ASSERT (
 392         sizeof (UINTN) == sizeof (UINT64),
 393         "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
 394         );
 395
 396       Address = 0;
 397       CopyMem (
 398         &Address,
 399         InstructionData->Immediate,
 400         InstructionData->ImmediateSize
 401         );
 402
 403       Status = ValidateMmioMemory (Ghcb, Address, Bytes);
 404       if (Status != 0) {
 405         return Status;
 406       }
 407
 408       ExitInfo1 = Address;
 409       ExitInfo2 = Bytes;
 410
 411       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 412       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 413       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 414       if (Status != 0) {
 415         return Status;
 416       }
 417
 418       if (Bytes == 4) {
 419         //
 420         // Zero-extend for 32-bit operation
 421         //
 422         Regs->Rax = 0;
 423       }
 424
 425       CopyMem (&Regs->Rax, Ghcb->SharedBuffer, Bytes);
 426       break;
 427
 428     //
 429     // MMIO read w/ zero-extension ((MOVZX regX, reg/memX)
 430     //
 431     case 0xB6:
 432       Bytes = 1;
 433     //
 434     // fall through
 435     //
 436     case 0xB7:
 437       CcDecodeModRm (Regs, InstructionData);
 438       Bytes = (Bytes != 0) ? Bytes : 2;
 439
 440       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 441       if (Status != 0) {
 442         return Status;
 443       }
 444
 445       ExitInfo1 = InstructionData->Ext.RmData;
 446       ExitInfo2 = Bytes;
 447
 448       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 449       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 450       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 451       if (Status != 0) {
 452         return Status;
 453       }
 454
 455       Register = CcGetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 456       SetMem (Register, (UINTN)(1 << InstructionData->DataSize), 0);
 457       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 458       break;
 459
 460     //
 461     // MMIO read w/ sign-extension (MOVSX regX, reg/memX)
 462     //
 463     case 0xBE:
 464       Bytes = 1;
 465     //
 466     // fall through
 467     //
 468     case 0xBF:
 469       CcDecodeModRm (Regs, InstructionData);
 470       Bytes = (Bytes != 0) ? Bytes : 2;
 471
 472       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 473       if (Status != 0) {
 474         return Status;
 475       }
 476
 477       ExitInfo1 = InstructionData->Ext.RmData;
 478       ExitInfo2 = Bytes;
 479
 480       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 481       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 482       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 483       if (Status != 0) {
 484         return Status;
 485       }
 486
 487       if (Bytes == 1) {
 488         UINT8  *Data;
 489
 490         Data     = (UINT8 *)Ghcb->SharedBuffer;
 491         SignByte = ((*Data & BIT7) != 0) ? 0xFF : 0x00;
 492       } else {
 493         UINT16  *Data;
 494
 495         Data     = (UINT16 *)Ghcb->SharedBuffer;
 496         SignByte = ((*Data & BIT15) != 0) ? 0xFF : 0x00;
 497       }
 498
 499       Register = CcGetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 500       SetMem (Register, (UINTN)(1 << InstructionData->DataSize), SignByte);
 501       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 502       break;
 503
 504     default:
 505       DEBUG ((DEBUG_ERROR, "Invalid MMIO opcode (%x)\n", OpCode));
 506       Status = GP_EXCEPTION;
 507       ASSERT (FALSE);
 508   }
 509
 510   return Status;
 511 }
 512
 513 /**
 514   Handle a MWAIT event.
 515
 516   Use the VMGEXIT instruction to handle a MWAIT event.
 517
 518   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 519                                    Block
 520   @param[in, out] Regs             x64 processor context
 521   @param[in]      InstructionData  Instruction parsing context
 522
 523   @retval 0                        Event handled successfully
 524   @return                          New exception value to propagate
 525
 526 **/
 527 STATIC
 528 UINT64
 529 MwaitExit (
 530   IN OUT GHCB                    *Ghcb,
 531   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
 532   IN     CC_INSTRUCTION_DATA     *InstructionData
 533   )
 534 {
 535   CcDecodeModRm (Regs, InstructionData);
 536
 537   Ghcb->SaveArea.Rax = Regs->Rax;
 538   CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
 539   Ghcb->SaveArea.Rcx = Regs->Rcx;
 540   CcExitVmgSetOffsetValid (Ghcb, GhcbRcx);
 541
 542   return CcExitVmgExit (Ghcb, SVM_EXIT_MWAIT, 0, 0);
 543 }
 544
 545 /**
 546   Handle a MONITOR event.
 547
 548   Use the VMGEXIT instruction to handle a MONITOR event.
 549
 550   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 551                                    Block
 552   @param[in, out] Regs             x64 processor context
 553   @param[in]      InstructionData  Instruction parsing context
 554
 555   @retval 0                        Event handled successfully
 556   @return                          New exception value to propagate
 557
 558 **/
 559 STATIC
 560 UINT64
 561 MonitorExit (
 562   IN OUT GHCB                    *Ghcb,
 563   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
 564   IN     CC_INSTRUCTION_DATA     *InstructionData
 565   )
 566 {
 567   CcDecodeModRm (Regs, InstructionData);
 568
 569   Ghcb->SaveArea.Rax = Regs->Rax;  // Identity mapped, so VA = PA
 570   CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
 571   Ghcb->SaveArea.Rcx = Regs->Rcx;
 572   CcExitVmgSetOffsetValid (Ghcb, GhcbRcx);
 573   Ghcb->SaveArea.Rdx = Regs->Rdx;
 574   CcExitVmgSetOffsetValid (Ghcb, GhcbRdx);
 575
 576   return CcExitVmgExit (Ghcb, SVM_EXIT_MONITOR, 0, 0);
 577 }
 578
 579 /**
 580   Handle a WBINVD event.
 581
 582   Use the VMGEXIT instruction to handle a WBINVD event.
 583
 584   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 585                                    Block
 586   @param[in, out] Regs             x64 processor context
 587   @param[in]      InstructionData  Instruction parsing context
 588
 589   @retval 0                        Event handled successfully
 590   @return                          New exception value to propagate
 591
 592 **/
 593 STATIC
 594 UINT64
 595 WbinvdExit (
 596   IN OUT GHCB                    *Ghcb,
 597   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
 598   IN     CC_INSTRUCTION_DATA     *InstructionData
 599   )
 600 {
 601   return CcExitVmgExit (Ghcb, SVM_EXIT_WBINVD, 0, 0);
 602 }
 603
 604 /**
 605   Handle a RDTSCP event.
 606
 607   Use the VMGEXIT instruction to handle a RDTSCP event.
 608
 609   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 610                                    Block
 611   @param[in, out] Regs             x64 processor context
 612   @param[in]      InstructionData  Instruction parsing context
 613
 614   @retval 0                        Event handled successfully
 615   @return                          New exception value to propagate
 616
 617 **/
 618 STATIC
 619 UINT64
 620 RdtscpExit (
 621   IN OUT GHCB                    *Ghcb,
 622   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
 623   IN     CC_INSTRUCTION_DATA     *InstructionData
 624   )
 625 {
 626   UINT64  Status;
 627
 628   CcDecodeModRm (Regs, InstructionData);
 629
 630   Status = CcExitVmgExit (Ghcb, SVM_EXIT_RDTSCP, 0, 0);
 631   if (Status != 0) {
 632     return Status;
 633   }
 634
 635   if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax) ||
 636       !CcExitVmgIsOffsetValid (Ghcb, GhcbRcx) ||
 637       !CcExitVmgIsOffsetValid (Ghcb, GhcbRdx))
 638   {
 639     return UnsupportedExit (Ghcb, Regs, InstructionData);
 640   }
 641
 642   Regs->Rax = Ghcb->SaveArea.Rax;
 643   Regs->Rcx = Ghcb->SaveArea.Rcx;
 644   Regs->Rdx = Ghcb->SaveArea.Rdx;
 645
 646   return 0;
 647 }
 648
 649 /**
 650   Handle a VMMCALL event.
 651
 652   Use the VMGEXIT instruction to handle a VMMCALL event.
 653
 654   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 655                                    Block
 656   @param[in, out] Regs             x64 processor context
 657   @param[in]      InstructionData  Instruction parsing context
 658
 659   @retval 0                        Event handled successfully
 660   @return                          New exception value to propagate
 661
 662 **/
 663 STATIC
 664 UINT64
 665 VmmCallExit (
 666   IN OUT GHCB                    *Ghcb,
 667   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
 668   IN     CC_INSTRUCTION_DATA     *InstructionData
 669   )
 670 {
 671   UINT64  Status;
 672
 673   CcDecodeModRm (Regs, InstructionData);
 674
 675   Ghcb->SaveArea.Rax = Regs->Rax;
 676   CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
 677   Ghcb->SaveArea.Cpl = (UINT8)(Regs->Cs & 0x3);
 678   CcExitVmgSetOffsetValid (Ghcb, GhcbCpl);
 679
 680   Status = CcExitVmgExit (Ghcb, SVM_EXIT_VMMCALL, 0, 0);
 681   if (Status != 0) {
 682     return Status;
 683   }
 684
 685   if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax)) {
 686     return UnsupportedExit (Ghcb, Regs, InstructionData);
 687   }
 688
 689   Regs->Rax = Ghcb->SaveArea.Rax;
 690
 691   return 0;
 692 }
 693
 694 /**
 695   Handle an MSR event.
 696
 697   Use the VMGEXIT instruction to handle either a RDMSR or WRMSR event.
 698
 699   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 700                                    Block
 701   @param[in, out] Regs             x64 processor context
 702   @param[in]      InstructionData  Instruction parsing context
 703
 704   @retval 0                        Event handled successfully
 705   @return                          New exception value to propagate
 706
 707 **/
 708 STATIC
 709 UINT64
 710 MsrExit (
 711   IN OUT GHCB                    *Ghcb,
 712   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
 713   IN     CC_INSTRUCTION_DATA     *InstructionData
 714   )
 715 {
 716   UINT64  ExitInfo1, Status;
 717
 718   ExitInfo1 = 0;
 719
 720   switch (*(InstructionData->OpCodes + 1)) {
 721     case 0x30: // WRMSR
 722       ExitInfo1          = 1;
 723       Ghcb->SaveArea.Rax = Regs->Rax;
 724       CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
 725       Ghcb->SaveArea.Rdx = Regs->Rdx;
 726       CcExitVmgSetOffsetValid (Ghcb, GhcbRdx);
 727     //
 728     // fall through
 729     //
 730     case 0x32: // RDMSR
 731       Ghcb->SaveArea.Rcx = Regs->Rcx;
 732       CcExitVmgSetOffsetValid (Ghcb, GhcbRcx);
 733       break;
 734     default:
 735       return UnsupportedExit (Ghcb, Regs, InstructionData);
 736   }
 737
 738   Status = CcExitVmgExit (Ghcb, SVM_EXIT_MSR, ExitInfo1, 0);
 739   if (Status != 0) {
 740     return Status;
 741   }
 742
 743   if (ExitInfo1 == 0) {
 744     if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax) ||
 745         !CcExitVmgIsOffsetValid (Ghcb, GhcbRdx))
 746     {
 747       return UnsupportedExit (Ghcb, Regs, InstructionData);
 748     }
 749
 750     Regs->Rax = Ghcb->SaveArea.Rax;
 751     Regs->Rdx = Ghcb->SaveArea.Rdx;
 752   }
 753
 754   return 0;
 755 }
 756
 757 /**
 758   Build the IOIO event information.
 759
 760   The IOIO event information identifies the type of IO operation to be performed
 761   by the hypervisor. Build this information based on the instruction data.
 762
 763   @param[in]       Regs             x64 processor context
 764   @param[in, out]  InstructionData  Instruction parsing context
 765
 766   @return                           IOIO event information value
 767
 768 **/
 769 STATIC
 770 UINT64
 771 IoioExitInfo (
 772   IN     EFI_SYSTEM_CONTEXT_X64  *Regs,
 773   IN OUT CC_INSTRUCTION_DATA     *InstructionData
 774   )
 775 {
 776   UINT64  ExitInfo;
 777
 778   ExitInfo = 0;
 779
 780   switch (*(InstructionData->OpCodes)) {
 781     //
 782     // INS opcodes
 783     //
 784     case 0x6C:
 785     case 0x6D:
 786       ExitInfo |= IOIO_TYPE_INS;
 787       ExitInfo |= IOIO_SEG_ES;
 788       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
 789       break;
 790
 791     //
 792     // OUTS opcodes
 793     //
 794     case 0x6E:
 795     case 0x6F:
 796       ExitInfo |= IOIO_TYPE_OUTS;
 797       ExitInfo |= IOIO_SEG_DS;
 798       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
 799       break;
 800
 801     //
 802     // IN immediate opcodes
 803     //
 804     case 0xE4:
 805     case 0xE5:
 806       InstructionData->ImmediateSize = 1;
 807       InstructionData->End++;
 808       ExitInfo |= IOIO_TYPE_IN;
 809       ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16);
 810       break;
 811
 812     //
 813     // OUT immediate opcodes
 814     //
 815     case 0xE6:
 816     case 0xE7:
 817       InstructionData->ImmediateSize = 1;
 818       InstructionData->End++;
 819       ExitInfo |= IOIO_TYPE_OUT;
 820       ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16) | IOIO_TYPE_OUT;
 821       break;
 822
 823     //
 824     // IN register opcodes
 825     //
 826     case 0xEC:
 827     case 0xED:
 828       ExitInfo |= IOIO_TYPE_IN;
 829       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
 830       break;
 831
 832     //
 833     // OUT register opcodes
 834     //
 835     case 0xEE:
 836     case 0xEF:
 837       ExitInfo |= IOIO_TYPE_OUT;
 838       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
 839       break;
 840
 841     default:
 842       return 0;
 843   }
 844
 845   switch (*(InstructionData->OpCodes)) {
 846     //
 847     // Single-byte opcodes
 848     //
 849     case 0x6C:
 850     case 0x6E:
 851     case 0xE4:
 852     case 0xE6:
 853     case 0xEC:
 854     case 0xEE:
 855       ExitInfo |= IOIO_DATA_8;
 856       break;
 857
 858     //
 859     // Length determined by instruction parsing
 860     //
 861     default:
 862       ExitInfo |= (InstructionData->DataSize == Size16Bits) ? IOIO_DATA_16
 863                                                           : IOIO_DATA_32;
 864   }
 865
 866   switch (InstructionData->AddrSize) {
 867     case Size16Bits:
 868       ExitInfo |= IOIO_ADDR_16;
 869       break;
 870
 871     case Size32Bits:
 872       ExitInfo |= IOIO_ADDR_32;
 873       break;
 874
 875     case Size64Bits:
 876       ExitInfo |= IOIO_ADDR_64;
 877       break;
 878
 879     default:
 880       break;
 881   }
 882
 883   if (InstructionData->RepMode != 0) {
 884     ExitInfo |= IOIO_REP;
 885   }
 886
 887   return ExitInfo;
 888 }
 889
 890 /**
 891   Handle an IOIO event.
 892
 893   Use the VMGEXIT instruction to handle an IOIO event.
 894
 895   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 896                                    Block
 897   @param[in, out] Regs             x64 processor context
 898   @param[in]      InstructionData  Instruction parsing context
 899
 900   @retval 0                        Event handled successfully
 901   @return                          New exception value to propagate
 902
 903 **/
 904 STATIC
 905 UINT64
 906 IoioExit (
 907   IN OUT GHCB                    *Ghcb,
 908   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
 909   IN     CC_INSTRUCTION_DATA     *InstructionData
 910   )
 911 {
 912   UINT64   ExitInfo1, ExitInfo2, Status;
 913   BOOLEAN  IsString;
 914
 915   ExitInfo1 = IoioExitInfo (Regs, InstructionData);
 916   if (ExitInfo1 == 0) {
 917     return UnsupportedExit (Ghcb, Regs, InstructionData);
 918   }
 919
 920   IsString = ((ExitInfo1 & IOIO_TYPE_STR) != 0) ? TRUE : FALSE;
 921   if (IsString) {
 922     UINTN  IoBytes, VmgExitBytes;
 923     UINTN  GhcbCount, OpCount;
 924
 925     Status = 0;
 926
 927     IoBytes   = IOIO_DATA_BYTES (ExitInfo1);
 928     GhcbCount = sizeof (Ghcb->SharedBuffer) / IoBytes;
 929
 930     OpCount = ((ExitInfo1 & IOIO_REP) != 0) ? Regs->Rcx : 1;
 931     while (OpCount != 0) {
 932       ExitInfo2    = MIN (OpCount, GhcbCount);
 933       VmgExitBytes = ExitInfo2 * IoBytes;
 934
 935       if ((ExitInfo1 & IOIO_TYPE_IN) == 0) {
 936         CopyMem (Ghcb->SharedBuffer, (VOID *)Regs->Rsi, VmgExitBytes);
 937         Regs->Rsi += VmgExitBytes;
 938       }
 939
 940       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 941       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 942       Status = CcExitVmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, ExitInfo2);
 943       if (Status != 0) {
 944         return Status;
 945       }
 946
 947       if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
 948         CopyMem ((VOID *)Regs->Rdi, Ghcb->SharedBuffer, VmgExitBytes);
 949         Regs->Rdi += VmgExitBytes;
 950       }
 951
 952       if ((ExitInfo1 & IOIO_REP) != 0) {
 953         Regs->Rcx -= ExitInfo2;
 954       }
 955
 956       OpCount -= ExitInfo2;
 957     }
 958   } else {
 959     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
 960       Ghcb->SaveArea.Rax = 0;
 961     } else {
 962       CopyMem (&Ghcb->SaveArea.Rax, &Regs->Rax, IOIO_DATA_BYTES (ExitInfo1));
 963     }
 964
 965     CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
 966
 967     Status = CcExitVmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, 0);
 968     if (Status != 0) {
 969       return Status;
 970     }
 971
 972     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
 973       if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax)) {
 974         return UnsupportedExit (Ghcb, Regs, InstructionData);
 975       }
 976
 977       CopyMem (&Regs->Rax, &Ghcb->SaveArea.Rax, IOIO_DATA_BYTES (ExitInfo1));
 978     }
 979   }
 980
 981   return 0;
 982 }
 983
 984 /**
 985   Handle a INVD event.
 986
 987   Use the VMGEXIT instruction to handle a INVD event.
 988
 989   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 990                                    Block
 991   @param[in, out] Regs             x64 processor context
 992   @param[in]      InstructionData  Instruction parsing context
 993
 994   @retval 0                        Event handled successfully
 995   @return                          New exception value to propagate
 996
 997 **/
 998 STATIC
 999 UINT64
1000 InvdExit (
1001   IN OUT GHCB                    *Ghcb,
1002   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
1003   IN     CC_INSTRUCTION_DATA     *InstructionData
1004   )
1005 {
1006   return CcExitVmgExit (Ghcb, SVM_EXIT_INVD, 0, 0);
1007 }
1008
1009 /**
1010   Fetch CPUID leaf/function via hypervisor/VMGEXIT.
1011
1012   @param[in, out] Ghcb         Pointer to the Guest-Hypervisor Communication
1013                                Block
1014   @param[in]      EaxIn        EAX input for cpuid instruction
1015   @param[in]      EcxIn        ECX input for cpuid instruction
1016   @param[in]      Xcr0In       XCR0 at time of cpuid instruction
1017   @param[in, out] Eax          Pointer to store leaf's EAX value
1018   @param[in, out] Ebx          Pointer to store leaf's EBX value
1019   @param[in, out] Ecx          Pointer to store leaf's ECX value
1020   @param[in, out] Edx          Pointer to store leaf's EDX value
1021   @param[in, out] Status       Pointer to store status from VMGEXIT (always 0
1022                                unless return value indicates failure)
1023   @param[in, out] Unsupported  Pointer to store indication of unsupported
1024                                VMGEXIT (always false unless return value
1025                                indicates failure)
1026
1027   @retval TRUE                 CPUID leaf fetch successfully.
1028   @retval FALSE                Error occurred while fetching CPUID leaf. Callers
1029                                should Status and Unsupported and handle
1030                                accordingly if they indicate a more precise
1031                                error condition.
1032
1033 **/
1034 STATIC
1035 BOOLEAN
1036 GetCpuidHyp (
1037   IN OUT GHCB     *Ghcb,
1038   IN     UINT32   EaxIn,
1039   IN     UINT32   EcxIn,
1040   IN     UINT64   XCr0,
1041   IN OUT UINT32   *Eax,
1042   IN OUT UINT32   *Ebx,
1043   IN OUT UINT32   *Ecx,
1044   IN OUT UINT32   *Edx,
1045   IN OUT UINT64   *Status,
1046   IN OUT BOOLEAN  *UnsupportedExit
1047   )
1048 {
1049   *UnsupportedExit   = FALSE;
1050   Ghcb->SaveArea.Rax = EaxIn;
1051   CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
1052   Ghcb->SaveArea.Rcx = EcxIn;
1053   CcExitVmgSetOffsetValid (Ghcb, GhcbRcx);
1054   if (EaxIn == CPUID_EXTENDED_STATE) {
1055     Ghcb->SaveArea.XCr0 = XCr0;
1056     CcExitVmgSetOffsetValid (Ghcb, GhcbXCr0);
1057   }
1058
1059   *Status = CcExitVmgExit (Ghcb, SVM_EXIT_CPUID, 0, 0);
1060   if (*Status != 0) {
1061     return FALSE;
1062   }
1063
1064   if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax) ||
1065       !CcExitVmgIsOffsetValid (Ghcb, GhcbRbx) ||
1066       !CcExitVmgIsOffsetValid (Ghcb, GhcbRcx) ||
1067       !CcExitVmgIsOffsetValid (Ghcb, GhcbRdx))
1068   {
1069     *UnsupportedExit = TRUE;
1070     return FALSE;
1071   }
1072
1073   if (Eax) {
1074     *Eax = (UINT32)(UINTN)Ghcb->SaveArea.Rax;
1075   }
1076
1077   if (Ebx) {
1078     *Ebx = (UINT32)(UINTN)Ghcb->SaveArea.Rbx;
1079   }
1080
1081   if (Ecx) {
1082     *Ecx = (UINT32)(UINTN)Ghcb->SaveArea.Rcx;
1083   }
1084
1085   if (Edx) {
1086     *Edx = (UINT32)(UINTN)Ghcb->SaveArea.Rdx;
1087   }
1088
1089   return TRUE;
1090 }
1091
1092 /**
1093   Check if SEV-SNP enabled.
1094
1095   @retval TRUE      SEV-SNP is enabled.
1096   @retval FALSE     SEV-SNP is disabled.
1097
1098 **/
1099 STATIC
1100 BOOLEAN
1101 SnpEnabled (
1102   VOID
1103   )
1104 {
1105   MSR_SEV_STATUS_REGISTER  Msr;
1106
1107   Msr.Uint32 = AsmReadMsr32 (MSR_SEV_STATUS);
1108
1109   return !!Msr.Bits.SevSnpBit;
1110 }
1111
1112 /**
1113   Calculate the total XSAVE area size for enabled XSAVE areas
1114
1115   @param[in]      XFeaturesEnabled  Bit-mask of enabled XSAVE features/areas as
1116                                     indicated by XCR0/MSR_IA32_XSS bits
1117   @param[in]      XSaveBaseSize     Base/legacy XSAVE area size (e.g. when
1118                                     XCR0 is 1)
1119   @param[in, out] XSaveSize         Pointer to storage for calculated XSAVE area
1120                                     size
1121   @param[in]      Compacted         Whether or not the calculation is for the
1122                                     normal XSAVE area size (leaf 0xD,0x0,EBX) or
1123                                     compacted XSAVE area size (leaf 0xD,0x1,EBX)
1124
1125
1126   @retval TRUE                      XSAVE size calculation was successful.
1127   @retval FALSE                     XSAVE size calculation was unsuccessful.
1128 **/
1129 STATIC
1130 BOOLEAN
1131 GetCpuidXSaveSize (
1132   IN     UINT64   XFeaturesEnabled,
1133   IN     UINT32   XSaveBaseSize,
1134   IN OUT UINT32   *XSaveSize,
1135   IN     BOOLEAN  Compacted
1136   )
1137 {
1138   SEV_SNP_CPUID_INFO  *CpuidInfo;
1139   UINT64              XFeaturesFound = 0;
1140   UINT32              Idx;
1141
1142   *XSaveSize = XSaveBaseSize;
1143   CpuidInfo  = (SEV_SNP_CPUID_INFO *)(UINT64)PcdGet32 (PcdOvmfCpuidBase);
1144
1145   for (Idx = 0; Idx < CpuidInfo->Count; Idx++) {
1146     SEV_SNP_CPUID_FUNCTION  *CpuidFn = &CpuidInfo->function[Idx];
1147
1148     if (!((CpuidFn->EaxIn == 0xD) &&
1149           ((CpuidFn->EcxIn == 0) || (CpuidFn->EcxIn == 1))))
1150     {
1151       continue;
1152     }
1153
1154     if (XFeaturesFound & (1ULL << CpuidFn->EcxIn) ||
1155         !(XFeaturesEnabled & (1ULL << CpuidFn->EcxIn)))
1156     {
1157       continue;
1158     }
1159
1160     XFeaturesFound |= (1ULL << CpuidFn->EcxIn);
1161     if (Compacted) {
1162       *XSaveSize += CpuidFn->Eax;
1163     } else {
1164       *XSaveSize = MAX (*XSaveSize, CpuidFn->Eax + CpuidFn->Ebx);
1165     }
1166   }
1167
1168   /*
1169    * Either the guest set unsupported XCR0/XSS bits, or the corresponding
1170    * entries in the CPUID table were not present. This is an invalid state.
1171    */
1172   if (XFeaturesFound != (XFeaturesEnabled & ~3UL)) {
1173     return FALSE;
1174   }
1175
1176   return TRUE;
1177 }
1178
1179 /**
1180   Check if a CPUID leaf/function is indexed via ECX sub-leaf/sub-function
1181
1182   @param[in]      EaxIn        EAX input for cpuid instruction
1183
1184   @retval FALSE                cpuid leaf/function is not indexed by ECX input
1185   @retval TRUE                 cpuid leaf/function is indexed by ECX input
1186
1187 **/
1188 STATIC
1189 BOOLEAN
1190 IsFunctionIndexed (
1191   IN     UINT32  EaxIn
1192   )
1193 {
1194   switch (EaxIn) {
1195     case CPUID_CACHE_PARAMS:
1196     case CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS:
1197     case CPUID_EXTENDED_TOPOLOGY:
1198     case CPUID_EXTENDED_STATE:
1199     case CPUID_INTEL_RDT_MONITORING:
1200     case CPUID_INTEL_RDT_ALLOCATION:
1201     case CPUID_INTEL_SGX:
1202     case CPUID_INTEL_PROCESSOR_TRACE:
1203     case CPUID_DETERMINISTIC_ADDRESS_TRANSLATION_PARAMETERS:
1204     case CPUID_V2_EXTENDED_TOPOLOGY:
1205     case 0x8000001D: /* Cache Topology Information */
1206       return TRUE;
1207   }
1208
1209   return FALSE;
1210 }
1211
1212 /**
1213   Fetch CPUID leaf/function via SEV-SNP CPUID table.
1214
1215   @param[in, out] Ghcb         Pointer to the Guest-Hypervisor Communication
1216                                Block
1217   @param[in]      EaxIn        EAX input for cpuid instruction
1218   @param[in]      EcxIn        ECX input for cpuid instruction
1219   @param[in]      Xcr0In       XCR0 at time of cpuid instruction
1220   @param[in, out] Eax          Pointer to store leaf's EAX value
1221   @param[in, out] Ebx          Pointer to store leaf's EBX value
1222   @param[in, out] Ecx          Pointer to store leaf's ECX value
1223   @param[in, out] Edx          Pointer to store leaf's EDX value
1224   @param[in, out] Status       Pointer to store status from VMGEXIT (always 0
1225                                unless return value indicates failure)
1226   @param[in, out] Unsupported  Pointer to store indication of unsupported
1227                                VMGEXIT (always false unless return value
1228                                indicates failure)
1229
1230   @retval TRUE                 CPUID leaf fetch successfully.
1231   @retval FALSE                Error occurred while fetching CPUID leaf. Callers
1232                                should Status and Unsupported and handle
1233                                accordingly if they indicate a more precise
1234                                error condition.
1235
1236 **/
1237 STATIC
1238 BOOLEAN
1239 GetCpuidFw (
1240   IN OUT GHCB     *Ghcb,
1241   IN     UINT32   EaxIn,
1242   IN     UINT32   EcxIn,
1243   IN     UINT64   XCr0,
1244   IN OUT UINT32   *Eax,
1245   IN OUT UINT32   *Ebx,
1246   IN OUT UINT32   *Ecx,
1247   IN OUT UINT32   *Edx,
1248   IN OUT UINT64   *Status,
1249   IN OUT BOOLEAN  *Unsupported
1250   )
1251 {
1252   SEV_SNP_CPUID_INFO  *CpuidInfo;
1253   BOOLEAN             Found;
1254   UINT32              Idx;
1255
1256   CpuidInfo = (SEV_SNP_CPUID_INFO *)(UINT64)PcdGet32 (PcdOvmfCpuidBase);
1257   Found     = FALSE;
1258
1259   for (Idx = 0; Idx < CpuidInfo->Count; Idx++) {
1260     SEV_SNP_CPUID_FUNCTION  *CpuidFn = &CpuidInfo->function[Idx];
1261
1262     if (CpuidFn->EaxIn != EaxIn) {
1263       continue;
1264     }
1265
1266     if (IsFunctionIndexed (CpuidFn->EaxIn) && (CpuidFn->EcxIn != EcxIn)) {
1267       continue;
1268     }
1269
1270     *Eax = CpuidFn->Eax;
1271     *Ebx = CpuidFn->Ebx;
1272     *Ecx = CpuidFn->Ecx;
1273     *Edx = CpuidFn->Edx;
1274
1275     Found = TRUE;
1276     break;
1277   }
1278
1279   if (!Found) {
1280     *Eax = *Ebx = *Ecx = *Edx = 0;
1281     goto Out;
1282   }
1283
1284   if (EaxIn == CPUID_VERSION_INFO) {
1285     IA32_CR4  Cr4;
1286     UINT32    Ebx2;
1287     UINT32    Edx2;
1288
1289     if (!GetCpuidHyp (
1290            Ghcb,
1291            EaxIn,
1292            EcxIn,
1293            XCr0,
1294            NULL,
1295            &Ebx2,
1296            NULL,
1297            &Edx2,
1298            Status,
1299            Unsupported
1300            ))
1301     {
1302       return FALSE;
1303     }
1304
1305     /* initial APIC ID */
1306     *Ebx = (*Ebx & 0x00FFFFFF) | (Ebx2 & 0xFF000000);
1307     /* APIC enabled bit */
1308     *Edx = (*Edx & ~BIT9) | (Edx2 & BIT9);
1309     /* OSXSAVE enabled bit */
1310     Cr4.UintN = AsmReadCr4 ();
1311     *Ecx      = (Cr4.Bits.OSXSAVE) ? (*Ecx & ~BIT27) | (*Ecx & BIT27)
1312                               : (*Ecx & ~BIT27);
1313   } else if (EaxIn == CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS) {
1314     IA32_CR4  Cr4;
1315
1316     Cr4.UintN = AsmReadCr4 ();
1317     /* OSPKE enabled bit */
1318     *Ecx = (Cr4.Bits.PKE) ? (*Ecx | BIT4) : (*Ecx & ~BIT4);
1319   } else if (EaxIn == CPUID_EXTENDED_TOPOLOGY) {
1320     if (!GetCpuidHyp (
1321            Ghcb,
1322            EaxIn,
1323            EcxIn,
1324            XCr0,
1325            NULL,
1326            NULL,
1327            NULL,
1328            Edx,
1329            Status,
1330            Unsupported
1331            ))
1332     {
1333       return FALSE;
1334     }
1335   } else if ((EaxIn == CPUID_EXTENDED_STATE) && ((EcxIn == 0) || (EcxIn == 1))) {
1336     MSR_IA32_XSS_REGISTER  XssMsr;
1337     BOOLEAN                Compacted;
1338     UINT32                 XSaveSize;
1339
1340     XssMsr.Uint64 = 0;
1341     Compacted     = FALSE;
1342     if (EcxIn == 1) {
1343       /*
1344        * The PPR and APM aren't clear on what size should be encoded in
1345        * 0xD:0x1:EBX when compaction is not enabled by either XSAVEC or
1346        * XSAVES, as these are generally fixed to 1 on real CPUs. Report
1347        * this undefined case as an error.
1348        */
1349       if (!(*Eax & (BIT3 | BIT1))) {
1350         /* (XSAVES | XSAVEC) */
1351         return FALSE;
1352       }
1353
1354       Compacted     = TRUE;
1355       XssMsr.Uint64 = AsmReadMsr64 (MSR_IA32_XSS);
1356     }
1357
1358     if (!GetCpuidXSaveSize (
1359            XCr0 | XssMsr.Uint64,
1360            *Ebx,
1361            &XSaveSize,
1362            Compacted
1363            ))
1364     {
1365       return FALSE;
1366     }
1367
1368     *Ebx = XSaveSize;
1369   } else if (EaxIn == 0x8000001E) {
1370     UINT32  Ebx2;
1371     UINT32  Ecx2;
1372
1373     /* extended APIC ID */
1374     if (!GetCpuidHyp (
1375            Ghcb,
1376            EaxIn,
1377            EcxIn,
1378            XCr0,
1379            Eax,
1380            &Ebx2,
1381            &Ecx2,
1382            NULL,
1383            Status,
1384            Unsupported
1385            ))
1386     {
1387       return FALSE;
1388     }
1389
1390     /* compute ID */
1391     *Ebx = (*Ebx & 0xFFFFFF00) | (Ebx2 & 0x000000FF);
1392     /* node ID */
1393     *Ecx = (*Ecx & 0xFFFFFF00) | (Ecx2 & 0x000000FF);
1394   }
1395
1396 Out:
1397   *Status      = 0;
1398   *Unsupported = FALSE;
1399   return TRUE;
1400 }
1401
1402 /**
1403   Handle a CPUID event.
1404
1405   Use VMGEXIT instruction or CPUID table to handle a CPUID event.
1406
1407   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1408                                    Block
1409   @param[in, out] Regs             x64 processor context
1410   @param[in]      InstructionData  Instruction parsing context
1411
1412   @retval 0                        Event handled successfully
1413   @return                          New exception value to propagate
1414
1415 **/
1416 STATIC
1417 UINT64
1418 CpuidExit (
1419   IN OUT GHCB                    *Ghcb,
1420   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
1421   IN     CC_INSTRUCTION_DATA     *InstructionData
1422   )
1423 {
1424   BOOLEAN  Unsupported;
1425   UINT64   Status;
1426   UINT32   EaxIn;
1427   UINT32   EcxIn;
1428   UINT64   XCr0;
1429   UINT32   Eax;
1430   UINT32   Ebx;
1431   UINT32   Ecx;
1432   UINT32   Edx;
1433
1434   EaxIn = (UINT32)(UINTN)Regs->Rax;
1435   EcxIn = (UINT32)(UINTN)Regs->Rcx;
1436
1437   if (EaxIn == CPUID_EXTENDED_STATE) {
1438     IA32_CR4  Cr4;
1439
1440     Cr4.UintN           = AsmReadCr4 ();
1441     Ghcb->SaveArea.XCr0 = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
1442     XCr0                = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
1443   }
1444
1445   if (SnpEnabled ()) {
1446     if (!GetCpuidFw (
1447            Ghcb,
1448            EaxIn,
1449            EcxIn,
1450            XCr0,
1451            &Eax,
1452            &Ebx,
1453            &Ecx,
1454            &Edx,
1455            &Status,
1456            &Unsupported
1457            ))
1458     {
1459       goto CpuidFail;
1460     }
1461   } else {
1462     if (!GetCpuidHyp (
1463            Ghcb,
1464            EaxIn,
1465            EcxIn,
1466            XCr0,
1467            &Eax,
1468            &Ebx,
1469            &Ecx,
1470            &Edx,
1471            &Status,
1472            &Unsupported
1473            ))
1474     {
1475       goto CpuidFail;
1476     }
1477   }
1478
1479   Regs->Rax = Eax;
1480   Regs->Rbx = Ebx;
1481   Regs->Rcx = Ecx;
1482   Regs->Rdx = Edx;
1483
1484   return 0;
1485
1486 CpuidFail:
1487   if (Unsupported) {
1488     return UnsupportedExit (Ghcb, Regs, InstructionData);
1489   }
1490
1491   return Status;
1492 }
1493
1494 /**
1495   Handle a RDPMC event.
1496
1497   Use the VMGEXIT instruction to handle a RDPMC event.
1498
1499   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1500                                    Block
1501   @param[in, out] Regs             x64 processor context
1502   @param[in]      InstructionData  Instruction parsing context
1503
1504   @retval 0                        Event handled successfully
1505   @return                          New exception value to propagate
1506
1507 **/
1508 STATIC
1509 UINT64
1510 RdpmcExit (
1511   IN OUT GHCB                    *Ghcb,
1512   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
1513   IN     CC_INSTRUCTION_DATA     *InstructionData
1514   )
1515 {
1516   UINT64  Status;
1517
1518   Ghcb->SaveArea.Rcx = Regs->Rcx;
1519   CcExitVmgSetOffsetValid (Ghcb, GhcbRcx);
1520
1521   Status = CcExitVmgExit (Ghcb, SVM_EXIT_RDPMC, 0, 0);
1522   if (Status != 0) {
1523     return Status;
1524   }
1525
1526   if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax) ||
1527       !CcExitVmgIsOffsetValid (Ghcb, GhcbRdx))
1528   {
1529     return UnsupportedExit (Ghcb, Regs, InstructionData);
1530   }
1531
1532   Regs->Rax = Ghcb->SaveArea.Rax;
1533   Regs->Rdx = Ghcb->SaveArea.Rdx;
1534
1535   return 0;
1536 }
1537
1538 /**
1539   Handle a RDTSC event.
1540
1541   Use the VMGEXIT instruction to handle a RDTSC event.
1542
1543   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1544                                    Block
1545   @param[in, out] Regs             x64 processor context
1546   @param[in]      InstructionData  Instruction parsing context
1547
1548   @retval 0                        Event handled successfully
1549   @return                          New exception value to propagate
1550
1551 **/
1552 STATIC
1553 UINT64
1554 RdtscExit (
1555   IN OUT GHCB                    *Ghcb,
1556   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
1557   IN     CC_INSTRUCTION_DATA     *InstructionData
1558   )
1559 {
1560   UINT64  Status;
1561
1562   Status = CcExitVmgExit (Ghcb, SVM_EXIT_RDTSC, 0, 0);
1563   if (Status != 0) {
1564     return Status;
1565   }
1566
1567   if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax) ||
1568       !CcExitVmgIsOffsetValid (Ghcb, GhcbRdx))
1569   {
1570     return UnsupportedExit (Ghcb, Regs, InstructionData);
1571   }
1572
1573   Regs->Rax = Ghcb->SaveArea.Rax;
1574   Regs->Rdx = Ghcb->SaveArea.Rdx;
1575
1576   return 0;
1577 }
1578
1579 /**
1580   Handle a DR7 register write event.
1581
1582   Use the VMGEXIT instruction to handle a DR7 write event.
1583
1584   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1585                                    Block
1586   @param[in, out] Regs             x64 processor context
1587   @param[in]      InstructionData  Instruction parsing context
1588
1589   @retval 0                        Event handled successfully
1590   @return                          New exception value to propagate
1591
1592 **/
1593 STATIC
1594 UINT64
1595 Dr7WriteExit (
1596   IN OUT GHCB                    *Ghcb,
1597   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
1598   IN     CC_INSTRUCTION_DATA     *InstructionData
1599   )
1600 {
1601   CC_INSTRUCTION_OPCODE_EXT  *Ext;
1602   SEV_ES_PER_CPU_DATA        *SevEsData;
1603   UINT64                     *Register;
1604   UINT64                     Status;
1605
1606   Ext       = &InstructionData->Ext;
1607   SevEsData = (SEV_ES_PER_CPU_DATA *)(Ghcb + 1);
1608
1609   CcDecodeModRm (Regs, InstructionData);
1610
1611   //
1612   // MOV DRn always treats MOD == 3 no matter how encoded
1613   //
1614   Register = CcGetRegisterPointer (Regs, Ext->ModRm.Rm);
1615
1616   //
1617   // Using a value of 0 for ExitInfo1 means RAX holds the value
1618   //
1619   Ghcb->SaveArea.Rax = *Register;
1620   CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
1621
1622   Status = CcExitVmgExit (Ghcb, SVM_EXIT_DR7_WRITE, 0, 0);
1623   if (Status != 0) {
1624     return Status;
1625   }
1626
1627   SevEsData->Dr7       = *Register;
1628   SevEsData->Dr7Cached = 1;
1629
1630   return 0;
1631 }
1632
1633 /**
1634   Handle a DR7 register read event.
1635
1636   Use the VMGEXIT instruction to handle a DR7 read event.
1637
1638   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1639                                    Block
1640   @param[in, out] Regs             x64 processor context
1641   @param[in]      InstructionData  Instruction parsing context
1642
1643   @retval 0                        Event handled successfully
1644
1645 **/
1646 STATIC
1647 UINT64
1648 Dr7ReadExit (
1649   IN OUT GHCB                    *Ghcb,
1650   IN OUT EFI_SYSTEM_CONTEXT_X64  *Regs,
1651   IN     CC_INSTRUCTION_DATA     *InstructionData
1652   )
1653 {
1654   CC_INSTRUCTION_OPCODE_EXT  *Ext;
1655   SEV_ES_PER_CPU_DATA        *SevEsData;
1656   UINT64                     *Register;
1657
1658   Ext       = &InstructionData->Ext;
1659   SevEsData = (SEV_ES_PER_CPU_DATA *)(Ghcb + 1);
1660
1661   CcDecodeModRm (Regs, InstructionData);
1662
1663   //
1664   // MOV DRn always treats MOD == 3 no matter how encoded
1665   //
1666   Register = CcGetRegisterPointer (Regs, Ext->ModRm.Rm);
1667
1668   //
1669   // If there is a cached valued for DR7, return that. Otherwise return the
1670   // DR7 standard reset value of 0x400 (no debug breakpoints set).
1671   //
1672   *Register = (SevEsData->Dr7Cached == 1) ? SevEsData->Dr7 : 0x400;
1673
1674   return 0;
1675 }
1676
1677 /**
1678   Handle a #VC exception.
1679
1680   Performs the necessary processing to handle a #VC exception.
1681
1682   @param[in, out]  Ghcb           Pointer to the GHCB
1683   @param[in, out]  ExceptionType  Pointer to an EFI_EXCEPTION_TYPE to be set
1684                                   as value to use on error.
1685   @param[in, out]  SystemContext  Pointer to EFI_SYSTEM_CONTEXT
1686
1687   @retval  EFI_SUCCESS            Exception handled
1688   @retval  EFI_UNSUPPORTED        #VC not supported, (new) exception value to
1689                                   propagate provided
1690   @retval  EFI_PROTOCOL_ERROR     #VC handling failed, (new) exception value to
1691                                   propagate provided
1692
1693 **/
1694 EFI_STATUS
1695 EFIAPI
1696 InternalVmgExitHandleVc (
1697   IN OUT GHCB                *Ghcb,
1698   IN OUT EFI_EXCEPTION_TYPE  *ExceptionType,
1699   IN OUT EFI_SYSTEM_CONTEXT  SystemContext
1700   )
1701 {
1702   EFI_SYSTEM_CONTEXT_X64  *Regs;
1703   NAE_EXIT                NaeExit;
1704   CC_INSTRUCTION_DATA     InstructionData;
1705   UINT64                  ExitCode, Status;
1706   EFI_STATUS              VcRet;
1707   BOOLEAN                 InterruptState;
1708
1709   VcRet = EFI_SUCCESS;
1710
1711   Regs = SystemContext.SystemContextX64;
1712
1713   CcExitVmgInit (Ghcb, &InterruptState);
1714
1715   ExitCode = Regs->ExceptionData;
1716   switch (ExitCode) {
1717     case SVM_EXIT_DR7_READ:
1718       NaeExit = Dr7ReadExit;
1719       break;
1720
1721     case SVM_EXIT_DR7_WRITE:
1722       NaeExit = Dr7WriteExit;
1723       break;
1724
1725     case SVM_EXIT_RDTSC:
1726       NaeExit = RdtscExit;
1727       break;
1728
1729     case SVM_EXIT_RDPMC:
1730       NaeExit = RdpmcExit;
1731       break;
1732
1733     case SVM_EXIT_CPUID:
1734       NaeExit = CpuidExit;
1735       break;
1736
1737     case SVM_EXIT_INVD:
1738       NaeExit = InvdExit;
1739       break;
1740
1741     case SVM_EXIT_IOIO_PROT:
1742       NaeExit = IoioExit;
1743       break;
1744
1745     case SVM_EXIT_MSR:
1746       NaeExit = MsrExit;
1747       break;
1748
1749     case SVM_EXIT_VMMCALL:
1750       NaeExit = VmmCallExit;
1751       break;
1752
1753     case SVM_EXIT_RDTSCP:
1754       NaeExit = RdtscpExit;
1755       break;
1756
1757     case SVM_EXIT_WBINVD:
1758       NaeExit = WbinvdExit;
1759       break;
1760
1761     case SVM_EXIT_MONITOR:
1762       NaeExit = MonitorExit;
1763       break;
1764
1765     case SVM_EXIT_MWAIT:
1766       NaeExit = MwaitExit;
1767       break;
1768
1769     case SVM_EXIT_NPF:
1770       NaeExit = MmioExit;
1771       break;
1772
1773     default:
1774       NaeExit = UnsupportedExit;
1775   }
1776
1777   CcInitInstructionData (&InstructionData, Ghcb, Regs);
1778
1779   Status = NaeExit (Ghcb, Regs, &InstructionData);
1780   if (Status == 0) {
1781     Regs->Rip += CcInstructionLength (&InstructionData);
1782   } else {
1783     GHCB_EVENT_INJECTION  Event;
1784
1785     Event.Uint64 = Status;
1786     if (Event.Elements.ErrorCodeValid != 0) {
1787       Regs->ExceptionData = Event.Elements.ErrorCode;
1788     } else {
1789       Regs->ExceptionData = 0;
1790     }
1791
1792     *ExceptionType = Event.Elements.Vector;
1793
1794     VcRet = EFI_PROTOCOL_ERROR;
1795   }
1796
1797   CcExitVmgDone (Ghcb, InterruptState);
1798
1799   return VcRet;
1800 }
1801
1802 /**
1803   Routine to allow ASSERT from within #VC.
1804
1805   @param[in, out]  SevEsData  Pointer to the per-CPU data
1806
1807 **/
1808 VOID
1809 EFIAPI
1810 VmgExitIssueAssert (
1811   IN OUT SEV_ES_PER_CPU_DATA  *SevEsData
1812   )
1813 {
1814   //
1815   // Progress will be halted, so set VcCount to allow for ASSERT output
1816   // to be seen.
1817   //
1818   SevEsData->VcCount = 0;
1819
1820   ASSERT (FALSE);
1821   CpuDeadLoop ();
1822 }