OvmfPkg/Library/VmgExitLib/VmgExitVcHandler.c

   1 /** @file
   2   X64 #VC Exception Handler functon.
   3
   4   Copyright (C) 2020, Advanced Micro Devices, Inc. All rights reserved.<BR>
   5   SPDX-License-Identifier: BSD-2-Clause-Patent
   6
   7 **/
   8
   9 #include <Base.h>
  10 #include <Uefi.h>
  11 #include <Library/BaseMemoryLib.h>
  12 #include <Library/LocalApicLib.h>
  13 #include <Library/MemEncryptSevLib.h>
  14 #include <Library/VmgExitLib.h>
  15 #include <Register/Amd/Msr.h>
  16 #include <Register/Intel/Cpuid.h>
  17 #include <IndustryStandard/InstructionParsing.h>
  18
  19 #include "VmgExitVcHandler.h"
  20 // #include <Library/MemEncryptSevLib.h>
  21
  22 //
  23 // Instruction execution mode definition
  24 //
  25 typedef enum {
  26   LongMode64Bit = 0,
  27   LongModeCompat32Bit,
  28   LongModeCompat16Bit,
  29 } SEV_ES_INSTRUCTION_MODE;
  30
  31 //
  32 // Instruction size definition (for operand and address)
  33 //
  34 typedef enum {
  35   Size8Bits = 0,
  36   Size16Bits,
  37   Size32Bits,
  38   Size64Bits,
  39 } SEV_ES_INSTRUCTION_SIZE;
  40
  41 //
  42 // Intruction segment definition
  43 //
  44 typedef enum {
  45   SegmentEs = 0,
  46   SegmentCs,
  47   SegmentSs,
  48   SegmentDs,
  49   SegmentFs,
  50   SegmentGs,
  51 } SEV_ES_INSTRUCTION_SEGMENT;
  52
  53 //
  54 // Instruction rep function definition
  55 //
  56 typedef enum {
  57   RepNone = 0,
  58   RepZ,
  59   RepNZ,
  60 } SEV_ES_INSTRUCTION_REP;
  61
  62 typedef struct {
  63   UINT8    Rm;
  64   UINT8    Reg;
  65   UINT8    Mod;
  66 } SEV_ES_INSTRUCTION_MODRM_EXT;
  67
  68 typedef struct {
  69   UINT8    Base;
  70   UINT8    Index;
  71   UINT8    Scale;
  72 } SEV_ES_INSTRUCTION_SIB_EXT;
  73
  74 //
  75 // Instruction opcode definition
  76 //
  77 typedef struct {
  78   SEV_ES_INSTRUCTION_MODRM_EXT    ModRm;
  79
  80   SEV_ES_INSTRUCTION_SIB_EXT      Sib;
  81
  82   UINTN                           RegData;
  83   UINTN                           RmData;
  84 } SEV_ES_INSTRUCTION_OPCODE_EXT;
  85
  86 //
  87 // Instruction parsing context definition
  88 //
  89 typedef struct {
  90   GHCB                             *Ghcb;
  91
  92   SEV_ES_INSTRUCTION_MODE          Mode;
  93   SEV_ES_INSTRUCTION_SIZE          DataSize;
  94   SEV_ES_INSTRUCTION_SIZE          AddrSize;
  95   BOOLEAN                          SegmentSpecified;
  96   SEV_ES_INSTRUCTION_SEGMENT       Segment;
  97   SEV_ES_INSTRUCTION_REP           RepMode;
  98
  99   UINT8                            *Begin;
 100   UINT8                            *End;
 101
 102   UINT8                            *Prefixes;
 103   UINT8                            *OpCodes;
 104   UINT8                            *Displacement;
 105   UINT8                            *Immediate;
 106
 107   INSTRUCTION_REX_PREFIX           RexPrefix;
 108
 109   BOOLEAN                          ModRmPresent;
 110   INSTRUCTION_MODRM                ModRm;
 111
 112   BOOLEAN                          SibPresent;
 113   INSTRUCTION_SIB                  Sib;
 114
 115   UINTN                            PrefixSize;
 116   UINTN                            OpCodeSize;
 117   UINTN                            DisplacementSize;
 118   UINTN                            ImmediateSize;
 119
 120   SEV_ES_INSTRUCTION_OPCODE_EXT    Ext;
 121 } SEV_ES_INSTRUCTION_DATA;
 122
 123 //
 124 // Non-automatic Exit function prototype
 125 //
 126 typedef
 127 UINT64
 128 (*NAE_EXIT) (
 129   GHCB                     *Ghcb,
 130   EFI_SYSTEM_CONTEXT_X64   *Regs,
 131   SEV_ES_INSTRUCTION_DATA  *InstructionData
 132   );
 133
 134 //
 135 // SEV-SNP Cpuid table entry/function
 136 //
 137 typedef PACKED struct {
 138   UINT32    EaxIn;
 139   UINT32    EcxIn;
 140   UINT64    Unused;
 141   UINT64    Unused2;
 142   UINT32    Eax;
 143   UINT32    Ebx;
 144   UINT32    Ecx;
 145   UINT32    Edx;
 146   UINT64    Reserved;
 147 } SEV_SNP_CPUID_FUNCTION;
 148
 149 //
 150 // SEV-SNP Cpuid page format
 151 //
 152 typedef PACKED struct {
 153   UINT32                    Count;
 154   UINT32                    Reserved1;
 155   UINT64                    Reserved2;
 156   SEV_SNP_CPUID_FUNCTION    function[0];
 157 } SEV_SNP_CPUID_INFO;
 158
 159 /**
 160   Return a pointer to the contents of the specified register.
 161
 162   Based upon the input register, return a pointer to the registers contents
 163   in the x86 processor context.
 164
 165   @param[in] Regs      x64 processor context
 166   @param[in] Register  Register to obtain pointer for
 167
 168   @return              Pointer to the contents of the requested register
 169
 170 **/
 171 STATIC
 172 UINT64 *
 173 GetRegisterPointer (
 174   IN EFI_SYSTEM_CONTEXT_X64  *Regs,
 175   IN UINT8                   Register
 176   )
 177 {
 178   UINT64  *Reg;
 179
 180   switch (Register) {
 181     case 0:
 182       Reg = &Regs->Rax;
 183       break;
 184     case 1:
 185       Reg = &Regs->Rcx;
 186       break;
 187     case 2:
 188       Reg = &Regs->Rdx;
 189       break;
 190     case 3:
 191       Reg = &Regs->Rbx;
 192       break;
 193     case 4:
 194       Reg = &Regs->Rsp;
 195       break;
 196     case 5:
 197       Reg = &Regs->Rbp;
 198       break;
 199     case 6:
 200       Reg = &Regs->Rsi;
 201       break;
 202     case 7:
 203       Reg = &Regs->Rdi;
 204       break;
 205     case 8:
 206       Reg = &Regs->R8;
 207       break;
 208     case 9:
 209       Reg = &Regs->R9;
 210       break;
 211     case 10:
 212       Reg = &Regs->R10;
 213       break;
 214     case 11:
 215       Reg = &Regs->R11;
 216       break;
 217     case 12:
 218       Reg = &Regs->R12;
 219       break;
 220     case 13:
 221       Reg = &Regs->R13;
 222       break;
 223     case 14:
 224       Reg = &Regs->R14;
 225       break;
 226     case 15:
 227       Reg = &Regs->R15;
 228       break;
 229     default:
 230       Reg = NULL;
 231   }
 232
 233   ASSERT (Reg != NULL);
 234
 235   return Reg;
 236 }
 237
 238 /**
 239   Update the instruction parsing context for displacement bytes.
 240
 241   @param[in, out] InstructionData  Instruction parsing context
 242   @param[in]      Size             The instruction displacement size
 243
 244 **/
 245 STATIC
 246 VOID
 247 UpdateForDisplacement (
 248   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 249   IN     UINTN                    Size
 250   )
 251 {
 252   InstructionData->DisplacementSize = Size;
 253   InstructionData->Immediate       += Size;
 254   InstructionData->End             += Size;
 255 }
 256
 257 /**
 258   Determine if an instruction address if RIP relative.
 259
 260   Examine the instruction parsing context to determine if the address offset
 261   is relative to the instruction pointer.
 262
 263   @param[in] InstructionData  Instruction parsing context
 264
 265   @retval TRUE                Instruction addressing is RIP relative
 266   @retval FALSE               Instruction addressing is not RIP relative
 267
 268 **/
 269 STATIC
 270 BOOLEAN
 271 IsRipRelative (
 272   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 273   )
 274 {
 275   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 276
 277   Ext = &InstructionData->Ext;
 278
 279   return ((InstructionData->Mode == LongMode64Bit) &&
 280           (Ext->ModRm.Mod == 0) &&
 281           (Ext->ModRm.Rm == 5)  &&
 282           (InstructionData->SibPresent == FALSE));
 283 }
 284
 285 /**
 286   Return the effective address of a memory operand.
 287
 288   Examine the instruction parsing context to obtain the effective memory
 289   address of a memory operand.
 290
 291   @param[in] Regs             x64 processor context
 292   @param[in] InstructionData  Instruction parsing context
 293
 294   @return                     The memory operand effective address
 295
 296 **/
 297 STATIC
 298 UINT64
 299 GetEffectiveMemoryAddress (
 300   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 301   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 302   )
 303 {
 304   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 305   UINT64                         EffectiveAddress;
 306
 307   Ext              = &InstructionData->Ext;
 308   EffectiveAddress = 0;
 309
 310   if (IsRipRelative (InstructionData)) {
 311     //
 312     // RIP-relative displacement is a 32-bit signed value
 313     //
 314     INT32  RipRelative;
 315
 316     RipRelative = *(INT32 *)InstructionData->Displacement;
 317
 318     UpdateForDisplacement (InstructionData, 4);
 319
 320     //
 321     // Negative displacement is handled by standard UINT64 wrap-around.
 322     //
 323     return Regs->Rip + (UINT64)RipRelative;
 324   }
 325
 326   switch (Ext->ModRm.Mod) {
 327     case 1:
 328       UpdateForDisplacement (InstructionData, 1);
 329       EffectiveAddress += (UINT64)(*(INT8 *)(InstructionData->Displacement));
 330       break;
 331     case 2:
 332       switch (InstructionData->AddrSize) {
 333         case Size16Bits:
 334           UpdateForDisplacement (InstructionData, 2);
 335           EffectiveAddress += (UINT64)(*(INT16 *)(InstructionData->Displacement));
 336           break;
 337         default:
 338           UpdateForDisplacement (InstructionData, 4);
 339           EffectiveAddress += (UINT64)(*(INT32 *)(InstructionData->Displacement));
 340           break;
 341       }
 342
 343       break;
 344   }
 345
 346   if (InstructionData->SibPresent) {
 347     INT64  Displacement;
 348
 349     if (Ext->Sib.Index != 4) {
 350       CopyMem (
 351         &Displacement,
 352         GetRegisterPointer (Regs, Ext->Sib.Index),
 353         sizeof (Displacement)
 354         );
 355       Displacement *= (INT64)(1 << Ext->Sib.Scale);
 356
 357       //
 358       // Negative displacement is handled by standard UINT64 wrap-around.
 359       //
 360       EffectiveAddress += (UINT64)Displacement;
 361     }
 362
 363     if ((Ext->Sib.Base != 5) || Ext->ModRm.Mod) {
 364       EffectiveAddress += *GetRegisterPointer (Regs, Ext->Sib.Base);
 365     } else {
 366       UpdateForDisplacement (InstructionData, 4);
 367       EffectiveAddress += (UINT64)(*(INT32 *)(InstructionData->Displacement));
 368     }
 369   } else {
 370     EffectiveAddress += *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 371   }
 372
 373   return EffectiveAddress;
 374 }
 375
 376 /**
 377   Decode a ModRM byte.
 378
 379   Examine the instruction parsing context to decode a ModRM byte and the SIB
 380   byte, if present.
 381
 382   @param[in]      Regs             x64 processor context
 383   @param[in, out] InstructionData  Instruction parsing context
 384
 385 **/
 386 STATIC
 387 VOID
 388 DecodeModRm (
 389   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 390   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 391   )
 392 {
 393   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 394   INSTRUCTION_REX_PREFIX         *RexPrefix;
 395   INSTRUCTION_MODRM              *ModRm;
 396   INSTRUCTION_SIB                *Sib;
 397
 398   RexPrefix = &InstructionData->RexPrefix;
 399   Ext       = &InstructionData->Ext;
 400   ModRm     = &InstructionData->ModRm;
 401   Sib       = &InstructionData->Sib;
 402
 403   InstructionData->ModRmPresent = TRUE;
 404   ModRm->Uint8                  = *(InstructionData->End);
 405
 406   InstructionData->Displacement++;
 407   InstructionData->Immediate++;
 408   InstructionData->End++;
 409
 410   Ext->ModRm.Mod = ModRm->Bits.Mod;
 411   Ext->ModRm.Reg = (RexPrefix->Bits.BitR << 3) | ModRm->Bits.Reg;
 412   Ext->ModRm.Rm  = (RexPrefix->Bits.BitB << 3) | ModRm->Bits.Rm;
 413
 414   Ext->RegData = *GetRegisterPointer (Regs, Ext->ModRm.Reg);
 415
 416   if (Ext->ModRm.Mod == 3) {
 417     Ext->RmData = *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 418   } else {
 419     if (ModRm->Bits.Rm == 4) {
 420       InstructionData->SibPresent = TRUE;
 421       Sib->Uint8                  = *(InstructionData->End);
 422
 423       InstructionData->Displacement++;
 424       InstructionData->Immediate++;
 425       InstructionData->End++;
 426
 427       Ext->Sib.Scale = Sib->Bits.Scale;
 428       Ext->Sib.Index = (RexPrefix->Bits.BitX << 3) | Sib->Bits.Index;
 429       Ext->Sib.Base  = (RexPrefix->Bits.BitB << 3) | Sib->Bits.Base;
 430     }
 431
 432     Ext->RmData = GetEffectiveMemoryAddress (Regs, InstructionData);
 433   }
 434 }
 435
 436 /**
 437   Decode instruction prefixes.
 438
 439   Parse the instruction data to track the instruction prefixes that have
 440   been used.
 441
 442   @param[in]      Regs             x64 processor context
 443   @param[in, out] InstructionData  Instruction parsing context
 444
 445 **/
 446 STATIC
 447 VOID
 448 DecodePrefixes (
 449   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 450   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 451   )
 452 {
 453   SEV_ES_INSTRUCTION_MODE  Mode;
 454   SEV_ES_INSTRUCTION_SIZE  ModeDataSize;
 455   SEV_ES_INSTRUCTION_SIZE  ModeAddrSize;
 456   UINT8                    *Byte;
 457
 458   //
 459   // Always in 64-bit mode
 460   //
 461   Mode         = LongMode64Bit;
 462   ModeDataSize = Size32Bits;
 463   ModeAddrSize = Size64Bits;
 464
 465   InstructionData->Mode     = Mode;
 466   InstructionData->DataSize = ModeDataSize;
 467   InstructionData->AddrSize = ModeAddrSize;
 468
 469   InstructionData->Prefixes = InstructionData->Begin;
 470
 471   Byte = InstructionData->Prefixes;
 472   for ( ; ; Byte++, InstructionData->PrefixSize++) {
 473     //
 474     // Check the 0x40 to 0x4F range using an if statement here since some
 475     // compilers don't like the "case 0x40 ... 0x4F:" syntax. This avoids
 476     // 16 case statements below.
 477     //
 478     if ((*Byte >= REX_PREFIX_START) && (*Byte <= REX_PREFIX_STOP)) {
 479       InstructionData->RexPrefix.Uint8 = *Byte;
 480       if ((*Byte & REX_64BIT_OPERAND_SIZE_MASK) != 0) {
 481         InstructionData->DataSize = Size64Bits;
 482       }
 483
 484       continue;
 485     }
 486
 487     switch (*Byte) {
 488       case OVERRIDE_SEGMENT_CS:
 489       case OVERRIDE_SEGMENT_DS:
 490       case OVERRIDE_SEGMENT_ES:
 491       case OVERRIDE_SEGMENT_SS:
 492         if (Mode != LongMode64Bit) {
 493           InstructionData->SegmentSpecified = TRUE;
 494           InstructionData->Segment          = (*Byte >> 3) & 3;
 495         }
 496
 497         break;
 498
 499       case OVERRIDE_SEGMENT_FS:
 500       case OVERRIDE_SEGMENT_GS:
 501         InstructionData->SegmentSpecified = TRUE;
 502         InstructionData->Segment          = *Byte & 7;
 503         break;
 504
 505       case OVERRIDE_OPERAND_SIZE:
 506         if (InstructionData->RexPrefix.Uint8 == 0) {
 507           InstructionData->DataSize =
 508             (Mode == LongMode64Bit)       ? Size16Bits :
 509             (Mode == LongModeCompat32Bit) ? Size16Bits :
 510             (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 511         }
 512
 513         break;
 514
 515       case OVERRIDE_ADDRESS_SIZE:
 516         InstructionData->AddrSize =
 517           (Mode == LongMode64Bit)       ? Size32Bits :
 518           (Mode == LongModeCompat32Bit) ? Size16Bits :
 519           (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 520         break;
 521
 522       case LOCK_PREFIX:
 523         break;
 524
 525       case REPZ_PREFIX:
 526         InstructionData->RepMode = RepZ;
 527         break;
 528
 529       case REPNZ_PREFIX:
 530         InstructionData->RepMode = RepNZ;
 531         break;
 532
 533       default:
 534         InstructionData->OpCodes    = Byte;
 535         InstructionData->OpCodeSize = (*Byte == TWO_BYTE_OPCODE_ESCAPE) ? 2 : 1;
 536
 537         InstructionData->End          = Byte + InstructionData->OpCodeSize;
 538         InstructionData->Displacement = InstructionData->End;
 539         InstructionData->Immediate    = InstructionData->End;
 540         return;
 541     }
 542   }
 543 }
 544
 545 /**
 546   Determine instruction length
 547
 548   Return the total length of the parsed instruction.
 549
 550   @param[in] InstructionData  Instruction parsing context
 551
 552   @return                     Length of parsed instruction
 553
 554 **/
 555 STATIC
 556 UINT64
 557 InstructionLength (
 558   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 559   )
 560 {
 561   return (UINT64)(InstructionData->End - InstructionData->Begin);
 562 }
 563
 564 /**
 565   Initialize the instruction parsing context.
 566
 567   Initialize the instruction parsing context, which includes decoding the
 568   instruction prefixes.
 569
 570   @param[in, out] InstructionData  Instruction parsing context
 571   @param[in]      Ghcb             Pointer to the Guest-Hypervisor Communication
 572                                    Block
 573   @param[in]      Regs             x64 processor context
 574
 575 **/
 576 STATIC
 577 VOID
 578 InitInstructionData (
 579   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 580   IN     GHCB                     *Ghcb,
 581   IN     EFI_SYSTEM_CONTEXT_X64   *Regs
 582   )
 583 {
 584   SetMem (InstructionData, sizeof (*InstructionData), 0);
 585   InstructionData->Ghcb  = Ghcb;
 586   InstructionData->Begin = (UINT8 *)Regs->Rip;
 587   InstructionData->End   = (UINT8 *)Regs->Rip;
 588
 589   DecodePrefixes (Regs, InstructionData);
 590 }
 591
 592 /**
 593   Report an unsupported event to the hypervisor
 594
 595   Use the VMGEXIT support to report an unsupported event to the hypervisor.
 596
 597   @param[in] Ghcb             Pointer to the Guest-Hypervisor Communication
 598                               Block
 599   @param[in] Regs             x64 processor context
 600   @param[in] InstructionData  Instruction parsing context
 601
 602   @return                     New exception value to propagate
 603
 604 **/
 605 STATIC
 606 UINT64
 607 UnsupportedExit (
 608   IN GHCB                     *Ghcb,
 609   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 610   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 611   )
 612 {
 613   UINT64  Status;
 614
 615   Status = VmgExit (Ghcb, SVM_EXIT_UNSUPPORTED, Regs->ExceptionData, 0);
 616   if (Status == 0) {
 617     GHCB_EVENT_INJECTION  Event;
 618
 619     Event.Uint64          = 0;
 620     Event.Elements.Vector = GP_EXCEPTION;
 621     Event.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
 622     Event.Elements.Valid  = 1;
 623
 624     Status = Event.Uint64;
 625   }
 626
 627   return Status;
 628 }
 629
 630 /**
 631   Validate that the MMIO memory access is not to encrypted memory.
 632
 633   Examine the pagetable entry for the memory specified. MMIO should not be
 634   performed against encrypted memory. MMIO to the APIC page is always allowed.
 635
 636   @param[in] Ghcb           Pointer to the Guest-Hypervisor Communication Block
 637   @param[in] MemoryAddress  Memory address to validate
 638   @param[in] MemoryLength   Memory length to validate
 639
 640   @retval 0          Memory is not encrypted
 641   @return            New exception value to propogate
 642
 643 **/
 644 STATIC
 645 UINT64
 646 ValidateMmioMemory (
 647   IN GHCB   *Ghcb,
 648   IN UINTN  MemoryAddress,
 649   IN UINTN  MemoryLength
 650   )
 651 {
 652   MEM_ENCRYPT_SEV_ADDRESS_RANGE_STATE  State;
 653   GHCB_EVENT_INJECTION                 GpEvent;
 654   UINTN                                Address;
 655
 656   //
 657   // Allow APIC accesses (which will have the encryption bit set during
 658   // SEC and PEI phases).
 659   //
 660   Address = MemoryAddress & ~(SIZE_4KB - 1);
 661   if (Address == GetLocalApicBaseAddress ()) {
 662     return 0;
 663   }
 664
 665   State = MemEncryptSevGetAddressRangeState (
 666             0,
 667             MemoryAddress,
 668             MemoryLength
 669             );
 670   if (State == MemEncryptSevAddressRangeUnencrypted) {
 671     return 0;
 672   }
 673
 674   //
 675   // Any state other than unencrypted is an error, issue a #GP.
 676   //
 677   DEBUG ((
 678     DEBUG_ERROR,
 679     "MMIO using encrypted memory: %lx\n",
 680     (UINT64)MemoryAddress
 681     ));
 682   GpEvent.Uint64          = 0;
 683   GpEvent.Elements.Vector = GP_EXCEPTION;
 684   GpEvent.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
 685   GpEvent.Elements.Valid  = 1;
 686
 687   return GpEvent.Uint64;
 688 }
 689
 690 /**
 691   Handle an MMIO event.
 692
 693   Use the VMGEXIT instruction to handle either an MMIO read or an MMIO write.
 694
 695   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 696                                    Block
 697   @param[in, out] Regs             x64 processor context
 698   @param[in, out] InstructionData  Instruction parsing context
 699
 700   @retval 0                        Event handled successfully
 701   @return                          New exception value to propagate
 702
 703 **/
 704 STATIC
 705 UINT64
 706 MmioExit (
 707   IN OUT GHCB                     *Ghcb,
 708   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 709   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 710   )
 711 {
 712   UINT64  ExitInfo1, ExitInfo2, Status;
 713   UINTN   Bytes;
 714   UINT64  *Register;
 715   UINT8   OpCode, SignByte;
 716   UINTN   Address;
 717
 718   Bytes = 0;
 719
 720   OpCode = *(InstructionData->OpCodes);
 721   if (OpCode == TWO_BYTE_OPCODE_ESCAPE) {
 722     OpCode = *(InstructionData->OpCodes + 1);
 723   }
 724
 725   switch (OpCode) {
 726     //
 727     // MMIO write (MOV reg/memX, regX)
 728     //
 729     case 0x88:
 730       Bytes = 1;
 731     //
 732     // fall through
 733     //
 734     case 0x89:
 735       DecodeModRm (Regs, InstructionData);
 736       Bytes = ((Bytes != 0) ? Bytes :
 737                (InstructionData->DataSize == Size16Bits) ? 2 :
 738                (InstructionData->DataSize == Size32Bits) ? 4 :
 739                (InstructionData->DataSize == Size64Bits) ? 8 :
 740                0);
 741
 742       if (InstructionData->Ext.ModRm.Mod == 3) {
 743         //
 744         // NPF on two register operands???
 745         //
 746         return UnsupportedExit (Ghcb, Regs, InstructionData);
 747       }
 748
 749       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 750       if (Status != 0) {
 751         return Status;
 752       }
 753
 754       ExitInfo1 = InstructionData->Ext.RmData;
 755       ExitInfo2 = Bytes;
 756       CopyMem (Ghcb->SharedBuffer, &InstructionData->Ext.RegData, Bytes);
 757
 758       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 759       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 760       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 761       if (Status != 0) {
 762         return Status;
 763       }
 764
 765       break;
 766
 767     //
 768     // MMIO write (MOV moffsetX, aX)
 769     //
 770     case 0xA2:
 771       Bytes = 1;
 772     //
 773     // fall through
 774     //
 775     case 0xA3:
 776       Bytes = ((Bytes != 0) ? Bytes :
 777                (InstructionData->DataSize == Size16Bits) ? 2 :
 778                (InstructionData->DataSize == Size32Bits) ? 4 :
 779                (InstructionData->DataSize == Size64Bits) ? 8 :
 780                0);
 781
 782       InstructionData->ImmediateSize = (UINTN)(1 << InstructionData->AddrSize);
 783       InstructionData->End          += InstructionData->ImmediateSize;
 784
 785       //
 786       // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
 787       // Use a STATIC_ASSERT to be certain the code is being built as X64.
 788       //
 789       STATIC_ASSERT (
 790         sizeof (UINTN) == sizeof (UINT64),
 791         "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
 792         );
 793
 794       Address = 0;
 795       CopyMem (
 796         &Address,
 797         InstructionData->Immediate,
 798         InstructionData->ImmediateSize
 799         );
 800
 801       Status = ValidateMmioMemory (Ghcb, Address, Bytes);
 802       if (Status != 0) {
 803         return Status;
 804       }
 805
 806       ExitInfo1 = Address;
 807       ExitInfo2 = Bytes;
 808       CopyMem (Ghcb->SharedBuffer, &Regs->Rax, Bytes);
 809
 810       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 811       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 812       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 813       if (Status != 0) {
 814         return Status;
 815       }
 816
 817       break;
 818
 819     //
 820     // MMIO write (MOV reg/memX, immX)
 821     //
 822     case 0xC6:
 823       Bytes = 1;
 824     //
 825     // fall through
 826     //
 827     case 0xC7:
 828       DecodeModRm (Regs, InstructionData);
 829       Bytes = ((Bytes != 0) ? Bytes :
 830                (InstructionData->DataSize == Size16Bits) ? 2 :
 831                (InstructionData->DataSize == Size32Bits) ? 4 :
 832                0);
 833
 834       InstructionData->ImmediateSize = Bytes;
 835       InstructionData->End          += Bytes;
 836
 837       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 838       if (Status != 0) {
 839         return Status;
 840       }
 841
 842       ExitInfo1 = InstructionData->Ext.RmData;
 843       ExitInfo2 = Bytes;
 844       CopyMem (Ghcb->SharedBuffer, InstructionData->Immediate, Bytes);
 845
 846       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 847       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 848       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 849       if (Status != 0) {
 850         return Status;
 851       }
 852
 853       break;
 854
 855     //
 856     // MMIO read (MOV regX, reg/memX)
 857     //
 858     case 0x8A:
 859       Bytes = 1;
 860     //
 861     // fall through
 862     //
 863     case 0x8B:
 864       DecodeModRm (Regs, InstructionData);
 865       Bytes = ((Bytes != 0) ? Bytes :
 866                (InstructionData->DataSize == Size16Bits) ? 2 :
 867                (InstructionData->DataSize == Size32Bits) ? 4 :
 868                (InstructionData->DataSize == Size64Bits) ? 8 :
 869                0);
 870       if (InstructionData->Ext.ModRm.Mod == 3) {
 871         //
 872         // NPF on two register operands???
 873         //
 874         return UnsupportedExit (Ghcb, Regs, InstructionData);
 875       }
 876
 877       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 878       if (Status != 0) {
 879         return Status;
 880       }
 881
 882       ExitInfo1 = InstructionData->Ext.RmData;
 883       ExitInfo2 = Bytes;
 884
 885       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 886       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 887       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 888       if (Status != 0) {
 889         return Status;
 890       }
 891
 892       Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 893       if (Bytes == 4) {
 894         //
 895         // Zero-extend for 32-bit operation
 896         //
 897         *Register = 0;
 898       }
 899
 900       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 901       break;
 902
 903     //
 904     // MMIO read (MOV aX, moffsetX)
 905     //
 906     case 0xA0:
 907       Bytes = 1;
 908     //
 909     // fall through
 910     //
 911     case 0xA1:
 912       Bytes = ((Bytes != 0) ? Bytes :
 913                (InstructionData->DataSize == Size16Bits) ? 2 :
 914                (InstructionData->DataSize == Size32Bits) ? 4 :
 915                (InstructionData->DataSize == Size64Bits) ? 8 :
 916                0);
 917
 918       InstructionData->ImmediateSize = (UINTN)(1 << InstructionData->AddrSize);
 919       InstructionData->End          += InstructionData->ImmediateSize;
 920
 921       //
 922       // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
 923       // Use a STATIC_ASSERT to be certain the code is being built as X64.
 924       //
 925       STATIC_ASSERT (
 926         sizeof (UINTN) == sizeof (UINT64),
 927         "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
 928         );
 929
 930       Address = 0;
 931       CopyMem (
 932         &Address,
 933         InstructionData->Immediate,
 934         InstructionData->ImmediateSize
 935         );
 936
 937       Status = ValidateMmioMemory (Ghcb, Address, Bytes);
 938       if (Status != 0) {
 939         return Status;
 940       }
 941
 942       ExitInfo1 = Address;
 943       ExitInfo2 = Bytes;
 944
 945       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 946       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 947       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 948       if (Status != 0) {
 949         return Status;
 950       }
 951
 952       if (Bytes == 4) {
 953         //
 954         // Zero-extend for 32-bit operation
 955         //
 956         Regs->Rax = 0;
 957       }
 958
 959       CopyMem (&Regs->Rax, Ghcb->SharedBuffer, Bytes);
 960       break;
 961
 962     //
 963     // MMIO read w/ zero-extension ((MOVZX regX, reg/memX)
 964     //
 965     case 0xB6:
 966       Bytes = 1;
 967     //
 968     // fall through
 969     //
 970     case 0xB7:
 971       DecodeModRm (Regs, InstructionData);
 972       Bytes = (Bytes != 0) ? Bytes : 2;
 973
 974       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 975       if (Status != 0) {
 976         return Status;
 977       }
 978
 979       ExitInfo1 = InstructionData->Ext.RmData;
 980       ExitInfo2 = Bytes;
 981
 982       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 983       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 984       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 985       if (Status != 0) {
 986         return Status;
 987       }
 988
 989       Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 990       SetMem (Register, (UINTN)(1 << InstructionData->DataSize), 0);
 991       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 992       break;
 993
 994     //
 995     // MMIO read w/ sign-extension (MOVSX regX, reg/memX)
 996     //
 997     case 0xBE:
 998       Bytes = 1;
 999     //
1000     // fall through
1001     //
1002     case 0xBF:
1003       DecodeModRm (Regs, InstructionData);
1004       Bytes = (Bytes != 0) ? Bytes : 2;
1005
1006       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
1007       if (Status != 0) {
1008         return Status;
1009       }
1010
1011       ExitInfo1 = InstructionData->Ext.RmData;
1012       ExitInfo2 = Bytes;
1013
1014       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
1015       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
1016       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
1017       if (Status != 0) {
1018         return Status;
1019       }
1020
1021       if (Bytes == 1) {
1022         UINT8  *Data;
1023
1024         Data     = (UINT8 *)Ghcb->SharedBuffer;
1025         SignByte = ((*Data & BIT7) != 0) ? 0xFF : 0x00;
1026       } else {
1027         UINT16  *Data;
1028
1029         Data     = (UINT16 *)Ghcb->SharedBuffer;
1030         SignByte = ((*Data & BIT15) != 0) ? 0xFF : 0x00;
1031       }
1032
1033       Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
1034       SetMem (Register, (UINTN)(1 << InstructionData->DataSize), SignByte);
1035       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
1036       break;
1037
1038     default:
1039       DEBUG ((DEBUG_ERROR, "Invalid MMIO opcode (%x)\n", OpCode));
1040       Status = GP_EXCEPTION;
1041       ASSERT (FALSE);
1042   }
1043
1044   return Status;
1045 }
1046
1047 /**
1048   Handle a MWAIT event.
1049
1050   Use the VMGEXIT instruction to handle a MWAIT event.
1051
1052   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1053                                    Block
1054   @param[in, out] Regs             x64 processor context
1055   @param[in]      InstructionData  Instruction parsing context
1056
1057   @retval 0                        Event handled successfully
1058   @return                          New exception value to propagate
1059
1060 **/
1061 STATIC
1062 UINT64
1063 MwaitExit (
1064   IN OUT GHCB                     *Ghcb,
1065   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1066   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1067   )
1068 {
1069   DecodeModRm (Regs, InstructionData);
1070
1071   Ghcb->SaveArea.Rax = Regs->Rax;
1072   VmgSetOffsetValid (Ghcb, GhcbRax);
1073   Ghcb->SaveArea.Rcx = Regs->Rcx;
1074   VmgSetOffsetValid (Ghcb, GhcbRcx);
1075
1076   return VmgExit (Ghcb, SVM_EXIT_MWAIT, 0, 0);
1077 }
1078
1079 /**
1080   Handle a MONITOR event.
1081
1082   Use the VMGEXIT instruction to handle a MONITOR event.
1083
1084   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1085                                    Block
1086   @param[in, out] Regs             x64 processor context
1087   @param[in]      InstructionData  Instruction parsing context
1088
1089   @retval 0                        Event handled successfully
1090   @return                          New exception value to propagate
1091
1092 **/
1093 STATIC
1094 UINT64
1095 MonitorExit (
1096   IN OUT GHCB                     *Ghcb,
1097   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1098   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1099   )
1100 {
1101   DecodeModRm (Regs, InstructionData);
1102
1103   Ghcb->SaveArea.Rax = Regs->Rax;  // Identity mapped, so VA = PA
1104   VmgSetOffsetValid (Ghcb, GhcbRax);
1105   Ghcb->SaveArea.Rcx = Regs->Rcx;
1106   VmgSetOffsetValid (Ghcb, GhcbRcx);
1107   Ghcb->SaveArea.Rdx = Regs->Rdx;
1108   VmgSetOffsetValid (Ghcb, GhcbRdx);
1109
1110   return VmgExit (Ghcb, SVM_EXIT_MONITOR, 0, 0);
1111 }
1112
1113 /**
1114   Handle a WBINVD event.
1115
1116   Use the VMGEXIT instruction to handle a WBINVD event.
1117
1118   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1119                                    Block
1120   @param[in, out] Regs             x64 processor context
1121   @param[in]      InstructionData  Instruction parsing context
1122
1123   @retval 0                        Event handled successfully
1124   @return                          New exception value to propagate
1125
1126 **/
1127 STATIC
1128 UINT64
1129 WbinvdExit (
1130   IN OUT GHCB                     *Ghcb,
1131   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1132   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1133   )
1134 {
1135   return VmgExit (Ghcb, SVM_EXIT_WBINVD, 0, 0);
1136 }
1137
1138 /**
1139   Handle a RDTSCP event.
1140
1141   Use the VMGEXIT instruction to handle a RDTSCP event.
1142
1143   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1144                                    Block
1145   @param[in, out] Regs             x64 processor context
1146   @param[in]      InstructionData  Instruction parsing context
1147
1148   @retval 0                        Event handled successfully
1149   @return                          New exception value to propagate
1150
1151 **/
1152 STATIC
1153 UINT64
1154 RdtscpExit (
1155   IN OUT GHCB                     *Ghcb,
1156   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1157   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1158   )
1159 {
1160   UINT64  Status;
1161
1162   DecodeModRm (Regs, InstructionData);
1163
1164   Status = VmgExit (Ghcb, SVM_EXIT_RDTSCP, 0, 0);
1165   if (Status != 0) {
1166     return Status;
1167   }
1168
1169   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1170       !VmgIsOffsetValid (Ghcb, GhcbRcx) ||
1171       !VmgIsOffsetValid (Ghcb, GhcbRdx))
1172   {
1173     return UnsupportedExit (Ghcb, Regs, InstructionData);
1174   }
1175
1176   Regs->Rax = Ghcb->SaveArea.Rax;
1177   Regs->Rcx = Ghcb->SaveArea.Rcx;
1178   Regs->Rdx = Ghcb->SaveArea.Rdx;
1179
1180   return 0;
1181 }
1182
1183 /**
1184   Handle a VMMCALL event.
1185
1186   Use the VMGEXIT instruction to handle a VMMCALL event.
1187
1188   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1189                                    Block
1190   @param[in, out] Regs             x64 processor context
1191   @param[in]      InstructionData  Instruction parsing context
1192
1193   @retval 0                        Event handled successfully
1194   @return                          New exception value to propagate
1195
1196 **/
1197 STATIC
1198 UINT64
1199 VmmCallExit (
1200   IN OUT GHCB                     *Ghcb,
1201   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1202   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1203   )
1204 {
1205   UINT64  Status;
1206
1207   DecodeModRm (Regs, InstructionData);
1208
1209   Ghcb->SaveArea.Rax = Regs->Rax;
1210   VmgSetOffsetValid (Ghcb, GhcbRax);
1211   Ghcb->SaveArea.Cpl = (UINT8)(Regs->Cs & 0x3);
1212   VmgSetOffsetValid (Ghcb, GhcbCpl);
1213
1214   Status = VmgExit (Ghcb, SVM_EXIT_VMMCALL, 0, 0);
1215   if (Status != 0) {
1216     return Status;
1217   }
1218
1219   if (!VmgIsOffsetValid (Ghcb, GhcbRax)) {
1220     return UnsupportedExit (Ghcb, Regs, InstructionData);
1221   }
1222
1223   Regs->Rax = Ghcb->SaveArea.Rax;
1224
1225   return 0;
1226 }
1227
1228 /**
1229   Handle an MSR event.
1230
1231   Use the VMGEXIT instruction to handle either a RDMSR or WRMSR event.
1232
1233   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1234                                    Block
1235   @param[in, out] Regs             x64 processor context
1236   @param[in]      InstructionData  Instruction parsing context
1237
1238   @retval 0                        Event handled successfully
1239   @return                          New exception value to propagate
1240
1241 **/
1242 STATIC
1243 UINT64
1244 MsrExit (
1245   IN OUT GHCB                     *Ghcb,
1246   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1247   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1248   )
1249 {
1250   UINT64  ExitInfo1, Status;
1251
1252   ExitInfo1 = 0;
1253
1254   switch (*(InstructionData->OpCodes + 1)) {
1255     case 0x30: // WRMSR
1256       ExitInfo1          = 1;
1257       Ghcb->SaveArea.Rax = Regs->Rax;
1258       VmgSetOffsetValid (Ghcb, GhcbRax);
1259       Ghcb->SaveArea.Rdx = Regs->Rdx;
1260       VmgSetOffsetValid (Ghcb, GhcbRdx);
1261     //
1262     // fall through
1263     //
1264     case 0x32: // RDMSR
1265       Ghcb->SaveArea.Rcx = Regs->Rcx;
1266       VmgSetOffsetValid (Ghcb, GhcbRcx);
1267       break;
1268     default:
1269       return UnsupportedExit (Ghcb, Regs, InstructionData);
1270   }
1271
1272   Status = VmgExit (Ghcb, SVM_EXIT_MSR, ExitInfo1, 0);
1273   if (Status != 0) {
1274     return Status;
1275   }
1276
1277   if (ExitInfo1 == 0) {
1278     if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1279         !VmgIsOffsetValid (Ghcb, GhcbRdx))
1280     {
1281       return UnsupportedExit (Ghcb, Regs, InstructionData);
1282     }
1283
1284     Regs->Rax = Ghcb->SaveArea.Rax;
1285     Regs->Rdx = Ghcb->SaveArea.Rdx;
1286   }
1287
1288   return 0;
1289 }
1290
1291 /**
1292   Build the IOIO event information.
1293
1294   The IOIO event information identifies the type of IO operation to be performed
1295   by the hypervisor. Build this information based on the instruction data.
1296
1297   @param[in]       Regs             x64 processor context
1298   @param[in, out]  InstructionData  Instruction parsing context
1299
1300   @return                           IOIO event information value
1301
1302 **/
1303 STATIC
1304 UINT64
1305 IoioExitInfo (
1306   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
1307   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
1308   )
1309 {
1310   UINT64  ExitInfo;
1311
1312   ExitInfo = 0;
1313
1314   switch (*(InstructionData->OpCodes)) {
1315     //
1316     // INS opcodes
1317     //
1318     case 0x6C:
1319     case 0x6D:
1320       ExitInfo |= IOIO_TYPE_INS;
1321       ExitInfo |= IOIO_SEG_ES;
1322       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1323       break;
1324
1325     //
1326     // OUTS opcodes
1327     //
1328     case 0x6E:
1329     case 0x6F:
1330       ExitInfo |= IOIO_TYPE_OUTS;
1331       ExitInfo |= IOIO_SEG_DS;
1332       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1333       break;
1334
1335     //
1336     // IN immediate opcodes
1337     //
1338     case 0xE4:
1339     case 0xE5:
1340       InstructionData->ImmediateSize = 1;
1341       InstructionData->End++;
1342       ExitInfo |= IOIO_TYPE_IN;
1343       ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16);
1344       break;
1345
1346     //
1347     // OUT immediate opcodes
1348     //
1349     case 0xE6:
1350     case 0xE7:
1351       InstructionData->ImmediateSize = 1;
1352       InstructionData->End++;
1353       ExitInfo |= IOIO_TYPE_OUT;
1354       ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16) | IOIO_TYPE_OUT;
1355       break;
1356
1357     //
1358     // IN register opcodes
1359     //
1360     case 0xEC:
1361     case 0xED:
1362       ExitInfo |= IOIO_TYPE_IN;
1363       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1364       break;
1365
1366     //
1367     // OUT register opcodes
1368     //
1369     case 0xEE:
1370     case 0xEF:
1371       ExitInfo |= IOIO_TYPE_OUT;
1372       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1373       break;
1374
1375     default:
1376       return 0;
1377   }
1378
1379   switch (*(InstructionData->OpCodes)) {
1380     //
1381     // Single-byte opcodes
1382     //
1383     case 0x6C:
1384     case 0x6E:
1385     case 0xE4:
1386     case 0xE6:
1387     case 0xEC:
1388     case 0xEE:
1389       ExitInfo |= IOIO_DATA_8;
1390       break;
1391
1392     //
1393     // Length determined by instruction parsing
1394     //
1395     default:
1396       ExitInfo |= (InstructionData->DataSize == Size16Bits) ? IOIO_DATA_16
1397                                                           : IOIO_DATA_32;
1398   }
1399
1400   switch (InstructionData->AddrSize) {
1401     case Size16Bits:
1402       ExitInfo |= IOIO_ADDR_16;
1403       break;
1404
1405     case Size32Bits:
1406       ExitInfo |= IOIO_ADDR_32;
1407       break;
1408
1409     case Size64Bits:
1410       ExitInfo |= IOIO_ADDR_64;
1411       break;
1412
1413     default:
1414       break;
1415   }
1416
1417   if (InstructionData->RepMode != 0) {
1418     ExitInfo |= IOIO_REP;
1419   }
1420
1421   return ExitInfo;
1422 }
1423
1424 /**
1425   Handle an IOIO event.
1426
1427   Use the VMGEXIT instruction to handle an IOIO event.
1428
1429   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1430                                    Block
1431   @param[in, out] Regs             x64 processor context
1432   @param[in]      InstructionData  Instruction parsing context
1433
1434   @retval 0                        Event handled successfully
1435   @return                          New exception value to propagate
1436
1437 **/
1438 STATIC
1439 UINT64
1440 IoioExit (
1441   IN OUT GHCB                     *Ghcb,
1442   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1443   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1444   )
1445 {
1446   UINT64   ExitInfo1, ExitInfo2, Status;
1447   BOOLEAN  IsString;
1448
1449   ExitInfo1 = IoioExitInfo (Regs, InstructionData);
1450   if (ExitInfo1 == 0) {
1451     return UnsupportedExit (Ghcb, Regs, InstructionData);
1452   }
1453
1454   IsString = ((ExitInfo1 & IOIO_TYPE_STR) != 0) ? TRUE : FALSE;
1455   if (IsString) {
1456     UINTN  IoBytes, VmgExitBytes;
1457     UINTN  GhcbCount, OpCount;
1458
1459     Status = 0;
1460
1461     IoBytes   = IOIO_DATA_BYTES (ExitInfo1);
1462     GhcbCount = sizeof (Ghcb->SharedBuffer) / IoBytes;
1463
1464     OpCount = ((ExitInfo1 & IOIO_REP) != 0) ? Regs->Rcx : 1;
1465     while (OpCount != 0) {
1466       ExitInfo2    = MIN (OpCount, GhcbCount);
1467       VmgExitBytes = ExitInfo2 * IoBytes;
1468
1469       if ((ExitInfo1 & IOIO_TYPE_IN) == 0) {
1470         CopyMem (Ghcb->SharedBuffer, (VOID *)Regs->Rsi, VmgExitBytes);
1471         Regs->Rsi += VmgExitBytes;
1472       }
1473
1474       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
1475       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
1476       Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, ExitInfo2);
1477       if (Status != 0) {
1478         return Status;
1479       }
1480
1481       if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1482         CopyMem ((VOID *)Regs->Rdi, Ghcb->SharedBuffer, VmgExitBytes);
1483         Regs->Rdi += VmgExitBytes;
1484       }
1485
1486       if ((ExitInfo1 & IOIO_REP) != 0) {
1487         Regs->Rcx -= ExitInfo2;
1488       }
1489
1490       OpCount -= ExitInfo2;
1491     }
1492   } else {
1493     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1494       Ghcb->SaveArea.Rax = 0;
1495     } else {
1496       CopyMem (&Ghcb->SaveArea.Rax, &Regs->Rax, IOIO_DATA_BYTES (ExitInfo1));
1497     }
1498
1499     VmgSetOffsetValid (Ghcb, GhcbRax);
1500
1501     Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, 0);
1502     if (Status != 0) {
1503       return Status;
1504     }
1505
1506     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1507       if (!VmgIsOffsetValid (Ghcb, GhcbRax)) {
1508         return UnsupportedExit (Ghcb, Regs, InstructionData);
1509       }
1510
1511       CopyMem (&Regs->Rax, &Ghcb->SaveArea.Rax, IOIO_DATA_BYTES (ExitInfo1));
1512     }
1513   }
1514
1515   return 0;
1516 }
1517
1518 /**
1519   Handle a INVD event.
1520
1521   Use the VMGEXIT instruction to handle a INVD event.
1522
1523   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1524                                    Block
1525   @param[in, out] Regs             x64 processor context
1526   @param[in]      InstructionData  Instruction parsing context
1527
1528   @retval 0                        Event handled successfully
1529   @return                          New exception value to propagate
1530
1531 **/
1532 STATIC
1533 UINT64
1534 InvdExit (
1535   IN OUT GHCB                     *Ghcb,
1536   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1537   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1538   )
1539 {
1540   return VmgExit (Ghcb, SVM_EXIT_INVD, 0, 0);
1541 }
1542
1543 /**
1544   Fetch CPUID leaf/function via hypervisor/VMGEXIT.
1545
1546   @param[in, out] Ghcb         Pointer to the Guest-Hypervisor Communication
1547                                Block
1548   @param[in]      EaxIn        EAX input for cpuid instruction
1549   @param[in]      EcxIn        ECX input for cpuid instruction
1550   @param[in]      Xcr0In       XCR0 at time of cpuid instruction
1551   @param[in, out] Eax          Pointer to store leaf's EAX value
1552   @param[in, out] Ebx          Pointer to store leaf's EBX value
1553   @param[in, out] Ecx          Pointer to store leaf's ECX value
1554   @param[in, out] Edx          Pointer to store leaf's EDX value
1555   @param[in, out] Status       Pointer to store status from VMGEXIT (always 0
1556                                unless return value indicates failure)
1557   @param[in, out] Unsupported  Pointer to store indication of unsupported
1558                                VMGEXIT (always false unless return value
1559                                indicates failure)
1560
1561   @retval TRUE                 CPUID leaf fetch successfully.
1562   @retval FALSE                Error occurred while fetching CPUID leaf. Callers
1563                                should Status and Unsupported and handle
1564                                accordingly if they indicate a more precise
1565                                error condition.
1566
1567 **/
1568 STATIC
1569 BOOLEAN
1570 GetCpuidHyp (
1571   IN OUT GHCB     *Ghcb,
1572   IN     UINT32   EaxIn,
1573   IN     UINT32   EcxIn,
1574   IN     UINT64   XCr0,
1575   IN OUT UINT32   *Eax,
1576   IN OUT UINT32   *Ebx,
1577   IN OUT UINT32   *Ecx,
1578   IN OUT UINT32   *Edx,
1579   IN OUT UINT64   *Status,
1580   IN OUT BOOLEAN  *UnsupportedExit
1581   )
1582 {
1583   *UnsupportedExit   = FALSE;
1584   Ghcb->SaveArea.Rax = EaxIn;
1585   VmgSetOffsetValid (Ghcb, GhcbRax);
1586   Ghcb->SaveArea.Rcx = EcxIn;
1587   VmgSetOffsetValid (Ghcb, GhcbRcx);
1588   if (EaxIn == CPUID_EXTENDED_STATE) {
1589     Ghcb->SaveArea.XCr0 = XCr0;
1590     VmgSetOffsetValid (Ghcb, GhcbXCr0);
1591   }
1592
1593   *Status = VmgExit (Ghcb, SVM_EXIT_CPUID, 0, 0);
1594   if (*Status != 0) {
1595     return FALSE;
1596   }
1597
1598   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1599       !VmgIsOffsetValid (Ghcb, GhcbRbx) ||
1600       !VmgIsOffsetValid (Ghcb, GhcbRcx) ||
1601       !VmgIsOffsetValid (Ghcb, GhcbRdx))
1602   {
1603     *UnsupportedExit = TRUE;
1604     return FALSE;
1605   }
1606
1607   if (Eax) {
1608     *Eax = (UINT32)(UINTN)Ghcb->SaveArea.Rax;
1609   }
1610
1611   if (Ebx) {
1612     *Ebx = (UINT32)(UINTN)Ghcb->SaveArea.Rbx;
1613   }
1614
1615   if (Ecx) {
1616     *Ecx = (UINT32)(UINTN)Ghcb->SaveArea.Rcx;
1617   }
1618
1619   if (Edx) {
1620     *Edx = (UINT32)(UINTN)Ghcb->SaveArea.Rdx;
1621   }
1622
1623   return TRUE;
1624 }
1625
1626 /**
1627   Check if SEV-SNP enabled.
1628
1629   @retval TRUE      SEV-SNP is enabled.
1630   @retval FALSE     SEV-SNP is disabled.
1631
1632 **/
1633 STATIC
1634 BOOLEAN
1635 SnpEnabled (
1636   VOID
1637   )
1638 {
1639   MSR_SEV_STATUS_REGISTER  Msr;
1640
1641   Msr.Uint32 = AsmReadMsr32 (MSR_SEV_STATUS);
1642
1643   return !!Msr.Bits.SevSnpBit;
1644 }
1645
1646 /**
1647   Calculate the total XSAVE area size for enabled XSAVE areas
1648
1649   @param[in]      XFeaturesEnabled  Bit-mask of enabled XSAVE features/areas as
1650                                     indicated by XCR0/MSR_IA32_XSS bits
1651   @param[in]      XSaveBaseSize     Base/legacy XSAVE area size (e.g. when
1652                                     XCR0 is 1)
1653   @param[in, out] XSaveSize         Pointer to storage for calculated XSAVE area
1654                                     size
1655   @param[in]      Compacted         Whether or not the calculation is for the
1656                                     normal XSAVE area size (leaf 0xD,0x0,EBX) or
1657                                     compacted XSAVE area size (leaf 0xD,0x1,EBX)
1658
1659
1660   @retval TRUE                      XSAVE size calculation was successful.
1661   @retval FALSE                     XSAVE size calculation was unsuccessful.
1662 **/
1663 STATIC
1664 BOOLEAN
1665 GetCpuidXSaveSize (
1666   IN     UINT64   XFeaturesEnabled,
1667   IN     UINT32   XSaveBaseSize,
1668   IN OUT UINT32   *XSaveSize,
1669   IN     BOOLEAN  Compacted
1670   )
1671 {
1672   SEV_SNP_CPUID_INFO  *CpuidInfo;
1673   UINT64              XFeaturesFound = 0;
1674   UINT32              Idx;
1675
1676   *XSaveSize = XSaveBaseSize;
1677   CpuidInfo  = (SEV_SNP_CPUID_INFO *)(UINT64)PcdGet32 (PcdOvmfCpuidBase);
1678
1679   for (Idx = 0; Idx < CpuidInfo->Count; Idx++) {
1680     SEV_SNP_CPUID_FUNCTION  *CpuidFn = &CpuidInfo->function[Idx];
1681
1682     if (!((CpuidFn->EaxIn == 0xD) &&
1683           ((CpuidFn->EcxIn == 0) || (CpuidFn->EcxIn == 1))))
1684     {
1685       continue;
1686     }
1687
1688     if (XFeaturesFound & (1ULL << CpuidFn->EcxIn) ||
1689         !(XFeaturesEnabled & (1ULL << CpuidFn->EcxIn)))
1690     {
1691       continue;
1692     }
1693
1694     XFeaturesFound |= (1ULL << CpuidFn->EcxIn);
1695     if (Compacted) {
1696       *XSaveSize += CpuidFn->Eax;
1697     } else {
1698       *XSaveSize = MAX (*XSaveSize, CpuidFn->Eax + CpuidFn->Ebx);
1699     }
1700   }
1701
1702   /*
1703    * Either the guest set unsupported XCR0/XSS bits, or the corresponding
1704    * entries in the CPUID table were not present. This is an invalid state.
1705    */
1706   if (XFeaturesFound != (XFeaturesEnabled & ~3UL)) {
1707     return FALSE;
1708   }
1709
1710   return TRUE;
1711 }
1712
1713 /**
1714   Check if a CPUID leaf/function is indexed via ECX sub-leaf/sub-function
1715
1716   @param[in]      EaxIn        EAX input for cpuid instruction
1717
1718   @retval FALSE                cpuid leaf/function is not indexed by ECX input
1719   @retval TRUE                 cpuid leaf/function is indexed by ECX input
1720
1721 **/
1722 STATIC
1723 BOOLEAN
1724 IsFunctionIndexed (
1725   IN     UINT32  EaxIn
1726   )
1727 {
1728   switch (EaxIn) {
1729     case CPUID_CACHE_PARAMS:
1730     case CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS:
1731     case CPUID_EXTENDED_TOPOLOGY:
1732     case CPUID_EXTENDED_STATE:
1733     case CPUID_INTEL_RDT_MONITORING:
1734     case CPUID_INTEL_RDT_ALLOCATION:
1735     case CPUID_INTEL_SGX:
1736     case CPUID_INTEL_PROCESSOR_TRACE:
1737     case CPUID_DETERMINISTIC_ADDRESS_TRANSLATION_PARAMETERS:
1738     case CPUID_V2_EXTENDED_TOPOLOGY:
1739     case 0x8000001D: /* Cache Topology Information */
1740       return TRUE;
1741   }
1742
1743   return FALSE;
1744 }
1745
1746 /**
1747   Fetch CPUID leaf/function via SEV-SNP CPUID table.
1748
1749   @param[in, out] Ghcb         Pointer to the Guest-Hypervisor Communication
1750                                Block
1751   @param[in]      EaxIn        EAX input for cpuid instruction
1752   @param[in]      EcxIn        ECX input for cpuid instruction
1753   @param[in]      Xcr0In       XCR0 at time of cpuid instruction
1754   @param[in, out] Eax          Pointer to store leaf's EAX value
1755   @param[in, out] Ebx          Pointer to store leaf's EBX value
1756   @param[in, out] Ecx          Pointer to store leaf's ECX value
1757   @param[in, out] Edx          Pointer to store leaf's EDX value
1758   @param[in, out] Status       Pointer to store status from VMGEXIT (always 0
1759                                unless return value indicates failure)
1760   @param[in, out] Unsupported  Pointer to store indication of unsupported
1761                                VMGEXIT (always false unless return value
1762                                indicates failure)
1763
1764   @retval TRUE                 CPUID leaf fetch successfully.
1765   @retval FALSE                Error occurred while fetching CPUID leaf. Callers
1766                                should Status and Unsupported and handle
1767                                accordingly if they indicate a more precise
1768                                error condition.
1769
1770 **/
1771 STATIC
1772 BOOLEAN
1773 GetCpuidFw (
1774   IN OUT GHCB     *Ghcb,
1775   IN     UINT32   EaxIn,
1776   IN     UINT32   EcxIn,
1777   IN     UINT64   XCr0,
1778   IN OUT UINT32   *Eax,
1779   IN OUT UINT32   *Ebx,
1780   IN OUT UINT32   *Ecx,
1781   IN OUT UINT32   *Edx,
1782   IN OUT UINT64   *Status,
1783   IN OUT BOOLEAN  *Unsupported
1784   )
1785 {
1786   SEV_SNP_CPUID_INFO  *CpuidInfo;
1787   BOOLEAN             Found;
1788   UINT32              Idx;
1789
1790   CpuidInfo = (SEV_SNP_CPUID_INFO *)(UINT64)PcdGet32 (PcdOvmfCpuidBase);
1791   Found     = FALSE;
1792
1793   for (Idx = 0; Idx < CpuidInfo->Count; Idx++) {
1794     SEV_SNP_CPUID_FUNCTION  *CpuidFn = &CpuidInfo->function[Idx];
1795
1796     if (CpuidFn->EaxIn != EaxIn) {
1797       continue;
1798     }
1799
1800     if (IsFunctionIndexed (CpuidFn->EaxIn) && (CpuidFn->EcxIn != EcxIn)) {
1801       continue;
1802     }
1803
1804     *Eax = CpuidFn->Eax;
1805     *Ebx = CpuidFn->Ebx;
1806     *Ecx = CpuidFn->Ecx;
1807     *Edx = CpuidFn->Edx;
1808
1809     Found = TRUE;
1810     break;
1811   }
1812
1813   if (!Found) {
1814     *Eax = *Ebx = *Ecx = *Edx = 0;
1815     goto Out;
1816   }
1817
1818   if (EaxIn == CPUID_VERSION_INFO) {
1819     IA32_CR4  Cr4;
1820     UINT32    Ebx2;
1821     UINT32    Edx2;
1822
1823     if (!GetCpuidHyp (
1824            Ghcb,
1825            EaxIn,
1826            EcxIn,
1827            XCr0,
1828            NULL,
1829            &Ebx2,
1830            NULL,
1831            &Edx2,
1832            Status,
1833            Unsupported
1834            ))
1835     {
1836       return FALSE;
1837     }
1838
1839     /* initial APIC ID */
1840     *Ebx = (*Ebx & 0x00FFFFFF) | (Ebx2 & 0xFF000000);
1841     /* APIC enabled bit */
1842     *Edx = (*Edx & ~BIT9) | (Edx2 & BIT9);
1843     /* OSXSAVE enabled bit */
1844     Cr4.UintN = AsmReadCr4 ();
1845     *Ecx      = (Cr4.Bits.OSXSAVE) ? (*Ecx & ~BIT27) | (*Ecx & BIT27)
1846                               : (*Ecx & ~BIT27);
1847   } else if (EaxIn == CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS) {
1848     IA32_CR4  Cr4;
1849
1850     Cr4.UintN = AsmReadCr4 ();
1851     /* OSPKE enabled bit */
1852     *Ecx = (Cr4.Bits.PKE) ? (*Ecx | BIT4) : (*Ecx & ~BIT4);
1853   } else if (EaxIn == CPUID_EXTENDED_TOPOLOGY) {
1854     if (!GetCpuidHyp (
1855            Ghcb,
1856            EaxIn,
1857            EcxIn,
1858            XCr0,
1859            NULL,
1860            NULL,
1861            NULL,
1862            Edx,
1863            Status,
1864            Unsupported
1865            ))
1866     {
1867       return FALSE;
1868     }
1869   } else if ((EaxIn == CPUID_EXTENDED_STATE) && ((EcxIn == 0) || (EcxIn == 1))) {
1870     MSR_IA32_XSS_REGISTER  XssMsr;
1871     BOOLEAN                Compacted;
1872     UINT32                 XSaveSize;
1873
1874     XssMsr.Uint64 = 0;
1875     if (EcxIn == 1) {
1876       /*
1877        * The PPR and APM aren't clear on what size should be encoded in
1878        * 0xD:0x1:EBX when compaction is not enabled by either XSAVEC or
1879        * XSAVES, as these are generally fixed to 1 on real CPUs. Report
1880        * this undefined case as an error.
1881        */
1882       if (!(*Eax & (BIT3 | BIT1))) {
1883         /* (XSAVES | XSAVEC) */
1884         return FALSE;
1885       }
1886
1887       Compacted     = TRUE;
1888       XssMsr.Uint64 = AsmReadMsr64 (MSR_IA32_XSS);
1889     }
1890
1891     if (!GetCpuidXSaveSize (
1892            XCr0 | XssMsr.Uint64,
1893            *Ebx,
1894            &XSaveSize,
1895            Compacted
1896            ))
1897     {
1898       return FALSE;
1899     }
1900
1901     *Ebx = XSaveSize;
1902   } else if (EaxIn == 0x8000001E) {
1903     UINT32  Ebx2;
1904     UINT32  Ecx2;
1905
1906     /* extended APIC ID */
1907     if (!GetCpuidHyp (
1908            Ghcb,
1909            EaxIn,
1910            EcxIn,
1911            XCr0,
1912            Eax,
1913            &Ebx2,
1914            &Ecx2,
1915            NULL,
1916            Status,
1917            Unsupported
1918            ))
1919     {
1920       return FALSE;
1921     }
1922
1923     /* compute ID */
1924     *Ebx = (*Ebx & 0xFFFFFF00) | (Ebx2 & 0x000000FF);
1925     /* node ID */
1926     *Ecx = (*Ecx & 0xFFFFFF00) | (Ecx2 & 0x000000FF);
1927   }
1928
1929 Out:
1930   *Status      = 0;
1931   *Unsupported = FALSE;
1932   return TRUE;
1933 }
1934
1935 /**
1936   Handle a CPUID event.
1937
1938   Use VMGEXIT instruction or CPUID table to handle a CPUID event.
1939
1940   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1941                                    Block
1942   @param[in, out] Regs             x64 processor context
1943   @param[in]      InstructionData  Instruction parsing context
1944
1945   @retval 0                        Event handled successfully
1946   @return                          New exception value to propagate
1947
1948 **/
1949 STATIC
1950 UINT64
1951 CpuidExit (
1952   IN OUT GHCB                     *Ghcb,
1953   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1954   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1955   )
1956 {
1957   BOOLEAN  Unsupported;
1958   UINT64   Status;
1959   UINT32   EaxIn;
1960   UINT32   EcxIn;
1961   UINT64   XCr0;
1962   UINT32   Eax;
1963   UINT32   Ebx;
1964   UINT32   Ecx;
1965   UINT32   Edx;
1966
1967   EaxIn = (UINT32)(UINTN)Regs->Rax;
1968   EcxIn = (UINT32)(UINTN)Regs->Rcx;
1969
1970   if (EaxIn == CPUID_EXTENDED_STATE) {
1971     IA32_CR4  Cr4;
1972
1973     Cr4.UintN           = AsmReadCr4 ();
1974     Ghcb->SaveArea.XCr0 = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
1975     XCr0                = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
1976   }
1977
1978   if (SnpEnabled ()) {
1979     if (!GetCpuidFw (
1980            Ghcb,
1981            EaxIn,
1982            EcxIn,
1983            XCr0,
1984            &Eax,
1985            &Ebx,
1986            &Ecx,
1987            &Edx,
1988            &Status,
1989            &Unsupported
1990            ))
1991     {
1992       goto CpuidFail;
1993     }
1994   } else {
1995     if (!GetCpuidHyp (
1996            Ghcb,
1997            EaxIn,
1998            EcxIn,
1999            XCr0,
2000            &Eax,
2001            &Ebx,
2002            &Ecx,
2003            &Edx,
2004            &Status,
2005            &Unsupported
2006            ))
2007     {
2008       goto CpuidFail;
2009     }
2010   }
2011
2012   Regs->Rax = Eax;
2013   Regs->Rbx = Ebx;
2014   Regs->Rcx = Ecx;
2015   Regs->Rdx = Edx;
2016
2017   return 0;
2018
2019 CpuidFail:
2020   if (Unsupported) {
2021     return UnsupportedExit (Ghcb, Regs, InstructionData);
2022   }
2023
2024   return Status;
2025 }
2026
2027 /**
2028   Handle a RDPMC event.
2029
2030   Use the VMGEXIT instruction to handle a RDPMC event.
2031
2032   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
2033                                    Block
2034   @param[in, out] Regs             x64 processor context
2035   @param[in]      InstructionData  Instruction parsing context
2036
2037   @retval 0                        Event handled successfully
2038   @return                          New exception value to propagate
2039
2040 **/
2041 STATIC
2042 UINT64
2043 RdpmcExit (
2044   IN OUT GHCB                     *Ghcb,
2045   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
2046   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
2047   )
2048 {
2049   UINT64  Status;
2050
2051   Ghcb->SaveArea.Rcx = Regs->Rcx;
2052   VmgSetOffsetValid (Ghcb, GhcbRcx);
2053
2054   Status = VmgExit (Ghcb, SVM_EXIT_RDPMC, 0, 0);
2055   if (Status != 0) {
2056     return Status;
2057   }
2058
2059   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
2060       !VmgIsOffsetValid (Ghcb, GhcbRdx))
2061   {
2062     return UnsupportedExit (Ghcb, Regs, InstructionData);
2063   }
2064
2065   Regs->Rax = Ghcb->SaveArea.Rax;
2066   Regs->Rdx = Ghcb->SaveArea.Rdx;
2067
2068   return 0;
2069 }
2070
2071 /**
2072   Handle a RDTSC event.
2073
2074   Use the VMGEXIT instruction to handle a RDTSC event.
2075
2076   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
2077                                    Block
2078   @param[in, out] Regs             x64 processor context
2079   @param[in]      InstructionData  Instruction parsing context
2080
2081   @retval 0                        Event handled successfully
2082   @return                          New exception value to propagate
2083
2084 **/
2085 STATIC
2086 UINT64
2087 RdtscExit (
2088   IN OUT GHCB                     *Ghcb,
2089   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
2090   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
2091   )
2092 {
2093   UINT64  Status;
2094
2095   Status = VmgExit (Ghcb, SVM_EXIT_RDTSC, 0, 0);
2096   if (Status != 0) {
2097     return Status;
2098   }
2099
2100   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
2101       !VmgIsOffsetValid (Ghcb, GhcbRdx))
2102   {
2103     return UnsupportedExit (Ghcb, Regs, InstructionData);
2104   }
2105
2106   Regs->Rax = Ghcb->SaveArea.Rax;
2107   Regs->Rdx = Ghcb->SaveArea.Rdx;
2108
2109   return 0;
2110 }
2111
2112 /**
2113   Handle a DR7 register write event.
2114
2115   Use the VMGEXIT instruction to handle a DR7 write event.
2116
2117   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
2118                                    Block
2119   @param[in, out] Regs             x64 processor context
2120   @param[in]      InstructionData  Instruction parsing context
2121
2122   @retval 0                        Event handled successfully
2123   @return                          New exception value to propagate
2124
2125 **/
2126 STATIC
2127 UINT64
2128 Dr7WriteExit (
2129   IN OUT GHCB                     *Ghcb,
2130   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
2131   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
2132   )
2133 {
2134   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
2135   SEV_ES_PER_CPU_DATA            *SevEsData;
2136   UINT64                         *Register;
2137   UINT64                         Status;
2138
2139   Ext       = &InstructionData->Ext;
2140   SevEsData = (SEV_ES_PER_CPU_DATA *)(Ghcb + 1);
2141
2142   DecodeModRm (Regs, InstructionData);
2143
2144   //
2145   // MOV DRn always treats MOD == 3 no matter how encoded
2146   //
2147   Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
2148
2149   //
2150   // Using a value of 0 for ExitInfo1 means RAX holds the value
2151   //
2152   Ghcb->SaveArea.Rax = *Register;
2153   VmgSetOffsetValid (Ghcb, GhcbRax);
2154
2155   Status = VmgExit (Ghcb, SVM_EXIT_DR7_WRITE, 0, 0);
2156   if (Status != 0) {
2157     return Status;
2158   }
2159
2160   SevEsData->Dr7       = *Register;
2161   SevEsData->Dr7Cached = 1;
2162
2163   return 0;
2164 }
2165
2166 /**
2167   Handle a DR7 register read event.
2168
2169   Use the VMGEXIT instruction to handle a DR7 read event.
2170
2171   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
2172                                    Block
2173   @param[in, out] Regs             x64 processor context
2174   @param[in]      InstructionData  Instruction parsing context
2175
2176   @retval 0                        Event handled successfully
2177
2178 **/
2179 STATIC
2180 UINT64
2181 Dr7ReadExit (
2182   IN OUT GHCB                     *Ghcb,
2183   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
2184   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
2185   )
2186 {
2187   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
2188   SEV_ES_PER_CPU_DATA            *SevEsData;
2189   UINT64                         *Register;
2190
2191   Ext       = &InstructionData->Ext;
2192   SevEsData = (SEV_ES_PER_CPU_DATA *)(Ghcb + 1);
2193
2194   DecodeModRm (Regs, InstructionData);
2195
2196   //
2197   // MOV DRn always treats MOD == 3 no matter how encoded
2198   //
2199   Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
2200
2201   //
2202   // If there is a cached valued for DR7, return that. Otherwise return the
2203   // DR7 standard reset value of 0x400 (no debug breakpoints set).
2204   //
2205   *Register = (SevEsData->Dr7Cached == 1) ? SevEsData->Dr7 : 0x400;
2206
2207   return 0;
2208 }
2209
2210 /**
2211   Handle a #VC exception.
2212
2213   Performs the necessary processing to handle a #VC exception.
2214
2215   @param[in, out]  Ghcb           Pointer to the GHCB
2216   @param[in, out]  ExceptionType  Pointer to an EFI_EXCEPTION_TYPE to be set
2217                                   as value to use on error.
2218   @param[in, out]  SystemContext  Pointer to EFI_SYSTEM_CONTEXT
2219
2220   @retval  EFI_SUCCESS            Exception handled
2221   @retval  EFI_UNSUPPORTED        #VC not supported, (new) exception value to
2222                                   propagate provided
2223   @retval  EFI_PROTOCOL_ERROR     #VC handling failed, (new) exception value to
2224                                   propagate provided
2225
2226 **/
2227 EFI_STATUS
2228 EFIAPI
2229 InternalVmgExitHandleVc (
2230   IN OUT GHCB                *Ghcb,
2231   IN OUT EFI_EXCEPTION_TYPE  *ExceptionType,
2232   IN OUT EFI_SYSTEM_CONTEXT  SystemContext
2233   )
2234 {
2235   EFI_SYSTEM_CONTEXT_X64   *Regs;
2236   NAE_EXIT                 NaeExit;
2237   SEV_ES_INSTRUCTION_DATA  InstructionData;
2238   UINT64                   ExitCode, Status;
2239   EFI_STATUS               VcRet;
2240   BOOLEAN                  InterruptState;
2241
2242   VcRet = EFI_SUCCESS;
2243
2244   Regs = SystemContext.SystemContextX64;
2245
2246   VmgInit (Ghcb, &InterruptState);
2247
2248   ExitCode = Regs->ExceptionData;
2249   switch (ExitCode) {
2250     case SVM_EXIT_DR7_READ:
2251       NaeExit = Dr7ReadExit;
2252       break;
2253
2254     case SVM_EXIT_DR7_WRITE:
2255       NaeExit = Dr7WriteExit;
2256       break;
2257
2258     case SVM_EXIT_RDTSC:
2259       NaeExit = RdtscExit;
2260       break;
2261
2262     case SVM_EXIT_RDPMC:
2263       NaeExit = RdpmcExit;
2264       break;
2265
2266     case SVM_EXIT_CPUID:
2267       NaeExit = CpuidExit;
2268       break;
2269
2270     case SVM_EXIT_INVD:
2271       NaeExit = InvdExit;
2272       break;
2273
2274     case SVM_EXIT_IOIO_PROT:
2275       NaeExit = IoioExit;
2276       break;
2277
2278     case SVM_EXIT_MSR:
2279       NaeExit = MsrExit;
2280       break;
2281
2282     case SVM_EXIT_VMMCALL:
2283       NaeExit = VmmCallExit;
2284       break;
2285
2286     case SVM_EXIT_RDTSCP:
2287       NaeExit = RdtscpExit;
2288       break;
2289
2290     case SVM_EXIT_WBINVD:
2291       NaeExit = WbinvdExit;
2292       break;
2293
2294     case SVM_EXIT_MONITOR:
2295       NaeExit = MonitorExit;
2296       break;
2297
2298     case SVM_EXIT_MWAIT:
2299       NaeExit = MwaitExit;
2300       break;
2301
2302     case SVM_EXIT_NPF:
2303       NaeExit = MmioExit;
2304       break;
2305
2306     default:
2307       NaeExit = UnsupportedExit;
2308   }
2309
2310   InitInstructionData (&InstructionData, Ghcb, Regs);
2311
2312   Status = NaeExit (Ghcb, Regs, &InstructionData);
2313   if (Status == 0) {
2314     Regs->Rip += InstructionLength (&InstructionData);
2315   } else {
2316     GHCB_EVENT_INJECTION  Event;
2317
2318     Event.Uint64 = Status;
2319     if (Event.Elements.ErrorCodeValid != 0) {
2320       Regs->ExceptionData = Event.Elements.ErrorCode;
2321     } else {
2322       Regs->ExceptionData = 0;
2323     }
2324
2325     *ExceptionType = Event.Elements.Vector;
2326
2327     VcRet = EFI_PROTOCOL_ERROR;
2328   }
2329
2330   VmgDone (Ghcb, InterruptState);
2331
2332   return VcRet;
2333 }
2334
2335 /**
2336   Routine to allow ASSERT from within #VC.
2337
2338   @param[in, out]  SevEsData  Pointer to the per-CPU data
2339
2340 **/
2341 VOID
2342 EFIAPI
2343 VmgExitIssueAssert (
2344   IN OUT SEV_ES_PER_CPU_DATA  *SevEsData
2345   )
2346 {
2347   //
2348   // Progress will be halted, so set VcCount to allow for ASSERT output
2349   // to be seen.
2350   //
2351   SevEsData->VcCount = 0;
2352
2353   ASSERT (FALSE);
2354   CpuDeadLoop ();
2355 }