OvmfPkg/Library/VmgExitLib/VmgExitVcHandler.c

   1 /** @file
   2   X64 #VC Exception Handler functon.
   3
   4   Copyright (C) 2020, Advanced Micro Devices, Inc. All rights reserved.<BR>
   5   SPDX-License-Identifier: BSD-2-Clause-Patent
   6
   7 **/
   8
   9 #include <Base.h>
  10 #include <Uefi.h>
  11 #include <Library/BaseMemoryLib.h>
  12 #include <Library/VmgExitLib.h>
  13 #include <Register/Amd/Msr.h>
  14 #include <Register/Intel/Cpuid.h>
  15 #include <IndustryStandard/InstructionParsing.h>
  16
  17 //
  18 // Instruction execution mode definition
  19 //
  20 typedef enum {
  21   LongMode64Bit        = 0,
  22   LongModeCompat32Bit,
  23   LongModeCompat16Bit,
  24 } SEV_ES_INSTRUCTION_MODE;
  25
  26 //
  27 // Instruction size definition (for operand and address)
  28 //
  29 typedef enum {
  30   Size8Bits            = 0,
  31   Size16Bits,
  32   Size32Bits,
  33   Size64Bits,
  34 } SEV_ES_INSTRUCTION_SIZE;
  35
  36 //
  37 // Intruction segment definition
  38 //
  39 typedef enum {
  40   SegmentEs            = 0,
  41   SegmentCs,
  42   SegmentSs,
  43   SegmentDs,
  44   SegmentFs,
  45   SegmentGs,
  46 } SEV_ES_INSTRUCTION_SEGMENT;
  47
  48 //
  49 // Instruction rep function definition
  50 //
  51 typedef enum {
  52   RepNone              = 0,
  53   RepZ,
  54   RepNZ,
  55 } SEV_ES_INSTRUCTION_REP;
  56
  57 typedef struct {
  58   UINT8  Rm;
  59   UINT8  Reg;
  60   UINT8  Mod;
  61 } SEV_ES_INSTRUCTION_MODRM_EXT;
  62
  63 typedef struct {
  64   UINT8  Base;
  65   UINT8  Index;
  66   UINT8  Scale;
  67 } SEV_ES_INSTRUCTION_SIB_EXT;
  68
  69 //
  70 // Instruction opcode definition
  71 //
  72 typedef struct {
  73   SEV_ES_INSTRUCTION_MODRM_EXT  ModRm;
  74
  75   SEV_ES_INSTRUCTION_SIB_EXT    Sib;
  76
  77   UINTN                         RegData;
  78   UINTN                         RmData;
  79 } SEV_ES_INSTRUCTION_OPCODE_EXT;
  80
  81 //
  82 // Instruction parsing context definition
  83 //
  84 typedef struct {
  85   GHCB                           *Ghcb;
  86
  87   SEV_ES_INSTRUCTION_MODE        Mode;
  88   SEV_ES_INSTRUCTION_SIZE        DataSize;
  89   SEV_ES_INSTRUCTION_SIZE        AddrSize;
  90   BOOLEAN                        SegmentSpecified;
  91   SEV_ES_INSTRUCTION_SEGMENT     Segment;
  92   SEV_ES_INSTRUCTION_REP         RepMode;
  93
  94   UINT8                          *Begin;
  95   UINT8                          *End;
  96
  97   UINT8                          *Prefixes;
  98   UINT8                          *OpCodes;
  99   UINT8                          *Displacement;
 100   UINT8                          *Immediate;
 101
 102   INSTRUCTION_REX_PREFIX         RexPrefix;
 103
 104   BOOLEAN                        ModRmPresent;
 105   INSTRUCTION_MODRM              ModRm;
 106
 107   BOOLEAN                        SibPresent;
 108   INSTRUCTION_SIB                Sib;
 109
 110   UINTN                          PrefixSize;
 111   UINTN                          OpCodeSize;
 112   UINTN                          DisplacementSize;
 113   UINTN                          ImmediateSize;
 114
 115   SEV_ES_INSTRUCTION_OPCODE_EXT  Ext;
 116 } SEV_ES_INSTRUCTION_DATA;
 117
 118 //
 119 // Non-automatic Exit function prototype
 120 //
 121 typedef
 122 UINT64
 123 (*NAE_EXIT) (
 124   GHCB                     *Ghcb,
 125   EFI_SYSTEM_CONTEXT_X64   *Regs,
 126   SEV_ES_INSTRUCTION_DATA  *InstructionData
 127   );
 128
 129
 130 /**
 131   Checks the GHCB to determine if the specified register has been marked valid.
 132
 133   The ValidBitmap area represents the areas of the GHCB that have been marked
 134   valid. Return an indication of whether the area of the GHCB that holds the
 135   specified register has been marked valid.
 136
 137   @param[in] Ghcb    Pointer to the Guest-Hypervisor Communication Block
 138   @param[in] Reg     Offset in the GHCB of the register to check
 139
 140   @retval TRUE       Register has been marked vald in the GHCB
 141   @retval FALSE      Register has not been marked valid in the GHCB
 142
 143 **/
 144 STATIC
 145 BOOLEAN
 146 GhcbIsRegValid (
 147   IN GHCB                *Ghcb,
 148   IN GHCB_REGISTER       Reg
 149   )
 150 {
 151   UINT32  RegIndex;
 152   UINT32  RegBit;
 153
 154   RegIndex = Reg / 8;
 155   RegBit   = Reg & 0x07;
 156
 157   return ((Ghcb->SaveArea.ValidBitmap[RegIndex] & (1 << RegBit)) != 0);
 158 }
 159
 160 /**
 161   Marks a register as valid in the GHCB.
 162
 163   The ValidBitmap area represents the areas of the GHCB that have been marked
 164   valid. Set the area of the GHCB that holds the specified register as valid.
 165
 166   @param[in, out] Ghcb    Pointer to the Guest-Hypervisor Communication Block
 167   @param[in] Reg          Offset in the GHCB of the register to mark valid
 168
 169 **/
 170 STATIC
 171 VOID
 172 GhcbSetRegValid (
 173   IN OUT GHCB                *Ghcb,
 174   IN     GHCB_REGISTER       Reg
 175   )
 176 {
 177   UINT32  RegIndex;
 178   UINT32  RegBit;
 179
 180   RegIndex = Reg / 8;
 181   RegBit   = Reg & 0x07;
 182
 183   Ghcb->SaveArea.ValidBitmap[RegIndex] |= (1 << RegBit);
 184 }
 185
 186 /**
 187   Return a pointer to the contents of the specified register.
 188
 189   Based upon the input register, return a pointer to the registers contents
 190   in the x86 processor context.
 191
 192   @param[in] Regs      x64 processor context
 193   @param[in] Register  Register to obtain pointer for
 194
 195   @return              Pointer to the contents of the requested register
 196
 197 **/
 198 STATIC
 199 UINT64 *
 200 GetRegisterPointer (
 201   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 202   IN UINT8                    Register
 203   )
 204 {
 205   UINT64 *Reg;
 206
 207   switch (Register) {
 208   case 0:
 209     Reg = &Regs->Rax;
 210     break;
 211   case 1:
 212     Reg = &Regs->Rcx;
 213     break;
 214   case 2:
 215     Reg = &Regs->Rdx;
 216     break;
 217   case 3:
 218     Reg = &Regs->Rbx;
 219     break;
 220   case 4:
 221     Reg = &Regs->Rsp;
 222     break;
 223   case 5:
 224     Reg = &Regs->Rbp;
 225     break;
 226   case 6:
 227     Reg = &Regs->Rsi;
 228     break;
 229   case 7:
 230     Reg = &Regs->Rdi;
 231     break;
 232   case 8:
 233     Reg = &Regs->R8;
 234     break;
 235   case 9:
 236     Reg = &Regs->R9;
 237     break;
 238   case 10:
 239     Reg = &Regs->R10;
 240     break;
 241   case 11:
 242     Reg = &Regs->R11;
 243     break;
 244   case 12:
 245     Reg = &Regs->R12;
 246     break;
 247   case 13:
 248     Reg = &Regs->R13;
 249     break;
 250   case 14:
 251     Reg = &Regs->R14;
 252     break;
 253   case 15:
 254     Reg = &Regs->R15;
 255     break;
 256   default:
 257     Reg = NULL;
 258   }
 259   ASSERT (Reg != NULL);
 260
 261   return Reg;
 262 }
 263
 264 /**
 265   Update the instruction parsing context for displacement bytes.
 266
 267   @param[in, out] InstructionData  Instruction parsing context
 268   @param[in]      Size             The instruction displacement size
 269
 270 **/
 271 STATIC
 272 VOID
 273 UpdateForDisplacement (
 274   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 275   IN     UINTN                    Size
 276   )
 277 {
 278   InstructionData->DisplacementSize = Size;
 279   InstructionData->Immediate += Size;
 280   InstructionData->End += Size;
 281 }
 282
 283 /**
 284   Determine if an instruction address if RIP relative.
 285
 286   Examine the instruction parsing context to determine if the address offset
 287   is relative to the instruction pointer.
 288
 289   @param[in] InstructionData  Instruction parsing context
 290
 291   @retval TRUE                Instruction addressing is RIP relative
 292   @retval FALSE               Instruction addressing is not RIP relative
 293
 294 **/
 295 STATIC
 296 BOOLEAN
 297 IsRipRelative (
 298   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 299   )
 300 {
 301   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 302
 303   Ext = &InstructionData->Ext;
 304
 305   return ((InstructionData->Mode == LongMode64Bit) &&
 306           (Ext->ModRm.Mod == 0) &&
 307           (Ext->ModRm.Rm == 5)  &&
 308           (InstructionData->SibPresent == FALSE));
 309 }
 310
 311 /**
 312   Return the effective address of a memory operand.
 313
 314   Examine the instruction parsing context to obtain the effective memory
 315   address of a memory operand.
 316
 317   @param[in] Regs             x64 processor context
 318   @param[in] InstructionData  Instruction parsing context
 319
 320   @return                     The memory operand effective address
 321
 322 **/
 323 STATIC
 324 UINT64
 325 GetEffectiveMemoryAddress (
 326   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 327   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 328   )
 329 {
 330   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 331   UINT64                         EffectiveAddress;
 332
 333   Ext = &InstructionData->Ext;
 334   EffectiveAddress = 0;
 335
 336   if (IsRipRelative (InstructionData)) {
 337     //
 338     // RIP-relative displacement is a 32-bit signed value
 339     //
 340     INT32 RipRelative;
 341
 342     RipRelative = *(INT32 *) InstructionData->Displacement;
 343
 344     UpdateForDisplacement (InstructionData, 4);
 345
 346     //
 347     // Negative displacement is handled by standard UINT64 wrap-around.
 348     //
 349     return Regs->Rip + (UINT64) RipRelative;
 350   }
 351
 352   switch (Ext->ModRm.Mod) {
 353   case 1:
 354     UpdateForDisplacement (InstructionData, 1);
 355     EffectiveAddress += (UINT64) (*(INT8 *) (InstructionData->Displacement));
 356     break;
 357   case 2:
 358     switch (InstructionData->AddrSize) {
 359     case Size16Bits:
 360       UpdateForDisplacement (InstructionData, 2);
 361       EffectiveAddress += (UINT64) (*(INT16 *) (InstructionData->Displacement));
 362       break;
 363     default:
 364       UpdateForDisplacement (InstructionData, 4);
 365       EffectiveAddress += (UINT64) (*(INT32 *) (InstructionData->Displacement));
 366       break;
 367     }
 368     break;
 369   }
 370
 371   if (InstructionData->SibPresent) {
 372     INT64  Displacement;
 373
 374     if (Ext->Sib.Index != 4) {
 375       CopyMem (
 376         &Displacement,
 377         GetRegisterPointer (Regs, Ext->Sib.Index),
 378         sizeof (Displacement)
 379         );
 380       Displacement *= (INT64)(1 << Ext->Sib.Scale);
 381
 382       //
 383       // Negative displacement is handled by standard UINT64 wrap-around.
 384       //
 385       EffectiveAddress += (UINT64) Displacement;
 386     }
 387
 388     if ((Ext->Sib.Base != 5) || Ext->ModRm.Mod) {
 389       EffectiveAddress += *GetRegisterPointer (Regs, Ext->Sib.Base);
 390     } else {
 391       UpdateForDisplacement (InstructionData, 4);
 392       EffectiveAddress += (UINT64) (*(INT32 *) (InstructionData->Displacement));
 393     }
 394   } else {
 395     EffectiveAddress += *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 396   }
 397
 398   return EffectiveAddress;
 399 }
 400
 401 /**
 402   Decode a ModRM byte.
 403
 404   Examine the instruction parsing context to decode a ModRM byte and the SIB
 405   byte, if present.
 406
 407   @param[in]      Regs             x64 processor context
 408   @param[in, out] InstructionData  Instruction parsing context
 409
 410 **/
 411 STATIC
 412 VOID
 413 DecodeModRm (
 414   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 415   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 416   )
 417 {
 418   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 419   INSTRUCTION_REX_PREFIX         *RexPrefix;
 420   INSTRUCTION_MODRM              *ModRm;
 421   INSTRUCTION_SIB                *Sib;
 422
 423   RexPrefix = &InstructionData->RexPrefix;
 424   Ext = &InstructionData->Ext;
 425   ModRm = &InstructionData->ModRm;
 426   Sib = &InstructionData->Sib;
 427
 428   InstructionData->ModRmPresent = TRUE;
 429   ModRm->Uint8 = *(InstructionData->End);
 430
 431   InstructionData->Displacement++;
 432   InstructionData->Immediate++;
 433   InstructionData->End++;
 434
 435   Ext->ModRm.Mod = ModRm->Bits.Mod;
 436   Ext->ModRm.Reg = (RexPrefix->Bits.BitR << 3) | ModRm->Bits.Reg;
 437   Ext->ModRm.Rm  = (RexPrefix->Bits.BitB << 3) | ModRm->Bits.Rm;
 438
 439   Ext->RegData = *GetRegisterPointer (Regs, Ext->ModRm.Reg);
 440
 441   if (Ext->ModRm.Mod == 3) {
 442     Ext->RmData = *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 443   } else {
 444     if (ModRm->Bits.Rm == 4) {
 445       InstructionData->SibPresent = TRUE;
 446       Sib->Uint8 = *(InstructionData->End);
 447
 448       InstructionData->Displacement++;
 449       InstructionData->Immediate++;
 450       InstructionData->End++;
 451
 452       Ext->Sib.Scale = Sib->Bits.Scale;
 453       Ext->Sib.Index = (RexPrefix->Bits.BitX << 3) | Sib->Bits.Index;
 454       Ext->Sib.Base  = (RexPrefix->Bits.BitB << 3) | Sib->Bits.Base;
 455     }
 456
 457     Ext->RmData = GetEffectiveMemoryAddress (Regs, InstructionData);
 458   }
 459 }
 460
 461 /**
 462   Decode instruction prefixes.
 463
 464   Parse the instruction data to track the instruction prefixes that have
 465   been used.
 466
 467   @param[in]      Regs             x64 processor context
 468   @param[in, out] InstructionData  Instruction parsing context
 469
 470 **/
 471 STATIC
 472 VOID
 473 DecodePrefixes (
 474   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 475   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 476   )
 477 {
 478   SEV_ES_INSTRUCTION_MODE  Mode;
 479   SEV_ES_INSTRUCTION_SIZE  ModeDataSize;
 480   SEV_ES_INSTRUCTION_SIZE  ModeAddrSize;
 481   UINT8                    *Byte;
 482
 483   //
 484   // Always in 64-bit mode
 485   //
 486   Mode = LongMode64Bit;
 487   ModeDataSize = Size32Bits;
 488   ModeAddrSize = Size64Bits;
 489
 490   InstructionData->Mode = Mode;
 491   InstructionData->DataSize = ModeDataSize;
 492   InstructionData->AddrSize = ModeAddrSize;
 493
 494   InstructionData->Prefixes = InstructionData->Begin;
 495
 496   Byte = InstructionData->Prefixes;
 497   for ( ; ; Byte++, InstructionData->PrefixSize++) {
 498     //
 499     // Check the 0x40 to 0x4F range using an if statement here since some
 500     // compilers don't like the "case 0x40 ... 0x4F:" syntax. This avoids
 501     // 16 case statements below.
 502     //
 503     if ((*Byte >= REX_PREFIX_START) && (*Byte <= REX_PREFIX_STOP)) {
 504       InstructionData->RexPrefix.Uint8 = *Byte;
 505       if ((*Byte & REX_64BIT_OPERAND_SIZE_MASK) != 0) {
 506         InstructionData->DataSize = Size64Bits;
 507       }
 508       continue;
 509     }
 510
 511     switch (*Byte) {
 512     case OVERRIDE_SEGMENT_CS:
 513     case OVERRIDE_SEGMENT_DS:
 514     case OVERRIDE_SEGMENT_ES:
 515     case OVERRIDE_SEGMENT_SS:
 516       if (Mode != LongMode64Bit) {
 517         InstructionData->SegmentSpecified = TRUE;
 518         InstructionData->Segment = (*Byte >> 3) & 3;
 519       }
 520       break;
 521
 522     case OVERRIDE_SEGMENT_FS:
 523     case OVERRIDE_SEGMENT_GS:
 524       InstructionData->SegmentSpecified = TRUE;
 525       InstructionData->Segment = *Byte & 7;
 526       break;
 527
 528     case OVERRIDE_OPERAND_SIZE:
 529       if (InstructionData->RexPrefix.Uint8 == 0) {
 530         InstructionData->DataSize =
 531           (Mode == LongMode64Bit)       ? Size16Bits :
 532           (Mode == LongModeCompat32Bit) ? Size16Bits :
 533           (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 534       }
 535       break;
 536
 537     case OVERRIDE_ADDRESS_SIZE:
 538       InstructionData->AddrSize =
 539         (Mode == LongMode64Bit)       ? Size32Bits :
 540         (Mode == LongModeCompat32Bit) ? Size16Bits :
 541         (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 542       break;
 543
 544     case LOCK_PREFIX:
 545       break;
 546
 547     case REPZ_PREFIX:
 548       InstructionData->RepMode = RepZ;
 549       break;
 550
 551     case REPNZ_PREFIX:
 552       InstructionData->RepMode = RepNZ;
 553       break;
 554
 555     default:
 556       InstructionData->OpCodes = Byte;
 557       InstructionData->OpCodeSize = (*Byte == TWO_BYTE_OPCODE_ESCAPE) ? 2 : 1;
 558
 559       InstructionData->End = Byte + InstructionData->OpCodeSize;
 560       InstructionData->Displacement = InstructionData->End;
 561       InstructionData->Immediate = InstructionData->End;
 562       return;
 563     }
 564   }
 565 }
 566
 567 /**
 568   Determine instruction length
 569
 570   Return the total length of the parsed instruction.
 571
 572   @param[in] InstructionData  Instruction parsing context
 573
 574   @return                     Length of parsed instruction
 575
 576 **/
 577 STATIC
 578 UINT64
 579 InstructionLength (
 580   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 581   )
 582 {
 583   return (UINT64) (InstructionData->End - InstructionData->Begin);
 584 }
 585
 586 /**
 587   Initialize the instruction parsing context.
 588
 589   Initialize the instruction parsing context, which includes decoding the
 590   instruction prefixes.
 591
 592   @param[in, out] InstructionData  Instruction parsing context
 593   @param[in]      Ghcb             Pointer to the Guest-Hypervisor Communication
 594                                    Block
 595   @param[in]      Regs             x64 processor context
 596
 597 **/
 598 STATIC
 599 VOID
 600 InitInstructionData (
 601   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 602   IN     GHCB                     *Ghcb,
 603   IN     EFI_SYSTEM_CONTEXT_X64   *Regs
 604   )
 605 {
 606   SetMem (InstructionData, sizeof (*InstructionData), 0);
 607   InstructionData->Ghcb = Ghcb;
 608   InstructionData->Begin = (UINT8 *) Regs->Rip;
 609   InstructionData->End = (UINT8 *) Regs->Rip;
 610
 611   DecodePrefixes (Regs, InstructionData);
 612 }
 613
 614 /**
 615   Report an unsupported event to the hypervisor
 616
 617   Use the VMGEXIT support to report an unsupported event to the hypervisor.
 618
 619   @param[in] Ghcb             Pointer to the Guest-Hypervisor Communication
 620                               Block
 621   @param[in] Regs             x64 processor context
 622   @param[in] InstructionData  Instruction parsing context
 623
 624   @return                     New exception value to propagate
 625
 626 **/
 627 STATIC
 628 UINT64
 629 UnsupportedExit (
 630   IN GHCB                     *Ghcb,
 631   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 632   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 633   )
 634 {
 635   UINT64  Status;
 636
 637   Status = VmgExit (Ghcb, SVM_EXIT_UNSUPPORTED, Regs->ExceptionData, 0);
 638   if (Status == 0) {
 639     GHCB_EVENT_INJECTION  Event;
 640
 641     Event.Uint64 = 0;
 642     Event.Elements.Vector = GP_EXCEPTION;
 643     Event.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
 644     Event.Elements.Valid  = 1;
 645
 646     Status = Event.Uint64;
 647   }
 648
 649   return Status;
 650 }
 651
 652 /**
 653   Handle an MMIO event.
 654
 655   Use the VMGEXIT instruction to handle either an MMIO read or an MMIO write.
 656
 657   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 658                                    Block
 659   @param[in, out] Regs             x64 processor context
 660   @param[in, out] InstructionData  Instruction parsing context
 661
 662   @retval 0                        Event handled successfully
 663   @return                          New exception value to propagate
 664
 665 **/
 666 STATIC
 667 UINT64
 668 MmioExit (
 669   IN OUT GHCB                     *Ghcb,
 670   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 671   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 672   )
 673 {
 674   UINT64  ExitInfo1, ExitInfo2, Status;
 675   UINTN   Bytes;
 676   UINT64  *Register;
 677   UINT8   OpCode, SignByte;
 678
 679   Bytes = 0;
 680
 681   OpCode = *(InstructionData->OpCodes);
 682   if (OpCode == TWO_BYTE_OPCODE_ESCAPE) {
 683     OpCode = *(InstructionData->OpCodes + 1);
 684   }
 685
 686   switch (OpCode) {
 687   //
 688   // MMIO write (MOV reg/memX, regX)
 689   //
 690   case 0x88:
 691     Bytes = 1;
 692     //
 693     // fall through
 694     //
 695   case 0x89:
 696     DecodeModRm (Regs, InstructionData);
 697     Bytes = ((Bytes != 0) ? Bytes :
 698              (InstructionData->DataSize == Size16Bits) ? 2 :
 699              (InstructionData->DataSize == Size32Bits) ? 4 :
 700              (InstructionData->DataSize == Size64Bits) ? 8 :
 701              0);
 702
 703     if (InstructionData->Ext.ModRm.Mod == 3) {
 704       //
 705       // NPF on two register operands???
 706       //
 707       return UnsupportedExit (Ghcb, Regs, InstructionData);
 708     }
 709
 710     ExitInfo1 = InstructionData->Ext.RmData;
 711     ExitInfo2 = Bytes;
 712     CopyMem (Ghcb->SharedBuffer, &InstructionData->Ext.RegData, Bytes);
 713
 714     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 715     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 716     if (Status != 0) {
 717       return Status;
 718     }
 719     break;
 720
 721   //
 722   // MMIO write (MOV reg/memX, immX)
 723   //
 724   case 0xC6:
 725     Bytes = 1;
 726     //
 727     // fall through
 728     //
 729   case 0xC7:
 730     DecodeModRm (Regs, InstructionData);
 731     Bytes = ((Bytes != 0) ? Bytes :
 732              (InstructionData->DataSize == Size16Bits) ? 2 :
 733              (InstructionData->DataSize == Size32Bits) ? 4 :
 734              0);
 735
 736     InstructionData->ImmediateSize = Bytes;
 737     InstructionData->End += Bytes;
 738
 739     ExitInfo1 = InstructionData->Ext.RmData;
 740     ExitInfo2 = Bytes;
 741     CopyMem (Ghcb->SharedBuffer, InstructionData->Immediate, Bytes);
 742
 743     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 744     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 745     if (Status != 0) {
 746       return Status;
 747     }
 748     break;
 749
 750   //
 751   // MMIO read (MOV regX, reg/memX)
 752   //
 753   case 0x8A:
 754     Bytes = 1;
 755     //
 756     // fall through
 757     //
 758   case 0x8B:
 759     DecodeModRm (Regs, InstructionData);
 760     Bytes = ((Bytes != 0) ? Bytes :
 761              (InstructionData->DataSize == Size16Bits) ? 2 :
 762              (InstructionData->DataSize == Size32Bits) ? 4 :
 763              (InstructionData->DataSize == Size64Bits) ? 8 :
 764              0);
 765     if (InstructionData->Ext.ModRm.Mod == 3) {
 766       //
 767       // NPF on two register operands???
 768       //
 769       return UnsupportedExit (Ghcb, Regs, InstructionData);
 770     }
 771
 772     ExitInfo1 = InstructionData->Ext.RmData;
 773     ExitInfo2 = Bytes;
 774
 775     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 776     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 777     if (Status != 0) {
 778       return Status;
 779     }
 780
 781     Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 782     if (Bytes == 4) {
 783       //
 784       // Zero-extend for 32-bit operation
 785       //
 786       *Register = 0;
 787     }
 788     CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 789     break;
 790
 791   //
 792   // MMIO read w/ zero-extension ((MOVZX regX, reg/memX)
 793   //
 794   case 0xB6:
 795     Bytes = 1;
 796     //
 797     // fall through
 798     //
 799   case 0xB7:
 800     Bytes = (Bytes != 0) ? Bytes : 2;
 801
 802     ExitInfo1 = InstructionData->Ext.RmData;
 803     ExitInfo2 = Bytes;
 804
 805     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 806     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 807     if (Status != 0) {
 808       return Status;
 809     }
 810
 811     Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 812     SetMem (Register, InstructionData->DataSize, 0);
 813     CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 814     break;
 815
 816   //
 817   // MMIO read w/ sign-extension (MOVSX regX, reg/memX)
 818   //
 819   case 0xBE:
 820     Bytes = 1;
 821     //
 822     // fall through
 823     //
 824   case 0xBF:
 825     Bytes = (Bytes != 0) ? Bytes : 2;
 826
 827     ExitInfo1 = InstructionData->Ext.RmData;
 828     ExitInfo2 = Bytes;
 829
 830     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 831     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 832     if (Status != 0) {
 833       return Status;
 834     }
 835
 836     if (Bytes == 1) {
 837       UINT8 *Data;
 838
 839       Data = (UINT8 *) Ghcb->SharedBuffer;
 840       SignByte = ((*Data & BIT7) != 0) ? 0xFF : 0x00;
 841     } else {
 842       UINT16 *Data;
 843
 844       Data = (UINT16 *) Ghcb->SharedBuffer;
 845       SignByte = ((*Data & BIT15) != 0) ? 0xFF : 0x00;
 846     }
 847
 848     Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 849     SetMem (Register, InstructionData->DataSize, SignByte);
 850     CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 851     break;
 852
 853   default:
 854     Status = GP_EXCEPTION;
 855     ASSERT (FALSE);
 856   }
 857
 858   return Status;
 859 }
 860
 861 /**
 862   Handle a WBINVD event.
 863
 864   Use the VMGEXIT instruction to handle a WBINVD event.
 865
 866   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 867                                    Block
 868   @param[in, out] Regs             x64 processor context
 869   @param[in]      InstructionData  Instruction parsing context
 870
 871   @retval 0                        Event handled successfully
 872   @return                          New exception value to propagate
 873
 874 **/
 875 STATIC
 876 UINT64
 877 WbinvdExit (
 878   IN OUT GHCB                     *Ghcb,
 879   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 880   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
 881   )
 882 {
 883   return VmgExit (Ghcb, SVM_EXIT_WBINVD, 0, 0);
 884 }
 885
 886 /**
 887   Handle an MSR event.
 888
 889   Use the VMGEXIT instruction to handle either a RDMSR or WRMSR event.
 890
 891   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 892                                    Block
 893   @param[in, out] Regs             x64 processor context
 894   @param[in]      InstructionData  Instruction parsing context
 895
 896   @retval 0                        Event handled successfully
 897   @return                          New exception value to propagate
 898
 899 **/
 900 STATIC
 901 UINT64
 902 MsrExit (
 903   IN OUT GHCB                     *Ghcb,
 904   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 905   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
 906   )
 907 {
 908   UINT64  ExitInfo1, Status;
 909
 910   ExitInfo1 = 0;
 911
 912   switch (*(InstructionData->OpCodes + 1)) {
 913   case 0x30: // WRMSR
 914     ExitInfo1 = 1;
 915     Ghcb->SaveArea.Rax = Regs->Rax;
 916     GhcbSetRegValid (Ghcb, GhcbRax);
 917     Ghcb->SaveArea.Rdx = Regs->Rdx;
 918     GhcbSetRegValid (Ghcb, GhcbRdx);
 919     //
 920     // fall through
 921     //
 922   case 0x32: // RDMSR
 923     Ghcb->SaveArea.Rcx = Regs->Rcx;
 924     GhcbSetRegValid (Ghcb, GhcbRcx);
 925     break;
 926   default:
 927     return UnsupportedExit (Ghcb, Regs, InstructionData);
 928   }
 929
 930   Status = VmgExit (Ghcb, SVM_EXIT_MSR, ExitInfo1, 0);
 931   if (Status != 0) {
 932     return Status;
 933   }
 934
 935   if (ExitInfo1 == 0) {
 936     if (!GhcbIsRegValid (Ghcb, GhcbRax) ||
 937         !GhcbIsRegValid (Ghcb, GhcbRdx)) {
 938       return UnsupportedExit (Ghcb, Regs, InstructionData);
 939     }
 940     Regs->Rax = Ghcb->SaveArea.Rax;
 941     Regs->Rdx = Ghcb->SaveArea.Rdx;
 942   }
 943
 944   return 0;
 945 }
 946
 947 /**
 948   Build the IOIO event information.
 949
 950   The IOIO event information identifies the type of IO operation to be performed
 951   by the hypervisor. Build this information based on the instruction data.
 952
 953   @param[in]       Regs             x64 processor context
 954   @param[in, out]  InstructionData  Instruction parsing context
 955
 956   @return                           IOIO event information value
 957
 958 **/
 959 STATIC
 960 UINT64
 961 IoioExitInfo (
 962   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 963   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 964   )
 965 {
 966   UINT64  ExitInfo;
 967
 968   ExitInfo = 0;
 969
 970   switch (*(InstructionData->OpCodes)) {
 971   //
 972   // INS opcodes
 973   //
 974   case 0x6C:
 975   case 0x6D:
 976     ExitInfo |= IOIO_TYPE_INS;
 977     ExitInfo |= IOIO_SEG_ES;
 978     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
 979     break;
 980
 981   //
 982   // OUTS opcodes
 983   //
 984   case 0x6E:
 985   case 0x6F:
 986     ExitInfo |= IOIO_TYPE_OUTS;
 987     ExitInfo |= IOIO_SEG_DS;
 988     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
 989     break;
 990
 991   //
 992   // IN immediate opcodes
 993   //
 994   case 0xE4:
 995   case 0xE5:
 996     InstructionData->ImmediateSize = 1;
 997     InstructionData->End++;
 998     ExitInfo |= IOIO_TYPE_IN;
 999     ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16);
1000     break;
1001
1002   //
1003   // OUT immediate opcodes
1004   //
1005   case 0xE6:
1006   case 0xE7:
1007     InstructionData->ImmediateSize = 1;
1008     InstructionData->End++;
1009     ExitInfo |= IOIO_TYPE_OUT;
1010     ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16) | IOIO_TYPE_OUT;
1011     break;
1012
1013   //
1014   // IN register opcodes
1015   //
1016   case 0xEC:
1017   case 0xED:
1018     ExitInfo |= IOIO_TYPE_IN;
1019     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1020     break;
1021
1022   //
1023   // OUT register opcodes
1024   //
1025   case 0xEE:
1026   case 0xEF:
1027     ExitInfo |= IOIO_TYPE_OUT;
1028     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1029     break;
1030
1031   default:
1032     return 0;
1033   }
1034
1035   switch (*(InstructionData->OpCodes)) {
1036   //
1037   // Single-byte opcodes
1038   //
1039   case 0x6C:
1040   case 0x6E:
1041   case 0xE4:
1042   case 0xE6:
1043   case 0xEC:
1044   case 0xEE:
1045     ExitInfo |= IOIO_DATA_8;
1046     break;
1047
1048   //
1049   // Length determined by instruction parsing
1050   //
1051   default:
1052     ExitInfo |= (InstructionData->DataSize == Size16Bits) ? IOIO_DATA_16
1053                                                           : IOIO_DATA_32;
1054   }
1055
1056   switch (InstructionData->AddrSize) {
1057   case Size16Bits:
1058     ExitInfo |= IOIO_ADDR_16;
1059     break;
1060
1061   case Size32Bits:
1062     ExitInfo |= IOIO_ADDR_32;
1063     break;
1064
1065   case Size64Bits:
1066     ExitInfo |= IOIO_ADDR_64;
1067     break;
1068
1069   default:
1070     break;
1071   }
1072
1073   if (InstructionData->RepMode != 0) {
1074     ExitInfo |= IOIO_REP;
1075   }
1076
1077   return ExitInfo;
1078 }
1079
1080 /**
1081   Handle an IOIO event.
1082
1083   Use the VMGEXIT instruction to handle an IOIO event.
1084
1085   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1086                                    Block
1087   @param[in, out] Regs             x64 processor context
1088   @param[in]      InstructionData  Instruction parsing context
1089
1090   @retval 0                        Event handled successfully
1091   @return                          New exception value to propagate
1092
1093 **/
1094 STATIC
1095 UINT64
1096 IoioExit (
1097   IN OUT GHCB                     *Ghcb,
1098   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1099   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1100   )
1101 {
1102   UINT64   ExitInfo1, ExitInfo2, Status;
1103   BOOLEAN  IsString;
1104
1105   ExitInfo1 = IoioExitInfo (Regs, InstructionData);
1106   if (ExitInfo1 == 0) {
1107     return UnsupportedExit (Ghcb, Regs, InstructionData);
1108   }
1109
1110   IsString = ((ExitInfo1 & IOIO_TYPE_STR) != 0) ? TRUE : FALSE;
1111   if (IsString) {
1112     UINTN  IoBytes, VmgExitBytes;
1113     UINTN  GhcbCount, OpCount;
1114
1115     Status = 0;
1116
1117     IoBytes = IOIO_DATA_BYTES (ExitInfo1);
1118     GhcbCount = sizeof (Ghcb->SharedBuffer) / IoBytes;
1119
1120     OpCount = ((ExitInfo1 & IOIO_REP) != 0) ? Regs->Rcx : 1;
1121     while (OpCount != 0) {
1122       ExitInfo2 = MIN (OpCount, GhcbCount);
1123       VmgExitBytes = ExitInfo2 * IoBytes;
1124
1125       if ((ExitInfo1 & IOIO_TYPE_IN) == 0) {
1126         CopyMem (Ghcb->SharedBuffer, (VOID *) Regs->Rsi, VmgExitBytes);
1127         Regs->Rsi += VmgExitBytes;
1128       }
1129
1130       Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
1131       Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, ExitInfo2);
1132       if (Status != 0) {
1133         return Status;
1134       }
1135
1136       if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1137         CopyMem ((VOID *) Regs->Rdi, Ghcb->SharedBuffer, VmgExitBytes);
1138         Regs->Rdi += VmgExitBytes;
1139       }
1140
1141       if ((ExitInfo1 & IOIO_REP) != 0) {
1142         Regs->Rcx -= ExitInfo2;
1143       }
1144
1145       OpCount -= ExitInfo2;
1146     }
1147   } else {
1148     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1149       Ghcb->SaveArea.Rax = 0;
1150     } else {
1151       CopyMem (&Ghcb->SaveArea.Rax, &Regs->Rax, IOIO_DATA_BYTES (ExitInfo1));
1152     }
1153     GhcbSetRegValid (Ghcb, GhcbRax);
1154
1155     Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, 0);
1156     if (Status != 0) {
1157       return Status;
1158     }
1159
1160     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1161       if (!GhcbIsRegValid (Ghcb, GhcbRax)) {
1162         return UnsupportedExit (Ghcb, Regs, InstructionData);
1163       }
1164       CopyMem (&Regs->Rax, &Ghcb->SaveArea.Rax, IOIO_DATA_BYTES (ExitInfo1));
1165     }
1166   }
1167
1168   return 0;
1169 }
1170
1171 /**
1172   Handle a INVD event.
1173
1174   Use the VMGEXIT instruction to handle a INVD event.
1175
1176   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1177                                    Block
1178   @param[in, out] Regs             x64 processor context
1179   @param[in]      InstructionData  Instruction parsing context
1180
1181   @retval 0                        Event handled successfully
1182   @return                          New exception value to propagate
1183
1184 **/
1185 STATIC
1186 UINT64
1187 InvdExit (
1188   IN OUT GHCB                     *Ghcb,
1189   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1190   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1191   )
1192 {
1193   return VmgExit (Ghcb, SVM_EXIT_INVD, 0, 0);
1194 }
1195
1196 /**
1197   Handle a CPUID event.
1198
1199   Use the VMGEXIT instruction to handle a CPUID event.
1200
1201   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1202                                    Block
1203   @param[in, out] Regs             x64 processor context
1204   @param[in]      InstructionData  Instruction parsing context
1205
1206   @retval 0                        Event handled successfully
1207   @return                          New exception value to propagate
1208
1209 **/
1210 STATIC
1211 UINT64
1212 CpuidExit (
1213   IN OUT GHCB                     *Ghcb,
1214   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1215   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1216   )
1217 {
1218   UINT64  Status;
1219
1220   Ghcb->SaveArea.Rax = Regs->Rax;
1221   GhcbSetRegValid (Ghcb, GhcbRax);
1222   Ghcb->SaveArea.Rcx = Regs->Rcx;
1223   GhcbSetRegValid (Ghcb, GhcbRcx);
1224   if (Regs->Rax == CPUID_EXTENDED_STATE) {
1225     IA32_CR4  Cr4;
1226
1227     Cr4.UintN = AsmReadCr4 ();
1228     Ghcb->SaveArea.XCr0 = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
1229     GhcbSetRegValid (Ghcb, GhcbXCr0);
1230   }
1231
1232   Status = VmgExit (Ghcb, SVM_EXIT_CPUID, 0, 0);
1233   if (Status != 0) {
1234     return Status;
1235   }
1236
1237   if (!GhcbIsRegValid (Ghcb, GhcbRax) ||
1238       !GhcbIsRegValid (Ghcb, GhcbRbx) ||
1239       !GhcbIsRegValid (Ghcb, GhcbRcx) ||
1240       !GhcbIsRegValid (Ghcb, GhcbRdx)) {
1241     return UnsupportedExit (Ghcb, Regs, InstructionData);
1242   }
1243   Regs->Rax = Ghcb->SaveArea.Rax;
1244   Regs->Rbx = Ghcb->SaveArea.Rbx;
1245   Regs->Rcx = Ghcb->SaveArea.Rcx;
1246   Regs->Rdx = Ghcb->SaveArea.Rdx;
1247
1248   return 0;
1249 }
1250
1251 /**
1252   Handle a RDPMC event.
1253
1254   Use the VMGEXIT instruction to handle a RDPMC event.
1255
1256   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1257                                    Block
1258   @param[in, out] Regs             x64 processor context
1259   @param[in]      InstructionData  Instruction parsing context
1260
1261   @retval 0                        Event handled successfully
1262   @return                          New exception value to propagate
1263
1264 **/
1265 STATIC
1266 UINT64
1267 RdpmcExit (
1268   IN OUT GHCB                     *Ghcb,
1269   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1270   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1271   )
1272 {
1273   UINT64  Status;
1274
1275   Ghcb->SaveArea.Rcx = Regs->Rcx;
1276   GhcbSetRegValid (Ghcb, GhcbRcx);
1277
1278   Status = VmgExit (Ghcb, SVM_EXIT_RDPMC, 0, 0);
1279   if (Status != 0) {
1280     return Status;
1281   }
1282
1283   if (!GhcbIsRegValid (Ghcb, GhcbRax) ||
1284       !GhcbIsRegValid (Ghcb, GhcbRdx)) {
1285     return UnsupportedExit (Ghcb, Regs, InstructionData);
1286   }
1287   Regs->Rax = Ghcb->SaveArea.Rax;
1288   Regs->Rdx = Ghcb->SaveArea.Rdx;
1289
1290   return 0;
1291 }
1292
1293 /**
1294   Handle a RDTSC event.
1295
1296   Use the VMGEXIT instruction to handle a RDTSC event.
1297
1298   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1299                                    Block
1300   @param[in, out] Regs             x64 processor context
1301   @param[in]      InstructionData  Instruction parsing context
1302
1303   @retval 0                        Event handled successfully
1304   @return                          New exception value to propagate
1305
1306 **/
1307 STATIC
1308 UINT64
1309 RdtscExit (
1310   IN OUT GHCB                     *Ghcb,
1311   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1312   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1313   )
1314 {
1315   UINT64  Status;
1316
1317   Status = VmgExit (Ghcb, SVM_EXIT_RDTSC, 0, 0);
1318   if (Status != 0) {
1319     return Status;
1320   }
1321
1322   if (!GhcbIsRegValid (Ghcb, GhcbRax) ||
1323       !GhcbIsRegValid (Ghcb, GhcbRdx)) {
1324     return UnsupportedExit (Ghcb, Regs, InstructionData);
1325   }
1326   Regs->Rax = Ghcb->SaveArea.Rax;
1327   Regs->Rdx = Ghcb->SaveArea.Rdx;
1328
1329   return 0;
1330 }
1331
1332 /**
1333   Handle a #VC exception.
1334
1335   Performs the necessary processing to handle a #VC exception.
1336
1337   @param[in, out]  ExceptionType  Pointer to an EFI_EXCEPTION_TYPE to be set
1338                                   as value to use on error.
1339   @param[in, out]  SystemContext  Pointer to EFI_SYSTEM_CONTEXT
1340
1341   @retval  EFI_SUCCESS            Exception handled
1342   @retval  EFI_UNSUPPORTED        #VC not supported, (new) exception value to
1343                                   propagate provided
1344   @retval  EFI_PROTOCOL_ERROR     #VC handling failed, (new) exception value to
1345                                   propagate provided
1346
1347 **/
1348 EFI_STATUS
1349 EFIAPI
1350 VmgExitHandleVc (
1351   IN OUT EFI_EXCEPTION_TYPE  *ExceptionType,
1352   IN OUT EFI_SYSTEM_CONTEXT  SystemContext
1353   )
1354 {
1355   MSR_SEV_ES_GHCB_REGISTER  Msr;
1356   EFI_SYSTEM_CONTEXT_X64    *Regs;
1357   GHCB                      *Ghcb;
1358   NAE_EXIT                  NaeExit;
1359   SEV_ES_INSTRUCTION_DATA   InstructionData;
1360   UINT64                    ExitCode, Status;
1361   EFI_STATUS                VcRet;
1362
1363   VcRet = EFI_SUCCESS;
1364
1365   Msr.GhcbPhysicalAddress = AsmReadMsr64 (MSR_SEV_ES_GHCB);
1366   ASSERT (Msr.GhcbInfo.Function == 0);
1367   ASSERT (Msr.Ghcb != 0);
1368
1369   Regs = SystemContext.SystemContextX64;
1370   Ghcb = Msr.Ghcb;
1371
1372   VmgInit (Ghcb);
1373
1374   ExitCode = Regs->ExceptionData;
1375   switch (ExitCode) {
1376   case SVM_EXIT_RDTSC:
1377     NaeExit = RdtscExit;
1378     break;
1379
1380   case SVM_EXIT_RDPMC:
1381     NaeExit = RdpmcExit;
1382     break;
1383
1384   case SVM_EXIT_CPUID:
1385     NaeExit = CpuidExit;
1386     break;
1387
1388   case SVM_EXIT_INVD:
1389     NaeExit = InvdExit;
1390     break;
1391
1392   case SVM_EXIT_IOIO_PROT:
1393     NaeExit = IoioExit;
1394     break;
1395
1396   case SVM_EXIT_MSR:
1397     NaeExit = MsrExit;
1398     break;
1399
1400   case SVM_EXIT_WBINVD:
1401     NaeExit = WbinvdExit;
1402     break;
1403
1404   case SVM_EXIT_NPF:
1405     NaeExit = MmioExit;
1406     break;
1407
1408   default:
1409     NaeExit = UnsupportedExit;
1410   }
1411
1412   InitInstructionData (&InstructionData, Ghcb, Regs);
1413
1414   Status = NaeExit (Ghcb, Regs, &InstructionData);
1415   if (Status == 0) {
1416     Regs->Rip += InstructionLength (&InstructionData);
1417   } else {
1418     GHCB_EVENT_INJECTION  Event;
1419
1420     Event.Uint64 = Status;
1421     if (Event.Elements.ErrorCodeValid != 0) {
1422       Regs->ExceptionData = Event.Elements.ErrorCode;
1423     } else {
1424       Regs->ExceptionData = 0;
1425     }
1426
1427     *ExceptionType = Event.Elements.Vector;
1428
1429     VcRet = EFI_PROTOCOL_ERROR;
1430   }
1431
1432   VmgDone (Ghcb);
1433
1434   return VcRet;
1435 }