OvmfPkg/Library/VmgExitLib/VmgExitVcHandler.c

   1 /** @file
   2   X64 #VC Exception Handler functon.
   3
   4   Copyright (C) 2020, Advanced Micro Devices, Inc. All rights reserved.<BR>
   5   SPDX-License-Identifier: BSD-2-Clause-Patent
   6
   7 **/
   8
   9 #include <Base.h>
  10 #include <Uefi.h>
  11 #include <Library/BaseMemoryLib.h>
  12 #include <Library/VmgExitLib.h>
  13 #include <Register/Amd/Msr.h>
  14 #include <Register/Intel/Cpuid.h>
  15 #include <IndustryStandard/InstructionParsing.h>
  16
  17 //
  18 // Instruction execution mode definition
  19 //
  20 typedef enum {
  21   LongMode64Bit        = 0,
  22   LongModeCompat32Bit,
  23   LongModeCompat16Bit,
  24 } SEV_ES_INSTRUCTION_MODE;
  25
  26 //
  27 // Instruction size definition (for operand and address)
  28 //
  29 typedef enum {
  30   Size8Bits            = 0,
  31   Size16Bits,
  32   Size32Bits,
  33   Size64Bits,
  34 } SEV_ES_INSTRUCTION_SIZE;
  35
  36 //
  37 // Intruction segment definition
  38 //
  39 typedef enum {
  40   SegmentEs            = 0,
  41   SegmentCs,
  42   SegmentSs,
  43   SegmentDs,
  44   SegmentFs,
  45   SegmentGs,
  46 } SEV_ES_INSTRUCTION_SEGMENT;
  47
  48 //
  49 // Instruction rep function definition
  50 //
  51 typedef enum {
  52   RepNone              = 0,
  53   RepZ,
  54   RepNZ,
  55 } SEV_ES_INSTRUCTION_REP;
  56
  57 typedef struct {
  58   UINT8  Rm;
  59   UINT8  Reg;
  60   UINT8  Mod;
  61 } SEV_ES_INSTRUCTION_MODRM_EXT;
  62
  63 typedef struct {
  64   UINT8  Base;
  65   UINT8  Index;
  66   UINT8  Scale;
  67 } SEV_ES_INSTRUCTION_SIB_EXT;
  68
  69 //
  70 // Instruction opcode definition
  71 //
  72 typedef struct {
  73   SEV_ES_INSTRUCTION_MODRM_EXT  ModRm;
  74
  75   SEV_ES_INSTRUCTION_SIB_EXT    Sib;
  76
  77   UINTN                         RegData;
  78   UINTN                         RmData;
  79 } SEV_ES_INSTRUCTION_OPCODE_EXT;
  80
  81 //
  82 // Instruction parsing context definition
  83 //
  84 typedef struct {
  85   GHCB                           *Ghcb;
  86
  87   SEV_ES_INSTRUCTION_MODE        Mode;
  88   SEV_ES_INSTRUCTION_SIZE        DataSize;
  89   SEV_ES_INSTRUCTION_SIZE        AddrSize;
  90   BOOLEAN                        SegmentSpecified;
  91   SEV_ES_INSTRUCTION_SEGMENT     Segment;
  92   SEV_ES_INSTRUCTION_REP         RepMode;
  93
  94   UINT8                          *Begin;
  95   UINT8                          *End;
  96
  97   UINT8                          *Prefixes;
  98   UINT8                          *OpCodes;
  99   UINT8                          *Displacement;
 100   UINT8                          *Immediate;
 101
 102   INSTRUCTION_REX_PREFIX         RexPrefix;
 103
 104   BOOLEAN                        ModRmPresent;
 105   INSTRUCTION_MODRM              ModRm;
 106
 107   BOOLEAN                        SibPresent;
 108   INSTRUCTION_SIB                Sib;
 109
 110   UINTN                          PrefixSize;
 111   UINTN                          OpCodeSize;
 112   UINTN                          DisplacementSize;
 113   UINTN                          ImmediateSize;
 114
 115   SEV_ES_INSTRUCTION_OPCODE_EXT  Ext;
 116 } SEV_ES_INSTRUCTION_DATA;
 117
 118 //
 119 // Non-automatic Exit function prototype
 120 //
 121 typedef
 122 UINT64
 123 (*NAE_EXIT) (
 124   GHCB                     *Ghcb,
 125   EFI_SYSTEM_CONTEXT_X64   *Regs,
 126   SEV_ES_INSTRUCTION_DATA  *InstructionData
 127   );
 128
 129 //
 130 // Per-CPU data mapping structure
 131 //
 132 typedef struct {
 133   BOOLEAN  Dr7Cached;
 134   UINT64   Dr7;
 135 } SEV_ES_PER_CPU_DATA;
 136
 137
 138 /**
 139   Return a pointer to the contents of the specified register.
 140
 141   Based upon the input register, return a pointer to the registers contents
 142   in the x86 processor context.
 143
 144   @param[in] Regs      x64 processor context
 145   @param[in] Register  Register to obtain pointer for
 146
 147   @return              Pointer to the contents of the requested register
 148
 149 **/
 150 STATIC
 151 UINT64 *
 152 GetRegisterPointer (
 153   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 154   IN UINT8                    Register
 155   )
 156 {
 157   UINT64 *Reg;
 158
 159   switch (Register) {
 160   case 0:
 161     Reg = &Regs->Rax;
 162     break;
 163   case 1:
 164     Reg = &Regs->Rcx;
 165     break;
 166   case 2:
 167     Reg = &Regs->Rdx;
 168     break;
 169   case 3:
 170     Reg = &Regs->Rbx;
 171     break;
 172   case 4:
 173     Reg = &Regs->Rsp;
 174     break;
 175   case 5:
 176     Reg = &Regs->Rbp;
 177     break;
 178   case 6:
 179     Reg = &Regs->Rsi;
 180     break;
 181   case 7:
 182     Reg = &Regs->Rdi;
 183     break;
 184   case 8:
 185     Reg = &Regs->R8;
 186     break;
 187   case 9:
 188     Reg = &Regs->R9;
 189     break;
 190   case 10:
 191     Reg = &Regs->R10;
 192     break;
 193   case 11:
 194     Reg = &Regs->R11;
 195     break;
 196   case 12:
 197     Reg = &Regs->R12;
 198     break;
 199   case 13:
 200     Reg = &Regs->R13;
 201     break;
 202   case 14:
 203     Reg = &Regs->R14;
 204     break;
 205   case 15:
 206     Reg = &Regs->R15;
 207     break;
 208   default:
 209     Reg = NULL;
 210   }
 211   ASSERT (Reg != NULL);
 212
 213   return Reg;
 214 }
 215
 216 /**
 217   Update the instruction parsing context for displacement bytes.
 218
 219   @param[in, out] InstructionData  Instruction parsing context
 220   @param[in]      Size             The instruction displacement size
 221
 222 **/
 223 STATIC
 224 VOID
 225 UpdateForDisplacement (
 226   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 227   IN     UINTN                    Size
 228   )
 229 {
 230   InstructionData->DisplacementSize = Size;
 231   InstructionData->Immediate += Size;
 232   InstructionData->End += Size;
 233 }
 234
 235 /**
 236   Determine if an instruction address if RIP relative.
 237
 238   Examine the instruction parsing context to determine if the address offset
 239   is relative to the instruction pointer.
 240
 241   @param[in] InstructionData  Instruction parsing context
 242
 243   @retval TRUE                Instruction addressing is RIP relative
 244   @retval FALSE               Instruction addressing is not RIP relative
 245
 246 **/
 247 STATIC
 248 BOOLEAN
 249 IsRipRelative (
 250   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 251   )
 252 {
 253   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 254
 255   Ext = &InstructionData->Ext;
 256
 257   return ((InstructionData->Mode == LongMode64Bit) &&
 258           (Ext->ModRm.Mod == 0) &&
 259           (Ext->ModRm.Rm == 5)  &&
 260           (InstructionData->SibPresent == FALSE));
 261 }
 262
 263 /**
 264   Return the effective address of a memory operand.
 265
 266   Examine the instruction parsing context to obtain the effective memory
 267   address of a memory operand.
 268
 269   @param[in] Regs             x64 processor context
 270   @param[in] InstructionData  Instruction parsing context
 271
 272   @return                     The memory operand effective address
 273
 274 **/
 275 STATIC
 276 UINT64
 277 GetEffectiveMemoryAddress (
 278   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 279   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 280   )
 281 {
 282   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 283   UINT64                         EffectiveAddress;
 284
 285   Ext = &InstructionData->Ext;
 286   EffectiveAddress = 0;
 287
 288   if (IsRipRelative (InstructionData)) {
 289     //
 290     // RIP-relative displacement is a 32-bit signed value
 291     //
 292     INT32 RipRelative;
 293
 294     RipRelative = *(INT32 *) InstructionData->Displacement;
 295
 296     UpdateForDisplacement (InstructionData, 4);
 297
 298     //
 299     // Negative displacement is handled by standard UINT64 wrap-around.
 300     //
 301     return Regs->Rip + (UINT64) RipRelative;
 302   }
 303
 304   switch (Ext->ModRm.Mod) {
 305   case 1:
 306     UpdateForDisplacement (InstructionData, 1);
 307     EffectiveAddress += (UINT64) (*(INT8 *) (InstructionData->Displacement));
 308     break;
 309   case 2:
 310     switch (InstructionData->AddrSize) {
 311     case Size16Bits:
 312       UpdateForDisplacement (InstructionData, 2);
 313       EffectiveAddress += (UINT64) (*(INT16 *) (InstructionData->Displacement));
 314       break;
 315     default:
 316       UpdateForDisplacement (InstructionData, 4);
 317       EffectiveAddress += (UINT64) (*(INT32 *) (InstructionData->Displacement));
 318       break;
 319     }
 320     break;
 321   }
 322
 323   if (InstructionData->SibPresent) {
 324     INT64  Displacement;
 325
 326     if (Ext->Sib.Index != 4) {
 327       CopyMem (
 328         &Displacement,
 329         GetRegisterPointer (Regs, Ext->Sib.Index),
 330         sizeof (Displacement)
 331         );
 332       Displacement *= (INT64)(1 << Ext->Sib.Scale);
 333
 334       //
 335       // Negative displacement is handled by standard UINT64 wrap-around.
 336       //
 337       EffectiveAddress += (UINT64) Displacement;
 338     }
 339
 340     if ((Ext->Sib.Base != 5) || Ext->ModRm.Mod) {
 341       EffectiveAddress += *GetRegisterPointer (Regs, Ext->Sib.Base);
 342     } else {
 343       UpdateForDisplacement (InstructionData, 4);
 344       EffectiveAddress += (UINT64) (*(INT32 *) (InstructionData->Displacement));
 345     }
 346   } else {
 347     EffectiveAddress += *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 348   }
 349
 350   return EffectiveAddress;
 351 }
 352
 353 /**
 354   Decode a ModRM byte.
 355
 356   Examine the instruction parsing context to decode a ModRM byte and the SIB
 357   byte, if present.
 358
 359   @param[in]      Regs             x64 processor context
 360   @param[in, out] InstructionData  Instruction parsing context
 361
 362 **/
 363 STATIC
 364 VOID
 365 DecodeModRm (
 366   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 367   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 368   )
 369 {
 370   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 371   INSTRUCTION_REX_PREFIX         *RexPrefix;
 372   INSTRUCTION_MODRM              *ModRm;
 373   INSTRUCTION_SIB                *Sib;
 374
 375   RexPrefix = &InstructionData->RexPrefix;
 376   Ext = &InstructionData->Ext;
 377   ModRm = &InstructionData->ModRm;
 378   Sib = &InstructionData->Sib;
 379
 380   InstructionData->ModRmPresent = TRUE;
 381   ModRm->Uint8 = *(InstructionData->End);
 382
 383   InstructionData->Displacement++;
 384   InstructionData->Immediate++;
 385   InstructionData->End++;
 386
 387   Ext->ModRm.Mod = ModRm->Bits.Mod;
 388   Ext->ModRm.Reg = (RexPrefix->Bits.BitR << 3) | ModRm->Bits.Reg;
 389   Ext->ModRm.Rm  = (RexPrefix->Bits.BitB << 3) | ModRm->Bits.Rm;
 390
 391   Ext->RegData = *GetRegisterPointer (Regs, Ext->ModRm.Reg);
 392
 393   if (Ext->ModRm.Mod == 3) {
 394     Ext->RmData = *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 395   } else {
 396     if (ModRm->Bits.Rm == 4) {
 397       InstructionData->SibPresent = TRUE;
 398       Sib->Uint8 = *(InstructionData->End);
 399
 400       InstructionData->Displacement++;
 401       InstructionData->Immediate++;
 402       InstructionData->End++;
 403
 404       Ext->Sib.Scale = Sib->Bits.Scale;
 405       Ext->Sib.Index = (RexPrefix->Bits.BitX << 3) | Sib->Bits.Index;
 406       Ext->Sib.Base  = (RexPrefix->Bits.BitB << 3) | Sib->Bits.Base;
 407     }
 408
 409     Ext->RmData = GetEffectiveMemoryAddress (Regs, InstructionData);
 410   }
 411 }
 412
 413 /**
 414   Decode instruction prefixes.
 415
 416   Parse the instruction data to track the instruction prefixes that have
 417   been used.
 418
 419   @param[in]      Regs             x64 processor context
 420   @param[in, out] InstructionData  Instruction parsing context
 421
 422 **/
 423 STATIC
 424 VOID
 425 DecodePrefixes (
 426   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 427   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 428   )
 429 {
 430   SEV_ES_INSTRUCTION_MODE  Mode;
 431   SEV_ES_INSTRUCTION_SIZE  ModeDataSize;
 432   SEV_ES_INSTRUCTION_SIZE  ModeAddrSize;
 433   UINT8                    *Byte;
 434
 435   //
 436   // Always in 64-bit mode
 437   //
 438   Mode = LongMode64Bit;
 439   ModeDataSize = Size32Bits;
 440   ModeAddrSize = Size64Bits;
 441
 442   InstructionData->Mode = Mode;
 443   InstructionData->DataSize = ModeDataSize;
 444   InstructionData->AddrSize = ModeAddrSize;
 445
 446   InstructionData->Prefixes = InstructionData->Begin;
 447
 448   Byte = InstructionData->Prefixes;
 449   for ( ; ; Byte++, InstructionData->PrefixSize++) {
 450     //
 451     // Check the 0x40 to 0x4F range using an if statement here since some
 452     // compilers don't like the "case 0x40 ... 0x4F:" syntax. This avoids
 453     // 16 case statements below.
 454     //
 455     if ((*Byte >= REX_PREFIX_START) && (*Byte <= REX_PREFIX_STOP)) {
 456       InstructionData->RexPrefix.Uint8 = *Byte;
 457       if ((*Byte & REX_64BIT_OPERAND_SIZE_MASK) != 0) {
 458         InstructionData->DataSize = Size64Bits;
 459       }
 460       continue;
 461     }
 462
 463     switch (*Byte) {
 464     case OVERRIDE_SEGMENT_CS:
 465     case OVERRIDE_SEGMENT_DS:
 466     case OVERRIDE_SEGMENT_ES:
 467     case OVERRIDE_SEGMENT_SS:
 468       if (Mode != LongMode64Bit) {
 469         InstructionData->SegmentSpecified = TRUE;
 470         InstructionData->Segment = (*Byte >> 3) & 3;
 471       }
 472       break;
 473
 474     case OVERRIDE_SEGMENT_FS:
 475     case OVERRIDE_SEGMENT_GS:
 476       InstructionData->SegmentSpecified = TRUE;
 477       InstructionData->Segment = *Byte & 7;
 478       break;
 479
 480     case OVERRIDE_OPERAND_SIZE:
 481       if (InstructionData->RexPrefix.Uint8 == 0) {
 482         InstructionData->DataSize =
 483           (Mode == LongMode64Bit)       ? Size16Bits :
 484           (Mode == LongModeCompat32Bit) ? Size16Bits :
 485           (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 486       }
 487       break;
 488
 489     case OVERRIDE_ADDRESS_SIZE:
 490       InstructionData->AddrSize =
 491         (Mode == LongMode64Bit)       ? Size32Bits :
 492         (Mode == LongModeCompat32Bit) ? Size16Bits :
 493         (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 494       break;
 495
 496     case LOCK_PREFIX:
 497       break;
 498
 499     case REPZ_PREFIX:
 500       InstructionData->RepMode = RepZ;
 501       break;
 502
 503     case REPNZ_PREFIX:
 504       InstructionData->RepMode = RepNZ;
 505       break;
 506
 507     default:
 508       InstructionData->OpCodes = Byte;
 509       InstructionData->OpCodeSize = (*Byte == TWO_BYTE_OPCODE_ESCAPE) ? 2 : 1;
 510
 511       InstructionData->End = Byte + InstructionData->OpCodeSize;
 512       InstructionData->Displacement = InstructionData->End;
 513       InstructionData->Immediate = InstructionData->End;
 514       return;
 515     }
 516   }
 517 }
 518
 519 /**
 520   Determine instruction length
 521
 522   Return the total length of the parsed instruction.
 523
 524   @param[in] InstructionData  Instruction parsing context
 525
 526   @return                     Length of parsed instruction
 527
 528 **/
 529 STATIC
 530 UINT64
 531 InstructionLength (
 532   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 533   )
 534 {
 535   return (UINT64) (InstructionData->End - InstructionData->Begin);
 536 }
 537
 538 /**
 539   Initialize the instruction parsing context.
 540
 541   Initialize the instruction parsing context, which includes decoding the
 542   instruction prefixes.
 543
 544   @param[in, out] InstructionData  Instruction parsing context
 545   @param[in]      Ghcb             Pointer to the Guest-Hypervisor Communication
 546                                    Block
 547   @param[in]      Regs             x64 processor context
 548
 549 **/
 550 STATIC
 551 VOID
 552 InitInstructionData (
 553   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 554   IN     GHCB                     *Ghcb,
 555   IN     EFI_SYSTEM_CONTEXT_X64   *Regs
 556   )
 557 {
 558   SetMem (InstructionData, sizeof (*InstructionData), 0);
 559   InstructionData->Ghcb = Ghcb;
 560   InstructionData->Begin = (UINT8 *) Regs->Rip;
 561   InstructionData->End = (UINT8 *) Regs->Rip;
 562
 563   DecodePrefixes (Regs, InstructionData);
 564 }
 565
 566 /**
 567   Report an unsupported event to the hypervisor
 568
 569   Use the VMGEXIT support to report an unsupported event to the hypervisor.
 570
 571   @param[in] Ghcb             Pointer to the Guest-Hypervisor Communication
 572                               Block
 573   @param[in] Regs             x64 processor context
 574   @param[in] InstructionData  Instruction parsing context
 575
 576   @return                     New exception value to propagate
 577
 578 **/
 579 STATIC
 580 UINT64
 581 UnsupportedExit (
 582   IN GHCB                     *Ghcb,
 583   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 584   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 585   )
 586 {
 587   UINT64  Status;
 588
 589   Status = VmgExit (Ghcb, SVM_EXIT_UNSUPPORTED, Regs->ExceptionData, 0);
 590   if (Status == 0) {
 591     GHCB_EVENT_INJECTION  Event;
 592
 593     Event.Uint64 = 0;
 594     Event.Elements.Vector = GP_EXCEPTION;
 595     Event.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
 596     Event.Elements.Valid  = 1;
 597
 598     Status = Event.Uint64;
 599   }
 600
 601   return Status;
 602 }
 603
 604 /**
 605   Handle an MMIO event.
 606
 607   Use the VMGEXIT instruction to handle either an MMIO read or an MMIO write.
 608
 609   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 610                                    Block
 611   @param[in, out] Regs             x64 processor context
 612   @param[in, out] InstructionData  Instruction parsing context
 613
 614   @retval 0                        Event handled successfully
 615   @return                          New exception value to propagate
 616
 617 **/
 618 STATIC
 619 UINT64
 620 MmioExit (
 621   IN OUT GHCB                     *Ghcb,
 622   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 623   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 624   )
 625 {
 626   UINT64  ExitInfo1, ExitInfo2, Status;
 627   UINTN   Bytes;
 628   UINT64  *Register;
 629   UINT8   OpCode, SignByte;
 630
 631   Bytes = 0;
 632
 633   OpCode = *(InstructionData->OpCodes);
 634   if (OpCode == TWO_BYTE_OPCODE_ESCAPE) {
 635     OpCode = *(InstructionData->OpCodes + 1);
 636   }
 637
 638   switch (OpCode) {
 639   //
 640   // MMIO write (MOV reg/memX, regX)
 641   //
 642   case 0x88:
 643     Bytes = 1;
 644     //
 645     // fall through
 646     //
 647   case 0x89:
 648     DecodeModRm (Regs, InstructionData);
 649     Bytes = ((Bytes != 0) ? Bytes :
 650              (InstructionData->DataSize == Size16Bits) ? 2 :
 651              (InstructionData->DataSize == Size32Bits) ? 4 :
 652              (InstructionData->DataSize == Size64Bits) ? 8 :
 653              0);
 654
 655     if (InstructionData->Ext.ModRm.Mod == 3) {
 656       //
 657       // NPF on two register operands???
 658       //
 659       return UnsupportedExit (Ghcb, Regs, InstructionData);
 660     }
 661
 662     ExitInfo1 = InstructionData->Ext.RmData;
 663     ExitInfo2 = Bytes;
 664     CopyMem (Ghcb->SharedBuffer, &InstructionData->Ext.RegData, Bytes);
 665
 666     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 667     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 668     if (Status != 0) {
 669       return Status;
 670     }
 671     break;
 672
 673   //
 674   // MMIO write (MOV reg/memX, immX)
 675   //
 676   case 0xC6:
 677     Bytes = 1;
 678     //
 679     // fall through
 680     //
 681   case 0xC7:
 682     DecodeModRm (Regs, InstructionData);
 683     Bytes = ((Bytes != 0) ? Bytes :
 684              (InstructionData->DataSize == Size16Bits) ? 2 :
 685              (InstructionData->DataSize == Size32Bits) ? 4 :
 686              0);
 687
 688     InstructionData->ImmediateSize = Bytes;
 689     InstructionData->End += Bytes;
 690
 691     ExitInfo1 = InstructionData->Ext.RmData;
 692     ExitInfo2 = Bytes;
 693     CopyMem (Ghcb->SharedBuffer, InstructionData->Immediate, Bytes);
 694
 695     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 696     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 697     if (Status != 0) {
 698       return Status;
 699     }
 700     break;
 701
 702   //
 703   // MMIO read (MOV regX, reg/memX)
 704   //
 705   case 0x8A:
 706     Bytes = 1;
 707     //
 708     // fall through
 709     //
 710   case 0x8B:
 711     DecodeModRm (Regs, InstructionData);
 712     Bytes = ((Bytes != 0) ? Bytes :
 713              (InstructionData->DataSize == Size16Bits) ? 2 :
 714              (InstructionData->DataSize == Size32Bits) ? 4 :
 715              (InstructionData->DataSize == Size64Bits) ? 8 :
 716              0);
 717     if (InstructionData->Ext.ModRm.Mod == 3) {
 718       //
 719       // NPF on two register operands???
 720       //
 721       return UnsupportedExit (Ghcb, Regs, InstructionData);
 722     }
 723
 724     ExitInfo1 = InstructionData->Ext.RmData;
 725     ExitInfo2 = Bytes;
 726
 727     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 728     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 729     if (Status != 0) {
 730       return Status;
 731     }
 732
 733     Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 734     if (Bytes == 4) {
 735       //
 736       // Zero-extend for 32-bit operation
 737       //
 738       *Register = 0;
 739     }
 740     CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 741     break;
 742
 743   //
 744   // MMIO read w/ zero-extension ((MOVZX regX, reg/memX)
 745   //
 746   case 0xB6:
 747     Bytes = 1;
 748     //
 749     // fall through
 750     //
 751   case 0xB7:
 752     Bytes = (Bytes != 0) ? Bytes : 2;
 753
 754     ExitInfo1 = InstructionData->Ext.RmData;
 755     ExitInfo2 = Bytes;
 756
 757     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 758     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 759     if (Status != 0) {
 760       return Status;
 761     }
 762
 763     Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 764     SetMem (Register, InstructionData->DataSize, 0);
 765     CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 766     break;
 767
 768   //
 769   // MMIO read w/ sign-extension (MOVSX regX, reg/memX)
 770   //
 771   case 0xBE:
 772     Bytes = 1;
 773     //
 774     // fall through
 775     //
 776   case 0xBF:
 777     Bytes = (Bytes != 0) ? Bytes : 2;
 778
 779     ExitInfo1 = InstructionData->Ext.RmData;
 780     ExitInfo2 = Bytes;
 781
 782     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 783     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 784     if (Status != 0) {
 785       return Status;
 786     }
 787
 788     if (Bytes == 1) {
 789       UINT8 *Data;
 790
 791       Data = (UINT8 *) Ghcb->SharedBuffer;
 792       SignByte = ((*Data & BIT7) != 0) ? 0xFF : 0x00;
 793     } else {
 794       UINT16 *Data;
 795
 796       Data = (UINT16 *) Ghcb->SharedBuffer;
 797       SignByte = ((*Data & BIT15) != 0) ? 0xFF : 0x00;
 798     }
 799
 800     Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 801     SetMem (Register, InstructionData->DataSize, SignByte);
 802     CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 803     break;
 804
 805   default:
 806     Status = GP_EXCEPTION;
 807     ASSERT (FALSE);
 808   }
 809
 810   return Status;
 811 }
 812
 813 /**
 814   Handle a MWAIT event.
 815
 816   Use the VMGEXIT instruction to handle a MWAIT event.
 817
 818   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 819                                    Block
 820   @param[in, out] Regs             x64 processor context
 821   @param[in]      InstructionData  Instruction parsing context
 822
 823   @retval 0                        Event handled successfully
 824   @return                          New exception value to propagate
 825
 826 **/
 827 STATIC
 828 UINT64
 829 MwaitExit (
 830   IN OUT GHCB                     *Ghcb,
 831   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 832   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
 833   )
 834 {
 835   DecodeModRm (Regs, InstructionData);
 836
 837   Ghcb->SaveArea.Rax = Regs->Rax;
 838   VmgSetOffsetValid (Ghcb, GhcbRax);
 839   Ghcb->SaveArea.Rcx = Regs->Rcx;
 840   VmgSetOffsetValid (Ghcb, GhcbRcx);
 841
 842   return VmgExit (Ghcb, SVM_EXIT_MWAIT, 0, 0);
 843 }
 844
 845 /**
 846   Handle a MONITOR event.
 847
 848   Use the VMGEXIT instruction to handle a MONITOR event.
 849
 850   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 851                                    Block
 852   @param[in, out] Regs             x64 processor context
 853   @param[in]      InstructionData  Instruction parsing context
 854
 855   @retval 0                        Event handled successfully
 856   @return                          New exception value to propagate
 857
 858 **/
 859 STATIC
 860 UINT64
 861 MonitorExit (
 862   IN OUT GHCB                     *Ghcb,
 863   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 864   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
 865   )
 866 {
 867   DecodeModRm (Regs, InstructionData);
 868
 869   Ghcb->SaveArea.Rax = Regs->Rax;  // Identity mapped, so VA = PA
 870   VmgSetOffsetValid (Ghcb, GhcbRax);
 871   Ghcb->SaveArea.Rcx = Regs->Rcx;
 872   VmgSetOffsetValid (Ghcb, GhcbRcx);
 873   Ghcb->SaveArea.Rdx = Regs->Rdx;
 874   VmgSetOffsetValid (Ghcb, GhcbRdx);
 875
 876   return VmgExit (Ghcb, SVM_EXIT_MONITOR, 0, 0);
 877 }
 878
 879 /**
 880   Handle a WBINVD event.
 881
 882   Use the VMGEXIT instruction to handle a WBINVD event.
 883
 884   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 885                                    Block
 886   @param[in, out] Regs             x64 processor context
 887   @param[in]      InstructionData  Instruction parsing context
 888
 889   @retval 0                        Event handled successfully
 890   @return                          New exception value to propagate
 891
 892 **/
 893 STATIC
 894 UINT64
 895 WbinvdExit (
 896   IN OUT GHCB                     *Ghcb,
 897   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 898   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
 899   )
 900 {
 901   return VmgExit (Ghcb, SVM_EXIT_WBINVD, 0, 0);
 902 }
 903
 904 /**
 905   Handle a RDTSCP event.
 906
 907   Use the VMGEXIT instruction to handle a RDTSCP event.
 908
 909   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 910                                    Block
 911   @param[in, out] Regs             x64 processor context
 912   @param[in]      InstructionData  Instruction parsing context
 913
 914   @retval 0                        Event handled successfully
 915   @return                          New exception value to propagate
 916
 917 **/
 918 STATIC
 919 UINT64
 920 RdtscpExit (
 921   IN OUT GHCB                     *Ghcb,
 922   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 923   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
 924   )
 925 {
 926   UINT64  Status;
 927
 928   DecodeModRm (Regs, InstructionData);
 929
 930   Status = VmgExit (Ghcb, SVM_EXIT_RDTSCP, 0, 0);
 931   if (Status != 0) {
 932     return Status;
 933   }
 934
 935   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
 936       !VmgIsOffsetValid (Ghcb, GhcbRcx) ||
 937       !VmgIsOffsetValid (Ghcb, GhcbRdx)) {
 938     return UnsupportedExit (Ghcb, Regs, InstructionData);
 939   }
 940   Regs->Rax = Ghcb->SaveArea.Rax;
 941   Regs->Rcx = Ghcb->SaveArea.Rcx;
 942   Regs->Rdx = Ghcb->SaveArea.Rdx;
 943
 944   return 0;
 945 }
 946
 947 /**
 948   Handle a VMMCALL event.
 949
 950   Use the VMGEXIT instruction to handle a VMMCALL event.
 951
 952   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 953                                    Block
 954   @param[in, out] Regs             x64 processor context
 955   @param[in]      InstructionData  Instruction parsing context
 956
 957   @retval 0                        Event handled successfully
 958   @return                          New exception value to propagate
 959
 960 **/
 961 STATIC
 962 UINT64
 963 VmmCallExit (
 964   IN OUT GHCB                     *Ghcb,
 965   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 966   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
 967   )
 968 {
 969   UINT64  Status;
 970
 971   DecodeModRm (Regs, InstructionData);
 972
 973   Ghcb->SaveArea.Rax = Regs->Rax;
 974   VmgSetOffsetValid (Ghcb, GhcbRax);
 975   Ghcb->SaveArea.Cpl = (UINT8) (Regs->Cs & 0x3);
 976   VmgSetOffsetValid (Ghcb, GhcbCpl);
 977
 978   Status = VmgExit (Ghcb, SVM_EXIT_VMMCALL, 0, 0);
 979   if (Status != 0) {
 980     return Status;
 981   }
 982
 983   if (!VmgIsOffsetValid (Ghcb, GhcbRax)) {
 984     return UnsupportedExit (Ghcb, Regs, InstructionData);
 985   }
 986   Regs->Rax = Ghcb->SaveArea.Rax;
 987
 988   return 0;
 989 }
 990
 991 /**
 992   Handle an MSR event.
 993
 994   Use the VMGEXIT instruction to handle either a RDMSR or WRMSR event.
 995
 996   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 997                                    Block
 998   @param[in, out] Regs             x64 processor context
 999   @param[in]      InstructionData  Instruction parsing context
1000
1001   @retval 0                        Event handled successfully
1002   @return                          New exception value to propagate
1003
1004 **/
1005 STATIC
1006 UINT64
1007 MsrExit (
1008   IN OUT GHCB                     *Ghcb,
1009   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1010   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1011   )
1012 {
1013   UINT64  ExitInfo1, Status;
1014
1015   ExitInfo1 = 0;
1016
1017   switch (*(InstructionData->OpCodes + 1)) {
1018   case 0x30: // WRMSR
1019     ExitInfo1 = 1;
1020     Ghcb->SaveArea.Rax = Regs->Rax;
1021     VmgSetOffsetValid (Ghcb, GhcbRax);
1022     Ghcb->SaveArea.Rdx = Regs->Rdx;
1023     VmgSetOffsetValid (Ghcb, GhcbRdx);
1024     //
1025     // fall through
1026     //
1027   case 0x32: // RDMSR
1028     Ghcb->SaveArea.Rcx = Regs->Rcx;
1029     VmgSetOffsetValid (Ghcb, GhcbRcx);
1030     break;
1031   default:
1032     return UnsupportedExit (Ghcb, Regs, InstructionData);
1033   }
1034
1035   Status = VmgExit (Ghcb, SVM_EXIT_MSR, ExitInfo1, 0);
1036   if (Status != 0) {
1037     return Status;
1038   }
1039
1040   if (ExitInfo1 == 0) {
1041     if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1042         !VmgIsOffsetValid (Ghcb, GhcbRdx)) {
1043       return UnsupportedExit (Ghcb, Regs, InstructionData);
1044     }
1045     Regs->Rax = Ghcb->SaveArea.Rax;
1046     Regs->Rdx = Ghcb->SaveArea.Rdx;
1047   }
1048
1049   return 0;
1050 }
1051
1052 /**
1053   Build the IOIO event information.
1054
1055   The IOIO event information identifies the type of IO operation to be performed
1056   by the hypervisor. Build this information based on the instruction data.
1057
1058   @param[in]       Regs             x64 processor context
1059   @param[in, out]  InstructionData  Instruction parsing context
1060
1061   @return                           IOIO event information value
1062
1063 **/
1064 STATIC
1065 UINT64
1066 IoioExitInfo (
1067   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
1068   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
1069   )
1070 {
1071   UINT64  ExitInfo;
1072
1073   ExitInfo = 0;
1074
1075   switch (*(InstructionData->OpCodes)) {
1076   //
1077   // INS opcodes
1078   //
1079   case 0x6C:
1080   case 0x6D:
1081     ExitInfo |= IOIO_TYPE_INS;
1082     ExitInfo |= IOIO_SEG_ES;
1083     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1084     break;
1085
1086   //
1087   // OUTS opcodes
1088   //
1089   case 0x6E:
1090   case 0x6F:
1091     ExitInfo |= IOIO_TYPE_OUTS;
1092     ExitInfo |= IOIO_SEG_DS;
1093     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1094     break;
1095
1096   //
1097   // IN immediate opcodes
1098   //
1099   case 0xE4:
1100   case 0xE5:
1101     InstructionData->ImmediateSize = 1;
1102     InstructionData->End++;
1103     ExitInfo |= IOIO_TYPE_IN;
1104     ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16);
1105     break;
1106
1107   //
1108   // OUT immediate opcodes
1109   //
1110   case 0xE6:
1111   case 0xE7:
1112     InstructionData->ImmediateSize = 1;
1113     InstructionData->End++;
1114     ExitInfo |= IOIO_TYPE_OUT;
1115     ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16) | IOIO_TYPE_OUT;
1116     break;
1117
1118   //
1119   // IN register opcodes
1120   //
1121   case 0xEC:
1122   case 0xED:
1123     ExitInfo |= IOIO_TYPE_IN;
1124     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1125     break;
1126
1127   //
1128   // OUT register opcodes
1129   //
1130   case 0xEE:
1131   case 0xEF:
1132     ExitInfo |= IOIO_TYPE_OUT;
1133     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1134     break;
1135
1136   default:
1137     return 0;
1138   }
1139
1140   switch (*(InstructionData->OpCodes)) {
1141   //
1142   // Single-byte opcodes
1143   //
1144   case 0x6C:
1145   case 0x6E:
1146   case 0xE4:
1147   case 0xE6:
1148   case 0xEC:
1149   case 0xEE:
1150     ExitInfo |= IOIO_DATA_8;
1151     break;
1152
1153   //
1154   // Length determined by instruction parsing
1155   //
1156   default:
1157     ExitInfo |= (InstructionData->DataSize == Size16Bits) ? IOIO_DATA_16
1158                                                           : IOIO_DATA_32;
1159   }
1160
1161   switch (InstructionData->AddrSize) {
1162   case Size16Bits:
1163     ExitInfo |= IOIO_ADDR_16;
1164     break;
1165
1166   case Size32Bits:
1167     ExitInfo |= IOIO_ADDR_32;
1168     break;
1169
1170   case Size64Bits:
1171     ExitInfo |= IOIO_ADDR_64;
1172     break;
1173
1174   default:
1175     break;
1176   }
1177
1178   if (InstructionData->RepMode != 0) {
1179     ExitInfo |= IOIO_REP;
1180   }
1181
1182   return ExitInfo;
1183 }
1184
1185 /**
1186   Handle an IOIO event.
1187
1188   Use the VMGEXIT instruction to handle an IOIO event.
1189
1190   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1191                                    Block
1192   @param[in, out] Regs             x64 processor context
1193   @param[in]      InstructionData  Instruction parsing context
1194
1195   @retval 0                        Event handled successfully
1196   @return                          New exception value to propagate
1197
1198 **/
1199 STATIC
1200 UINT64
1201 IoioExit (
1202   IN OUT GHCB                     *Ghcb,
1203   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1204   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1205   )
1206 {
1207   UINT64   ExitInfo1, ExitInfo2, Status;
1208   BOOLEAN  IsString;
1209
1210   ExitInfo1 = IoioExitInfo (Regs, InstructionData);
1211   if (ExitInfo1 == 0) {
1212     return UnsupportedExit (Ghcb, Regs, InstructionData);
1213   }
1214
1215   IsString = ((ExitInfo1 & IOIO_TYPE_STR) != 0) ? TRUE : FALSE;
1216   if (IsString) {
1217     UINTN  IoBytes, VmgExitBytes;
1218     UINTN  GhcbCount, OpCount;
1219
1220     Status = 0;
1221
1222     IoBytes = IOIO_DATA_BYTES (ExitInfo1);
1223     GhcbCount = sizeof (Ghcb->SharedBuffer) / IoBytes;
1224
1225     OpCount = ((ExitInfo1 & IOIO_REP) != 0) ? Regs->Rcx : 1;
1226     while (OpCount != 0) {
1227       ExitInfo2 = MIN (OpCount, GhcbCount);
1228       VmgExitBytes = ExitInfo2 * IoBytes;
1229
1230       if ((ExitInfo1 & IOIO_TYPE_IN) == 0) {
1231         CopyMem (Ghcb->SharedBuffer, (VOID *) Regs->Rsi, VmgExitBytes);
1232         Regs->Rsi += VmgExitBytes;
1233       }
1234
1235       Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
1236       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
1237       Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, ExitInfo2);
1238       if (Status != 0) {
1239         return Status;
1240       }
1241
1242       if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1243         CopyMem ((VOID *) Regs->Rdi, Ghcb->SharedBuffer, VmgExitBytes);
1244         Regs->Rdi += VmgExitBytes;
1245       }
1246
1247       if ((ExitInfo1 & IOIO_REP) != 0) {
1248         Regs->Rcx -= ExitInfo2;
1249       }
1250
1251       OpCount -= ExitInfo2;
1252     }
1253   } else {
1254     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1255       Ghcb->SaveArea.Rax = 0;
1256     } else {
1257       CopyMem (&Ghcb->SaveArea.Rax, &Regs->Rax, IOIO_DATA_BYTES (ExitInfo1));
1258     }
1259     VmgSetOffsetValid (Ghcb, GhcbRax);
1260
1261     Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, 0);
1262     if (Status != 0) {
1263       return Status;
1264     }
1265
1266     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1267       if (!VmgIsOffsetValid (Ghcb, GhcbRax)) {
1268         return UnsupportedExit (Ghcb, Regs, InstructionData);
1269       }
1270       CopyMem (&Regs->Rax, &Ghcb->SaveArea.Rax, IOIO_DATA_BYTES (ExitInfo1));
1271     }
1272   }
1273
1274   return 0;
1275 }
1276
1277 /**
1278   Handle a INVD event.
1279
1280   Use the VMGEXIT instruction to handle a INVD event.
1281
1282   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1283                                    Block
1284   @param[in, out] Regs             x64 processor context
1285   @param[in]      InstructionData  Instruction parsing context
1286
1287   @retval 0                        Event handled successfully
1288   @return                          New exception value to propagate
1289
1290 **/
1291 STATIC
1292 UINT64
1293 InvdExit (
1294   IN OUT GHCB                     *Ghcb,
1295   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1296   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1297   )
1298 {
1299   return VmgExit (Ghcb, SVM_EXIT_INVD, 0, 0);
1300 }
1301
1302 /**
1303   Handle a CPUID event.
1304
1305   Use the VMGEXIT instruction to handle a CPUID event.
1306
1307   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1308                                    Block
1309   @param[in, out] Regs             x64 processor context
1310   @param[in]      InstructionData  Instruction parsing context
1311
1312   @retval 0                        Event handled successfully
1313   @return                          New exception value to propagate
1314
1315 **/
1316 STATIC
1317 UINT64
1318 CpuidExit (
1319   IN OUT GHCB                     *Ghcb,
1320   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1321   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1322   )
1323 {
1324   UINT64  Status;
1325
1326   Ghcb->SaveArea.Rax = Regs->Rax;
1327   VmgSetOffsetValid (Ghcb, GhcbRax);
1328   Ghcb->SaveArea.Rcx = Regs->Rcx;
1329   VmgSetOffsetValid (Ghcb, GhcbRcx);
1330   if (Regs->Rax == CPUID_EXTENDED_STATE) {
1331     IA32_CR4  Cr4;
1332
1333     Cr4.UintN = AsmReadCr4 ();
1334     Ghcb->SaveArea.XCr0 = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
1335     VmgSetOffsetValid (Ghcb, GhcbXCr0);
1336   }
1337
1338   Status = VmgExit (Ghcb, SVM_EXIT_CPUID, 0, 0);
1339   if (Status != 0) {
1340     return Status;
1341   }
1342
1343   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1344       !VmgIsOffsetValid (Ghcb, GhcbRbx) ||
1345       !VmgIsOffsetValid (Ghcb, GhcbRcx) ||
1346       !VmgIsOffsetValid (Ghcb, GhcbRdx)) {
1347     return UnsupportedExit (Ghcb, Regs, InstructionData);
1348   }
1349   Regs->Rax = Ghcb->SaveArea.Rax;
1350   Regs->Rbx = Ghcb->SaveArea.Rbx;
1351   Regs->Rcx = Ghcb->SaveArea.Rcx;
1352   Regs->Rdx = Ghcb->SaveArea.Rdx;
1353
1354   return 0;
1355 }
1356
1357 /**
1358   Handle a RDPMC event.
1359
1360   Use the VMGEXIT instruction to handle a RDPMC event.
1361
1362   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1363                                    Block
1364   @param[in, out] Regs             x64 processor context
1365   @param[in]      InstructionData  Instruction parsing context
1366
1367   @retval 0                        Event handled successfully
1368   @return                          New exception value to propagate
1369
1370 **/
1371 STATIC
1372 UINT64
1373 RdpmcExit (
1374   IN OUT GHCB                     *Ghcb,
1375   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1376   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1377   )
1378 {
1379   UINT64  Status;
1380
1381   Ghcb->SaveArea.Rcx = Regs->Rcx;
1382   VmgSetOffsetValid (Ghcb, GhcbRcx);
1383
1384   Status = VmgExit (Ghcb, SVM_EXIT_RDPMC, 0, 0);
1385   if (Status != 0) {
1386     return Status;
1387   }
1388
1389   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1390       !VmgIsOffsetValid (Ghcb, GhcbRdx)) {
1391     return UnsupportedExit (Ghcb, Regs, InstructionData);
1392   }
1393   Regs->Rax = Ghcb->SaveArea.Rax;
1394   Regs->Rdx = Ghcb->SaveArea.Rdx;
1395
1396   return 0;
1397 }
1398
1399 /**
1400   Handle a RDTSC event.
1401
1402   Use the VMGEXIT instruction to handle a RDTSC event.
1403
1404   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1405                                    Block
1406   @param[in, out] Regs             x64 processor context
1407   @param[in]      InstructionData  Instruction parsing context
1408
1409   @retval 0                        Event handled successfully
1410   @return                          New exception value to propagate
1411
1412 **/
1413 STATIC
1414 UINT64
1415 RdtscExit (
1416   IN OUT GHCB                     *Ghcb,
1417   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1418   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1419   )
1420 {
1421   UINT64  Status;
1422
1423   Status = VmgExit (Ghcb, SVM_EXIT_RDTSC, 0, 0);
1424   if (Status != 0) {
1425     return Status;
1426   }
1427
1428   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1429       !VmgIsOffsetValid (Ghcb, GhcbRdx)) {
1430     return UnsupportedExit (Ghcb, Regs, InstructionData);
1431   }
1432   Regs->Rax = Ghcb->SaveArea.Rax;
1433   Regs->Rdx = Ghcb->SaveArea.Rdx;
1434
1435   return 0;
1436 }
1437
1438 /**
1439   Handle a DR7 register write event.
1440
1441   Use the VMGEXIT instruction to handle a DR7 write event.
1442
1443   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1444                                    Block
1445   @param[in, out] Regs             x64 processor context
1446   @param[in]      InstructionData  Instruction parsing context
1447
1448   @retval 0                        Event handled successfully
1449   @return                          New exception value to propagate
1450
1451 **/
1452 STATIC
1453 UINT64
1454 Dr7WriteExit (
1455   IN OUT GHCB                     *Ghcb,
1456   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1457   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1458   )
1459 {
1460   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
1461   SEV_ES_PER_CPU_DATA            *SevEsData;
1462   UINT64                         *Register;
1463   UINT64                         Status;
1464
1465   Ext = &InstructionData->Ext;
1466   SevEsData = (SEV_ES_PER_CPU_DATA *) (Ghcb + 1);
1467
1468   DecodeModRm (Regs, InstructionData);
1469
1470   //
1471   // MOV DRn always treats MOD == 3 no matter how encoded
1472   //
1473   Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
1474
1475   //
1476   // Using a value of 0 for ExitInfo1 means RAX holds the value
1477   //
1478   Ghcb->SaveArea.Rax = *Register;
1479   VmgSetOffsetValid (Ghcb, GhcbRax);
1480
1481   Status = VmgExit (Ghcb, SVM_EXIT_DR7_WRITE, 0, 0);
1482   if (Status != 0) {
1483     return Status;
1484   }
1485
1486   SevEsData->Dr7 = *Register;
1487   SevEsData->Dr7Cached = TRUE;
1488
1489   return 0;
1490 }
1491
1492 /**
1493   Handle a DR7 register read event.
1494
1495   Use the VMGEXIT instruction to handle a DR7 read event.
1496
1497   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1498                                    Block
1499   @param[in, out] Regs             x64 processor context
1500   @param[in]      InstructionData  Instruction parsing context
1501
1502   @retval 0                        Event handled successfully
1503
1504 **/
1505 STATIC
1506 UINT64
1507 Dr7ReadExit (
1508   IN OUT GHCB                     *Ghcb,
1509   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1510   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1511   )
1512 {
1513   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
1514   SEV_ES_PER_CPU_DATA            *SevEsData;
1515   UINT64                         *Register;
1516
1517   Ext = &InstructionData->Ext;
1518   SevEsData = (SEV_ES_PER_CPU_DATA *) (Ghcb + 1);
1519
1520   DecodeModRm (Regs, InstructionData);
1521
1522   //
1523   // MOV DRn always treats MOD == 3 no matter how encoded
1524   //
1525   Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
1526
1527   //
1528   // If there is a cached valued for DR7, return that. Otherwise return the
1529   // DR7 standard reset value of 0x400 (no debug breakpoints set).
1530   //
1531   *Register = (SevEsData->Dr7Cached) ? SevEsData->Dr7 : 0x400;
1532
1533   return 0;
1534 }
1535
1536 /**
1537   Handle a #VC exception.
1538
1539   Performs the necessary processing to handle a #VC exception.
1540
1541   @param[in, out]  ExceptionType  Pointer to an EFI_EXCEPTION_TYPE to be set
1542                                   as value to use on error.
1543   @param[in, out]  SystemContext  Pointer to EFI_SYSTEM_CONTEXT
1544
1545   @retval  EFI_SUCCESS            Exception handled
1546   @retval  EFI_UNSUPPORTED        #VC not supported, (new) exception value to
1547                                   propagate provided
1548   @retval  EFI_PROTOCOL_ERROR     #VC handling failed, (new) exception value to
1549                                   propagate provided
1550
1551 **/
1552 EFI_STATUS
1553 EFIAPI
1554 VmgExitHandleVc (
1555   IN OUT EFI_EXCEPTION_TYPE  *ExceptionType,
1556   IN OUT EFI_SYSTEM_CONTEXT  SystemContext
1557   )
1558 {
1559   MSR_SEV_ES_GHCB_REGISTER  Msr;
1560   EFI_SYSTEM_CONTEXT_X64    *Regs;
1561   GHCB                      *Ghcb;
1562   NAE_EXIT                  NaeExit;
1563   SEV_ES_INSTRUCTION_DATA   InstructionData;
1564   UINT64                    ExitCode, Status;
1565   EFI_STATUS                VcRet;
1566
1567   VcRet = EFI_SUCCESS;
1568
1569   Msr.GhcbPhysicalAddress = AsmReadMsr64 (MSR_SEV_ES_GHCB);
1570   ASSERT (Msr.GhcbInfo.Function == 0);
1571   ASSERT (Msr.Ghcb != 0);
1572
1573   Regs = SystemContext.SystemContextX64;
1574   Ghcb = Msr.Ghcb;
1575
1576   VmgInit (Ghcb);
1577
1578   ExitCode = Regs->ExceptionData;
1579   switch (ExitCode) {
1580   case SVM_EXIT_DR7_READ:
1581     NaeExit = Dr7ReadExit;
1582     break;
1583
1584   case SVM_EXIT_DR7_WRITE:
1585     NaeExit = Dr7WriteExit;
1586     break;
1587
1588   case SVM_EXIT_RDTSC:
1589     NaeExit = RdtscExit;
1590     break;
1591
1592   case SVM_EXIT_RDPMC:
1593     NaeExit = RdpmcExit;
1594     break;
1595
1596   case SVM_EXIT_CPUID:
1597     NaeExit = CpuidExit;
1598     break;
1599
1600   case SVM_EXIT_INVD:
1601     NaeExit = InvdExit;
1602     break;
1603
1604   case SVM_EXIT_IOIO_PROT:
1605     NaeExit = IoioExit;
1606     break;
1607
1608   case SVM_EXIT_MSR:
1609     NaeExit = MsrExit;
1610     break;
1611
1612   case SVM_EXIT_VMMCALL:
1613     NaeExit = VmmCallExit;
1614     break;
1615
1616   case SVM_EXIT_RDTSCP:
1617     NaeExit = RdtscpExit;
1618     break;
1619
1620   case SVM_EXIT_WBINVD:
1621     NaeExit = WbinvdExit;
1622     break;
1623
1624   case SVM_EXIT_MONITOR:
1625     NaeExit = MonitorExit;
1626     break;
1627
1628   case SVM_EXIT_MWAIT:
1629     NaeExit = MwaitExit;
1630     break;
1631
1632   case SVM_EXIT_NPF:
1633     NaeExit = MmioExit;
1634     break;
1635
1636   default:
1637     NaeExit = UnsupportedExit;
1638   }
1639
1640   InitInstructionData (&InstructionData, Ghcb, Regs);
1641
1642   Status = NaeExit (Ghcb, Regs, &InstructionData);
1643   if (Status == 0) {
1644     Regs->Rip += InstructionLength (&InstructionData);
1645   } else {
1646     GHCB_EVENT_INJECTION  Event;
1647
1648     Event.Uint64 = Status;
1649     if (Event.Elements.ErrorCodeValid != 0) {
1650       Regs->ExceptionData = Event.Elements.ErrorCode;
1651     } else {
1652       Regs->ExceptionData = 0;
1653     }
1654
1655     *ExceptionType = Event.Elements.Vector;
1656
1657     VcRet = EFI_PROTOCOL_ERROR;
1658   }
1659
1660   VmgDone (Ghcb);
1661
1662   return VcRet;
1663 }