OvmfPkg/Library/VmgExitLib/VmgExitVcHandler.c

   1 /** @file
   2   X64 #VC Exception Handler functon.
   3
   4   Copyright (C) 2020, Advanced Micro Devices, Inc. All rights reserved.<BR>
   5   SPDX-License-Identifier: BSD-2-Clause-Patent
   6
   7 **/
   8
   9 #include <Base.h>
  10 #include <Uefi.h>
  11 #include <Library/BaseMemoryLib.h>
  12 #include <Library/LocalApicLib.h>
  13 #include <Library/MemEncryptSevLib.h>
  14 #include <Library/VmgExitLib.h>
  15 #include <Register/Amd/Msr.h>
  16 #include <Register/Intel/Cpuid.h>
  17 #include <IndustryStandard/InstructionParsing.h>
  18
  19 #include "VmgExitVcHandler.h"
  20
  21 //
  22 // Instruction execution mode definition
  23 //
  24 typedef enum {
  25   LongMode64Bit        = 0,
  26   LongModeCompat32Bit,
  27   LongModeCompat16Bit,
  28 } SEV_ES_INSTRUCTION_MODE;
  29
  30 //
  31 // Instruction size definition (for operand and address)
  32 //
  33 typedef enum {
  34   Size8Bits            = 0,
  35   Size16Bits,
  36   Size32Bits,
  37   Size64Bits,
  38 } SEV_ES_INSTRUCTION_SIZE;
  39
  40 //
  41 // Intruction segment definition
  42 //
  43 typedef enum {
  44   SegmentEs            = 0,
  45   SegmentCs,
  46   SegmentSs,
  47   SegmentDs,
  48   SegmentFs,
  49   SegmentGs,
  50 } SEV_ES_INSTRUCTION_SEGMENT;
  51
  52 //
  53 // Instruction rep function definition
  54 //
  55 typedef enum {
  56   RepNone              = 0,
  57   RepZ,
  58   RepNZ,
  59 } SEV_ES_INSTRUCTION_REP;
  60
  61 typedef struct {
  62   UINT8  Rm;
  63   UINT8  Reg;
  64   UINT8  Mod;
  65 } SEV_ES_INSTRUCTION_MODRM_EXT;
  66
  67 typedef struct {
  68   UINT8  Base;
  69   UINT8  Index;
  70   UINT8  Scale;
  71 } SEV_ES_INSTRUCTION_SIB_EXT;
  72
  73 //
  74 // Instruction opcode definition
  75 //
  76 typedef struct {
  77   SEV_ES_INSTRUCTION_MODRM_EXT  ModRm;
  78
  79   SEV_ES_INSTRUCTION_SIB_EXT    Sib;
  80
  81   UINTN                         RegData;
  82   UINTN                         RmData;
  83 } SEV_ES_INSTRUCTION_OPCODE_EXT;
  84
  85 //
  86 // Instruction parsing context definition
  87 //
  88 typedef struct {
  89   GHCB                           *Ghcb;
  90
  91   SEV_ES_INSTRUCTION_MODE        Mode;
  92   SEV_ES_INSTRUCTION_SIZE        DataSize;
  93   SEV_ES_INSTRUCTION_SIZE        AddrSize;
  94   BOOLEAN                        SegmentSpecified;
  95   SEV_ES_INSTRUCTION_SEGMENT     Segment;
  96   SEV_ES_INSTRUCTION_REP         RepMode;
  97
  98   UINT8                          *Begin;
  99   UINT8                          *End;
 100
 101   UINT8                          *Prefixes;
 102   UINT8                          *OpCodes;
 103   UINT8                          *Displacement;
 104   UINT8                          *Immediate;
 105
 106   INSTRUCTION_REX_PREFIX         RexPrefix;
 107
 108   BOOLEAN                        ModRmPresent;
 109   INSTRUCTION_MODRM              ModRm;
 110
 111   BOOLEAN                        SibPresent;
 112   INSTRUCTION_SIB                Sib;
 113
 114   UINTN                          PrefixSize;
 115   UINTN                          OpCodeSize;
 116   UINTN                          DisplacementSize;
 117   UINTN                          ImmediateSize;
 118
 119   SEV_ES_INSTRUCTION_OPCODE_EXT  Ext;
 120 } SEV_ES_INSTRUCTION_DATA;
 121
 122 //
 123 // Non-automatic Exit function prototype
 124 //
 125 typedef
 126 UINT64
 127 (*NAE_EXIT) (
 128   GHCB                     *Ghcb,
 129   EFI_SYSTEM_CONTEXT_X64   *Regs,
 130   SEV_ES_INSTRUCTION_DATA  *InstructionData
 131   );
 132
 133 /**
 134   Return a pointer to the contents of the specified register.
 135
 136   Based upon the input register, return a pointer to the registers contents
 137   in the x86 processor context.
 138
 139   @param[in] Regs      x64 processor context
 140   @param[in] Register  Register to obtain pointer for
 141
 142   @return              Pointer to the contents of the requested register
 143
 144 **/
 145 STATIC
 146 UINT64 *
 147 GetRegisterPointer (
 148   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 149   IN UINT8                    Register
 150   )
 151 {
 152   UINT64 *Reg;
 153
 154   switch (Register) {
 155   case 0:
 156     Reg = &Regs->Rax;
 157     break;
 158   case 1:
 159     Reg = &Regs->Rcx;
 160     break;
 161   case 2:
 162     Reg = &Regs->Rdx;
 163     break;
 164   case 3:
 165     Reg = &Regs->Rbx;
 166     break;
 167   case 4:
 168     Reg = &Regs->Rsp;
 169     break;
 170   case 5:
 171     Reg = &Regs->Rbp;
 172     break;
 173   case 6:
 174     Reg = &Regs->Rsi;
 175     break;
 176   case 7:
 177     Reg = &Regs->Rdi;
 178     break;
 179   case 8:
 180     Reg = &Regs->R8;
 181     break;
 182   case 9:
 183     Reg = &Regs->R9;
 184     break;
 185   case 10:
 186     Reg = &Regs->R10;
 187     break;
 188   case 11:
 189     Reg = &Regs->R11;
 190     break;
 191   case 12:
 192     Reg = &Regs->R12;
 193     break;
 194   case 13:
 195     Reg = &Regs->R13;
 196     break;
 197   case 14:
 198     Reg = &Regs->R14;
 199     break;
 200   case 15:
 201     Reg = &Regs->R15;
 202     break;
 203   default:
 204     Reg = NULL;
 205   }
 206   ASSERT (Reg != NULL);
 207
 208   return Reg;
 209 }
 210
 211 /**
 212   Update the instruction parsing context for displacement bytes.
 213
 214   @param[in, out] InstructionData  Instruction parsing context
 215   @param[in]      Size             The instruction displacement size
 216
 217 **/
 218 STATIC
 219 VOID
 220 UpdateForDisplacement (
 221   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 222   IN     UINTN                    Size
 223   )
 224 {
 225   InstructionData->DisplacementSize = Size;
 226   InstructionData->Immediate += Size;
 227   InstructionData->End += Size;
 228 }
 229
 230 /**
 231   Determine if an instruction address if RIP relative.
 232
 233   Examine the instruction parsing context to determine if the address offset
 234   is relative to the instruction pointer.
 235
 236   @param[in] InstructionData  Instruction parsing context
 237
 238   @retval TRUE                Instruction addressing is RIP relative
 239   @retval FALSE               Instruction addressing is not RIP relative
 240
 241 **/
 242 STATIC
 243 BOOLEAN
 244 IsRipRelative (
 245   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 246   )
 247 {
 248   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 249
 250   Ext = &InstructionData->Ext;
 251
 252   return ((InstructionData->Mode == LongMode64Bit) &&
 253           (Ext->ModRm.Mod == 0) &&
 254           (Ext->ModRm.Rm == 5)  &&
 255           (InstructionData->SibPresent == FALSE));
 256 }
 257
 258 /**
 259   Return the effective address of a memory operand.
 260
 261   Examine the instruction parsing context to obtain the effective memory
 262   address of a memory operand.
 263
 264   @param[in] Regs             x64 processor context
 265   @param[in] InstructionData  Instruction parsing context
 266
 267   @return                     The memory operand effective address
 268
 269 **/
 270 STATIC
 271 UINT64
 272 GetEffectiveMemoryAddress (
 273   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 274   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 275   )
 276 {
 277   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 278   UINT64                         EffectiveAddress;
 279
 280   Ext = &InstructionData->Ext;
 281   EffectiveAddress = 0;
 282
 283   if (IsRipRelative (InstructionData)) {
 284     //
 285     // RIP-relative displacement is a 32-bit signed value
 286     //
 287     INT32 RipRelative;
 288
 289     RipRelative = *(INT32 *) InstructionData->Displacement;
 290
 291     UpdateForDisplacement (InstructionData, 4);
 292
 293     //
 294     // Negative displacement is handled by standard UINT64 wrap-around.
 295     //
 296     return Regs->Rip + (UINT64) RipRelative;
 297   }
 298
 299   switch (Ext->ModRm.Mod) {
 300   case 1:
 301     UpdateForDisplacement (InstructionData, 1);
 302     EffectiveAddress += (UINT64) (*(INT8 *) (InstructionData->Displacement));
 303     break;
 304   case 2:
 305     switch (InstructionData->AddrSize) {
 306     case Size16Bits:
 307       UpdateForDisplacement (InstructionData, 2);
 308       EffectiveAddress += (UINT64) (*(INT16 *) (InstructionData->Displacement));
 309       break;
 310     default:
 311       UpdateForDisplacement (InstructionData, 4);
 312       EffectiveAddress += (UINT64) (*(INT32 *) (InstructionData->Displacement));
 313       break;
 314     }
 315     break;
 316   }
 317
 318   if (InstructionData->SibPresent) {
 319     INT64  Displacement;
 320
 321     if (Ext->Sib.Index != 4) {
 322       CopyMem (
 323         &Displacement,
 324         GetRegisterPointer (Regs, Ext->Sib.Index),
 325         sizeof (Displacement)
 326         );
 327       Displacement *= (INT64)(1 << Ext->Sib.Scale);
 328
 329       //
 330       // Negative displacement is handled by standard UINT64 wrap-around.
 331       //
 332       EffectiveAddress += (UINT64) Displacement;
 333     }
 334
 335     if ((Ext->Sib.Base != 5) || Ext->ModRm.Mod) {
 336       EffectiveAddress += *GetRegisterPointer (Regs, Ext->Sib.Base);
 337     } else {
 338       UpdateForDisplacement (InstructionData, 4);
 339       EffectiveAddress += (UINT64) (*(INT32 *) (InstructionData->Displacement));
 340     }
 341   } else {
 342     EffectiveAddress += *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 343   }
 344
 345   return EffectiveAddress;
 346 }
 347
 348 /**
 349   Decode a ModRM byte.
 350
 351   Examine the instruction parsing context to decode a ModRM byte and the SIB
 352   byte, if present.
 353
 354   @param[in]      Regs             x64 processor context
 355   @param[in, out] InstructionData  Instruction parsing context
 356
 357 **/
 358 STATIC
 359 VOID
 360 DecodeModRm (
 361   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 362   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 363   )
 364 {
 365   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 366   INSTRUCTION_REX_PREFIX         *RexPrefix;
 367   INSTRUCTION_MODRM              *ModRm;
 368   INSTRUCTION_SIB                *Sib;
 369
 370   RexPrefix = &InstructionData->RexPrefix;
 371   Ext = &InstructionData->Ext;
 372   ModRm = &InstructionData->ModRm;
 373   Sib = &InstructionData->Sib;
 374
 375   InstructionData->ModRmPresent = TRUE;
 376   ModRm->Uint8 = *(InstructionData->End);
 377
 378   InstructionData->Displacement++;
 379   InstructionData->Immediate++;
 380   InstructionData->End++;
 381
 382   Ext->ModRm.Mod = ModRm->Bits.Mod;
 383   Ext->ModRm.Reg = (RexPrefix->Bits.BitR << 3) | ModRm->Bits.Reg;
 384   Ext->ModRm.Rm  = (RexPrefix->Bits.BitB << 3) | ModRm->Bits.Rm;
 385
 386   Ext->RegData = *GetRegisterPointer (Regs, Ext->ModRm.Reg);
 387
 388   if (Ext->ModRm.Mod == 3) {
 389     Ext->RmData = *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 390   } else {
 391     if (ModRm->Bits.Rm == 4) {
 392       InstructionData->SibPresent = TRUE;
 393       Sib->Uint8 = *(InstructionData->End);
 394
 395       InstructionData->Displacement++;
 396       InstructionData->Immediate++;
 397       InstructionData->End++;
 398
 399       Ext->Sib.Scale = Sib->Bits.Scale;
 400       Ext->Sib.Index = (RexPrefix->Bits.BitX << 3) | Sib->Bits.Index;
 401       Ext->Sib.Base  = (RexPrefix->Bits.BitB << 3) | Sib->Bits.Base;
 402     }
 403
 404     Ext->RmData = GetEffectiveMemoryAddress (Regs, InstructionData);
 405   }
 406 }
 407
 408 /**
 409   Decode instruction prefixes.
 410
 411   Parse the instruction data to track the instruction prefixes that have
 412   been used.
 413
 414   @param[in]      Regs             x64 processor context
 415   @param[in, out] InstructionData  Instruction parsing context
 416
 417 **/
 418 STATIC
 419 VOID
 420 DecodePrefixes (
 421   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 422   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 423   )
 424 {
 425   SEV_ES_INSTRUCTION_MODE  Mode;
 426   SEV_ES_INSTRUCTION_SIZE  ModeDataSize;
 427   SEV_ES_INSTRUCTION_SIZE  ModeAddrSize;
 428   UINT8                    *Byte;
 429
 430   //
 431   // Always in 64-bit mode
 432   //
 433   Mode = LongMode64Bit;
 434   ModeDataSize = Size32Bits;
 435   ModeAddrSize = Size64Bits;
 436
 437   InstructionData->Mode = Mode;
 438   InstructionData->DataSize = ModeDataSize;
 439   InstructionData->AddrSize = ModeAddrSize;
 440
 441   InstructionData->Prefixes = InstructionData->Begin;
 442
 443   Byte = InstructionData->Prefixes;
 444   for ( ; ; Byte++, InstructionData->PrefixSize++) {
 445     //
 446     // Check the 0x40 to 0x4F range using an if statement here since some
 447     // compilers don't like the "case 0x40 ... 0x4F:" syntax. This avoids
 448     // 16 case statements below.
 449     //
 450     if ((*Byte >= REX_PREFIX_START) && (*Byte <= REX_PREFIX_STOP)) {
 451       InstructionData->RexPrefix.Uint8 = *Byte;
 452       if ((*Byte & REX_64BIT_OPERAND_SIZE_MASK) != 0) {
 453         InstructionData->DataSize = Size64Bits;
 454       }
 455       continue;
 456     }
 457
 458     switch (*Byte) {
 459     case OVERRIDE_SEGMENT_CS:
 460     case OVERRIDE_SEGMENT_DS:
 461     case OVERRIDE_SEGMENT_ES:
 462     case OVERRIDE_SEGMENT_SS:
 463       if (Mode != LongMode64Bit) {
 464         InstructionData->SegmentSpecified = TRUE;
 465         InstructionData->Segment = (*Byte >> 3) & 3;
 466       }
 467       break;
 468
 469     case OVERRIDE_SEGMENT_FS:
 470     case OVERRIDE_SEGMENT_GS:
 471       InstructionData->SegmentSpecified = TRUE;
 472       InstructionData->Segment = *Byte & 7;
 473       break;
 474
 475     case OVERRIDE_OPERAND_SIZE:
 476       if (InstructionData->RexPrefix.Uint8 == 0) {
 477         InstructionData->DataSize =
 478           (Mode == LongMode64Bit)       ? Size16Bits :
 479           (Mode == LongModeCompat32Bit) ? Size16Bits :
 480           (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 481       }
 482       break;
 483
 484     case OVERRIDE_ADDRESS_SIZE:
 485       InstructionData->AddrSize =
 486         (Mode == LongMode64Bit)       ? Size32Bits :
 487         (Mode == LongModeCompat32Bit) ? Size16Bits :
 488         (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 489       break;
 490
 491     case LOCK_PREFIX:
 492       break;
 493
 494     case REPZ_PREFIX:
 495       InstructionData->RepMode = RepZ;
 496       break;
 497
 498     case REPNZ_PREFIX:
 499       InstructionData->RepMode = RepNZ;
 500       break;
 501
 502     default:
 503       InstructionData->OpCodes = Byte;
 504       InstructionData->OpCodeSize = (*Byte == TWO_BYTE_OPCODE_ESCAPE) ? 2 : 1;
 505
 506       InstructionData->End = Byte + InstructionData->OpCodeSize;
 507       InstructionData->Displacement = InstructionData->End;
 508       InstructionData->Immediate = InstructionData->End;
 509       return;
 510     }
 511   }
 512 }
 513
 514 /**
 515   Determine instruction length
 516
 517   Return the total length of the parsed instruction.
 518
 519   @param[in] InstructionData  Instruction parsing context
 520
 521   @return                     Length of parsed instruction
 522
 523 **/
 524 STATIC
 525 UINT64
 526 InstructionLength (
 527   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 528   )
 529 {
 530   return (UINT64) (InstructionData->End - InstructionData->Begin);
 531 }
 532
 533 /**
 534   Initialize the instruction parsing context.
 535
 536   Initialize the instruction parsing context, which includes decoding the
 537   instruction prefixes.
 538
 539   @param[in, out] InstructionData  Instruction parsing context
 540   @param[in]      Ghcb             Pointer to the Guest-Hypervisor Communication
 541                                    Block
 542   @param[in]      Regs             x64 processor context
 543
 544 **/
 545 STATIC
 546 VOID
 547 InitInstructionData (
 548   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 549   IN     GHCB                     *Ghcb,
 550   IN     EFI_SYSTEM_CONTEXT_X64   *Regs
 551   )
 552 {
 553   SetMem (InstructionData, sizeof (*InstructionData), 0);
 554   InstructionData->Ghcb = Ghcb;
 555   InstructionData->Begin = (UINT8 *) Regs->Rip;
 556   InstructionData->End = (UINT8 *) Regs->Rip;
 557
 558   DecodePrefixes (Regs, InstructionData);
 559 }
 560
 561 /**
 562   Report an unsupported event to the hypervisor
 563
 564   Use the VMGEXIT support to report an unsupported event to the hypervisor.
 565
 566   @param[in] Ghcb             Pointer to the Guest-Hypervisor Communication
 567                               Block
 568   @param[in] Regs             x64 processor context
 569   @param[in] InstructionData  Instruction parsing context
 570
 571   @return                     New exception value to propagate
 572
 573 **/
 574 STATIC
 575 UINT64
 576 UnsupportedExit (
 577   IN GHCB                     *Ghcb,
 578   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 579   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 580   )
 581 {
 582   UINT64  Status;
 583
 584   Status = VmgExit (Ghcb, SVM_EXIT_UNSUPPORTED, Regs->ExceptionData, 0);
 585   if (Status == 0) {
 586     GHCB_EVENT_INJECTION  Event;
 587
 588     Event.Uint64 = 0;
 589     Event.Elements.Vector = GP_EXCEPTION;
 590     Event.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
 591     Event.Elements.Valid  = 1;
 592
 593     Status = Event.Uint64;
 594   }
 595
 596   return Status;
 597 }
 598
 599 /**
 600   Validate that the MMIO memory access is not to encrypted memory.
 601
 602   Examine the pagetable entry for the memory specified. MMIO should not be
 603   performed against encrypted memory. MMIO to the APIC page is always allowed.
 604
 605   @param[in] Ghcb           Pointer to the Guest-Hypervisor Communication Block
 606   @param[in] MemoryAddress  Memory address to validate
 607   @param[in] MemoryLength   Memory length to validate
 608
 609   @retval 0          Memory is not encrypted
 610   @return            New exception value to propogate
 611
 612 **/
 613 STATIC
 614 UINT64
 615 ValidateMmioMemory (
 616   IN GHCB   *Ghcb,
 617   IN UINTN  MemoryAddress,
 618   IN UINTN  MemoryLength
 619   )
 620 {
 621   MEM_ENCRYPT_SEV_ADDRESS_RANGE_STATE  State;
 622   GHCB_EVENT_INJECTION                 GpEvent;
 623   UINTN                                Address;
 624
 625   //
 626   // Allow APIC accesses (which will have the encryption bit set during
 627   // SEC and PEI phases).
 628   //
 629   Address = MemoryAddress & ~(SIZE_4KB - 1);
 630   if (Address == GetLocalApicBaseAddress ()) {
 631     return 0;
 632   }
 633
 634   State = MemEncryptSevGetAddressRangeState (
 635             0,
 636             MemoryAddress,
 637             MemoryLength
 638             );
 639   if (State == MemEncryptSevAddressRangeUnencrypted) {
 640     return 0;
 641   }
 642
 643   //
 644   // Any state other than unencrypted is an error, issue a #GP.
 645   //
 646   DEBUG ((DEBUG_ERROR,
 647     "MMIO using encrypted memory: %lx\n",
 648     (UINT64) MemoryAddress));
 649   GpEvent.Uint64 = 0;
 650   GpEvent.Elements.Vector = GP_EXCEPTION;
 651   GpEvent.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
 652   GpEvent.Elements.Valid  = 1;
 653
 654   return GpEvent.Uint64;
 655 }
 656
 657 /**
 658   Handle an MMIO event.
 659
 660   Use the VMGEXIT instruction to handle either an MMIO read or an MMIO write.
 661
 662   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 663                                    Block
 664   @param[in, out] Regs             x64 processor context
 665   @param[in, out] InstructionData  Instruction parsing context
 666
 667   @retval 0                        Event handled successfully
 668   @return                          New exception value to propagate
 669
 670 **/
 671 STATIC
 672 UINT64
 673 MmioExit (
 674   IN OUT GHCB                     *Ghcb,
 675   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 676   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 677   )
 678 {
 679   UINT64  ExitInfo1, ExitInfo2, Status;
 680   UINTN   Bytes;
 681   UINT64  *Register;
 682   UINT8   OpCode, SignByte;
 683   UINTN   Address;
 684
 685   Bytes = 0;
 686
 687   OpCode = *(InstructionData->OpCodes);
 688   if (OpCode == TWO_BYTE_OPCODE_ESCAPE) {
 689     OpCode = *(InstructionData->OpCodes + 1);
 690   }
 691
 692   switch (OpCode) {
 693   //
 694   // MMIO write (MOV reg/memX, regX)
 695   //
 696   case 0x88:
 697     Bytes = 1;
 698     //
 699     // fall through
 700     //
 701   case 0x89:
 702     DecodeModRm (Regs, InstructionData);
 703     Bytes = ((Bytes != 0) ? Bytes :
 704              (InstructionData->DataSize == Size16Bits) ? 2 :
 705              (InstructionData->DataSize == Size32Bits) ? 4 :
 706              (InstructionData->DataSize == Size64Bits) ? 8 :
 707              0);
 708
 709     if (InstructionData->Ext.ModRm.Mod == 3) {
 710       //
 711       // NPF on two register operands???
 712       //
 713       return UnsupportedExit (Ghcb, Regs, InstructionData);
 714     }
 715
 716     Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 717     if (Status != 0) {
 718       return Status;
 719     }
 720
 721     ExitInfo1 = InstructionData->Ext.RmData;
 722     ExitInfo2 = Bytes;
 723     CopyMem (Ghcb->SharedBuffer, &InstructionData->Ext.RegData, Bytes);
 724
 725     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 726     VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 727     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 728     if (Status != 0) {
 729       return Status;
 730     }
 731     break;
 732
 733   //
 734   // MMIO write (MOV moffsetX, aX)
 735   //
 736   case 0xA2:
 737     Bytes = 1;
 738     //
 739     // fall through
 740     //
 741   case 0xA3:
 742     Bytes = ((Bytes != 0) ? Bytes :
 743              (InstructionData->DataSize == Size16Bits) ? 2 :
 744              (InstructionData->DataSize == Size32Bits) ? 4 :
 745              (InstructionData->DataSize == Size64Bits) ? 8 :
 746              0);
 747
 748     InstructionData->ImmediateSize = (UINTN) (1 << InstructionData->AddrSize);
 749     InstructionData->End += InstructionData->ImmediateSize;
 750
 751     //
 752     // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
 753     // Use a STATIC_ASSERT to be certain the code is being built as X64.
 754     //
 755     STATIC_ASSERT (
 756       sizeof (UINTN) == sizeof (UINT64),
 757       "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
 758       );
 759
 760     Address = 0;
 761     CopyMem (
 762       &Address,
 763       InstructionData->Immediate,
 764       InstructionData->ImmediateSize
 765       );
 766
 767     Status = ValidateMmioMemory (Ghcb, Address, Bytes);
 768     if (Status != 0) {
 769       return Status;
 770     }
 771
 772     ExitInfo1 = Address;
 773     ExitInfo2 = Bytes;
 774     CopyMem (Ghcb->SharedBuffer, &Regs->Rax, Bytes);
 775
 776     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 777     VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 778     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 779     if (Status != 0) {
 780       return Status;
 781     }
 782     break;
 783
 784   //
 785   // MMIO write (MOV reg/memX, immX)
 786   //
 787   case 0xC6:
 788     Bytes = 1;
 789     //
 790     // fall through
 791     //
 792   case 0xC7:
 793     DecodeModRm (Regs, InstructionData);
 794     Bytes = ((Bytes != 0) ? Bytes :
 795              (InstructionData->DataSize == Size16Bits) ? 2 :
 796              (InstructionData->DataSize == Size32Bits) ? 4 :
 797              0);
 798
 799     InstructionData->ImmediateSize = Bytes;
 800     InstructionData->End += Bytes;
 801
 802     Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 803     if (Status != 0) {
 804       return Status;
 805     }
 806
 807     ExitInfo1 = InstructionData->Ext.RmData;
 808     ExitInfo2 = Bytes;
 809     CopyMem (Ghcb->SharedBuffer, InstructionData->Immediate, Bytes);
 810
 811     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 812     VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 813     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 814     if (Status != 0) {
 815       return Status;
 816     }
 817     break;
 818
 819   //
 820   // MMIO read (MOV regX, reg/memX)
 821   //
 822   case 0x8A:
 823     Bytes = 1;
 824     //
 825     // fall through
 826     //
 827   case 0x8B:
 828     DecodeModRm (Regs, InstructionData);
 829     Bytes = ((Bytes != 0) ? Bytes :
 830              (InstructionData->DataSize == Size16Bits) ? 2 :
 831              (InstructionData->DataSize == Size32Bits) ? 4 :
 832              (InstructionData->DataSize == Size64Bits) ? 8 :
 833              0);
 834     if (InstructionData->Ext.ModRm.Mod == 3) {
 835       //
 836       // NPF on two register operands???
 837       //
 838       return UnsupportedExit (Ghcb, Regs, InstructionData);
 839     }
 840
 841     Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 842     if (Status != 0) {
 843       return Status;
 844     }
 845
 846     ExitInfo1 = InstructionData->Ext.RmData;
 847     ExitInfo2 = Bytes;
 848
 849     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 850     VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 851     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 852     if (Status != 0) {
 853       return Status;
 854     }
 855
 856     Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 857     if (Bytes == 4) {
 858       //
 859       // Zero-extend for 32-bit operation
 860       //
 861       *Register = 0;
 862     }
 863     CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 864     break;
 865
 866   //
 867   // MMIO read (MOV aX, moffsetX)
 868   //
 869   case 0xA0:
 870     Bytes = 1;
 871     //
 872     // fall through
 873     //
 874   case 0xA1:
 875     Bytes = ((Bytes != 0) ? Bytes :
 876              (InstructionData->DataSize == Size16Bits) ? 2 :
 877              (InstructionData->DataSize == Size32Bits) ? 4 :
 878              (InstructionData->DataSize == Size64Bits) ? 8 :
 879              0);
 880
 881     InstructionData->ImmediateSize = (UINTN) (1 << InstructionData->AddrSize);
 882     InstructionData->End += InstructionData->ImmediateSize;
 883
 884     //
 885     // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
 886     // Use a STATIC_ASSERT to be certain the code is being built as X64.
 887     //
 888     STATIC_ASSERT (
 889       sizeof (UINTN) == sizeof (UINT64),
 890       "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
 891       );
 892
 893     Address = 0;
 894     CopyMem (
 895       &Address,
 896       InstructionData->Immediate,
 897       InstructionData->ImmediateSize
 898       );
 899
 900     Status = ValidateMmioMemory (Ghcb, Address, Bytes);
 901     if (Status != 0) {
 902       return Status;
 903     }
 904
 905     ExitInfo1 = Address;
 906     ExitInfo2 = Bytes;
 907
 908     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 909     VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 910     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 911     if (Status != 0) {
 912       return Status;
 913     }
 914
 915     if (Bytes == 4) {
 916       //
 917       // Zero-extend for 32-bit operation
 918       //
 919       Regs->Rax = 0;
 920     }
 921     CopyMem (&Regs->Rax, Ghcb->SharedBuffer, Bytes);
 922     break;
 923
 924   //
 925   // MMIO read w/ zero-extension ((MOVZX regX, reg/memX)
 926   //
 927   case 0xB6:
 928     Bytes = 1;
 929     //
 930     // fall through
 931     //
 932   case 0xB7:
 933     DecodeModRm (Regs, InstructionData);
 934     Bytes = (Bytes != 0) ? Bytes : 2;
 935
 936     Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 937     if (Status != 0) {
 938       return Status;
 939     }
 940
 941     ExitInfo1 = InstructionData->Ext.RmData;
 942     ExitInfo2 = Bytes;
 943
 944     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 945     VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 946     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 947     if (Status != 0) {
 948       return Status;
 949     }
 950
 951     Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 952     SetMem (Register, (UINTN) (1 << InstructionData->DataSize), 0);
 953     CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 954     break;
 955
 956   //
 957   // MMIO read w/ sign-extension (MOVSX regX, reg/memX)
 958   //
 959   case 0xBE:
 960     Bytes = 1;
 961     //
 962     // fall through
 963     //
 964   case 0xBF:
 965     DecodeModRm (Regs, InstructionData);
 966     Bytes = (Bytes != 0) ? Bytes : 2;
 967
 968     Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 969     if (Status != 0) {
 970       return Status;
 971     }
 972
 973     ExitInfo1 = InstructionData->Ext.RmData;
 974     ExitInfo2 = Bytes;
 975
 976     Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
 977     VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 978     Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 979     if (Status != 0) {
 980       return Status;
 981     }
 982
 983     if (Bytes == 1) {
 984       UINT8 *Data;
 985
 986       Data = (UINT8 *) Ghcb->SharedBuffer;
 987       SignByte = ((*Data & BIT7) != 0) ? 0xFF : 0x00;
 988     } else {
 989       UINT16 *Data;
 990
 991       Data = (UINT16 *) Ghcb->SharedBuffer;
 992       SignByte = ((*Data & BIT15) != 0) ? 0xFF : 0x00;
 993     }
 994
 995     Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 996     SetMem (Register, (UINTN) (1 << InstructionData->DataSize), SignByte);
 997     CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 998     break;
 999
1000   default:
1001     DEBUG ((DEBUG_ERROR, "Invalid MMIO opcode (%x)\n", OpCode));
1002     Status = GP_EXCEPTION;
1003     ASSERT (FALSE);
1004   }
1005
1006   return Status;
1007 }
1008
1009 /**
1010   Handle a MWAIT event.
1011
1012   Use the VMGEXIT instruction to handle a MWAIT event.
1013
1014   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1015                                    Block
1016   @param[in, out] Regs             x64 processor context
1017   @param[in]      InstructionData  Instruction parsing context
1018
1019   @retval 0                        Event handled successfully
1020   @return                          New exception value to propagate
1021
1022 **/
1023 STATIC
1024 UINT64
1025 MwaitExit (
1026   IN OUT GHCB                     *Ghcb,
1027   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1028   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1029   )
1030 {
1031   DecodeModRm (Regs, InstructionData);
1032
1033   Ghcb->SaveArea.Rax = Regs->Rax;
1034   VmgSetOffsetValid (Ghcb, GhcbRax);
1035   Ghcb->SaveArea.Rcx = Regs->Rcx;
1036   VmgSetOffsetValid (Ghcb, GhcbRcx);
1037
1038   return VmgExit (Ghcb, SVM_EXIT_MWAIT, 0, 0);
1039 }
1040
1041 /**
1042   Handle a MONITOR event.
1043
1044   Use the VMGEXIT instruction to handle a MONITOR event.
1045
1046   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1047                                    Block
1048   @param[in, out] Regs             x64 processor context
1049   @param[in]      InstructionData  Instruction parsing context
1050
1051   @retval 0                        Event handled successfully
1052   @return                          New exception value to propagate
1053
1054 **/
1055 STATIC
1056 UINT64
1057 MonitorExit (
1058   IN OUT GHCB                     *Ghcb,
1059   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1060   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1061   )
1062 {
1063   DecodeModRm (Regs, InstructionData);
1064
1065   Ghcb->SaveArea.Rax = Regs->Rax;  // Identity mapped, so VA = PA
1066   VmgSetOffsetValid (Ghcb, GhcbRax);
1067   Ghcb->SaveArea.Rcx = Regs->Rcx;
1068   VmgSetOffsetValid (Ghcb, GhcbRcx);
1069   Ghcb->SaveArea.Rdx = Regs->Rdx;
1070   VmgSetOffsetValid (Ghcb, GhcbRdx);
1071
1072   return VmgExit (Ghcb, SVM_EXIT_MONITOR, 0, 0);
1073 }
1074
1075 /**
1076   Handle a WBINVD event.
1077
1078   Use the VMGEXIT instruction to handle a WBINVD event.
1079
1080   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1081                                    Block
1082   @param[in, out] Regs             x64 processor context
1083   @param[in]      InstructionData  Instruction parsing context
1084
1085   @retval 0                        Event handled successfully
1086   @return                          New exception value to propagate
1087
1088 **/
1089 STATIC
1090 UINT64
1091 WbinvdExit (
1092   IN OUT GHCB                     *Ghcb,
1093   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1094   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1095   )
1096 {
1097   return VmgExit (Ghcb, SVM_EXIT_WBINVD, 0, 0);
1098 }
1099
1100 /**
1101   Handle a RDTSCP event.
1102
1103   Use the VMGEXIT instruction to handle a RDTSCP event.
1104
1105   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1106                                    Block
1107   @param[in, out] Regs             x64 processor context
1108   @param[in]      InstructionData  Instruction parsing context
1109
1110   @retval 0                        Event handled successfully
1111   @return                          New exception value to propagate
1112
1113 **/
1114 STATIC
1115 UINT64
1116 RdtscpExit (
1117   IN OUT GHCB                     *Ghcb,
1118   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1119   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1120   )
1121 {
1122   UINT64  Status;
1123
1124   DecodeModRm (Regs, InstructionData);
1125
1126   Status = VmgExit (Ghcb, SVM_EXIT_RDTSCP, 0, 0);
1127   if (Status != 0) {
1128     return Status;
1129   }
1130
1131   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1132       !VmgIsOffsetValid (Ghcb, GhcbRcx) ||
1133       !VmgIsOffsetValid (Ghcb, GhcbRdx)) {
1134     return UnsupportedExit (Ghcb, Regs, InstructionData);
1135   }
1136   Regs->Rax = Ghcb->SaveArea.Rax;
1137   Regs->Rcx = Ghcb->SaveArea.Rcx;
1138   Regs->Rdx = Ghcb->SaveArea.Rdx;
1139
1140   return 0;
1141 }
1142
1143 /**
1144   Handle a VMMCALL event.
1145
1146   Use the VMGEXIT instruction to handle a VMMCALL event.
1147
1148   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1149                                    Block
1150   @param[in, out] Regs             x64 processor context
1151   @param[in]      InstructionData  Instruction parsing context
1152
1153   @retval 0                        Event handled successfully
1154   @return                          New exception value to propagate
1155
1156 **/
1157 STATIC
1158 UINT64
1159 VmmCallExit (
1160   IN OUT GHCB                     *Ghcb,
1161   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1162   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1163   )
1164 {
1165   UINT64  Status;
1166
1167   DecodeModRm (Regs, InstructionData);
1168
1169   Ghcb->SaveArea.Rax = Regs->Rax;
1170   VmgSetOffsetValid (Ghcb, GhcbRax);
1171   Ghcb->SaveArea.Cpl = (UINT8) (Regs->Cs & 0x3);
1172   VmgSetOffsetValid (Ghcb, GhcbCpl);
1173
1174   Status = VmgExit (Ghcb, SVM_EXIT_VMMCALL, 0, 0);
1175   if (Status != 0) {
1176     return Status;
1177   }
1178
1179   if (!VmgIsOffsetValid (Ghcb, GhcbRax)) {
1180     return UnsupportedExit (Ghcb, Regs, InstructionData);
1181   }
1182   Regs->Rax = Ghcb->SaveArea.Rax;
1183
1184   return 0;
1185 }
1186
1187 /**
1188   Handle an MSR event.
1189
1190   Use the VMGEXIT instruction to handle either a RDMSR or WRMSR event.
1191
1192   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1193                                    Block
1194   @param[in, out] Regs             x64 processor context
1195   @param[in]      InstructionData  Instruction parsing context
1196
1197   @retval 0                        Event handled successfully
1198   @return                          New exception value to propagate
1199
1200 **/
1201 STATIC
1202 UINT64
1203 MsrExit (
1204   IN OUT GHCB                     *Ghcb,
1205   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1206   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1207   )
1208 {
1209   UINT64  ExitInfo1, Status;
1210
1211   ExitInfo1 = 0;
1212
1213   switch (*(InstructionData->OpCodes + 1)) {
1214   case 0x30: // WRMSR
1215     ExitInfo1 = 1;
1216     Ghcb->SaveArea.Rax = Regs->Rax;
1217     VmgSetOffsetValid (Ghcb, GhcbRax);
1218     Ghcb->SaveArea.Rdx = Regs->Rdx;
1219     VmgSetOffsetValid (Ghcb, GhcbRdx);
1220     //
1221     // fall through
1222     //
1223   case 0x32: // RDMSR
1224     Ghcb->SaveArea.Rcx = Regs->Rcx;
1225     VmgSetOffsetValid (Ghcb, GhcbRcx);
1226     break;
1227   default:
1228     return UnsupportedExit (Ghcb, Regs, InstructionData);
1229   }
1230
1231   Status = VmgExit (Ghcb, SVM_EXIT_MSR, ExitInfo1, 0);
1232   if (Status != 0) {
1233     return Status;
1234   }
1235
1236   if (ExitInfo1 == 0) {
1237     if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1238         !VmgIsOffsetValid (Ghcb, GhcbRdx)) {
1239       return UnsupportedExit (Ghcb, Regs, InstructionData);
1240     }
1241     Regs->Rax = Ghcb->SaveArea.Rax;
1242     Regs->Rdx = Ghcb->SaveArea.Rdx;
1243   }
1244
1245   return 0;
1246 }
1247
1248 /**
1249   Build the IOIO event information.
1250
1251   The IOIO event information identifies the type of IO operation to be performed
1252   by the hypervisor. Build this information based on the instruction data.
1253
1254   @param[in]       Regs             x64 processor context
1255   @param[in, out]  InstructionData  Instruction parsing context
1256
1257   @return                           IOIO event information value
1258
1259 **/
1260 STATIC
1261 UINT64
1262 IoioExitInfo (
1263   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
1264   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
1265   )
1266 {
1267   UINT64  ExitInfo;
1268
1269   ExitInfo = 0;
1270
1271   switch (*(InstructionData->OpCodes)) {
1272   //
1273   // INS opcodes
1274   //
1275   case 0x6C:
1276   case 0x6D:
1277     ExitInfo |= IOIO_TYPE_INS;
1278     ExitInfo |= IOIO_SEG_ES;
1279     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1280     break;
1281
1282   //
1283   // OUTS opcodes
1284   //
1285   case 0x6E:
1286   case 0x6F:
1287     ExitInfo |= IOIO_TYPE_OUTS;
1288     ExitInfo |= IOIO_SEG_DS;
1289     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1290     break;
1291
1292   //
1293   // IN immediate opcodes
1294   //
1295   case 0xE4:
1296   case 0xE5:
1297     InstructionData->ImmediateSize = 1;
1298     InstructionData->End++;
1299     ExitInfo |= IOIO_TYPE_IN;
1300     ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16);
1301     break;
1302
1303   //
1304   // OUT immediate opcodes
1305   //
1306   case 0xE6:
1307   case 0xE7:
1308     InstructionData->ImmediateSize = 1;
1309     InstructionData->End++;
1310     ExitInfo |= IOIO_TYPE_OUT;
1311     ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16) | IOIO_TYPE_OUT;
1312     break;
1313
1314   //
1315   // IN register opcodes
1316   //
1317   case 0xEC:
1318   case 0xED:
1319     ExitInfo |= IOIO_TYPE_IN;
1320     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1321     break;
1322
1323   //
1324   // OUT register opcodes
1325   //
1326   case 0xEE:
1327   case 0xEF:
1328     ExitInfo |= IOIO_TYPE_OUT;
1329     ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1330     break;
1331
1332   default:
1333     return 0;
1334   }
1335
1336   switch (*(InstructionData->OpCodes)) {
1337   //
1338   // Single-byte opcodes
1339   //
1340   case 0x6C:
1341   case 0x6E:
1342   case 0xE4:
1343   case 0xE6:
1344   case 0xEC:
1345   case 0xEE:
1346     ExitInfo |= IOIO_DATA_8;
1347     break;
1348
1349   //
1350   // Length determined by instruction parsing
1351   //
1352   default:
1353     ExitInfo |= (InstructionData->DataSize == Size16Bits) ? IOIO_DATA_16
1354                                                           : IOIO_DATA_32;
1355   }
1356
1357   switch (InstructionData->AddrSize) {
1358   case Size16Bits:
1359     ExitInfo |= IOIO_ADDR_16;
1360     break;
1361
1362   case Size32Bits:
1363     ExitInfo |= IOIO_ADDR_32;
1364     break;
1365
1366   case Size64Bits:
1367     ExitInfo |= IOIO_ADDR_64;
1368     break;
1369
1370   default:
1371     break;
1372   }
1373
1374   if (InstructionData->RepMode != 0) {
1375     ExitInfo |= IOIO_REP;
1376   }
1377
1378   return ExitInfo;
1379 }
1380
1381 /**
1382   Handle an IOIO event.
1383
1384   Use the VMGEXIT instruction to handle an IOIO event.
1385
1386   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1387                                    Block
1388   @param[in, out] Regs             x64 processor context
1389   @param[in]      InstructionData  Instruction parsing context
1390
1391   @retval 0                        Event handled successfully
1392   @return                          New exception value to propagate
1393
1394 **/
1395 STATIC
1396 UINT64
1397 IoioExit (
1398   IN OUT GHCB                     *Ghcb,
1399   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1400   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1401   )
1402 {
1403   UINT64   ExitInfo1, ExitInfo2, Status;
1404   BOOLEAN  IsString;
1405
1406   ExitInfo1 = IoioExitInfo (Regs, InstructionData);
1407   if (ExitInfo1 == 0) {
1408     return UnsupportedExit (Ghcb, Regs, InstructionData);
1409   }
1410
1411   IsString = ((ExitInfo1 & IOIO_TYPE_STR) != 0) ? TRUE : FALSE;
1412   if (IsString) {
1413     UINTN  IoBytes, VmgExitBytes;
1414     UINTN  GhcbCount, OpCount;
1415
1416     Status = 0;
1417
1418     IoBytes = IOIO_DATA_BYTES (ExitInfo1);
1419     GhcbCount = sizeof (Ghcb->SharedBuffer) / IoBytes;
1420
1421     OpCount = ((ExitInfo1 & IOIO_REP) != 0) ? Regs->Rcx : 1;
1422     while (OpCount != 0) {
1423       ExitInfo2 = MIN (OpCount, GhcbCount);
1424       VmgExitBytes = ExitInfo2 * IoBytes;
1425
1426       if ((ExitInfo1 & IOIO_TYPE_IN) == 0) {
1427         CopyMem (Ghcb->SharedBuffer, (VOID *) Regs->Rsi, VmgExitBytes);
1428         Regs->Rsi += VmgExitBytes;
1429       }
1430
1431       Ghcb->SaveArea.SwScratch = (UINT64) Ghcb->SharedBuffer;
1432       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
1433       Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, ExitInfo2);
1434       if (Status != 0) {
1435         return Status;
1436       }
1437
1438       if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1439         CopyMem ((VOID *) Regs->Rdi, Ghcb->SharedBuffer, VmgExitBytes);
1440         Regs->Rdi += VmgExitBytes;
1441       }
1442
1443       if ((ExitInfo1 & IOIO_REP) != 0) {
1444         Regs->Rcx -= ExitInfo2;
1445       }
1446
1447       OpCount -= ExitInfo2;
1448     }
1449   } else {
1450     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1451       Ghcb->SaveArea.Rax = 0;
1452     } else {
1453       CopyMem (&Ghcb->SaveArea.Rax, &Regs->Rax, IOIO_DATA_BYTES (ExitInfo1));
1454     }
1455     VmgSetOffsetValid (Ghcb, GhcbRax);
1456
1457     Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, 0);
1458     if (Status != 0) {
1459       return Status;
1460     }
1461
1462     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1463       if (!VmgIsOffsetValid (Ghcb, GhcbRax)) {
1464         return UnsupportedExit (Ghcb, Regs, InstructionData);
1465       }
1466       CopyMem (&Regs->Rax, &Ghcb->SaveArea.Rax, IOIO_DATA_BYTES (ExitInfo1));
1467     }
1468   }
1469
1470   return 0;
1471 }
1472
1473 /**
1474   Handle a INVD event.
1475
1476   Use the VMGEXIT instruction to handle a INVD event.
1477
1478   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1479                                    Block
1480   @param[in, out] Regs             x64 processor context
1481   @param[in]      InstructionData  Instruction parsing context
1482
1483   @retval 0                        Event handled successfully
1484   @return                          New exception value to propagate
1485
1486 **/
1487 STATIC
1488 UINT64
1489 InvdExit (
1490   IN OUT GHCB                     *Ghcb,
1491   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1492   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1493   )
1494 {
1495   return VmgExit (Ghcb, SVM_EXIT_INVD, 0, 0);
1496 }
1497
1498 /**
1499   Handle a CPUID event.
1500
1501   Use the VMGEXIT instruction to handle a CPUID event.
1502
1503   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1504                                    Block
1505   @param[in, out] Regs             x64 processor context
1506   @param[in]      InstructionData  Instruction parsing context
1507
1508   @retval 0                        Event handled successfully
1509   @return                          New exception value to propagate
1510
1511 **/
1512 STATIC
1513 UINT64
1514 CpuidExit (
1515   IN OUT GHCB                     *Ghcb,
1516   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1517   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1518   )
1519 {
1520   UINT64  Status;
1521
1522   Ghcb->SaveArea.Rax = Regs->Rax;
1523   VmgSetOffsetValid (Ghcb, GhcbRax);
1524   Ghcb->SaveArea.Rcx = Regs->Rcx;
1525   VmgSetOffsetValid (Ghcb, GhcbRcx);
1526   if (Regs->Rax == CPUID_EXTENDED_STATE) {
1527     IA32_CR4  Cr4;
1528
1529     Cr4.UintN = AsmReadCr4 ();
1530     Ghcb->SaveArea.XCr0 = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
1531     VmgSetOffsetValid (Ghcb, GhcbXCr0);
1532   }
1533
1534   Status = VmgExit (Ghcb, SVM_EXIT_CPUID, 0, 0);
1535   if (Status != 0) {
1536     return Status;
1537   }
1538
1539   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1540       !VmgIsOffsetValid (Ghcb, GhcbRbx) ||
1541       !VmgIsOffsetValid (Ghcb, GhcbRcx) ||
1542       !VmgIsOffsetValid (Ghcb, GhcbRdx)) {
1543     return UnsupportedExit (Ghcb, Regs, InstructionData);
1544   }
1545   Regs->Rax = Ghcb->SaveArea.Rax;
1546   Regs->Rbx = Ghcb->SaveArea.Rbx;
1547   Regs->Rcx = Ghcb->SaveArea.Rcx;
1548   Regs->Rdx = Ghcb->SaveArea.Rdx;
1549
1550   return 0;
1551 }
1552
1553 /**
1554   Handle a RDPMC event.
1555
1556   Use the VMGEXIT instruction to handle a RDPMC event.
1557
1558   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1559                                    Block
1560   @param[in, out] Regs             x64 processor context
1561   @param[in]      InstructionData  Instruction parsing context
1562
1563   @retval 0                        Event handled successfully
1564   @return                          New exception value to propagate
1565
1566 **/
1567 STATIC
1568 UINT64
1569 RdpmcExit (
1570   IN OUT GHCB                     *Ghcb,
1571   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1572   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1573   )
1574 {
1575   UINT64  Status;
1576
1577   Ghcb->SaveArea.Rcx = Regs->Rcx;
1578   VmgSetOffsetValid (Ghcb, GhcbRcx);
1579
1580   Status = VmgExit (Ghcb, SVM_EXIT_RDPMC, 0, 0);
1581   if (Status != 0) {
1582     return Status;
1583   }
1584
1585   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1586       !VmgIsOffsetValid (Ghcb, GhcbRdx)) {
1587     return UnsupportedExit (Ghcb, Regs, InstructionData);
1588   }
1589   Regs->Rax = Ghcb->SaveArea.Rax;
1590   Regs->Rdx = Ghcb->SaveArea.Rdx;
1591
1592   return 0;
1593 }
1594
1595 /**
1596   Handle a RDTSC event.
1597
1598   Use the VMGEXIT instruction to handle a RDTSC event.
1599
1600   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1601                                    Block
1602   @param[in, out] Regs             x64 processor context
1603   @param[in]      InstructionData  Instruction parsing context
1604
1605   @retval 0                        Event handled successfully
1606   @return                          New exception value to propagate
1607
1608 **/
1609 STATIC
1610 UINT64
1611 RdtscExit (
1612   IN OUT GHCB                     *Ghcb,
1613   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1614   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1615   )
1616 {
1617   UINT64  Status;
1618
1619   Status = VmgExit (Ghcb, SVM_EXIT_RDTSC, 0, 0);
1620   if (Status != 0) {
1621     return Status;
1622   }
1623
1624   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1625       !VmgIsOffsetValid (Ghcb, GhcbRdx)) {
1626     return UnsupportedExit (Ghcb, Regs, InstructionData);
1627   }
1628   Regs->Rax = Ghcb->SaveArea.Rax;
1629   Regs->Rdx = Ghcb->SaveArea.Rdx;
1630
1631   return 0;
1632 }
1633
1634 /**
1635   Handle a DR7 register write event.
1636
1637   Use the VMGEXIT instruction to handle a DR7 write event.
1638
1639   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1640                                    Block
1641   @param[in, out] Regs             x64 processor context
1642   @param[in]      InstructionData  Instruction parsing context
1643
1644   @retval 0                        Event handled successfully
1645   @return                          New exception value to propagate
1646
1647 **/
1648 STATIC
1649 UINT64
1650 Dr7WriteExit (
1651   IN OUT GHCB                     *Ghcb,
1652   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1653   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1654   )
1655 {
1656   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
1657   SEV_ES_PER_CPU_DATA            *SevEsData;
1658   UINT64                         *Register;
1659   UINT64                         Status;
1660
1661   Ext = &InstructionData->Ext;
1662   SevEsData = (SEV_ES_PER_CPU_DATA *) (Ghcb + 1);
1663
1664   DecodeModRm (Regs, InstructionData);
1665
1666   //
1667   // MOV DRn always treats MOD == 3 no matter how encoded
1668   //
1669   Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
1670
1671   //
1672   // Using a value of 0 for ExitInfo1 means RAX holds the value
1673   //
1674   Ghcb->SaveArea.Rax = *Register;
1675   VmgSetOffsetValid (Ghcb, GhcbRax);
1676
1677   Status = VmgExit (Ghcb, SVM_EXIT_DR7_WRITE, 0, 0);
1678   if (Status != 0) {
1679     return Status;
1680   }
1681
1682   SevEsData->Dr7 = *Register;
1683   SevEsData->Dr7Cached = 1;
1684
1685   return 0;
1686 }
1687
1688 /**
1689   Handle a DR7 register read event.
1690
1691   Use the VMGEXIT instruction to handle a DR7 read event.
1692
1693   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1694                                    Block
1695   @param[in, out] Regs             x64 processor context
1696   @param[in]      InstructionData  Instruction parsing context
1697
1698   @retval 0                        Event handled successfully
1699
1700 **/
1701 STATIC
1702 UINT64
1703 Dr7ReadExit (
1704   IN OUT GHCB                     *Ghcb,
1705   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1706   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1707   )
1708 {
1709   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
1710   SEV_ES_PER_CPU_DATA            *SevEsData;
1711   UINT64                         *Register;
1712
1713   Ext = &InstructionData->Ext;
1714   SevEsData = (SEV_ES_PER_CPU_DATA *) (Ghcb + 1);
1715
1716   DecodeModRm (Regs, InstructionData);
1717
1718   //
1719   // MOV DRn always treats MOD == 3 no matter how encoded
1720   //
1721   Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
1722
1723   //
1724   // If there is a cached valued for DR7, return that. Otherwise return the
1725   // DR7 standard reset value of 0x400 (no debug breakpoints set).
1726   //
1727   *Register = (SevEsData->Dr7Cached == 1) ? SevEsData->Dr7 : 0x400;
1728
1729   return 0;
1730 }
1731
1732 /**
1733   Handle a #VC exception.
1734
1735   Performs the necessary processing to handle a #VC exception.
1736
1737   @param[in, out]  Ghcb           Pointer to the GHCB
1738   @param[in, out]  ExceptionType  Pointer to an EFI_EXCEPTION_TYPE to be set
1739                                   as value to use on error.
1740   @param[in, out]  SystemContext  Pointer to EFI_SYSTEM_CONTEXT
1741
1742   @retval  EFI_SUCCESS            Exception handled
1743   @retval  EFI_UNSUPPORTED        #VC not supported, (new) exception value to
1744                                   propagate provided
1745   @retval  EFI_PROTOCOL_ERROR     #VC handling failed, (new) exception value to
1746                                   propagate provided
1747
1748 **/
1749 EFI_STATUS
1750 EFIAPI
1751 InternalVmgExitHandleVc (
1752   IN OUT GHCB                *Ghcb,
1753   IN OUT EFI_EXCEPTION_TYPE  *ExceptionType,
1754   IN OUT EFI_SYSTEM_CONTEXT  SystemContext
1755   )
1756 {
1757   EFI_SYSTEM_CONTEXT_X64    *Regs;
1758   NAE_EXIT                  NaeExit;
1759   SEV_ES_INSTRUCTION_DATA   InstructionData;
1760   UINT64                    ExitCode, Status;
1761   EFI_STATUS                VcRet;
1762   BOOLEAN                   InterruptState;
1763
1764   VcRet = EFI_SUCCESS;
1765
1766   Regs = SystemContext.SystemContextX64;
1767
1768   VmgInit (Ghcb, &InterruptState);
1769
1770   ExitCode = Regs->ExceptionData;
1771   switch (ExitCode) {
1772   case SVM_EXIT_DR7_READ:
1773     NaeExit = Dr7ReadExit;
1774     break;
1775
1776   case SVM_EXIT_DR7_WRITE:
1777     NaeExit = Dr7WriteExit;
1778     break;
1779
1780   case SVM_EXIT_RDTSC:
1781     NaeExit = RdtscExit;
1782     break;
1783
1784   case SVM_EXIT_RDPMC:
1785     NaeExit = RdpmcExit;
1786     break;
1787
1788   case SVM_EXIT_CPUID:
1789     NaeExit = CpuidExit;
1790     break;
1791
1792   case SVM_EXIT_INVD:
1793     NaeExit = InvdExit;
1794     break;
1795
1796   case SVM_EXIT_IOIO_PROT:
1797     NaeExit = IoioExit;
1798     break;
1799
1800   case SVM_EXIT_MSR:
1801     NaeExit = MsrExit;
1802     break;
1803
1804   case SVM_EXIT_VMMCALL:
1805     NaeExit = VmmCallExit;
1806     break;
1807
1808   case SVM_EXIT_RDTSCP:
1809     NaeExit = RdtscpExit;
1810     break;
1811
1812   case SVM_EXIT_WBINVD:
1813     NaeExit = WbinvdExit;
1814     break;
1815
1816   case SVM_EXIT_MONITOR:
1817     NaeExit = MonitorExit;
1818     break;
1819
1820   case SVM_EXIT_MWAIT:
1821     NaeExit = MwaitExit;
1822     break;
1823
1824   case SVM_EXIT_NPF:
1825     NaeExit = MmioExit;
1826     break;
1827
1828   default:
1829     NaeExit = UnsupportedExit;
1830   }
1831
1832   InitInstructionData (&InstructionData, Ghcb, Regs);
1833
1834   Status = NaeExit (Ghcb, Regs, &InstructionData);
1835   if (Status == 0) {
1836     Regs->Rip += InstructionLength (&InstructionData);
1837   } else {
1838     GHCB_EVENT_INJECTION  Event;
1839
1840     Event.Uint64 = Status;
1841     if (Event.Elements.ErrorCodeValid != 0) {
1842       Regs->ExceptionData = Event.Elements.ErrorCode;
1843     } else {
1844       Regs->ExceptionData = 0;
1845     }
1846
1847     *ExceptionType = Event.Elements.Vector;
1848
1849     VcRet = EFI_PROTOCOL_ERROR;
1850   }
1851
1852   VmgDone (Ghcb, InterruptState);
1853
1854   return VcRet;
1855 }
1856
1857 /**
1858   Routine to allow ASSERT from within #VC.
1859
1860   @param[in, out]  SevEsData  Pointer to the per-CPU data
1861
1862 **/
1863 VOID
1864 EFIAPI
1865 VmgExitIssueAssert (
1866   IN OUT SEV_ES_PER_CPU_DATA  *SevEsData
1867   )
1868 {
1869   //
1870   // Progress will be halted, so set VcCount to allow for ASSERT output
1871   // to be seen.
1872   //
1873   SevEsData->VcCount = 0;
1874
1875   ASSERT (FALSE);
1876   CpuDeadLoop ();
1877 }