OvmfPkg/Library/CcExitLib/CcExitVcHandler.c

   1 /** @file
   2   X64 #VC Exception Handler functon.
   3
   4   Copyright (C) 2020, Advanced Micro Devices, Inc. All rights reserved.<BR>
   5   SPDX-License-Identifier: BSD-2-Clause-Patent
   6
   7 **/
   8
   9 #include <Base.h>
  10 #include <Uefi.h>
  11 #include <Library/BaseMemoryLib.h>
  12 #include <Library/LocalApicLib.h>
  13 #include <Library/MemEncryptSevLib.h>
  14 #include <Library/CcExitLib.h>
  15 #include <Register/Amd/Msr.h>
  16 #include <Register/Intel/Cpuid.h>
  17 #include <IndustryStandard/InstructionParsing.h>
  18
  19 #include "CcExitVcHandler.h"
  20
  21 //
  22 // Instruction execution mode definition
  23 //
  24 typedef enum {
  25   LongMode64Bit = 0,
  26   LongModeCompat32Bit,
  27   LongModeCompat16Bit,
  28 } SEV_ES_INSTRUCTION_MODE;
  29
  30 //
  31 // Instruction size definition (for operand and address)
  32 //
  33 typedef enum {
  34   Size8Bits = 0,
  35   Size16Bits,
  36   Size32Bits,
  37   Size64Bits,
  38 } SEV_ES_INSTRUCTION_SIZE;
  39
  40 //
  41 // Intruction segment definition
  42 //
  43 typedef enum {
  44   SegmentEs = 0,
  45   SegmentCs,
  46   SegmentSs,
  47   SegmentDs,
  48   SegmentFs,
  49   SegmentGs,
  50 } SEV_ES_INSTRUCTION_SEGMENT;
  51
  52 //
  53 // Instruction rep function definition
  54 //
  55 typedef enum {
  56   RepNone = 0,
  57   RepZ,
  58   RepNZ,
  59 } SEV_ES_INSTRUCTION_REP;
  60
  61 typedef struct {
  62   UINT8    Rm;
  63   UINT8    Reg;
  64   UINT8    Mod;
  65 } SEV_ES_INSTRUCTION_MODRM_EXT;
  66
  67 typedef struct {
  68   UINT8    Base;
  69   UINT8    Index;
  70   UINT8    Scale;
  71 } SEV_ES_INSTRUCTION_SIB_EXT;
  72
  73 //
  74 // Instruction opcode definition
  75 //
  76 typedef struct {
  77   SEV_ES_INSTRUCTION_MODRM_EXT    ModRm;
  78
  79   SEV_ES_INSTRUCTION_SIB_EXT      Sib;
  80
  81   UINTN                           RegData;
  82   UINTN                           RmData;
  83 } SEV_ES_INSTRUCTION_OPCODE_EXT;
  84
  85 //
  86 // Instruction parsing context definition
  87 //
  88 typedef struct {
  89   GHCB                             *Ghcb;
  90
  91   SEV_ES_INSTRUCTION_MODE          Mode;
  92   SEV_ES_INSTRUCTION_SIZE          DataSize;
  93   SEV_ES_INSTRUCTION_SIZE          AddrSize;
  94   BOOLEAN                          SegmentSpecified;
  95   SEV_ES_INSTRUCTION_SEGMENT       Segment;
  96   SEV_ES_INSTRUCTION_REP           RepMode;
  97
  98   UINT8                            *Begin;
  99   UINT8                            *End;
 100
 101   UINT8                            *Prefixes;
 102   UINT8                            *OpCodes;
 103   UINT8                            *Displacement;
 104   UINT8                            *Immediate;
 105
 106   INSTRUCTION_REX_PREFIX           RexPrefix;
 107
 108   BOOLEAN                          ModRmPresent;
 109   INSTRUCTION_MODRM                ModRm;
 110
 111   BOOLEAN                          SibPresent;
 112   INSTRUCTION_SIB                  Sib;
 113
 114   UINTN                            PrefixSize;
 115   UINTN                            OpCodeSize;
 116   UINTN                            DisplacementSize;
 117   UINTN                            ImmediateSize;
 118
 119   SEV_ES_INSTRUCTION_OPCODE_EXT    Ext;
 120 } SEV_ES_INSTRUCTION_DATA;
 121
 122 //
 123 // Non-automatic Exit function prototype
 124 //
 125 typedef
 126 UINT64
 127 (*NAE_EXIT) (
 128   GHCB                     *Ghcb,
 129   EFI_SYSTEM_CONTEXT_X64   *Regs,
 130   SEV_ES_INSTRUCTION_DATA  *InstructionData
 131   );
 132
 133 //
 134 // SEV-SNP Cpuid table entry/function
 135 //
 136 typedef PACKED struct {
 137   UINT32    EaxIn;
 138   UINT32    EcxIn;
 139   UINT64    Unused;
 140   UINT64    Unused2;
 141   UINT32    Eax;
 142   UINT32    Ebx;
 143   UINT32    Ecx;
 144   UINT32    Edx;
 145   UINT64    Reserved;
 146 } SEV_SNP_CPUID_FUNCTION;
 147
 148 //
 149 // SEV-SNP Cpuid page format
 150 //
 151 typedef PACKED struct {
 152   UINT32                    Count;
 153   UINT32                    Reserved1;
 154   UINT64                    Reserved2;
 155   SEV_SNP_CPUID_FUNCTION    function[0];
 156 } SEV_SNP_CPUID_INFO;
 157
 158 /**
 159   Return a pointer to the contents of the specified register.
 160
 161   Based upon the input register, return a pointer to the registers contents
 162   in the x86 processor context.
 163
 164   @param[in] Regs      x64 processor context
 165   @param[in] Register  Register to obtain pointer for
 166
 167   @return              Pointer to the contents of the requested register
 168
 169 **/
 170 STATIC
 171 UINT64 *
 172 GetRegisterPointer (
 173   IN EFI_SYSTEM_CONTEXT_X64  *Regs,
 174   IN UINT8                   Register
 175   )
 176 {
 177   UINT64  *Reg;
 178
 179   switch (Register) {
 180     case 0:
 181       Reg = &Regs->Rax;
 182       break;
 183     case 1:
 184       Reg = &Regs->Rcx;
 185       break;
 186     case 2:
 187       Reg = &Regs->Rdx;
 188       break;
 189     case 3:
 190       Reg = &Regs->Rbx;
 191       break;
 192     case 4:
 193       Reg = &Regs->Rsp;
 194       break;
 195     case 5:
 196       Reg = &Regs->Rbp;
 197       break;
 198     case 6:
 199       Reg = &Regs->Rsi;
 200       break;
 201     case 7:
 202       Reg = &Regs->Rdi;
 203       break;
 204     case 8:
 205       Reg = &Regs->R8;
 206       break;
 207     case 9:
 208       Reg = &Regs->R9;
 209       break;
 210     case 10:
 211       Reg = &Regs->R10;
 212       break;
 213     case 11:
 214       Reg = &Regs->R11;
 215       break;
 216     case 12:
 217       Reg = &Regs->R12;
 218       break;
 219     case 13:
 220       Reg = &Regs->R13;
 221       break;
 222     case 14:
 223       Reg = &Regs->R14;
 224       break;
 225     case 15:
 226       Reg = &Regs->R15;
 227       break;
 228     default:
 229       Reg = NULL;
 230   }
 231
 232   ASSERT (Reg != NULL);
 233
 234   return Reg;
 235 }
 236
 237 /**
 238   Update the instruction parsing context for displacement bytes.
 239
 240   @param[in, out] InstructionData  Instruction parsing context
 241   @param[in]      Size             The instruction displacement size
 242
 243 **/
 244 STATIC
 245 VOID
 246 UpdateForDisplacement (
 247   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 248   IN     UINTN                    Size
 249   )
 250 {
 251   InstructionData->DisplacementSize = Size;
 252   InstructionData->Immediate       += Size;
 253   InstructionData->End             += Size;
 254 }
 255
 256 /**
 257   Determine if an instruction address if RIP relative.
 258
 259   Examine the instruction parsing context to determine if the address offset
 260   is relative to the instruction pointer.
 261
 262   @param[in] InstructionData  Instruction parsing context
 263
 264   @retval TRUE                Instruction addressing is RIP relative
 265   @retval FALSE               Instruction addressing is not RIP relative
 266
 267 **/
 268 STATIC
 269 BOOLEAN
 270 IsRipRelative (
 271   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 272   )
 273 {
 274   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 275
 276   Ext = &InstructionData->Ext;
 277
 278   return ((InstructionData->Mode == LongMode64Bit) &&
 279           (Ext->ModRm.Mod == 0) &&
 280           (Ext->ModRm.Rm == 5)  &&
 281           (InstructionData->SibPresent == FALSE));
 282 }
 283
 284 /**
 285   Return the effective address of a memory operand.
 286
 287   Examine the instruction parsing context to obtain the effective memory
 288   address of a memory operand.
 289
 290   @param[in] Regs             x64 processor context
 291   @param[in] InstructionData  Instruction parsing context
 292
 293   @return                     The memory operand effective address
 294
 295 **/
 296 STATIC
 297 UINT64
 298 GetEffectiveMemoryAddress (
 299   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 300   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 301   )
 302 {
 303   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 304   UINT64                         EffectiveAddress;
 305
 306   Ext              = &InstructionData->Ext;
 307   EffectiveAddress = 0;
 308
 309   if (IsRipRelative (InstructionData)) {
 310     //
 311     // RIP-relative displacement is a 32-bit signed value
 312     //
 313     INT32  RipRelative;
 314
 315     RipRelative = *(INT32 *)InstructionData->Displacement;
 316
 317     UpdateForDisplacement (InstructionData, 4);
 318
 319     //
 320     // Negative displacement is handled by standard UINT64 wrap-around.
 321     //
 322     return Regs->Rip + (UINT64)RipRelative;
 323   }
 324
 325   switch (Ext->ModRm.Mod) {
 326     case 1:
 327       UpdateForDisplacement (InstructionData, 1);
 328       EffectiveAddress += (UINT64)(*(INT8 *)(InstructionData->Displacement));
 329       break;
 330     case 2:
 331       switch (InstructionData->AddrSize) {
 332         case Size16Bits:
 333           UpdateForDisplacement (InstructionData, 2);
 334           EffectiveAddress += (UINT64)(*(INT16 *)(InstructionData->Displacement));
 335           break;
 336         default:
 337           UpdateForDisplacement (InstructionData, 4);
 338           EffectiveAddress += (UINT64)(*(INT32 *)(InstructionData->Displacement));
 339           break;
 340       }
 341
 342       break;
 343   }
 344
 345   if (InstructionData->SibPresent) {
 346     INT64  Displacement;
 347
 348     if (Ext->Sib.Index != 4) {
 349       CopyMem (
 350         &Displacement,
 351         GetRegisterPointer (Regs, Ext->Sib.Index),
 352         sizeof (Displacement)
 353         );
 354       Displacement *= (INT64)(1 << Ext->Sib.Scale);
 355
 356       //
 357       // Negative displacement is handled by standard UINT64 wrap-around.
 358       //
 359       EffectiveAddress += (UINT64)Displacement;
 360     }
 361
 362     if ((Ext->Sib.Base != 5) || Ext->ModRm.Mod) {
 363       EffectiveAddress += *GetRegisterPointer (Regs, Ext->Sib.Base);
 364     } else {
 365       UpdateForDisplacement (InstructionData, 4);
 366       EffectiveAddress += (UINT64)(*(INT32 *)(InstructionData->Displacement));
 367     }
 368   } else {
 369     EffectiveAddress += *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 370   }
 371
 372   return EffectiveAddress;
 373 }
 374
 375 /**
 376   Decode a ModRM byte.
 377
 378   Examine the instruction parsing context to decode a ModRM byte and the SIB
 379   byte, if present.
 380
 381   @param[in]      Regs             x64 processor context
 382   @param[in, out] InstructionData  Instruction parsing context
 383
 384 **/
 385 STATIC
 386 VOID
 387 DecodeModRm (
 388   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 389   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 390   )
 391 {
 392   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 393   INSTRUCTION_REX_PREFIX         *RexPrefix;
 394   INSTRUCTION_MODRM              *ModRm;
 395   INSTRUCTION_SIB                *Sib;
 396
 397   RexPrefix = &InstructionData->RexPrefix;
 398   Ext       = &InstructionData->Ext;
 399   ModRm     = &InstructionData->ModRm;
 400   Sib       = &InstructionData->Sib;
 401
 402   InstructionData->ModRmPresent = TRUE;
 403   ModRm->Uint8                  = *(InstructionData->End);
 404
 405   InstructionData->Displacement++;
 406   InstructionData->Immediate++;
 407   InstructionData->End++;
 408
 409   Ext->ModRm.Mod = ModRm->Bits.Mod;
 410   Ext->ModRm.Reg = (RexPrefix->Bits.BitR << 3) | ModRm->Bits.Reg;
 411   Ext->ModRm.Rm  = (RexPrefix->Bits.BitB << 3) | ModRm->Bits.Rm;
 412
 413   Ext->RegData = *GetRegisterPointer (Regs, Ext->ModRm.Reg);
 414
 415   if (Ext->ModRm.Mod == 3) {
 416     Ext->RmData = *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 417   } else {
 418     if (ModRm->Bits.Rm == 4) {
 419       InstructionData->SibPresent = TRUE;
 420       Sib->Uint8                  = *(InstructionData->End);
 421
 422       InstructionData->Displacement++;
 423       InstructionData->Immediate++;
 424       InstructionData->End++;
 425
 426       Ext->Sib.Scale = Sib->Bits.Scale;
 427       Ext->Sib.Index = (RexPrefix->Bits.BitX << 3) | Sib->Bits.Index;
 428       Ext->Sib.Base  = (RexPrefix->Bits.BitB << 3) | Sib->Bits.Base;
 429     }
 430
 431     Ext->RmData = GetEffectiveMemoryAddress (Regs, InstructionData);
 432   }
 433 }
 434
 435 /**
 436   Decode instruction prefixes.
 437
 438   Parse the instruction data to track the instruction prefixes that have
 439   been used.
 440
 441   @param[in]      Regs             x64 processor context
 442   @param[in, out] InstructionData  Instruction parsing context
 443
 444 **/
 445 STATIC
 446 VOID
 447 DecodePrefixes (
 448   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 449   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 450   )
 451 {
 452   SEV_ES_INSTRUCTION_MODE  Mode;
 453   SEV_ES_INSTRUCTION_SIZE  ModeDataSize;
 454   SEV_ES_INSTRUCTION_SIZE  ModeAddrSize;
 455   UINT8                    *Byte;
 456
 457   //
 458   // Always in 64-bit mode
 459   //
 460   Mode         = LongMode64Bit;
 461   ModeDataSize = Size32Bits;
 462   ModeAddrSize = Size64Bits;
 463
 464   InstructionData->Mode     = Mode;
 465   InstructionData->DataSize = ModeDataSize;
 466   InstructionData->AddrSize = ModeAddrSize;
 467
 468   InstructionData->Prefixes = InstructionData->Begin;
 469
 470   Byte = InstructionData->Prefixes;
 471   for ( ; ; Byte++, InstructionData->PrefixSize++) {
 472     //
 473     // Check the 0x40 to 0x4F range using an if statement here since some
 474     // compilers don't like the "case 0x40 ... 0x4F:" syntax. This avoids
 475     // 16 case statements below.
 476     //
 477     if ((*Byte >= REX_PREFIX_START) && (*Byte <= REX_PREFIX_STOP)) {
 478       InstructionData->RexPrefix.Uint8 = *Byte;
 479       if ((*Byte & REX_64BIT_OPERAND_SIZE_MASK) != 0) {
 480         InstructionData->DataSize = Size64Bits;
 481       }
 482
 483       continue;
 484     }
 485
 486     switch (*Byte) {
 487       case OVERRIDE_SEGMENT_CS:
 488       case OVERRIDE_SEGMENT_DS:
 489       case OVERRIDE_SEGMENT_ES:
 490       case OVERRIDE_SEGMENT_SS:
 491         if (Mode != LongMode64Bit) {
 492           InstructionData->SegmentSpecified = TRUE;
 493           InstructionData->Segment          = (*Byte >> 3) & 3;
 494         }
 495
 496         break;
 497
 498       case OVERRIDE_SEGMENT_FS:
 499       case OVERRIDE_SEGMENT_GS:
 500         InstructionData->SegmentSpecified = TRUE;
 501         InstructionData->Segment          = *Byte & 7;
 502         break;
 503
 504       case OVERRIDE_OPERAND_SIZE:
 505         if (InstructionData->RexPrefix.Uint8 == 0) {
 506           InstructionData->DataSize =
 507             (Mode == LongMode64Bit)       ? Size16Bits :
 508             (Mode == LongModeCompat32Bit) ? Size16Bits :
 509             (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 510         }
 511
 512         break;
 513
 514       case OVERRIDE_ADDRESS_SIZE:
 515         InstructionData->AddrSize =
 516           (Mode == LongMode64Bit)       ? Size32Bits :
 517           (Mode == LongModeCompat32Bit) ? Size16Bits :
 518           (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 519         break;
 520
 521       case LOCK_PREFIX:
 522         break;
 523
 524       case REPZ_PREFIX:
 525         InstructionData->RepMode = RepZ;
 526         break;
 527
 528       case REPNZ_PREFIX:
 529         InstructionData->RepMode = RepNZ;
 530         break;
 531
 532       default:
 533         InstructionData->OpCodes    = Byte;
 534         InstructionData->OpCodeSize = (*Byte == TWO_BYTE_OPCODE_ESCAPE) ? 2 : 1;
 535
 536         InstructionData->End          = Byte + InstructionData->OpCodeSize;
 537         InstructionData->Displacement = InstructionData->End;
 538         InstructionData->Immediate    = InstructionData->End;
 539         return;
 540     }
 541   }
 542 }
 543
 544 /**
 545   Determine instruction length
 546
 547   Return the total length of the parsed instruction.
 548
 549   @param[in] InstructionData  Instruction parsing context
 550
 551   @return                     Length of parsed instruction
 552
 553 **/
 554 STATIC
 555 UINT64
 556 InstructionLength (
 557   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 558   )
 559 {
 560   return (UINT64)(InstructionData->End - InstructionData->Begin);
 561 }
 562
 563 /**
 564   Initialize the instruction parsing context.
 565
 566   Initialize the instruction parsing context, which includes decoding the
 567   instruction prefixes.
 568
 569   @param[in, out] InstructionData  Instruction parsing context
 570   @param[in]      Ghcb             Pointer to the Guest-Hypervisor Communication
 571                                    Block
 572   @param[in]      Regs             x64 processor context
 573
 574 **/
 575 STATIC
 576 VOID
 577 InitInstructionData (
 578   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 579   IN     GHCB                     *Ghcb,
 580   IN     EFI_SYSTEM_CONTEXT_X64   *Regs
 581   )
 582 {
 583   SetMem (InstructionData, sizeof (*InstructionData), 0);
 584   InstructionData->Ghcb  = Ghcb;
 585   InstructionData->Begin = (UINT8 *)Regs->Rip;
 586   InstructionData->End   = (UINT8 *)Regs->Rip;
 587
 588   DecodePrefixes (Regs, InstructionData);
 589 }
 590
 591 /**
 592   Report an unsupported event to the hypervisor
 593
 594   Use the VMGEXIT support to report an unsupported event to the hypervisor.
 595
 596   @param[in] Ghcb             Pointer to the Guest-Hypervisor Communication
 597                               Block
 598   @param[in] Regs             x64 processor context
 599   @param[in] InstructionData  Instruction parsing context
 600
 601   @return                     New exception value to propagate
 602
 603 **/
 604 STATIC
 605 UINT64
 606 UnsupportedExit (
 607   IN GHCB                     *Ghcb,
 608   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 609   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 610   )
 611 {
 612   UINT64  Status;
 613
 614   Status = CcExitVmgExit (Ghcb, SVM_EXIT_UNSUPPORTED, Regs->ExceptionData, 0);
 615   if (Status == 0) {
 616     GHCB_EVENT_INJECTION  Event;
 617
 618     Event.Uint64          = 0;
 619     Event.Elements.Vector = GP_EXCEPTION;
 620     Event.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
 621     Event.Elements.Valid  = 1;
 622
 623     Status = Event.Uint64;
 624   }
 625
 626   return Status;
 627 }
 628
 629 /**
 630   Validate that the MMIO memory access is not to encrypted memory.
 631
 632   Examine the pagetable entry for the memory specified. MMIO should not be
 633   performed against encrypted memory. MMIO to the APIC page is always allowed.
 634
 635   @param[in] Ghcb           Pointer to the Guest-Hypervisor Communication Block
 636   @param[in] MemoryAddress  Memory address to validate
 637   @param[in] MemoryLength   Memory length to validate
 638
 639   @retval 0          Memory is not encrypted
 640   @return            New exception value to propogate
 641
 642 **/
 643 STATIC
 644 UINT64
 645 ValidateMmioMemory (
 646   IN GHCB   *Ghcb,
 647   IN UINTN  MemoryAddress,
 648   IN UINTN  MemoryLength
 649   )
 650 {
 651   MEM_ENCRYPT_SEV_ADDRESS_RANGE_STATE  State;
 652   GHCB_EVENT_INJECTION                 GpEvent;
 653   UINTN                                Address;
 654
 655   //
 656   // Allow APIC accesses (which will have the encryption bit set during
 657   // SEC and PEI phases).
 658   //
 659   Address = MemoryAddress & ~(SIZE_4KB - 1);
 660   if (Address == GetLocalApicBaseAddress ()) {
 661     return 0;
 662   }
 663
 664   State = MemEncryptSevGetAddressRangeState (
 665             0,
 666             MemoryAddress,
 667             MemoryLength
 668             );
 669   if (State == MemEncryptSevAddressRangeUnencrypted) {
 670     return 0;
 671   }
 672
 673   //
 674   // Any state other than unencrypted is an error, issue a #GP.
 675   //
 676   DEBUG ((
 677     DEBUG_ERROR,
 678     "MMIO using encrypted memory: %lx\n",
 679     (UINT64)MemoryAddress
 680     ));
 681   GpEvent.Uint64          = 0;
 682   GpEvent.Elements.Vector = GP_EXCEPTION;
 683   GpEvent.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
 684   GpEvent.Elements.Valid  = 1;
 685
 686   return GpEvent.Uint64;
 687 }
 688
 689 /**
 690   Handle an MMIO event.
 691
 692   Use the VMGEXIT instruction to handle either an MMIO read or an MMIO write.
 693
 694   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 695                                    Block
 696   @param[in, out] Regs             x64 processor context
 697   @param[in, out] InstructionData  Instruction parsing context
 698
 699   @retval 0                        Event handled successfully
 700   @return                          New exception value to propagate
 701
 702 **/
 703 STATIC
 704 UINT64
 705 MmioExit (
 706   IN OUT GHCB                     *Ghcb,
 707   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 708   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 709   )
 710 {
 711   UINT64  ExitInfo1, ExitInfo2, Status;
 712   UINTN   Bytes;
 713   UINT64  *Register;
 714   UINT8   OpCode, SignByte;
 715   UINTN   Address;
 716
 717   Bytes = 0;
 718
 719   OpCode = *(InstructionData->OpCodes);
 720   if (OpCode == TWO_BYTE_OPCODE_ESCAPE) {
 721     OpCode = *(InstructionData->OpCodes + 1);
 722   }
 723
 724   switch (OpCode) {
 725     //
 726     // MMIO write (MOV reg/memX, regX)
 727     //
 728     case 0x88:
 729       Bytes = 1;
 730     //
 731     // fall through
 732     //
 733     case 0x89:
 734       DecodeModRm (Regs, InstructionData);
 735       Bytes = ((Bytes != 0) ? Bytes :
 736                (InstructionData->DataSize == Size16Bits) ? 2 :
 737                (InstructionData->DataSize == Size32Bits) ? 4 :
 738                (InstructionData->DataSize == Size64Bits) ? 8 :
 739                0);
 740
 741       if (InstructionData->Ext.ModRm.Mod == 3) {
 742         //
 743         // NPF on two register operands???
 744         //
 745         return UnsupportedExit (Ghcb, Regs, InstructionData);
 746       }
 747
 748       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 749       if (Status != 0) {
 750         return Status;
 751       }
 752
 753       ExitInfo1 = InstructionData->Ext.RmData;
 754       ExitInfo2 = Bytes;
 755       CopyMem (Ghcb->SharedBuffer, &InstructionData->Ext.RegData, Bytes);
 756
 757       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 758       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 759       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 760       if (Status != 0) {
 761         return Status;
 762       }
 763
 764       break;
 765
 766     //
 767     // MMIO write (MOV moffsetX, aX)
 768     //
 769     case 0xA2:
 770       Bytes = 1;
 771     //
 772     // fall through
 773     //
 774     case 0xA3:
 775       Bytes = ((Bytes != 0) ? Bytes :
 776                (InstructionData->DataSize == Size16Bits) ? 2 :
 777                (InstructionData->DataSize == Size32Bits) ? 4 :
 778                (InstructionData->DataSize == Size64Bits) ? 8 :
 779                0);
 780
 781       InstructionData->ImmediateSize = (UINTN)(1 << InstructionData->AddrSize);
 782       InstructionData->End          += InstructionData->ImmediateSize;
 783
 784       //
 785       // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
 786       // Use a STATIC_ASSERT to be certain the code is being built as X64.
 787       //
 788       STATIC_ASSERT (
 789         sizeof (UINTN) == sizeof (UINT64),
 790         "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
 791         );
 792
 793       Address = 0;
 794       CopyMem (
 795         &Address,
 796         InstructionData->Immediate,
 797         InstructionData->ImmediateSize
 798         );
 799
 800       Status = ValidateMmioMemory (Ghcb, Address, Bytes);
 801       if (Status != 0) {
 802         return Status;
 803       }
 804
 805       ExitInfo1 = Address;
 806       ExitInfo2 = Bytes;
 807       CopyMem (Ghcb->SharedBuffer, &Regs->Rax, Bytes);
 808
 809       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 810       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 811       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 812       if (Status != 0) {
 813         return Status;
 814       }
 815
 816       break;
 817
 818     //
 819     // MMIO write (MOV reg/memX, immX)
 820     //
 821     case 0xC6:
 822       Bytes = 1;
 823     //
 824     // fall through
 825     //
 826     case 0xC7:
 827       DecodeModRm (Regs, InstructionData);
 828       Bytes = ((Bytes != 0) ? Bytes :
 829                (InstructionData->DataSize == Size16Bits) ? 2 :
 830                (InstructionData->DataSize == Size32Bits) ? 4 :
 831                0);
 832
 833       InstructionData->ImmediateSize = Bytes;
 834       InstructionData->End          += Bytes;
 835
 836       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 837       if (Status != 0) {
 838         return Status;
 839       }
 840
 841       ExitInfo1 = InstructionData->Ext.RmData;
 842       ExitInfo2 = Bytes;
 843       CopyMem (Ghcb->SharedBuffer, InstructionData->Immediate, Bytes);
 844
 845       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 846       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 847       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 848       if (Status != 0) {
 849         return Status;
 850       }
 851
 852       break;
 853
 854     //
 855     // MMIO read (MOV regX, reg/memX)
 856     //
 857     case 0x8A:
 858       Bytes = 1;
 859     //
 860     // fall through
 861     //
 862     case 0x8B:
 863       DecodeModRm (Regs, InstructionData);
 864       Bytes = ((Bytes != 0) ? Bytes :
 865                (InstructionData->DataSize == Size16Bits) ? 2 :
 866                (InstructionData->DataSize == Size32Bits) ? 4 :
 867                (InstructionData->DataSize == Size64Bits) ? 8 :
 868                0);
 869       if (InstructionData->Ext.ModRm.Mod == 3) {
 870         //
 871         // NPF on two register operands???
 872         //
 873         return UnsupportedExit (Ghcb, Regs, InstructionData);
 874       }
 875
 876       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 877       if (Status != 0) {
 878         return Status;
 879       }
 880
 881       ExitInfo1 = InstructionData->Ext.RmData;
 882       ExitInfo2 = Bytes;
 883
 884       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 885       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 886       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 887       if (Status != 0) {
 888         return Status;
 889       }
 890
 891       Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 892       if (Bytes == 4) {
 893         //
 894         // Zero-extend for 32-bit operation
 895         //
 896         *Register = 0;
 897       }
 898
 899       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 900       break;
 901
 902     //
 903     // MMIO read (MOV aX, moffsetX)
 904     //
 905     case 0xA0:
 906       Bytes = 1;
 907     //
 908     // fall through
 909     //
 910     case 0xA1:
 911       Bytes = ((Bytes != 0) ? Bytes :
 912                (InstructionData->DataSize == Size16Bits) ? 2 :
 913                (InstructionData->DataSize == Size32Bits) ? 4 :
 914                (InstructionData->DataSize == Size64Bits) ? 8 :
 915                0);
 916
 917       InstructionData->ImmediateSize = (UINTN)(1 << InstructionData->AddrSize);
 918       InstructionData->End          += InstructionData->ImmediateSize;
 919
 920       //
 921       // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
 922       // Use a STATIC_ASSERT to be certain the code is being built as X64.
 923       //
 924       STATIC_ASSERT (
 925         sizeof (UINTN) == sizeof (UINT64),
 926         "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
 927         );
 928
 929       Address = 0;
 930       CopyMem (
 931         &Address,
 932         InstructionData->Immediate,
 933         InstructionData->ImmediateSize
 934         );
 935
 936       Status = ValidateMmioMemory (Ghcb, Address, Bytes);
 937       if (Status != 0) {
 938         return Status;
 939       }
 940
 941       ExitInfo1 = Address;
 942       ExitInfo2 = Bytes;
 943
 944       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 945       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 946       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 947       if (Status != 0) {
 948         return Status;
 949       }
 950
 951       if (Bytes == 4) {
 952         //
 953         // Zero-extend for 32-bit operation
 954         //
 955         Regs->Rax = 0;
 956       }
 957
 958       CopyMem (&Regs->Rax, Ghcb->SharedBuffer, Bytes);
 959       break;
 960
 961     //
 962     // MMIO read w/ zero-extension ((MOVZX regX, reg/memX)
 963     //
 964     case 0xB6:
 965       Bytes = 1;
 966     //
 967     // fall through
 968     //
 969     case 0xB7:
 970       DecodeModRm (Regs, InstructionData);
 971       Bytes = (Bytes != 0) ? Bytes : 2;
 972
 973       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 974       if (Status != 0) {
 975         return Status;
 976       }
 977
 978       ExitInfo1 = InstructionData->Ext.RmData;
 979       ExitInfo2 = Bytes;
 980
 981       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 982       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
 983       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 984       if (Status != 0) {
 985         return Status;
 986       }
 987
 988       Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 989       SetMem (Register, (UINTN)(1 << InstructionData->DataSize), 0);
 990       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 991       break;
 992
 993     //
 994     // MMIO read w/ sign-extension (MOVSX regX, reg/memX)
 995     //
 996     case 0xBE:
 997       Bytes = 1;
 998     //
 999     // fall through
1000     //
1001     case 0xBF:
1002       DecodeModRm (Regs, InstructionData);
1003       Bytes = (Bytes != 0) ? Bytes : 2;
1004
1005       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
1006       if (Status != 0) {
1007         return Status;
1008       }
1009
1010       ExitInfo1 = InstructionData->Ext.RmData;
1011       ExitInfo2 = Bytes;
1012
1013       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
1014       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
1015       Status = CcExitVmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
1016       if (Status != 0) {
1017         return Status;
1018       }
1019
1020       if (Bytes == 1) {
1021         UINT8  *Data;
1022
1023         Data     = (UINT8 *)Ghcb->SharedBuffer;
1024         SignByte = ((*Data & BIT7) != 0) ? 0xFF : 0x00;
1025       } else {
1026         UINT16  *Data;
1027
1028         Data     = (UINT16 *)Ghcb->SharedBuffer;
1029         SignByte = ((*Data & BIT15) != 0) ? 0xFF : 0x00;
1030       }
1031
1032       Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
1033       SetMem (Register, (UINTN)(1 << InstructionData->DataSize), SignByte);
1034       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
1035       break;
1036
1037     default:
1038       DEBUG ((DEBUG_ERROR, "Invalid MMIO opcode (%x)\n", OpCode));
1039       Status = GP_EXCEPTION;
1040       ASSERT (FALSE);
1041   }
1042
1043   return Status;
1044 }
1045
1046 /**
1047   Handle a MWAIT event.
1048
1049   Use the VMGEXIT instruction to handle a MWAIT event.
1050
1051   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1052                                    Block
1053   @param[in, out] Regs             x64 processor context
1054   @param[in]      InstructionData  Instruction parsing context
1055
1056   @retval 0                        Event handled successfully
1057   @return                          New exception value to propagate
1058
1059 **/
1060 STATIC
1061 UINT64
1062 MwaitExit (
1063   IN OUT GHCB                     *Ghcb,
1064   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1065   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1066   )
1067 {
1068   DecodeModRm (Regs, InstructionData);
1069
1070   Ghcb->SaveArea.Rax = Regs->Rax;
1071   CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
1072   Ghcb->SaveArea.Rcx = Regs->Rcx;
1073   CcExitVmgSetOffsetValid (Ghcb, GhcbRcx);
1074
1075   return CcExitVmgExit (Ghcb, SVM_EXIT_MWAIT, 0, 0);
1076 }
1077
1078 /**
1079   Handle a MONITOR event.
1080
1081   Use the VMGEXIT instruction to handle a MONITOR event.
1082
1083   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1084                                    Block
1085   @param[in, out] Regs             x64 processor context
1086   @param[in]      InstructionData  Instruction parsing context
1087
1088   @retval 0                        Event handled successfully
1089   @return                          New exception value to propagate
1090
1091 **/
1092 STATIC
1093 UINT64
1094 MonitorExit (
1095   IN OUT GHCB                     *Ghcb,
1096   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1097   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1098   )
1099 {
1100   DecodeModRm (Regs, InstructionData);
1101
1102   Ghcb->SaveArea.Rax = Regs->Rax;  // Identity mapped, so VA = PA
1103   CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
1104   Ghcb->SaveArea.Rcx = Regs->Rcx;
1105   CcExitVmgSetOffsetValid (Ghcb, GhcbRcx);
1106   Ghcb->SaveArea.Rdx = Regs->Rdx;
1107   CcExitVmgSetOffsetValid (Ghcb, GhcbRdx);
1108
1109   return CcExitVmgExit (Ghcb, SVM_EXIT_MONITOR, 0, 0);
1110 }
1111
1112 /**
1113   Handle a WBINVD event.
1114
1115   Use the VMGEXIT instruction to handle a WBINVD event.
1116
1117   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1118                                    Block
1119   @param[in, out] Regs             x64 processor context
1120   @param[in]      InstructionData  Instruction parsing context
1121
1122   @retval 0                        Event handled successfully
1123   @return                          New exception value to propagate
1124
1125 **/
1126 STATIC
1127 UINT64
1128 WbinvdExit (
1129   IN OUT GHCB                     *Ghcb,
1130   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1131   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1132   )
1133 {
1134   return CcExitVmgExit (Ghcb, SVM_EXIT_WBINVD, 0, 0);
1135 }
1136
1137 /**
1138   Handle a RDTSCP event.
1139
1140   Use the VMGEXIT instruction to handle a RDTSCP event.
1141
1142   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1143                                    Block
1144   @param[in, out] Regs             x64 processor context
1145   @param[in]      InstructionData  Instruction parsing context
1146
1147   @retval 0                        Event handled successfully
1148   @return                          New exception value to propagate
1149
1150 **/
1151 STATIC
1152 UINT64
1153 RdtscpExit (
1154   IN OUT GHCB                     *Ghcb,
1155   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1156   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1157   )
1158 {
1159   UINT64  Status;
1160
1161   DecodeModRm (Regs, InstructionData);
1162
1163   Status = CcExitVmgExit (Ghcb, SVM_EXIT_RDTSCP, 0, 0);
1164   if (Status != 0) {
1165     return Status;
1166   }
1167
1168   if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax) ||
1169       !CcExitVmgIsOffsetValid (Ghcb, GhcbRcx) ||
1170       !CcExitVmgIsOffsetValid (Ghcb, GhcbRdx))
1171   {
1172     return UnsupportedExit (Ghcb, Regs, InstructionData);
1173   }
1174
1175   Regs->Rax = Ghcb->SaveArea.Rax;
1176   Regs->Rcx = Ghcb->SaveArea.Rcx;
1177   Regs->Rdx = Ghcb->SaveArea.Rdx;
1178
1179   return 0;
1180 }
1181
1182 /**
1183   Handle a VMMCALL event.
1184
1185   Use the VMGEXIT instruction to handle a VMMCALL event.
1186
1187   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1188                                    Block
1189   @param[in, out] Regs             x64 processor context
1190   @param[in]      InstructionData  Instruction parsing context
1191
1192   @retval 0                        Event handled successfully
1193   @return                          New exception value to propagate
1194
1195 **/
1196 STATIC
1197 UINT64
1198 VmmCallExit (
1199   IN OUT GHCB                     *Ghcb,
1200   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1201   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1202   )
1203 {
1204   UINT64  Status;
1205
1206   DecodeModRm (Regs, InstructionData);
1207
1208   Ghcb->SaveArea.Rax = Regs->Rax;
1209   CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
1210   Ghcb->SaveArea.Cpl = (UINT8)(Regs->Cs & 0x3);
1211   CcExitVmgSetOffsetValid (Ghcb, GhcbCpl);
1212
1213   Status = CcExitVmgExit (Ghcb, SVM_EXIT_VMMCALL, 0, 0);
1214   if (Status != 0) {
1215     return Status;
1216   }
1217
1218   if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax)) {
1219     return UnsupportedExit (Ghcb, Regs, InstructionData);
1220   }
1221
1222   Regs->Rax = Ghcb->SaveArea.Rax;
1223
1224   return 0;
1225 }
1226
1227 /**
1228   Handle an MSR event.
1229
1230   Use the VMGEXIT instruction to handle either a RDMSR or WRMSR event.
1231
1232   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1233                                    Block
1234   @param[in, out] Regs             x64 processor context
1235   @param[in]      InstructionData  Instruction parsing context
1236
1237   @retval 0                        Event handled successfully
1238   @return                          New exception value to propagate
1239
1240 **/
1241 STATIC
1242 UINT64
1243 MsrExit (
1244   IN OUT GHCB                     *Ghcb,
1245   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1246   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1247   )
1248 {
1249   UINT64  ExitInfo1, Status;
1250
1251   ExitInfo1 = 0;
1252
1253   switch (*(InstructionData->OpCodes + 1)) {
1254     case 0x30: // WRMSR
1255       ExitInfo1          = 1;
1256       Ghcb->SaveArea.Rax = Regs->Rax;
1257       CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
1258       Ghcb->SaveArea.Rdx = Regs->Rdx;
1259       CcExitVmgSetOffsetValid (Ghcb, GhcbRdx);
1260     //
1261     // fall through
1262     //
1263     case 0x32: // RDMSR
1264       Ghcb->SaveArea.Rcx = Regs->Rcx;
1265       CcExitVmgSetOffsetValid (Ghcb, GhcbRcx);
1266       break;
1267     default:
1268       return UnsupportedExit (Ghcb, Regs, InstructionData);
1269   }
1270
1271   Status = CcExitVmgExit (Ghcb, SVM_EXIT_MSR, ExitInfo1, 0);
1272   if (Status != 0) {
1273     return Status;
1274   }
1275
1276   if (ExitInfo1 == 0) {
1277     if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax) ||
1278         !CcExitVmgIsOffsetValid (Ghcb, GhcbRdx))
1279     {
1280       return UnsupportedExit (Ghcb, Regs, InstructionData);
1281     }
1282
1283     Regs->Rax = Ghcb->SaveArea.Rax;
1284     Regs->Rdx = Ghcb->SaveArea.Rdx;
1285   }
1286
1287   return 0;
1288 }
1289
1290 /**
1291   Build the IOIO event information.
1292
1293   The IOIO event information identifies the type of IO operation to be performed
1294   by the hypervisor. Build this information based on the instruction data.
1295
1296   @param[in]       Regs             x64 processor context
1297   @param[in, out]  InstructionData  Instruction parsing context
1298
1299   @return                           IOIO event information value
1300
1301 **/
1302 STATIC
1303 UINT64
1304 IoioExitInfo (
1305   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
1306   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
1307   )
1308 {
1309   UINT64  ExitInfo;
1310
1311   ExitInfo = 0;
1312
1313   switch (*(InstructionData->OpCodes)) {
1314     //
1315     // INS opcodes
1316     //
1317     case 0x6C:
1318     case 0x6D:
1319       ExitInfo |= IOIO_TYPE_INS;
1320       ExitInfo |= IOIO_SEG_ES;
1321       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1322       break;
1323
1324     //
1325     // OUTS opcodes
1326     //
1327     case 0x6E:
1328     case 0x6F:
1329       ExitInfo |= IOIO_TYPE_OUTS;
1330       ExitInfo |= IOIO_SEG_DS;
1331       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1332       break;
1333
1334     //
1335     // IN immediate opcodes
1336     //
1337     case 0xE4:
1338     case 0xE5:
1339       InstructionData->ImmediateSize = 1;
1340       InstructionData->End++;
1341       ExitInfo |= IOIO_TYPE_IN;
1342       ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16);
1343       break;
1344
1345     //
1346     // OUT immediate opcodes
1347     //
1348     case 0xE6:
1349     case 0xE7:
1350       InstructionData->ImmediateSize = 1;
1351       InstructionData->End++;
1352       ExitInfo |= IOIO_TYPE_OUT;
1353       ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16) | IOIO_TYPE_OUT;
1354       break;
1355
1356     //
1357     // IN register opcodes
1358     //
1359     case 0xEC:
1360     case 0xED:
1361       ExitInfo |= IOIO_TYPE_IN;
1362       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1363       break;
1364
1365     //
1366     // OUT register opcodes
1367     //
1368     case 0xEE:
1369     case 0xEF:
1370       ExitInfo |= IOIO_TYPE_OUT;
1371       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1372       break;
1373
1374     default:
1375       return 0;
1376   }
1377
1378   switch (*(InstructionData->OpCodes)) {
1379     //
1380     // Single-byte opcodes
1381     //
1382     case 0x6C:
1383     case 0x6E:
1384     case 0xE4:
1385     case 0xE6:
1386     case 0xEC:
1387     case 0xEE:
1388       ExitInfo |= IOIO_DATA_8;
1389       break;
1390
1391     //
1392     // Length determined by instruction parsing
1393     //
1394     default:
1395       ExitInfo |= (InstructionData->DataSize == Size16Bits) ? IOIO_DATA_16
1396                                                           : IOIO_DATA_32;
1397   }
1398
1399   switch (InstructionData->AddrSize) {
1400     case Size16Bits:
1401       ExitInfo |= IOIO_ADDR_16;
1402       break;
1403
1404     case Size32Bits:
1405       ExitInfo |= IOIO_ADDR_32;
1406       break;
1407
1408     case Size64Bits:
1409       ExitInfo |= IOIO_ADDR_64;
1410       break;
1411
1412     default:
1413       break;
1414   }
1415
1416   if (InstructionData->RepMode != 0) {
1417     ExitInfo |= IOIO_REP;
1418   }
1419
1420   return ExitInfo;
1421 }
1422
1423 /**
1424   Handle an IOIO event.
1425
1426   Use the VMGEXIT instruction to handle an IOIO event.
1427
1428   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1429                                    Block
1430   @param[in, out] Regs             x64 processor context
1431   @param[in]      InstructionData  Instruction parsing context
1432
1433   @retval 0                        Event handled successfully
1434   @return                          New exception value to propagate
1435
1436 **/
1437 STATIC
1438 UINT64
1439 IoioExit (
1440   IN OUT GHCB                     *Ghcb,
1441   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1442   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1443   )
1444 {
1445   UINT64   ExitInfo1, ExitInfo2, Status;
1446   BOOLEAN  IsString;
1447
1448   ExitInfo1 = IoioExitInfo (Regs, InstructionData);
1449   if (ExitInfo1 == 0) {
1450     return UnsupportedExit (Ghcb, Regs, InstructionData);
1451   }
1452
1453   IsString = ((ExitInfo1 & IOIO_TYPE_STR) != 0) ? TRUE : FALSE;
1454   if (IsString) {
1455     UINTN  IoBytes, VmgExitBytes;
1456     UINTN  GhcbCount, OpCount;
1457
1458     Status = 0;
1459
1460     IoBytes   = IOIO_DATA_BYTES (ExitInfo1);
1461     GhcbCount = sizeof (Ghcb->SharedBuffer) / IoBytes;
1462
1463     OpCount = ((ExitInfo1 & IOIO_REP) != 0) ? Regs->Rcx : 1;
1464     while (OpCount != 0) {
1465       ExitInfo2    = MIN (OpCount, GhcbCount);
1466       VmgExitBytes = ExitInfo2 * IoBytes;
1467
1468       if ((ExitInfo1 & IOIO_TYPE_IN) == 0) {
1469         CopyMem (Ghcb->SharedBuffer, (VOID *)Regs->Rsi, VmgExitBytes);
1470         Regs->Rsi += VmgExitBytes;
1471       }
1472
1473       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
1474       CcExitVmgSetOffsetValid (Ghcb, GhcbSwScratch);
1475       Status = CcExitVmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, ExitInfo2);
1476       if (Status != 0) {
1477         return Status;
1478       }
1479
1480       if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1481         CopyMem ((VOID *)Regs->Rdi, Ghcb->SharedBuffer, VmgExitBytes);
1482         Regs->Rdi += VmgExitBytes;
1483       }
1484
1485       if ((ExitInfo1 & IOIO_REP) != 0) {
1486         Regs->Rcx -= ExitInfo2;
1487       }
1488
1489       OpCount -= ExitInfo2;
1490     }
1491   } else {
1492     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1493       Ghcb->SaveArea.Rax = 0;
1494     } else {
1495       CopyMem (&Ghcb->SaveArea.Rax, &Regs->Rax, IOIO_DATA_BYTES (ExitInfo1));
1496     }
1497
1498     CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
1499
1500     Status = CcExitVmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, 0);
1501     if (Status != 0) {
1502       return Status;
1503     }
1504
1505     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1506       if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax)) {
1507         return UnsupportedExit (Ghcb, Regs, InstructionData);
1508       }
1509
1510       CopyMem (&Regs->Rax, &Ghcb->SaveArea.Rax, IOIO_DATA_BYTES (ExitInfo1));
1511     }
1512   }
1513
1514   return 0;
1515 }
1516
1517 /**
1518   Handle a INVD event.
1519
1520   Use the VMGEXIT instruction to handle a INVD event.
1521
1522   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1523                                    Block
1524   @param[in, out] Regs             x64 processor context
1525   @param[in]      InstructionData  Instruction parsing context
1526
1527   @retval 0                        Event handled successfully
1528   @return                          New exception value to propagate
1529
1530 **/
1531 STATIC
1532 UINT64
1533 InvdExit (
1534   IN OUT GHCB                     *Ghcb,
1535   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1536   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1537   )
1538 {
1539   return CcExitVmgExit (Ghcb, SVM_EXIT_INVD, 0, 0);
1540 }
1541
1542 /**
1543   Fetch CPUID leaf/function via hypervisor/VMGEXIT.
1544
1545   @param[in, out] Ghcb         Pointer to the Guest-Hypervisor Communication
1546                                Block
1547   @param[in]      EaxIn        EAX input for cpuid instruction
1548   @param[in]      EcxIn        ECX input for cpuid instruction
1549   @param[in]      Xcr0In       XCR0 at time of cpuid instruction
1550   @param[in, out] Eax          Pointer to store leaf's EAX value
1551   @param[in, out] Ebx          Pointer to store leaf's EBX value
1552   @param[in, out] Ecx          Pointer to store leaf's ECX value
1553   @param[in, out] Edx          Pointer to store leaf's EDX value
1554   @param[in, out] Status       Pointer to store status from VMGEXIT (always 0
1555                                unless return value indicates failure)
1556   @param[in, out] Unsupported  Pointer to store indication of unsupported
1557                                VMGEXIT (always false unless return value
1558                                indicates failure)
1559
1560   @retval TRUE                 CPUID leaf fetch successfully.
1561   @retval FALSE                Error occurred while fetching CPUID leaf. Callers
1562                                should Status and Unsupported and handle
1563                                accordingly if they indicate a more precise
1564                                error condition.
1565
1566 **/
1567 STATIC
1568 BOOLEAN
1569 GetCpuidHyp (
1570   IN OUT GHCB     *Ghcb,
1571   IN     UINT32   EaxIn,
1572   IN     UINT32   EcxIn,
1573   IN     UINT64   XCr0,
1574   IN OUT UINT32   *Eax,
1575   IN OUT UINT32   *Ebx,
1576   IN OUT UINT32   *Ecx,
1577   IN OUT UINT32   *Edx,
1578   IN OUT UINT64   *Status,
1579   IN OUT BOOLEAN  *UnsupportedExit
1580   )
1581 {
1582   *UnsupportedExit   = FALSE;
1583   Ghcb->SaveArea.Rax = EaxIn;
1584   CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
1585   Ghcb->SaveArea.Rcx = EcxIn;
1586   CcExitVmgSetOffsetValid (Ghcb, GhcbRcx);
1587   if (EaxIn == CPUID_EXTENDED_STATE) {
1588     Ghcb->SaveArea.XCr0 = XCr0;
1589     CcExitVmgSetOffsetValid (Ghcb, GhcbXCr0);
1590   }
1591
1592   *Status = CcExitVmgExit (Ghcb, SVM_EXIT_CPUID, 0, 0);
1593   if (*Status != 0) {
1594     return FALSE;
1595   }
1596
1597   if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax) ||
1598       !CcExitVmgIsOffsetValid (Ghcb, GhcbRbx) ||
1599       !CcExitVmgIsOffsetValid (Ghcb, GhcbRcx) ||
1600       !CcExitVmgIsOffsetValid (Ghcb, GhcbRdx))
1601   {
1602     *UnsupportedExit = TRUE;
1603     return FALSE;
1604   }
1605
1606   if (Eax) {
1607     *Eax = (UINT32)(UINTN)Ghcb->SaveArea.Rax;
1608   }
1609
1610   if (Ebx) {
1611     *Ebx = (UINT32)(UINTN)Ghcb->SaveArea.Rbx;
1612   }
1613
1614   if (Ecx) {
1615     *Ecx = (UINT32)(UINTN)Ghcb->SaveArea.Rcx;
1616   }
1617
1618   if (Edx) {
1619     *Edx = (UINT32)(UINTN)Ghcb->SaveArea.Rdx;
1620   }
1621
1622   return TRUE;
1623 }
1624
1625 /**
1626   Check if SEV-SNP enabled.
1627
1628   @retval TRUE      SEV-SNP is enabled.
1629   @retval FALSE     SEV-SNP is disabled.
1630
1631 **/
1632 STATIC
1633 BOOLEAN
1634 SnpEnabled (
1635   VOID
1636   )
1637 {
1638   MSR_SEV_STATUS_REGISTER  Msr;
1639
1640   Msr.Uint32 = AsmReadMsr32 (MSR_SEV_STATUS);
1641
1642   return !!Msr.Bits.SevSnpBit;
1643 }
1644
1645 /**
1646   Calculate the total XSAVE area size for enabled XSAVE areas
1647
1648   @param[in]      XFeaturesEnabled  Bit-mask of enabled XSAVE features/areas as
1649                                     indicated by XCR0/MSR_IA32_XSS bits
1650   @param[in]      XSaveBaseSize     Base/legacy XSAVE area size (e.g. when
1651                                     XCR0 is 1)
1652   @param[in, out] XSaveSize         Pointer to storage for calculated XSAVE area
1653                                     size
1654   @param[in]      Compacted         Whether or not the calculation is for the
1655                                     normal XSAVE area size (leaf 0xD,0x0,EBX) or
1656                                     compacted XSAVE area size (leaf 0xD,0x1,EBX)
1657
1658
1659   @retval TRUE                      XSAVE size calculation was successful.
1660   @retval FALSE                     XSAVE size calculation was unsuccessful.
1661 **/
1662 STATIC
1663 BOOLEAN
1664 GetCpuidXSaveSize (
1665   IN     UINT64   XFeaturesEnabled,
1666   IN     UINT32   XSaveBaseSize,
1667   IN OUT UINT32   *XSaveSize,
1668   IN     BOOLEAN  Compacted
1669   )
1670 {
1671   SEV_SNP_CPUID_INFO  *CpuidInfo;
1672   UINT64              XFeaturesFound = 0;
1673   UINT32              Idx;
1674
1675   *XSaveSize = XSaveBaseSize;
1676   CpuidInfo  = (SEV_SNP_CPUID_INFO *)(UINT64)PcdGet32 (PcdOvmfCpuidBase);
1677
1678   for (Idx = 0; Idx < CpuidInfo->Count; Idx++) {
1679     SEV_SNP_CPUID_FUNCTION  *CpuidFn = &CpuidInfo->function[Idx];
1680
1681     if (!((CpuidFn->EaxIn == 0xD) &&
1682           ((CpuidFn->EcxIn == 0) || (CpuidFn->EcxIn == 1))))
1683     {
1684       continue;
1685     }
1686
1687     if (XFeaturesFound & (1ULL << CpuidFn->EcxIn) ||
1688         !(XFeaturesEnabled & (1ULL << CpuidFn->EcxIn)))
1689     {
1690       continue;
1691     }
1692
1693     XFeaturesFound |= (1ULL << CpuidFn->EcxIn);
1694     if (Compacted) {
1695       *XSaveSize += CpuidFn->Eax;
1696     } else {
1697       *XSaveSize = MAX (*XSaveSize, CpuidFn->Eax + CpuidFn->Ebx);
1698     }
1699   }
1700
1701   /*
1702    * Either the guest set unsupported XCR0/XSS bits, or the corresponding
1703    * entries in the CPUID table were not present. This is an invalid state.
1704    */
1705   if (XFeaturesFound != (XFeaturesEnabled & ~3UL)) {
1706     return FALSE;
1707   }
1708
1709   return TRUE;
1710 }
1711
1712 /**
1713   Check if a CPUID leaf/function is indexed via ECX sub-leaf/sub-function
1714
1715   @param[in]      EaxIn        EAX input for cpuid instruction
1716
1717   @retval FALSE                cpuid leaf/function is not indexed by ECX input
1718   @retval TRUE                 cpuid leaf/function is indexed by ECX input
1719
1720 **/
1721 STATIC
1722 BOOLEAN
1723 IsFunctionIndexed (
1724   IN     UINT32  EaxIn
1725   )
1726 {
1727   switch (EaxIn) {
1728     case CPUID_CACHE_PARAMS:
1729     case CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS:
1730     case CPUID_EXTENDED_TOPOLOGY:
1731     case CPUID_EXTENDED_STATE:
1732     case CPUID_INTEL_RDT_MONITORING:
1733     case CPUID_INTEL_RDT_ALLOCATION:
1734     case CPUID_INTEL_SGX:
1735     case CPUID_INTEL_PROCESSOR_TRACE:
1736     case CPUID_DETERMINISTIC_ADDRESS_TRANSLATION_PARAMETERS:
1737     case CPUID_V2_EXTENDED_TOPOLOGY:
1738     case 0x8000001D: /* Cache Topology Information */
1739       return TRUE;
1740   }
1741
1742   return FALSE;
1743 }
1744
1745 /**
1746   Fetch CPUID leaf/function via SEV-SNP CPUID table.
1747
1748   @param[in, out] Ghcb         Pointer to the Guest-Hypervisor Communication
1749                                Block
1750   @param[in]      EaxIn        EAX input for cpuid instruction
1751   @param[in]      EcxIn        ECX input for cpuid instruction
1752   @param[in]      Xcr0In       XCR0 at time of cpuid instruction
1753   @param[in, out] Eax          Pointer to store leaf's EAX value
1754   @param[in, out] Ebx          Pointer to store leaf's EBX value
1755   @param[in, out] Ecx          Pointer to store leaf's ECX value
1756   @param[in, out] Edx          Pointer to store leaf's EDX value
1757   @param[in, out] Status       Pointer to store status from VMGEXIT (always 0
1758                                unless return value indicates failure)
1759   @param[in, out] Unsupported  Pointer to store indication of unsupported
1760                                VMGEXIT (always false unless return value
1761                                indicates failure)
1762
1763   @retval TRUE                 CPUID leaf fetch successfully.
1764   @retval FALSE                Error occurred while fetching CPUID leaf. Callers
1765                                should Status and Unsupported and handle
1766                                accordingly if they indicate a more precise
1767                                error condition.
1768
1769 **/
1770 STATIC
1771 BOOLEAN
1772 GetCpuidFw (
1773   IN OUT GHCB     *Ghcb,
1774   IN     UINT32   EaxIn,
1775   IN     UINT32   EcxIn,
1776   IN     UINT64   XCr0,
1777   IN OUT UINT32   *Eax,
1778   IN OUT UINT32   *Ebx,
1779   IN OUT UINT32   *Ecx,
1780   IN OUT UINT32   *Edx,
1781   IN OUT UINT64   *Status,
1782   IN OUT BOOLEAN  *Unsupported
1783   )
1784 {
1785   SEV_SNP_CPUID_INFO  *CpuidInfo;
1786   BOOLEAN             Found;
1787   UINT32              Idx;
1788
1789   CpuidInfo = (SEV_SNP_CPUID_INFO *)(UINT64)PcdGet32 (PcdOvmfCpuidBase);
1790   Found     = FALSE;
1791
1792   for (Idx = 0; Idx < CpuidInfo->Count; Idx++) {
1793     SEV_SNP_CPUID_FUNCTION  *CpuidFn = &CpuidInfo->function[Idx];
1794
1795     if (CpuidFn->EaxIn != EaxIn) {
1796       continue;
1797     }
1798
1799     if (IsFunctionIndexed (CpuidFn->EaxIn) && (CpuidFn->EcxIn != EcxIn)) {
1800       continue;
1801     }
1802
1803     *Eax = CpuidFn->Eax;
1804     *Ebx = CpuidFn->Ebx;
1805     *Ecx = CpuidFn->Ecx;
1806     *Edx = CpuidFn->Edx;
1807
1808     Found = TRUE;
1809     break;
1810   }
1811
1812   if (!Found) {
1813     *Eax = *Ebx = *Ecx = *Edx = 0;
1814     goto Out;
1815   }
1816
1817   if (EaxIn == CPUID_VERSION_INFO) {
1818     IA32_CR4  Cr4;
1819     UINT32    Ebx2;
1820     UINT32    Edx2;
1821
1822     if (!GetCpuidHyp (
1823            Ghcb,
1824            EaxIn,
1825            EcxIn,
1826            XCr0,
1827            NULL,
1828            &Ebx2,
1829            NULL,
1830            &Edx2,
1831            Status,
1832            Unsupported
1833            ))
1834     {
1835       return FALSE;
1836     }
1837
1838     /* initial APIC ID */
1839     *Ebx = (*Ebx & 0x00FFFFFF) | (Ebx2 & 0xFF000000);
1840     /* APIC enabled bit */
1841     *Edx = (*Edx & ~BIT9) | (Edx2 & BIT9);
1842     /* OSXSAVE enabled bit */
1843     Cr4.UintN = AsmReadCr4 ();
1844     *Ecx      = (Cr4.Bits.OSXSAVE) ? (*Ecx & ~BIT27) | (*Ecx & BIT27)
1845                               : (*Ecx & ~BIT27);
1846   } else if (EaxIn == CPUID_STRUCTURED_EXTENDED_FEATURE_FLAGS) {
1847     IA32_CR4  Cr4;
1848
1849     Cr4.UintN = AsmReadCr4 ();
1850     /* OSPKE enabled bit */
1851     *Ecx = (Cr4.Bits.PKE) ? (*Ecx | BIT4) : (*Ecx & ~BIT4);
1852   } else if (EaxIn == CPUID_EXTENDED_TOPOLOGY) {
1853     if (!GetCpuidHyp (
1854            Ghcb,
1855            EaxIn,
1856            EcxIn,
1857            XCr0,
1858            NULL,
1859            NULL,
1860            NULL,
1861            Edx,
1862            Status,
1863            Unsupported
1864            ))
1865     {
1866       return FALSE;
1867     }
1868   } else if ((EaxIn == CPUID_EXTENDED_STATE) && ((EcxIn == 0) || (EcxIn == 1))) {
1869     MSR_IA32_XSS_REGISTER  XssMsr;
1870     BOOLEAN                Compacted;
1871     UINT32                 XSaveSize;
1872
1873     XssMsr.Uint64 = 0;
1874     Compacted     = FALSE;
1875     if (EcxIn == 1) {
1876       /*
1877        * The PPR and APM aren't clear on what size should be encoded in
1878        * 0xD:0x1:EBX when compaction is not enabled by either XSAVEC or
1879        * XSAVES, as these are generally fixed to 1 on real CPUs. Report
1880        * this undefined case as an error.
1881        */
1882       if (!(*Eax & (BIT3 | BIT1))) {
1883         /* (XSAVES | XSAVEC) */
1884         return FALSE;
1885       }
1886
1887       Compacted     = TRUE;
1888       XssMsr.Uint64 = AsmReadMsr64 (MSR_IA32_XSS);
1889     }
1890
1891     if (!GetCpuidXSaveSize (
1892            XCr0 | XssMsr.Uint64,
1893            *Ebx,
1894            &XSaveSize,
1895            Compacted
1896            ))
1897     {
1898       return FALSE;
1899     }
1900
1901     *Ebx = XSaveSize;
1902   } else if (EaxIn == 0x8000001E) {
1903     UINT32  Ebx2;
1904     UINT32  Ecx2;
1905
1906     /* extended APIC ID */
1907     if (!GetCpuidHyp (
1908            Ghcb,
1909            EaxIn,
1910            EcxIn,
1911            XCr0,
1912            Eax,
1913            &Ebx2,
1914            &Ecx2,
1915            NULL,
1916            Status,
1917            Unsupported
1918            ))
1919     {
1920       return FALSE;
1921     }
1922
1923     /* compute ID */
1924     *Ebx = (*Ebx & 0xFFFFFF00) | (Ebx2 & 0x000000FF);
1925     /* node ID */
1926     *Ecx = (*Ecx & 0xFFFFFF00) | (Ecx2 & 0x000000FF);
1927   }
1928
1929 Out:
1930   *Status      = 0;
1931   *Unsupported = FALSE;
1932   return TRUE;
1933 }
1934
1935 /**
1936   Handle a CPUID event.
1937
1938   Use VMGEXIT instruction or CPUID table to handle a CPUID event.
1939
1940   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1941                                    Block
1942   @param[in, out] Regs             x64 processor context
1943   @param[in]      InstructionData  Instruction parsing context
1944
1945   @retval 0                        Event handled successfully
1946   @return                          New exception value to propagate
1947
1948 **/
1949 STATIC
1950 UINT64
1951 CpuidExit (
1952   IN OUT GHCB                     *Ghcb,
1953   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1954   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1955   )
1956 {
1957   BOOLEAN  Unsupported;
1958   UINT64   Status;
1959   UINT32   EaxIn;
1960   UINT32   EcxIn;
1961   UINT64   XCr0;
1962   UINT32   Eax;
1963   UINT32   Ebx;
1964   UINT32   Ecx;
1965   UINT32   Edx;
1966
1967   EaxIn = (UINT32)(UINTN)Regs->Rax;
1968   EcxIn = (UINT32)(UINTN)Regs->Rcx;
1969
1970   if (EaxIn == CPUID_EXTENDED_STATE) {
1971     IA32_CR4  Cr4;
1972
1973     Cr4.UintN           = AsmReadCr4 ();
1974     Ghcb->SaveArea.XCr0 = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
1975     XCr0                = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
1976   }
1977
1978   if (SnpEnabled ()) {
1979     if (!GetCpuidFw (
1980            Ghcb,
1981            EaxIn,
1982            EcxIn,
1983            XCr0,
1984            &Eax,
1985            &Ebx,
1986            &Ecx,
1987            &Edx,
1988            &Status,
1989            &Unsupported
1990            ))
1991     {
1992       goto CpuidFail;
1993     }
1994   } else {
1995     if (!GetCpuidHyp (
1996            Ghcb,
1997            EaxIn,
1998            EcxIn,
1999            XCr0,
2000            &Eax,
2001            &Ebx,
2002            &Ecx,
2003            &Edx,
2004            &Status,
2005            &Unsupported
2006            ))
2007     {
2008       goto CpuidFail;
2009     }
2010   }
2011
2012   Regs->Rax = Eax;
2013   Regs->Rbx = Ebx;
2014   Regs->Rcx = Ecx;
2015   Regs->Rdx = Edx;
2016
2017   return 0;
2018
2019 CpuidFail:
2020   if (Unsupported) {
2021     return UnsupportedExit (Ghcb, Regs, InstructionData);
2022   }
2023
2024   return Status;
2025 }
2026
2027 /**
2028   Handle a RDPMC event.
2029
2030   Use the VMGEXIT instruction to handle a RDPMC event.
2031
2032   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
2033                                    Block
2034   @param[in, out] Regs             x64 processor context
2035   @param[in]      InstructionData  Instruction parsing context
2036
2037   @retval 0                        Event handled successfully
2038   @return                          New exception value to propagate
2039
2040 **/
2041 STATIC
2042 UINT64
2043 RdpmcExit (
2044   IN OUT GHCB                     *Ghcb,
2045   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
2046   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
2047   )
2048 {
2049   UINT64  Status;
2050
2051   Ghcb->SaveArea.Rcx = Regs->Rcx;
2052   CcExitVmgSetOffsetValid (Ghcb, GhcbRcx);
2053
2054   Status = CcExitVmgExit (Ghcb, SVM_EXIT_RDPMC, 0, 0);
2055   if (Status != 0) {
2056     return Status;
2057   }
2058
2059   if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax) ||
2060       !CcExitVmgIsOffsetValid (Ghcb, GhcbRdx))
2061   {
2062     return UnsupportedExit (Ghcb, Regs, InstructionData);
2063   }
2064
2065   Regs->Rax = Ghcb->SaveArea.Rax;
2066   Regs->Rdx = Ghcb->SaveArea.Rdx;
2067
2068   return 0;
2069 }
2070
2071 /**
2072   Handle a RDTSC event.
2073
2074   Use the VMGEXIT instruction to handle a RDTSC event.
2075
2076   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
2077                                    Block
2078   @param[in, out] Regs             x64 processor context
2079   @param[in]      InstructionData  Instruction parsing context
2080
2081   @retval 0                        Event handled successfully
2082   @return                          New exception value to propagate
2083
2084 **/
2085 STATIC
2086 UINT64
2087 RdtscExit (
2088   IN OUT GHCB                     *Ghcb,
2089   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
2090   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
2091   )
2092 {
2093   UINT64  Status;
2094
2095   Status = CcExitVmgExit (Ghcb, SVM_EXIT_RDTSC, 0, 0);
2096   if (Status != 0) {
2097     return Status;
2098   }
2099
2100   if (!CcExitVmgIsOffsetValid (Ghcb, GhcbRax) ||
2101       !CcExitVmgIsOffsetValid (Ghcb, GhcbRdx))
2102   {
2103     return UnsupportedExit (Ghcb, Regs, InstructionData);
2104   }
2105
2106   Regs->Rax = Ghcb->SaveArea.Rax;
2107   Regs->Rdx = Ghcb->SaveArea.Rdx;
2108
2109   return 0;
2110 }
2111
2112 /**
2113   Handle a DR7 register write event.
2114
2115   Use the VMGEXIT instruction to handle a DR7 write event.
2116
2117   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
2118                                    Block
2119   @param[in, out] Regs             x64 processor context
2120   @param[in]      InstructionData  Instruction parsing context
2121
2122   @retval 0                        Event handled successfully
2123   @return                          New exception value to propagate
2124
2125 **/
2126 STATIC
2127 UINT64
2128 Dr7WriteExit (
2129   IN OUT GHCB                     *Ghcb,
2130   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
2131   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
2132   )
2133 {
2134   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
2135   SEV_ES_PER_CPU_DATA            *SevEsData;
2136   UINT64                         *Register;
2137   UINT64                         Status;
2138
2139   Ext       = &InstructionData->Ext;
2140   SevEsData = (SEV_ES_PER_CPU_DATA *)(Ghcb + 1);
2141
2142   DecodeModRm (Regs, InstructionData);
2143
2144   //
2145   // MOV DRn always treats MOD == 3 no matter how encoded
2146   //
2147   Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
2148
2149   //
2150   // Using a value of 0 for ExitInfo1 means RAX holds the value
2151   //
2152   Ghcb->SaveArea.Rax = *Register;
2153   CcExitVmgSetOffsetValid (Ghcb, GhcbRax);
2154
2155   Status = CcExitVmgExit (Ghcb, SVM_EXIT_DR7_WRITE, 0, 0);
2156   if (Status != 0) {
2157     return Status;
2158   }
2159
2160   SevEsData->Dr7       = *Register;
2161   SevEsData->Dr7Cached = 1;
2162
2163   return 0;
2164 }
2165
2166 /**
2167   Handle a DR7 register read event.
2168
2169   Use the VMGEXIT instruction to handle a DR7 read event.
2170
2171   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
2172                                    Block
2173   @param[in, out] Regs             x64 processor context
2174   @param[in]      InstructionData  Instruction parsing context
2175
2176   @retval 0                        Event handled successfully
2177
2178 **/
2179 STATIC
2180 UINT64
2181 Dr7ReadExit (
2182   IN OUT GHCB                     *Ghcb,
2183   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
2184   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
2185   )
2186 {
2187   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
2188   SEV_ES_PER_CPU_DATA            *SevEsData;
2189   UINT64                         *Register;
2190
2191   Ext       = &InstructionData->Ext;
2192   SevEsData = (SEV_ES_PER_CPU_DATA *)(Ghcb + 1);
2193
2194   DecodeModRm (Regs, InstructionData);
2195
2196   //
2197   // MOV DRn always treats MOD == 3 no matter how encoded
2198   //
2199   Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
2200
2201   //
2202   // If there is a cached valued for DR7, return that. Otherwise return the
2203   // DR7 standard reset value of 0x400 (no debug breakpoints set).
2204   //
2205   *Register = (SevEsData->Dr7Cached == 1) ? SevEsData->Dr7 : 0x400;
2206
2207   return 0;
2208 }
2209
2210 /**
2211   Handle a #VC exception.
2212
2213   Performs the necessary processing to handle a #VC exception.
2214
2215   @param[in, out]  Ghcb           Pointer to the GHCB
2216   @param[in, out]  ExceptionType  Pointer to an EFI_EXCEPTION_TYPE to be set
2217                                   as value to use on error.
2218   @param[in, out]  SystemContext  Pointer to EFI_SYSTEM_CONTEXT
2219
2220   @retval  EFI_SUCCESS            Exception handled
2221   @retval  EFI_UNSUPPORTED        #VC not supported, (new) exception value to
2222                                   propagate provided
2223   @retval  EFI_PROTOCOL_ERROR     #VC handling failed, (new) exception value to
2224                                   propagate provided
2225
2226 **/
2227 EFI_STATUS
2228 EFIAPI
2229 InternalVmgExitHandleVc (
2230   IN OUT GHCB                *Ghcb,
2231   IN OUT EFI_EXCEPTION_TYPE  *ExceptionType,
2232   IN OUT EFI_SYSTEM_CONTEXT  SystemContext
2233   )
2234 {
2235   EFI_SYSTEM_CONTEXT_X64   *Regs;
2236   NAE_EXIT                 NaeExit;
2237   SEV_ES_INSTRUCTION_DATA  InstructionData;
2238   UINT64                   ExitCode, Status;
2239   EFI_STATUS               VcRet;
2240   BOOLEAN                  InterruptState;
2241
2242   VcRet = EFI_SUCCESS;
2243
2244   Regs = SystemContext.SystemContextX64;
2245
2246   CcExitVmgInit (Ghcb, &InterruptState);
2247
2248   ExitCode = Regs->ExceptionData;
2249   switch (ExitCode) {
2250     case SVM_EXIT_DR7_READ:
2251       NaeExit = Dr7ReadExit;
2252       break;
2253
2254     case SVM_EXIT_DR7_WRITE:
2255       NaeExit = Dr7WriteExit;
2256       break;
2257
2258     case SVM_EXIT_RDTSC:
2259       NaeExit = RdtscExit;
2260       break;
2261
2262     case SVM_EXIT_RDPMC:
2263       NaeExit = RdpmcExit;
2264       break;
2265
2266     case SVM_EXIT_CPUID:
2267       NaeExit = CpuidExit;
2268       break;
2269
2270     case SVM_EXIT_INVD:
2271       NaeExit = InvdExit;
2272       break;
2273
2274     case SVM_EXIT_IOIO_PROT:
2275       NaeExit = IoioExit;
2276       break;
2277
2278     case SVM_EXIT_MSR:
2279       NaeExit = MsrExit;
2280       break;
2281
2282     case SVM_EXIT_VMMCALL:
2283       NaeExit = VmmCallExit;
2284       break;
2285
2286     case SVM_EXIT_RDTSCP:
2287       NaeExit = RdtscpExit;
2288       break;
2289
2290     case SVM_EXIT_WBINVD:
2291       NaeExit = WbinvdExit;
2292       break;
2293
2294     case SVM_EXIT_MONITOR:
2295       NaeExit = MonitorExit;
2296       break;
2297
2298     case SVM_EXIT_MWAIT:
2299       NaeExit = MwaitExit;
2300       break;
2301
2302     case SVM_EXIT_NPF:
2303       NaeExit = MmioExit;
2304       break;
2305
2306     default:
2307       NaeExit = UnsupportedExit;
2308   }
2309
2310   InitInstructionData (&InstructionData, Ghcb, Regs);
2311
2312   Status = NaeExit (Ghcb, Regs, &InstructionData);
2313   if (Status == 0) {
2314     Regs->Rip += InstructionLength (&InstructionData);
2315   } else {
2316     GHCB_EVENT_INJECTION  Event;
2317
2318     Event.Uint64 = Status;
2319     if (Event.Elements.ErrorCodeValid != 0) {
2320       Regs->ExceptionData = Event.Elements.ErrorCode;
2321     } else {
2322       Regs->ExceptionData = 0;
2323     }
2324
2325     *ExceptionType = Event.Elements.Vector;
2326
2327     VcRet = EFI_PROTOCOL_ERROR;
2328   }
2329
2330   CcExitVmgDone (Ghcb, InterruptState);
2331
2332   return VcRet;
2333 }
2334
2335 /**
2336   Routine to allow ASSERT from within #VC.
2337
2338   @param[in, out]  SevEsData  Pointer to the per-CPU data
2339
2340 **/
2341 VOID
2342 EFIAPI
2343 VmgExitIssueAssert (
2344   IN OUT SEV_ES_PER_CPU_DATA  *SevEsData
2345   )
2346 {
2347   //
2348   // Progress will be halted, so set VcCount to allow for ASSERT output
2349   // to be seen.
2350   //
2351   SevEsData->VcCount = 0;
2352
2353   ASSERT (FALSE);
2354   CpuDeadLoop ();
2355 }