OvmfPkg/Library/VmgExitLib/VmgExitVcHandler.c

   1 /** @file
   2   X64 #VC Exception Handler functon.
   3
   4   Copyright (C) 2020, Advanced Micro Devices, Inc. All rights reserved.<BR>
   5   SPDX-License-Identifier: BSD-2-Clause-Patent
   6
   7 **/
   8
   9 #include <Base.h>
  10 #include <Uefi.h>
  11 #include <Library/BaseMemoryLib.h>
  12 #include <Library/LocalApicLib.h>
  13 #include <Library/MemEncryptSevLib.h>
  14 #include <Library/VmgExitLib.h>
  15 #include <Register/Amd/Msr.h>
  16 #include <Register/Intel/Cpuid.h>
  17 #include <IndustryStandard/InstructionParsing.h>
  18
  19 #include "VmgExitVcHandler.h"
  20
  21 //
  22 // Instruction execution mode definition
  23 //
  24 typedef enum {
  25   LongMode64Bit = 0,
  26   LongModeCompat32Bit,
  27   LongModeCompat16Bit,
  28 } SEV_ES_INSTRUCTION_MODE;
  29
  30 //
  31 // Instruction size definition (for operand and address)
  32 //
  33 typedef enum {
  34   Size8Bits = 0,
  35   Size16Bits,
  36   Size32Bits,
  37   Size64Bits,
  38 } SEV_ES_INSTRUCTION_SIZE;
  39
  40 //
  41 // Intruction segment definition
  42 //
  43 typedef enum {
  44   SegmentEs = 0,
  45   SegmentCs,
  46   SegmentSs,
  47   SegmentDs,
  48   SegmentFs,
  49   SegmentGs,
  50 } SEV_ES_INSTRUCTION_SEGMENT;
  51
  52 //
  53 // Instruction rep function definition
  54 //
  55 typedef enum {
  56   RepNone = 0,
  57   RepZ,
  58   RepNZ,
  59 } SEV_ES_INSTRUCTION_REP;
  60
  61 typedef struct {
  62   UINT8    Rm;
  63   UINT8    Reg;
  64   UINT8    Mod;
  65 } SEV_ES_INSTRUCTION_MODRM_EXT;
  66
  67 typedef struct {
  68   UINT8    Base;
  69   UINT8    Index;
  70   UINT8    Scale;
  71 } SEV_ES_INSTRUCTION_SIB_EXT;
  72
  73 //
  74 // Instruction opcode definition
  75 //
  76 typedef struct {
  77   SEV_ES_INSTRUCTION_MODRM_EXT    ModRm;
  78
  79   SEV_ES_INSTRUCTION_SIB_EXT      Sib;
  80
  81   UINTN                           RegData;
  82   UINTN                           RmData;
  83 } SEV_ES_INSTRUCTION_OPCODE_EXT;
  84
  85 //
  86 // Instruction parsing context definition
  87 //
  88 typedef struct {
  89   GHCB                             *Ghcb;
  90
  91   SEV_ES_INSTRUCTION_MODE          Mode;
  92   SEV_ES_INSTRUCTION_SIZE          DataSize;
  93   SEV_ES_INSTRUCTION_SIZE          AddrSize;
  94   BOOLEAN                          SegmentSpecified;
  95   SEV_ES_INSTRUCTION_SEGMENT       Segment;
  96   SEV_ES_INSTRUCTION_REP           RepMode;
  97
  98   UINT8                            *Begin;
  99   UINT8                            *End;
 100
 101   UINT8                            *Prefixes;
 102   UINT8                            *OpCodes;
 103   UINT8                            *Displacement;
 104   UINT8                            *Immediate;
 105
 106   INSTRUCTION_REX_PREFIX           RexPrefix;
 107
 108   BOOLEAN                          ModRmPresent;
 109   INSTRUCTION_MODRM                ModRm;
 110
 111   BOOLEAN                          SibPresent;
 112   INSTRUCTION_SIB                  Sib;
 113
 114   UINTN                            PrefixSize;
 115   UINTN                            OpCodeSize;
 116   UINTN                            DisplacementSize;
 117   UINTN                            ImmediateSize;
 118
 119   SEV_ES_INSTRUCTION_OPCODE_EXT    Ext;
 120 } SEV_ES_INSTRUCTION_DATA;
 121
 122 //
 123 // Non-automatic Exit function prototype
 124 //
 125 typedef
 126 UINT64
 127 (*NAE_EXIT) (
 128   GHCB                     *Ghcb,
 129   EFI_SYSTEM_CONTEXT_X64   *Regs,
 130   SEV_ES_INSTRUCTION_DATA  *InstructionData
 131   );
 132
 133 /**
 134   Return a pointer to the contents of the specified register.
 135
 136   Based upon the input register, return a pointer to the registers contents
 137   in the x86 processor context.
 138
 139   @param[in] Regs      x64 processor context
 140   @param[in] Register  Register to obtain pointer for
 141
 142   @return              Pointer to the contents of the requested register
 143
 144 **/
 145 STATIC
 146 UINT64 *
 147 GetRegisterPointer (
 148   IN EFI_SYSTEM_CONTEXT_X64  *Regs,
 149   IN UINT8                   Register
 150   )
 151 {
 152   UINT64  *Reg;
 153
 154   switch (Register) {
 155     case 0:
 156       Reg = &Regs->Rax;
 157       break;
 158     case 1:
 159       Reg = &Regs->Rcx;
 160       break;
 161     case 2:
 162       Reg = &Regs->Rdx;
 163       break;
 164     case 3:
 165       Reg = &Regs->Rbx;
 166       break;
 167     case 4:
 168       Reg = &Regs->Rsp;
 169       break;
 170     case 5:
 171       Reg = &Regs->Rbp;
 172       break;
 173     case 6:
 174       Reg = &Regs->Rsi;
 175       break;
 176     case 7:
 177       Reg = &Regs->Rdi;
 178       break;
 179     case 8:
 180       Reg = &Regs->R8;
 181       break;
 182     case 9:
 183       Reg = &Regs->R9;
 184       break;
 185     case 10:
 186       Reg = &Regs->R10;
 187       break;
 188     case 11:
 189       Reg = &Regs->R11;
 190       break;
 191     case 12:
 192       Reg = &Regs->R12;
 193       break;
 194     case 13:
 195       Reg = &Regs->R13;
 196       break;
 197     case 14:
 198       Reg = &Regs->R14;
 199       break;
 200     case 15:
 201       Reg = &Regs->R15;
 202       break;
 203     default:
 204       Reg = NULL;
 205   }
 206
 207   ASSERT (Reg != NULL);
 208
 209   return Reg;
 210 }
 211
 212 /**
 213   Update the instruction parsing context for displacement bytes.
 214
 215   @param[in, out] InstructionData  Instruction parsing context
 216   @param[in]      Size             The instruction displacement size
 217
 218 **/
 219 STATIC
 220 VOID
 221 UpdateForDisplacement (
 222   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 223   IN     UINTN                    Size
 224   )
 225 {
 226   InstructionData->DisplacementSize = Size;
 227   InstructionData->Immediate       += Size;
 228   InstructionData->End             += Size;
 229 }
 230
 231 /**
 232   Determine if an instruction address if RIP relative.
 233
 234   Examine the instruction parsing context to determine if the address offset
 235   is relative to the instruction pointer.
 236
 237   @param[in] InstructionData  Instruction parsing context
 238
 239   @retval TRUE                Instruction addressing is RIP relative
 240   @retval FALSE               Instruction addressing is not RIP relative
 241
 242 **/
 243 STATIC
 244 BOOLEAN
 245 IsRipRelative (
 246   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 247   )
 248 {
 249   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 250
 251   Ext = &InstructionData->Ext;
 252
 253   return ((InstructionData->Mode == LongMode64Bit) &&
 254           (Ext->ModRm.Mod == 0) &&
 255           (Ext->ModRm.Rm == 5)  &&
 256           (InstructionData->SibPresent == FALSE));
 257 }
 258
 259 /**
 260   Return the effective address of a memory operand.
 261
 262   Examine the instruction parsing context to obtain the effective memory
 263   address of a memory operand.
 264
 265   @param[in] Regs             x64 processor context
 266   @param[in] InstructionData  Instruction parsing context
 267
 268   @return                     The memory operand effective address
 269
 270 **/
 271 STATIC
 272 UINT64
 273 GetEffectiveMemoryAddress (
 274   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 275   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 276   )
 277 {
 278   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 279   UINT64                         EffectiveAddress;
 280
 281   Ext              = &InstructionData->Ext;
 282   EffectiveAddress = 0;
 283
 284   if (IsRipRelative (InstructionData)) {
 285     //
 286     // RIP-relative displacement is a 32-bit signed value
 287     //
 288     INT32  RipRelative;
 289
 290     RipRelative = *(INT32 *)InstructionData->Displacement;
 291
 292     UpdateForDisplacement (InstructionData, 4);
 293
 294     //
 295     // Negative displacement is handled by standard UINT64 wrap-around.
 296     //
 297     return Regs->Rip + (UINT64)RipRelative;
 298   }
 299
 300   switch (Ext->ModRm.Mod) {
 301     case 1:
 302       UpdateForDisplacement (InstructionData, 1);
 303       EffectiveAddress += (UINT64)(*(INT8 *)(InstructionData->Displacement));
 304       break;
 305     case 2:
 306       switch (InstructionData->AddrSize) {
 307         case Size16Bits:
 308           UpdateForDisplacement (InstructionData, 2);
 309           EffectiveAddress += (UINT64)(*(INT16 *)(InstructionData->Displacement));
 310           break;
 311         default:
 312           UpdateForDisplacement (InstructionData, 4);
 313           EffectiveAddress += (UINT64)(*(INT32 *)(InstructionData->Displacement));
 314           break;
 315       }
 316
 317       break;
 318   }
 319
 320   if (InstructionData->SibPresent) {
 321     INT64  Displacement;
 322
 323     if (Ext->Sib.Index != 4) {
 324       CopyMem (
 325         &Displacement,
 326         GetRegisterPointer (Regs, Ext->Sib.Index),
 327         sizeof (Displacement)
 328         );
 329       Displacement *= (INT64)(1 << Ext->Sib.Scale);
 330
 331       //
 332       // Negative displacement is handled by standard UINT64 wrap-around.
 333       //
 334       EffectiveAddress += (UINT64)Displacement;
 335     }
 336
 337     if ((Ext->Sib.Base != 5) || Ext->ModRm.Mod) {
 338       EffectiveAddress += *GetRegisterPointer (Regs, Ext->Sib.Base);
 339     } else {
 340       UpdateForDisplacement (InstructionData, 4);
 341       EffectiveAddress += (UINT64)(*(INT32 *)(InstructionData->Displacement));
 342     }
 343   } else {
 344     EffectiveAddress += *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 345   }
 346
 347   return EffectiveAddress;
 348 }
 349
 350 /**
 351   Decode a ModRM byte.
 352
 353   Examine the instruction parsing context to decode a ModRM byte and the SIB
 354   byte, if present.
 355
 356   @param[in]      Regs             x64 processor context
 357   @param[in, out] InstructionData  Instruction parsing context
 358
 359 **/
 360 STATIC
 361 VOID
 362 DecodeModRm (
 363   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 364   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 365   )
 366 {
 367   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
 368   INSTRUCTION_REX_PREFIX         *RexPrefix;
 369   INSTRUCTION_MODRM              *ModRm;
 370   INSTRUCTION_SIB                *Sib;
 371
 372   RexPrefix = &InstructionData->RexPrefix;
 373   Ext       = &InstructionData->Ext;
 374   ModRm     = &InstructionData->ModRm;
 375   Sib       = &InstructionData->Sib;
 376
 377   InstructionData->ModRmPresent = TRUE;
 378   ModRm->Uint8                  = *(InstructionData->End);
 379
 380   InstructionData->Displacement++;
 381   InstructionData->Immediate++;
 382   InstructionData->End++;
 383
 384   Ext->ModRm.Mod = ModRm->Bits.Mod;
 385   Ext->ModRm.Reg = (RexPrefix->Bits.BitR << 3) | ModRm->Bits.Reg;
 386   Ext->ModRm.Rm  = (RexPrefix->Bits.BitB << 3) | ModRm->Bits.Rm;
 387
 388   Ext->RegData = *GetRegisterPointer (Regs, Ext->ModRm.Reg);
 389
 390   if (Ext->ModRm.Mod == 3) {
 391     Ext->RmData = *GetRegisterPointer (Regs, Ext->ModRm.Rm);
 392   } else {
 393     if (ModRm->Bits.Rm == 4) {
 394       InstructionData->SibPresent = TRUE;
 395       Sib->Uint8                  = *(InstructionData->End);
 396
 397       InstructionData->Displacement++;
 398       InstructionData->Immediate++;
 399       InstructionData->End++;
 400
 401       Ext->Sib.Scale = Sib->Bits.Scale;
 402       Ext->Sib.Index = (RexPrefix->Bits.BitX << 3) | Sib->Bits.Index;
 403       Ext->Sib.Base  = (RexPrefix->Bits.BitB << 3) | Sib->Bits.Base;
 404     }
 405
 406     Ext->RmData = GetEffectiveMemoryAddress (Regs, InstructionData);
 407   }
 408 }
 409
 410 /**
 411   Decode instruction prefixes.
 412
 413   Parse the instruction data to track the instruction prefixes that have
 414   been used.
 415
 416   @param[in]      Regs             x64 processor context
 417   @param[in, out] InstructionData  Instruction parsing context
 418
 419 **/
 420 STATIC
 421 VOID
 422 DecodePrefixes (
 423   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
 424   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 425   )
 426 {
 427   SEV_ES_INSTRUCTION_MODE  Mode;
 428   SEV_ES_INSTRUCTION_SIZE  ModeDataSize;
 429   SEV_ES_INSTRUCTION_SIZE  ModeAddrSize;
 430   UINT8                    *Byte;
 431
 432   //
 433   // Always in 64-bit mode
 434   //
 435   Mode         = LongMode64Bit;
 436   ModeDataSize = Size32Bits;
 437   ModeAddrSize = Size64Bits;
 438
 439   InstructionData->Mode     = Mode;
 440   InstructionData->DataSize = ModeDataSize;
 441   InstructionData->AddrSize = ModeAddrSize;
 442
 443   InstructionData->Prefixes = InstructionData->Begin;
 444
 445   Byte = InstructionData->Prefixes;
 446   for ( ; ; Byte++, InstructionData->PrefixSize++) {
 447     //
 448     // Check the 0x40 to 0x4F range using an if statement here since some
 449     // compilers don't like the "case 0x40 ... 0x4F:" syntax. This avoids
 450     // 16 case statements below.
 451     //
 452     if ((*Byte >= REX_PREFIX_START) && (*Byte <= REX_PREFIX_STOP)) {
 453       InstructionData->RexPrefix.Uint8 = *Byte;
 454       if ((*Byte & REX_64BIT_OPERAND_SIZE_MASK) != 0) {
 455         InstructionData->DataSize = Size64Bits;
 456       }
 457
 458       continue;
 459     }
 460
 461     switch (*Byte) {
 462       case OVERRIDE_SEGMENT_CS:
 463       case OVERRIDE_SEGMENT_DS:
 464       case OVERRIDE_SEGMENT_ES:
 465       case OVERRIDE_SEGMENT_SS:
 466         if (Mode != LongMode64Bit) {
 467           InstructionData->SegmentSpecified = TRUE;
 468           InstructionData->Segment          = (*Byte >> 3) & 3;
 469         }
 470
 471         break;
 472
 473       case OVERRIDE_SEGMENT_FS:
 474       case OVERRIDE_SEGMENT_GS:
 475         InstructionData->SegmentSpecified = TRUE;
 476         InstructionData->Segment          = *Byte & 7;
 477         break;
 478
 479       case OVERRIDE_OPERAND_SIZE:
 480         if (InstructionData->RexPrefix.Uint8 == 0) {
 481           InstructionData->DataSize =
 482             (Mode == LongMode64Bit)       ? Size16Bits :
 483             (Mode == LongModeCompat32Bit) ? Size16Bits :
 484             (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 485         }
 486
 487         break;
 488
 489       case OVERRIDE_ADDRESS_SIZE:
 490         InstructionData->AddrSize =
 491           (Mode == LongMode64Bit)       ? Size32Bits :
 492           (Mode == LongModeCompat32Bit) ? Size16Bits :
 493           (Mode == LongModeCompat16Bit) ? Size32Bits : 0;
 494         break;
 495
 496       case LOCK_PREFIX:
 497         break;
 498
 499       case REPZ_PREFIX:
 500         InstructionData->RepMode = RepZ;
 501         break;
 502
 503       case REPNZ_PREFIX:
 504         InstructionData->RepMode = RepNZ;
 505         break;
 506
 507       default:
 508         InstructionData->OpCodes    = Byte;
 509         InstructionData->OpCodeSize = (*Byte == TWO_BYTE_OPCODE_ESCAPE) ? 2 : 1;
 510
 511         InstructionData->End          = Byte + InstructionData->OpCodeSize;
 512         InstructionData->Displacement = InstructionData->End;
 513         InstructionData->Immediate    = InstructionData->End;
 514         return;
 515     }
 516   }
 517 }
 518
 519 /**
 520   Determine instruction length
 521
 522   Return the total length of the parsed instruction.
 523
 524   @param[in] InstructionData  Instruction parsing context
 525
 526   @return                     Length of parsed instruction
 527
 528 **/
 529 STATIC
 530 UINT64
 531 InstructionLength (
 532   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 533   )
 534 {
 535   return (UINT64)(InstructionData->End - InstructionData->Begin);
 536 }
 537
 538 /**
 539   Initialize the instruction parsing context.
 540
 541   Initialize the instruction parsing context, which includes decoding the
 542   instruction prefixes.
 543
 544   @param[in, out] InstructionData  Instruction parsing context
 545   @param[in]      Ghcb             Pointer to the Guest-Hypervisor Communication
 546                                    Block
 547   @param[in]      Regs             x64 processor context
 548
 549 **/
 550 STATIC
 551 VOID
 552 InitInstructionData (
 553   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData,
 554   IN     GHCB                     *Ghcb,
 555   IN     EFI_SYSTEM_CONTEXT_X64   *Regs
 556   )
 557 {
 558   SetMem (InstructionData, sizeof (*InstructionData), 0);
 559   InstructionData->Ghcb  = Ghcb;
 560   InstructionData->Begin = (UINT8 *)Regs->Rip;
 561   InstructionData->End   = (UINT8 *)Regs->Rip;
 562
 563   DecodePrefixes (Regs, InstructionData);
 564 }
 565
 566 /**
 567   Report an unsupported event to the hypervisor
 568
 569   Use the VMGEXIT support to report an unsupported event to the hypervisor.
 570
 571   @param[in] Ghcb             Pointer to the Guest-Hypervisor Communication
 572                               Block
 573   @param[in] Regs             x64 processor context
 574   @param[in] InstructionData  Instruction parsing context
 575
 576   @return                     New exception value to propagate
 577
 578 **/
 579 STATIC
 580 UINT64
 581 UnsupportedExit (
 582   IN GHCB                     *Ghcb,
 583   IN EFI_SYSTEM_CONTEXT_X64   *Regs,
 584   IN SEV_ES_INSTRUCTION_DATA  *InstructionData
 585   )
 586 {
 587   UINT64  Status;
 588
 589   Status = VmgExit (Ghcb, SVM_EXIT_UNSUPPORTED, Regs->ExceptionData, 0);
 590   if (Status == 0) {
 591     GHCB_EVENT_INJECTION  Event;
 592
 593     Event.Uint64          = 0;
 594     Event.Elements.Vector = GP_EXCEPTION;
 595     Event.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
 596     Event.Elements.Valid  = 1;
 597
 598     Status = Event.Uint64;
 599   }
 600
 601   return Status;
 602 }
 603
 604 /**
 605   Validate that the MMIO memory access is not to encrypted memory.
 606
 607   Examine the pagetable entry for the memory specified. MMIO should not be
 608   performed against encrypted memory. MMIO to the APIC page is always allowed.
 609
 610   @param[in] Ghcb           Pointer to the Guest-Hypervisor Communication Block
 611   @param[in] MemoryAddress  Memory address to validate
 612   @param[in] MemoryLength   Memory length to validate
 613
 614   @retval 0          Memory is not encrypted
 615   @return            New exception value to propogate
 616
 617 **/
 618 STATIC
 619 UINT64
 620 ValidateMmioMemory (
 621   IN GHCB   *Ghcb,
 622   IN UINTN  MemoryAddress,
 623   IN UINTN  MemoryLength
 624   )
 625 {
 626   MEM_ENCRYPT_SEV_ADDRESS_RANGE_STATE  State;
 627   GHCB_EVENT_INJECTION                 GpEvent;
 628   UINTN                                Address;
 629
 630   //
 631   // Allow APIC accesses (which will have the encryption bit set during
 632   // SEC and PEI phases).
 633   //
 634   Address = MemoryAddress & ~(SIZE_4KB - 1);
 635   if (Address == GetLocalApicBaseAddress ()) {
 636     return 0;
 637   }
 638
 639   State = MemEncryptSevGetAddressRangeState (
 640             0,
 641             MemoryAddress,
 642             MemoryLength
 643             );
 644   if (State == MemEncryptSevAddressRangeUnencrypted) {
 645     return 0;
 646   }
 647
 648   //
 649   // Any state other than unencrypted is an error, issue a #GP.
 650   //
 651   DEBUG ((
 652     DEBUG_ERROR,
 653     "MMIO using encrypted memory: %lx\n",
 654     (UINT64)MemoryAddress
 655     ));
 656   GpEvent.Uint64          = 0;
 657   GpEvent.Elements.Vector = GP_EXCEPTION;
 658   GpEvent.Elements.Type   = GHCB_EVENT_INJECTION_TYPE_EXCEPTION;
 659   GpEvent.Elements.Valid  = 1;
 660
 661   return GpEvent.Uint64;
 662 }
 663
 664 /**
 665   Handle an MMIO event.
 666
 667   Use the VMGEXIT instruction to handle either an MMIO read or an MMIO write.
 668
 669   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
 670                                    Block
 671   @param[in, out] Regs             x64 processor context
 672   @param[in, out] InstructionData  Instruction parsing context
 673
 674   @retval 0                        Event handled successfully
 675   @return                          New exception value to propagate
 676
 677 **/
 678 STATIC
 679 UINT64
 680 MmioExit (
 681   IN OUT GHCB                     *Ghcb,
 682   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
 683   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
 684   )
 685 {
 686   UINT64  ExitInfo1, ExitInfo2, Status;
 687   UINTN   Bytes;
 688   UINT64  *Register;
 689   UINT8   OpCode, SignByte;
 690   UINTN   Address;
 691
 692   Bytes = 0;
 693
 694   OpCode = *(InstructionData->OpCodes);
 695   if (OpCode == TWO_BYTE_OPCODE_ESCAPE) {
 696     OpCode = *(InstructionData->OpCodes + 1);
 697   }
 698
 699   switch (OpCode) {
 700     //
 701     // MMIO write (MOV reg/memX, regX)
 702     //
 703     case 0x88:
 704       Bytes = 1;
 705     //
 706     // fall through
 707     //
 708     case 0x89:
 709       DecodeModRm (Regs, InstructionData);
 710       Bytes = ((Bytes != 0) ? Bytes :
 711                (InstructionData->DataSize == Size16Bits) ? 2 :
 712                (InstructionData->DataSize == Size32Bits) ? 4 :
 713                (InstructionData->DataSize == Size64Bits) ? 8 :
 714                0);
 715
 716       if (InstructionData->Ext.ModRm.Mod == 3) {
 717         //
 718         // NPF on two register operands???
 719         //
 720         return UnsupportedExit (Ghcb, Regs, InstructionData);
 721       }
 722
 723       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 724       if (Status != 0) {
 725         return Status;
 726       }
 727
 728       ExitInfo1 = InstructionData->Ext.RmData;
 729       ExitInfo2 = Bytes;
 730       CopyMem (Ghcb->SharedBuffer, &InstructionData->Ext.RegData, Bytes);
 731
 732       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 733       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 734       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 735       if (Status != 0) {
 736         return Status;
 737       }
 738
 739       break;
 740
 741     //
 742     // MMIO write (MOV moffsetX, aX)
 743     //
 744     case 0xA2:
 745       Bytes = 1;
 746     //
 747     // fall through
 748     //
 749     case 0xA3:
 750       Bytes = ((Bytes != 0) ? Bytes :
 751                (InstructionData->DataSize == Size16Bits) ? 2 :
 752                (InstructionData->DataSize == Size32Bits) ? 4 :
 753                (InstructionData->DataSize == Size64Bits) ? 8 :
 754                0);
 755
 756       InstructionData->ImmediateSize = (UINTN)(1 << InstructionData->AddrSize);
 757       InstructionData->End          += InstructionData->ImmediateSize;
 758
 759       //
 760       // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
 761       // Use a STATIC_ASSERT to be certain the code is being built as X64.
 762       //
 763       STATIC_ASSERT (
 764         sizeof (UINTN) == sizeof (UINT64),
 765         "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
 766         );
 767
 768       Address = 0;
 769       CopyMem (
 770         &Address,
 771         InstructionData->Immediate,
 772         InstructionData->ImmediateSize
 773         );
 774
 775       Status = ValidateMmioMemory (Ghcb, Address, Bytes);
 776       if (Status != 0) {
 777         return Status;
 778       }
 779
 780       ExitInfo1 = Address;
 781       ExitInfo2 = Bytes;
 782       CopyMem (Ghcb->SharedBuffer, &Regs->Rax, Bytes);
 783
 784       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 785       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 786       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 787       if (Status != 0) {
 788         return Status;
 789       }
 790
 791       break;
 792
 793     //
 794     // MMIO write (MOV reg/memX, immX)
 795     //
 796     case 0xC6:
 797       Bytes = 1;
 798     //
 799     // fall through
 800     //
 801     case 0xC7:
 802       DecodeModRm (Regs, InstructionData);
 803       Bytes = ((Bytes != 0) ? Bytes :
 804                (InstructionData->DataSize == Size16Bits) ? 2 :
 805                (InstructionData->DataSize == Size32Bits) ? 4 :
 806                0);
 807
 808       InstructionData->ImmediateSize = Bytes;
 809       InstructionData->End          += Bytes;
 810
 811       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 812       if (Status != 0) {
 813         return Status;
 814       }
 815
 816       ExitInfo1 = InstructionData->Ext.RmData;
 817       ExitInfo2 = Bytes;
 818       CopyMem (Ghcb->SharedBuffer, InstructionData->Immediate, Bytes);
 819
 820       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 821       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 822       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_WRITE, ExitInfo1, ExitInfo2);
 823       if (Status != 0) {
 824         return Status;
 825       }
 826
 827       break;
 828
 829     //
 830     // MMIO read (MOV regX, reg/memX)
 831     //
 832     case 0x8A:
 833       Bytes = 1;
 834     //
 835     // fall through
 836     //
 837     case 0x8B:
 838       DecodeModRm (Regs, InstructionData);
 839       Bytes = ((Bytes != 0) ? Bytes :
 840                (InstructionData->DataSize == Size16Bits) ? 2 :
 841                (InstructionData->DataSize == Size32Bits) ? 4 :
 842                (InstructionData->DataSize == Size64Bits) ? 8 :
 843                0);
 844       if (InstructionData->Ext.ModRm.Mod == 3) {
 845         //
 846         // NPF on two register operands???
 847         //
 848         return UnsupportedExit (Ghcb, Regs, InstructionData);
 849       }
 850
 851       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 852       if (Status != 0) {
 853         return Status;
 854       }
 855
 856       ExitInfo1 = InstructionData->Ext.RmData;
 857       ExitInfo2 = Bytes;
 858
 859       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 860       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 861       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 862       if (Status != 0) {
 863         return Status;
 864       }
 865
 866       Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 867       if (Bytes == 4) {
 868         //
 869         // Zero-extend for 32-bit operation
 870         //
 871         *Register = 0;
 872       }
 873
 874       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 875       break;
 876
 877     //
 878     // MMIO read (MOV aX, moffsetX)
 879     //
 880     case 0xA0:
 881       Bytes = 1;
 882     //
 883     // fall through
 884     //
 885     case 0xA1:
 886       Bytes = ((Bytes != 0) ? Bytes :
 887                (InstructionData->DataSize == Size16Bits) ? 2 :
 888                (InstructionData->DataSize == Size32Bits) ? 4 :
 889                (InstructionData->DataSize == Size64Bits) ? 8 :
 890                0);
 891
 892       InstructionData->ImmediateSize = (UINTN)(1 << InstructionData->AddrSize);
 893       InstructionData->End          += InstructionData->ImmediateSize;
 894
 895       //
 896       // This code is X64 only, so a possible 8-byte copy to a UINTN is ok.
 897       // Use a STATIC_ASSERT to be certain the code is being built as X64.
 898       //
 899       STATIC_ASSERT (
 900         sizeof (UINTN) == sizeof (UINT64),
 901         "sizeof (UINTN) != sizeof (UINT64), this file must be built as X64"
 902         );
 903
 904       Address = 0;
 905       CopyMem (
 906         &Address,
 907         InstructionData->Immediate,
 908         InstructionData->ImmediateSize
 909         );
 910
 911       Status = ValidateMmioMemory (Ghcb, Address, Bytes);
 912       if (Status != 0) {
 913         return Status;
 914       }
 915
 916       ExitInfo1 = Address;
 917       ExitInfo2 = Bytes;
 918
 919       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 920       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 921       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 922       if (Status != 0) {
 923         return Status;
 924       }
 925
 926       if (Bytes == 4) {
 927         //
 928         // Zero-extend for 32-bit operation
 929         //
 930         Regs->Rax = 0;
 931       }
 932
 933       CopyMem (&Regs->Rax, Ghcb->SharedBuffer, Bytes);
 934       break;
 935
 936     //
 937     // MMIO read w/ zero-extension ((MOVZX regX, reg/memX)
 938     //
 939     case 0xB6:
 940       Bytes = 1;
 941     //
 942     // fall through
 943     //
 944     case 0xB7:
 945       DecodeModRm (Regs, InstructionData);
 946       Bytes = (Bytes != 0) ? Bytes : 2;
 947
 948       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 949       if (Status != 0) {
 950         return Status;
 951       }
 952
 953       ExitInfo1 = InstructionData->Ext.RmData;
 954       ExitInfo2 = Bytes;
 955
 956       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 957       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 958       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 959       if (Status != 0) {
 960         return Status;
 961       }
 962
 963       Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
 964       SetMem (Register, (UINTN)(1 << InstructionData->DataSize), 0);
 965       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
 966       break;
 967
 968     //
 969     // MMIO read w/ sign-extension (MOVSX regX, reg/memX)
 970     //
 971     case 0xBE:
 972       Bytes = 1;
 973     //
 974     // fall through
 975     //
 976     case 0xBF:
 977       DecodeModRm (Regs, InstructionData);
 978       Bytes = (Bytes != 0) ? Bytes : 2;
 979
 980       Status = ValidateMmioMemory (Ghcb, InstructionData->Ext.RmData, Bytes);
 981       if (Status != 0) {
 982         return Status;
 983       }
 984
 985       ExitInfo1 = InstructionData->Ext.RmData;
 986       ExitInfo2 = Bytes;
 987
 988       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
 989       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
 990       Status = VmgExit (Ghcb, SVM_EXIT_MMIO_READ, ExitInfo1, ExitInfo2);
 991       if (Status != 0) {
 992         return Status;
 993       }
 994
 995       if (Bytes == 1) {
 996         UINT8  *Data;
 997
 998         Data     = (UINT8 *)Ghcb->SharedBuffer;
 999         SignByte = ((*Data & BIT7) != 0) ? 0xFF : 0x00;
1000       } else {
1001         UINT16  *Data;
1002
1003         Data     = (UINT16 *)Ghcb->SharedBuffer;
1004         SignByte = ((*Data & BIT15) != 0) ? 0xFF : 0x00;
1005       }
1006
1007       Register = GetRegisterPointer (Regs, InstructionData->Ext.ModRm.Reg);
1008       SetMem (Register, (UINTN)(1 << InstructionData->DataSize), SignByte);
1009       CopyMem (Register, Ghcb->SharedBuffer, Bytes);
1010       break;
1011
1012     default:
1013       DEBUG ((DEBUG_ERROR, "Invalid MMIO opcode (%x)\n", OpCode));
1014       Status = GP_EXCEPTION;
1015       ASSERT (FALSE);
1016   }
1017
1018   return Status;
1019 }
1020
1021 /**
1022   Handle a MWAIT event.
1023
1024   Use the VMGEXIT instruction to handle a MWAIT event.
1025
1026   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1027                                    Block
1028   @param[in, out] Regs             x64 processor context
1029   @param[in]      InstructionData  Instruction parsing context
1030
1031   @retval 0                        Event handled successfully
1032   @return                          New exception value to propagate
1033
1034 **/
1035 STATIC
1036 UINT64
1037 MwaitExit (
1038   IN OUT GHCB                     *Ghcb,
1039   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1040   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1041   )
1042 {
1043   DecodeModRm (Regs, InstructionData);
1044
1045   Ghcb->SaveArea.Rax = Regs->Rax;
1046   VmgSetOffsetValid (Ghcb, GhcbRax);
1047   Ghcb->SaveArea.Rcx = Regs->Rcx;
1048   VmgSetOffsetValid (Ghcb, GhcbRcx);
1049
1050   return VmgExit (Ghcb, SVM_EXIT_MWAIT, 0, 0);
1051 }
1052
1053 /**
1054   Handle a MONITOR event.
1055
1056   Use the VMGEXIT instruction to handle a MONITOR event.
1057
1058   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1059                                    Block
1060   @param[in, out] Regs             x64 processor context
1061   @param[in]      InstructionData  Instruction parsing context
1062
1063   @retval 0                        Event handled successfully
1064   @return                          New exception value to propagate
1065
1066 **/
1067 STATIC
1068 UINT64
1069 MonitorExit (
1070   IN OUT GHCB                     *Ghcb,
1071   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1072   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1073   )
1074 {
1075   DecodeModRm (Regs, InstructionData);
1076
1077   Ghcb->SaveArea.Rax = Regs->Rax;  // Identity mapped, so VA = PA
1078   VmgSetOffsetValid (Ghcb, GhcbRax);
1079   Ghcb->SaveArea.Rcx = Regs->Rcx;
1080   VmgSetOffsetValid (Ghcb, GhcbRcx);
1081   Ghcb->SaveArea.Rdx = Regs->Rdx;
1082   VmgSetOffsetValid (Ghcb, GhcbRdx);
1083
1084   return VmgExit (Ghcb, SVM_EXIT_MONITOR, 0, 0);
1085 }
1086
1087 /**
1088   Handle a WBINVD event.
1089
1090   Use the VMGEXIT instruction to handle a WBINVD event.
1091
1092   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1093                                    Block
1094   @param[in, out] Regs             x64 processor context
1095   @param[in]      InstructionData  Instruction parsing context
1096
1097   @retval 0                        Event handled successfully
1098   @return                          New exception value to propagate
1099
1100 **/
1101 STATIC
1102 UINT64
1103 WbinvdExit (
1104   IN OUT GHCB                     *Ghcb,
1105   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1106   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1107   )
1108 {
1109   return VmgExit (Ghcb, SVM_EXIT_WBINVD, 0, 0);
1110 }
1111
1112 /**
1113   Handle a RDTSCP event.
1114
1115   Use the VMGEXIT instruction to handle a RDTSCP event.
1116
1117   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1118                                    Block
1119   @param[in, out] Regs             x64 processor context
1120   @param[in]      InstructionData  Instruction parsing context
1121
1122   @retval 0                        Event handled successfully
1123   @return                          New exception value to propagate
1124
1125 **/
1126 STATIC
1127 UINT64
1128 RdtscpExit (
1129   IN OUT GHCB                     *Ghcb,
1130   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1131   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1132   )
1133 {
1134   UINT64  Status;
1135
1136   DecodeModRm (Regs, InstructionData);
1137
1138   Status = VmgExit (Ghcb, SVM_EXIT_RDTSCP, 0, 0);
1139   if (Status != 0) {
1140     return Status;
1141   }
1142
1143   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1144       !VmgIsOffsetValid (Ghcb, GhcbRcx) ||
1145       !VmgIsOffsetValid (Ghcb, GhcbRdx))
1146   {
1147     return UnsupportedExit (Ghcb, Regs, InstructionData);
1148   }
1149
1150   Regs->Rax = Ghcb->SaveArea.Rax;
1151   Regs->Rcx = Ghcb->SaveArea.Rcx;
1152   Regs->Rdx = Ghcb->SaveArea.Rdx;
1153
1154   return 0;
1155 }
1156
1157 /**
1158   Handle a VMMCALL event.
1159
1160   Use the VMGEXIT instruction to handle a VMMCALL event.
1161
1162   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1163                                    Block
1164   @param[in, out] Regs             x64 processor context
1165   @param[in]      InstructionData  Instruction parsing context
1166
1167   @retval 0                        Event handled successfully
1168   @return                          New exception value to propagate
1169
1170 **/
1171 STATIC
1172 UINT64
1173 VmmCallExit (
1174   IN OUT GHCB                     *Ghcb,
1175   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1176   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1177   )
1178 {
1179   UINT64  Status;
1180
1181   DecodeModRm (Regs, InstructionData);
1182
1183   Ghcb->SaveArea.Rax = Regs->Rax;
1184   VmgSetOffsetValid (Ghcb, GhcbRax);
1185   Ghcb->SaveArea.Cpl = (UINT8)(Regs->Cs & 0x3);
1186   VmgSetOffsetValid (Ghcb, GhcbCpl);
1187
1188   Status = VmgExit (Ghcb, SVM_EXIT_VMMCALL, 0, 0);
1189   if (Status != 0) {
1190     return Status;
1191   }
1192
1193   if (!VmgIsOffsetValid (Ghcb, GhcbRax)) {
1194     return UnsupportedExit (Ghcb, Regs, InstructionData);
1195   }
1196
1197   Regs->Rax = Ghcb->SaveArea.Rax;
1198
1199   return 0;
1200 }
1201
1202 /**
1203   Handle an MSR event.
1204
1205   Use the VMGEXIT instruction to handle either a RDMSR or WRMSR event.
1206
1207   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1208                                    Block
1209   @param[in, out] Regs             x64 processor context
1210   @param[in]      InstructionData  Instruction parsing context
1211
1212   @retval 0                        Event handled successfully
1213   @return                          New exception value to propagate
1214
1215 **/
1216 STATIC
1217 UINT64
1218 MsrExit (
1219   IN OUT GHCB                     *Ghcb,
1220   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1221   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1222   )
1223 {
1224   UINT64  ExitInfo1, Status;
1225
1226   ExitInfo1 = 0;
1227
1228   switch (*(InstructionData->OpCodes + 1)) {
1229     case 0x30: // WRMSR
1230       ExitInfo1          = 1;
1231       Ghcb->SaveArea.Rax = Regs->Rax;
1232       VmgSetOffsetValid (Ghcb, GhcbRax);
1233       Ghcb->SaveArea.Rdx = Regs->Rdx;
1234       VmgSetOffsetValid (Ghcb, GhcbRdx);
1235     //
1236     // fall through
1237     //
1238     case 0x32: // RDMSR
1239       Ghcb->SaveArea.Rcx = Regs->Rcx;
1240       VmgSetOffsetValid (Ghcb, GhcbRcx);
1241       break;
1242     default:
1243       return UnsupportedExit (Ghcb, Regs, InstructionData);
1244   }
1245
1246   Status = VmgExit (Ghcb, SVM_EXIT_MSR, ExitInfo1, 0);
1247   if (Status != 0) {
1248     return Status;
1249   }
1250
1251   if (ExitInfo1 == 0) {
1252     if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1253         !VmgIsOffsetValid (Ghcb, GhcbRdx))
1254     {
1255       return UnsupportedExit (Ghcb, Regs, InstructionData);
1256     }
1257
1258     Regs->Rax = Ghcb->SaveArea.Rax;
1259     Regs->Rdx = Ghcb->SaveArea.Rdx;
1260   }
1261
1262   return 0;
1263 }
1264
1265 /**
1266   Build the IOIO event information.
1267
1268   The IOIO event information identifies the type of IO operation to be performed
1269   by the hypervisor. Build this information based on the instruction data.
1270
1271   @param[in]       Regs             x64 processor context
1272   @param[in, out]  InstructionData  Instruction parsing context
1273
1274   @return                           IOIO event information value
1275
1276 **/
1277 STATIC
1278 UINT64
1279 IoioExitInfo (
1280   IN     EFI_SYSTEM_CONTEXT_X64   *Regs,
1281   IN OUT SEV_ES_INSTRUCTION_DATA  *InstructionData
1282   )
1283 {
1284   UINT64  ExitInfo;
1285
1286   ExitInfo = 0;
1287
1288   switch (*(InstructionData->OpCodes)) {
1289     //
1290     // INS opcodes
1291     //
1292     case 0x6C:
1293     case 0x6D:
1294       ExitInfo |= IOIO_TYPE_INS;
1295       ExitInfo |= IOIO_SEG_ES;
1296       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1297       break;
1298
1299     //
1300     // OUTS opcodes
1301     //
1302     case 0x6E:
1303     case 0x6F:
1304       ExitInfo |= IOIO_TYPE_OUTS;
1305       ExitInfo |= IOIO_SEG_DS;
1306       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1307       break;
1308
1309     //
1310     // IN immediate opcodes
1311     //
1312     case 0xE4:
1313     case 0xE5:
1314       InstructionData->ImmediateSize = 1;
1315       InstructionData->End++;
1316       ExitInfo |= IOIO_TYPE_IN;
1317       ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16);
1318       break;
1319
1320     //
1321     // OUT immediate opcodes
1322     //
1323     case 0xE6:
1324     case 0xE7:
1325       InstructionData->ImmediateSize = 1;
1326       InstructionData->End++;
1327       ExitInfo |= IOIO_TYPE_OUT;
1328       ExitInfo |= ((*(InstructionData->OpCodes + 1)) << 16) | IOIO_TYPE_OUT;
1329       break;
1330
1331     //
1332     // IN register opcodes
1333     //
1334     case 0xEC:
1335     case 0xED:
1336       ExitInfo |= IOIO_TYPE_IN;
1337       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1338       break;
1339
1340     //
1341     // OUT register opcodes
1342     //
1343     case 0xEE:
1344     case 0xEF:
1345       ExitInfo |= IOIO_TYPE_OUT;
1346       ExitInfo |= ((Regs->Rdx & 0xffff) << 16);
1347       break;
1348
1349     default:
1350       return 0;
1351   }
1352
1353   switch (*(InstructionData->OpCodes)) {
1354     //
1355     // Single-byte opcodes
1356     //
1357     case 0x6C:
1358     case 0x6E:
1359     case 0xE4:
1360     case 0xE6:
1361     case 0xEC:
1362     case 0xEE:
1363       ExitInfo |= IOIO_DATA_8;
1364       break;
1365
1366     //
1367     // Length determined by instruction parsing
1368     //
1369     default:
1370       ExitInfo |= (InstructionData->DataSize == Size16Bits) ? IOIO_DATA_16
1371                                                           : IOIO_DATA_32;
1372   }
1373
1374   switch (InstructionData->AddrSize) {
1375     case Size16Bits:
1376       ExitInfo |= IOIO_ADDR_16;
1377       break;
1378
1379     case Size32Bits:
1380       ExitInfo |= IOIO_ADDR_32;
1381       break;
1382
1383     case Size64Bits:
1384       ExitInfo |= IOIO_ADDR_64;
1385       break;
1386
1387     default:
1388       break;
1389   }
1390
1391   if (InstructionData->RepMode != 0) {
1392     ExitInfo |= IOIO_REP;
1393   }
1394
1395   return ExitInfo;
1396 }
1397
1398 /**
1399   Handle an IOIO event.
1400
1401   Use the VMGEXIT instruction to handle an IOIO event.
1402
1403   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1404                                    Block
1405   @param[in, out] Regs             x64 processor context
1406   @param[in]      InstructionData  Instruction parsing context
1407
1408   @retval 0                        Event handled successfully
1409   @return                          New exception value to propagate
1410
1411 **/
1412 STATIC
1413 UINT64
1414 IoioExit (
1415   IN OUT GHCB                     *Ghcb,
1416   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1417   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1418   )
1419 {
1420   UINT64   ExitInfo1, ExitInfo2, Status;
1421   BOOLEAN  IsString;
1422
1423   ExitInfo1 = IoioExitInfo (Regs, InstructionData);
1424   if (ExitInfo1 == 0) {
1425     return UnsupportedExit (Ghcb, Regs, InstructionData);
1426   }
1427
1428   IsString = ((ExitInfo1 & IOIO_TYPE_STR) != 0) ? TRUE : FALSE;
1429   if (IsString) {
1430     UINTN  IoBytes, VmgExitBytes;
1431     UINTN  GhcbCount, OpCount;
1432
1433     Status = 0;
1434
1435     IoBytes   = IOIO_DATA_BYTES (ExitInfo1);
1436     GhcbCount = sizeof (Ghcb->SharedBuffer) / IoBytes;
1437
1438     OpCount = ((ExitInfo1 & IOIO_REP) != 0) ? Regs->Rcx : 1;
1439     while (OpCount != 0) {
1440       ExitInfo2    = MIN (OpCount, GhcbCount);
1441       VmgExitBytes = ExitInfo2 * IoBytes;
1442
1443       if ((ExitInfo1 & IOIO_TYPE_IN) == 0) {
1444         CopyMem (Ghcb->SharedBuffer, (VOID *)Regs->Rsi, VmgExitBytes);
1445         Regs->Rsi += VmgExitBytes;
1446       }
1447
1448       Ghcb->SaveArea.SwScratch = (UINT64)Ghcb->SharedBuffer;
1449       VmgSetOffsetValid (Ghcb, GhcbSwScratch);
1450       Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, ExitInfo2);
1451       if (Status != 0) {
1452         return Status;
1453       }
1454
1455       if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1456         CopyMem ((VOID *)Regs->Rdi, Ghcb->SharedBuffer, VmgExitBytes);
1457         Regs->Rdi += VmgExitBytes;
1458       }
1459
1460       if ((ExitInfo1 & IOIO_REP) != 0) {
1461         Regs->Rcx -= ExitInfo2;
1462       }
1463
1464       OpCount -= ExitInfo2;
1465     }
1466   } else {
1467     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1468       Ghcb->SaveArea.Rax = 0;
1469     } else {
1470       CopyMem (&Ghcb->SaveArea.Rax, &Regs->Rax, IOIO_DATA_BYTES (ExitInfo1));
1471     }
1472
1473     VmgSetOffsetValid (Ghcb, GhcbRax);
1474
1475     Status = VmgExit (Ghcb, SVM_EXIT_IOIO_PROT, ExitInfo1, 0);
1476     if (Status != 0) {
1477       return Status;
1478     }
1479
1480     if ((ExitInfo1 & IOIO_TYPE_IN) != 0) {
1481       if (!VmgIsOffsetValid (Ghcb, GhcbRax)) {
1482         return UnsupportedExit (Ghcb, Regs, InstructionData);
1483       }
1484
1485       CopyMem (&Regs->Rax, &Ghcb->SaveArea.Rax, IOIO_DATA_BYTES (ExitInfo1));
1486     }
1487   }
1488
1489   return 0;
1490 }
1491
1492 /**
1493   Handle a INVD event.
1494
1495   Use the VMGEXIT instruction to handle a INVD event.
1496
1497   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1498                                    Block
1499   @param[in, out] Regs             x64 processor context
1500   @param[in]      InstructionData  Instruction parsing context
1501
1502   @retval 0                        Event handled successfully
1503   @return                          New exception value to propagate
1504
1505 **/
1506 STATIC
1507 UINT64
1508 InvdExit (
1509   IN OUT GHCB                     *Ghcb,
1510   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1511   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1512   )
1513 {
1514   return VmgExit (Ghcb, SVM_EXIT_INVD, 0, 0);
1515 }
1516
1517 /**
1518   Handle a CPUID event.
1519
1520   Use the VMGEXIT instruction to handle a CPUID event.
1521
1522   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1523                                    Block
1524   @param[in, out] Regs             x64 processor context
1525   @param[in]      InstructionData  Instruction parsing context
1526
1527   @retval 0                        Event handled successfully
1528   @return                          New exception value to propagate
1529
1530 **/
1531 STATIC
1532 UINT64
1533 CpuidExit (
1534   IN OUT GHCB                     *Ghcb,
1535   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1536   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1537   )
1538 {
1539   UINT64  Status;
1540
1541   Ghcb->SaveArea.Rax = Regs->Rax;
1542   VmgSetOffsetValid (Ghcb, GhcbRax);
1543   Ghcb->SaveArea.Rcx = Regs->Rcx;
1544   VmgSetOffsetValid (Ghcb, GhcbRcx);
1545   if (Regs->Rax == CPUID_EXTENDED_STATE) {
1546     IA32_CR4  Cr4;
1547
1548     Cr4.UintN           = AsmReadCr4 ();
1549     Ghcb->SaveArea.XCr0 = (Cr4.Bits.OSXSAVE == 1) ? AsmXGetBv (0) : 1;
1550     VmgSetOffsetValid (Ghcb, GhcbXCr0);
1551   }
1552
1553   Status = VmgExit (Ghcb, SVM_EXIT_CPUID, 0, 0);
1554   if (Status != 0) {
1555     return Status;
1556   }
1557
1558   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1559       !VmgIsOffsetValid (Ghcb, GhcbRbx) ||
1560       !VmgIsOffsetValid (Ghcb, GhcbRcx) ||
1561       !VmgIsOffsetValid (Ghcb, GhcbRdx))
1562   {
1563     return UnsupportedExit (Ghcb, Regs, InstructionData);
1564   }
1565
1566   Regs->Rax = Ghcb->SaveArea.Rax;
1567   Regs->Rbx = Ghcb->SaveArea.Rbx;
1568   Regs->Rcx = Ghcb->SaveArea.Rcx;
1569   Regs->Rdx = Ghcb->SaveArea.Rdx;
1570
1571   return 0;
1572 }
1573
1574 /**
1575   Handle a RDPMC event.
1576
1577   Use the VMGEXIT instruction to handle a RDPMC event.
1578
1579   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1580                                    Block
1581   @param[in, out] Regs             x64 processor context
1582   @param[in]      InstructionData  Instruction parsing context
1583
1584   @retval 0                        Event handled successfully
1585   @return                          New exception value to propagate
1586
1587 **/
1588 STATIC
1589 UINT64
1590 RdpmcExit (
1591   IN OUT GHCB                     *Ghcb,
1592   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1593   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1594   )
1595 {
1596   UINT64  Status;
1597
1598   Ghcb->SaveArea.Rcx = Regs->Rcx;
1599   VmgSetOffsetValid (Ghcb, GhcbRcx);
1600
1601   Status = VmgExit (Ghcb, SVM_EXIT_RDPMC, 0, 0);
1602   if (Status != 0) {
1603     return Status;
1604   }
1605
1606   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1607       !VmgIsOffsetValid (Ghcb, GhcbRdx))
1608   {
1609     return UnsupportedExit (Ghcb, Regs, InstructionData);
1610   }
1611
1612   Regs->Rax = Ghcb->SaveArea.Rax;
1613   Regs->Rdx = Ghcb->SaveArea.Rdx;
1614
1615   return 0;
1616 }
1617
1618 /**
1619   Handle a RDTSC event.
1620
1621   Use the VMGEXIT instruction to handle a RDTSC event.
1622
1623   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1624                                    Block
1625   @param[in, out] Regs             x64 processor context
1626   @param[in]      InstructionData  Instruction parsing context
1627
1628   @retval 0                        Event handled successfully
1629   @return                          New exception value to propagate
1630
1631 **/
1632 STATIC
1633 UINT64
1634 RdtscExit (
1635   IN OUT GHCB                     *Ghcb,
1636   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1637   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1638   )
1639 {
1640   UINT64  Status;
1641
1642   Status = VmgExit (Ghcb, SVM_EXIT_RDTSC, 0, 0);
1643   if (Status != 0) {
1644     return Status;
1645   }
1646
1647   if (!VmgIsOffsetValid (Ghcb, GhcbRax) ||
1648       !VmgIsOffsetValid (Ghcb, GhcbRdx))
1649   {
1650     return UnsupportedExit (Ghcb, Regs, InstructionData);
1651   }
1652
1653   Regs->Rax = Ghcb->SaveArea.Rax;
1654   Regs->Rdx = Ghcb->SaveArea.Rdx;
1655
1656   return 0;
1657 }
1658
1659 /**
1660   Handle a DR7 register write event.
1661
1662   Use the VMGEXIT instruction to handle a DR7 write event.
1663
1664   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1665                                    Block
1666   @param[in, out] Regs             x64 processor context
1667   @param[in]      InstructionData  Instruction parsing context
1668
1669   @retval 0                        Event handled successfully
1670   @return                          New exception value to propagate
1671
1672 **/
1673 STATIC
1674 UINT64
1675 Dr7WriteExit (
1676   IN OUT GHCB                     *Ghcb,
1677   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1678   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1679   )
1680 {
1681   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
1682   SEV_ES_PER_CPU_DATA            *SevEsData;
1683   UINT64                         *Register;
1684   UINT64                         Status;
1685
1686   Ext       = &InstructionData->Ext;
1687   SevEsData = (SEV_ES_PER_CPU_DATA *)(Ghcb + 1);
1688
1689   DecodeModRm (Regs, InstructionData);
1690
1691   //
1692   // MOV DRn always treats MOD == 3 no matter how encoded
1693   //
1694   Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
1695
1696   //
1697   // Using a value of 0 for ExitInfo1 means RAX holds the value
1698   //
1699   Ghcb->SaveArea.Rax = *Register;
1700   VmgSetOffsetValid (Ghcb, GhcbRax);
1701
1702   Status = VmgExit (Ghcb, SVM_EXIT_DR7_WRITE, 0, 0);
1703   if (Status != 0) {
1704     return Status;
1705   }
1706
1707   SevEsData->Dr7       = *Register;
1708   SevEsData->Dr7Cached = 1;
1709
1710   return 0;
1711 }
1712
1713 /**
1714   Handle a DR7 register read event.
1715
1716   Use the VMGEXIT instruction to handle a DR7 read event.
1717
1718   @param[in, out] Ghcb             Pointer to the Guest-Hypervisor Communication
1719                                    Block
1720   @param[in, out] Regs             x64 processor context
1721   @param[in]      InstructionData  Instruction parsing context
1722
1723   @retval 0                        Event handled successfully
1724
1725 **/
1726 STATIC
1727 UINT64
1728 Dr7ReadExit (
1729   IN OUT GHCB                     *Ghcb,
1730   IN OUT EFI_SYSTEM_CONTEXT_X64   *Regs,
1731   IN     SEV_ES_INSTRUCTION_DATA  *InstructionData
1732   )
1733 {
1734   SEV_ES_INSTRUCTION_OPCODE_EXT  *Ext;
1735   SEV_ES_PER_CPU_DATA            *SevEsData;
1736   UINT64                         *Register;
1737
1738   Ext       = &InstructionData->Ext;
1739   SevEsData = (SEV_ES_PER_CPU_DATA *)(Ghcb + 1);
1740
1741   DecodeModRm (Regs, InstructionData);
1742
1743   //
1744   // MOV DRn always treats MOD == 3 no matter how encoded
1745   //
1746   Register = GetRegisterPointer (Regs, Ext->ModRm.Rm);
1747
1748   //
1749   // If there is a cached valued for DR7, return that. Otherwise return the
1750   // DR7 standard reset value of 0x400 (no debug breakpoints set).
1751   //
1752   *Register = (SevEsData->Dr7Cached == 1) ? SevEsData->Dr7 : 0x400;
1753
1754   return 0;
1755 }
1756
1757 /**
1758   Handle a #VC exception.
1759
1760   Performs the necessary processing to handle a #VC exception.
1761
1762   @param[in, out]  Ghcb           Pointer to the GHCB
1763   @param[in, out]  ExceptionType  Pointer to an EFI_EXCEPTION_TYPE to be set
1764                                   as value to use on error.
1765   @param[in, out]  SystemContext  Pointer to EFI_SYSTEM_CONTEXT
1766
1767   @retval  EFI_SUCCESS            Exception handled
1768   @retval  EFI_UNSUPPORTED        #VC not supported, (new) exception value to
1769                                   propagate provided
1770   @retval  EFI_PROTOCOL_ERROR     #VC handling failed, (new) exception value to
1771                                   propagate provided
1772
1773 **/
1774 EFI_STATUS
1775 EFIAPI
1776 InternalVmgExitHandleVc (
1777   IN OUT GHCB                *Ghcb,
1778   IN OUT EFI_EXCEPTION_TYPE  *ExceptionType,
1779   IN OUT EFI_SYSTEM_CONTEXT  SystemContext
1780   )
1781 {
1782   EFI_SYSTEM_CONTEXT_X64   *Regs;
1783   NAE_EXIT                 NaeExit;
1784   SEV_ES_INSTRUCTION_DATA  InstructionData;
1785   UINT64                   ExitCode, Status;
1786   EFI_STATUS               VcRet;
1787   BOOLEAN                  InterruptState;
1788
1789   VcRet = EFI_SUCCESS;
1790
1791   Regs = SystemContext.SystemContextX64;
1792
1793   VmgInit (Ghcb, &InterruptState);
1794
1795   ExitCode = Regs->ExceptionData;
1796   switch (ExitCode) {
1797     case SVM_EXIT_DR7_READ:
1798       NaeExit = Dr7ReadExit;
1799       break;
1800
1801     case SVM_EXIT_DR7_WRITE:
1802       NaeExit = Dr7WriteExit;
1803       break;
1804
1805     case SVM_EXIT_RDTSC:
1806       NaeExit = RdtscExit;
1807       break;
1808
1809     case SVM_EXIT_RDPMC:
1810       NaeExit = RdpmcExit;
1811       break;
1812
1813     case SVM_EXIT_CPUID:
1814       NaeExit = CpuidExit;
1815       break;
1816
1817     case SVM_EXIT_INVD:
1818       NaeExit = InvdExit;
1819       break;
1820
1821     case SVM_EXIT_IOIO_PROT:
1822       NaeExit = IoioExit;
1823       break;
1824
1825     case SVM_EXIT_MSR:
1826       NaeExit = MsrExit;
1827       break;
1828
1829     case SVM_EXIT_VMMCALL:
1830       NaeExit = VmmCallExit;
1831       break;
1832
1833     case SVM_EXIT_RDTSCP:
1834       NaeExit = RdtscpExit;
1835       break;
1836
1837     case SVM_EXIT_WBINVD:
1838       NaeExit = WbinvdExit;
1839       break;
1840
1841     case SVM_EXIT_MONITOR:
1842       NaeExit = MonitorExit;
1843       break;
1844
1845     case SVM_EXIT_MWAIT:
1846       NaeExit = MwaitExit;
1847       break;
1848
1849     case SVM_EXIT_NPF:
1850       NaeExit = MmioExit;
1851       break;
1852
1853     default:
1854       NaeExit = UnsupportedExit;
1855   }
1856
1857   InitInstructionData (&InstructionData, Ghcb, Regs);
1858
1859   Status = NaeExit (Ghcb, Regs, &InstructionData);
1860   if (Status == 0) {
1861     Regs->Rip += InstructionLength (&InstructionData);
1862   } else {
1863     GHCB_EVENT_INJECTION  Event;
1864
1865     Event.Uint64 = Status;
1866     if (Event.Elements.ErrorCodeValid != 0) {
1867       Regs->ExceptionData = Event.Elements.ErrorCode;
1868     } else {
1869       Regs->ExceptionData = 0;
1870     }
1871
1872     *ExceptionType = Event.Elements.Vector;
1873
1874     VcRet = EFI_PROTOCOL_ERROR;
1875   }
1876
1877   VmgDone (Ghcb, InterruptState);
1878
1879   return VcRet;
1880 }
1881
1882 /**
1883   Routine to allow ASSERT from within #VC.
1884
1885   @param[in, out]  SevEsData  Pointer to the per-CPU data
1886
1887 **/
1888 VOID
1889 EFIAPI
1890 VmgExitIssueAssert (
1891   IN OUT SEV_ES_PER_CPU_DATA  *SevEsData
1892   )
1893 {
1894   //
1895   // Progress will be halted, so set VcCount to allow for ASSERT output
1896   // to be seen.
1897   //
1898   SevEsData->VcCount = 0;
1899
1900   ASSERT (FALSE);
1901   CpuDeadLoop ();
1902 }