[mirror_edk2.git] / MdeModulePkg / Bus / Pci / NonDiscoverablePciDeviceDxe / NonDiscoverablePciDeviceIo.c

/** @file\r
\r
  Copyright (c) 2008 - 2009, Apple Inc. All rights reserved.<BR>\r
  Copyright (c) 2016, Linaro, Ltd. All rights reserved.<BR>\r
\r
  SPDX-License-Identifier: BSD-2-Clause-Patent\r
\r
**/\r
\r
#include "NonDiscoverablePciDeviceIo.h"\r
\r
#include <Library/DxeServicesTableLib.h>\r
\r
#include <IndustryStandard/Acpi.h>\r
\r
#include <Protocol/PciRootBridgeIo.h>\r
\r
typedef struct {\r
  EFI_PHYSICAL_ADDRESS            AllocAddress;\r
  VOID                            *HostAddress;\r
  EFI_PCI_IO_PROTOCOL_OPERATION   Operation;\r
  UINTN                           NumberOfBytes;\r
} NON_DISCOVERABLE_PCI_DEVICE_MAP_INFO;\r
\r
/**\r
  Get the resource associated with BAR number 'BarIndex'.\r
\r
  @param  Dev           Point to the NON_DISCOVERABLE_PCI_DEVICE instance.\r
  @param  BarIndex      The BAR index of the standard PCI Configuration header to use as the\r
                        base address for the memory operation to perform.\r
  @param  Descriptor    Points to the address space descriptor\r
**/\r
STATIC\r
EFI_STATUS\r
GetBarResource (\r
  IN  NON_DISCOVERABLE_PCI_DEVICE         *Dev,\r
  IN  UINT8                               BarIndex,\r
  OUT EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   **Descriptor\r
  )\r
{\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   *Desc;\r
\r
  if (BarIndex < Dev->BarOffset) {\r
    return EFI_NOT_FOUND;\r
  }\r
\r
  BarIndex -= (UINT8)Dev->BarOffset;\r
\r
  if (BarIndex >= Dev->BarCount) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  for (Desc = Dev->Device->Resources;\r
       Desc->Desc != ACPI_END_TAG_DESCRIPTOR;\r
       Desc = (VOID *)((UINT8 *)Desc + Desc->Len + 3)) {\r
\r
    if (BarIndex == 0) {\r
      *Descriptor = Desc;\r
      return EFI_SUCCESS;\r
    }\r
\r
    BarIndex -= 1;\r
  }\r
  return EFI_NOT_FOUND;\r
}\r
\r
/**\r
  Reads from the memory space of a PCI controller. Returns either when the polling exit criteria is\r
  satisfied or after a defined duration.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Width                 Signifies the width of the memory or I/O operations.\r
  @param  BarIndex              The BAR index of the standard PCI Configuration header to use as the\r
                                base address for the memory operation to perform.\r
  @param  Offset                The offset within the selected BAR to start the memory operation.\r
  @param  Mask                  Mask used for the polling criteria.\r
  @param  Value                 The comparison value used for the polling exit criteria.\r
  @param  Delay                 The number of 100 ns units to poll.\r
  @param  Result                Pointer to the last value read from the memory location.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoPollMem (\r
  IN  EFI_PCI_IO_PROTOCOL         *This,\r
  IN  EFI_PCI_IO_PROTOCOL_WIDTH   Width,\r
  IN  UINT8                       BarIndex,\r
  IN  UINT64                      Offset,\r
  IN  UINT64                      Mask,\r
  IN  UINT64                      Value,\r
  IN  UINT64                      Delay,\r
  OUT UINT64                      *Result\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE         *Dev;\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   *Desc;\r
  UINTN                               Count;\r
  EFI_STATUS                          Status;\r
\r
  if ((UINT32)Width > EfiPciIoWidthUint64) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  if (Result == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
  Count = 1;\r
\r
  Status = GetBarResource (Dev, BarIndex, &Desc);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  if (Offset + (Count << (Width & 0x3)) > Desc->AddrLen) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  ASSERT (FALSE);\r
  return EFI_UNSUPPORTED;\r
}\r
\r
/**\r
  Reads from the memory space of a PCI controller. Returns either when the polling exit criteria is\r
  satisfied or after a defined duration.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Width                 Signifies the width of the memory or I/O operations.\r
  @param  BarIndex              The BAR index of the standard PCI Configuration header to use as the\r
                                base address for the memory operation to perform.\r
  @param  Offset                The offset within the selected BAR to start the memory operation.\r
  @param  Mask                  Mask used for the polling criteria.\r
  @param  Value                 The comparison value used for the polling exit criteria.\r
  @param  Delay                 The number of 100 ns units to poll.\r
  @param  Result                Pointer to the last value read from the memory location.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoPollIo (\r
  IN  EFI_PCI_IO_PROTOCOL         *This,\r
  IN  EFI_PCI_IO_PROTOCOL_WIDTH   Width,\r
  IN  UINT8                       BarIndex,\r
  IN  UINT64                      Offset,\r
  IN  UINT64                      Mask,\r
  IN  UINT64                      Value,\r
  IN  UINT64                      Delay,\r
  OUT UINT64                      *Result\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE         *Dev;\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   *Desc;\r
  UINTN                               Count;\r
  EFI_STATUS                          Status;\r
\r
  if ((UINT32)Width > EfiPciIoWidthUint64) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  if (Result == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
  Count = 1;\r
\r
  Status = GetBarResource (Dev, BarIndex, &Desc);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  if (Offset + (Count << (Width & 0x3)) > Desc->AddrLen) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  ASSERT (FALSE);\r
  return EFI_UNSUPPORTED;\r
}\r
\r
/**\r
  Enable a PCI driver to access PCI controller registers in the PCI memory or I/O space.\r
\r
  @param  Width         Signifies the width of the memory or I/O operations.\r
  @param  Count         The number of memory or I/O operations to perform.\r
  @param  DstStride     The stride of the destination buffer.\r
  @param  Dst           For read operations, the destination buffer to store the results. For write\r
                        operations, the destination buffer to write data to.\r
  @param  SrcStride     The stride of the source buffer.\r
  @param  Src           For read operations, the source buffer to read data from. For write\r
                        operations, the source buffer to write data from.\r
\r
  @retval EFI_SUCCESS            The data was read from or written to the PCI controller.\r
  @retval EFI_INVALID_PARAMETER  One or more parameters are invalid.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoMemRW (\r
  IN  EFI_PCI_IO_PROTOCOL_WIDTH   Width,\r
  IN  UINTN                       Count,\r
  IN  UINTN                       DstStride,\r
  IN  VOID                        *Dst,\r
  IN  UINTN                       SrcStride,\r
  OUT CONST VOID                  *Src\r
  )\r
{\r
  volatile UINT8             *Dst8;\r
  volatile UINT16            *Dst16;\r
  volatile UINT32            *Dst32;\r
  volatile CONST UINT8       *Src8;\r
  volatile CONST UINT16      *Src16;\r
  volatile CONST UINT32      *Src32;\r
\r
  //\r
  // Loop for each iteration and move the data\r
  //\r
  switch (Width & 0x3) {\r
  case EfiPciWidthUint8:\r
    Dst8 = (UINT8 *)Dst;\r
    Src8 = (UINT8 *)Src;\r
    for (;Count > 0; Count--, Dst8 += DstStride, Src8 += SrcStride) {\r
      *Dst8 = *Src8;\r
    }\r
    break;\r
  case EfiPciWidthUint16:\r
    Dst16 = (UINT16 *)Dst;\r
    Src16 = (UINT16 *)Src;\r
    for (;Count > 0; Count--, Dst16 += DstStride, Src16 += SrcStride) {\r
      *Dst16 = *Src16;\r
    }\r
    break;\r
  case EfiPciWidthUint32:\r
    Dst32 = (UINT32 *)Dst;\r
    Src32 = (UINT32 *)Src;\r
    for (;Count > 0; Count--, Dst32 += DstStride, Src32 += SrcStride) {\r
      *Dst32 = *Src32;\r
    }\r
    break;\r
  default:\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Enable a PCI driver to access PCI controller registers in the PCI memory or I/O space.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Width                 Signifies the width of the memory or I/O operations.\r
  @param  BarIndex              The BAR index of the standard PCI Configuration header to use as the\r
                                base address for the memory or I/O operation to perform.\r
  @param  Offset                The offset within the selected BAR to start the memory or I/O operation.\r
  @param  Count                 The number of memory or I/O operations to perform.\r
  @param  Buffer                For read operations, the destination buffer to store the results. For write\r
                                operations, the source buffer to write data from.\r
\r
  @retval EFI_SUCCESS           The data was read from or written to the PCI controller.\r
  @retval EFI_UNSUPPORTED       BarIndex not valid for this PCI controller.\r
  @retval EFI_UNSUPPORTED       The address range specified by Offset, Width, and Count is not\r
                                valid for the PCI BAR specified by BarIndex.\r
  @retval EFI_OUT_OF_RESOURCES  The request could not be completed due to a lack of resources.\r
  @retval EFI_INVALID_PARAMETER One or more parameters are invalid.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoMemRead (\r
  IN     EFI_PCI_IO_PROTOCOL          *This,\r
  IN     EFI_PCI_IO_PROTOCOL_WIDTH    Width,\r
  IN     UINT8                        BarIndex,\r
  IN     UINT64                       Offset,\r
  IN     UINTN                        Count,\r
  IN OUT VOID                         *Buffer\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE         *Dev;\r
  UINTN                               AlignMask;\r
  VOID                                *Address;\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   *Desc;\r
  EFI_STATUS                          Status;\r
\r
  if (Buffer == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
\r
  //\r
  // Only allow accesses to the BARs we emulate\r
  //\r
  Status = GetBarResource (Dev, BarIndex, &Desc);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  if (Offset + (Count << (Width & 0x3)) > Desc->AddrLen) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  Address = (VOID *)(UINTN)(Desc->AddrRangeMin + Offset);\r
  AlignMask = (1 << (Width & 0x03)) - 1;\r
  if ((UINTN)Address & AlignMask) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  switch (Width) {\r
  case EfiPciIoWidthUint8:\r
  case EfiPciIoWidthUint16:\r
  case EfiPciIoWidthUint32:\r
  case EfiPciIoWidthUint64:\r
    return PciIoMemRW (Width, Count, 1, Buffer, 1, Address);\r
\r
  case EfiPciIoWidthFifoUint8:\r
  case EfiPciIoWidthFifoUint16:\r
  case EfiPciIoWidthFifoUint32:\r
  case EfiPciIoWidthFifoUint64:\r
    return PciIoMemRW (Width, Count, 1, Buffer, 0, Address);\r
\r
  case EfiPciIoWidthFillUint8:\r
  case EfiPciIoWidthFillUint16:\r
  case EfiPciIoWidthFillUint32:\r
  case EfiPciIoWidthFillUint64:\r
    return PciIoMemRW (Width, Count, 0, Buffer, 1, Address);\r
\r
  default:\r
    break;\r
  }\r
  return EFI_INVALID_PARAMETER;\r
}\r
\r
/**\r
  Enable a PCI driver to access PCI controller registers in the PCI memory or I/O space.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Width                 Signifies the width of the memory or I/O operations.\r
  @param  BarIndex              The BAR index of the standard PCI Configuration header to use as the\r
                                base address for the memory or I/O operation to perform.\r
  @param  Offset                The offset within the selected BAR to start the memory or I/O operation.\r
  @param  Count                 The number of memory or I/O operations to perform.\r
  @param  Buffer                For read operations, the destination buffer to store the results. For write\r
                                operations, the source buffer to write data from.\r
\r
  @retval EFI_SUCCESS           The data was read from or written to the PCI controller.\r
  @retval EFI_UNSUPPORTED       BarIndex not valid for this PCI controller.\r
  @retval EFI_UNSUPPORTED       The address range specified by Offset, Width, and Count is not\r
                                valid for the PCI BAR specified by BarIndex.\r
  @retval EFI_OUT_OF_RESOURCES  The request could not be completed due to a lack of resources.\r
  @retval EFI_INVALID_PARAMETER One or more parameters are invalid.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoMemWrite (\r
  IN     EFI_PCI_IO_PROTOCOL          *This,\r
  IN     EFI_PCI_IO_PROTOCOL_WIDTH    Width,\r
  IN     UINT8                        BarIndex,\r
  IN     UINT64                       Offset,\r
  IN     UINTN                        Count,\r
  IN OUT VOID                         *Buffer\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE         *Dev;\r
  UINTN                               AlignMask;\r
  VOID                                *Address;\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   *Desc;\r
  EFI_STATUS                          Status;\r
\r
  if (Buffer == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
\r
  //\r
  // Only allow accesses to the BARs we emulate\r
  //\r
  Status = GetBarResource (Dev, BarIndex, &Desc);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  if (Offset + (Count << (Width & 0x3)) > Desc->AddrLen) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  Address = (VOID *)(UINTN)(Desc->AddrRangeMin + Offset);\r
  AlignMask = (1 << (Width & 0x03)) - 1;\r
  if ((UINTN)Address & AlignMask) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  switch (Width) {\r
  case EfiPciIoWidthUint8:\r
  case EfiPciIoWidthUint16:\r
  case EfiPciIoWidthUint32:\r
  case EfiPciIoWidthUint64:\r
    return PciIoMemRW (Width, Count, 1, Address, 1, Buffer);\r
\r
  case EfiPciIoWidthFifoUint8:\r
  case EfiPciIoWidthFifoUint16:\r
  case EfiPciIoWidthFifoUint32:\r
  case EfiPciIoWidthFifoUint64:\r
    return PciIoMemRW (Width, Count, 0, Address, 1, Buffer);\r
\r
  case EfiPciIoWidthFillUint8:\r
  case EfiPciIoWidthFillUint16:\r
  case EfiPciIoWidthFillUint32:\r
  case EfiPciIoWidthFillUint64:\r
    return PciIoMemRW (Width, Count, 1, Address, 0, Buffer);\r
\r
  default:\r
    break;\r
  }\r
  return EFI_INVALID_PARAMETER;\r
}\r
\r
/**\r
  Enable a PCI driver to access PCI controller registers in the PCI memory or I/O space.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Width                 Signifies the width of the memory or I/O operations.\r
  @param  BarIndex              The BAR index of the standard PCI Configuration header to use as the\r
                                base address for the memory or I/O operation to perform.\r
  @param  Offset                The offset within the selected BAR to start the memory or I/O operation.\r
  @param  Count                 The number of memory or I/O operations to perform.\r
  @param  Buffer                For read operations, the destination buffer to store the results. For write\r
                                operations, the source buffer to write data from.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoIoRead (\r
  IN EFI_PCI_IO_PROTOCOL              *This,\r
  IN     EFI_PCI_IO_PROTOCOL_WIDTH    Width,\r
  IN     UINT8                        BarIndex,\r
  IN     UINT64                       Offset,\r
  IN     UINTN                        Count,\r
  IN OUT VOID                         *Buffer\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE         *Dev;\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   *Desc;\r
  EFI_STATUS                          Status;\r
\r
  if ((UINT32)Width >= EfiPciIoWidthMaximum) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  if (Buffer == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
\r
  Status = GetBarResource (Dev, BarIndex, &Desc);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  if (Offset + (Count << (Width & 0x3)) > Desc->AddrLen) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  ASSERT (FALSE);\r
  return EFI_UNSUPPORTED;\r
}\r
\r
/**\r
  Enable a PCI driver to access PCI controller registers in the PCI memory or I/O space.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Width                 Signifies the width of the memory or I/O operations.\r
  @param  BarIndex              The BAR index of the standard PCI Configuration header to use as the\r
                                base address for the memory or I/O operation to perform.\r
  @param  Offset                The offset within the selected BAR to start the memory or I/O operation.\r
  @param  Count                 The number of memory or I/O operations to perform.\r
  @param  Buffer                For read operations, the destination buffer to store the results. For write\r
                                operations, the source buffer to write data from.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoIoWrite (\r
  IN     EFI_PCI_IO_PROTOCOL          *This,\r
  IN     EFI_PCI_IO_PROTOCOL_WIDTH    Width,\r
  IN     UINT8                        BarIndex,\r
  IN     UINT64                       Offset,\r
  IN     UINTN                        Count,\r
  IN OUT VOID                         *Buffer\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE         *Dev;\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   *Desc;\r
  EFI_STATUS                          Status;\r
\r
  if ((UINT32)Width >= EfiPciIoWidthMaximum) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  if (Buffer == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
\r
  Status = GetBarResource (Dev, BarIndex, &Desc);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  if (Offset + (Count << (Width & 0x3)) > Desc->AddrLen) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  ASSERT (FALSE);\r
  return EFI_UNSUPPORTED;\r
}\r
\r
/**\r
  Enable a PCI driver to access PCI config space.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Width                 Signifies the width of the memory or I/O operations.\r
  @param  Offset                The offset within the selected BAR to start the memory or I/O operation.\r
  @param  Count                 The number of memory or I/O operations to perform.\r
  @param  Buffer                For read operations, the destination buffer to store the results. For write\r
                                operations, the source buffer to write data from.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoPciRead (\r
  IN     EFI_PCI_IO_PROTOCOL        *This,\r
  IN     EFI_PCI_IO_PROTOCOL_WIDTH  Width,\r
  IN     UINT32                     Offset,\r
  IN     UINTN                      Count,\r
  IN OUT VOID                       *Buffer\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE   *Dev;\r
  VOID                          *Address;\r
  UINTN                         Length;\r
\r
  if (Width < 0 || Width >= EfiPciIoWidthMaximum || Buffer == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
  Address = (UINT8 *)&Dev->ConfigSpace + Offset;\r
  Length = Count << ((UINTN)Width & 0x3);\r
\r
  if (Offset >= sizeof (Dev->ConfigSpace)) {\r
    ZeroMem (Buffer, Length);\r
    return EFI_SUCCESS;\r
  }\r
\r
  if (Offset + Length > sizeof (Dev->ConfigSpace)) {\r
    //\r
    // Read all zeroes for config space accesses beyond the first\r
    // 64 bytes\r
    //\r
    Length -= sizeof (Dev->ConfigSpace) - Offset;\r
    ZeroMem ((UINT8 *)Buffer + sizeof (Dev->ConfigSpace) - Offset, Length);\r
\r
    Count -= Length >> ((UINTN)Width & 0x3);\r
  }\r
  return PciIoMemRW (Width, Count, 1, Buffer, 1, Address);\r
}\r
\r
/**\r
  Enable a PCI driver to access PCI config space.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Width                 Signifies the width of the memory or I/O operations.\r
  @param  Offset                The offset within the selected BAR to start the memory or I/O operation.\r
  @param  Count                 The number of memory or I/O operations to perform.\r
  @param  Buffer                For read operations, the destination buffer to store the results. For write\r
                                operations, the source buffer to write data from\r
\r
  @retval EFI_SUCCESS           The data was read from or written to the PCI controller.\r
  @retval EFI_UNSUPPORTED       The address range specified by Offset, Width, and Count is not\r
                                valid for the PCI BAR specified by BarIndex.\r
  @retval EFI_INVALID_PARAMETER One or more parameters are invalid.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoPciWrite (\r
  IN EFI_PCI_IO_PROTOCOL              *This,\r
  IN     EFI_PCI_IO_PROTOCOL_WIDTH    Width,\r
  IN     UINT32                       Offset,\r
  IN     UINTN                        Count,\r
  IN OUT VOID                         *Buffer\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE   *Dev;\r
  VOID                          *Address;\r
\r
  if (Width < 0 || Width >= EfiPciIoWidthMaximum || Buffer == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
  Address = (UINT8 *)&Dev->ConfigSpace + Offset;\r
\r
  if (Offset + (Count << ((UINTN)Width & 0x3)) > sizeof (Dev->ConfigSpace)) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  return PciIoMemRW (Width, Count, 1, Address, 1, Buffer);\r
}\r
\r
/**\r
  Enables a PCI driver to copy one region of PCI memory space to another region of PCI\r
  memory space.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Width                 Signifies the width of the memory operations.\r
  @param  DestBarIndex          The BAR index in the standard PCI Configuration header to use as the\r
                                base address for the memory operation to perform.\r
  @param  DestOffset            The destination offset within the BAR specified by DestBarIndex to\r
                                start the memory writes for the copy operation.\r
  @param  SrcBarIndex           The BAR index in the standard PCI Configuration header to use as the\r
                                base address for the memory operation to perform.\r
  @param  SrcOffset             The source offset within the BAR specified by SrcBarIndex to start\r
                                the memory reads for the copy operation.\r
  @param  Count                 The number of memory operations to perform. Bytes moved is Width\r
                                size * Count, starting at DestOffset and SrcOffset.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoCopyMem (\r
  IN EFI_PCI_IO_PROTOCOL              *This,\r
  IN     EFI_PCI_IO_PROTOCOL_WIDTH    Width,\r
  IN     UINT8                        DestBarIndex,\r
  IN     UINT64                       DestOffset,\r
  IN     UINT8                        SrcBarIndex,\r
  IN     UINT64                       SrcOffset,\r
  IN     UINTN                        Count\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE         *Dev;\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   *DestDesc;\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   *SrcDesc;\r
  EFI_STATUS                          Status;\r
\r
  if ((UINT32)Width > EfiPciIoWidthUint64) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
\r
  Status = GetBarResource (Dev, DestBarIndex, &DestDesc);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  if (DestOffset + (Count << (Width & 0x3)) > DestDesc->AddrLen) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  Status = GetBarResource (Dev, SrcBarIndex, &SrcDesc);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  if (SrcOffset + (Count << (Width & 0x3)) > SrcDesc->AddrLen) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  ASSERT (FALSE);\r
  return EFI_UNSUPPORTED;\r
}\r
\r
/**\r
  Provides the PCI controller-specific addresses needed to access system memory.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Operation             Indicates if the bus master is going to read or write to system memory.\r
  @param  HostAddress           The system memory address to map to the PCI controller.\r
  @param  NumberOfBytes         On input the number of bytes to map. On output the number of bytes\r
                                that were mapped.\r
  @param  DeviceAddress         The resulting map address for the bus master PCI controller to use to\r
                                access the hosts HostAddress.\r
  @param  Mapping               A resulting value to pass to Unmap().\r
\r
  @retval EFI_SUCCESS           The range was mapped for the returned NumberOfBytes.\r
  @retval EFI_UNSUPPORTED       The HostAddress cannot be mapped as a common buffer.\r
  @retval EFI_INVALID_PARAMETER One or more parameters are invalid.\r
  @retval EFI_OUT_OF_RESOURCES  The request could not be completed due to a lack of resources.\r
  @retval EFI_DEVICE_ERROR      The system hardware could not map the requested address.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
CoherentPciIoMap (\r
  IN     EFI_PCI_IO_PROTOCOL            *This,\r
  IN     EFI_PCI_IO_PROTOCOL_OPERATION  Operation,\r
  IN     VOID                           *HostAddress,\r
  IN OUT UINTN                          *NumberOfBytes,\r
  OUT    EFI_PHYSICAL_ADDRESS           *DeviceAddress,\r
  OUT    VOID                           **Mapping\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE           *Dev;\r
  EFI_STATUS                            Status;\r
  NON_DISCOVERABLE_PCI_DEVICE_MAP_INFO  *MapInfo;\r
\r
  if (Operation != EfiPciIoOperationBusMasterRead &&\r
      Operation != EfiPciIoOperationBusMasterWrite &&\r
      Operation != EfiPciIoOperationBusMasterCommonBuffer) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  if (HostAddress   == NULL ||\r
      NumberOfBytes == NULL ||\r
      DeviceAddress == NULL ||\r
      Mapping       == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  //\r
  // If HostAddress exceeds 4 GB, and this device does not support 64-bit DMA\r
  // addressing, we need to allocate a bounce buffer and copy over the data.\r
  //\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
  if ((Dev->Attributes & EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE) == 0 &&\r
      (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress + *NumberOfBytes > SIZE_4GB) {\r
\r
    //\r
    // Bounce buffering is not possible for consistent mappings\r
    //\r
    if (Operation == EfiPciIoOperationBusMasterCommonBuffer) {\r
      return EFI_UNSUPPORTED;\r
    }\r
\r
    MapInfo = AllocatePool (sizeof *MapInfo);\r
    if (MapInfo == NULL) {\r
      return EFI_OUT_OF_RESOURCES;\r
    }\r
\r
    MapInfo->AllocAddress = MAX_UINT32;\r
    MapInfo->HostAddress = HostAddress;\r
    MapInfo->Operation = Operation;\r
    MapInfo->NumberOfBytes = *NumberOfBytes;\r
\r
    Status = gBS->AllocatePages (AllocateMaxAddress, EfiBootServicesData,\r
                    EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes),\r
                    &MapInfo->AllocAddress);\r
    if (EFI_ERROR (Status)) {\r
      //\r
      // If we fail here, it is likely because the system has no memory below\r
      // 4 GB to begin with. There is not much we can do about that other than\r
      // fail the map request.\r
      //\r
      FreePool (MapInfo);\r
      return EFI_DEVICE_ERROR;\r
    }\r
    if (Operation == EfiPciIoOperationBusMasterRead) {\r
      gBS->CopyMem ((VOID *)(UINTN)MapInfo->AllocAddress, HostAddress,\r
             *NumberOfBytes);\r
    }\r
    *DeviceAddress = MapInfo->AllocAddress;\r
    *Mapping = MapInfo;\r
  } else {\r
    *DeviceAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress;\r
    *Mapping = NULL;\r
  }\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Completes the Map() operation and releases any corresponding resources.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Mapping               The mapping value returned from Map().\r
\r
  @retval EFI_SUCCESS           The range was unmapped.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
CoherentPciIoUnmap (\r
  IN  EFI_PCI_IO_PROTOCOL          *This,\r
  IN  VOID                         *Mapping\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE_MAP_INFO  *MapInfo;\r
\r
  MapInfo = Mapping;\r
  if (MapInfo != NULL) {\r
    if (MapInfo->Operation == EfiPciIoOperationBusMasterWrite) {\r
      gBS->CopyMem (MapInfo->HostAddress, (VOID *)(UINTN)MapInfo->AllocAddress,\r
             MapInfo->NumberOfBytes);\r
    }\r
    gBS->FreePages (MapInfo->AllocAddress,\r
           EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes));\r
    FreePool (MapInfo);\r
  }\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Allocates pages.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Type                  This parameter is not used and must be ignored.\r
  @param  MemoryType            The type of memory to allocate, EfiBootServicesData or\r
                                EfiRuntimeServicesData.\r
  @param  Pages                 The number of pages to allocate.\r
  @param  HostAddress           A pointer to store the base system memory address of the\r
                                allocated range.\r
  @param  Attributes            The requested bit mask of attributes for the allocated range.\r
\r
  @retval EFI_SUCCESS           The requested memory pages were allocated.\r
  @retval EFI_UNSUPPORTED       Attributes is unsupported. The only legal attribute bits are\r
                                MEMORY_WRITE_COMBINE and MEMORY_CACHED.\r
  @retval EFI_INVALID_PARAMETER One or more parameters are invalid.\r
  @retval EFI_OUT_OF_RESOURCES  The memory pages could not be allocated.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
CoherentPciIoAllocateBuffer (\r
  IN  EFI_PCI_IO_PROTOCOL         *This,\r
  IN  EFI_ALLOCATE_TYPE           Type,\r
  IN  EFI_MEMORY_TYPE             MemoryType,\r
  IN  UINTN                       Pages,\r
  OUT VOID                        **HostAddress,\r
  IN  UINT64                      Attributes\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE       *Dev;\r
  EFI_PHYSICAL_ADDRESS              AllocAddress;\r
  EFI_ALLOCATE_TYPE                 AllocType;\r
  EFI_STATUS                        Status;\r
\r
  if ((Attributes & ~(EFI_PCI_ATTRIBUTE_MEMORY_WRITE_COMBINE |\r
                      EFI_PCI_ATTRIBUTE_MEMORY_CACHED)) != 0) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  if (HostAddress == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  if ((MemoryType != EfiBootServicesData) &&\r
      (MemoryType != EfiRuntimeServicesData)) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  //\r
  // Allocate below 4 GB if the dual address cycle attribute has not\r
  // been set. If the system has no memory available below 4 GB, there\r
  // is little we can do except propagate the error.\r
  //\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
  if ((Dev->Attributes & EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE) == 0) {\r
    AllocAddress = MAX_UINT32;\r
    AllocType = AllocateMaxAddress;\r
  } else {\r
    AllocType = AllocateAnyPages;\r
  }\r
\r
  Status = gBS->AllocatePages (AllocType, MemoryType, Pages, &AllocAddress);\r
  if (!EFI_ERROR (Status)) {\r
    *HostAddress = (VOID *)(UINTN)AllocAddress;\r
  }\r
  return Status;\r
}\r
\r
/**\r
  Frees memory that was allocated in function CoherentPciIoAllocateBuffer ().\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Pages                 The number of pages to free.\r
  @param  HostAddress           The base system memory address of the allocated range.\r
\r
  @retval EFI_SUCCESS           The requested memory pages were freed.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
CoherentPciIoFreeBuffer (\r
  IN  EFI_PCI_IO_PROTOCOL         *This,\r
  IN  UINTN                       Pages,\r
  IN  VOID                        *HostAddress\r
  )\r
{\r
  FreePages (HostAddress, Pages);\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Frees memory that was allocated in function NonCoherentPciIoAllocateBuffer ().\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Pages                 The number of pages to free.\r
  @param  HostAddress           The base system memory address of the allocated range.\r
\r
  @retval EFI_SUCCESS           The requested memory pages were freed.\r
  @retval others                The operation contain some errors.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
NonCoherentPciIoFreeBuffer (\r
  IN  EFI_PCI_IO_PROTOCOL         *This,\r
  IN  UINTN                       Pages,\r
  IN  VOID                        *HostAddress\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE                   *Dev;\r
  LIST_ENTRY                                    *Entry;\r
  EFI_STATUS                                    Status;\r
  NON_DISCOVERABLE_DEVICE_UNCACHED_ALLOCATION   *Alloc;\r
  BOOLEAN                                       Found;\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
\r
  Found = FALSE;\r
  Alloc = NULL;\r
\r
  //\r
  // Find the uncached allocation list entry associated\r
  // with this allocation\r
  //\r
  for (Entry = Dev->UncachedAllocationList.ForwardLink;\r
       Entry != &Dev->UncachedAllocationList;\r
       Entry = Entry->ForwardLink) {\r
\r
    Alloc = BASE_CR (Entry, NON_DISCOVERABLE_DEVICE_UNCACHED_ALLOCATION, List);\r
    if (Alloc->HostAddress == HostAddress && Alloc->NumPages == Pages) {\r
      //\r
      // We are freeing the exact allocation we were given\r
      // before by AllocateBuffer()\r
      //\r
      Found = TRUE;\r
      break;\r
    }\r
  }\r
\r
  if (!Found) {\r
    ASSERT_EFI_ERROR (EFI_NOT_FOUND);\r
    return EFI_NOT_FOUND;\r
  }\r
\r
  RemoveEntryList (&Alloc->List);\r
\r
  Status = gDS->SetMemorySpaceAttributes (\r
                  (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress,\r
                  EFI_PAGES_TO_SIZE (Pages),\r
                  Alloc->Attributes);\r
  if (EFI_ERROR (Status)) {\r
    goto FreeAlloc;\r
  }\r
\r
  //\r
  // If we fail to restore the original attributes, it is better to leak the\r
  // memory than to return it to the heap\r
  //\r
  FreePages (HostAddress, Pages);\r
\r
FreeAlloc:\r
  FreePool (Alloc);\r
  return Status;\r
}\r
\r
/**\r
  Allocates pages.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Type                  This parameter is not used and must be ignored.\r
  @param  MemoryType            The type of memory to allocate, EfiBootServicesData or\r
                                EfiRuntimeServicesData.\r
  @param  Pages                 The number of pages to allocate.\r
  @param  HostAddress           A pointer to store the base system memory address of the\r
                                allocated range.\r
  @param  Attributes            The requested bit mask of attributes for the allocated range.\r
\r
  @retval EFI_SUCCESS           The requested memory pages were allocated.\r
  @retval EFI_UNSUPPORTED       Attributes is unsupported. The only legal attribute bits are\r
                                MEMORY_WRITE_COMBINE and MEMORY_CACHED.\r
  @retval EFI_INVALID_PARAMETER One or more parameters are invalid.\r
  @retval EFI_OUT_OF_RESOURCES  The memory pages could not be allocated.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
NonCoherentPciIoAllocateBuffer (\r
  IN  EFI_PCI_IO_PROTOCOL         *This,\r
  IN  EFI_ALLOCATE_TYPE           Type,\r
  IN  EFI_MEMORY_TYPE             MemoryType,\r
  IN  UINTN                       Pages,\r
  OUT VOID                        **HostAddress,\r
  IN  UINT64                      Attributes\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE                 *Dev;\r
  EFI_GCD_MEMORY_SPACE_DESCRIPTOR             GcdDescriptor;\r
  EFI_STATUS                                  Status;\r
  UINT64                                      MemType;\r
  NON_DISCOVERABLE_DEVICE_UNCACHED_ALLOCATION *Alloc;\r
  VOID                                        *AllocAddress;\r
\r
  if (HostAddress == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
\r
  Status = CoherentPciIoAllocateBuffer (This, Type, MemoryType, Pages,\r
             &AllocAddress, Attributes);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  Status = gDS->GetMemorySpaceDescriptor (\r
                  (EFI_PHYSICAL_ADDRESS)(UINTN)AllocAddress,\r
                  &GcdDescriptor);\r
  if (EFI_ERROR (Status)) {\r
    goto FreeBuffer;\r
  }\r
\r
  if ((GcdDescriptor.Capabilities & (EFI_MEMORY_WC | EFI_MEMORY_UC)) == 0) {\r
    Status = EFI_UNSUPPORTED;\r
    goto FreeBuffer;\r
  }\r
\r
  //\r
  // Set the preferred memory attributes\r
  //\r
  if ((Attributes & EFI_PCI_ATTRIBUTE_MEMORY_WRITE_COMBINE) != 0 ||\r
      (GcdDescriptor.Capabilities & EFI_MEMORY_UC) == 0) {\r
    //\r
    // Use write combining if it was requested, or if it is the only\r
    // type supported by the region.\r
    //\r
    MemType = EFI_MEMORY_WC;\r
  } else {\r
    MemType = EFI_MEMORY_UC;\r
  }\r
\r
  Alloc = AllocatePool (sizeof *Alloc);\r
  if (Alloc == NULL) {\r
    goto FreeBuffer;\r
  }\r
\r
  Alloc->HostAddress = AllocAddress;\r
  Alloc->NumPages = Pages;\r
  Alloc->Attributes = GcdDescriptor.Attributes;\r
\r
  //\r
  // Record this allocation in the linked list, so we\r
  // can restore the memory space attributes later\r
  //\r
  InsertHeadList (&Dev->UncachedAllocationList, &Alloc->List);\r
\r
  Status = gDS->SetMemorySpaceAttributes (\r
                  (EFI_PHYSICAL_ADDRESS)(UINTN)AllocAddress,\r
                  EFI_PAGES_TO_SIZE (Pages),\r
                  MemType);\r
  if (EFI_ERROR (Status)) {\r
    goto RemoveList;\r
  }\r
\r
  Status = mCpu->FlushDataCache (\r
                   mCpu,\r
                   (EFI_PHYSICAL_ADDRESS)(UINTN)AllocAddress,\r
                   EFI_PAGES_TO_SIZE (Pages),\r
                   EfiCpuFlushTypeInvalidate);\r
  if (EFI_ERROR (Status)) {\r
    goto RemoveList;\r
  }\r
\r
  *HostAddress = AllocAddress;\r
\r
  return EFI_SUCCESS;\r
\r
RemoveList:\r
  RemoveEntryList (&Alloc->List);\r
  FreePool (Alloc);\r
\r
FreeBuffer:\r
  CoherentPciIoFreeBuffer (This, Pages, AllocAddress);\r
  return Status;\r
}\r
\r
/**\r
  Provides the PCI controller-specific addresses needed to access system memory.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Operation             Indicates if the bus master is going to read or write to system memory.\r
  @param  HostAddress           The system memory address to map to the PCI controller.\r
  @param  NumberOfBytes         On input the number of bytes to map. On output the number of bytes\r
                                that were mapped.\r
  @param  DeviceAddress         The resulting map address for the bus master PCI controller to use to\r
                                access the hosts HostAddress.\r
  @param  Mapping               A resulting value to pass to Unmap().\r
\r
  @retval EFI_SUCCESS           The range was mapped for the returned NumberOfBytes.\r
  @retval EFI_UNSUPPORTED       The HostAddress cannot be mapped as a common buffer.\r
  @retval EFI_INVALID_PARAMETER One or more parameters are invalid.\r
  @retval EFI_OUT_OF_RESOURCES  The request could not be completed due to a lack of resources.\r
  @retval EFI_DEVICE_ERROR      The system hardware could not map the requested address.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
NonCoherentPciIoMap (\r
  IN     EFI_PCI_IO_PROTOCOL            *This,\r
  IN     EFI_PCI_IO_PROTOCOL_OPERATION  Operation,\r
  IN     VOID                           *HostAddress,\r
  IN OUT UINTN                          *NumberOfBytes,\r
  OUT    EFI_PHYSICAL_ADDRESS           *DeviceAddress,\r
  OUT    VOID                           **Mapping\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE           *Dev;\r
  EFI_STATUS                            Status;\r
  NON_DISCOVERABLE_PCI_DEVICE_MAP_INFO  *MapInfo;\r
  UINTN                                 AlignMask;\r
  VOID                                  *AllocAddress;\r
  EFI_GCD_MEMORY_SPACE_DESCRIPTOR       GcdDescriptor;\r
  BOOLEAN                               Bounce;\r
\r
  if (HostAddress   == NULL ||\r
      NumberOfBytes == NULL ||\r
      DeviceAddress == NULL ||\r
      Mapping       == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  if (Operation != EfiPciIoOperationBusMasterRead &&\r
      Operation != EfiPciIoOperationBusMasterWrite &&\r
      Operation != EfiPciIoOperationBusMasterCommonBuffer) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  MapInfo = AllocatePool (sizeof *MapInfo);\r
  if (MapInfo == NULL) {\r
    return EFI_OUT_OF_RESOURCES;\r
  }\r
\r
  MapInfo->HostAddress = HostAddress;\r
  MapInfo->Operation = Operation;\r
  MapInfo->NumberOfBytes = *NumberOfBytes;\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
\r
  //\r
  // If this device does not support 64-bit DMA addressing, we need to allocate\r
  // a bounce buffer and copy over the data in case HostAddress >= 4 GB.\r
  //\r
  Bounce = ((Dev->Attributes & EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE) == 0 &&\r
            (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress + *NumberOfBytes > SIZE_4GB);\r
\r
  if (!Bounce) {\r
    switch (Operation) {\r
    case EfiPciIoOperationBusMasterRead:\r
    case EfiPciIoOperationBusMasterWrite:\r
      //\r
      // For streaming DMA, it is sufficient if the buffer is aligned to\r
      // the CPUs DMA buffer alignment.\r
      //\r
      AlignMask = mCpu->DmaBufferAlignment - 1;\r
      if ((((UINTN) HostAddress | *NumberOfBytes) & AlignMask) == 0) {\r
        break;\r
      }\r
      // fall through\r
\r
    case EfiPciIoOperationBusMasterCommonBuffer:\r
      //\r
      // Check whether the host address refers to an uncached mapping.\r
      //\r
      Status = gDS->GetMemorySpaceDescriptor (\r
                      (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress,\r
                      &GcdDescriptor);\r
      if (EFI_ERROR (Status) ||\r
          (GcdDescriptor.Attributes & (EFI_MEMORY_WB|EFI_MEMORY_WT)) != 0) {\r
        Bounce = TRUE;\r
      }\r
      break;\r
\r
    default:\r
      ASSERT (FALSE);\r
    }\r
  }\r
\r
  if (Bounce) {\r
    if (Operation == EfiPciIoOperationBusMasterCommonBuffer) {\r
      Status = EFI_DEVICE_ERROR;\r
      goto FreeMapInfo;\r
    }\r
\r
    Status = NonCoherentPciIoAllocateBuffer (This, AllocateAnyPages,\r
               EfiBootServicesData, EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes),\r
               &AllocAddress, EFI_PCI_ATTRIBUTE_MEMORY_WRITE_COMBINE);\r
    if (EFI_ERROR (Status)) {\r
      goto FreeMapInfo;\r
    }\r
    MapInfo->AllocAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)AllocAddress;\r
    if (Operation == EfiPciIoOperationBusMasterRead) {\r
      gBS->CopyMem (AllocAddress, HostAddress, *NumberOfBytes);\r
    }\r
    *DeviceAddress = MapInfo->AllocAddress;\r
  } else {\r
    MapInfo->AllocAddress = 0;\r
    *DeviceAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress;\r
\r
    //\r
    // We are not using a bounce buffer: the mapping is sufficiently\r
    // aligned to allow us to simply flush the caches. Note that cleaning\r
    // the caches is necessary for both data directions:\r
    // - for bus master read, we want the latest data to be present\r
    //   in main memory\r
    // - for bus master write, we don't want any stale dirty cachelines that\r
    //   may be written back unexpectedly, and clobber the data written to\r
    //   main memory by the device.\r
    //\r
    mCpu->FlushDataCache (mCpu, (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress,\r
            *NumberOfBytes, EfiCpuFlushTypeWriteBack);\r
  }\r
\r
  *Mapping = MapInfo;\r
  return EFI_SUCCESS;\r
\r
FreeMapInfo:\r
  FreePool (MapInfo);\r
\r
  return Status;\r
}\r
\r
/**\r
  Completes the Map() operation and releases any corresponding resources.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Mapping               The mapping value returned from Map().\r
\r
  @retval EFI_SUCCESS           The range was unmapped.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
NonCoherentPciIoUnmap (\r
  IN  EFI_PCI_IO_PROTOCOL          *This,\r
  IN  VOID                         *Mapping\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE_MAP_INFO  *MapInfo;\r
\r
  if (Mapping == NULL) {\r
    return EFI_DEVICE_ERROR;\r
  }\r
\r
  MapInfo = Mapping;\r
  if (MapInfo->AllocAddress != 0) {\r
    //\r
    // We are using a bounce buffer: copy back the data if necessary,\r
    // and free the buffer.\r
    //\r
    if (MapInfo->Operation == EfiPciIoOperationBusMasterWrite) {\r
      gBS->CopyMem (MapInfo->HostAddress, (VOID *)(UINTN)MapInfo->AllocAddress,\r
             MapInfo->NumberOfBytes);\r
    }\r
    NonCoherentPciIoFreeBuffer (This,\r
      EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes),\r
      (VOID *)(UINTN)MapInfo->AllocAddress);\r
  } else {\r
    //\r
    // We are *not* using a bounce buffer: if this is a bus master write,\r
    // we have to invalidate the caches so the CPU will see the uncached\r
    // data written by the device.\r
    //\r
    if (MapInfo->Operation == EfiPciIoOperationBusMasterWrite) {\r
      mCpu->FlushDataCache (mCpu,\r
              (EFI_PHYSICAL_ADDRESS)(UINTN)MapInfo->HostAddress,\r
              MapInfo->NumberOfBytes, EfiCpuFlushTypeInvalidate);\r
    }\r
  }\r
  FreePool (MapInfo);\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Flushes all PCI posted write transactions from a PCI host bridge to system memory.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoFlush (\r
  IN EFI_PCI_IO_PROTOCOL          *This\r
  )\r
{\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Retrieves this PCI controller's current PCI bus number, device number, and function number.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  SegmentNumber         The PCI controller's current PCI segment number.\r
  @param  BusNumber             The PCI controller's current PCI bus number.\r
  @param  DeviceNumber          The PCI controller's current PCI device number.\r
  @param  FunctionNumber        The PCI controller's current PCI function number.\r
\r
  @retval EFI_SUCCESS           The PCI controller location was returned.\r
  @retval EFI_INVALID_PARAMETER One or more parameters are invalid.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoGetLocation (\r
  IN   EFI_PCI_IO_PROTOCOL  *This,\r
  OUT  UINTN                *SegmentNumber,\r
  OUT  UINTN                *BusNumber,\r
  OUT  UINTN                *DeviceNumber,\r
  OUT  UINTN                *FunctionNumber\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE         *Dev;\r
\r
  if (SegmentNumber == NULL ||\r
      BusNumber == NULL ||\r
      DeviceNumber == NULL ||\r
      FunctionNumber == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
\r
  *SegmentNumber  = 0xff;\r
  *BusNumber      = Dev->UniqueId >> 5;\r
  *DeviceNumber   = Dev->UniqueId & 0x1f;\r
  *FunctionNumber = 0;\r
\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Performs an operation on the attributes that this PCI controller supports. The operations include\r
  getting the set of supported attributes, retrieving the current attributes, setting the current\r
  attributes, enabling attributes, and disabling attributes.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Operation             The operation to perform on the attributes for this PCI controller.\r
  @param  Attributes            The mask of attributes that are used for Set, Enable, and Disable\r
                                operations.\r
  @param  Result                A pointer to the result mask of attributes that are returned for the Get\r
                                and Supported operations.\r
\r
  @retval EFI_SUCCESS           The operation on the PCI controller's attributes was completed.\r
  @retval EFI_INVALID_PARAMETER One or more parameters are invalid.\r
  @retval EFI_UNSUPPORTED       one or more of the bits set in\r
                                Attributes are not supported by this PCI controller or one of\r
                                its parent bridges when Operation is Set, Enable or Disable.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoAttributes (\r
  IN  EFI_PCI_IO_PROTOCOL                      *This,\r
  IN  EFI_PCI_IO_PROTOCOL_ATTRIBUTE_OPERATION  Operation,\r
  IN  UINT64                                   Attributes,\r
  OUT UINT64                                   *Result OPTIONAL\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE   *Dev;\r
  BOOLEAN                       Enable;\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
\r
  if ((Attributes & (~(DEV_SUPPORTED_ATTRIBUTES))) != 0) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  Enable = FALSE;\r
  switch (Operation) {\r
  case EfiPciIoAttributeOperationGet:\r
    if (Result == NULL) {\r
      return EFI_INVALID_PARAMETER;\r
    }\r
    *Result = Dev->Attributes;\r
    break;\r
\r
  case EfiPciIoAttributeOperationSupported:\r
    if (Result == NULL) {\r
      return EFI_INVALID_PARAMETER;\r
    }\r
    *Result = DEV_SUPPORTED_ATTRIBUTES;\r
    break;\r
\r
  case EfiPciIoAttributeOperationEnable:\r
    Attributes |= Dev->Attributes;\r
  case EfiPciIoAttributeOperationSet:\r
    Enable = ((~Dev->Attributes & Attributes) & EFI_PCI_DEVICE_ENABLE) != 0;\r
    Dev->Attributes = Attributes;\r
    break;\r
\r
  case EfiPciIoAttributeOperationDisable:\r
    Dev->Attributes &= ~Attributes;\r
    break;\r
\r
  default:\r
    return EFI_INVALID_PARAMETER;\r
  };\r
\r
  //\r
  // If we're setting any of the EFI_PCI_DEVICE_ENABLE bits, perform\r
  // the device specific initialization now.\r
  //\r
  if (Enable && !Dev->Enabled && Dev->Device->Initialize != NULL) {\r
    Dev->Device->Initialize (Dev->Device);\r
    Dev->Enabled = TRUE;\r
  }\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Gets the attributes that this PCI controller supports setting on a BAR using\r
  SetBarAttributes(), and retrieves the list of resource descriptors for a BAR.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  BarIndex              The BAR index of the standard PCI Configuration header to use as the\r
                                base address for resource range. The legal range for this field is 0..5.\r
  @param  Supports              A pointer to the mask of attributes that this PCI controller supports\r
                                setting for this BAR with SetBarAttributes().\r
  @param  Resources             A pointer to the ACPI 2.0 resource descriptors that describe the current\r
                                configuration of this BAR of the PCI controller.\r
\r
  @retval EFI_SUCCESS           If Supports is not NULL, then the attributes that the PCI\r
                                controller supports are returned in Supports. If Resources\r
                                is not NULL, then the ACPI 2.0 resource descriptors that the PCI\r
                                controller is currently using are returned in Resources.\r
  @retval EFI_INVALID_PARAMETER Both Supports and Attributes are NULL.\r
  @retval EFI_UNSUPPORTED       BarIndex not valid for this PCI controller.\r
  @retval EFI_OUT_OF_RESOURCES  There are not enough resources available to allocate\r
                                Resources.\r
\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoGetBarAttributes (\r
  IN EFI_PCI_IO_PROTOCOL             *This,\r
  IN  UINT8                          BarIndex,\r
  OUT UINT64                         *Supports OPTIONAL,\r
  OUT VOID                           **Resources OPTIONAL\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE       *Dev;\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR *Descriptor;\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR *BarDesc;\r
  EFI_ACPI_END_TAG_DESCRIPTOR       *End;\r
  EFI_STATUS                        Status;\r
\r
  if (Supports == NULL && Resources == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
\r
  Status = GetBarResource (Dev, BarIndex, &BarDesc);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  //\r
  // Don't expose any configurable attributes for our emulated BAR\r
  //\r
  if (Supports != NULL) {\r
    *Supports = 0;\r
  }\r
\r
  if (Resources != NULL) {\r
    Descriptor = AllocatePool (sizeof (EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR) +\r
                               sizeof (EFI_ACPI_END_TAG_DESCRIPTOR));\r
    if (Descriptor == NULL) {\r
      return EFI_OUT_OF_RESOURCES;\r
    }\r
\r
    CopyMem (Descriptor, BarDesc, sizeof *Descriptor);\r
\r
    End           = (EFI_ACPI_END_TAG_DESCRIPTOR *) (Descriptor + 1);\r
    End->Desc     = ACPI_END_TAG_DESCRIPTOR;\r
    End->Checksum = 0;\r
\r
    *Resources = Descriptor;\r
  }\r
  return EFI_SUCCESS;\r
}\r
\r
/**\r
  Sets the attributes for a range of a BAR on a PCI controller.\r
\r
  @param  This                  A pointer to the EFI_PCI_IO_PROTOCOL instance.\r
  @param  Attributes            The mask of attributes to set for the resource range specified by\r
                                BarIndex, Offset, and Length.\r
  @param  BarIndex              The BAR index of the standard PCI Configuration header to use as the\r
                                base address for resource range. The legal range for this field is 0..5.\r
  @param  Offset                A pointer to the BAR relative base address of the resource range to be\r
                                modified by the attributes specified by Attributes.\r
  @param  Length                A pointer to the length of the resource range to be modified by the\r
                                attributes specified by Attributes.\r
**/\r
STATIC\r
EFI_STATUS\r
EFIAPI\r
PciIoSetBarAttributes (\r
  IN     EFI_PCI_IO_PROTOCOL          *This,\r
  IN     UINT64                       Attributes,\r
  IN     UINT8                        BarIndex,\r
  IN OUT UINT64                       *Offset,\r
  IN OUT UINT64                       *Length\r
  )\r
{\r
  NON_DISCOVERABLE_PCI_DEVICE         *Dev;\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   *Desc;\r
  EFI_PCI_IO_PROTOCOL_WIDTH           Width;\r
  UINTN                               Count;\r
  EFI_STATUS                          Status;\r
\r
  if ((Attributes & (~DEV_SUPPORTED_ATTRIBUTES)) != 0) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  if (Offset == NULL || Length == NULL) {\r
    return EFI_INVALID_PARAMETER;\r
  }\r
\r
  Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This);\r
  Width = EfiPciIoWidthUint8;\r
  Count = (UINT32) *Length;\r
\r
  Status = GetBarResource(Dev, BarIndex, &Desc);\r
  if (EFI_ERROR (Status)) {\r
    return Status;\r
  }\r
\r
  if (*Offset + (Count << (Width & 0x3)) > Desc->AddrLen) {\r
    return EFI_UNSUPPORTED;\r
  }\r
\r
  ASSERT (FALSE);\r
  return EFI_UNSUPPORTED;\r
}\r
\r
STATIC CONST EFI_PCI_IO_PROTOCOL PciIoTemplate =\r
{\r
  PciIoPollMem,\r
  PciIoPollIo,\r
  { PciIoMemRead, PciIoMemWrite },\r
  { PciIoIoRead,  PciIoIoWrite },\r
  { PciIoPciRead, PciIoPciWrite },\r
  PciIoCopyMem,\r
  CoherentPciIoMap,\r
  CoherentPciIoUnmap,\r
  CoherentPciIoAllocateBuffer,\r
  CoherentPciIoFreeBuffer,\r
  PciIoFlush,\r
  PciIoGetLocation,\r
  PciIoAttributes,\r
  PciIoGetBarAttributes,\r
  PciIoSetBarAttributes,\r
  0,\r
  0\r
};\r
\r
/**\r
  Initialize PciIo Protocol.\r
\r
  @param  Dev      Point to NON_DISCOVERABLE_PCI_DEVICE instance.\r
\r
**/\r
VOID\r
InitializePciIoProtocol (\r
  NON_DISCOVERABLE_PCI_DEVICE     *Dev\r
  )\r
{\r
  EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR   *Desc;\r
  INTN                                Idx;\r
\r
  InitializeListHead (&Dev->UncachedAllocationList);\r
\r
  Dev->ConfigSpace.Hdr.VendorId = PCI_ID_VENDOR_UNKNOWN;\r
  Dev->ConfigSpace.Hdr.DeviceId = PCI_ID_DEVICE_DONTCARE;\r
\r
  // Copy protocol structure\r
  CopyMem(&Dev->PciIo, &PciIoTemplate, sizeof PciIoTemplate);\r
\r
  if (Dev->Device->DmaType == NonDiscoverableDeviceDmaTypeNonCoherent) {\r
    Dev->PciIo.AllocateBuffer   = NonCoherentPciIoAllocateBuffer;\r
    Dev->PciIo.FreeBuffer       = NonCoherentPciIoFreeBuffer;\r
    Dev->PciIo.Map              = NonCoherentPciIoMap;\r
    Dev->PciIo.Unmap            = NonCoherentPciIoUnmap;\r
  }\r
\r
  if (CompareGuid (Dev->Device->Type, &gEdkiiNonDiscoverableAhciDeviceGuid)) {\r
    Dev->ConfigSpace.Hdr.ClassCode[0] = PCI_IF_MASS_STORAGE_AHCI;\r
    Dev->ConfigSpace.Hdr.ClassCode[1] = PCI_CLASS_MASS_STORAGE_SATADPA;\r
    Dev->ConfigSpace.Hdr.ClassCode[2] = PCI_CLASS_MASS_STORAGE;\r
    Dev->BarOffset = 5;\r
  } else if (CompareGuid (Dev->Device->Type,\r
                          &gEdkiiNonDiscoverableEhciDeviceGuid)) {\r
    Dev->ConfigSpace.Hdr.ClassCode[0] = PCI_IF_EHCI;\r
    Dev->ConfigSpace.Hdr.ClassCode[1] = PCI_CLASS_SERIAL_USB;\r
    Dev->ConfigSpace.Hdr.ClassCode[2] = PCI_CLASS_SERIAL;\r
    Dev->BarOffset = 0;\r
  } else if (CompareGuid (Dev->Device->Type,\r
                          &gEdkiiNonDiscoverableNvmeDeviceGuid)) {\r
    Dev->ConfigSpace.Hdr.ClassCode[0] = 0x2; // PCI_IF_NVMHCI\r
    Dev->ConfigSpace.Hdr.ClassCode[1] = 0x8; // PCI_CLASS_MASS_STORAGE_NVM\r
    Dev->ConfigSpace.Hdr.ClassCode[2] = PCI_CLASS_MASS_STORAGE;\r
    Dev->BarOffset = 0;\r
  } else if (CompareGuid (Dev->Device->Type,\r
                          &gEdkiiNonDiscoverableOhciDeviceGuid)) {\r
    Dev->ConfigSpace.Hdr.ClassCode[0] = PCI_IF_OHCI;\r
    Dev->ConfigSpace.Hdr.ClassCode[1] = PCI_CLASS_SERIAL_USB;\r
    Dev->ConfigSpace.Hdr.ClassCode[2] = PCI_CLASS_SERIAL;\r
    Dev->BarOffset = 0;\r
  } else if (CompareGuid (Dev->Device->Type,\r
                          &gEdkiiNonDiscoverableSdhciDeviceGuid)) {\r
    Dev->ConfigSpace.Hdr.ClassCode[0] = 0x0; // don't care\r
    Dev->ConfigSpace.Hdr.ClassCode[1] = PCI_SUBCLASS_SD_HOST_CONTROLLER;\r
    Dev->ConfigSpace.Hdr.ClassCode[2] = PCI_CLASS_SYSTEM_PERIPHERAL;\r
    Dev->BarOffset = 0;\r
  } else if (CompareGuid (Dev->Device->Type,\r
                          &gEdkiiNonDiscoverableXhciDeviceGuid)) {\r
    Dev->ConfigSpace.Hdr.ClassCode[0] = PCI_IF_XHCI;\r
    Dev->ConfigSpace.Hdr.ClassCode[1] = PCI_CLASS_SERIAL_USB;\r
    Dev->ConfigSpace.Hdr.ClassCode[2] = PCI_CLASS_SERIAL;\r
    Dev->BarOffset = 0;\r
  } else if (CompareGuid (Dev->Device->Type,\r
                          &gEdkiiNonDiscoverableUhciDeviceGuid)) {\r
    Dev->ConfigSpace.Hdr.ClassCode[0] = PCI_IF_UHCI;\r
    Dev->ConfigSpace.Hdr.ClassCode[1] = PCI_CLASS_SERIAL_USB;\r
    Dev->ConfigSpace.Hdr.ClassCode[2] = PCI_CLASS_SERIAL;\r
    Dev->BarOffset = 0;\r
  } else if (CompareGuid (Dev->Device->Type,\r
                          &gEdkiiNonDiscoverableUfsDeviceGuid)) {\r
    Dev->ConfigSpace.Hdr.ClassCode[0] = 0x0; // don't care\r
    Dev->ConfigSpace.Hdr.ClassCode[1] = 0x9; // UFS controller subclass;\r
    Dev->ConfigSpace.Hdr.ClassCode[2] = PCI_CLASS_MASS_STORAGE;\r
    Dev->BarOffset = 0;\r
  } else {\r
    ASSERT_EFI_ERROR (EFI_INVALID_PARAMETER);\r
  }\r
\r
  //\r
  // Iterate over the resources to populate the virtual BARs\r
  //\r
  Idx = Dev->BarOffset;\r
  for (Desc = Dev->Device->Resources, Dev->BarCount = 0;\r
       Desc->Desc != ACPI_END_TAG_DESCRIPTOR;\r
       Desc = (VOID *)((UINT8 *)Desc + Desc->Len + 3)) {\r
\r
    ASSERT (Desc->Desc == ACPI_ADDRESS_SPACE_DESCRIPTOR);\r
    ASSERT (Desc->ResType == ACPI_ADDRESS_SPACE_TYPE_MEM);\r
\r
    if (Idx >= PCI_MAX_BAR ||\r
        (Idx == PCI_MAX_BAR - 1 && Desc->AddrSpaceGranularity == 64)) {\r
      DEBUG ((DEBUG_ERROR,\r
        "%a: resource count exceeds number of emulated BARs\n",\r
        __FUNCTION__));\r
      ASSERT (FALSE);\r
      break;\r
    }\r
\r
    Dev->ConfigSpace.Device.Bar[Idx] = (UINT32)Desc->AddrRangeMin;\r
    Dev->BarCount++;\r
\r
    if (Desc->AddrSpaceGranularity == 64) {\r
      Dev->ConfigSpace.Device.Bar[Idx] |= 0x4;\r
      Dev->ConfigSpace.Device.Bar[++Idx] = (UINT32)RShiftU64 (\r
                                                     Desc->AddrRangeMin, 32);\r
    }\r
  }\r
}\r