From: Ard Biesheuvel Date: Fri, 9 Dec 2016 15:04:34 +0000 (+0000) Subject: MdeModulePkg/NonDiscoverablePciDeviceDxe: add support for non-coherent DMA X-Git-Tag: edk2-stable201903~4929 X-Git-Url: https://git.proxmox.com/?p=mirror_edk2.git;a=commitdiff_plain;h=16296a126c99174ad27a6d78229df70e62049310 MdeModulePkg/NonDiscoverablePciDeviceDxe: add support for non-coherent DMA Add support for non-coherent DMA, either by performing explicit cache maintenance when DMA mappings are aligned to the CPU's DMA buffer alignment, or by bounce buffering via uncached mappings otherwise. Contributed-under: TianoCore Contribution Agreement 1.0 Signed-off-by: Ard Biesheuvel Tested-by: Marcin Wojtas Reviewed-by: Ruiyu Ni --- diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c index ee765d7a5d..0fcf2b2ec1 100644 --- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c +++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.c @@ -16,6 +16,8 @@ #include +EFI_CPU_ARCH_PROTOCOL *mCpu; + // // We only support the following device types // @@ -69,14 +71,7 @@ NonDiscoverablePciDeviceSupported ( return Status; } - // - // Restricted to DMA coherent for now - // Status = EFI_UNSUPPORTED; - if (Device->DmaType != NonDiscoverableDeviceDmaTypeCoherent) { - goto CloseProtocol; - } - for (Idx = 0; Idx < ARRAY_SIZE (SupportedNonDiscoverableDevices); Idx++) { if (CompareGuid (Device->Type, SupportedNonDiscoverableDevices [Idx])) { Status = EFI_SUCCESS; @@ -224,6 +219,11 @@ NonDiscoverablePciDeviceDxeEntryPoint ( IN EFI_SYSTEM_TABLE *SystemTable ) { + EFI_STATUS Status; + + Status = gBS->LocateProtocol (&gEfiCpuArchProtocolGuid, NULL, (VOID **)&mCpu); + ASSERT_EFI_ERROR(Status); + return EfiLibInstallDriverBindingComponentName2 ( ImageHandle, SystemTable, diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf index 996fe310e0..5faa894513 100644 --- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf +++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceDxe.inf @@ -32,6 +32,7 @@ [LibraryClasses] BaseMemoryLib DebugLib + DxeServicesTableLib MemoryAllocationLib UefiBootServicesTableLib UefiDriverEntryPoint @@ -40,6 +41,7 @@ [Protocols] gEfiPciIoProtocolGuid ## BY_START gEdkiiNonDiscoverableDeviceProtocolGuid ## TO_START + gEfiCpuArchProtocolGuid ## CONSUMES [Guids] gEdkiiNonDiscoverableAhciDeviceGuid diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c index 82ee9d1190..ce73a770c1 100644 --- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c +++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.c @@ -15,6 +15,8 @@ #include "NonDiscoverablePciDeviceIo.h" +#include + #include #include @@ -537,6 +539,324 @@ CoherentPciIoFreeBuffer ( return EFI_SUCCESS; } +STATIC +EFI_STATUS +EFIAPI +NonCoherentPciIoFreeBuffer ( + IN EFI_PCI_IO_PROTOCOL *This, + IN UINTN Pages, + IN VOID *HostAddress + ) +{ + NON_DISCOVERABLE_PCI_DEVICE *Dev; + LIST_ENTRY *Entry; + EFI_STATUS Status; + NON_DISCOVERABLE_DEVICE_UNCACHED_ALLOCATION *Alloc; + BOOLEAN Found; + + Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This); + + Found = FALSE; + + // + // Find the uncached allocation list entry associated + // with this allocation + // + for (Entry = Dev->UncachedAllocationList.ForwardLink; + Entry != &Dev->UncachedAllocationList; + Entry = Entry->ForwardLink) { + + Alloc = BASE_CR (Entry, NON_DISCOVERABLE_DEVICE_UNCACHED_ALLOCATION, List); + if (Alloc->HostAddress == HostAddress && Alloc->NumPages == Pages) { + // + // We are freeing the exact allocation we were given + // before by AllocateBuffer() + // + Found = TRUE; + break; + } + } + + if (!Found) { + ASSERT_EFI_ERROR (EFI_NOT_FOUND); + return EFI_NOT_FOUND; + } + + RemoveEntryList (&Alloc->List); + + Status = gDS->SetMemorySpaceAttributes ( + (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress, + EFI_PAGES_TO_SIZE (Pages), + Alloc->Attributes); + if (EFI_ERROR (Status)) { + goto FreeAlloc; + } + + // + // If we fail to restore the original attributes, it is better to leak the + // memory than to return it to the heap + // + FreePages (HostAddress, Pages); + +FreeAlloc: + FreePool (Alloc); + return Status; +} + +STATIC +EFI_STATUS +EFIAPI +NonCoherentPciIoAllocateBuffer ( + IN EFI_PCI_IO_PROTOCOL *This, + IN EFI_ALLOCATE_TYPE Type, + IN EFI_MEMORY_TYPE MemoryType, + IN UINTN Pages, + OUT VOID **HostAddress, + IN UINT64 Attributes + ) +{ + NON_DISCOVERABLE_PCI_DEVICE *Dev; + EFI_GCD_MEMORY_SPACE_DESCRIPTOR GcdDescriptor; + EFI_STATUS Status; + UINT64 MemType; + NON_DISCOVERABLE_DEVICE_UNCACHED_ALLOCATION *Alloc; + VOID *AllocAddress; + + Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This); + + Status = CoherentPciIoAllocateBuffer (This, Type, MemoryType, Pages, + &AllocAddress, Attributes); + if (EFI_ERROR (Status)) { + return Status; + } + + Status = gDS->GetMemorySpaceDescriptor ( + (EFI_PHYSICAL_ADDRESS)(UINTN)AllocAddress, + &GcdDescriptor); + if (EFI_ERROR (Status)) { + goto FreeBuffer; + } + + if ((GcdDescriptor.Capabilities & (EFI_MEMORY_WC | EFI_MEMORY_UC)) == 0) { + Status = EFI_UNSUPPORTED; + goto FreeBuffer; + } + + // + // Set the preferred memory attributes + // + if ((Attributes & EFI_PCI_ATTRIBUTE_MEMORY_WRITE_COMBINE) != 0 || + (GcdDescriptor.Capabilities & EFI_MEMORY_UC) == 0) { + // + // Use write combining if it was requested, or if it is the only + // type supported by the region. + // + MemType = EFI_MEMORY_WC; + } else { + MemType = EFI_MEMORY_UC; + } + + Alloc = AllocatePool (sizeof *Alloc); + if (Alloc == NULL) { + goto FreeBuffer; + } + + Alloc->HostAddress = AllocAddress; + Alloc->NumPages = Pages; + Alloc->Attributes = GcdDescriptor.Attributes; + + // + // Record this allocation in the linked list, so we + // can restore the memory space attributes later + // + InsertHeadList (&Dev->UncachedAllocationList, &Alloc->List); + + Status = gDS->SetMemorySpaceAttributes ( + (EFI_PHYSICAL_ADDRESS)(UINTN)AllocAddress, + EFI_PAGES_TO_SIZE (Pages), + MemType); + if (EFI_ERROR (Status)) { + goto RemoveList; + } + + Status = mCpu->FlushDataCache ( + mCpu, + (EFI_PHYSICAL_ADDRESS)(UINTN)AllocAddress, + EFI_PAGES_TO_SIZE (Pages), + EfiCpuFlushTypeInvalidate); + if (EFI_ERROR (Status)) { + goto RemoveList; + } + + *HostAddress = AllocAddress; + + return EFI_SUCCESS; + +RemoveList: + RemoveEntryList (&Alloc->List); + FreePool (Alloc); + +FreeBuffer: + CoherentPciIoFreeBuffer (This, Pages, AllocAddress); + return Status; +} + +STATIC +EFI_STATUS +EFIAPI +NonCoherentPciIoMap ( + IN EFI_PCI_IO_PROTOCOL *This, + IN EFI_PCI_IO_PROTOCOL_OPERATION Operation, + IN VOID *HostAddress, + IN OUT UINTN *NumberOfBytes, + OUT EFI_PHYSICAL_ADDRESS *DeviceAddress, + OUT VOID **Mapping + ) +{ + NON_DISCOVERABLE_PCI_DEVICE *Dev; + EFI_STATUS Status; + NON_DISCOVERABLE_PCI_DEVICE_MAP_INFO *MapInfo; + UINTN AlignMask; + VOID *AllocAddress; + EFI_GCD_MEMORY_SPACE_DESCRIPTOR GcdDescriptor; + BOOLEAN Bounce; + + MapInfo = AllocatePool (sizeof *MapInfo); + if (MapInfo == NULL) { + return EFI_OUT_OF_RESOURCES; + } + + MapInfo->HostAddress = HostAddress; + MapInfo->Operation = Operation; + MapInfo->NumberOfBytes = *NumberOfBytes; + + Dev = NON_DISCOVERABLE_PCI_DEVICE_FROM_PCI_IO(This); + + // + // If this device does not support 64-bit DMA addressing, we need to allocate + // a bounce buffer and copy over the data in case HostAddress >= 4 GB. + // + Bounce = ((Dev->Attributes & EFI_PCI_IO_ATTRIBUTE_DUAL_ADDRESS_CYCLE) == 0 && + (UINTN)HostAddress + *NumberOfBytes > SIZE_4GB); + + if (!Bounce) { + switch (Operation) { + case EfiPciIoOperationBusMasterRead: + case EfiPciIoOperationBusMasterWrite: + // + // For streaming DMA, it is sufficient if the buffer is aligned to + // the CPUs DMA buffer alignment. + // + AlignMask = mCpu->DmaBufferAlignment - 1; + if ((((UINTN) HostAddress | *NumberOfBytes) & AlignMask) == 0) { + break; + } + // fall through + + case EfiPciIoOperationBusMasterCommonBuffer: + // + // Check whether the host address refers to an uncached mapping. + // + Status = gDS->GetMemorySpaceDescriptor ( + (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress, + &GcdDescriptor); + if (EFI_ERROR (Status) || + (GcdDescriptor.Attributes & (EFI_MEMORY_WB|EFI_MEMORY_WT)) != 0) { + Bounce = TRUE; + } + break; + + default: + ASSERT (FALSE); + } + } + + if (Bounce) { + if (Operation == EfiPciIoOperationBusMasterCommonBuffer) { + Status = EFI_DEVICE_ERROR; + goto FreeMapInfo; + } + + Status = NonCoherentPciIoAllocateBuffer (This, AllocateAnyPages, + EfiBootServicesData, EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes), + &AllocAddress, EFI_PCI_ATTRIBUTE_MEMORY_WRITE_COMBINE); + if (EFI_ERROR (Status)) { + goto FreeMapInfo; + } + MapInfo->AllocAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)AllocAddress; + if (Operation == EfiPciIoOperationBusMasterRead) { + gBS->CopyMem (AllocAddress, HostAddress, *NumberOfBytes); + } + *DeviceAddress = MapInfo->AllocAddress; + } else { + MapInfo->AllocAddress = 0; + *DeviceAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress; + + // + // We are not using a bounce buffer: the mapping is sufficiently + // aligned to allow us to simply flush the caches. Note that cleaning + // the caches is necessary for both data directions: + // - for bus master read, we want the latest data to be present + // in main memory + // - for bus master write, we don't want any stale dirty cachelines that + // may be written back unexpectedly, and clobber the data written to + // main memory by the device. + // + mCpu->FlushDataCache (mCpu, (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress, + *NumberOfBytes, EfiCpuFlushTypeWriteBack); + } + + *Mapping = MapInfo; + return EFI_SUCCESS; + +FreeMapInfo: + FreePool (MapInfo); + + return Status; +} + +STATIC +EFI_STATUS +EFIAPI +NonCoherentPciIoUnmap ( + IN EFI_PCI_IO_PROTOCOL *This, + IN VOID *Mapping + ) +{ + NON_DISCOVERABLE_PCI_DEVICE_MAP_INFO *MapInfo; + + if (Mapping == NULL) { + return EFI_DEVICE_ERROR; + } + + MapInfo = Mapping; + if (MapInfo->AllocAddress != 0) { + // + // We are using a bounce buffer: copy back the data if necessary, + // and free the buffer. + // + if (MapInfo->Operation == EfiPciIoOperationBusMasterWrite) { + gBS->CopyMem (MapInfo->HostAddress, (VOID *)(UINTN)MapInfo->AllocAddress, + MapInfo->NumberOfBytes); + } + NonCoherentPciIoFreeBuffer (This, + EFI_SIZE_TO_PAGES (MapInfo->NumberOfBytes), + (VOID *)(UINTN)MapInfo->AllocAddress); + } else { + // + // We are *not* using a bounce buffer: if this is a bus master write, + // we have to invalidate the caches so the CPU will see the uncached + // data written by the device. + // + if (MapInfo->Operation == EfiPciIoOperationBusMasterWrite) { + mCpu->FlushDataCache (mCpu, + (EFI_PHYSICAL_ADDRESS)(UINTN)MapInfo->HostAddress, + MapInfo->NumberOfBytes, EfiCpuFlushTypeInvalidate); + } + } + FreePool (MapInfo); + return EFI_SUCCESS; +} STATIC EFI_STATUS @@ -726,12 +1046,21 @@ InitializePciIoProtocol ( EFI_ACPI_ADDRESS_SPACE_DESCRIPTOR *Desc; INTN Idx; + InitializeListHead (&Dev->UncachedAllocationList); + Dev->ConfigSpace.Hdr.VendorId = PCI_ID_VENDOR_UNKNOWN; Dev->ConfigSpace.Hdr.DeviceId = PCI_ID_DEVICE_DONTCARE; // Copy protocol structure CopyMem(&Dev->PciIo, &PciIoTemplate, sizeof PciIoTemplate); + if (Dev->Device->DmaType == NonDiscoverableDeviceDmaTypeNonCoherent) { + Dev->PciIo.AllocateBuffer = NonCoherentPciIoAllocateBuffer; + Dev->PciIo.FreeBuffer = NonCoherentPciIoFreeBuffer; + Dev->PciIo.Map = NonCoherentPciIoMap; + Dev->PciIo.Unmap = NonCoherentPciIoUnmap; + } + if (CompareGuid (Dev->Device->Type, &gEdkiiNonDiscoverableAhciDeviceGuid)) { Dev->ConfigSpace.Hdr.ClassCode[0] = PCI_IF_MASS_STORAGE_AHCI; Dev->ConfigSpace.Hdr.ClassCode[1] = PCI_CLASS_MASS_STORAGE_SATADPA; diff --git a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h index bc0a3d3258..4496148629 100644 --- a/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h +++ b/MdeModulePkg/Bus/Pci/NonDiscoverablePciDeviceDxe/NonDiscoverablePciDeviceIo.h @@ -15,6 +15,8 @@ #ifndef __NON_DISCOVERABLE_PCI_DEVICE_IO_H__ #define __NON_DISCOVERABLE_PCI_DEVICE_IO_H__ +#include + #include #include #include @@ -25,6 +27,7 @@ #include #include +#include #include #define NON_DISCOVERABLE_PCI_DEVICE_SIG SIGNATURE_32 ('P', 'P', 'I', 'D') @@ -38,6 +41,27 @@ #define PCI_MAX_BARS 6 +extern EFI_CPU_ARCH_PROTOCOL *mCpu; + +typedef struct { + // + // The linked-list next pointer + // + LIST_ENTRY List; + // + // The address of the uncached allocation + // + VOID *HostAddress; + // + // The number of pages in the allocation + // + UINTN NumPages; + // + // The attributes of the allocation + // + UINT64 Attributes; +} NON_DISCOVERABLE_DEVICE_UNCACHED_ALLOCATION; + typedef struct { UINT32 Signature; // @@ -71,6 +95,11 @@ typedef struct { // Whether this device has been enabled // BOOLEAN Enabled; + // + // Linked list to keep track of uncached allocations performed + // on behalf of this device + // + LIST_ENTRY UncachedAllocationList; } NON_DISCOVERABLE_PCI_DEVICE; VOID