arch/powerpc/platforms/powernv/npu-dma.c

   1 /*
   2  * This file implements the DMA operations for NVLink devices. The NPU
   3  * devices all point to the same iommu table as the parent PCI device.
   4  *
   5  * Copyright Alistair Popple, IBM Corporation 2015.
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of version 2 of the GNU General Public
   9  * License as published by the Free Software Foundation.
  10  */
  11
  12 #include <linux/export.h>
  13 #include <linux/pci.h>
  14 #include <linux/memblock.h>
  15 #include <linux/iommu.h>
  16
  17 #include <asm/iommu.h>
  18 #include <asm/pnv-pci.h>
  19 #include <asm/msi_bitmap.h>
  20 #include <asm/opal.h>
  21
  22 #include "powernv.h"
  23 #include "pci.h"
  24
  25 /*
  26  * Other types of TCE cache invalidation are not functional in the
  27  * hardware.
  28  */
  29 static struct pci_dev *get_pci_dev(struct device_node *dn)
  30 {
  31         return PCI_DN(dn)->pcidev;
  32 }
  33
  34 /* Given a NPU device get the associated PCI device. */
  35 struct pci_dev *pnv_pci_get_gpu_dev(struct pci_dev *npdev)
  36 {
  37         struct device_node *dn;
  38         struct pci_dev *gpdev;
  39
  40         if (WARN_ON(!npdev))
  41                 return NULL;
  42
  43         if (WARN_ON(!npdev->dev.of_node))
  44                 return NULL;
  45
  46         /* Get assoicated PCI device */
  47         dn = of_parse_phandle(npdev->dev.of_node, "ibm,gpu", 0);
  48         if (!dn)
  49                 return NULL;
  50
  51         gpdev = get_pci_dev(dn);
  52         of_node_put(dn);
  53
  54         return gpdev;
  55 }
  56 EXPORT_SYMBOL(pnv_pci_get_gpu_dev);
  57
  58 /* Given the real PCI device get a linked NPU device. */
  59 struct pci_dev *pnv_pci_get_npu_dev(struct pci_dev *gpdev, int index)
  60 {
  61         struct device_node *dn;
  62         struct pci_dev *npdev;
  63
  64         if (WARN_ON(!gpdev))
  65                 return NULL;
  66
  67         if (WARN_ON(!gpdev->dev.of_node))
  68                 return NULL;
  69
  70         /* Get assoicated PCI device */
  71         dn = of_parse_phandle(gpdev->dev.of_node, "ibm,npu", index);
  72         if (!dn)
  73                 return NULL;
  74
  75         npdev = get_pci_dev(dn);
  76         of_node_put(dn);
  77
  78         return npdev;
  79 }
  80 EXPORT_SYMBOL(pnv_pci_get_npu_dev);
  81
  82 #define NPU_DMA_OP_UNSUPPORTED()                                        \
  83         dev_err_once(dev, "%s operation unsupported for NVLink devices\n", \
  84                 __func__)
  85
  86 static void *dma_npu_alloc(struct device *dev, size_t size,
  87                            dma_addr_t *dma_handle, gfp_t flag,
  88                            unsigned long attrs)
  89 {
  90         NPU_DMA_OP_UNSUPPORTED();
  91         return NULL;
  92 }
  93
  94 static void dma_npu_free(struct device *dev, size_t size,
  95                          void *vaddr, dma_addr_t dma_handle,
  96                          unsigned long attrs)
  97 {
  98         NPU_DMA_OP_UNSUPPORTED();
  99 }
 100
 101 static dma_addr_t dma_npu_map_page(struct device *dev, struct page *page,
 102                                    unsigned long offset, size_t size,
 103                                    enum dma_data_direction direction,
 104                                    unsigned long attrs)
 105 {
 106         NPU_DMA_OP_UNSUPPORTED();
 107         return 0;
 108 }
 109
 110 static int dma_npu_map_sg(struct device *dev, struct scatterlist *sglist,
 111                           int nelems, enum dma_data_direction direction,
 112                           unsigned long attrs)
 113 {
 114         NPU_DMA_OP_UNSUPPORTED();
 115         return 0;
 116 }
 117
 118 static int dma_npu_dma_supported(struct device *dev, u64 mask)
 119 {
 120         NPU_DMA_OP_UNSUPPORTED();
 121         return 0;
 122 }
 123
 124 static u64 dma_npu_get_required_mask(struct device *dev)
 125 {
 126         NPU_DMA_OP_UNSUPPORTED();
 127         return 0;
 128 }
 129
 130 static struct dma_map_ops dma_npu_ops = {
 131         .map_page               = dma_npu_map_page,
 132         .map_sg                 = dma_npu_map_sg,
 133         .alloc                  = dma_npu_alloc,
 134         .free                   = dma_npu_free,
 135         .dma_supported          = dma_npu_dma_supported,
 136         .get_required_mask      = dma_npu_get_required_mask,
 137 };
 138
 139 /*
 140  * Returns the PE assoicated with the PCI device of the given
 141  * NPU. Returns the linked pci device if pci_dev != NULL.
 142  */
 143 static struct pnv_ioda_pe *get_gpu_pci_dev_and_pe(struct pnv_ioda_pe *npe,
 144                                                   struct pci_dev **gpdev)
 145 {
 146         struct pnv_phb *phb;
 147         struct pci_controller *hose;
 148         struct pci_dev *pdev;
 149         struct pnv_ioda_pe *pe;
 150         struct pci_dn *pdn;
 151
 152         pdev = pnv_pci_get_gpu_dev(npe->pdev);
 153         if (!pdev)
 154                 return NULL;
 155
 156         pdn = pci_get_pdn(pdev);
 157         if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
 158                 return NULL;
 159
 160         hose = pci_bus_to_host(pdev->bus);
 161         phb = hose->private_data;
 162         pe = &phb->ioda.pe_array[pdn->pe_number];
 163
 164         if (gpdev)
 165                 *gpdev = pdev;
 166
 167         return pe;
 168 }
 169
 170 long pnv_npu_set_window(struct pnv_ioda_pe *npe, int num,
 171                 struct iommu_table *tbl)
 172 {
 173         struct pnv_phb *phb = npe->phb;
 174         int64_t rc;
 175         const unsigned long size = tbl->it_indirect_levels ?
 176                 tbl->it_level_size : tbl->it_size;
 177         const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
 178         const __u64 win_size = tbl->it_size << tbl->it_page_shift;
 179
 180         pe_info(npe, "Setting up window %llx..%llx pg=%lx\n",
 181                         start_addr, start_addr + win_size - 1,
 182                         IOMMU_PAGE_SIZE(tbl));
 183
 184         rc = opal_pci_map_pe_dma_window(phb->opal_id,
 185                         npe->pe_number,
 186                         npe->pe_number,
 187                         tbl->it_indirect_levels + 1,
 188                         __pa(tbl->it_base),
 189                         size << 3,
 190                         IOMMU_PAGE_SIZE(tbl));
 191         if (rc) {
 192                 pe_err(npe, "Failed to configure TCE table, err %lld\n", rc);
 193                 return rc;
 194         }
 195         pnv_pci_ioda2_tce_invalidate_entire(phb, false);
 196
 197         /* Add the table to the list so its TCE cache will get invalidated */
 198         pnv_pci_link_table_and_group(phb->hose->node, num,
 199                         tbl, &npe->table_group);
 200
 201         return 0;
 202 }
 203
 204 long pnv_npu_unset_window(struct pnv_ioda_pe *npe, int num)
 205 {
 206         struct pnv_phb *phb = npe->phb;
 207         int64_t rc;
 208
 209         pe_info(npe, "Removing DMA window\n");
 210
 211         rc = opal_pci_map_pe_dma_window(phb->opal_id, npe->pe_number,
 212                         npe->pe_number,
 213                         0/* levels */, 0/* table address */,
 214                         0/* table size */, 0/* page size */);
 215         if (rc) {
 216                 pe_err(npe, "Unmapping failed, ret = %lld\n", rc);
 217                 return rc;
 218         }
 219         pnv_pci_ioda2_tce_invalidate_entire(phb, false);
 220
 221         pnv_pci_unlink_table_and_group(npe->table_group.tables[num],
 222                         &npe->table_group);
 223
 224         return 0;
 225 }
 226
 227 /*
 228  * Enables 32 bit DMA on NPU.
 229  */
 230 static void pnv_npu_dma_set_32(struct pnv_ioda_pe *npe)
 231 {
 232         struct pci_dev *gpdev;
 233         struct pnv_ioda_pe *gpe;
 234         int64_t rc;
 235
 236         /*
 237          * Find the assoicated PCI devices and get the dma window
 238          * information from there.
 239          */
 240         if (!npe->pdev || !(npe->flags & PNV_IODA_PE_DEV))
 241                 return;
 242
 243         gpe = get_gpu_pci_dev_and_pe(npe, &gpdev);
 244         if (!gpe)
 245                 return;
 246
 247         rc = pnv_npu_set_window(npe, 0, gpe->table_group.tables[0]);
 248
 249         /*
 250          * We don't initialise npu_pe->tce32_table as we always use
 251          * dma_npu_ops which are nops.
 252          */
 253         set_dma_ops(&npe->pdev->dev, &dma_npu_ops);
 254 }
 255
 256 /*
 257  * Enables bypass mode on the NPU. The NPU only supports one
 258  * window per link, so bypass needs to be explicitly enabled or
 259  * disabled. Unlike for a PHB3 bypass and non-bypass modes can't be
 260  * active at the same time.
 261  */
 262 static int pnv_npu_dma_set_bypass(struct pnv_ioda_pe *npe)
 263 {
 264         struct pnv_phb *phb = npe->phb;
 265         int64_t rc = 0;
 266         phys_addr_t top = memblock_end_of_DRAM();
 267
 268         if (phb->type != PNV_PHB_NPU || !npe->pdev)
 269                 return -EINVAL;
 270
 271         rc = pnv_npu_unset_window(npe, 0);
 272         if (rc != OPAL_SUCCESS)
 273                 return rc;
 274
 275         /* Enable the bypass window */
 276
 277         top = roundup_pow_of_two(top);
 278         dev_info(&npe->pdev->dev, "Enabling bypass for PE %x\n",
 279                         npe->pe_number);
 280         rc = opal_pci_map_pe_dma_window_real(phb->opal_id,
 281                         npe->pe_number, npe->pe_number,
 282                         0 /* bypass base */, top);
 283
 284         if (rc == OPAL_SUCCESS)
 285                 pnv_pci_ioda2_tce_invalidate_entire(phb, false);
 286
 287         return rc;
 288 }
 289
 290 void pnv_npu_try_dma_set_bypass(struct pci_dev *gpdev, bool bypass)
 291 {
 292         int i;
 293         struct pnv_phb *phb;
 294         struct pci_dn *pdn;
 295         struct pnv_ioda_pe *npe;
 296         struct pci_dev *npdev;
 297
 298         for (i = 0; ; ++i) {
 299                 npdev = pnv_pci_get_npu_dev(gpdev, i);
 300
 301                 if (!npdev)
 302                         break;
 303
 304                 pdn = pci_get_pdn(npdev);
 305                 if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
 306                         return;
 307
 308                 phb = pci_bus_to_host(npdev->bus)->private_data;
 309
 310                 /* We only do bypass if it's enabled on the linked device */
 311                 npe = &phb->ioda.pe_array[pdn->pe_number];
 312
 313                 if (bypass) {
 314                         dev_info(&npdev->dev,
 315                                         "Using 64-bit DMA iommu bypass\n");
 316                         pnv_npu_dma_set_bypass(npe);
 317                 } else {
 318                         dev_info(&npdev->dev, "Using 32-bit DMA via iommu\n");
 319                         pnv_npu_dma_set_32(npe);
 320                 }
 321         }
 322 }
 323
 324 /* Switch ownership from platform code to external user (e.g. VFIO) */
 325 void pnv_npu_take_ownership(struct pnv_ioda_pe *npe)
 326 {
 327         struct pnv_phb *phb = npe->phb;
 328         int64_t rc;
 329
 330         /*
 331          * Note: NPU has just a single TVE in the hardware which means that
 332          * while used by the kernel, it can have either 32bit window or
 333          * DMA bypass but never both. So we deconfigure 32bit window only
 334          * if it was enabled at the moment of ownership change.
 335          */
 336         if (npe->table_group.tables[0]) {
 337                 pnv_npu_unset_window(npe, 0);
 338                 return;
 339         }
 340
 341         /* Disable bypass */
 342         rc = opal_pci_map_pe_dma_window_real(phb->opal_id,
 343                         npe->pe_number, npe->pe_number,
 344                         0 /* bypass base */, 0);
 345         if (rc) {
 346                 pe_err(npe, "Failed to disable bypass, err %lld\n", rc);
 347                 return;
 348         }
 349         pnv_pci_ioda2_tce_invalidate_entire(npe->phb, false);
 350 }
 351
 352 struct pnv_ioda_pe *pnv_pci_npu_setup_iommu(struct pnv_ioda_pe *npe)
 353 {
 354         struct pnv_phb *phb = npe->phb;
 355         struct pci_bus *pbus = phb->hose->bus;
 356         struct pci_dev *npdev, *gpdev = NULL, *gptmp;
 357         struct pnv_ioda_pe *gpe = get_gpu_pci_dev_and_pe(npe, &gpdev);
 358
 359         if (!gpe || !gpdev)
 360                 return NULL;
 361
 362         list_for_each_entry(npdev, &pbus->devices, bus_list) {
 363                 gptmp = pnv_pci_get_gpu_dev(npdev);
 364
 365                 if (gptmp != gpdev)
 366                         continue;
 367
 368                 pe_info(gpe, "Attached NPU %s\n", dev_name(&npdev->dev));
 369                 iommu_group_add_device(gpe->table_group.group, &npdev->dev);
 370         }
 371
 372         return gpe;
 373 }