drivers/iommu/fsl_pamu_domain.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *
   4  * Copyright (C) 2013 Freescale Semiconductor, Inc.
   5  * Author: Varun Sethi <varun.sethi@freescale.com>
   6  */
   7
   8 #define pr_fmt(fmt)    "fsl-pamu-domain: %s: " fmt, __func__
   9
  10 #include "fsl_pamu_domain.h"
  11
  12 #include <sysdev/fsl_pci.h>
  13
  14 /*
  15  * Global spinlock that needs to be held while
  16  * configuring PAMU.
  17  */
  18 static DEFINE_SPINLOCK(iommu_lock);
  19
  20 static struct kmem_cache *fsl_pamu_domain_cache;
  21 static struct kmem_cache *iommu_devinfo_cache;
  22 static DEFINE_SPINLOCK(device_domain_lock);
  23
  24 struct iommu_device pamu_iommu; /* IOMMU core code handle */
  25
  26 static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
  27 {
  28         return container_of(dom, struct fsl_dma_domain, iommu_domain);
  29 }
  30
  31 static int __init iommu_init_mempool(void)
  32 {
  33         fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
  34                                                   sizeof(struct fsl_dma_domain),
  35                                                   0,
  36                                                   SLAB_HWCACHE_ALIGN,
  37                                                   NULL);
  38         if (!fsl_pamu_domain_cache) {
  39                 pr_debug("Couldn't create fsl iommu_domain cache\n");
  40                 return -ENOMEM;
  41         }
  42
  43         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
  44                                                 sizeof(struct device_domain_info),
  45                                                 0,
  46                                                 SLAB_HWCACHE_ALIGN,
  47                                                 NULL);
  48         if (!iommu_devinfo_cache) {
  49                 pr_debug("Couldn't create devinfo cache\n");
  50                 kmem_cache_destroy(fsl_pamu_domain_cache);
  51                 return -ENOMEM;
  52         }
  53
  54         return 0;
  55 }
  56
  57 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,
  58                               u32 val)
  59 {
  60         int ret = 0;
  61         unsigned long flags;
  62
  63         spin_lock_irqsave(&iommu_lock, flags);
  64         ret = pamu_update_paace_stash(liodn, val);
  65         if (ret) {
  66                 pr_debug("Failed to update SPAACE for liodn %d\n ", liodn);
  67                 spin_unlock_irqrestore(&iommu_lock, flags);
  68                 return ret;
  69         }
  70
  71         spin_unlock_irqrestore(&iommu_lock, flags);
  72
  73         return ret;
  74 }
  75
  76 /* Set the geometry parameters for a LIODN */
  77 static int pamu_set_liodn(struct fsl_dma_domain *dma_domain, struct device *dev,
  78                           int liodn)
  79 {
  80         u32 omi_index = ~(u32)0;
  81         unsigned long flags;
  82         int ret;
  83
  84         /*
  85          * Configure the omi_index at the geometry setup time.
  86          * This is a static value which depends on the type of
  87          * device and would not change thereafter.
  88          */
  89         get_ome_index(&omi_index, dev);
  90
  91         spin_lock_irqsave(&iommu_lock, flags);
  92         ret = pamu_disable_liodn(liodn);
  93         if (ret)
  94                 goto out_unlock;
  95         ret = pamu_config_ppaace(liodn, omi_index, dma_domain->stash_id, 0);
  96         if (ret)
  97                 goto out_unlock;
  98         ret = pamu_config_ppaace(liodn, ~(u32)0, dma_domain->stash_id,
  99                                  PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE);
 100 out_unlock:
 101         spin_unlock_irqrestore(&iommu_lock, flags);
 102         if (ret) {
 103                 pr_debug("PAACE configuration failed for liodn %d\n",
 104                          liodn);
 105         }
 106         return ret;
 107 }
 108
 109 static void remove_device_ref(struct device_domain_info *info)
 110 {
 111         unsigned long flags;
 112
 113         list_del(&info->link);
 114         spin_lock_irqsave(&iommu_lock, flags);
 115         pamu_disable_liodn(info->liodn);
 116         spin_unlock_irqrestore(&iommu_lock, flags);
 117         spin_lock_irqsave(&device_domain_lock, flags);
 118         dev_iommu_priv_set(info->dev, NULL);
 119         kmem_cache_free(iommu_devinfo_cache, info);
 120         spin_unlock_irqrestore(&device_domain_lock, flags);
 121 }
 122
 123 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)
 124 {
 125         struct device_domain_info *info, *tmp;
 126         unsigned long flags;
 127
 128         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 129         /* Remove the device from the domain device list */
 130         list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {
 131                 if (!dev || (info->dev == dev))
 132                         remove_device_ref(info);
 133         }
 134         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 135 }
 136
 137 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)
 138 {
 139         struct device_domain_info *info, *old_domain_info;
 140         unsigned long flags;
 141
 142         spin_lock_irqsave(&device_domain_lock, flags);
 143         /*
 144          * Check here if the device is already attached to domain or not.
 145          * If the device is already attached to a domain detach it.
 146          */
 147         old_domain_info = dev_iommu_priv_get(dev);
 148         if (old_domain_info && old_domain_info->domain != dma_domain) {
 149                 spin_unlock_irqrestore(&device_domain_lock, flags);
 150                 detach_device(dev, old_domain_info->domain);
 151                 spin_lock_irqsave(&device_domain_lock, flags);
 152         }
 153
 154         info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC);
 155
 156         info->dev = dev;
 157         info->liodn = liodn;
 158         info->domain = dma_domain;
 159
 160         list_add(&info->link, &dma_domain->devices);
 161         /*
 162          * In case of devices with multiple LIODNs just store
 163          * the info for the first LIODN as all
 164          * LIODNs share the same domain
 165          */
 166         if (!dev_iommu_priv_get(dev))
 167                 dev_iommu_priv_set(dev, info);
 168         spin_unlock_irqrestore(&device_domain_lock, flags);
 169 }
 170
 171 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
 172                                          dma_addr_t iova)
 173 {
 174         if (iova < domain->geometry.aperture_start ||
 175             iova > domain->geometry.aperture_end)
 176                 return 0;
 177         return iova;
 178 }
 179
 180 static bool fsl_pamu_capable(enum iommu_cap cap)
 181 {
 182         return cap == IOMMU_CAP_CACHE_COHERENCY;
 183 }
 184
 185 static void fsl_pamu_domain_free(struct iommu_domain *domain)
 186 {
 187         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 188
 189         /* remove all the devices from the device list */
 190         detach_device(NULL, dma_domain);
 191         kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
 192 }
 193
 194 static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type)
 195 {
 196         struct fsl_dma_domain *dma_domain;
 197
 198         if (type != IOMMU_DOMAIN_UNMANAGED)
 199                 return NULL;
 200
 201         dma_domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL);
 202         if (!dma_domain)
 203                 return NULL;
 204
 205         dma_domain->stash_id = ~(u32)0;
 206         INIT_LIST_HEAD(&dma_domain->devices);
 207         spin_lock_init(&dma_domain->domain_lock);
 208
 209         /* default geometry 64 GB i.e. maximum system address */
 210         dma_domain->iommu_domain. geometry.aperture_start = 0;
 211         dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1;
 212         dma_domain->iommu_domain.geometry.force_aperture = true;
 213
 214         return &dma_domain->iommu_domain;
 215 }
 216
 217 /* Update stash destination for all LIODNs associated with the domain */
 218 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)
 219 {
 220         struct device_domain_info *info;
 221         int ret = 0;
 222
 223         list_for_each_entry(info, &dma_domain->devices, link) {
 224                 ret = update_liodn_stash(info->liodn, dma_domain, val);
 225                 if (ret)
 226                         break;
 227         }
 228
 229         return ret;
 230 }
 231
 232 static int fsl_pamu_attach_device(struct iommu_domain *domain,
 233                                   struct device *dev)
 234 {
 235         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 236         unsigned long flags;
 237         int len, ret = 0, i;
 238         const u32 *liodn;
 239         struct pci_dev *pdev = NULL;
 240         struct pci_controller *pci_ctl;
 241
 242         /*
 243          * Use LIODN of the PCI controller while attaching a
 244          * PCI device.
 245          */
 246         if (dev_is_pci(dev)) {
 247                 pdev = to_pci_dev(dev);
 248                 pci_ctl = pci_bus_to_host(pdev->bus);
 249                 /*
 250                  * make dev point to pci controller device
 251                  * so we can get the LIODN programmed by
 252                  * u-boot.
 253                  */
 254                 dev = pci_ctl->parent;
 255         }
 256
 257         liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
 258         if (!liodn) {
 259                 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
 260                 return -EINVAL;
 261         }
 262
 263         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 264         for (i = 0; i < len / sizeof(u32); i++) {
 265                 /* Ensure that LIODN value is valid */
 266                 if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
 267                         pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
 268                                  liodn[i], dev->of_node);
 269                         ret = -EINVAL;
 270                         break;
 271                 }
 272
 273                 attach_device(dma_domain, liodn[i], dev);
 274                 ret = pamu_set_liodn(dma_domain, dev, liodn[i]);
 275                 if (ret)
 276                         break;
 277                 ret = pamu_enable_liodn(liodn[i]);
 278                 if (ret)
 279                         break;
 280         }
 281         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 282         return ret;
 283 }
 284
 285 static void fsl_pamu_detach_device(struct iommu_domain *domain,
 286                                    struct device *dev)
 287 {
 288         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 289         const u32 *prop;
 290         int len;
 291         struct pci_dev *pdev = NULL;
 292         struct pci_controller *pci_ctl;
 293
 294         /*
 295          * Use LIODN of the PCI controller while detaching a
 296          * PCI device.
 297          */
 298         if (dev_is_pci(dev)) {
 299                 pdev = to_pci_dev(dev);
 300                 pci_ctl = pci_bus_to_host(pdev->bus);
 301                 /*
 302                  * make dev point to pci controller device
 303                  * so we can get the LIODN programmed by
 304                  * u-boot.
 305                  */
 306                 dev = pci_ctl->parent;
 307         }
 308
 309         prop = of_get_property(dev->of_node, "fsl,liodn", &len);
 310         if (prop)
 311                 detach_device(dev, dma_domain);
 312         else
 313                 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
 314 }
 315
 316 /* Set the domain stash attribute */
 317 int fsl_pamu_configure_l1_stash(struct iommu_domain *domain, u32 cpu)
 318 {
 319         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 320         unsigned long flags;
 321         int ret;
 322
 323         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 324         dma_domain->stash_id = get_stash_id(PAMU_ATTR_CACHE_L1, cpu);
 325         if (dma_domain->stash_id == ~(u32)0) {
 326                 pr_debug("Invalid stash attributes\n");
 327                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 328                 return -EINVAL;
 329         }
 330         ret = update_domain_stash(dma_domain, dma_domain->stash_id);
 331         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 332
 333         return ret;
 334 }
 335
 336 static struct iommu_group *get_device_iommu_group(struct device *dev)
 337 {
 338         struct iommu_group *group;
 339
 340         group = iommu_group_get(dev);
 341         if (!group)
 342                 group = iommu_group_alloc();
 343
 344         return group;
 345 }
 346
 347 static  bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl)
 348 {
 349         u32 version;
 350
 351         /* Check the PCI controller version number by readding BRR1 register */
 352         version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2));
 353         version &= PCI_FSL_BRR1_VER;
 354         /* If PCI controller version is >= 0x204 we can partition endpoints */
 355         return version >= 0x204;
 356 }
 357
 358 /* Get iommu group information from peer devices or devices on the parent bus */
 359 static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev)
 360 {
 361         struct pci_dev *tmp;
 362         struct iommu_group *group;
 363         struct pci_bus *bus = pdev->bus;
 364
 365         /*
 366          * Traverese the pci bus device list to get
 367          * the shared iommu group.
 368          */
 369         while (bus) {
 370                 list_for_each_entry(tmp, &bus->devices, bus_list) {
 371                         if (tmp == pdev)
 372                                 continue;
 373                         group = iommu_group_get(&tmp->dev);
 374                         if (group)
 375                                 return group;
 376                 }
 377
 378                 bus = bus->parent;
 379         }
 380
 381         return NULL;
 382 }
 383
 384 static struct iommu_group *get_pci_device_group(struct pci_dev *pdev)
 385 {
 386         struct pci_controller *pci_ctl;
 387         bool pci_endpt_partitioning;
 388         struct iommu_group *group = NULL;
 389
 390         pci_ctl = pci_bus_to_host(pdev->bus);
 391         pci_endpt_partitioning = check_pci_ctl_endpt_part(pci_ctl);
 392         /* We can partition PCIe devices so assign device group to the device */
 393         if (pci_endpt_partitioning) {
 394                 group = pci_device_group(&pdev->dev);
 395
 396                 /*
 397                  * PCIe controller is not a paritionable entity
 398                  * free the controller device iommu_group.
 399                  */
 400                 if (pci_ctl->parent->iommu_group)
 401                         iommu_group_remove_device(pci_ctl->parent);
 402         } else {
 403                 /*
 404                  * All devices connected to the controller will share the
 405                  * PCI controllers device group. If this is the first
 406                  * device to be probed for the pci controller, copy the
 407                  * device group information from the PCI controller device
 408                  * node and remove the PCI controller iommu group.
 409                  * For subsequent devices, the iommu group information can
 410                  * be obtained from sibling devices (i.e. from the bus_devices
 411                  * link list).
 412                  */
 413                 if (pci_ctl->parent->iommu_group) {
 414                         group = get_device_iommu_group(pci_ctl->parent);
 415                         iommu_group_remove_device(pci_ctl->parent);
 416                 } else {
 417                         group = get_shared_pci_device_group(pdev);
 418                 }
 419         }
 420
 421         if (!group)
 422                 group = ERR_PTR(-ENODEV);
 423
 424         return group;
 425 }
 426
 427 static struct iommu_group *fsl_pamu_device_group(struct device *dev)
 428 {
 429         struct iommu_group *group = ERR_PTR(-ENODEV);
 430         int len;
 431
 432         /*
 433          * For platform devices we allocate a separate group for
 434          * each of the devices.
 435          */
 436         if (dev_is_pci(dev))
 437                 group = get_pci_device_group(to_pci_dev(dev));
 438         else if (of_get_property(dev->of_node, "fsl,liodn", &len))
 439                 group = get_device_iommu_group(dev);
 440
 441         return group;
 442 }
 443
 444 static struct iommu_device *fsl_pamu_probe_device(struct device *dev)
 445 {
 446         return &pamu_iommu;
 447 }
 448
 449 static void fsl_pamu_release_device(struct device *dev)
 450 {
 451 }
 452
 453 static const struct iommu_ops fsl_pamu_ops = {
 454         .capable        = fsl_pamu_capable,
 455         .domain_alloc   = fsl_pamu_domain_alloc,
 456         .domain_free    = fsl_pamu_domain_free,
 457         .attach_dev     = fsl_pamu_attach_device,
 458         .detach_dev     = fsl_pamu_detach_device,
 459         .iova_to_phys   = fsl_pamu_iova_to_phys,
 460         .probe_device   = fsl_pamu_probe_device,
 461         .release_device = fsl_pamu_release_device,
 462         .device_group   = fsl_pamu_device_group,
 463 };
 464
 465 int __init pamu_domain_init(void)
 466 {
 467         int ret = 0;
 468
 469         ret = iommu_init_mempool();
 470         if (ret)
 471                 return ret;
 472
 473         ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0");
 474         if (ret)
 475                 return ret;
 476
 477         ret = iommu_device_register(&pamu_iommu, &fsl_pamu_ops, NULL);
 478         if (ret) {
 479                 iommu_device_sysfs_remove(&pamu_iommu);
 480                 pr_err("Can't register iommu device\n");
 481                 return ret;
 482         }
 483
 484         bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
 485         bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);
 486
 487         return ret;
 488 }