// SPDX-License-Identifier: GPL-2.0
/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+#include <linux/atomic.h>
#include <linux/bitfield.h>
#include <linux/delay.h>
+#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/iopoll.h>
#include <linux/iommu.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
+#include <linux/shmem_fs.h>
#include <linux/sizes.h>
#include "panfrost_device.h"
#define mmu_write(dev, reg, data) writel(data, dev->iomem + reg)
#define mmu_read(dev, reg) readl(dev->iomem + reg)
-struct panfrost_mmu {
- struct io_pgtable_cfg pgtbl_cfg;
- struct io_pgtable_ops *pgtbl_ops;
- struct mutex lock;
-};
-
static int wait_ready(struct panfrost_device *pfdev, u32 as_nr)
{
int ret;
}
-static int mmu_hw_do_operation(struct panfrost_device *pfdev, u32 as_nr,
- u64 iova, size_t size, u32 op)
+static int mmu_hw_do_operation_locked(struct panfrost_device *pfdev, int as_nr,
+ u64 iova, size_t size, u32 op)
{
- unsigned long flags;
- int ret;
-
- spin_lock_irqsave(&pfdev->hwaccess_lock, flags);
+ if (as_nr < 0)
+ return 0;
if (op != AS_COMMAND_UNLOCK)
lock_region(pfdev, as_nr, iova, size);
write_cmd(pfdev, as_nr, op);
/* Wait for the flush to complete */
- ret = wait_ready(pfdev, as_nr);
+ return wait_ready(pfdev, as_nr);
+}
- spin_unlock_irqrestore(&pfdev->hwaccess_lock, flags);
+static int mmu_hw_do_operation(struct panfrost_device *pfdev,
+ struct panfrost_mmu *mmu,
+ u64 iova, size_t size, u32 op)
+{
+ int ret;
+ spin_lock(&pfdev->as_lock);
+ ret = mmu_hw_do_operation_locked(pfdev, mmu->as, iova, size, op);
+ spin_unlock(&pfdev->as_lock);
return ret;
}
-void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr)
+static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
{
- struct io_pgtable_cfg *cfg = &pfdev->mmu->pgtbl_cfg;
+ int as_nr = mmu->as;
+ struct io_pgtable_cfg *cfg = &mmu->pgtbl_cfg;
u64 transtab = cfg->arm_mali_lpae_cfg.transtab;
u64 memattr = cfg->arm_mali_lpae_cfg.memattr;
- mmu_write(pfdev, MMU_INT_CLEAR, ~0);
- mmu_write(pfdev, MMU_INT_MASK, ~0);
+ mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM);
mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), transtab & 0xffffffffUL);
mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), transtab >> 32);
write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
}
-static void mmu_disable(struct panfrost_device *pfdev, u32 as_nr)
+static void panfrost_mmu_disable(struct panfrost_device *pfdev, u32 as_nr)
{
+ mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM);
+
mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), 0);
mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), 0);
write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
}
+u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
+{
+ int as;
+
+ spin_lock(&pfdev->as_lock);
+
+ as = mmu->as;
+ if (as >= 0) {
+ int en = atomic_inc_return(&mmu->as_count);
+ WARN_ON(en >= NUM_JOB_SLOTS);
+
+ list_move(&mmu->list, &pfdev->as_lru_list);
+ goto out;
+ }
+
+ /* Check for a free AS */
+ as = ffz(pfdev->as_alloc_mask);
+ if (!(BIT(as) & pfdev->features.as_present)) {
+ struct panfrost_mmu *lru_mmu;
+
+ list_for_each_entry_reverse(lru_mmu, &pfdev->as_lru_list, list) {
+ if (!atomic_read(&lru_mmu->as_count))
+ break;
+ }
+ WARN_ON(&lru_mmu->list == &pfdev->as_lru_list);
+
+ list_del_init(&lru_mmu->list);
+ as = lru_mmu->as;
+
+ WARN_ON(as < 0);
+ lru_mmu->as = -1;
+ }
+
+ /* Assign the free or reclaimed AS to the FD */
+ mmu->as = as;
+ set_bit(as, &pfdev->as_alloc_mask);
+ atomic_set(&mmu->as_count, 1);
+ list_add(&mmu->list, &pfdev->as_lru_list);
+
+ dev_dbg(pfdev->dev, "Assigned AS%d to mmu %p, alloc_mask=%lx", as, mmu, pfdev->as_alloc_mask);
+
+ panfrost_mmu_enable(pfdev, mmu);
+
+out:
+ spin_unlock(&pfdev->as_lock);
+ return as;
+}
+
+void panfrost_mmu_as_put(struct panfrost_device *pfdev, struct panfrost_mmu *mmu)
+{
+ atomic_dec(&mmu->as_count);
+ WARN_ON(atomic_read(&mmu->as_count) < 0);
+}
+
+void panfrost_mmu_reset(struct panfrost_device *pfdev)
+{
+ struct panfrost_mmu *mmu, *mmu_tmp;
+
+ spin_lock(&pfdev->as_lock);
+
+ pfdev->as_alloc_mask = 0;
+
+ list_for_each_entry_safe(mmu, mmu_tmp, &pfdev->as_lru_list, list) {
+ mmu->as = -1;
+ atomic_set(&mmu->as_count, 0);
+ list_del_init(&mmu->list);
+ }
+
+ spin_unlock(&pfdev->as_lock);
+
+ mmu_write(pfdev, MMU_INT_CLEAR, ~0);
+ mmu_write(pfdev, MMU_INT_MASK, ~0);
+}
+
static size_t get_pgsize(u64 addr, size_t size)
{
if (addr & (SZ_2M - 1) || size < SZ_2M)
return SZ_2M;
}
-int panfrost_mmu_map(struct panfrost_gem_object *bo)
+void panfrost_mmu_flush_range(struct panfrost_device *pfdev,
+ struct panfrost_mmu *mmu,
+ u64 iova, size_t size)
{
- struct drm_gem_object *obj = &bo->base.base;
- struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
- struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops;
- u64 iova = bo->node.start << PAGE_SHIFT;
- unsigned int count;
- struct scatterlist *sgl;
- struct sg_table *sgt;
- int ret;
+ if (mmu->as < 0)
+ return;
- if (WARN_ON(bo->is_mapped))
- return 0;
+ pm_runtime_get_noresume(pfdev->dev);
- sgt = drm_gem_shmem_get_pages_sgt(obj);
- if (WARN_ON(IS_ERR(sgt)))
- return PTR_ERR(sgt);
+ /* Flush the PTs only if we're already awake */
+ if (pm_runtime_active(pfdev->dev))
+ mmu_hw_do_operation(pfdev, mmu, iova, size, AS_COMMAND_FLUSH_PT);
- ret = pm_runtime_get_sync(pfdev->dev);
- if (ret < 0)
- return ret;
+ pm_runtime_put_sync_autosuspend(pfdev->dev);
+}
- mutex_lock(&pfdev->mmu->lock);
+static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu,
+ u64 iova, int prot, struct sg_table *sgt)
+{
+ unsigned int count;
+ struct scatterlist *sgl;
+ struct io_pgtable_ops *ops = mmu->pgtbl_ops;
+ u64 start_iova = iova;
for_each_sg(sgt->sgl, sgl, sgt->nents, count) {
unsigned long paddr = sg_dma_address(sgl);
size_t len = sg_dma_len(sgl);
- dev_dbg(pfdev->dev, "map: iova=%llx, paddr=%lx, len=%zx", iova, paddr, len);
+ dev_dbg(pfdev->dev, "map: as=%d, iova=%llx, paddr=%lx, len=%zx", mmu->as, iova, paddr, len);
while (len) {
size_t pgsize = get_pgsize(iova | paddr, len);
- ops->map(ops, iova, paddr, pgsize, IOMMU_WRITE | IOMMU_READ);
+ ops->map(ops, iova, paddr, pgsize, prot);
iova += pgsize;
paddr += pgsize;
len -= pgsize;
}
}
- mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT,
- bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT);
+ panfrost_mmu_flush_range(pfdev, mmu, start_iova, iova - start_iova);
+
+ return 0;
+}
+
+int panfrost_mmu_map(struct panfrost_gem_object *bo)
+{
+ struct drm_gem_object *obj = &bo->base.base;
+ struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
+ struct sg_table *sgt;
+ int prot = IOMMU_READ | IOMMU_WRITE;
- mutex_unlock(&pfdev->mmu->lock);
+ if (WARN_ON(bo->is_mapped))
+ return 0;
+
+ if (bo->noexec)
+ prot |= IOMMU_NOEXEC;
- pm_runtime_mark_last_busy(pfdev->dev);
- pm_runtime_put_autosuspend(pfdev->dev);
+ sgt = drm_gem_shmem_get_pages_sgt(obj);
+ if (WARN_ON(IS_ERR(sgt)))
+ return PTR_ERR(sgt);
+
+ mmu_map_sg(pfdev, bo->mmu, bo->node.start << PAGE_SHIFT, prot, sgt);
bo->is_mapped = true;
return 0;
{
struct drm_gem_object *obj = &bo->base.base;
struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
- struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops;
+ struct io_pgtable_ops *ops = bo->mmu->pgtbl_ops;
u64 iova = bo->node.start << PAGE_SHIFT;
size_t len = bo->node.size << PAGE_SHIFT;
size_t unmapped_len = 0;
- int ret;
if (WARN_ON(!bo->is_mapped))
return;
- dev_dbg(pfdev->dev, "unmap: iova=%llx, len=%zx", iova, len);
-
- ret = pm_runtime_get_sync(pfdev->dev);
- if (ret < 0)
- return;
-
- mutex_lock(&pfdev->mmu->lock);
+ dev_dbg(pfdev->dev, "unmap: as=%d, iova=%llx, len=%zx", bo->mmu->as, iova, len);
while (unmapped_len < len) {
size_t unmapped_page;
size_t pgsize = get_pgsize(iova, len - unmapped_len);
- unmapped_page = ops->unmap(ops, iova, pgsize, NULL);
- if (!unmapped_page)
- break;
-
- iova += unmapped_page;
- unmapped_len += unmapped_page;
+ if (ops->iova_to_phys(ops, iova)) {
+ unmapped_page = ops->unmap(ops, iova, pgsize, NULL);
+ WARN_ON(unmapped_page != pgsize);
+ }
+ iova += pgsize;
+ unmapped_len += pgsize;
}
- mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT,
- bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT);
-
- mutex_unlock(&pfdev->mmu->lock);
-
- pm_runtime_mark_last_busy(pfdev->dev);
- pm_runtime_put_autosuspend(pfdev->dev);
+ panfrost_mmu_flush_range(pfdev, bo->mmu, bo->node.start << PAGE_SHIFT, len);
bo->is_mapped = false;
}
static void mmu_tlb_inv_context_s1(void *cookie)
-{
- struct panfrost_device *pfdev = cookie;
-
- mmu_hw_do_operation(pfdev, 0, 0, ~0UL, AS_COMMAND_FLUSH_MEM);
-}
+{}
static void mmu_tlb_sync_context(void *cookie)
{
.tlb_flush_leaf = mmu_tlb_flush_leaf,
};
+int panfrost_mmu_pgtable_alloc(struct panfrost_file_priv *priv)
+{
+ struct panfrost_mmu *mmu = &priv->mmu;
+ struct panfrost_device *pfdev = priv->pfdev;
+
+ INIT_LIST_HEAD(&mmu->list);
+ mmu->as = -1;
+
+ mmu->pgtbl_cfg = (struct io_pgtable_cfg) {
+ .pgsize_bitmap = SZ_4K | SZ_2M,
+ .ias = FIELD_GET(0xff, pfdev->features.mmu_features),
+ .oas = FIELD_GET(0xff00, pfdev->features.mmu_features),
+ .tlb = &mmu_tlb_ops,
+ .iommu_dev = pfdev->dev,
+ };
+
+ mmu->pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &mmu->pgtbl_cfg,
+ priv);
+ if (!mmu->pgtbl_ops)
+ return -EINVAL;
+
+ return 0;
+}
+
+void panfrost_mmu_pgtable_free(struct panfrost_file_priv *priv)
+{
+ struct panfrost_device *pfdev = priv->pfdev;
+ struct panfrost_mmu *mmu = &priv->mmu;
+
+ spin_lock(&pfdev->as_lock);
+ if (mmu->as >= 0) {
+ pm_runtime_get_noresume(pfdev->dev);
+ if (pm_runtime_active(pfdev->dev))
+ panfrost_mmu_disable(pfdev, mmu->as);
+ pm_runtime_put_autosuspend(pfdev->dev);
+
+ clear_bit(mmu->as, &pfdev->as_alloc_mask);
+ clear_bit(mmu->as, &pfdev->as_in_use_mask);
+ list_del(&mmu->list);
+ }
+ spin_unlock(&pfdev->as_lock);
+
+ free_io_pgtable_ops(mmu->pgtbl_ops);
+}
+
+static struct drm_mm_node *addr_to_drm_mm_node(struct panfrost_device *pfdev, int as, u64 addr)
+{
+ struct drm_mm_node *node = NULL;
+ u64 offset = addr >> PAGE_SHIFT;
+ struct panfrost_mmu *mmu;
+
+ spin_lock(&pfdev->as_lock);
+ list_for_each_entry(mmu, &pfdev->as_lru_list, list) {
+ struct panfrost_file_priv *priv;
+ if (as != mmu->as)
+ continue;
+
+ priv = container_of(mmu, struct panfrost_file_priv, mmu);
+ drm_mm_for_each_node(node, &priv->mm) {
+ if (offset >= node->start && offset < (node->start + node->size))
+ goto out;
+ }
+ }
+
+out:
+ spin_unlock(&pfdev->as_lock);
+ return node;
+}
+
+#define NUM_FAULT_PAGES (SZ_2M / PAGE_SIZE)
+
+int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, u64 addr)
+{
+ int ret, i;
+ struct drm_mm_node *node;
+ struct panfrost_gem_object *bo;
+ struct address_space *mapping;
+ pgoff_t page_offset;
+ struct sg_table *sgt;
+ struct page **pages;
+
+ node = addr_to_drm_mm_node(pfdev, as, addr);
+ if (!node)
+ return -ENOENT;
+
+ bo = drm_mm_node_to_panfrost_bo(node);
+ if (!bo->is_heap) {
+ dev_WARN(pfdev->dev, "matching BO is not heap type (GPU VA = %llx)",
+ node->start << PAGE_SHIFT);
+ return -EINVAL;
+ }
+ WARN_ON(bo->mmu->as != as);
+
+ /* Assume 2MB alignment and size multiple */
+ addr &= ~((u64)SZ_2M - 1);
+ page_offset = addr >> PAGE_SHIFT;
+ page_offset -= node->start;
+
+ mutex_lock(&bo->base.pages_lock);
+
+ if (!bo->base.pages) {
+ bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M,
+ sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO);
+ if (!bo->sgts) {
+ mutex_unlock(&bo->base.pages_lock);
+ return -ENOMEM;
+ }
+
+ pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT,
+ sizeof(struct page *), GFP_KERNEL | __GFP_ZERO);
+ if (!pages) {
+ kfree(bo->sgts);
+ bo->sgts = NULL;
+ mutex_unlock(&bo->base.pages_lock);
+ return -ENOMEM;
+ }
+ bo->base.pages = pages;
+ bo->base.pages_use_count = 1;
+ } else
+ pages = bo->base.pages;
+
+ mapping = bo->base.base.filp->f_mapping;
+ mapping_set_unevictable(mapping);
+
+ for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) {
+ pages[i] = shmem_read_mapping_page(mapping, i);
+ if (IS_ERR(pages[i])) {
+ mutex_unlock(&bo->base.pages_lock);
+ ret = PTR_ERR(pages[i]);
+ goto err_pages;
+ }
+ }
+
+ mutex_unlock(&bo->base.pages_lock);
+
+ sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)];
+ ret = sg_alloc_table_from_pages(sgt, pages + page_offset,
+ NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL);
+ if (ret)
+ goto err_pages;
+
+ if (!dma_map_sg(pfdev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL)) {
+ ret = -EINVAL;
+ goto err_map;
+ }
+
+ mmu_map_sg(pfdev, bo->mmu, addr, IOMMU_WRITE | IOMMU_READ | IOMMU_NOEXEC, sgt);
+
+ bo->is_mapped = true;
+
+ dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr);
+
+ return 0;
+
+err_map:
+ sg_free_table(sgt);
+err_pages:
+ drm_gem_shmem_put_pages(&bo->base);
+ return ret;
+}
+
static const char *access_type_name(struct panfrost_device *pfdev,
u32 fault_status)
{
static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data)
{
struct panfrost_device *pfdev = data;
- u32 status = mmu_read(pfdev, MMU_INT_STAT);
- int i;
- if (!status)
+ if (!mmu_read(pfdev, MMU_INT_STAT))
return IRQ_NONE;
- dev_err(pfdev->dev, "mmu irq status=%x\n", status);
+ mmu_write(pfdev, MMU_INT_MASK, 0);
+ return IRQ_WAKE_THREAD;
+}
+
+static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data)
+{
+ struct panfrost_device *pfdev = data;
+ u32 status = mmu_read(pfdev, MMU_INT_RAWSTAT);
+ int i, ret;
for (i = 0; status; i++) {
u32 mask = BIT(i) | BIT(i + 16);
access_type = (fault_status >> 8) & 0x3;
source_id = (fault_status >> 16);
+ /* Page fault only */
+ if ((status & mask) == BIT(i)) {
+ WARN_ON(exception_type < 0xC1 || exception_type > 0xC4);
+
+ ret = panfrost_mmu_map_fault_addr(pfdev, i, addr);
+ if (!ret) {
+ mmu_write(pfdev, MMU_INT_CLEAR, BIT(i));
+ status &= ~mask;
+ continue;
+ }
+ }
+
/* terminal fault, print info about the fault */
dev_err(pfdev->dev,
"Unhandled Page fault in AS%d at VA 0x%016llX\n"
status &= ~mask;
}
+ mmu_write(pfdev, MMU_INT_MASK, ~0);
return IRQ_HANDLED;
};
int panfrost_mmu_init(struct panfrost_device *pfdev)
{
- struct io_pgtable_ops *pgtbl_ops;
int err, irq;
- pfdev->mmu = devm_kzalloc(pfdev->dev, sizeof(*pfdev->mmu), GFP_KERNEL);
- if (!pfdev->mmu)
- return -ENOMEM;
-
- mutex_init(&pfdev->mmu->lock);
-
irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu");
if (irq <= 0)
return -ENODEV;
- err = devm_request_irq(pfdev->dev, irq, panfrost_mmu_irq_handler,
- IRQF_SHARED, "mmu", pfdev);
+ err = devm_request_threaded_irq(pfdev->dev, irq, panfrost_mmu_irq_handler,
+ panfrost_mmu_irq_handler_thread,
+ IRQF_SHARED, "mmu", pfdev);
if (err) {
dev_err(pfdev->dev, "failed to request mmu irq");
return err;
}
- mmu_write(pfdev, MMU_INT_CLEAR, ~0);
- mmu_write(pfdev, MMU_INT_MASK, ~0);
-
- pfdev->mmu->pgtbl_cfg = (struct io_pgtable_cfg) {
- .pgsize_bitmap = SZ_4K | SZ_2M,
- .ias = FIELD_GET(0xff, pfdev->features.mmu_features),
- .oas = FIELD_GET(0xff00, pfdev->features.mmu_features),
- .tlb = &mmu_tlb_ops,
- .iommu_dev = pfdev->dev,
- };
-
- pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &pfdev->mmu->pgtbl_cfg,
- pfdev);
- if (!pgtbl_ops)
- return -ENOMEM;
-
- pfdev->mmu->pgtbl_ops = pgtbl_ops;
-
- panfrost_mmu_enable(pfdev, 0);
return 0;
}
void panfrost_mmu_fini(struct panfrost_device *pfdev)
{
mmu_write(pfdev, MMU_INT_MASK, 0);
- mmu_disable(pfdev, 0);
-
- free_io_pgtable_ops(pfdev->mmu->pgtbl_ops);
}
projects / mirror_ubuntu-focal-kernel.git / blobdiff