include/linux/io-pgtable.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __IO_PGTABLE_H
   3 #define __IO_PGTABLE_H
   4
   5 #include <linux/bitops.h>
   6 #include <linux/iommu.h>
   7
   8 /*
   9  * Public API for use by IOMMU drivers
  10  */
  11 enum io_pgtable_fmt {
  12         ARM_32_LPAE_S1,
  13         ARM_32_LPAE_S2,
  14         ARM_64_LPAE_S1,
  15         ARM_64_LPAE_S2,
  16         ARM_V7S,
  17         ARM_MALI_LPAE,
  18         IO_PGTABLE_NUM_FMTS,
  19 };
  20
  21 /**
  22  * struct iommu_flush_ops - IOMMU callbacks for TLB and page table management.
  23  *
  24  * @tlb_flush_all:  Synchronously invalidate the entire TLB context.
  25  * @tlb_flush_walk: Synchronously invalidate all intermediate TLB state
  26  *                  (sometimes referred to as the "walk cache") for a virtual
  27  *                  address range.
  28  * @tlb_flush_leaf: Synchronously invalidate all leaf TLB state for a virtual
  29  *                  address range.
  30  * @tlb_add_page:   Optional callback to queue up leaf TLB invalidation for a
  31  *                  single page.  IOMMUs that cannot batch TLB invalidation
  32  *                  operations efficiently will typically issue them here, but
  33  *                  others may decide to update the iommu_iotlb_gather structure
  34  *                  and defer the invalidation until iommu_tlb_sync() instead.
  35  *
  36  * Note that these can all be called in atomic context and must therefore
  37  * not block.
  38  */
  39 struct iommu_flush_ops {
  40         void (*tlb_flush_all)(void *cookie);
  41         void (*tlb_flush_walk)(unsigned long iova, size_t size, size_t granule,
  42                                void *cookie);
  43         void (*tlb_flush_leaf)(unsigned long iova, size_t size, size_t granule,
  44                                void *cookie);
  45         void (*tlb_add_page)(struct iommu_iotlb_gather *gather,
  46                              unsigned long iova, size_t granule, void *cookie);
  47 };
  48
  49 /**
  50  * struct io_pgtable_cfg - Configuration data for a set of page tables.
  51  *
  52  * @quirks:        A bitmap of hardware quirks that require some special
  53  *                 action by the low-level page table allocator.
  54  * @pgsize_bitmap: A bitmap of page sizes supported by this set of page
  55  *                 tables.
  56  * @ias:           Input address (iova) size, in bits.
  57  * @oas:           Output address (paddr) size, in bits.
  58  * @coherent_walk  A flag to indicate whether or not page table walks made
  59  *                 by the IOMMU are coherent with the CPU caches.
  60  * @tlb:           TLB management callbacks for this set of tables.
  61  * @iommu_dev:     The device representing the DMA configuration for the
  62  *                 page table walker.
  63  */
  64 struct io_pgtable_cfg {
  65         /*
  66          * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in
  67          *      stage 1 PTEs, for hardware which insists on validating them
  68          *      even in non-secure state where they should normally be ignored.
  69          *
  70          * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and
  71          *      IOMMU_NOEXEC flags and map everything with full access, for
  72          *      hardware which does not implement the permissions of a given
  73          *      format, and/or requires some format-specific default value.
  74          *
  75          * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid
  76          *      (unmapped) entries but the hardware might do so anyway, perform
  77          *      TLB maintenance when mapping as well as when unmapping.
  78          *
  79          * IO_PGTABLE_QUIRK_ARM_MTK_EXT: (ARM v7s format) MediaTek IOMMUs extend
  80          *      to support up to 34 bits PA where the bit32 and bit33 are
  81          *      encoded in the bit9 and bit4 of the PTE respectively.
  82          *
  83          * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs
  84          *      on unmap, for DMA domains using the flush queue mechanism for
  85          *      delayed invalidation.
  86          */
  87         #define IO_PGTABLE_QUIRK_ARM_NS         BIT(0)
  88         #define IO_PGTABLE_QUIRK_NO_PERMS       BIT(1)
  89         #define IO_PGTABLE_QUIRK_TLBI_ON_MAP    BIT(2)
  90         #define IO_PGTABLE_QUIRK_ARM_MTK_EXT    BIT(3)
  91         #define IO_PGTABLE_QUIRK_NON_STRICT     BIT(4)
  92         unsigned long                   quirks;
  93         unsigned long                   pgsize_bitmap;
  94         unsigned int                    ias;
  95         unsigned int                    oas;
  96         bool                            coherent_walk;
  97         const struct iommu_flush_ops    *tlb;
  98         struct device                   *iommu_dev;
  99
 100         /* Low-level data specific to the table format */
 101         union {
 102                 struct {
 103                         u64     ttbr[2];
 104                         u64     tcr;
 105                         u64     mair;
 106                 } arm_lpae_s1_cfg;
 107
 108                 struct {
 109                         u64     vttbr;
 110                         u64     vtcr;
 111                 } arm_lpae_s2_cfg;
 112
 113                 struct {
 114                         u32     ttbr[2];
 115                         u32     tcr;
 116                         u32     nmrr;
 117                         u32     prrr;
 118                 } arm_v7s_cfg;
 119
 120                 struct {
 121                         u64     transtab;
 122                         u64     memattr;
 123                 } arm_mali_lpae_cfg;
 124         };
 125 };
 126
 127 /**
 128  * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers.
 129  *
 130  * @map:          Map a physically contiguous memory region.
 131  * @unmap:        Unmap a physically contiguous memory region.
 132  * @iova_to_phys: Translate iova to physical address.
 133  *
 134  * These functions map directly onto the iommu_ops member functions with
 135  * the same names.
 136  */
 137 struct io_pgtable_ops {
 138         int (*map)(struct io_pgtable_ops *ops, unsigned long iova,
 139                    phys_addr_t paddr, size_t size, int prot);
 140         size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
 141                         size_t size, struct iommu_iotlb_gather *gather);
 142         phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops,
 143                                     unsigned long iova);
 144 };
 145
 146 /**
 147  * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU.
 148  *
 149  * @fmt:    The page table format.
 150  * @cfg:    The page table configuration. This will be modified to represent
 151  *          the configuration actually provided by the allocator (e.g. the
 152  *          pgsize_bitmap may be restricted).
 153  * @cookie: An opaque token provided by the IOMMU driver and passed back to
 154  *          the callback routines in cfg->tlb.
 155  */
 156 struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt,
 157                                             struct io_pgtable_cfg *cfg,
 158                                             void *cookie);
 159
 160 /**
 161  * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller
 162  *                         *must* ensure that the page table is no longer
 163  *                         live, but the TLB can be dirty.
 164  *
 165  * @ops: The ops returned from alloc_io_pgtable_ops.
 166  */
 167 void free_io_pgtable_ops(struct io_pgtable_ops *ops);
 168
 169
 170 /*
 171  * Internal structures for page table allocator implementations.
 172  */
 173
 174 /**
 175  * struct io_pgtable - Internal structure describing a set of page tables.
 176  *
 177  * @fmt:    The page table format.
 178  * @cookie: An opaque token provided by the IOMMU driver and passed back to
 179  *          any callback routines.
 180  * @cfg:    A copy of the page table configuration.
 181  * @ops:    The page table operations in use for this set of page tables.
 182  */
 183 struct io_pgtable {
 184         enum io_pgtable_fmt     fmt;
 185         void                    *cookie;
 186         struct io_pgtable_cfg   cfg;
 187         struct io_pgtable_ops   ops;
 188 };
 189
 190 #define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops)
 191
 192 static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop)
 193 {
 194         iop->cfg.tlb->tlb_flush_all(iop->cookie);
 195 }
 196
 197 static inline void
 198 io_pgtable_tlb_flush_walk(struct io_pgtable *iop, unsigned long iova,
 199                           size_t size, size_t granule)
 200 {
 201         iop->cfg.tlb->tlb_flush_walk(iova, size, granule, iop->cookie);
 202 }
 203
 204 static inline void
 205 io_pgtable_tlb_flush_leaf(struct io_pgtable *iop, unsigned long iova,
 206                           size_t size, size_t granule)
 207 {
 208         iop->cfg.tlb->tlb_flush_leaf(iova, size, granule, iop->cookie);
 209 }
 210
 211 static inline void
 212 io_pgtable_tlb_add_page(struct io_pgtable *iop,
 213                         struct iommu_iotlb_gather * gather, unsigned long iova,
 214                         size_t granule)
 215 {
 216         if (iop->cfg.tlb->tlb_add_page)
 217                 iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie);
 218 }
 219
 220 /**
 221  * struct io_pgtable_init_fns - Alloc/free a set of page tables for a
 222  *                              particular format.
 223  *
 224  * @alloc: Allocate a set of page tables described by cfg.
 225  * @free:  Free the page tables associated with iop.
 226  */
 227 struct io_pgtable_init_fns {
 228         struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie);
 229         void (*free)(struct io_pgtable *iop);
 230 };
 231
 232 extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns;
 233 extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns;
 234 extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
 235 extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
 236 extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns;
 237 extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns;
 238
 239 #endif /* __IO_PGTABLE_H */