1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * Copyright (c) 2006, Intel Corporation.
5 * Copyright (C) 2006-2008 Intel Corporation
6 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
12 #include <linux/types.h>
13 #include <linux/kernel.h>
14 #include <linux/rbtree.h>
15 #include <linux/atomic.h>
16 #include <linux/dma-mapping.h>
21 unsigned long pfn_hi
; /* Highest allocated pfn */
22 unsigned long pfn_lo
; /* Lowest allocated pfn */
26 struct iova_cpu_rcache
;
28 #define IOVA_RANGE_CACHE_MAX_SIZE 6 /* log of max cached IOVA range size (in pages) */
29 #define MAX_GLOBAL_MAGS 32 /* magazines per bin */
33 unsigned long depot_size
;
34 struct iova_magazine
*depot
[MAX_GLOBAL_MAGS
];
35 struct iova_cpu_rcache __percpu
*cpu_rcaches
;
40 /* Call-Back from IOVA code into IOMMU drivers */
41 typedef void (* iova_flush_cb
)(struct iova_domain
*domain
);
43 /* Destructor for per-entry data */
44 typedef void (* iova_entry_dtor
)(unsigned long data
);
46 /* Number of entries per Flush Queue */
47 #define IOVA_FQ_SIZE 256
49 /* Timeout (in ms) after which entries are flushed from the Flush-Queue */
50 #define IOVA_FQ_TIMEOUT 10
52 /* Flush Queue entry for defered flushing */
53 struct iova_fq_entry
{
54 unsigned long iova_pfn
;
57 u64 counter
; /* Flush counter when this entrie was added */
60 /* Per-CPU Flush Queue structure */
62 struct iova_fq_entry entries
[IOVA_FQ_SIZE
];
67 /* holds all the iova translations for a domain */
69 spinlock_t iova_rbtree_lock
; /* Lock to protect update of rbtree */
70 struct rb_root rbroot
; /* iova domain rbtree root */
71 struct rb_node
*cached_node
; /* Save last alloced node */
72 struct rb_node
*cached32_node
; /* Save last 32-bit alloced node */
73 unsigned long granule
; /* pfn granularity for this domain */
74 unsigned long start_pfn
; /* Lower limit for this domain */
75 unsigned long dma_32bit_pfn
;
76 unsigned long max32_alloc_size
; /* Size of last failed allocation */
77 struct iova_fq __percpu
*fq
; /* Flush Queue */
79 atomic64_t fq_flush_start_cnt
; /* Number of TLB flushes that
82 atomic64_t fq_flush_finish_cnt
; /* Number of TLB flushes that
85 struct iova anchor
; /* rbtree lookup anchor */
86 struct iova_rcache rcaches
[IOVA_RANGE_CACHE_MAX_SIZE
]; /* IOVA range caches */
88 iova_flush_cb flush_cb
; /* Call-Back function to flush IOMMU
91 iova_entry_dtor entry_dtor
; /* IOMMU driver specific destructor for
94 struct timer_list fq_timer
; /* Timer to regularily empty the
96 atomic_t fq_timer_on
; /* 1 when timer is active, 0
98 struct hlist_node cpuhp_dead
;
101 static inline unsigned long iova_size(struct iova
*iova
)
103 return iova
->pfn_hi
- iova
->pfn_lo
+ 1;
106 static inline unsigned long iova_shift(struct iova_domain
*iovad
)
108 return __ffs(iovad
->granule
);
111 static inline unsigned long iova_mask(struct iova_domain
*iovad
)
113 return iovad
->granule
- 1;
116 static inline size_t iova_offset(struct iova_domain
*iovad
, dma_addr_t iova
)
118 return iova
& iova_mask(iovad
);
121 static inline size_t iova_align(struct iova_domain
*iovad
, size_t size
)
123 return ALIGN(size
, iovad
->granule
);
126 static inline dma_addr_t
iova_dma_addr(struct iova_domain
*iovad
, struct iova
*iova
)
128 return (dma_addr_t
)iova
->pfn_lo
<< iova_shift(iovad
);
131 static inline unsigned long iova_pfn(struct iova_domain
*iovad
, dma_addr_t iova
)
133 return iova
>> iova_shift(iovad
);
136 #if IS_ENABLED(CONFIG_IOMMU_IOVA)
137 int iova_cache_get(void);
138 void iova_cache_put(void);
140 void free_iova(struct iova_domain
*iovad
, unsigned long pfn
);
141 void __free_iova(struct iova_domain
*iovad
, struct iova
*iova
);
142 struct iova
*alloc_iova(struct iova_domain
*iovad
, unsigned long size
,
143 unsigned long limit_pfn
,
145 void free_iova_fast(struct iova_domain
*iovad
, unsigned long pfn
,
147 void queue_iova(struct iova_domain
*iovad
,
148 unsigned long pfn
, unsigned long pages
,
150 unsigned long alloc_iova_fast(struct iova_domain
*iovad
, unsigned long size
,
151 unsigned long limit_pfn
, bool flush_rcache
);
152 struct iova
*reserve_iova(struct iova_domain
*iovad
, unsigned long pfn_lo
,
153 unsigned long pfn_hi
);
154 void init_iova_domain(struct iova_domain
*iovad
, unsigned long granule
,
155 unsigned long start_pfn
);
156 int init_iova_flush_queue(struct iova_domain
*iovad
,
157 iova_flush_cb flush_cb
, iova_entry_dtor entry_dtor
);
158 struct iova
*find_iova(struct iova_domain
*iovad
, unsigned long pfn
);
159 void put_iova_domain(struct iova_domain
*iovad
);
161 static inline int iova_cache_get(void)
166 static inline void iova_cache_put(void)
170 static inline void free_iova(struct iova_domain
*iovad
, unsigned long pfn
)
174 static inline void __free_iova(struct iova_domain
*iovad
, struct iova
*iova
)
178 static inline struct iova
*alloc_iova(struct iova_domain
*iovad
,
180 unsigned long limit_pfn
,
186 static inline void free_iova_fast(struct iova_domain
*iovad
,
192 static inline void queue_iova(struct iova_domain
*iovad
,
193 unsigned long pfn
, unsigned long pages
,
198 static inline unsigned long alloc_iova_fast(struct iova_domain
*iovad
,
200 unsigned long limit_pfn
,
206 static inline struct iova
*reserve_iova(struct iova_domain
*iovad
,
207 unsigned long pfn_lo
,
208 unsigned long pfn_hi
)
213 static inline void init_iova_domain(struct iova_domain
*iovad
,
214 unsigned long granule
,
215 unsigned long start_pfn
)
219 static inline int init_iova_flush_queue(struct iova_domain
*iovad
,
220 iova_flush_cb flush_cb
,
221 iova_entry_dtor entry_dtor
)
226 static inline struct iova
*find_iova(struct iova_domain
*iovad
,
232 static inline void put_iova_domain(struct iova_domain
*iovad
)