2 * Copyright (c) 2006, Intel Corporation.
4 * This file is released under the GPLv2.
6 * Copyright (C) 2006-2008 Intel Corporation
7 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
14 #include <linux/types.h>
15 #include <linux/kernel.h>
16 #include <linux/rbtree.h>
17 #include <linux/atomic.h>
18 #include <linux/dma-mapping.h>
23 unsigned long pfn_hi
; /* Highest allocated pfn */
24 unsigned long pfn_lo
; /* Lowest allocated pfn */
28 struct iova_cpu_rcache
;
30 #define IOVA_RANGE_CACHE_MAX_SIZE 6 /* log of max cached IOVA range size (in pages) */
31 #define MAX_GLOBAL_MAGS 32 /* magazines per bin */
35 unsigned long depot_size
;
36 struct iova_magazine
*depot
[MAX_GLOBAL_MAGS
];
37 struct iova_cpu_rcache __percpu
*cpu_rcaches
;
42 /* Call-Back from IOVA code into IOMMU drivers */
43 typedef void (* iova_flush_cb
)(struct iova_domain
*domain
);
45 /* Destructor for per-entry data */
46 typedef void (* iova_entry_dtor
)(unsigned long data
);
48 /* Number of entries per Flush Queue */
49 #define IOVA_FQ_SIZE 256
51 /* Timeout (in ms) after which entries are flushed from the Flush-Queue */
52 #define IOVA_FQ_TIMEOUT 10
54 /* Flush Queue entry for defered flushing */
55 struct iova_fq_entry
{
56 unsigned long iova_pfn
;
59 u64 counter
; /* Flush counter when this entrie was added */
62 /* Per-CPU Flush Queue structure */
64 struct iova_fq_entry entries
[IOVA_FQ_SIZE
];
69 /* holds all the iova translations for a domain */
71 spinlock_t iova_rbtree_lock
; /* Lock to protect update of rbtree */
72 struct rb_root rbroot
; /* iova domain rbtree root */
73 struct rb_node
*cached32_node
; /* Save last alloced node */
74 unsigned long granule
; /* pfn granularity for this domain */
75 unsigned long start_pfn
; /* Lower limit for this domain */
76 unsigned long dma_32bit_pfn
;
77 struct iova_rcache rcaches
[IOVA_RANGE_CACHE_MAX_SIZE
]; /* IOVA range caches */
79 iova_flush_cb flush_cb
; /* Call-Back function to flush IOMMU
82 iova_entry_dtor entry_dtor
; /* IOMMU driver specific destructor for
85 struct iova_fq __percpu
*fq
; /* Flush Queue */
87 atomic64_t fq_flush_start_cnt
; /* Number of TLB flushes that
90 atomic64_t fq_flush_finish_cnt
; /* Number of TLB flushes that
93 struct timer_list fq_timer
; /* Timer to regularily empty the
95 atomic_t fq_timer_on
; /* 1 when timer is active, 0
99 static inline unsigned long iova_size(struct iova
*iova
)
101 return iova
->pfn_hi
- iova
->pfn_lo
+ 1;
104 static inline unsigned long iova_shift(struct iova_domain
*iovad
)
106 return __ffs(iovad
->granule
);
109 static inline unsigned long iova_mask(struct iova_domain
*iovad
)
111 return iovad
->granule
- 1;
114 static inline size_t iova_offset(struct iova_domain
*iovad
, dma_addr_t iova
)
116 return iova
& iova_mask(iovad
);
119 static inline size_t iova_align(struct iova_domain
*iovad
, size_t size
)
121 return ALIGN(size
, iovad
->granule
);
124 static inline dma_addr_t
iova_dma_addr(struct iova_domain
*iovad
, struct iova
*iova
)
126 return (dma_addr_t
)iova
->pfn_lo
<< iova_shift(iovad
);
129 static inline unsigned long iova_pfn(struct iova_domain
*iovad
, dma_addr_t iova
)
131 return iova
>> iova_shift(iovad
);
134 #if IS_ENABLED(CONFIG_IOMMU_IOVA)
135 int iova_cache_get(void);
136 void iova_cache_put(void);
138 struct iova
*alloc_iova_mem(void);
139 void free_iova_mem(struct iova
*iova
);
140 void free_iova(struct iova_domain
*iovad
, unsigned long pfn
);
141 void __free_iova(struct iova_domain
*iovad
, struct iova
*iova
);
142 struct iova
*alloc_iova(struct iova_domain
*iovad
, unsigned long size
,
143 unsigned long limit_pfn
,
145 void free_iova_fast(struct iova_domain
*iovad
, unsigned long pfn
,
147 void queue_iova(struct iova_domain
*iovad
,
148 unsigned long pfn
, unsigned long pages
,
150 unsigned long alloc_iova_fast(struct iova_domain
*iovad
, unsigned long size
,
151 unsigned long limit_pfn
);
152 struct iova
*reserve_iova(struct iova_domain
*iovad
, unsigned long pfn_lo
,
153 unsigned long pfn_hi
);
154 void copy_reserved_iova(struct iova_domain
*from
, struct iova_domain
*to
);
155 void init_iova_domain(struct iova_domain
*iovad
, unsigned long granule
,
156 unsigned long start_pfn
, unsigned long pfn_32bit
);
157 int init_iova_flush_queue(struct iova_domain
*iovad
,
158 iova_flush_cb flush_cb
, iova_entry_dtor entry_dtor
);
159 struct iova
*find_iova(struct iova_domain
*iovad
, unsigned long pfn
);
160 void put_iova_domain(struct iova_domain
*iovad
);
161 struct iova
*split_and_remove_iova(struct iova_domain
*iovad
,
162 struct iova
*iova
, unsigned long pfn_lo
, unsigned long pfn_hi
);
163 void free_cpu_cached_iovas(unsigned int cpu
, struct iova_domain
*iovad
);
165 static inline int iova_cache_get(void)
170 static inline void iova_cache_put(void)
174 static inline struct iova
*alloc_iova_mem(void)
179 static inline void free_iova_mem(struct iova
*iova
)
183 static inline void free_iova(struct iova_domain
*iovad
, unsigned long pfn
)
187 static inline void __free_iova(struct iova_domain
*iovad
, struct iova
*iova
)
191 static inline struct iova
*alloc_iova(struct iova_domain
*iovad
,
193 unsigned long limit_pfn
,
199 static inline void free_iova_fast(struct iova_domain
*iovad
,
205 static inline void queue_iova(struct iova_domain
*iovad
,
206 unsigned long pfn
, unsigned long pages
,
211 static inline unsigned long alloc_iova_fast(struct iova_domain
*iovad
,
213 unsigned long limit_pfn
)
218 static inline struct iova
*reserve_iova(struct iova_domain
*iovad
,
219 unsigned long pfn_lo
,
220 unsigned long pfn_hi
)
225 static inline void copy_reserved_iova(struct iova_domain
*from
,
226 struct iova_domain
*to
)
230 static inline void init_iova_domain(struct iova_domain
*iovad
,
231 unsigned long granule
,
232 unsigned long start_pfn
,
233 unsigned long pfn_32bit
)
237 static inline int init_iova_flush_queue(struct iova_domain
*iovad
,
238 iova_flush_cb flush_cb
,
239 iova_entry_dtor entry_dtor
)
244 static inline struct iova
*find_iova(struct iova_domain
*iovad
,
250 static inline void put_iova_domain(struct iova_domain
*iovad
)
254 static inline struct iova
*split_and_remove_iova(struct iova_domain
*iovad
,
256 unsigned long pfn_lo
,
257 unsigned long pfn_hi
)
262 static inline void free_cpu_cached_iovas(unsigned int cpu
,
263 struct iova_domain
*iovad
)