2 * Copyright (c) 2006, Intel Corporation.
4 * This file is released under the GPLv2.
6 * Copyright (C) 2006-2008 Intel Corporation
7 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
14 #include <linux/types.h>
15 #include <linux/kernel.h>
16 #include <linux/rbtree.h>
17 #include <linux/atomic.h>
18 #include <linux/dma-mapping.h>
23 unsigned long pfn_hi; /* Highest allocated pfn */
24 unsigned long pfn_lo; /* Lowest allocated pfn */
28 struct iova_cpu_rcache;
30 #define IOVA_RANGE_CACHE_MAX_SIZE 6 /* log of max cached IOVA range size (in pages) */
31 #define MAX_GLOBAL_MAGS 32 /* magazines per bin */
35 unsigned long depot_size;
36 struct iova_magazine *depot[MAX_GLOBAL_MAGS];
37 struct iova_cpu_rcache __percpu *cpu_rcaches;
42 /* Call-Back from IOVA code into IOMMU drivers */
43 typedef void (* iova_flush_cb)(struct iova_domain *domain);
45 /* Destructor for per-entry data */
46 typedef void (* iova_entry_dtor)(unsigned long data);
48 /* Number of entries per Flush Queue */
49 #define IOVA_FQ_SIZE 256
51 /* Timeout (in ms) after which entries are flushed from the Flush-Queue */
52 #define IOVA_FQ_TIMEOUT 10
54 /* Flush Queue entry for defered flushing */
55 struct iova_fq_entry {
56 unsigned long iova_pfn;
59 u64 counter; /* Flush counter when this entrie was added */
62 /* Per-CPU Flush Queue structure */
64 struct iova_fq_entry entries[IOVA_FQ_SIZE];
69 /* holds all the iova translations for a domain */
71 spinlock_t iova_rbtree_lock; /* Lock to protect update of rbtree */
72 struct rb_root rbroot; /* iova domain rbtree root */
73 struct rb_node *cached32_node; /* Save last alloced node */
74 unsigned long granule; /* pfn granularity for this domain */
75 unsigned long start_pfn; /* Lower limit for this domain */
76 unsigned long dma_32bit_pfn;
77 struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */
79 iova_flush_cb flush_cb; /* Call-Back function to flush IOMMU
82 iova_entry_dtor entry_dtor; /* IOMMU driver specific destructor for
85 struct iova_fq __percpu *fq; /* Flush Queue */
87 atomic64_t fq_flush_start_cnt; /* Number of TLB flushes that
90 atomic64_t fq_flush_finish_cnt; /* Number of TLB flushes that
93 struct timer_list fq_timer; /* Timer to regularily empty the
95 atomic_t fq_timer_on; /* 1 when timer is active, 0
99 static inline unsigned long iova_size(struct iova *iova)
101 return iova->pfn_hi - iova->pfn_lo + 1;
104 static inline unsigned long iova_shift(struct iova_domain *iovad)
106 return __ffs(iovad->granule);
109 static inline unsigned long iova_mask(struct iova_domain *iovad)
111 return iovad->granule - 1;
114 static inline size_t iova_offset(struct iova_domain *iovad, dma_addr_t iova)
116 return iova & iova_mask(iovad);
119 static inline size_t iova_align(struct iova_domain *iovad, size_t size)
121 return ALIGN(size, iovad->granule);
124 static inline dma_addr_t iova_dma_addr(struct iova_domain *iovad, struct iova *iova)
126 return (dma_addr_t)iova->pfn_lo << iova_shift(iovad);
129 static inline unsigned long iova_pfn(struct iova_domain *iovad, dma_addr_t iova)
131 return iova >> iova_shift(iovad);
134 #if IS_REACHABLE(CONFIG_IOMMU_IOVA)
135 int iova_cache_get(void);
136 void iova_cache_put(void);
138 struct iova *alloc_iova_mem(void);
139 void free_iova_mem(struct iova *iova);
140 void free_iova(struct iova_domain *iovad, unsigned long pfn);
141 void __free_iova(struct iova_domain *iovad, struct iova *iova);
142 struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
143 unsigned long limit_pfn,
145 void free_iova_fast(struct iova_domain *iovad, unsigned long pfn,
147 void queue_iova(struct iova_domain *iovad,
148 unsigned long pfn, unsigned long pages,
150 unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
151 unsigned long limit_pfn);
152 struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
153 unsigned long pfn_hi);
154 void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to);
155 void init_iova_domain(struct iova_domain *iovad, unsigned long granule,
156 unsigned long start_pfn, unsigned long pfn_32bit);
157 bool has_iova_flush_queue(struct iova_domain *iovad);
158 int init_iova_flush_queue(struct iova_domain *iovad,
159 iova_flush_cb flush_cb, iova_entry_dtor entry_dtor);
160 struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
161 void put_iova_domain(struct iova_domain *iovad);
162 struct iova *split_and_remove_iova(struct iova_domain *iovad,
163 struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi);
164 void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
166 static inline int iova_cache_get(void)
171 static inline void iova_cache_put(void)
175 static inline struct iova *alloc_iova_mem(void)
180 static inline void free_iova_mem(struct iova *iova)
184 static inline void free_iova(struct iova_domain *iovad, unsigned long pfn)
188 static inline void __free_iova(struct iova_domain *iovad, struct iova *iova)
192 static inline struct iova *alloc_iova(struct iova_domain *iovad,
194 unsigned long limit_pfn,
200 static inline void free_iova_fast(struct iova_domain *iovad,
206 static inline void queue_iova(struct iova_domain *iovad,
207 unsigned long pfn, unsigned long pages,
212 static inline unsigned long alloc_iova_fast(struct iova_domain *iovad,
214 unsigned long limit_pfn)
219 static inline struct iova *reserve_iova(struct iova_domain *iovad,
220 unsigned long pfn_lo,
221 unsigned long pfn_hi)
226 static inline void copy_reserved_iova(struct iova_domain *from,
227 struct iova_domain *to)
231 static inline void init_iova_domain(struct iova_domain *iovad,
232 unsigned long granule,
233 unsigned long start_pfn,
234 unsigned long pfn_32bit)
238 static inline bool has_iova_flush_queue(struct iova_domain *iovad)
243 static inline int init_iova_flush_queue(struct iova_domain *iovad,
244 iova_flush_cb flush_cb,
245 iova_entry_dtor entry_dtor)
250 static inline struct iova *find_iova(struct iova_domain *iovad,
256 static inline void put_iova_domain(struct iova_domain *iovad)
260 static inline struct iova *split_and_remove_iova(struct iova_domain *iovad,
262 unsigned long pfn_lo,
263 unsigned long pfn_hi)
268 static inline void free_cpu_cached_iovas(unsigned int cpu,
269 struct iova_domain *iovad)