Linux 6.7-rc7
[linux-modified.git] / drivers / parisc / iommu-helpers.h
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <linux/prefetch.h>
3
4 /**
5  * iommu_fill_pdir - Insert coalesced scatter/gather chunks into the I/O Pdir.
6  * @ioc: The I/O Controller.
7  * @startsg: The scatter/gather list of coalesced chunks.
8  * @nents: The number of entries in the scatter/gather list.
9  * @hint: The DMA Hint.
10  *
11  * This function inserts the coalesced scatter/gather list chunks into the
12  * I/O Controller's I/O Pdir.
13  */ 
14 static inline unsigned int
15 iommu_fill_pdir(struct ioc *ioc, struct scatterlist *startsg, int nents, 
16                 unsigned long hint,
17                 void (*iommu_io_pdir_entry)(__le64 *, space_t, unsigned long,
18                                             unsigned long))
19 {
20         struct scatterlist *dma_sg = startsg;   /* pointer to current DMA */
21         unsigned int n_mappings = 0;
22         unsigned long dma_offset = 0, dma_len = 0;
23         __le64 *pdirp = NULL;
24
25         /* Horrible hack.  For efficiency's sake, dma_sg starts one 
26          * entry below the true start (it is immediately incremented
27          * in the loop) */
28          dma_sg--;
29
30         while (nents-- > 0) {
31                 unsigned long vaddr;
32                 long size;
33
34                 DBG_RUN_SG(" %d : %08lx %p/%05x\n", nents,
35                            (unsigned long)sg_dma_address(startsg),
36                            sg_virt(startsg), startsg->length
37                 );
38
39
40                 /*
41                 ** Look for the start of a new DMA stream
42                 */
43                 
44                 if (sg_dma_address(startsg) & PIDE_FLAG) {
45                         u32 pide = sg_dma_address(startsg) & ~PIDE_FLAG;
46
47                         BUG_ON(pdirp && (dma_len != sg_dma_len(dma_sg)));
48
49                         dma_sg++;
50
51                         dma_len = sg_dma_len(startsg);
52                         sg_dma_len(startsg) = 0;
53                         dma_offset = (unsigned long) pide & ~IOVP_MASK;
54                         n_mappings++;
55 #if defined(ZX1_SUPPORT)
56                         /* Pluto IOMMU IO Virt Address is not zero based */
57                         sg_dma_address(dma_sg) = pide | ioc->ibase;
58 #else
59                         /* SBA, ccio, and dino are zero based.
60                          * Trying to save a few CPU cycles for most users.
61                          */
62                         sg_dma_address(dma_sg) = pide;
63 #endif
64                         pdirp = &(ioc->pdir_base[pide >> IOVP_SHIFT]);
65                         prefetchw(pdirp);
66                 }
67                 
68                 BUG_ON(pdirp == NULL);
69                 
70                 vaddr = (unsigned long)sg_virt(startsg);
71                 sg_dma_len(dma_sg) += startsg->length;
72                 size = startsg->length + dma_offset;
73                 dma_offset = 0;
74 #ifdef IOMMU_MAP_STATS
75                 ioc->msg_pages += startsg->length >> IOVP_SHIFT;
76 #endif
77                 do {
78                         iommu_io_pdir_entry(pdirp, KERNEL_SPACE, 
79                                             vaddr, hint);
80                         vaddr += IOVP_SIZE;
81                         size -= IOVP_SIZE;
82                         pdirp++;
83                 } while(unlikely(size > 0));
84                 startsg++;
85         }
86         return(n_mappings);
87 }
88
89
90 /*
91 ** First pass is to walk the SG list and determine where the breaks are
92 ** in the DMA stream. Allocates PDIR entries but does not fill them.
93 ** Returns the number of DMA chunks.
94 **
95 ** Doing the fill separate from the coalescing/allocation keeps the
96 ** code simpler. Future enhancement could make one pass through
97 ** the sglist do both.
98 */
99
100 static inline unsigned int
101 iommu_coalesce_chunks(struct ioc *ioc, struct device *dev,
102                 struct scatterlist *startsg, int nents,
103                 int (*iommu_alloc_range)(struct ioc *, struct device *, size_t))
104 {
105         struct scatterlist *contig_sg;     /* contig chunk head */
106         unsigned long dma_offset, dma_len; /* start/len of DMA stream */
107         unsigned int n_mappings = 0;
108         unsigned int max_seg_size = min(dma_get_max_seg_size(dev),
109                                         (unsigned)DMA_CHUNK_SIZE);
110         unsigned int max_seg_boundary = dma_get_seg_boundary(dev) + 1;
111         if (max_seg_boundary)   /* check if the addition above didn't overflow */
112                 max_seg_size = min(max_seg_size, max_seg_boundary);
113
114         while (nents > 0) {
115
116                 /*
117                 ** Prepare for first/next DMA stream
118                 */
119                 contig_sg = startsg;
120                 dma_len = startsg->length;
121                 dma_offset = startsg->offset;
122
123                 /* PARANOID: clear entries */
124                 sg_dma_address(startsg) = 0;
125                 sg_dma_len(startsg) = 0;
126
127                 /*
128                 ** This loop terminates one iteration "early" since
129                 ** it's always looking one "ahead".
130                 */
131                 while(--nents > 0) {
132                         unsigned long prev_end, sg_start;
133
134                         prev_end = (unsigned long)sg_virt(startsg) +
135                                                         startsg->length;
136
137                         startsg++;
138                         sg_start = (unsigned long)sg_virt(startsg);
139
140                         /* PARANOID: clear entries */
141                         sg_dma_address(startsg) = 0;
142                         sg_dma_len(startsg) = 0;
143
144                         /*
145                         ** First make sure current dma stream won't
146                         ** exceed max_seg_size if we coalesce the
147                         ** next entry.
148                         */   
149                         if (unlikely(ALIGN(dma_len + dma_offset + startsg->length, IOVP_SIZE) >
150                                      max_seg_size))
151                                 break;
152
153                         /*
154                         * Next see if we can append the next chunk (i.e.
155                         * it must end on one page and begin on another, or
156                         * it must start on the same address as the previous
157                         * entry ended.
158                         */
159                         if (unlikely((prev_end != sg_start) ||
160                                 ((prev_end | sg_start) & ~PAGE_MASK)))
161                                 break;
162                         
163                         dma_len += startsg->length;
164                 }
165
166                 /*
167                 ** End of DMA Stream
168                 ** Terminate last VCONTIG block.
169                 ** Allocate space for DMA stream.
170                 */
171                 sg_dma_len(contig_sg) = dma_len;
172                 dma_len = ALIGN(dma_len + dma_offset, IOVP_SIZE);
173                 sg_dma_address(contig_sg) =
174                         PIDE_FLAG 
175                         | (iommu_alloc_range(ioc, dev, dma_len) << IOVP_SHIFT)
176                         | dma_offset;
177                 n_mappings++;
178         }
179
180         return n_mappings;
181 }
182