1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (c) by Jaroslav Kysela <perex@perex.cz>
4 * Takashi Iwai <tiwai@suse.de>
6 * Generic memory allocators
9 #include <linux/slab.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/genalloc.h>
13 #include <linux/vmalloc.h>
15 #include <asm/set_memory.h>
17 #include <sound/memalloc.h>
18 #include "memalloc_local.h"
20 static const struct snd_malloc_ops *snd_dma_get_ops(struct snd_dma_buffer *dmab);
22 /* a cast to gfp flag from the dev pointer; for CONTINUOUS and VMALLOC types */
23 static inline gfp_t snd_mem_get_gfp_flags(const struct snd_dma_buffer *dmab,
29 return (__force gfp_t)(unsigned long)dmab->dev.dev;
32 static void *__snd_dma_alloc_pages(struct snd_dma_buffer *dmab, size_t size)
34 const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);
36 if (WARN_ON_ONCE(!ops || !ops->alloc))
38 return ops->alloc(dmab, size);
42 * snd_dma_alloc_pages - allocate the buffer area according to the given type
43 * @type: the DMA buffer type
44 * @device: the device pointer
45 * @size: the buffer size to allocate
46 * @dmab: buffer allocation record to store the allocated data
48 * Calls the memory-allocator function for the corresponding
51 * Return: Zero if the buffer with the given size is allocated successfully,
52 * otherwise a negative value on error.
54 int snd_dma_alloc_pages(int type, struct device *device, size_t size,
55 struct snd_dma_buffer *dmab)
62 size = PAGE_ALIGN(size);
63 dmab->dev.type = type;
64 dmab->dev.dev = device;
67 dmab->private_data = NULL;
68 dmab->area = __snd_dma_alloc_pages(dmab, size);
74 EXPORT_SYMBOL(snd_dma_alloc_pages);
77 * snd_dma_alloc_pages_fallback - allocate the buffer area according to the given type with fallback
78 * @type: the DMA buffer type
79 * @device: the device pointer
80 * @size: the buffer size to allocate
81 * @dmab: buffer allocation record to store the allocated data
83 * Calls the memory-allocator function for the corresponding
84 * buffer type. When no space is left, this function reduces the size and
85 * tries to allocate again. The size actually allocated is stored in
88 * Return: Zero if the buffer with the given size is allocated successfully,
89 * otherwise a negative value on error.
91 int snd_dma_alloc_pages_fallback(int type, struct device *device, size_t size,
92 struct snd_dma_buffer *dmab)
96 while ((err = snd_dma_alloc_pages(type, device, size, dmab)) < 0) {
99 if (size <= PAGE_SIZE)
102 size = PAGE_SIZE << get_order(size);
108 EXPORT_SYMBOL(snd_dma_alloc_pages_fallback);
111 * snd_dma_free_pages - release the allocated buffer
112 * @dmab: the buffer allocation record to release
114 * Releases the allocated buffer via snd_dma_alloc_pages().
116 void snd_dma_free_pages(struct snd_dma_buffer *dmab)
118 const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);
120 if (ops && ops->free)
123 EXPORT_SYMBOL(snd_dma_free_pages);
125 /* called by devres */
126 static void __snd_release_pages(struct device *dev, void *res)
128 snd_dma_free_pages(res);
132 * snd_devm_alloc_pages - allocate the buffer and manage with devres
133 * @dev: the device pointer
134 * @type: the DMA buffer type
135 * @size: the buffer size to allocate
137 * Allocate buffer pages depending on the given type and manage using devres.
138 * The pages will be released automatically at the device removal.
140 * Unlike snd_dma_alloc_pages(), this function requires the real device pointer,
141 * hence it can't work with SNDRV_DMA_TYPE_CONTINUOUS or
142 * SNDRV_DMA_TYPE_VMALLOC type.
144 * The function returns the snd_dma_buffer object at success, or NULL if failed.
146 struct snd_dma_buffer *
147 snd_devm_alloc_pages(struct device *dev, int type, size_t size)
149 struct snd_dma_buffer *dmab;
152 if (WARN_ON(type == SNDRV_DMA_TYPE_CONTINUOUS ||
153 type == SNDRV_DMA_TYPE_VMALLOC))
156 dmab = devres_alloc(__snd_release_pages, sizeof(*dmab), GFP_KERNEL);
160 err = snd_dma_alloc_pages(type, dev, size, dmab);
166 devres_add(dev, dmab);
169 EXPORT_SYMBOL_GPL(snd_devm_alloc_pages);
172 * snd_dma_buffer_mmap - perform mmap of the given DMA buffer
173 * @dmab: buffer allocation information
174 * @area: VM area information
176 int snd_dma_buffer_mmap(struct snd_dma_buffer *dmab,
177 struct vm_area_struct *area)
179 const struct snd_malloc_ops *ops;
183 ops = snd_dma_get_ops(dmab);
184 if (ops && ops->mmap)
185 return ops->mmap(dmab, area);
189 EXPORT_SYMBOL(snd_dma_buffer_mmap);
192 * snd_sgbuf_get_addr - return the physical address at the corresponding offset
193 * @dmab: buffer allocation information
194 * @offset: offset in the ring buffer
196 dma_addr_t snd_sgbuf_get_addr(struct snd_dma_buffer *dmab, size_t offset)
198 const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);
200 if (ops && ops->get_addr)
201 return ops->get_addr(dmab, offset);
203 return dmab->addr + offset;
205 EXPORT_SYMBOL(snd_sgbuf_get_addr);
208 * snd_sgbuf_get_page - return the physical page at the corresponding offset
209 * @dmab: buffer allocation information
210 * @offset: offset in the ring buffer
212 struct page *snd_sgbuf_get_page(struct snd_dma_buffer *dmab, size_t offset)
214 const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);
216 if (ops && ops->get_page)
217 return ops->get_page(dmab, offset);
219 return virt_to_page(dmab->area + offset);
221 EXPORT_SYMBOL(snd_sgbuf_get_page);
224 * snd_sgbuf_get_chunk_size - compute the max chunk size with continuous pages
226 * @dmab: buffer allocation information
227 * @ofs: offset in the ring buffer
228 * @size: the requested size
230 unsigned int snd_sgbuf_get_chunk_size(struct snd_dma_buffer *dmab,
231 unsigned int ofs, unsigned int size)
233 const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);
235 if (ops && ops->get_chunk_size)
236 return ops->get_chunk_size(dmab, ofs, size);
240 EXPORT_SYMBOL(snd_sgbuf_get_chunk_size);
243 * Continuous pages allocator
245 static void *snd_dma_continuous_alloc(struct snd_dma_buffer *dmab, size_t size)
247 gfp_t gfp = snd_mem_get_gfp_flags(dmab, GFP_KERNEL);
248 void *p = alloc_pages_exact(size, gfp);
251 dmab->addr = page_to_phys(virt_to_page(p));
255 static void snd_dma_continuous_free(struct snd_dma_buffer *dmab)
257 free_pages_exact(dmab->area, dmab->bytes);
260 static int snd_dma_continuous_mmap(struct snd_dma_buffer *dmab,
261 struct vm_area_struct *area)
263 return remap_pfn_range(area, area->vm_start,
264 dmab->addr >> PAGE_SHIFT,
265 area->vm_end - area->vm_start,
269 static const struct snd_malloc_ops snd_dma_continuous_ops = {
270 .alloc = snd_dma_continuous_alloc,
271 .free = snd_dma_continuous_free,
272 .mmap = snd_dma_continuous_mmap,
278 static void *snd_dma_vmalloc_alloc(struct snd_dma_buffer *dmab, size_t size)
280 gfp_t gfp = snd_mem_get_gfp_flags(dmab, GFP_KERNEL | __GFP_HIGHMEM);
282 return __vmalloc(size, gfp);
285 static void snd_dma_vmalloc_free(struct snd_dma_buffer *dmab)
290 static int snd_dma_vmalloc_mmap(struct snd_dma_buffer *dmab,
291 struct vm_area_struct *area)
293 return remap_vmalloc_range(area, dmab->area, 0);
296 #define get_vmalloc_page_addr(dmab, offset) \
297 page_to_phys(vmalloc_to_page((dmab)->area + (offset)))
299 static dma_addr_t snd_dma_vmalloc_get_addr(struct snd_dma_buffer *dmab,
302 return get_vmalloc_page_addr(dmab, offset) + offset % PAGE_SIZE;
305 static struct page *snd_dma_vmalloc_get_page(struct snd_dma_buffer *dmab,
308 return vmalloc_to_page(dmab->area + offset);
312 snd_dma_vmalloc_get_chunk_size(struct snd_dma_buffer *dmab,
313 unsigned int ofs, unsigned int size)
315 unsigned int start, end;
318 start = ALIGN_DOWN(ofs, PAGE_SIZE);
319 end = ofs + size - 1; /* the last byte address */
320 /* check page continuity */
321 addr = get_vmalloc_page_addr(dmab, start);
327 if (get_vmalloc_page_addr(dmab, start) != addr)
330 /* ok, all on continuous pages */
334 static const struct snd_malloc_ops snd_dma_vmalloc_ops = {
335 .alloc = snd_dma_vmalloc_alloc,
336 .free = snd_dma_vmalloc_free,
337 .mmap = snd_dma_vmalloc_mmap,
338 .get_addr = snd_dma_vmalloc_get_addr,
339 .get_page = snd_dma_vmalloc_get_page,
340 .get_chunk_size = snd_dma_vmalloc_get_chunk_size,
343 #ifdef CONFIG_HAS_DMA
347 #ifdef CONFIG_GENERIC_ALLOCATOR
348 static void *snd_dma_iram_alloc(struct snd_dma_buffer *dmab, size_t size)
350 struct device *dev = dmab->dev.dev;
351 struct gen_pool *pool;
355 pool = of_gen_pool_get(dev->of_node, "iram", 0);
356 /* Assign the pool into private_data field */
357 dmab->private_data = pool;
359 p = gen_pool_dma_alloc_align(pool, size, &dmab->addr, PAGE_SIZE);
364 /* Internal memory might have limited size and no enough space,
365 * so if we fail to malloc, try to fetch memory traditionally.
367 dmab->dev.type = SNDRV_DMA_TYPE_DEV;
368 return __snd_dma_alloc_pages(dmab, size);
371 static void snd_dma_iram_free(struct snd_dma_buffer *dmab)
373 struct gen_pool *pool = dmab->private_data;
375 if (pool && dmab->area)
376 gen_pool_free(pool, (unsigned long)dmab->area, dmab->bytes);
379 static int snd_dma_iram_mmap(struct snd_dma_buffer *dmab,
380 struct vm_area_struct *area)
382 area->vm_page_prot = pgprot_writecombine(area->vm_page_prot);
383 return remap_pfn_range(area, area->vm_start,
384 dmab->addr >> PAGE_SHIFT,
385 area->vm_end - area->vm_start,
389 static const struct snd_malloc_ops snd_dma_iram_ops = {
390 .alloc = snd_dma_iram_alloc,
391 .free = snd_dma_iram_free,
392 .mmap = snd_dma_iram_mmap,
394 #endif /* CONFIG_GENERIC_ALLOCATOR */
396 #define DEFAULT_GFP \
398 __GFP_COMP | /* compound page lets parts be mapped */ \
399 __GFP_NORETRY | /* don't trigger OOM-killer */ \
400 __GFP_NOWARN) /* no stack trace print - this call is non-critical */
403 * Coherent device pages allocator
405 static void *snd_dma_dev_alloc(struct snd_dma_buffer *dmab, size_t size)
409 p = dma_alloc_coherent(dmab->dev.dev, size, &dmab->addr, DEFAULT_GFP);
411 if (p && dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC)
412 set_memory_wc((unsigned long)p, PAGE_ALIGN(size) >> PAGE_SHIFT);
417 static void snd_dma_dev_free(struct snd_dma_buffer *dmab)
420 if (dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC)
421 set_memory_wb((unsigned long)dmab->area,
422 PAGE_ALIGN(dmab->bytes) >> PAGE_SHIFT);
424 dma_free_coherent(dmab->dev.dev, dmab->bytes, dmab->area, dmab->addr);
427 static int snd_dma_dev_mmap(struct snd_dma_buffer *dmab,
428 struct vm_area_struct *area)
431 if (dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC)
432 area->vm_page_prot = pgprot_writecombine(area->vm_page_prot);
434 return dma_mmap_coherent(dmab->dev.dev, area,
435 dmab->area, dmab->addr, dmab->bytes);
438 static const struct snd_malloc_ops snd_dma_dev_ops = {
439 .alloc = snd_dma_dev_alloc,
440 .free = snd_dma_dev_free,
441 .mmap = snd_dma_dev_mmap,
445 * Write-combined pages
448 /* On x86, share the same ops as the standard dev ops */
449 #define snd_dma_wc_ops snd_dma_dev_ops
450 #else /* CONFIG_X86 */
451 static void *snd_dma_wc_alloc(struct snd_dma_buffer *dmab, size_t size)
453 return dma_alloc_wc(dmab->dev.dev, size, &dmab->addr, DEFAULT_GFP);
456 static void snd_dma_wc_free(struct snd_dma_buffer *dmab)
458 dma_free_wc(dmab->dev.dev, dmab->bytes, dmab->area, dmab->addr);
461 static int snd_dma_wc_mmap(struct snd_dma_buffer *dmab,
462 struct vm_area_struct *area)
464 return dma_mmap_wc(dmab->dev.dev, area,
465 dmab->area, dmab->addr, dmab->bytes);
468 static const struct snd_malloc_ops snd_dma_wc_ops = {
469 .alloc = snd_dma_wc_alloc,
470 .free = snd_dma_wc_free,
471 .mmap = snd_dma_wc_mmap,
473 #endif /* CONFIG_X86 */
474 #endif /* CONFIG_HAS_DMA */
479 static const struct snd_malloc_ops *dma_ops[] = {
480 [SNDRV_DMA_TYPE_CONTINUOUS] = &snd_dma_continuous_ops,
481 [SNDRV_DMA_TYPE_VMALLOC] = &snd_dma_vmalloc_ops,
482 #ifdef CONFIG_HAS_DMA
483 [SNDRV_DMA_TYPE_DEV] = &snd_dma_dev_ops,
484 [SNDRV_DMA_TYPE_DEV_WC] = &snd_dma_wc_ops,
485 #ifdef CONFIG_GENERIC_ALLOCATOR
486 [SNDRV_DMA_TYPE_DEV_IRAM] = &snd_dma_iram_ops,
487 #endif /* CONFIG_GENERIC_ALLOCATOR */
488 #endif /* CONFIG_HAS_DMA */
489 #ifdef CONFIG_SND_DMA_SGBUF
490 [SNDRV_DMA_TYPE_DEV_SG] = &snd_dma_sg_ops,
491 [SNDRV_DMA_TYPE_DEV_WC_SG] = &snd_dma_sg_ops,
495 static const struct snd_malloc_ops *snd_dma_get_ops(struct snd_dma_buffer *dmab)
497 if (WARN_ON_ONCE(!dmab))
499 if (WARN_ON_ONCE(dmab->dev.type <= SNDRV_DMA_TYPE_UNKNOWN ||
500 dmab->dev.type >= ARRAY_SIZE(dma_ops)))
502 return dma_ops[dmab->dev.type];