1 /* SPDX-License-Identifier: GPL-2.0-only OR MIT */
3 #ifndef __DRM_GPUVM_H__
4 #define __DRM_GPUVM_H__
7 * Copyright (c) 2022 Red Hat.
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the "Software"),
11 * to deal in the Software without restriction, including without limitation
12 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 * and/or sell copies of the Software, and to permit persons to whom the
14 * Software is furnished to do so, subject to the following conditions:
16 * The above copyright notice and this permission notice shall be included in
17 * all copies or substantial portions of the Software.
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
23 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
24 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
25 * OTHER DEALINGS IN THE SOFTWARE.
28 #include <linux/list.h>
29 #include <linux/rbtree.h>
30 #include <linux/types.h>
32 #include <drm/drm_gem.h>
38 * enum drm_gpuva_flags - flags for struct drm_gpuva
40 enum drm_gpuva_flags {
42 * @DRM_GPUVA_INVALIDATED:
44 * Flag indicating that the &drm_gpuva's backing GEM is invalidated.
46 DRM_GPUVA_INVALIDATED = (1 << 0),
51 * Flag indicating that the &drm_gpuva is a sparse mapping.
53 DRM_GPUVA_SPARSE = (1 << 1),
56 * @DRM_GPUVA_USERBITS: user defined bits
58 DRM_GPUVA_USERBITS = (1 << 2),
62 * struct drm_gpuva - structure to track a GPU VA mapping
64 * This structure represents a GPU VA mapping and is associated with a
67 * Typically, this structure is embedded in bigger driver structures.
71 * @vm: the &drm_gpuvm this object is associated with
76 * @flags: the &drm_gpuva_flags for this mapping
78 enum drm_gpuva_flags flags;
81 * @va: structure containing the address and range of the &drm_gpuva
85 * @addr: the start address
96 * @gem: structure containing the &drm_gem_object and it's offset
100 * @offset: the offset within the &drm_gem_object
105 * @obj: the mapped &drm_gem_object
107 struct drm_gem_object *obj;
110 * @entry: the &list_head to attach this object to a &drm_gem_object
112 struct list_head entry;
116 * @rb: structure containing data to store &drm_gpuvas in a rb-tree
120 * @rb: the rb-tree node
125 * @entry: The &list_head to additionally connect &drm_gpuvas
126 * in the same order they appear in the interval tree. This is
127 * useful to keep iterating &drm_gpuvas from a start node found
128 * through the rb-tree while doing modifications on the rb-tree
131 struct list_head entry;
134 * @__subtree_last: needed by the interval tree, holding last-in-subtree
140 int drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va);
141 void drm_gpuva_remove(struct drm_gpuva *va);
143 void drm_gpuva_link(struct drm_gpuva *va);
144 void drm_gpuva_unlink(struct drm_gpuva *va);
146 struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm,
147 u64 addr, u64 range);
148 struct drm_gpuva *drm_gpuva_find_first(struct drm_gpuvm *gpuvm,
149 u64 addr, u64 range);
150 struct drm_gpuva *drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start);
151 struct drm_gpuva *drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end);
153 static inline void drm_gpuva_init(struct drm_gpuva *va, u64 addr, u64 range,
154 struct drm_gem_object *obj, u64 offset)
157 va->va.range = range;
159 va->gem.offset = offset;
163 * drm_gpuva_invalidate() - sets whether the backing GEM of this &drm_gpuva is
165 * @va: the &drm_gpuva to set the invalidate flag for
166 * @invalidate: indicates whether the &drm_gpuva is invalidated
168 static inline void drm_gpuva_invalidate(struct drm_gpuva *va, bool invalidate)
171 va->flags |= DRM_GPUVA_INVALIDATED;
173 va->flags &= ~DRM_GPUVA_INVALIDATED;
177 * drm_gpuva_invalidated() - indicates whether the backing BO of this &drm_gpuva
179 * @va: the &drm_gpuva to check
181 static inline bool drm_gpuva_invalidated(struct drm_gpuva *va)
183 return va->flags & DRM_GPUVA_INVALIDATED;
187 * struct drm_gpuvm - DRM GPU VA Manager
189 * The DRM GPU VA Manager keeps track of a GPU's virtual address space by using
190 * &maple_tree structures. Typically, this structure is embedded in bigger
193 * Drivers can pass addresses and ranges in an arbitrary unit, e.g. bytes or
196 * There should be one manager instance per GPU virtual address space.
200 * @name: the name of the DRM GPU VA space
205 * @mm_start: start of the VA space
210 * @mm_range: length of the VA space
215 * @rb: structures to track &drm_gpuva entries
219 * @tree: the rb-tree to track GPU VA mappings
221 struct rb_root_cached tree;
224 * @list: the &list_head to track GPU VA mappings
226 struct list_head list;
230 * @kernel_alloc_node:
232 * &drm_gpuva representing the address space cutout reserved for
235 struct drm_gpuva kernel_alloc_node;
238 * @ops: &drm_gpuvm_ops providing the split/merge steps to drivers
240 const struct drm_gpuvm_ops *ops;
243 void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
244 u64 start_offset, u64 range,
245 u64 reserve_offset, u64 reserve_range,
246 const struct drm_gpuvm_ops *ops);
247 void drm_gpuvm_destroy(struct drm_gpuvm *gpuvm);
249 bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range);
251 static inline struct drm_gpuva *
252 __drm_gpuva_next(struct drm_gpuva *va)
254 if (va && !list_is_last(&va->rb.entry, &va->vm->rb.list))
255 return list_next_entry(va, rb.entry);
261 * drm_gpuvm_for_each_va_range() - iterate over a range of &drm_gpuvas
262 * @va__: &drm_gpuva structure to assign to in each iteration step
263 * @gpuvm__: &drm_gpuvm to walk over
264 * @start__: starting offset, the first gpuva will overlap this
265 * @end__: ending offset, the last gpuva will start before this (but may
268 * This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie
269 * between @start__ and @end__. It is implemented similarly to list_for_each(),
270 * but is using the &drm_gpuvm's internal interval tree to accelerate
271 * the search for the starting &drm_gpuva, and hence isn't safe against removal
272 * of elements. It assumes that @end__ is within (or is the upper limit of) the
273 * &drm_gpuvm. This iterator does not skip over the &drm_gpuvm's
274 * @kernel_alloc_node.
276 #define drm_gpuvm_for_each_va_range(va__, gpuvm__, start__, end__) \
277 for (va__ = drm_gpuva_find_first((gpuvm__), (start__), (end__) - (start__)); \
278 va__ && (va__->va.addr < (end__)); \
279 va__ = __drm_gpuva_next(va__))
282 * drm_gpuvm_for_each_va_range_safe() - safely iterate over a range of
284 * @va__: &drm_gpuva to assign to in each iteration step
285 * @next__: another &drm_gpuva to use as temporary storage
286 * @gpuvm__: &drm_gpuvm to walk over
287 * @start__: starting offset, the first gpuva will overlap this
288 * @end__: ending offset, the last gpuva will start before this (but may
291 * This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie
292 * between @start__ and @end__. It is implemented similarly to
293 * list_for_each_safe(), but is using the &drm_gpuvm's internal interval
294 * tree to accelerate the search for the starting &drm_gpuva, and hence is safe
295 * against removal of elements. It assumes that @end__ is within (or is the
296 * upper limit of) the &drm_gpuvm. This iterator does not skip over the
297 * &drm_gpuvm's @kernel_alloc_node.
299 #define drm_gpuvm_for_each_va_range_safe(va__, next__, gpuvm__, start__, end__) \
300 for (va__ = drm_gpuva_find_first((gpuvm__), (start__), (end__) - (start__)), \
301 next__ = __drm_gpuva_next(va__); \
302 va__ && (va__->va.addr < (end__)); \
303 va__ = next__, next__ = __drm_gpuva_next(va__))
306 * drm_gpuvm_for_each_va() - iterate over all &drm_gpuvas
307 * @va__: &drm_gpuva to assign to in each iteration step
308 * @gpuvm__: &drm_gpuvm to walk over
310 * This iterator walks over all &drm_gpuva structures associated with the given
313 #define drm_gpuvm_for_each_va(va__, gpuvm__) \
314 list_for_each_entry(va__, &(gpuvm__)->rb.list, rb.entry)
317 * drm_gpuvm_for_each_va_safe() - safely iterate over all &drm_gpuvas
318 * @va__: &drm_gpuva to assign to in each iteration step
319 * @next__: another &drm_gpuva to use as temporary storage
320 * @gpuvm__: &drm_gpuvm to walk over
322 * This iterator walks over all &drm_gpuva structures associated with the given
323 * &drm_gpuvm. It is implemented with list_for_each_entry_safe(), and
324 * hence safe against the removal of elements.
326 #define drm_gpuvm_for_each_va_safe(va__, next__, gpuvm__) \
327 list_for_each_entry_safe(va__, next__, &(gpuvm__)->rb.list, rb.entry)
330 * enum drm_gpuva_op_type - GPU VA operation type
332 * Operations to alter the GPU VA mappings tracked by the &drm_gpuvm.
334 enum drm_gpuva_op_type {
336 * @DRM_GPUVA_OP_MAP: the map op type
341 * @DRM_GPUVA_OP_REMAP: the remap op type
346 * @DRM_GPUVA_OP_UNMAP: the unmap op type
351 * @DRM_GPUVA_OP_PREFETCH: the prefetch op type
353 DRM_GPUVA_OP_PREFETCH,
357 * struct drm_gpuva_op_map - GPU VA map operation
359 * This structure represents a single map operation generated by the
360 * DRM GPU VA manager.
362 struct drm_gpuva_op_map {
364 * @va: structure containing address and range of a map
369 * @addr: the base address of the new mapping
374 * @range: the range of the new mapping
380 * @gem: structure containing the &drm_gem_object and it's offset
384 * @offset: the offset within the &drm_gem_object
389 * @obj: the &drm_gem_object to map
391 struct drm_gem_object *obj;
396 * struct drm_gpuva_op_unmap - GPU VA unmap operation
398 * This structure represents a single unmap operation generated by the
399 * DRM GPU VA manager.
401 struct drm_gpuva_op_unmap {
403 * @va: the &drm_gpuva to unmap
405 struct drm_gpuva *va;
410 * Indicates whether this &drm_gpuva is physically contiguous with the
411 * original mapping request.
413 * Optionally, if &keep is set, drivers may keep the actual page table
414 * mappings for this &drm_gpuva, adding the missing page table entries
415 * only and update the &drm_gpuvm accordingly.
421 * struct drm_gpuva_op_remap - GPU VA remap operation
423 * This represents a single remap operation generated by the DRM GPU VA manager.
425 * A remap operation is generated when an existing GPU VA mmapping is split up
426 * by inserting a new GPU VA mapping or by partially unmapping existent
427 * mapping(s), hence it consists of a maximum of two map and one unmap
430 * The @unmap operation takes care of removing the original existing mapping.
431 * @prev is used to remap the preceding part, @next the subsequent part.
433 * If either a new mapping's start address is aligned with the start address
434 * of the old mapping or the new mapping's end address is aligned with the
435 * end address of the old mapping, either @prev or @next is NULL.
437 * Note, the reason for a dedicated remap operation, rather than arbitrary
438 * unmap and map operations, is to give drivers the chance of extracting driver
439 * specific data for creating the new mappings from the unmap operations's
440 * &drm_gpuva structure which typically is embedded in larger driver specific
443 struct drm_gpuva_op_remap {
445 * @prev: the preceding part of a split mapping
447 struct drm_gpuva_op_map *prev;
450 * @next: the subsequent part of a split mapping
452 struct drm_gpuva_op_map *next;
455 * @unmap: the unmap operation for the original existing mapping
457 struct drm_gpuva_op_unmap *unmap;
461 * struct drm_gpuva_op_prefetch - GPU VA prefetch operation
463 * This structure represents a single prefetch operation generated by the
464 * DRM GPU VA manager.
466 struct drm_gpuva_op_prefetch {
468 * @va: the &drm_gpuva to prefetch
470 struct drm_gpuva *va;
474 * struct drm_gpuva_op - GPU VA operation
476 * This structure represents a single generic operation.
478 * The particular type of the operation is defined by @op.
480 struct drm_gpuva_op {
484 * The &list_head used to distribute instances of this struct within
487 struct list_head entry;
490 * @op: the type of the operation
492 enum drm_gpuva_op_type op;
496 * @map: the map operation
498 struct drm_gpuva_op_map map;
501 * @remap: the remap operation
503 struct drm_gpuva_op_remap remap;
506 * @unmap: the unmap operation
508 struct drm_gpuva_op_unmap unmap;
511 * @prefetch: the prefetch operation
513 struct drm_gpuva_op_prefetch prefetch;
518 * struct drm_gpuva_ops - wraps a list of &drm_gpuva_op
520 struct drm_gpuva_ops {
522 * @list: the &list_head
524 struct list_head list;
528 * drm_gpuva_for_each_op() - iterator to walk over &drm_gpuva_ops
529 * @op: &drm_gpuva_op to assign in each iteration step
530 * @ops: &drm_gpuva_ops to walk
532 * This iterator walks over all ops within a given list of operations.
534 #define drm_gpuva_for_each_op(op, ops) list_for_each_entry(op, &(ops)->list, entry)
537 * drm_gpuva_for_each_op_safe() - iterator to safely walk over &drm_gpuva_ops
538 * @op: &drm_gpuva_op to assign in each iteration step
539 * @next: &next &drm_gpuva_op to store the next step
540 * @ops: &drm_gpuva_ops to walk
542 * This iterator walks over all ops within a given list of operations. It is
543 * implemented with list_for_each_safe(), so save against removal of elements.
545 #define drm_gpuva_for_each_op_safe(op, next, ops) \
546 list_for_each_entry_safe(op, next, &(ops)->list, entry)
549 * drm_gpuva_for_each_op_from_reverse() - iterate backwards from the given point
550 * @op: &drm_gpuva_op to assign in each iteration step
551 * @ops: &drm_gpuva_ops to walk
553 * This iterator walks over all ops within a given list of operations beginning
554 * from the given operation in reverse order.
556 #define drm_gpuva_for_each_op_from_reverse(op, ops) \
557 list_for_each_entry_from_reverse(op, &(ops)->list, entry)
560 * drm_gpuva_first_op() - returns the first &drm_gpuva_op from &drm_gpuva_ops
561 * @ops: the &drm_gpuva_ops to get the fist &drm_gpuva_op from
563 #define drm_gpuva_first_op(ops) \
564 list_first_entry(&(ops)->list, struct drm_gpuva_op, entry)
567 * drm_gpuva_last_op() - returns the last &drm_gpuva_op from &drm_gpuva_ops
568 * @ops: the &drm_gpuva_ops to get the last &drm_gpuva_op from
570 #define drm_gpuva_last_op(ops) \
571 list_last_entry(&(ops)->list, struct drm_gpuva_op, entry)
574 * drm_gpuva_prev_op() - previous &drm_gpuva_op in the list
575 * @op: the current &drm_gpuva_op
577 #define drm_gpuva_prev_op(op) list_prev_entry(op, entry)
580 * drm_gpuva_next_op() - next &drm_gpuva_op in the list
581 * @op: the current &drm_gpuva_op
583 #define drm_gpuva_next_op(op) list_next_entry(op, entry)
585 struct drm_gpuva_ops *
586 drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm,
588 struct drm_gem_object *obj, u64 offset);
589 struct drm_gpuva_ops *
590 drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm,
591 u64 addr, u64 range);
593 struct drm_gpuva_ops *
594 drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm,
595 u64 addr, u64 range);
597 struct drm_gpuva_ops *
598 drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm,
599 struct drm_gem_object *obj);
601 void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm,
602 struct drm_gpuva_ops *ops);
604 static inline void drm_gpuva_init_from_op(struct drm_gpuva *va,
605 struct drm_gpuva_op_map *op)
607 drm_gpuva_init(va, op->va.addr, op->va.range,
608 op->gem.obj, op->gem.offset);
612 * struct drm_gpuvm_ops - callbacks for split/merge steps
614 * This structure defines the callbacks used by &drm_gpuvm_sm_map and
615 * &drm_gpuvm_sm_unmap to provide the split/merge steps for map and unmap
616 * operations to drivers.
618 struct drm_gpuvm_ops {
620 * @op_alloc: called when the &drm_gpuvm allocates
621 * a struct drm_gpuva_op
623 * Some drivers may want to embed struct drm_gpuva_op into driver
624 * specific structures. By implementing this callback drivers can
625 * allocate memory accordingly.
627 * This callback is optional.
629 struct drm_gpuva_op *(*op_alloc)(void);
632 * @op_free: called when the &drm_gpuvm frees a
633 * struct drm_gpuva_op
635 * Some drivers may want to embed struct drm_gpuva_op into driver
636 * specific structures. By implementing this callback drivers can
637 * free the previously allocated memory accordingly.
639 * This callback is optional.
641 void (*op_free)(struct drm_gpuva_op *op);
644 * @sm_step_map: called from &drm_gpuvm_sm_map to finally insert the
645 * mapping once all previous steps were completed
647 * The &priv pointer matches the one the driver passed to
648 * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively.
650 * Can be NULL if &drm_gpuvm_sm_map is used.
652 int (*sm_step_map)(struct drm_gpuva_op *op, void *priv);
655 * @sm_step_remap: called from &drm_gpuvm_sm_map and
656 * &drm_gpuvm_sm_unmap to split up an existent mapping
658 * This callback is called when existent mapping needs to be split up.
659 * This is the case when either a newly requested mapping overlaps or
660 * is enclosed by an existent mapping or a partial unmap of an existent
661 * mapping is requested.
663 * The &priv pointer matches the one the driver passed to
664 * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively.
666 * Can be NULL if neither &drm_gpuvm_sm_map nor &drm_gpuvm_sm_unmap is
669 int (*sm_step_remap)(struct drm_gpuva_op *op, void *priv);
672 * @sm_step_unmap: called from &drm_gpuvm_sm_map and
673 * &drm_gpuvm_sm_unmap to unmap an existent mapping
675 * This callback is called when existent mapping needs to be unmapped.
676 * This is the case when either a newly requested mapping encloses an
677 * existent mapping or an unmap of an existent mapping is requested.
679 * The &priv pointer matches the one the driver passed to
680 * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively.
682 * Can be NULL if neither &drm_gpuvm_sm_map nor &drm_gpuvm_sm_unmap is
685 int (*sm_step_unmap)(struct drm_gpuva_op *op, void *priv);
688 int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv,
690 struct drm_gem_object *obj, u64 offset);
692 int drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv,
693 u64 addr, u64 range);
695 void drm_gpuva_map(struct drm_gpuvm *gpuvm,
696 struct drm_gpuva *va,
697 struct drm_gpuva_op_map *op);
699 void drm_gpuva_remap(struct drm_gpuva *prev,
700 struct drm_gpuva *next,
701 struct drm_gpuva_op_remap *op);
703 void drm_gpuva_unmap(struct drm_gpuva_op_unmap *op);
705 #endif /* __DRM_GPUVM_H__ */