1 // SPDX-License-Identifier: MIT
3 * Copyright © 2019 Intel Corporation
7 #include "gt/intel_context.h"
8 #include "gt/intel_engine_pm.h"
9 #include "i915_gem_client_blt.h"
10 #include "i915_gem_object_blt.h"
14 struct drm_i915_gem_object *obj;
15 struct sg_table *pages;
16 struct i915_page_sizes page_sizes;
19 static int vma_set_pages(struct i915_vma *vma)
21 struct i915_sleeve *sleeve = vma->private;
23 vma->pages = sleeve->pages;
24 vma->page_sizes = sleeve->page_sizes;
29 static void vma_clear_pages(struct i915_vma *vma)
31 GEM_BUG_ON(!vma->pages);
35 static void vma_bind(struct i915_address_space *vm,
36 struct i915_vm_pt_stash *stash,
38 enum i915_cache_level cache_level,
41 vm->vma_ops.bind_vma(vm, stash, vma, cache_level, flags);
44 static void vma_unbind(struct i915_address_space *vm, struct i915_vma *vma)
46 vm->vma_ops.unbind_vma(vm, vma);
49 static const struct i915_vma_ops proxy_vma_ops = {
50 .set_pages = vma_set_pages,
51 .clear_pages = vma_clear_pages,
53 .unbind_vma = vma_unbind,
56 static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
57 struct drm_i915_gem_object *obj,
58 struct sg_table *pages,
59 struct i915_page_sizes *page_sizes)
61 struct i915_sleeve *sleeve;
65 sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL);
67 return ERR_PTR(-ENOMEM);
69 vma = i915_vma_instance(obj, vm, NULL);
75 vma->private = sleeve;
76 vma->ops = &proxy_vma_ops;
79 sleeve->pages = pages;
80 sleeve->page_sizes = *page_sizes;
89 static void destroy_sleeve(struct i915_sleeve *sleeve)
94 struct clear_pages_work {
96 struct dma_fence_cb cb;
97 struct i915_sw_fence wait;
98 struct work_struct work;
99 struct irq_work irq_work;
100 struct i915_sleeve *sleeve;
101 struct intel_context *ce;
105 static const char *clear_pages_work_driver_name(struct dma_fence *fence)
110 static const char *clear_pages_work_timeline_name(struct dma_fence *fence)
115 static void clear_pages_work_release(struct dma_fence *fence)
117 struct clear_pages_work *w = container_of(fence, typeof(*w), dma);
119 destroy_sleeve(w->sleeve);
121 i915_sw_fence_fini(&w->wait);
123 BUILD_BUG_ON(offsetof(typeof(*w), dma));
124 dma_fence_free(&w->dma);
127 static const struct dma_fence_ops clear_pages_work_ops = {
128 .get_driver_name = clear_pages_work_driver_name,
129 .get_timeline_name = clear_pages_work_timeline_name,
130 .release = clear_pages_work_release,
133 static void clear_pages_signal_irq_worker(struct irq_work *work)
135 struct clear_pages_work *w = container_of(work, typeof(*w), irq_work);
137 dma_fence_signal(&w->dma);
138 dma_fence_put(&w->dma);
141 static void clear_pages_dma_fence_cb(struct dma_fence *fence,
142 struct dma_fence_cb *cb)
144 struct clear_pages_work *w = container_of(cb, typeof(*w), cb);
147 dma_fence_set_error(&w->dma, fence->error);
150 * Push the signalling of the fence into yet another worker to avoid
151 * the nightmare locking around the fence spinlock.
153 irq_work_queue(&w->irq_work);
156 static void clear_pages_worker(struct work_struct *work)
158 struct clear_pages_work *w = container_of(work, typeof(*w), work);
159 struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
160 struct i915_vma *vma = w->sleeve->vma;
161 struct i915_gem_ww_ctx ww;
162 struct i915_request *rq;
163 struct i915_vma *batch;
164 int err = w->dma.error;
169 if (obj->cache_dirty) {
170 if (i915_gem_object_has_struct_page(obj))
171 drm_clflush_sg(w->sleeve->pages);
172 obj->cache_dirty = false;
174 obj->read_domains = I915_GEM_GPU_DOMAINS;
175 obj->write_domain = 0;
177 i915_gem_ww_ctx_init(&ww, false);
178 intel_engine_pm_get(w->ce->engine);
180 err = intel_context_pin_ww(w->ce, &ww);
184 batch = intel_emit_vma_fill_blt(w->ce, vma, &ww, w->value);
186 err = PTR_ERR(batch);
190 rq = i915_request_create(w->ce);
196 /* There's no way the fence has signalled */
197 if (dma_fence_add_callback(&rq->fence, &w->cb,
198 clear_pages_dma_fence_cb))
201 err = intel_emit_vma_mark_active(batch, rq);
205 if (w->ce->engine->emit_init_breadcrumb) {
206 err = w->ce->engine->emit_init_breadcrumb(rq);
212 * w->dma is already exported via (vma|obj)->resv we need only
213 * keep track of the GPU activity within this vma/request, and
214 * propagate the signal from the request to w->dma.
216 err = __i915_vma_move_to_active(vma, rq);
220 err = w->ce->engine->emit_bb_start(rq,
221 batch->node.start, batch->node.size,
225 i915_request_set_error_once(rq, err);
229 i915_request_add(rq);
231 intel_emit_vma_release(w->ce, batch);
233 intel_context_unpin(w->ce);
235 if (err == -EDEADLK) {
236 err = i915_gem_ww_ctx_backoff(&ww);
240 i915_gem_ww_ctx_fini(&ww);
242 i915_vma_unpin(w->sleeve->vma);
243 intel_engine_pm_put(w->ce->engine);
246 dma_fence_set_error(&w->dma, err);
247 dma_fence_signal(&w->dma);
248 dma_fence_put(&w->dma);
252 static int pin_wait_clear_pages_work(struct clear_pages_work *w,
253 struct intel_context *ce)
255 struct i915_vma *vma = w->sleeve->vma;
256 struct i915_gem_ww_ctx ww;
259 i915_gem_ww_ctx_init(&ww, false);
261 err = i915_gem_object_lock(vma->obj, &ww);
265 err = i915_vma_pin_ww(vma, &ww, 0, 0, PIN_USER);
269 err = i915_sw_fence_await_reservation(&w->wait,
270 vma->obj->base.resv, NULL,
271 true, 0, I915_FENCE_GFP);
275 dma_resv_add_excl_fence(vma->obj->base.resv, &w->dma);
281 if (err == -EDEADLK) {
282 err = i915_gem_ww_ctx_backoff(&ww);
286 i915_gem_ww_ctx_fini(&ww);
290 static int __i915_sw_fence_call
291 clear_pages_work_notify(struct i915_sw_fence *fence,
292 enum i915_sw_fence_notify state)
294 struct clear_pages_work *w = container_of(fence, typeof(*w), wait);
298 schedule_work(&w->work);
302 dma_fence_put(&w->dma);
309 static DEFINE_SPINLOCK(fence_lock);
311 /* XXX: better name please */
312 int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj,
313 struct intel_context *ce,
314 struct sg_table *pages,
315 struct i915_page_sizes *page_sizes,
318 struct clear_pages_work *work;
319 struct i915_sleeve *sleeve;
322 sleeve = create_sleeve(ce->vm, obj, pages, page_sizes);
324 return PTR_ERR(sleeve);
326 work = kmalloc(sizeof(*work), GFP_KERNEL);
328 destroy_sleeve(sleeve);
333 work->sleeve = sleeve;
336 INIT_WORK(&work->work, clear_pages_worker);
338 init_irq_work(&work->irq_work, clear_pages_signal_irq_worker);
340 dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0);
341 i915_sw_fence_init(&work->wait, clear_pages_work_notify);
343 err = pin_wait_clear_pages_work(work, ce);
345 dma_fence_set_error(&work->dma, err);
347 dma_fence_get(&work->dma);
348 i915_sw_fence_commit(&work->wait);
353 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
354 #include "selftests/i915_gem_client_blt.c"