2 * Copyright (C) 2013 Red Hat
3 * Author: Rob Clark <robdclark@gmail.com>
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program. If not, see <http://www.gnu.org/licenses/>.
23 * Cmdstream submission:
26 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
27 #define BO_VALID 0x8000
28 #define BO_LOCKED 0x4000
29 #define BO_PINNED 0x2000
31 static inline void __user *to_user_ptr(u64 address)
33 return (void __user *)(uintptr_t)address;
36 static struct msm_gem_submit *submit_create(struct drm_device *dev,
37 struct msm_gpu *gpu, uint32_t nr)
39 struct msm_gem_submit *submit;
40 uint64_t sz = sizeof(*submit) + ((u64)nr * sizeof(submit->bos[0]));
45 submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
50 /* initially, until copy_from_user() and bo lookup succeeds: */
54 INIT_LIST_HEAD(&submit->bo_list);
55 ww_acquire_init(&submit->ticket, &reservation_ww_class);
61 static inline unsigned long __must_check
62 copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
64 if (access_ok(VERIFY_READ, from, n))
65 return __copy_from_user_inatomic(to, from, n);
69 static int submit_lookup_objects(struct msm_gem_submit *submit,
70 struct drm_msm_gem_submit *args, struct drm_file *file)
75 spin_lock(&file->table_lock);
78 for (i = 0; i < args->nr_bos; i++) {
79 struct drm_msm_gem_submit_bo submit_bo;
80 struct drm_gem_object *obj;
81 struct msm_gem_object *msm_obj;
82 void __user *userptr =
83 to_user_ptr(args->bos + (i * sizeof(submit_bo)));
85 ret = copy_from_user_inatomic(&submit_bo, userptr, sizeof(submit_bo));
88 spin_unlock(&file->table_lock);
89 ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
92 spin_lock(&file->table_lock);
96 if ((submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) ||
97 !(submit_bo.flags & MSM_SUBMIT_BO_FLAGS)) {
98 DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
103 submit->bos[i].flags = submit_bo.flags;
104 /* in validate_objects() we figure out if this is true: */
105 submit->bos[i].iova = submit_bo.presumed;
107 /* normally use drm_gem_object_lookup(), but for bulk lookup
108 * all under single table_lock just hit object_idr directly:
110 obj = idr_find(&file->object_idr, submit_bo.handle);
112 DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
117 msm_obj = to_msm_bo(obj);
119 if (!list_empty(&msm_obj->submit_entry)) {
120 DRM_ERROR("handle %u at index %u already on submit list\n",
121 submit_bo.handle, i);
126 drm_gem_object_reference(obj);
128 submit->bos[i].obj = msm_obj;
130 list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
135 spin_unlock(&file->table_lock);
143 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
145 struct msm_gem_object *msm_obj = submit->bos[i].obj;
147 if (submit->bos[i].flags & BO_PINNED)
148 msm_gem_put_iova(&msm_obj->base, submit->gpu->id);
150 if (submit->bos[i].flags & BO_LOCKED)
151 ww_mutex_unlock(&msm_obj->resv->lock);
153 if (!(submit->bos[i].flags & BO_VALID))
154 submit->bos[i].iova = 0;
156 submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED);
159 /* This is where we make sure all the bo's are reserved and pin'd: */
160 static int submit_validate_objects(struct msm_gem_submit *submit)
162 int contended, slow_locked = -1, i, ret = 0;
165 submit->valid = true;
167 for (i = 0; i < submit->nr_bos; i++) {
168 struct msm_gem_object *msm_obj = submit->bos[i].obj;
171 if (slow_locked == i)
176 if (!(submit->bos[i].flags & BO_LOCKED)) {
177 ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
181 submit->bos[i].flags |= BO_LOCKED;
185 /* if locking succeeded, pin bo: */
186 ret = msm_gem_get_iova_locked(&msm_obj->base,
187 submit->gpu->id, &iova);
189 /* this would break the logic in the fail path.. there is no
190 * reason for this to happen, but just to be on the safe side
191 * let's notice if this starts happening in the future:
193 WARN_ON(ret == -EDEADLK);
198 submit->bos[i].flags |= BO_PINNED;
200 if (iova == submit->bos[i].iova) {
201 submit->bos[i].flags |= BO_VALID;
203 submit->bos[i].iova = iova;
204 submit->bos[i].flags &= ~BO_VALID;
205 submit->valid = false;
209 ww_acquire_done(&submit->ticket);
215 submit_unlock_unpin_bo(submit, i);
218 submit_unlock_unpin_bo(submit, slow_locked);
220 if (ret == -EDEADLK) {
221 struct msm_gem_object *msm_obj = submit->bos[contended].obj;
222 /* we lost out in a seqno race, lock and retry.. */
223 ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
226 submit->bos[contended].flags |= BO_LOCKED;
227 slow_locked = contended;
235 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
236 struct msm_gem_object **obj, uint32_t *iova, bool *valid)
238 if (idx >= submit->nr_bos) {
239 DRM_ERROR("invalid buffer index: %u (out of %u)\n",
240 idx, submit->nr_bos);
245 *obj = submit->bos[idx].obj;
247 *iova = submit->bos[idx].iova;
249 *valid = !!(submit->bos[idx].flags & BO_VALID);
254 /* process the reloc's and patch up the cmdstream as needed: */
255 static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,
256 uint32_t offset, uint32_t nr_relocs, uint64_t relocs)
258 uint32_t i, last_offset = 0;
263 DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
267 /* For now, just map the entire thing. Eventually we probably
268 * to do it page-by-page, w/ kmap() if not vmap()d..
270 ptr = msm_gem_vaddr_locked(&obj->base);
274 DBG("failed to map: %d", ret);
278 for (i = 0; i < nr_relocs; i++) {
279 struct drm_msm_gem_submit_reloc submit_reloc;
280 void __user *userptr =
281 to_user_ptr(relocs + (i * sizeof(submit_reloc)));
285 ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc));
289 if (submit_reloc.submit_offset % 4) {
290 DRM_ERROR("non-aligned reloc offset: %u\n",
291 submit_reloc.submit_offset);
295 /* offset in dwords: */
296 off = submit_reloc.submit_offset / 4;
298 if ((off >= (obj->base.size / 4)) ||
299 (off < last_offset)) {
300 DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
304 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
311 iova += submit_reloc.reloc_offset;
313 if (submit_reloc.shift < 0)
314 iova >>= -submit_reloc.shift;
316 iova <<= submit_reloc.shift;
318 ptr[off] = iova | submit_reloc.or;
326 static void submit_cleanup(struct msm_gem_submit *submit, bool fail)
330 for (i = 0; i < submit->nr_bos; i++) {
331 struct msm_gem_object *msm_obj = submit->bos[i].obj;
332 submit_unlock_unpin_bo(submit, i);
333 list_del_init(&msm_obj->submit_entry);
334 drm_gem_object_unreference(&msm_obj->base);
337 ww_acquire_fini(&submit->ticket);
340 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
341 struct drm_file *file)
343 struct msm_drm_private *priv = dev->dev_private;
344 struct drm_msm_gem_submit *args = data;
345 struct msm_file_private *ctx = file->driver_priv;
346 struct msm_gem_submit *submit;
351 /* for now, we just have 3d pipe.. eventually this would need to
352 * be more clever to dispatch to appropriate gpu module:
354 if (args->pipe != MSM_PIPE_3D0)
359 if (args->nr_cmds > MAX_CMDS)
362 mutex_lock(&dev->struct_mutex);
364 submit = submit_create(dev, gpu, args->nr_bos);
370 ret = submit_lookup_objects(submit, args, file);
374 ret = submit_validate_objects(submit);
378 for (i = 0; i < args->nr_cmds; i++) {
379 struct drm_msm_gem_submit_cmd submit_cmd;
380 void __user *userptr =
381 to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
382 struct msm_gem_object *msm_obj;
385 ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
391 /* validate input from userspace: */
392 switch (submit_cmd.type) {
393 case MSM_SUBMIT_CMD_BUF:
394 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
395 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
398 DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
403 ret = submit_bo(submit, submit_cmd.submit_idx,
404 &msm_obj, &iova, NULL);
408 if (submit_cmd.size % 4) {
409 DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
415 if ((submit_cmd.size + submit_cmd.submit_offset) >=
416 msm_obj->base.size) {
417 DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size);
422 submit->cmd[i].type = submit_cmd.type;
423 submit->cmd[i].size = submit_cmd.size / 4;
424 submit->cmd[i].iova = iova + submit_cmd.submit_offset;
425 submit->cmd[i].idx = submit_cmd.submit_idx;
430 ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,
431 submit_cmd.nr_relocs, submit_cmd.relocs);
438 ret = msm_gpu_submit(gpu, submit, ctx);
440 args->fence = submit->fence;
444 submit_cleanup(submit, !!ret);
445 mutex_unlock(&dev->struct_mutex);