1 /**************************************************************************
3 * Copyright © 2014-2015 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 * Treat context OTables as resources to make use of the resource
29 * backing MOB eviction mechanism, that is used to read back the COTable
30 * whenever the backing MOB is evicted.
33 #include <drm/ttm/ttm_placement.h>
35 #include "vmwgfx_drv.h"
36 #include "vmwgfx_resource_priv.h"
37 #include "vmwgfx_so.h"
40 * struct vmw_cotable - Context Object Table resource
42 * @res: struct vmw_resource we are deriving from.
43 * @ctx: non-refcounted pointer to the owning context.
44 * @size_read_back: Size of data read back during eviction.
45 * @seen_entries: Seen entries in command stream for this cotable.
46 * @type: The cotable type.
47 * @scrubbed: Whether the cotable has been scrubbed.
48 * @resource_list: List of resources in the cotable.
51 struct vmw_resource res;
52 struct vmw_resource *ctx;
53 size_t size_read_back;
57 struct list_head resource_list;
61 * struct vmw_cotable_info - Static info about cotable types
63 * @min_initial_entries: Min number of initial intries at cotable allocation
64 * for this cotable type.
65 * @size: Size of each entry.
67 struct vmw_cotable_info {
68 u32 min_initial_entries;
70 void (*unbind_func)(struct vmw_private *, struct list_head *,
74 static const struct vmw_cotable_info co_info[] = {
75 {1, sizeof(SVGACOTableDXRTViewEntry), &vmw_view_cotable_list_destroy},
76 {1, sizeof(SVGACOTableDXDSViewEntry), &vmw_view_cotable_list_destroy},
77 {1, sizeof(SVGACOTableDXSRViewEntry), &vmw_view_cotable_list_destroy},
78 {1, sizeof(SVGACOTableDXElementLayoutEntry), NULL},
79 {1, sizeof(SVGACOTableDXBlendStateEntry), NULL},
80 {1, sizeof(SVGACOTableDXDepthStencilEntry), NULL},
81 {1, sizeof(SVGACOTableDXRasterizerStateEntry), NULL},
82 {1, sizeof(SVGACOTableDXSamplerEntry), NULL},
83 {1, sizeof(SVGACOTableDXStreamOutputEntry), NULL},
84 {1, sizeof(SVGACOTableDXQueryEntry), NULL},
85 {1, sizeof(SVGACOTableDXShaderEntry), &vmw_dx_shader_cotable_list_scrub}
89 * Cotables with bindings that we remove must be scrubbed first,
90 * otherwise, the device will swap in an invalid context when we remove
91 * bindings before scrubbing a cotable...
93 const SVGACOTableType vmw_cotable_scrub_order[] = {
97 SVGA_COTABLE_DXSHADER,
98 SVGA_COTABLE_ELEMENTLAYOUT,
99 SVGA_COTABLE_BLENDSTATE,
100 SVGA_COTABLE_DEPTHSTENCIL,
101 SVGA_COTABLE_RASTERIZERSTATE,
102 SVGA_COTABLE_SAMPLER,
103 SVGA_COTABLE_STREAMOUTPUT,
104 SVGA_COTABLE_DXQUERY,
107 static int vmw_cotable_bind(struct vmw_resource *res,
108 struct ttm_validate_buffer *val_buf);
109 static int vmw_cotable_unbind(struct vmw_resource *res,
111 struct ttm_validate_buffer *val_buf);
112 static int vmw_cotable_create(struct vmw_resource *res);
113 static int vmw_cotable_destroy(struct vmw_resource *res);
115 static const struct vmw_res_func vmw_cotable_func = {
116 .res_type = vmw_res_cotable,
117 .needs_backup = true,
119 .type_name = "context guest backed object tables",
120 .backup_placement = &vmw_mob_placement,
121 .create = vmw_cotable_create,
122 .destroy = vmw_cotable_destroy,
123 .bind = vmw_cotable_bind,
124 .unbind = vmw_cotable_unbind,
128 * vmw_cotable - Convert a struct vmw_resource pointer to a struct
129 * vmw_cotable pointer
131 * @res: Pointer to the resource.
133 static struct vmw_cotable *vmw_cotable(struct vmw_resource *res)
135 return container_of(res, struct vmw_cotable, res);
139 * vmw_cotable_destroy - Cotable resource destroy callback
141 * @res: Pointer to the cotable resource.
143 * There is no device cotable destroy command, so this function only
144 * makes sure that the resource id is set to invalid.
146 static int vmw_cotable_destroy(struct vmw_resource *res)
153 * vmw_cotable_unscrub - Undo a cotable unscrub operation
155 * @res: Pointer to the cotable resource
157 * This function issues commands to (re)bind the cotable to
158 * its backing mob, which needs to be validated and reserved at this point.
159 * This is identical to bind() except the function interface looks different.
161 static int vmw_cotable_unscrub(struct vmw_resource *res)
163 struct vmw_cotable *vcotbl = vmw_cotable(res);
164 struct vmw_private *dev_priv = res->dev_priv;
165 struct ttm_buffer_object *bo = &res->backup->base;
167 SVGA3dCmdHeader header;
168 SVGA3dCmdDXSetCOTable body;
171 WARN_ON_ONCE(bo->mem.mem_type != VMW_PL_MOB);
172 lockdep_assert_held(&bo->resv->lock.base);
174 cmd = vmw_fifo_reserve_dx(dev_priv, sizeof(*cmd), SVGA3D_INVALID_ID);
176 DRM_ERROR("Failed reserving FIFO space for cotable "
181 WARN_ON(vcotbl->ctx->id == SVGA3D_INVALID_ID);
182 WARN_ON(bo->mem.mem_type != VMW_PL_MOB);
183 cmd->header.id = SVGA_3D_CMD_DX_SET_COTABLE;
184 cmd->header.size = sizeof(cmd->body);
185 cmd->body.cid = vcotbl->ctx->id;
186 cmd->body.type = vcotbl->type;
187 cmd->body.mobid = bo->mem.start;
188 cmd->body.validSizeInBytes = vcotbl->size_read_back;
190 vmw_fifo_commit_flush(dev_priv, sizeof(*cmd));
191 vcotbl->scrubbed = false;
197 * vmw_cotable_bind - Undo a cotable unscrub operation
199 * @res: Pointer to the cotable resource
200 * @val_buf: Pointer to a struct ttm_validate_buffer prepared by the caller
201 * for convenience / fencing.
203 * This function issues commands to (re)bind the cotable to
204 * its backing mob, which needs to be validated and reserved at this point.
206 static int vmw_cotable_bind(struct vmw_resource *res,
207 struct ttm_validate_buffer *val_buf)
210 * The create() callback may have changed @res->backup without
211 * the caller noticing, and with val_buf->bo still pointing to
212 * the old backup buffer. Although hackish, and not used currently,
213 * take the opportunity to correct the value here so that it's not
214 * misused in the future.
216 val_buf->bo = &res->backup->base;
218 return vmw_cotable_unscrub(res);
222 * vmw_cotable_scrub - Scrub the cotable from the device.
224 * @res: Pointer to the cotable resource.
225 * @readback: Whether initiate a readback of the cotable data to the backup
228 * In some situations (context swapouts) it might be desirable to make the
229 * device forget about the cotable without performing a full unbind. A full
230 * unbind requires reserved backup buffers and it might not be possible to
231 * reserve them due to locking order violation issues. The vmw_cotable_scrub
232 * function implements a partial unbind() without that requirement but with the
233 * following restrictions.
234 * 1) Before the cotable is again used by the GPU, vmw_cotable_unscrub() must
236 * 2) Before the cotable backing buffer is used by the CPU, or during the
237 * resource destruction, vmw_cotable_unbind() must be called.
239 int vmw_cotable_scrub(struct vmw_resource *res, bool readback)
241 struct vmw_cotable *vcotbl = vmw_cotable(res);
242 struct vmw_private *dev_priv = res->dev_priv;
246 SVGA3dCmdHeader header;
247 SVGA3dCmdDXReadbackCOTable body;
250 SVGA3dCmdHeader header;
251 SVGA3dCmdDXSetCOTable body;
254 if (vcotbl->scrubbed)
257 if (co_info[vcotbl->type].unbind_func)
258 co_info[vcotbl->type].unbind_func(dev_priv,
259 &vcotbl->resource_list,
261 submit_size = sizeof(*cmd1);
263 submit_size += sizeof(*cmd0);
265 cmd1 = vmw_fifo_reserve_dx(dev_priv, submit_size, SVGA3D_INVALID_ID);
267 DRM_ERROR("Failed reserving FIFO space for cotable "
272 vcotbl->size_read_back = 0;
274 cmd0 = (void *) cmd1;
275 cmd0->header.id = SVGA_3D_CMD_DX_READBACK_COTABLE;
276 cmd0->header.size = sizeof(cmd0->body);
277 cmd0->body.cid = vcotbl->ctx->id;
278 cmd0->body.type = vcotbl->type;
279 cmd1 = (void *) &cmd0[1];
280 vcotbl->size_read_back = res->backup_size;
282 cmd1->header.id = SVGA_3D_CMD_DX_SET_COTABLE;
283 cmd1->header.size = sizeof(cmd1->body);
284 cmd1->body.cid = vcotbl->ctx->id;
285 cmd1->body.type = vcotbl->type;
286 cmd1->body.mobid = SVGA3D_INVALID_ID;
287 cmd1->body.validSizeInBytes = 0;
288 vmw_fifo_commit_flush(dev_priv, submit_size);
289 vcotbl->scrubbed = true;
291 /* Trigger a create() on next validate. */
298 * vmw_cotable_unbind - Cotable resource unbind callback
300 * @res: Pointer to the cotable resource.
301 * @readback: Whether to read back cotable data to the backup buffer.
302 * val_buf: Pointer to a struct ttm_validate_buffer prepared by the caller
303 * for convenience / fencing.
305 * Unbinds the cotable from the device and fences the backup buffer.
307 static int vmw_cotable_unbind(struct vmw_resource *res,
309 struct ttm_validate_buffer *val_buf)
311 struct vmw_cotable *vcotbl = vmw_cotable(res);
312 struct vmw_private *dev_priv = res->dev_priv;
313 struct ttm_buffer_object *bo = val_buf->bo;
314 struct vmw_fence_obj *fence;
316 if (list_empty(&res->mob_head))
319 WARN_ON_ONCE(bo->mem.mem_type != VMW_PL_MOB);
320 lockdep_assert_held(&bo->resv->lock.base);
322 mutex_lock(&dev_priv->binding_mutex);
323 if (!vcotbl->scrubbed)
324 vmw_dx_context_scrub_cotables(vcotbl->ctx, readback);
325 mutex_unlock(&dev_priv->binding_mutex);
326 (void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
327 vmw_fence_single_bo(bo, fence);
328 if (likely(fence != NULL))
329 vmw_fence_obj_unreference(&fence);
335 * vmw_cotable_readback - Read back a cotable without unbinding.
337 * @res: The cotable resource.
339 * Reads back a cotable to its backing mob without scrubbing the MOB from
340 * the cotable. The MOB is fenced for subsequent CPU access.
342 static int vmw_cotable_readback(struct vmw_resource *res)
344 struct vmw_cotable *vcotbl = vmw_cotable(res);
345 struct vmw_private *dev_priv = res->dev_priv;
348 SVGA3dCmdHeader header;
349 SVGA3dCmdDXReadbackCOTable body;
351 struct vmw_fence_obj *fence;
353 if (!vcotbl->scrubbed) {
354 cmd = vmw_fifo_reserve_dx(dev_priv, sizeof(*cmd),
357 DRM_ERROR("Failed reserving FIFO space for cotable "
361 cmd->header.id = SVGA_3D_CMD_DX_READBACK_COTABLE;
362 cmd->header.size = sizeof(cmd->body);
363 cmd->body.cid = vcotbl->ctx->id;
364 cmd->body.type = vcotbl->type;
365 vcotbl->size_read_back = res->backup_size;
366 vmw_fifo_commit(dev_priv, sizeof(*cmd));
369 (void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
370 vmw_fence_single_bo(&res->backup->base, fence);
371 vmw_fence_obj_unreference(&fence);
377 * vmw_cotable_resize - Resize a cotable.
379 * @res: The cotable resource.
380 * @new_size: The new size.
382 * Resizes a cotable and binds the new backup buffer.
383 * On failure the cotable is left intact.
384 * Important! This function may not fail once the MOB switch has been
385 * committed to hardware. That would put the device context in an
386 * invalid state which we can't currently recover from.
388 static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
390 struct vmw_private *dev_priv = res->dev_priv;
391 struct vmw_cotable *vcotbl = vmw_cotable(res);
392 struct vmw_dma_buffer *buf, *old_buf = res->backup;
393 struct ttm_buffer_object *bo, *old_bo = &res->backup->base;
394 size_t old_size = res->backup_size;
395 size_t old_size_read_back = vcotbl->size_read_back;
396 size_t cur_size_read_back;
397 struct ttm_bo_kmap_obj old_map, new_map;
401 ret = vmw_cotable_readback(res);
405 cur_size_read_back = vcotbl->size_read_back;
406 vcotbl->size_read_back = old_size_read_back;
409 * While device is processing, Allocate and reserve a buffer object
410 * for the new COTable. Initially pin the buffer object to make sure
411 * we can use tryreserve without failure.
413 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
417 ret = vmw_dmabuf_init(dev_priv, buf, new_size, &vmw_mob_ne_placement,
418 true, vmw_dmabuf_bo_free);
420 DRM_ERROR("Failed initializing new cotable MOB.\n");
425 WARN_ON_ONCE(ttm_bo_reserve(bo, false, true, NULL));
427 ret = ttm_bo_wait(old_bo, false, false);
428 if (unlikely(ret != 0)) {
429 DRM_ERROR("Failed waiting for cotable unbind.\n");
434 * Do a page by page copy of COTables. This eliminates slow vmap()s.
435 * This should really be a TTM utility.
437 for (i = 0; i < old_bo->num_pages; ++i) {
440 ret = ttm_bo_kmap(old_bo, i, 1, &old_map);
441 if (unlikely(ret != 0)) {
442 DRM_ERROR("Failed mapping old COTable on resize.\n");
445 ret = ttm_bo_kmap(bo, i, 1, &new_map);
446 if (unlikely(ret != 0)) {
447 DRM_ERROR("Failed mapping new COTable on resize.\n");
450 memcpy(ttm_kmap_obj_virtual(&new_map, &dummy),
451 ttm_kmap_obj_virtual(&old_map, &dummy),
453 ttm_bo_kunmap(&new_map);
454 ttm_bo_kunmap(&old_map);
457 /* Unpin new buffer, and switch backup buffers. */
458 ret = ttm_bo_validate(bo, &vmw_mob_placement, false, false);
459 if (unlikely(ret != 0)) {
460 DRM_ERROR("Failed validating new COTable backup buffer.\n");
465 res->backup_size = new_size;
466 vcotbl->size_read_back = cur_size_read_back;
469 * Now tell the device to switch. If this fails, then we need to
470 * revert the full resize.
472 ret = vmw_cotable_unscrub(res);
474 DRM_ERROR("Failed switching COTable backup buffer.\n");
475 res->backup = old_buf;
476 res->backup_size = old_size;
477 vcotbl->size_read_back = old_size_read_back;
481 /* Let go of the old mob. */
482 list_del(&res->mob_head);
483 list_add_tail(&res->mob_head, &buf->res_list);
484 vmw_dmabuf_unreference(&old_buf);
485 res->id = vcotbl->type;
490 ttm_bo_kunmap(&old_map);
492 ttm_bo_unreserve(bo);
493 vmw_dmabuf_unreference(&buf);
499 * vmw_cotable_create - Cotable resource create callback
501 * @res: Pointer to a cotable resource.
503 * There is no separate create command for cotables, so this callback, which
504 * is called before bind() in the validation sequence is instead used for two
506 * 1) Unscrub the cotable if it is scrubbed and still attached to a backup
507 * buffer, that is, if @res->mob_head is non-empty.
508 * 2) Resize the cotable if needed.
510 static int vmw_cotable_create(struct vmw_resource *res)
512 struct vmw_cotable *vcotbl = vmw_cotable(res);
513 size_t new_size = res->backup_size;
517 /* Check whether we need to resize the cotable */
518 needed_size = (vcotbl->seen_entries + 1) * co_info[vcotbl->type].size;
519 while (needed_size > new_size)
522 if (likely(new_size <= res->backup_size)) {
523 if (vcotbl->scrubbed && !list_empty(&res->mob_head)) {
524 ret = vmw_cotable_unscrub(res);
528 res->id = vcotbl->type;
532 return vmw_cotable_resize(res, new_size);
536 * vmw_hw_cotable_destroy - Cotable hw_destroy callback
538 * @res: Pointer to a cotable resource.
540 * The final (part of resource destruction) destroy callback.
542 static void vmw_hw_cotable_destroy(struct vmw_resource *res)
544 (void) vmw_cotable_destroy(res);
547 static size_t cotable_acc_size;
550 * vmw_cotable_free - Cotable resource destructor
552 * @res: Pointer to a cotable resource.
554 static void vmw_cotable_free(struct vmw_resource *res)
556 struct vmw_private *dev_priv = res->dev_priv;
559 ttm_mem_global_free(vmw_mem_glob(dev_priv), cotable_acc_size);
563 * vmw_cotable_alloc - Create a cotable resource
565 * @dev_priv: Pointer to a device private struct.
566 * @ctx: Pointer to the context resource.
567 * The cotable resource will not add a refcount.
568 * @type: The cotable type.
570 struct vmw_resource *vmw_cotable_alloc(struct vmw_private *dev_priv,
571 struct vmw_resource *ctx,
574 struct vmw_cotable *vcotbl;
578 if (unlikely(cotable_acc_size == 0))
579 cotable_acc_size = ttm_round_pot(sizeof(struct vmw_cotable));
581 ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv),
582 cotable_acc_size, false, true);
586 vcotbl = kzalloc(sizeof(*vcotbl), GFP_KERNEL);
587 if (unlikely(!vcotbl)) {
592 ret = vmw_resource_init(dev_priv, &vcotbl->res, true,
593 vmw_cotable_free, &vmw_cotable_func);
594 if (unlikely(ret != 0))
597 INIT_LIST_HEAD(&vcotbl->resource_list);
598 vcotbl->res.id = type;
599 vcotbl->res.backup_size = PAGE_SIZE;
600 num_entries = PAGE_SIZE / co_info[type].size;
601 if (num_entries < co_info[type].min_initial_entries) {
602 vcotbl->res.backup_size = co_info[type].min_initial_entries *
604 vcotbl->res.backup_size =
605 (vcotbl->res.backup_size + PAGE_SIZE - 1) & PAGE_MASK;
608 vcotbl->scrubbed = true;
609 vcotbl->seen_entries = -1;
613 vmw_resource_activate(&vcotbl->res, vmw_hw_cotable_destroy);
620 ttm_mem_global_free(vmw_mem_glob(dev_priv), cotable_acc_size);
625 * vmw_cotable_notify - Notify the cotable about an item creation
627 * @res: Pointer to a cotable resource.
630 int vmw_cotable_notify(struct vmw_resource *res, int id)
632 struct vmw_cotable *vcotbl = vmw_cotable(res);
634 if (id < 0 || id >= SVGA_COTABLE_MAX_IDS) {
635 DRM_ERROR("Illegal COTable id. Type is %u. Id is %d\n",
636 (unsigned) vcotbl->type, id);
640 if (vcotbl->seen_entries < id) {
641 /* Trigger a call to create() on next validate */
643 vcotbl->seen_entries = id;
650 * vmw_cotable_add_view - add a view to the cotable's list of active views.
652 * @res: pointer struct vmw_resource representing the cotable.
653 * @head: pointer to the struct list_head member of the resource, dedicated
654 * to the cotable active resource list.
656 void vmw_cotable_add_resource(struct vmw_resource *res, struct list_head *head)
658 struct vmw_cotable *vcotbl =
659 container_of(res, struct vmw_cotable, res);
661 list_add_tail(head, &vcotbl->resource_list);