drivers/gpu/drm/i915/gvt/scheduler.c

   1 /*
   2  * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Zhi Wang <zhi.a.wang@intel.com>
  25  *
  26  * Contributors:
  27  *    Ping Gao <ping.a.gao@intel.com>
  28  *    Tina Zhang <tina.zhang@intel.com>
  29  *    Chanbin Du <changbin.du@intel.com>
  30  *    Min He <min.he@intel.com>
  31  *    Bing Niu <bing.niu@intel.com>
  32  *    Zhenyu Wang <zhenyuw@linux.intel.com>
  33  *
  34  */
  35
  36 #include <linux/kthread.h>
  37
  38 #include "i915_drv.h"
  39 #include "gvt.h"
  40
  41 #define RING_CTX_OFF(x) \
  42         offsetof(struct execlist_ring_context, x)
  43
  44 static void set_context_pdp_root_pointer(
  45                 struct execlist_ring_context *ring_context,
  46                 u32 pdp[8])
  47 {
  48         struct execlist_mmio_pair *pdp_pair = &ring_context->pdp3_UDW;
  49         int i;
  50
  51         for (i = 0; i < 8; i++)
  52                 pdp_pair[i].val = pdp[7 - i];
  53 }
  54
  55 static int populate_shadow_context(struct intel_vgpu_workload *workload)
  56 {
  57         struct intel_vgpu *vgpu = workload->vgpu;
  58         struct intel_gvt *gvt = vgpu->gvt;
  59         int ring_id = workload->ring_id;
  60         struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
  61         struct drm_i915_gem_object *ctx_obj =
  62                 shadow_ctx->engine[ring_id].state->obj;
  63         struct execlist_ring_context *shadow_ring_context;
  64         struct page *page;
  65         void *dst;
  66         unsigned long context_gpa, context_page_num;
  67         int i;
  68
  69         gvt_dbg_sched("ring id %d workload lrca %x", ring_id,
  70                         workload->ctx_desc.lrca);
  71
  72         context_page_num = gvt->dev_priv->engine[ring_id]->context_size;
  73
  74         context_page_num = context_page_num >> PAGE_SHIFT;
  75
  76         if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS)
  77                 context_page_num = 19;
  78
  79         i = 2;
  80
  81         while (i < context_page_num) {
  82                 context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
  83                                 (u32)((workload->ctx_desc.lrca + i) <<
  84                                 GTT_PAGE_SHIFT));
  85                 if (context_gpa == INTEL_GVT_INVALID_ADDR) {
  86                         gvt_vgpu_err("Invalid guest context descriptor\n");
  87                         return -EINVAL;
  88                 }
  89
  90                 page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i);
  91                 dst = kmap(page);
  92                 intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst,
  93                                 GTT_PAGE_SIZE);
  94                 kunmap(page);
  95                 i++;
  96         }
  97
  98         page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
  99         shadow_ring_context = kmap(page);
 100
 101 #define COPY_REG(name) \
 102         intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
 103                 + RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
 104
 105         COPY_REG(ctx_ctrl);
 106         COPY_REG(ctx_timestamp);
 107
 108         if (ring_id == RCS) {
 109                 COPY_REG(bb_per_ctx_ptr);
 110                 COPY_REG(rcs_indirect_ctx);
 111                 COPY_REG(rcs_indirect_ctx_offset);
 112         }
 113 #undef COPY_REG
 114
 115         set_context_pdp_root_pointer(shadow_ring_context,
 116                                      workload->shadow_mm->shadow_page_table);
 117
 118         intel_gvt_hypervisor_read_gpa(vgpu,
 119                         workload->ring_context_gpa +
 120                         sizeof(*shadow_ring_context),
 121                         (void *)shadow_ring_context +
 122                         sizeof(*shadow_ring_context),
 123                         GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
 124
 125         kunmap(page);
 126         return 0;
 127 }
 128
 129 static inline bool is_gvt_request(struct drm_i915_gem_request *req)
 130 {
 131         return i915_gem_context_force_single_submission(req->ctx);
 132 }
 133
 134 static int shadow_context_status_change(struct notifier_block *nb,
 135                 unsigned long action, void *data)
 136 {
 137         struct drm_i915_gem_request *req = (struct drm_i915_gem_request *)data;
 138         struct intel_gvt *gvt = container_of(nb, struct intel_gvt,
 139                                 shadow_ctx_notifier_block[req->engine->id]);
 140         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 141         enum intel_engine_id ring_id = req->engine->id;
 142         struct intel_vgpu_workload *workload;
 143
 144         if (!is_gvt_request(req)) {
 145                 spin_lock_bh(&scheduler->mmio_context_lock);
 146                 if (action == INTEL_CONTEXT_SCHEDULE_IN &&
 147                     scheduler->engine_owner[ring_id]) {
 148                         /* Switch ring from vGPU to host. */
 149                         intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
 150                                               NULL, ring_id);
 151                         scheduler->engine_owner[ring_id] = NULL;
 152                 }
 153                 spin_unlock_bh(&scheduler->mmio_context_lock);
 154
 155                 return NOTIFY_OK;
 156         }
 157
 158         workload = scheduler->current_workload[ring_id];
 159         if (unlikely(!workload))
 160                 return NOTIFY_OK;
 161
 162         switch (action) {
 163         case INTEL_CONTEXT_SCHEDULE_IN:
 164                 spin_lock_bh(&scheduler->mmio_context_lock);
 165                 if (workload->vgpu != scheduler->engine_owner[ring_id]) {
 166                         /* Switch ring from host to vGPU or vGPU to vGPU. */
 167                         intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
 168                                               workload->vgpu, ring_id);
 169                         scheduler->engine_owner[ring_id] = workload->vgpu;
 170                 } else
 171                         gvt_dbg_sched("skip ring %d mmio switch for vgpu%d\n",
 172                                       ring_id, workload->vgpu->id);
 173                 spin_unlock_bh(&scheduler->mmio_context_lock);
 174                 atomic_set(&workload->shadow_ctx_active, 1);
 175                 break;
 176         case INTEL_CONTEXT_SCHEDULE_OUT:
 177                 atomic_set(&workload->shadow_ctx_active, 0);
 178                 break;
 179         default:
 180                 WARN_ON(1);
 181                 return NOTIFY_OK;
 182         }
 183         wake_up(&workload->shadow_ctx_status_wq);
 184         return NOTIFY_OK;
 185 }
 186
 187 static void shadow_context_descriptor_update(struct i915_gem_context *ctx,
 188                 struct intel_engine_cs *engine)
 189 {
 190         struct intel_context *ce = &ctx->engine[engine->id];
 191         u64 desc = 0;
 192
 193         desc = ce->lrc_desc;
 194
 195         /* Update bits 0-11 of the context descriptor which includes flags
 196          * like GEN8_CTX_* cached in desc_template
 197          */
 198         desc &= U64_MAX << 12;
 199         desc |= ctx->desc_template & ((1ULL << 12) - 1);
 200
 201         ce->lrc_desc = desc;
 202 }
 203
 204 /**
 205  * intel_gvt_scan_and_shadow_workload - audit the workload by scanning and
 206  * shadow it as well, include ringbuffer,wa_ctx and ctx.
 207  * @workload: an abstract entity for each execlist submission.
 208  *
 209  * This function is called before the workload submitting to i915, to make
 210  * sure the content of the workload is valid.
 211  */
 212 int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
 213 {
 214         int ring_id = workload->ring_id;
 215         struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
 216         struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
 217         struct drm_i915_gem_request *rq;
 218         struct intel_vgpu *vgpu = workload->vgpu;
 219         int ret;
 220
 221         lockdep_assert_held(&dev_priv->drm.struct_mutex);
 222
 223         if (workload->shadowed)
 224                 return 0;
 225
 226         shadow_ctx->desc_template &= ~(0x3 << GEN8_CTX_ADDRESSING_MODE_SHIFT);
 227         shadow_ctx->desc_template |= workload->ctx_desc.addressing_mode <<
 228                                     GEN8_CTX_ADDRESSING_MODE_SHIFT;
 229
 230         if (!test_and_set_bit(ring_id, vgpu->shadow_ctx_desc_updated))
 231                 shadow_context_descriptor_update(shadow_ctx,
 232                                         dev_priv->engine[ring_id]);
 233
 234         rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx);
 235         if (IS_ERR(rq)) {
 236                 gvt_vgpu_err("fail to allocate gem request\n");
 237                 ret = PTR_ERR(rq);
 238                 goto out;
 239         }
 240
 241         gvt_dbg_sched("ring id %d get i915 gem request %p\n", ring_id, rq);
 242
 243         workload->req = i915_gem_request_get(rq);
 244
 245         ret = intel_gvt_scan_and_shadow_ringbuffer(workload);
 246         if (ret)
 247                 goto out;
 248
 249         if ((workload->ring_id == RCS) &&
 250             (workload->wa_ctx.indirect_ctx.size != 0)) {
 251                 ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);
 252                 if (ret)
 253                         goto out;
 254         }
 255
 256         ret = populate_shadow_context(workload);
 257         if (ret)
 258                 goto out;
 259
 260         workload->shadowed = true;
 261
 262 out:
 263         return ret;
 264 }
 265
 266 static int dispatch_workload(struct intel_vgpu_workload *workload)
 267 {
 268         int ring_id = workload->ring_id;
 269         struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
 270         struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
 271         struct intel_engine_cs *engine = dev_priv->engine[ring_id];
 272         struct intel_vgpu *vgpu = workload->vgpu;
 273         struct intel_ring *ring;
 274         int ret = 0;
 275
 276         gvt_dbg_sched("ring id %d prepare to dispatch workload %p\n",
 277                 ring_id, workload);
 278
 279         mutex_lock(&dev_priv->drm.struct_mutex);
 280
 281         ret = intel_gvt_scan_and_shadow_workload(workload);
 282         if (ret)
 283                 goto out;
 284
 285         if (workload->prepare) {
 286                 ret = workload->prepare(workload);
 287                 if (ret)
 288                         goto out;
 289         }
 290
 291         /* pin shadow context by gvt even the shadow context will be pinned
 292          * when i915 alloc request. That is because gvt will update the guest
 293          * context from shadow context when workload is completed, and at that
 294          * moment, i915 may already unpined the shadow context to make the
 295          * shadow_ctx pages invalid. So gvt need to pin itself. After update
 296          * the guest context, gvt can unpin the shadow_ctx safely.
 297          */
 298         ring = engine->context_pin(engine, shadow_ctx);
 299         if (IS_ERR(ring)) {
 300                 ret = PTR_ERR(ring);
 301                 gvt_vgpu_err("fail to pin shadow context\n");
 302                 goto out;
 303         }
 304
 305 out:
 306         if (ret)
 307                 workload->status = ret;
 308
 309         if (!IS_ERR_OR_NULL(workload->req)) {
 310                 gvt_dbg_sched("ring id %d submit workload to i915 %p\n",
 311                                 ring_id, workload->req);
 312                 i915_add_request(workload->req);
 313                 workload->dispatched = true;
 314         }
 315
 316         mutex_unlock(&dev_priv->drm.struct_mutex);
 317         return ret;
 318 }
 319
 320 static struct intel_vgpu_workload *pick_next_workload(
 321                 struct intel_gvt *gvt, int ring_id)
 322 {
 323         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 324         struct intel_vgpu_workload *workload = NULL;
 325
 326         mutex_lock(&gvt->lock);
 327
 328         /*
 329          * no current vgpu / will be scheduled out / no workload
 330          * bail out
 331          */
 332         if (!scheduler->current_vgpu) {
 333                 gvt_dbg_sched("ring id %d stop - no current vgpu\n", ring_id);
 334                 goto out;
 335         }
 336
 337         if (scheduler->need_reschedule) {
 338                 gvt_dbg_sched("ring id %d stop - will reschedule\n", ring_id);
 339                 goto out;
 340         }
 341
 342         if (list_empty(workload_q_head(scheduler->current_vgpu, ring_id)))
 343                 goto out;
 344
 345         /*
 346          * still have current workload, maybe the workload disptacher
 347          * fail to submit it for some reason, resubmit it.
 348          */
 349         if (scheduler->current_workload[ring_id]) {
 350                 workload = scheduler->current_workload[ring_id];
 351                 gvt_dbg_sched("ring id %d still have current workload %p\n",
 352                                 ring_id, workload);
 353                 goto out;
 354         }
 355
 356         /*
 357          * pick a workload as current workload
 358          * once current workload is set, schedule policy routines
 359          * will wait the current workload is finished when trying to
 360          * schedule out a vgpu.
 361          */
 362         scheduler->current_workload[ring_id] = container_of(
 363                         workload_q_head(scheduler->current_vgpu, ring_id)->next,
 364                         struct intel_vgpu_workload, list);
 365
 366         workload = scheduler->current_workload[ring_id];
 367
 368         gvt_dbg_sched("ring id %d pick new workload %p\n", ring_id, workload);
 369
 370         atomic_inc(&workload->vgpu->running_workload_num);
 371 out:
 372         mutex_unlock(&gvt->lock);
 373         return workload;
 374 }
 375
 376 static void update_guest_context(struct intel_vgpu_workload *workload)
 377 {
 378         struct intel_vgpu *vgpu = workload->vgpu;
 379         struct intel_gvt *gvt = vgpu->gvt;
 380         int ring_id = workload->ring_id;
 381         struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
 382         struct drm_i915_gem_object *ctx_obj =
 383                 shadow_ctx->engine[ring_id].state->obj;
 384         struct execlist_ring_context *shadow_ring_context;
 385         struct page *page;
 386         void *src;
 387         unsigned long context_gpa, context_page_num;
 388         int i;
 389
 390         gvt_dbg_sched("ring id %d workload lrca %x\n", ring_id,
 391                         workload->ctx_desc.lrca);
 392
 393         context_page_num = gvt->dev_priv->engine[ring_id]->context_size;
 394
 395         context_page_num = context_page_num >> PAGE_SHIFT;
 396
 397         if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS)
 398                 context_page_num = 19;
 399
 400         i = 2;
 401
 402         while (i < context_page_num) {
 403                 context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
 404                                 (u32)((workload->ctx_desc.lrca + i) <<
 405                                         GTT_PAGE_SHIFT));
 406                 if (context_gpa == INTEL_GVT_INVALID_ADDR) {
 407                         gvt_vgpu_err("invalid guest context descriptor\n");
 408                         return;
 409                 }
 410
 411                 page = i915_gem_object_get_page(ctx_obj, LRC_PPHWSP_PN + i);
 412                 src = kmap(page);
 413                 intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src,
 414                                 GTT_PAGE_SIZE);
 415                 kunmap(page);
 416                 i++;
 417         }
 418
 419         intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa +
 420                 RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4);
 421
 422         page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
 423         shadow_ring_context = kmap(page);
 424
 425 #define COPY_REG(name) \
 426         intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \
 427                 RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
 428
 429         COPY_REG(ctx_ctrl);
 430         COPY_REG(ctx_timestamp);
 431
 432 #undef COPY_REG
 433
 434         intel_gvt_hypervisor_write_gpa(vgpu,
 435                         workload->ring_context_gpa +
 436                         sizeof(*shadow_ring_context),
 437                         (void *)shadow_ring_context +
 438                         sizeof(*shadow_ring_context),
 439                         GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
 440
 441         kunmap(page);
 442 }
 443
 444 static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 445 {
 446         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 447         struct intel_vgpu_workload *workload;
 448         struct intel_vgpu *vgpu;
 449         int event;
 450
 451         mutex_lock(&gvt->lock);
 452
 453         workload = scheduler->current_workload[ring_id];
 454         vgpu = workload->vgpu;
 455
 456         /* For the workload w/ request, needs to wait for the context
 457          * switch to make sure request is completed.
 458          * For the workload w/o request, directly complete the workload.
 459          */
 460         if (workload->req) {
 461                 struct drm_i915_private *dev_priv =
 462                         workload->vgpu->gvt->dev_priv;
 463                 struct intel_engine_cs *engine =
 464                         dev_priv->engine[workload->ring_id];
 465                 wait_event(workload->shadow_ctx_status_wq,
 466                            !atomic_read(&workload->shadow_ctx_active));
 467
 468                 /* If this request caused GPU hang, req->fence.error will
 469                  * be set to -EIO. Use -EIO to set workload status so
 470                  * that when this request caused GPU hang, didn't trigger
 471                  * context switch interrupt to guest.
 472                  */
 473                 if (likely(workload->status == -EINPROGRESS)) {
 474                         if (workload->req->fence.error == -EIO)
 475                                 workload->status = -EIO;
 476                         else
 477                                 workload->status = 0;
 478                 }
 479
 480                 i915_gem_request_put(fetch_and_zero(&workload->req));
 481
 482                 if (!workload->status && !(vgpu->resetting_eng &
 483                                            ENGINE_MASK(ring_id))) {
 484                         update_guest_context(workload);
 485
 486                         for_each_set_bit(event, workload->pending_events,
 487                                          INTEL_GVT_EVENT_MAX)
 488                                 intel_vgpu_trigger_virtual_event(vgpu, event);
 489                 }
 490                 mutex_lock(&dev_priv->drm.struct_mutex);
 491                 /* unpin shadow ctx as the shadow_ctx update is done */
 492                 engine->context_unpin(engine, workload->vgpu->shadow_ctx);
 493                 mutex_unlock(&dev_priv->drm.struct_mutex);
 494         }
 495
 496         gvt_dbg_sched("ring id %d complete workload %p status %d\n",
 497                         ring_id, workload, workload->status);
 498
 499         scheduler->current_workload[ring_id] = NULL;
 500
 501         list_del_init(&workload->list);
 502         workload->complete(workload);
 503
 504         atomic_dec(&vgpu->running_workload_num);
 505         wake_up(&scheduler->workload_complete_wq);
 506
 507         if (gvt->scheduler.need_reschedule)
 508                 intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED);
 509
 510         mutex_unlock(&gvt->lock);
 511 }
 512
 513 struct workload_thread_param {
 514         struct intel_gvt *gvt;
 515         int ring_id;
 516 };
 517
 518 static int workload_thread(void *priv)
 519 {
 520         struct workload_thread_param *p = (struct workload_thread_param *)priv;
 521         struct intel_gvt *gvt = p->gvt;
 522         int ring_id = p->ring_id;
 523         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 524         struct intel_vgpu_workload *workload = NULL;
 525         struct intel_vgpu *vgpu = NULL;
 526         int ret;
 527         bool need_force_wake = IS_SKYLAKE(gvt->dev_priv)
 528                         || IS_KABYLAKE(gvt->dev_priv);
 529         DEFINE_WAIT_FUNC(wait, woken_wake_function);
 530
 531         kfree(p);
 532
 533         gvt_dbg_core("workload thread for ring %d started\n", ring_id);
 534
 535         while (!kthread_should_stop()) {
 536                 add_wait_queue(&scheduler->waitq[ring_id], &wait);
 537                 do {
 538                         workload = pick_next_workload(gvt, ring_id);
 539                         if (workload)
 540                                 break;
 541                         wait_woken(&wait, TASK_INTERRUPTIBLE,
 542                                    MAX_SCHEDULE_TIMEOUT);
 543                 } while (!kthread_should_stop());
 544                 remove_wait_queue(&scheduler->waitq[ring_id], &wait);
 545
 546                 if (!workload)
 547                         break;
 548
 549                 gvt_dbg_sched("ring id %d next workload %p vgpu %d\n",
 550                                 workload->ring_id, workload,
 551                                 workload->vgpu->id);
 552
 553                 intel_runtime_pm_get(gvt->dev_priv);
 554
 555                 gvt_dbg_sched("ring id %d will dispatch workload %p\n",
 556                                 workload->ring_id, workload);
 557
 558                 if (need_force_wake)
 559                         intel_uncore_forcewake_get(gvt->dev_priv,
 560                                         FORCEWAKE_ALL);
 561
 562                 mutex_lock(&gvt->lock);
 563                 ret = dispatch_workload(workload);
 564                 mutex_unlock(&gvt->lock);
 565
 566                 if (ret) {
 567                         vgpu = workload->vgpu;
 568                         gvt_vgpu_err("fail to dispatch workload, skip\n");
 569                         goto complete;
 570                 }
 571
 572                 gvt_dbg_sched("ring id %d wait workload %p\n",
 573                                 workload->ring_id, workload);
 574                 i915_wait_request(workload->req, 0, MAX_SCHEDULE_TIMEOUT);
 575
 576 complete:
 577                 gvt_dbg_sched("will complete workload %p, status: %d\n",
 578                                 workload, workload->status);
 579
 580                 complete_current_workload(gvt, ring_id);
 581
 582                 if (need_force_wake)
 583                         intel_uncore_forcewake_put(gvt->dev_priv,
 584                                         FORCEWAKE_ALL);
 585
 586                 intel_runtime_pm_put(gvt->dev_priv);
 587         }
 588         return 0;
 589 }
 590
 591 void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu)
 592 {
 593         struct intel_gvt *gvt = vgpu->gvt;
 594         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 595
 596         if (atomic_read(&vgpu->running_workload_num)) {
 597                 gvt_dbg_sched("wait vgpu idle\n");
 598
 599                 wait_event(scheduler->workload_complete_wq,
 600                                 !atomic_read(&vgpu->running_workload_num));
 601         }
 602 }
 603
 604 void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt)
 605 {
 606         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 607         struct intel_engine_cs *engine;
 608         enum intel_engine_id i;
 609
 610         gvt_dbg_core("clean workload scheduler\n");
 611
 612         for_each_engine(engine, gvt->dev_priv, i) {
 613                 atomic_notifier_chain_unregister(
 614                                         &engine->context_status_notifier,
 615                                         &gvt->shadow_ctx_notifier_block[i]);
 616                 kthread_stop(scheduler->thread[i]);
 617         }
 618 }
 619
 620 int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt)
 621 {
 622         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 623         struct workload_thread_param *param = NULL;
 624         struct intel_engine_cs *engine;
 625         enum intel_engine_id i;
 626         int ret;
 627
 628         gvt_dbg_core("init workload scheduler\n");
 629
 630         init_waitqueue_head(&scheduler->workload_complete_wq);
 631
 632         for_each_engine(engine, gvt->dev_priv, i) {
 633                 init_waitqueue_head(&scheduler->waitq[i]);
 634
 635                 param = kzalloc(sizeof(*param), GFP_KERNEL);
 636                 if (!param) {
 637                         ret = -ENOMEM;
 638                         goto err;
 639                 }
 640
 641                 param->gvt = gvt;
 642                 param->ring_id = i;
 643
 644                 scheduler->thread[i] = kthread_run(workload_thread, param,
 645                         "gvt workload %d", i);
 646                 if (IS_ERR(scheduler->thread[i])) {
 647                         gvt_err("fail to create workload thread\n");
 648                         ret = PTR_ERR(scheduler->thread[i]);
 649                         goto err;
 650                 }
 651
 652                 gvt->shadow_ctx_notifier_block[i].notifier_call =
 653                                         shadow_context_status_change;
 654                 atomic_notifier_chain_register(&engine->context_status_notifier,
 655                                         &gvt->shadow_ctx_notifier_block[i]);
 656         }
 657         return 0;
 658 err:
 659         intel_gvt_clean_workload_scheduler(gvt);
 660         kfree(param);
 661         param = NULL;
 662         return ret;
 663 }
 664
 665 void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu)
 666 {
 667         i915_gem_context_put(vgpu->shadow_ctx);
 668 }
 669
 670 int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu)
 671 {
 672         atomic_set(&vgpu->running_workload_num, 0);
 673
 674         vgpu->shadow_ctx = i915_gem_context_create_gvt(
 675                         &vgpu->gvt->dev_priv->drm);
 676         if (IS_ERR(vgpu->shadow_ctx))
 677                 return PTR_ERR(vgpu->shadow_ctx);
 678
 679         vgpu->shadow_ctx->engine[RCS].initialised = true;
 680
 681         bitmap_zero(vgpu->shadow_ctx_desc_updated, I915_NUM_ENGINES);
 682
 683         return 0;
 684 }