2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/mutex.h>
24 #include <linux/log2.h>
25 #include <linux/sched.h>
26 #include <linux/sched/mm.h>
27 #include <linux/slab.h>
28 #include <linux/amd-iommu.h>
29 #include <linux/notifier.h>
30 #include <linux/compat.h>
35 #include "kfd_dbgmgr.h"
38 * Initial size for the array of queues.
39 * The allocated size is doubled each time
40 * it is exceeded up to MAX_PROCESS_QUEUES.
42 #define INITIAL_QUEUE_ARRAY_SIZE 16
45 * List of struct kfd_process (field kfd_process).
46 * Unique/indexed by mm_struct*
48 #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
49 static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
50 static DEFINE_MUTEX(kfd_processes_mutex);
52 DEFINE_STATIC_SRCU(kfd_processes_srcu);
54 static struct workqueue_struct *kfd_process_wq;
56 struct kfd_process_release_work {
57 struct work_struct kfd_work;
58 struct kfd_process *p;
61 static struct kfd_process *find_process(const struct task_struct *thread);
62 static struct kfd_process *create_process(const struct task_struct *thread);
64 void kfd_process_create_wq(void)
67 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
70 void kfd_process_destroy_wq(void)
73 destroy_workqueue(kfd_process_wq);
74 kfd_process_wq = NULL;
78 struct kfd_process *kfd_create_process(const struct task_struct *thread)
80 struct kfd_process *process;
83 return ERR_PTR(-EINVAL);
85 /* Only the pthreads threading model is supported. */
86 if (thread->group_leader->mm != thread->mm)
87 return ERR_PTR(-EINVAL);
89 /* Take mmap_sem because we call __mmu_notifier_register inside */
90 down_write(&thread->mm->mmap_sem);
93 * take kfd processes mutex before starting of process creation
94 * so there won't be a case where two threads of the same process
95 * create two kfd_process structures
97 mutex_lock(&kfd_processes_mutex);
99 /* A prior open of /dev/kfd could have already created the process. */
100 process = find_process(thread);
102 pr_debug("Process already found\n");
105 process = create_process(thread);
107 mutex_unlock(&kfd_processes_mutex);
109 up_write(&thread->mm->mmap_sem);
114 struct kfd_process *kfd_get_process(const struct task_struct *thread)
116 struct kfd_process *process;
119 return ERR_PTR(-EINVAL);
121 /* Only the pthreads threading model is supported. */
122 if (thread->group_leader->mm != thread->mm)
123 return ERR_PTR(-EINVAL);
125 process = find_process(thread);
127 return ERR_PTR(-EINVAL);
132 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
134 struct kfd_process *process;
136 hash_for_each_possible_rcu(kfd_processes_table, process,
137 kfd_processes, (uintptr_t)mm)
138 if (process->mm == mm)
144 static struct kfd_process *find_process(const struct task_struct *thread)
146 struct kfd_process *p;
149 idx = srcu_read_lock(&kfd_processes_srcu);
150 p = find_process_by_mm(thread->mm);
151 srcu_read_unlock(&kfd_processes_srcu, idx);
156 static void kfd_process_wq_release(struct work_struct *work)
158 struct kfd_process_release_work *my_work;
159 struct kfd_process_device *pdd, *temp;
160 struct kfd_process *p;
162 my_work = (struct kfd_process_release_work *) work;
166 pr_debug("Releasing process (pasid %d) in workqueue\n",
169 mutex_lock(&p->mutex);
171 list_for_each_entry_safe(pdd, temp, &p->per_device_data,
173 pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
174 pdd->dev->id, p->pasid);
176 if (pdd->reset_wavefronts)
177 dbgdev_wave_reset_wavefronts(pdd->dev, p);
179 amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
180 list_del(&pdd->per_device_list);
185 kfd_event_free_process(p);
187 kfd_pasid_free(p->pasid);
189 mutex_unlock(&p->mutex);
191 mutex_destroy(&p->mutex);
200 static void kfd_process_destroy_delayed(struct rcu_head *rcu)
202 struct kfd_process_release_work *work;
203 struct kfd_process *p;
205 p = container_of(rcu, struct kfd_process, rcu);
206 WARN_ON(atomic_read(&p->mm->mm_count) <= 0);
210 work = kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
213 INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
215 queue_work(kfd_process_wq, (struct work_struct *) work);
219 static void kfd_process_notifier_release(struct mmu_notifier *mn,
220 struct mm_struct *mm)
222 struct kfd_process *p;
223 struct kfd_process_device *pdd = NULL;
226 * The kfd_process structure can not be free because the
227 * mmu_notifier srcu is read locked
229 p = container_of(mn, struct kfd_process, mmu_notifier);
230 if (WARN_ON(p->mm != mm))
233 mutex_lock(&kfd_processes_mutex);
234 hash_del_rcu(&p->kfd_processes);
235 mutex_unlock(&kfd_processes_mutex);
236 synchronize_srcu(&kfd_processes_srcu);
238 mutex_lock(&p->mutex);
240 /* In case our notifier is called before IOMMU notifier */
243 /* Iterate over all process device data structure and check
244 * if we should delete debug managers and reset all wavefronts
246 list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
247 if ((pdd->dev->dbgmgr) &&
248 (pdd->dev->dbgmgr->pasid == p->pasid))
249 kfd_dbgmgr_destroy(pdd->dev->dbgmgr);
251 if (pdd->reset_wavefronts) {
252 pr_warn("Resetting all wave fronts\n");
253 dbgdev_wave_reset_wavefronts(pdd->dev, p);
254 pdd->reset_wavefronts = false;
258 mutex_unlock(&p->mutex);
261 * Because we drop mm_count inside kfd_process_destroy_delayed
262 * and because the mmu_notifier_unregister function also drop
263 * mm_count we need to take an extra count here.
266 mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
267 mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
270 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
271 .release = kfd_process_notifier_release,
274 static struct kfd_process *create_process(const struct task_struct *thread)
276 struct kfd_process *process;
279 process = kzalloc(sizeof(*process), GFP_KERNEL);
282 goto err_alloc_process;
284 process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
285 sizeof(process->queues[0]), GFP_KERNEL);
286 if (!process->queues)
287 goto err_alloc_queues;
289 process->pasid = kfd_pasid_alloc();
290 if (process->pasid == 0)
291 goto err_alloc_pasid;
293 mutex_init(&process->mutex);
295 process->mm = thread->mm;
297 /* register notifier */
298 process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
299 err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
301 goto err_mmu_notifier;
303 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
304 (uintptr_t)process->mm);
306 process->lead_thread = thread->group_leader;
308 process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
310 INIT_LIST_HEAD(&process->per_device_data);
312 kfd_event_init_process(process);
314 err = pqm_init(&process->pqm, process);
316 goto err_process_pqm_init;
318 /* init process apertures*/
319 process->is_32bit_user_mode = in_compat_syscall();
320 err = kfd_init_apertures(process);
322 goto err_init_apertures;
327 pqm_uninit(&process->pqm);
328 err_process_pqm_init:
329 hash_del_rcu(&process->kfd_processes);
331 mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
333 mutex_destroy(&process->mutex);
334 kfd_pasid_free(process->pasid);
336 kfree(process->queues);
343 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
344 struct kfd_process *p)
346 struct kfd_process_device *pdd = NULL;
348 list_for_each_entry(pdd, &p->per_device_data, per_device_list)
355 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
356 struct kfd_process *p)
358 struct kfd_process_device *pdd = NULL;
360 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
363 INIT_LIST_HEAD(&pdd->qpd.queues_list);
364 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
365 pdd->qpd.dqm = dev->dqm;
366 pdd->reset_wavefronts = false;
367 list_add(&pdd->per_device_list, &p->per_device_data);
374 * Direct the IOMMU to bind the process (specifically the pasid->mm)
376 * Unbinding occurs when the process dies or the device is removed.
378 * Assumes that the process lock is held.
380 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
381 struct kfd_process *p)
383 struct kfd_process_device *pdd;
386 pdd = kfd_get_process_device_data(dev, p);
388 pr_err("Process device data doesn't exist\n");
389 return ERR_PTR(-ENOMEM);
395 err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
404 void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
406 struct kfd_process *p;
407 struct kfd_process_device *pdd;
410 * Look for the process that matches the pasid. If there is no such
411 * process, we either released it in amdkfd's own notifier, or there
412 * is a bug. Unfortunately, there is no way to tell...
414 p = kfd_lookup_process_by_pasid(pasid);
418 pr_debug("Unbinding process %d from IOMMU\n", pasid);
420 if ((dev->dbgmgr) && (dev->dbgmgr->pasid == p->pasid))
421 kfd_dbgmgr_destroy(dev->dbgmgr);
425 pdd = kfd_get_process_device_data(dev, p);
428 mutex_unlock(&p->mutex);
432 if (pdd->reset_wavefronts) {
433 dbgdev_wave_reset_wavefronts(pdd->dev, p);
434 pdd->reset_wavefronts = false;
438 * Just mark pdd as unbound, because we still need it
439 * to call amd_iommu_unbind_pasid() in when the
441 * We don't call amd_iommu_unbind_pasid() here
442 * because the IOMMU called us.
446 mutex_unlock(&p->mutex);
449 struct kfd_process_device *kfd_get_first_process_device_data(
450 struct kfd_process *p)
452 return list_first_entry(&p->per_device_data,
453 struct kfd_process_device,
457 struct kfd_process_device *kfd_get_next_process_device_data(
458 struct kfd_process *p,
459 struct kfd_process_device *pdd)
461 if (list_is_last(&pdd->per_device_list, &p->per_device_data))
463 return list_next_entry(pdd, per_device_list);
466 bool kfd_has_process_device_data(struct kfd_process *p)
468 return !(list_empty(&p->per_device_data));
471 /* This returns with process->mutex locked. */
472 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
474 struct kfd_process *p;
477 int idx = srcu_read_lock(&kfd_processes_srcu);
479 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
480 if (p->pasid == pasid) {
481 mutex_lock(&p->mutex);
486 srcu_read_unlock(&kfd_processes_srcu, idx);