2 * Copyright(c) 2015 - 2017 Intel Corporation.
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47 #include <linux/topology.h>
48 #include <linux/cpumask.h>
49 #include <linux/module.h>
50 #include <linux/interrupt.h>
57 struct hfi1_affinity_node_list node_affinity = {
58 .list = LIST_HEAD_INIT(node_affinity.list),
59 .lock = __MUTEX_INITIALIZER(node_affinity.lock)
62 /* Name of IRQ types, indexed by enum irq_type */
63 static const char * const irq_type_names[] = {
70 /* Per NUMA node count of HFI devices */
71 static unsigned int *hfi1_per_node_cntr;
73 static inline void init_cpu_mask_set(struct cpu_mask_set *set)
75 cpumask_clear(&set->mask);
76 cpumask_clear(&set->used);
80 /* Initialize non-HT cpu cores mask */
81 void init_real_cpu_mask(void)
83 int possible, curr_cpu, i, ht;
85 cpumask_clear(&node_affinity.real_cpu_mask);
87 /* Start with cpu online mask as the real cpu mask */
88 cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask);
91 * Remove HT cores from the real cpu mask. Do this in two steps below.
93 possible = cpumask_weight(&node_affinity.real_cpu_mask);
94 ht = cpumask_weight(topology_sibling_cpumask(
95 cpumask_first(&node_affinity.real_cpu_mask)));
97 * Step 1. Skip over the first N HT siblings and use them as the
98 * "real" cores. Assumes that HT cores are not enumerated in
99 * succession (except in the single core case).
101 curr_cpu = cpumask_first(&node_affinity.real_cpu_mask);
102 for (i = 0; i < possible / ht; i++)
103 curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
105 * Step 2. Remove the remaining HT siblings. Use cpumask_next() to
108 for (; i < possible; i++) {
109 cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask);
110 curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
114 int node_affinity_init(void)
117 struct pci_dev *dev = NULL;
118 const struct pci_device_id *ids = hfi1_pci_tbl;
120 cpumask_clear(&node_affinity.proc.used);
121 cpumask_copy(&node_affinity.proc.mask, cpu_online_mask);
123 node_affinity.proc.gen = 0;
124 node_affinity.num_core_siblings =
125 cpumask_weight(topology_sibling_cpumask(
126 cpumask_first(&node_affinity.proc.mask)
128 node_affinity.num_possible_nodes = num_possible_nodes();
129 node_affinity.num_online_nodes = num_online_nodes();
130 node_affinity.num_online_cpus = num_online_cpus();
133 * The real cpu mask is part of the affinity struct but it has to be
134 * initialized early. It is needed to calculate the number of user
135 * contexts in set_up_context_variables().
137 init_real_cpu_mask();
139 hfi1_per_node_cntr = kcalloc(node_affinity.num_possible_nodes,
140 sizeof(*hfi1_per_node_cntr), GFP_KERNEL);
141 if (!hfi1_per_node_cntr)
144 while (ids->vendor) {
146 while ((dev = pci_get_device(ids->vendor, ids->device, dev))) {
147 node = pcibus_to_node(dev->bus);
151 hfi1_per_node_cntr[node]++;
160 * Invalid PCI NUMA node information found, note it, and populate
163 pr_err("HFI: Invalid PCI NUMA node. Performance may be affected\n");
164 pr_err("HFI: System BIOS may need to be upgraded\n");
165 for (node = 0; node < node_affinity.num_possible_nodes; node++)
166 hfi1_per_node_cntr[node] = 1;
171 void node_affinity_destroy(void)
173 struct list_head *pos, *q;
174 struct hfi1_affinity_node *entry;
176 mutex_lock(&node_affinity.lock);
177 list_for_each_safe(pos, q, &node_affinity.list) {
178 entry = list_entry(pos, struct hfi1_affinity_node,
183 mutex_unlock(&node_affinity.lock);
184 kfree(hfi1_per_node_cntr);
187 static struct hfi1_affinity_node *node_affinity_allocate(int node)
189 struct hfi1_affinity_node *entry;
191 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
195 INIT_LIST_HEAD(&entry->list);
201 * It appends an entry to the list.
202 * It *must* be called with node_affinity.lock held.
204 static void node_affinity_add_tail(struct hfi1_affinity_node *entry)
206 list_add_tail(&entry->list, &node_affinity.list);
209 /* It must be called with node_affinity.lock held */
210 static struct hfi1_affinity_node *node_affinity_lookup(int node)
212 struct list_head *pos;
213 struct hfi1_affinity_node *entry;
215 list_for_each(pos, &node_affinity.list) {
216 entry = list_entry(pos, struct hfi1_affinity_node, list);
217 if (entry->node == node)
225 * Interrupt affinity.
227 * non-rcv avail gets a default mask that
228 * starts as possible cpus with threads reset
229 * and each rcv avail reset.
231 * rcv avail gets node relative 1 wrapping back
232 * to the node relative 1 as necessary.
235 int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
237 int node = pcibus_to_node(dd->pcidev->bus);
238 struct hfi1_affinity_node *entry;
239 const struct cpumask *local_mask;
240 int curr_cpu, possible, i;
243 * If the BIOS does not have the NUMA node information set, select
244 * NUMA 0 so we get consistent performance.
247 dd_dev_err(dd, "Invalid PCI NUMA node. Performance may be affected\n");
252 local_mask = cpumask_of_node(dd->node);
253 if (cpumask_first(local_mask) >= nr_cpu_ids)
254 local_mask = topology_core_cpumask(0);
256 mutex_lock(&node_affinity.lock);
257 entry = node_affinity_lookup(dd->node);
260 * If this is the first time this NUMA node's affinity is used,
261 * create an entry in the global affinity structure and initialize it.
264 entry = node_affinity_allocate(node);
267 "Unable to allocate global affinity node\n");
268 mutex_unlock(&node_affinity.lock);
271 init_cpu_mask_set(&entry->def_intr);
272 init_cpu_mask_set(&entry->rcv_intr);
273 cpumask_clear(&entry->general_intr_mask);
274 /* Use the "real" cpu mask of this node as the default */
275 cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask,
278 /* fill in the receive list */
279 possible = cpumask_weight(&entry->def_intr.mask);
280 curr_cpu = cpumask_first(&entry->def_intr.mask);
283 /* only one CPU, everyone will use it */
284 cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask);
285 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
288 * The general/control context will be the first CPU in
289 * the default list, so it is removed from the default
290 * list and added to the general interrupt list.
292 cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask);
293 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
294 curr_cpu = cpumask_next(curr_cpu,
295 &entry->def_intr.mask);
298 * Remove the remaining kernel receive queues from
299 * the default list and add them to the receive list.
302 i < (dd->n_krcv_queues - 1) *
303 hfi1_per_node_cntr[dd->node];
305 cpumask_clear_cpu(curr_cpu,
306 &entry->def_intr.mask);
307 cpumask_set_cpu(curr_cpu,
308 &entry->rcv_intr.mask);
309 curr_cpu = cpumask_next(curr_cpu,
310 &entry->def_intr.mask);
311 if (curr_cpu >= nr_cpu_ids)
316 * If there ends up being 0 CPU cores leftover for SDMA
317 * engines, use the same CPU cores as general/control
320 if (cpumask_weight(&entry->def_intr.mask) == 0)
321 cpumask_copy(&entry->def_intr.mask,
322 &entry->general_intr_mask);
325 node_affinity_add_tail(entry);
327 mutex_unlock(&node_affinity.lock);
332 * Function updates the irq affinity hint for msix after it has been changed
333 * by the user using the /proc/irq interface. This function only accepts
334 * one cpu in the mask.
336 static void hfi1_update_sdma_affinity(struct hfi1_msix_entry *msix, int cpu)
338 struct sdma_engine *sde = msix->arg;
339 struct hfi1_devdata *dd = sde->dd;
340 struct hfi1_affinity_node *entry;
341 struct cpu_mask_set *set;
344 if (cpu > num_online_cpus() || cpu == sde->cpu)
347 mutex_lock(&node_affinity.lock);
348 entry = node_affinity_lookup(dd->node);
354 cpumask_clear(&msix->mask);
355 cpumask_set_cpu(cpu, &msix->mask);
356 dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n",
357 msix->irq, irq_type_names[msix->type],
359 irq_set_affinity_hint(msix->irq, &msix->mask);
362 * Set the new cpu in the hfi1_affinity_node and clean
363 * the old cpu if it is not used by any other IRQ
365 set = &entry->def_intr;
366 cpumask_set_cpu(cpu, &set->mask);
367 cpumask_set_cpu(cpu, &set->used);
368 for (i = 0; i < dd->num_msix_entries; i++) {
369 struct hfi1_msix_entry *other_msix;
371 other_msix = &dd->msix_entries[i];
372 if (other_msix->type != IRQ_SDMA || other_msix == msix)
375 if (cpumask_test_cpu(old_cpu, &other_msix->mask))
378 cpumask_clear_cpu(old_cpu, &set->mask);
379 cpumask_clear_cpu(old_cpu, &set->used);
381 mutex_unlock(&node_affinity.lock);
384 static void hfi1_irq_notifier_notify(struct irq_affinity_notify *notify,
385 const cpumask_t *mask)
387 int cpu = cpumask_first(mask);
388 struct hfi1_msix_entry *msix = container_of(notify,
389 struct hfi1_msix_entry,
392 /* Only one CPU configuration supported currently */
393 hfi1_update_sdma_affinity(msix, cpu);
396 static void hfi1_irq_notifier_release(struct kref *ref)
399 * This is required by affinity notifier. We don't have anything to
404 static void hfi1_setup_sdma_notifier(struct hfi1_msix_entry *msix)
406 struct irq_affinity_notify *notify = &msix->notify;
408 notify->irq = msix->irq;
409 notify->notify = hfi1_irq_notifier_notify;
410 notify->release = hfi1_irq_notifier_release;
412 if (irq_set_affinity_notifier(notify->irq, notify))
413 pr_err("Failed to register sdma irq affinity notifier for irq %d\n",
417 static void hfi1_cleanup_sdma_notifier(struct hfi1_msix_entry *msix)
419 struct irq_affinity_notify *notify = &msix->notify;
421 if (irq_set_affinity_notifier(notify->irq, NULL))
422 pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n",
427 * Function sets the irq affinity for msix.
428 * It *must* be called with node_affinity.lock held.
430 static int get_irq_affinity(struct hfi1_devdata *dd,
431 struct hfi1_msix_entry *msix)
434 struct hfi1_affinity_node *entry;
435 struct cpu_mask_set *set = NULL;
436 struct sdma_engine *sde = NULL;
437 struct hfi1_ctxtdata *rcd = NULL;
442 cpumask_clear(&msix->mask);
444 entry = node_affinity_lookup(dd->node);
446 switch (msix->type) {
448 sde = (struct sdma_engine *)msix->arg;
449 scnprintf(extra, 64, "engine %u", sde->this_idx);
450 set = &entry->def_intr;
453 cpu = cpumask_first(&entry->general_intr_mask);
456 rcd = (struct hfi1_ctxtdata *)msix->arg;
457 if (rcd->ctxt == HFI1_CTRL_CTXT)
458 cpu = cpumask_first(&entry->general_intr_mask);
460 set = &entry->rcv_intr;
461 scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
464 dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type);
469 * The general and control contexts are placed on a particular
470 * CPU, which is set above. Skip accounting for it. Everything else
471 * finds its CPU here.
473 if (cpu == -1 && set) {
474 if (!zalloc_cpumask_var(&diff, GFP_KERNEL))
477 if (cpumask_equal(&set->mask, &set->used)) {
479 * We've used up all the CPUs, bump up the generation
480 * and reset the 'used' map
483 cpumask_clear(&set->used);
485 cpumask_andnot(diff, &set->mask, &set->used);
486 cpu = cpumask_first(diff);
487 cpumask_set_cpu(cpu, &set->used);
489 free_cpumask_var(diff);
492 cpumask_set_cpu(cpu, &msix->mask);
493 dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n",
494 msix->irq, irq_type_names[msix->type],
496 irq_set_affinity_hint(msix->irq, &msix->mask);
498 if (msix->type == IRQ_SDMA) {
500 hfi1_setup_sdma_notifier(msix);
506 int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
510 mutex_lock(&node_affinity.lock);
511 ret = get_irq_affinity(dd, msix);
512 mutex_unlock(&node_affinity.lock);
516 void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
517 struct hfi1_msix_entry *msix)
519 struct cpu_mask_set *set = NULL;
520 struct hfi1_ctxtdata *rcd;
521 struct hfi1_affinity_node *entry;
523 mutex_lock(&node_affinity.lock);
524 entry = node_affinity_lookup(dd->node);
526 switch (msix->type) {
528 set = &entry->def_intr;
529 hfi1_cleanup_sdma_notifier(msix);
532 /* Don't do accounting for general contexts */
535 rcd = (struct hfi1_ctxtdata *)msix->arg;
536 /* Don't do accounting for control contexts */
537 if (rcd->ctxt != HFI1_CTRL_CTXT)
538 set = &entry->rcv_intr;
541 mutex_unlock(&node_affinity.lock);
546 cpumask_andnot(&set->used, &set->used, &msix->mask);
547 if (cpumask_empty(&set->used) && set->gen) {
549 cpumask_copy(&set->used, &set->mask);
553 irq_set_affinity_hint(msix->irq, NULL);
554 cpumask_clear(&msix->mask);
555 mutex_unlock(&node_affinity.lock);
558 /* This should be called with node_affinity.lock held */
559 static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,
560 struct hfi1_affinity_node_list *affinity)
562 int possible, curr_cpu, i;
563 uint num_cores_per_socket = node_affinity.num_online_cpus /
564 affinity->num_core_siblings /
565 node_affinity.num_online_nodes;
567 cpumask_copy(hw_thread_mask, &affinity->proc.mask);
568 if (affinity->num_core_siblings > 0) {
569 /* Removing other siblings not needed for now */
570 possible = cpumask_weight(hw_thread_mask);
571 curr_cpu = cpumask_first(hw_thread_mask);
573 i < num_cores_per_socket * node_affinity.num_online_nodes;
575 curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
577 for (; i < possible; i++) {
578 cpumask_clear_cpu(curr_cpu, hw_thread_mask);
579 curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
582 /* Identifying correct HW threads within physical cores */
583 cpumask_shift_left(hw_thread_mask, hw_thread_mask,
584 num_cores_per_socket *
585 node_affinity.num_online_nodes *
590 int hfi1_get_proc_affinity(int node)
592 int cpu = -1, ret, i;
593 struct hfi1_affinity_node *entry;
594 cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;
595 const struct cpumask *node_mask,
596 *proc_mask = ¤t->cpus_allowed;
597 struct hfi1_affinity_node_list *affinity = &node_affinity;
598 struct cpu_mask_set *set = &affinity->proc;
601 * check whether process/context affinity has already
604 if (cpumask_weight(proc_mask) == 1) {
605 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl",
606 current->pid, current->comm,
607 cpumask_pr_args(proc_mask));
609 * Mark the pre-set CPU as used. This is atomic so we don't
612 cpu = cpumask_first(proc_mask);
613 cpumask_set_cpu(cpu, &set->used);
615 } else if (cpumask_weight(proc_mask) < cpumask_weight(&set->mask)) {
616 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl",
617 current->pid, current->comm,
618 cpumask_pr_args(proc_mask));
623 * The process does not have a preset CPU affinity so find one to
624 * recommend using the following algorithm:
626 * For each user process that is opening a context on HFI Y:
627 * a) If all cores are filled, reinitialize the bitmask
628 * b) Fill real cores first, then HT cores (First set of HT
629 * cores on all physical cores, then second set of HT core,
630 * and, so on) in the following order:
632 * 1. Same NUMA node as HFI Y and not running an IRQ
634 * 2. Same NUMA node as HFI Y and running an IRQ handler
635 * 3. Different NUMA node to HFI Y and not running an IRQ
637 * 4. Different NUMA node to HFI Y and running an IRQ
639 * c) Mark core as filled in the bitmask. As user processes are
640 * done, clear cores from the bitmask.
643 ret = zalloc_cpumask_var(&diff, GFP_KERNEL);
646 ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL);
649 ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL);
651 goto free_hw_thread_mask;
652 ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL);
654 goto free_available_mask;
656 mutex_lock(&affinity->lock);
658 * If we've used all available HW threads, clear the mask and start
661 if (cpumask_equal(&set->mask, &set->used)) {
663 cpumask_clear(&set->used);
667 * If NUMA node has CPUs used by interrupt handlers, include them in the
668 * interrupt handler mask.
670 entry = node_affinity_lookup(node);
672 cpumask_copy(intrs_mask, (entry->def_intr.gen ?
673 &entry->def_intr.mask :
674 &entry->def_intr.used));
675 cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ?
676 &entry->rcv_intr.mask :
677 &entry->rcv_intr.used));
678 cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask);
680 hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl",
681 cpumask_pr_args(intrs_mask));
683 cpumask_copy(hw_thread_mask, &set->mask);
686 * If HT cores are enabled, identify which HW threads within the
687 * physical cores should be used.
689 if (affinity->num_core_siblings > 0) {
690 for (i = 0; i < affinity->num_core_siblings; i++) {
691 find_hw_thread_mask(i, hw_thread_mask, affinity);
694 * If there's at least one available core for this HW
695 * thread number, stop looking for a core.
697 * diff will always be not empty at least once in this
698 * loop as the used mask gets reset when
699 * (set->mask == set->used) before this loop.
701 cpumask_andnot(diff, hw_thread_mask, &set->used);
702 if (!cpumask_empty(diff))
706 hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",
707 cpumask_pr_args(hw_thread_mask));
709 node_mask = cpumask_of_node(node);
710 hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node,
711 cpumask_pr_args(node_mask));
713 /* Get cpumask of available CPUs on preferred NUMA */
714 cpumask_and(available_mask, hw_thread_mask, node_mask);
715 cpumask_andnot(available_mask, available_mask, &set->used);
716 hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node,
717 cpumask_pr_args(available_mask));
720 * At first, we don't want to place processes on the same
721 * CPUs as interrupt handlers. Then, CPUs running interrupt
724 * 1) If diff is not empty, then there are CPUs not running
725 * non-interrupt handlers available, so diff gets copied
726 * over to available_mask.
727 * 2) If diff is empty, then all CPUs not running interrupt
728 * handlers are taken, so available_mask contains all
729 * available CPUs running interrupt handlers.
730 * 3) If available_mask is empty, then all CPUs on the
731 * preferred NUMA node are taken, so other NUMA nodes are
732 * used for process assignments using the same method as
733 * the preferred NUMA node.
735 cpumask_andnot(diff, available_mask, intrs_mask);
736 if (!cpumask_empty(diff))
737 cpumask_copy(available_mask, diff);
739 /* If we don't have CPUs on the preferred node, use other NUMA nodes */
740 if (cpumask_empty(available_mask)) {
741 cpumask_andnot(available_mask, hw_thread_mask, &set->used);
742 /* Excluding preferred NUMA cores */
743 cpumask_andnot(available_mask, available_mask, node_mask);
745 "Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl",
746 cpumask_pr_args(available_mask));
749 * At first, we don't want to place processes on the same
750 * CPUs as interrupt handlers.
752 cpumask_andnot(diff, available_mask, intrs_mask);
753 if (!cpumask_empty(diff))
754 cpumask_copy(available_mask, diff);
756 hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",
757 cpumask_pr_args(available_mask));
759 cpu = cpumask_first(available_mask);
760 if (cpu >= nr_cpu_ids) /* empty */
763 cpumask_set_cpu(cpu, &set->used);
765 mutex_unlock(&affinity->lock);
766 hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu);
768 free_cpumask_var(intrs_mask);
770 free_cpumask_var(available_mask);
772 free_cpumask_var(hw_thread_mask);
774 free_cpumask_var(diff);
779 void hfi1_put_proc_affinity(int cpu)
781 struct hfi1_affinity_node_list *affinity = &node_affinity;
782 struct cpu_mask_set *set = &affinity->proc;
787 mutex_lock(&affinity->lock);
788 cpumask_clear_cpu(cpu, &set->used);
789 hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);
790 if (cpumask_empty(&set->used) && set->gen) {
792 cpumask_copy(&set->used, &set->mask);
794 mutex_unlock(&affinity->lock);