1 // SPDX-License-Identifier: GPL-2.0-only
3 * Housekeeping management. Manage the targets for routine code that can run on
4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
12 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
13 EXPORT_SYMBOL_GPL(housekeeping_overridden);
14 static cpumask_var_t housekeeping_mask;
15 static unsigned int housekeeping_flags;
17 bool housekeeping_enabled(enum hk_flags flags)
19 return !!(housekeeping_flags & flags);
21 EXPORT_SYMBOL_GPL(housekeeping_enabled);
23 int housekeeping_any_cpu(enum hk_flags flags)
27 if (static_branch_unlikely(&housekeeping_overridden)) {
28 if (housekeeping_flags & flags) {
29 cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id());
33 return cpumask_any_and(housekeeping_mask, cpu_online_mask);
36 return smp_processor_id();
38 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
40 const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
42 if (static_branch_unlikely(&housekeeping_overridden))
43 if (housekeeping_flags & flags)
44 return housekeeping_mask;
45 return cpu_possible_mask;
47 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
49 void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
51 if (static_branch_unlikely(&housekeeping_overridden))
52 if (housekeeping_flags & flags)
53 set_cpus_allowed_ptr(t, housekeeping_mask);
55 EXPORT_SYMBOL_GPL(housekeeping_affine);
57 bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
59 if (static_branch_unlikely(&housekeeping_overridden))
60 if (housekeeping_flags & flags)
61 return cpumask_test_cpu(cpu, housekeeping_mask);
64 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
66 void __init housekeeping_init(void)
68 if (!housekeeping_flags)
71 static_branch_enable(&housekeeping_overridden);
73 if (housekeeping_flags & HK_FLAG_TICK)
74 sched_tick_offload_init();
76 /* We need at least one CPU to handle housekeeping work */
77 WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
80 static int __init housekeeping_setup(char *str, enum hk_flags flags)
82 cpumask_var_t non_housekeeping_mask;
85 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
86 if (cpulist_parse(str, non_housekeeping_mask) < 0) {
87 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
88 free_bootmem_cpumask_var(non_housekeeping_mask);
92 alloc_bootmem_cpumask_var(&tmp);
93 if (!housekeeping_flags) {
94 alloc_bootmem_cpumask_var(&housekeeping_mask);
95 cpumask_andnot(housekeeping_mask,
96 cpu_possible_mask, non_housekeeping_mask);
98 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
99 if (cpumask_empty(tmp)) {
100 pr_warn("Housekeeping: must include one present CPU, "
101 "using boot CPU:%d\n", smp_processor_id());
102 __cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
103 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
106 cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
107 if (cpumask_empty(tmp))
108 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
109 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
110 if (!cpumask_equal(tmp, housekeeping_mask)) {
111 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
112 free_bootmem_cpumask_var(tmp);
113 free_bootmem_cpumask_var(non_housekeeping_mask);
117 free_bootmem_cpumask_var(tmp);
119 if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
120 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
121 tick_nohz_full_setup(non_housekeeping_mask);
123 pr_warn("Housekeeping: nohz unsupported."
124 " Build with CONFIG_NO_HZ_FULL\n");
125 free_bootmem_cpumask_var(non_housekeeping_mask);
130 housekeeping_flags |= flags;
132 free_bootmem_cpumask_var(non_housekeeping_mask);
137 static int __init housekeeping_nohz_full_setup(char *str)
141 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
142 HK_FLAG_MISC | HK_FLAG_KTHREAD;
144 return housekeeping_setup(str, flags);
146 __setup("nohz_full=", housekeeping_nohz_full_setup);
148 static int __init housekeeping_isolcpus_setup(char *str)
150 unsigned int flags = 0;
151 bool illegal = false;
155 while (isalpha(*str)) {
156 if (!strncmp(str, "nohz,", 5)) {
158 flags |= HK_FLAG_TICK;
162 if (!strncmp(str, "domain,", 7)) {
164 flags |= HK_FLAG_DOMAIN;
168 if (!strncmp(str, "managed_irq,", 12)) {
170 flags |= HK_FLAG_MANAGED_IRQ;
175 * Skip unknown sub-parameter and validate that it is not
176 * containing an invalid character.
178 for (par = str, len = 0; *str && *str != ','; str++, len++) {
179 if (!isalpha(*str) && *str != '_')
184 pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
188 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
192 /* Default behaviour for isolcpus without flags */
194 flags |= HK_FLAG_DOMAIN;
196 return housekeeping_setup(str, flags);
198 __setup("isolcpus=", housekeeping_isolcpus_setup);