From e797bda3fd29137f6c151dfa10ea6a61c17895ce Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 22 Jul 2019 20:47:16 +0200 Subject: smp/hotplug: Track booted once CPUs in a cpumask The booted once information which is required to deal with the MCE broadcast issue on X86 correctly is stored in the per cpu hotplug state, which is perfectly fine for the intended purpose. X86 needs that information for supporting NMI broadcasting via shortcuts, but retrieving it from per cpu data is cumbersome. Move it to a cpumask so the information can be checked against the cpu_present_mask quickly. No functional change intended. Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20190722105219.818822855@linutronix.de --- include/linux/cpumask.h | 2 ++ kernel/cpu.c | 11 +++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 21755471b1c3..693124900f0a 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -115,6 +115,8 @@ extern struct cpumask __cpu_active_mask; #define cpu_active(cpu) ((cpu) == 0) #endif +extern cpumask_t cpus_booted_once_mask; + static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits) { #ifdef CONFIG_DEBUG_PER_CPU_MAPS diff --git a/kernel/cpu.c b/kernel/cpu.c index e84c0873559e..05778e32674a 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -62,7 +62,6 @@ struct cpuhp_cpu_state { bool rollback; bool single; bool bringup; - bool booted_once; struct hlist_node *node; struct hlist_node *last; enum cpuhp_state cb_state; @@ -76,6 +75,10 @@ static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = { .fail = CPUHP_INVALID, }; +#ifdef CONFIG_SMP +cpumask_t cpus_booted_once_mask; +#endif + #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP) static struct lockdep_map cpuhp_state_up_map = STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map); @@ -433,7 +436,7 @@ static inline bool cpu_smt_allowed(unsigned int cpu) * CPU. Otherwise, a broadacasted MCE observing CR4.MCE=0b on any * core will shutdown the machine. */ - return !per_cpu(cpuhp_state, cpu).booted_once; + return !cpumask_test_cpu(cpu, &cpus_booted_once_mask); } #else static inline bool cpu_smt_allowed(unsigned int cpu) { return true; } @@ -1066,7 +1069,7 @@ void notify_cpu_starting(unsigned int cpu) int ret; rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */ - st->booted_once = true; + cpumask_set_cpu(cpu, &cpus_booted_once_mask); while (st->state < target) { st->state++; ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL); @@ -2334,7 +2337,7 @@ void __init boot_cpu_init(void) void __init boot_cpu_hotplug_init(void) { #ifdef CONFIG_SMP - this_cpu_write(cpuhp_state.booted_once, true); + cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask); #endif this_cpu_write(cpuhp_state.state, CPUHP_ONLINE); } -- cgit v1.2.3 From b9fa6442f7043e2cdd247905d4f3b80f2e9605cb Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 22 Jul 2019 20:47:24 +0200 Subject: cpumask: Implement cpumask_or_equal() The IPI code of x86 needs to evaluate whether the target cpumask is equal to the cpu_online_mask or equal except for the calling CPU. To replace the current implementation which requires the usage of a temporary cpumask, which might involve allocations, add a new function which compares a cpumask to the result of two other cpumasks which are or'ed together before comparison. This allows to make the required decision in one go and the calling code then can check for the calling CPU being set in the target mask with cpumask_test_cpu(). Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20190722105220.585449120@linutronix.de --- include/linux/bitmap.h | 23 +++++++++++++++++++++++ include/linux/cpumask.h | 14 ++++++++++++++ lib/bitmap.c | 20 ++++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index f58e97446abc..90528f12bdfa 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -120,6 +120,10 @@ extern int __bitmap_empty(const unsigned long *bitmap, unsigned int nbits); extern int __bitmap_full(const unsigned long *bitmap, unsigned int nbits); extern int __bitmap_equal(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); +extern bool __pure __bitmap_or_equal(const unsigned long *src1, + const unsigned long *src2, + const unsigned long *src3, + unsigned int nbits); extern void __bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int nbits); extern void __bitmap_shift_right(unsigned long *dst, const unsigned long *src, @@ -321,6 +325,25 @@ static inline int bitmap_equal(const unsigned long *src1, return __bitmap_equal(src1, src2, nbits); } +/** + * bitmap_or_equal - Check whether the or of two bitnaps is equal to a third + * @src1: Pointer to bitmap 1 + * @src2: Pointer to bitmap 2 will be or'ed with bitmap 1 + * @src3: Pointer to bitmap 3. Compare to the result of *@src1 | *@src2 + * + * Returns: True if (*@src1 | *@src2) == *@src3, false otherwise + */ +static inline bool bitmap_or_equal(const unsigned long *src1, + const unsigned long *src2, + const unsigned long *src3, + unsigned int nbits) +{ + if (!small_const_nbits(nbits)) + return __bitmap_or_equal(src1, src2, src3, nbits); + + return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits)); +} + static inline int bitmap_intersects(const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 693124900f0a..0c7db5efe66c 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -475,6 +475,20 @@ static inline bool cpumask_equal(const struct cpumask *src1p, nr_cpumask_bits); } +/** + * cpumask_or_equal - *src1p | *src2p == *src3p + * @src1p: the first input + * @src2p: the second input + * @src3p: the third input + */ +static inline bool cpumask_or_equal(const struct cpumask *src1p, + const struct cpumask *src2p, + const struct cpumask *src3p) +{ + return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p), + cpumask_bits(src3p), nr_cpumask_bits); +} + /** * cpumask_intersects - (*src1p & *src2p) != 0 * @src1p: the first input diff --git a/lib/bitmap.c b/lib/bitmap.c index bbe2589e8497..f9e834841e94 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -59,6 +59,26 @@ int __bitmap_equal(const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_equal); +bool __bitmap_or_equal(const unsigned long *bitmap1, + const unsigned long *bitmap2, + const unsigned long *bitmap3, + unsigned int bits) +{ + unsigned int k, lim = bits / BITS_PER_LONG; + unsigned long tmp; + + for (k = 0; k < lim; ++k) { + if ((bitmap1[k] | bitmap2[k]) != bitmap3[k]) + return false; + } + + if (!(bits % BITS_PER_LONG)) + return true; + + tmp = (bitmap1[k] | bitmap2[k]) ^ bitmap3[k]; + return (tmp & BITMAP_LAST_WORD_MASK(bits)) == 0; +} + void __bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int bits) { unsigned int k, lim = BITS_TO_LONGS(bits); -- cgit v1.2.3 From 0c09ab96fc820109d63097a2adcbbd20836b655f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 9 Jul 2019 16:23:40 +0200 Subject: cpu/hotplug: Cache number of online CPUs Re-evaluating the bitmap wheight of the online cpus bitmap in every invocation of num_online_cpus() over and over is a pretty useless exercise. Especially when num_online_cpus() is used in code paths like the IPI delivery of x86 or the membarrier code. Cache the number of online CPUs in the core and just return the cached variable. The accessor function provides only a snapshot when used without protection against concurrent CPU hotplug. The storage needs to use an atomic_t because the kexec and reboot code (ab)use set_cpu_online() in their 'shutdown' handlers without any form of serialization as pointed out by Mathieu. Regular CPU hotplug usage is properly serialized. Signed-off-by: Thomas Gleixner Reviewed-by: Mathieu Desnoyers Link: https://lkml.kernel.org/r/alpine.DEB.2.21.1907091622590.1634@nanos.tec.linutronix.de --- include/linux/cpumask.h | 25 ++++++++++++++++--------- kernel/cpu.c | 24 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 0c7db5efe66c..b5a5a1ed9efd 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -10,6 +10,7 @@ #include #include #include +#include #include /* Don't assign or return these: may not be this big! */ @@ -95,8 +96,21 @@ extern struct cpumask __cpu_active_mask; #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) +extern atomic_t __num_online_cpus; + #if NR_CPUS > 1 -#define num_online_cpus() cpumask_weight(cpu_online_mask) +/** + * num_online_cpus() - Read the number of online CPUs + * + * Despite the fact that __num_online_cpus is of type atomic_t, this + * interface gives only a momentary snapshot and is not protected against + * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held + * region. + */ +static inline unsigned int num_online_cpus(void) +{ + return atomic_read(&__num_online_cpus); +} #define num_possible_cpus() cpumask_weight(cpu_possible_mask) #define num_present_cpus() cpumask_weight(cpu_present_mask) #define num_active_cpus() cpumask_weight(cpu_active_mask) @@ -821,14 +835,7 @@ set_cpu_present(unsigned int cpu, bool present) cpumask_clear_cpu(cpu, &__cpu_present_mask); } -static inline void -set_cpu_online(unsigned int cpu, bool online) -{ - if (online) - cpumask_set_cpu(cpu, &__cpu_online_mask); - else - cpumask_clear_cpu(cpu, &__cpu_online_mask); -} +void set_cpu_online(unsigned int cpu, bool online); static inline void set_cpu_active(unsigned int cpu, bool active) diff --git a/kernel/cpu.c b/kernel/cpu.c index 05778e32674a..e1967e9eddc2 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -2298,6 +2298,9 @@ EXPORT_SYMBOL(__cpu_present_mask); struct cpumask __cpu_active_mask __read_mostly; EXPORT_SYMBOL(__cpu_active_mask); +atomic_t __num_online_cpus __read_mostly; +EXPORT_SYMBOL(__num_online_cpus); + void init_cpu_present(const struct cpumask *src) { cpumask_copy(&__cpu_present_mask, src); @@ -2313,6 +2316,27 @@ void init_cpu_online(const struct cpumask *src) cpumask_copy(&__cpu_online_mask, src); } +void set_cpu_online(unsigned int cpu, bool online) +{ + /* + * atomic_inc/dec() is required to handle the horrid abuse of this + * function by the reboot and kexec code which invoke it from + * IPI/NMI broadcasts when shutting down CPUs. Invocation from + * regular CPU hotplug is properly serialized. + * + * Note, that the fact that __num_online_cpus is of type atomic_t + * does not protect readers which are not serialized against + * concurrent hotplug operations. + */ + if (online) { + if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask)) + atomic_inc(&__num_online_cpus); + } else { + if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask)) + atomic_dec(&__num_online_cpus); + } +} + /* * Activate the first processor. */ -- cgit v1.2.3