diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 67874b8..cb6b94b 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1616,6 +1616,7 @@ config NR_CPUS config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" depends on SMP && HOTPLUG + select PERCPU_RWLOCK help Say Y here to experiment with turning CPUs off and on. CPUs can be controlled through /sys/devices/system/cpu. diff --git a/arch/blackfin/Kconfig b/arch/blackfin/Kconfig index b6f3ad5..83d9882 100644 --- a/arch/blackfin/Kconfig +++ b/arch/blackfin/Kconfig @@ -261,6 +261,7 @@ config NR_CPUS config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" depends on SMP && HOTPLUG + select PERCPU_RWLOCK default y config BF_REV_MIN diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 3279646..c246772 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -378,6 +378,7 @@ config HOTPLUG_CPU bool "Support for hot-pluggable CPUs (EXPERIMENTAL)" depends on SMP && EXPERIMENTAL select HOTPLUG + select PERCPU_RWLOCK default n ---help--- Say Y here to experiment with turning CPUs off and on. CPUs diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 2ac626a..f97c479 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -956,6 +956,7 @@ config SYS_HAS_EARLY_PRINTK config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" depends on SMP && HOTPLUG && SYS_SUPPORTS_HOTPLUG_CPU + select PERCPU_RWLOCK help Say Y here to allow turning CPUs off and on. CPUs can be controlled through /sys/devices/system/cpu. diff --git a/arch/mn10300/Kconfig b/arch/mn10300/Kconfig index e70001c..a64e488 100644 --- a/arch/mn10300/Kconfig +++ b/arch/mn10300/Kconfig @@ -60,6 +60,7 @@ config ARCH_HAS_ILOG2_U32 config HOTPLUG_CPU def_bool n + select PERCPU_RWLOCK source "init/Kconfig" diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index b77feff..6f55cd4 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -226,6 +226,7 @@ config HOTPLUG_CPU bool default y if SMP select HOTPLUG + select PERCPU_RWLOCK config ARCH_SELECT_MEMORY_MODEL def_bool y diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 17903f1..56b1f15 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -336,6 +336,7 @@ config HOTPLUG_CPU bool "Support for enabling/disabling CPUs" depends on SMP && HOTPLUG && EXPERIMENTAL && (PPC_PSERIES || \ PPC_PMAC || PPC_POWERNV || (PPC_85xx && !PPC_E500MC)) + select PERCPU_RWLOCK ---help--- Say Y here to be able to disable and re-enable individual CPUs at runtime on SMP machines. diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index b5ea38c..a9aafb4 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -299,6 +299,7 @@ config HOTPLUG_CPU prompt "Support for hot-pluggable CPUs" depends on SMP select HOTPLUG + select PERCPU_RWLOCK help Say Y here to be able to turn CPUs off and on. CPUs can be controlled through /sys/devices/system/cpu/cpu#. diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index babc2b8..8c92eef 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -765,6 +765,7 @@ config NR_CPUS config HOTPLUG_CPU bool "Support for hot-pluggable CPUs (EXPERIMENTAL)" depends on SMP && HOTPLUG && EXPERIMENTAL + select PERCPU_RWLOCK help Say Y here to experiment with turning CPUs off and on. CPUs can be controlled through /sys/devices/system/cpu. diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 9f2edb5..e2bd573 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -253,6 +253,7 @@ config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" depends on SPARC64 && SMP select HOTPLUG + select PERCPU_RWLOCK help Say Y here to experiment with turning CPUs off and on. CPUs can be controlled through /sys/devices/system/cpu/cpu#. diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 79795af..a225d12 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1689,6 +1689,7 @@ config PHYSICAL_ALIGN config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" depends on SMP && HOTPLUG + select PERCPU_RWLOCK ---help--- Say Y here to allow turning CPUs off and on. CPUs can be controlled through /sys/devices/system/cpu. diff --git a/include/linux/cpu.h b/include/linux/cpu.h index ce7a074..cf24da1 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -175,6 +175,8 @@ extern struct bus_type cpu_subsys; extern void get_online_cpus(void); extern void put_online_cpus(void); +extern void get_online_cpus_atomic(void); +extern void put_online_cpus_atomic(void); #define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri) #define register_hotcpu_notifier(nb) register_cpu_notifier(nb) #define unregister_hotcpu_notifier(nb) unregister_cpu_notifier(nb) @@ -198,6 +200,8 @@ static inline void cpu_hotplug_driver_unlock(void) #define get_online_cpus() do { } while (0) #define put_online_cpus() do { } while (0) +#define get_online_cpus_atomic() do { } while (0) +#define put_online_cpus_atomic() do { } while (0) #define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) /* These aren't inline functions due to a GCC bug. */ #define register_hotcpu_notifier(nb) ({ (void)(nb); 0; }) diff --git a/kernel/cpu.c b/kernel/cpu.c index 3046a50..1c84138 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -1,6 +1,18 @@ /* CPU control. * (C) 2001, 2002, 2003, 2004 Rusty Russell * + * Rework of the CPU hotplug offline mechanism to remove its dependence on + * the heavy-weight stop_machine() primitive, by Srivatsa S. Bhat and + * Paul E. McKenney. + * + * Copyright (C) IBM Corporation, 2012-2013 + * Authors: Srivatsa S. Bhat + * Paul E. McKenney + * + * With lots of invaluable suggestions from: + * Oleg Nesterov + * Tejun Heo + * * This code is licenced under the GPL. */ #include @@ -19,6 +31,7 @@ #include #include #include +#include #include "smpboot.h" @@ -133,6 +146,38 @@ static void cpu_hotplug_done(void) mutex_unlock(&cpu_hotplug.lock); } +/* + * Per-CPU Reader-Writer lock to synchronize between atomic hotplug + * readers and the CPU offline hotplug writer. + */ +DEFINE_STATIC_PERCPU_RWLOCK(hotplug_pcpu_rwlock); + +/* + * Invoked by atomic hotplug reader (a task which wants to prevent + * CPU offline, but which can't afford to sleep), to prevent CPUs from + * going offline. So, you can call this function from atomic contexts + * (including interrupt handlers). + * + * Note: This does NOT prevent CPUs from coming online! It only prevents + * CPUs from going offline. + * + * You can call this function recursively. + * + * Returns with preemption disabled (but interrupts remain as they are; + * they are not disabled). + */ +void get_online_cpus_atomic(void) +{ + percpu_read_lock_irqsafe(&hotplug_pcpu_rwlock); +} +EXPORT_SYMBOL_GPL(get_online_cpus_atomic); + +void put_online_cpus_atomic(void) +{ + percpu_read_unlock_irqsafe(&hotplug_pcpu_rwlock); +} +EXPORT_SYMBOL_GPL(put_online_cpus_atomic); + #else /* #if CONFIG_HOTPLUG_CPU */ static void cpu_hotplug_begin(void) {} static void cpu_hotplug_done(void) {} @@ -246,15 +291,21 @@ struct take_cpu_down_param { static int __ref take_cpu_down(void *_param) { struct take_cpu_down_param *param = _param; - int err; + unsigned long flags; + int err = 0; + + percpu_write_lock_irqsave(&hotplug_pcpu_rwlock, &flags); /* Ensure this CPU doesn't handle any more interrupts. */ err = __cpu_disable(); if (err < 0) - return err; + goto out; cpu_notify(CPU_DYING | param->mod, param->hcpu); - return 0; + +out: + percpu_write_unlock_irqrestore(&hotplug_pcpu_rwlock, &flags); + return err; } /* Requires cpu_add_remove_lock to be held */