diff options
| -rw-r--r-- | drivers/powercap/Kconfig | 10 | ||||
| -rw-r--r-- | drivers/powercap/Makefile | 1 | ||||
| -rw-r--r-- | drivers/powercap/idle_inject.c | 356 | ||||
| -rw-r--r-- | include/linux/idle_inject.h | 29 |
4 files changed, 396 insertions, 0 deletions
diff --git a/drivers/powercap/Kconfig b/drivers/powercap/Kconfig index 85727ef6ce8e..6ac27e5908f5 100644 --- a/drivers/powercap/Kconfig +++ b/drivers/powercap/Kconfig @@ -29,4 +29,14 @@ config INTEL_RAPL controller, CPU core (Power Plance 0), graphics uncore (Power Plane 1), etc. +config IDLE_INJECT + bool "Idle injection framework" + depends on CPU_IDLE + default n + help + This enables support for the idle injection framework. It + provides a way to force idle periods on a set of specified + CPUs for power capping. Idle period can be injected + synchronously on a set of specified CPUs or alternatively + on a per CPU basis. endif diff --git a/drivers/powercap/Makefile b/drivers/powercap/Makefile index 0a21ef31372b..1b328854b36e 100644 --- a/drivers/powercap/Makefile +++ b/drivers/powercap/Makefile @@ -1,2 +1,3 @@ obj-$(CONFIG_POWERCAP) += powercap_sys.o obj-$(CONFIG_INTEL_RAPL) += intel_rapl.o +obj-$(CONFIG_IDLE_INJECT) += idle_inject.o diff --git a/drivers/powercap/idle_inject.c b/drivers/powercap/idle_inject.c new file mode 100644 index 000000000000..24ff2a068978 --- /dev/null +++ b/drivers/powercap/idle_inject.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2018 Linaro Limited + * + * Author: Daniel Lezcano <daniel.lezcano@linaro.org> + * + * The idle injection framework provides a way to force CPUs to enter idle + * states for a specified fraction of time over a specified period. + * + * It relies on the smpboot kthreads feature providing common code for CPU + * hotplug and thread [un]parking. + * + * All of the kthreads used for idle injection are created at init time. + * + * Next, the users of the the idle injection framework provide a cpumask via + * its register function. The kthreads will be synchronized with respect to + * this cpumask. + * + * The idle + run duration is specified via separate helpers and that allows + * idle injection to be started. + * + * The idle injection kthreads will call play_idle() with the idle duration + * specified as per the above. + * + * After all of them have been woken up, a timer is set to start the next idle + * injection cycle. + * + * The timer interrupt handler will wake up the idle injection kthreads for + * all of the CPUs in the cpumask provided by the user. + * + * Idle injection is stopped synchronously and no leftover idle injection + * kthread activity after its completion is guaranteed. + * + * It is up to the user of this framework to provide a lock for higher-level + * synchronization to prevent race conditions like starting idle injection + * while unregistering from the framework. + */ +#define pr_fmt(fmt) "ii_dev: " fmt + +#include <linux/cpu.h> +#include <linux/hrtimer.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/smpboot.h> + +#include <uapi/linux/sched/types.h> + +/** + * struct idle_inject_thread - task on/off switch structure + * @tsk: task injecting the idle cycles + * @should_run: whether or not to run the task (for the smpboot kthread API) + */ +struct idle_inject_thread { + struct task_struct *tsk; + int should_run; +}; + +/** + * struct idle_inject_device - idle injection data + * @timer: idle injection period timer + * @idle_duration_ms: duration of CPU idle time to inject + * @run_duration_ms: duration of CPU run time to allow + * @cpumask: mask of CPUs affected by idle injection + */ +struct idle_inject_device { + struct hrtimer timer; + unsigned int idle_duration_ms; + unsigned int run_duration_ms; + unsigned long int cpumask[0]; +}; + +static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread); +static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device); + +/** + * idle_inject_wakeup - Wake up idle injection threads + * @ii_dev: target idle injection device + * + * Every idle injection task associated with the given idle injection device + * and running on an online CPU will be woken up. + */ +static void idle_inject_wakeup(struct idle_inject_device *ii_dev) +{ + struct idle_inject_thread *iit; + unsigned int cpu; + + for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) { + iit = per_cpu_ptr(&idle_inject_thread, cpu); + iit->should_run = 1; + wake_up_process(iit->tsk); + } +} + +/** + * idle_inject_timer_fn - idle injection timer function + * @timer: idle injection hrtimer + * + * This function is called when the idle injection timer expires. It wakes up + * idle injection tasks associated with the timer and they, in turn, invoke + * play_idle() to inject a specified amount of CPU idle time. + * + * Return: HRTIMER_RESTART. + */ +static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer) +{ + unsigned int duration_ms; + struct idle_inject_device *ii_dev = + container_of(timer, struct idle_inject_device, timer); + + duration_ms = READ_ONCE(ii_dev->run_duration_ms); + duration_ms += READ_ONCE(ii_dev->idle_duration_ms); + + idle_inject_wakeup(ii_dev); + + hrtimer_forward_now(timer, ms_to_ktime(duration_ms)); + + return HRTIMER_RESTART; +} + +/** + * idle_inject_fn - idle injection work function + * @cpu: the CPU owning the task + * + * This function calls play_idle() to inject a specified amount of CPU idle + * time. + */ +static void idle_inject_fn(unsigned int cpu) +{ + struct idle_inject_device *ii_dev; + struct idle_inject_thread *iit; + + ii_dev = per_cpu(idle_inject_device, cpu); + iit = per_cpu_ptr(&idle_inject_thread, cpu); + + /* + * Let the smpboot main loop know that the task should not run again. + */ + iit->should_run = 0; + + play_idle(READ_ONCE(ii_dev->idle_duration_ms)); +} + +/** + * idle_inject_set_duration - idle and run duration update helper + * @run_duration_ms: CPU run time to allow in milliseconds + * @idle_duration_ms: CPU idle time to inject in milliseconds + */ +void idle_inject_set_duration(struct idle_inject_device *ii_dev, + unsigned int run_duration_ms, + unsigned int idle_duration_ms) +{ + if (run_duration_ms && idle_duration_ms) { + WRITE_ONCE(ii_dev->run_duration_ms, run_duration_ms); + WRITE_ONCE(ii_dev->idle_duration_ms, idle_duration_ms); + } +} + +/** + * idle_inject_get_duration - idle and run duration retrieval helper + * @run_duration_ms: memory location to store the current CPU run time + * @idle_duration_ms: memory location to store the current CPU idle time + */ +void idle_inject_get_duration(struct idle_inject_device *ii_dev, + unsigned int *run_duration_ms, + unsigned int *idle_duration_ms) +{ + *run_duration_ms = READ_ONCE(ii_dev->run_duration_ms); + *idle_duration_ms = READ_ONCE(ii_dev->idle_duration_ms); +} + +/** + * idle_inject_start - start idle injections + * @ii_dev: idle injection control device structure + * + * The function starts idle injection by first waking up all of the idle + * injection kthreads associated with @ii_dev to let them inject CPU idle time + * sets up a timer to start the next idle injection period. + * + * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success. + */ +int idle_inject_start(struct idle_inject_device *ii_dev) +{ + unsigned int idle_duration_ms = READ_ONCE(ii_dev->idle_duration_ms); + unsigned int run_duration_ms = READ_ONCE(ii_dev->run_duration_ms); + + if (!idle_duration_ms || !run_duration_ms) + return -EINVAL; + + pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n", + cpumask_pr_args(to_cpumask(ii_dev->cpumask))); + + idle_inject_wakeup(ii_dev); + + hrtimer_start(&ii_dev->timer, + ms_to_ktime(idle_duration_ms + run_duration_ms), + HRTIMER_MODE_REL); + + return 0; +} + +/** + * idle_inject_stop - stops idle injections + * @ii_dev: idle injection control device structure + * + * The function stops idle injection and waits for the threads to finish work. + * If CPU idle time is being injected when this function runs, then it will + * wait until the end of the cycle. + * + * When it returns, there is no more idle injection kthread activity. The + * kthreads are scheduled out and the periodic timer is off. + */ +void idle_inject_stop(struct idle_inject_device *ii_dev) +{ + struct idle_inject_thread *iit; + unsigned int cpu; + + pr_debug("Stopping idle injection on CPUs '%*pbl'\n", + cpumask_pr_args(to_cpumask(ii_dev->cpumask))); + + hrtimer_cancel(&ii_dev->timer); + + /* + * Stopping idle injection requires all of the idle injection kthreads + * associated with the given cpumask to be parked and stay that way, so + * prevent CPUs from going online at this point. Any CPUs going online + * after the loop below will be covered by clearing the should_run flag + * that will cause the smpboot main loop to schedule them out. + */ + cpu_hotplug_disable(); + + /* + * Iterate over all (online + offline) CPUs here in case one of them + * goes offline with the should_run flag set so as to prevent its idle + * injection kthread from running when the CPU goes online again after + * the ii_dev has been freed. + */ + for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) { + iit = per_cpu_ptr(&idle_inject_thread, cpu); + iit->should_run = 0; + + wait_task_inactive(iit->tsk, 0); + } + + cpu_hotplug_enable(); +} + +/** + * idle_inject_setup - prepare the current task for idle injection + * @cpu: not used + * + * Called once, this function is in charge of setting the current task's + * scheduler parameters to make it an RT task. + */ +static void idle_inject_setup(unsigned int cpu) +{ + struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO / 2 }; + + sched_setscheduler(current, SCHED_FIFO, ¶m); +} + +/** + * idle_inject_should_run - function helper for the smpboot API + * @cpu: CPU the kthread is running on + * + * Return: whether or not the thread can run. + */ +static int idle_inject_should_run(unsigned int cpu) +{ + struct idle_inject_thread *iit = + per_cpu_ptr(&idle_inject_thread, cpu); + + return iit->should_run; +} + +/** + * idle_inject_register - initialize idle injection on a set of CPUs + * @cpumask: CPUs to be affected by idle injection + * + * This function creates an idle injection control device structure for the + * given set of CPUs and initializes the timer associated with it. It does not + * start any injection cycles. + * + * Return: NULL if memory allocation fails, idle injection control device + * pointer on success. + */ +struct idle_inject_device *idle_inject_register(struct cpumask *cpumask) +{ + struct idle_inject_device *ii_dev; + int cpu, cpu_rb; + + ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL); + if (!ii_dev) + return NULL; + + cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask); + hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + ii_dev->timer.function = idle_inject_timer_fn; + + for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) { + + if (per_cpu(idle_inject_device, cpu)) { + pr_err("cpu%d is already registered\n", cpu); + goto out_rollback; + } + + per_cpu(idle_inject_device, cpu) = ii_dev; + } + + return ii_dev; + +out_rollback: + for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) { + if (cpu == cpu_rb) + break; + per_cpu(idle_inject_device, cpu_rb) = NULL; + } + + kfree(ii_dev); + + return NULL; +} + +/** + * idle_inject_unregister - unregister idle injection control device + * @ii_dev: idle injection control device to unregister + * + * The function stops idle injection for the given control device, + * unregisters its kthreads and frees memory allocated when that device was + * created. + */ +void idle_inject_unregister(struct idle_inject_device *ii_dev) +{ + unsigned int cpu; + + idle_inject_stop(ii_dev); + + for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) + per_cpu(idle_inject_device, cpu) = NULL; + + kfree(ii_dev); +} + +static struct smp_hotplug_thread idle_inject_threads = { + .store = &idle_inject_thread.tsk, + .setup = idle_inject_setup, + .thread_fn = idle_inject_fn, + .thread_comm = "idle_inject/%u", + .thread_should_run = idle_inject_should_run, +}; + +static int __init idle_inject_init(void) +{ + return smpboot_register_percpu_thread(&idle_inject_threads); +} +early_initcall(idle_inject_init); diff --git a/include/linux/idle_inject.h b/include/linux/idle_inject.h new file mode 100644 index 000000000000..bdc0293fb6cb --- /dev/null +++ b/include/linux/idle_inject.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018 Linaro Ltd + * + * Author: Daniel Lezcano <daniel.lezcano@linaro.org> + * + */ +#ifndef __IDLE_INJECT_H__ +#define __IDLE_INJECT_H__ + +/* private idle injection device structure */ +struct idle_inject_device; + +struct idle_inject_device *idle_inject_register(struct cpumask *cpumask); + +void idle_inject_unregister(struct idle_inject_device *ii_dev); + +int idle_inject_start(struct idle_inject_device *ii_dev); + +void idle_inject_stop(struct idle_inject_device *ii_dev); + +void idle_inject_set_duration(struct idle_inject_device *ii_dev, + unsigned int run_duration_ms, + unsigned int idle_duration_ms); + +void idle_inject_get_duration(struct idle_inject_device *ii_dev, + unsigned int *run_duration_ms, + unsigned int *idle_duration_ms); +#endif /* __IDLE_INJECT_H__ */ |