cpuidle: lpm-levels: Track and predict next rescheduling ipi
Add changes to track and predict next rescheduling ipi based on past history. Add a module param to control enabling it. Change-Id: Ie495d8906288ee410708693ee15ed51643aefb44 Signed-off-by: Maulik Shah <mkshah@codeaurora.org>
This commit is contained in:
Maulik Shah
parent
763cf29b05
commit
97edfeda2e
@ -53,6 +53,8 @@
|
||||
#include <asm/mpu.h>
|
||||
#include <soc/qcom/minidump.h>
|
||||
|
||||
#include <soc/qcom/lpm_levels.h>
|
||||
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include <trace/events/ipi.h>
|
||||
|
||||
@ -725,6 +727,7 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
|
||||
|
||||
void smp_send_reschedule(int cpu)
|
||||
{
|
||||
update_ipi_history(cpu);
|
||||
smp_cross_call_common(cpumask_of(cpu), IPI_RESCHEDULE);
|
||||
}
|
||||
|
||||
|
||||
@ -63,6 +63,7 @@
|
||||
#include <soc/qcom/minidump.h>
|
||||
|
||||
#include <soc/qcom/scm.h>
|
||||
#include <soc/qcom/lpm_levels.h>
|
||||
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include <trace/events/ipi.h>
|
||||
@ -955,6 +956,7 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
|
||||
void smp_send_reschedule(int cpu)
|
||||
{
|
||||
BUG_ON(cpu_is_offline(cpu));
|
||||
update_ipi_history(cpu);
|
||||
smp_cross_call_common(cpumask_of(cpu), IPI_RESCHEDULE);
|
||||
}
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
|
||||
/*
|
||||
* Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2014-2019, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#define pr_fmt(fmt) "%s: " fmt, KBUILD_MODNAME
|
||||
@ -565,6 +565,9 @@ static int parse_cpu_levels(struct device_node *dn, struct lpm_cluster *c)
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
cpu->ipi_prediction = !(of_property_read_bool(dn,
|
||||
"qcom,disable-ipi-prediction"));
|
||||
|
||||
cpu->lpm_prediction = !(of_property_read_bool(dn,
|
||||
"qcom,disable-prediction"));
|
||||
|
||||
|
||||
@ -82,6 +82,9 @@ struct lpm_cluster *lpm_root_node;
|
||||
static bool lpm_prediction = true;
|
||||
module_param_named(lpm_prediction, lpm_prediction, bool, 0664);
|
||||
|
||||
static bool lpm_ipi_prediction;
|
||||
module_param_named(lpm_ipi_prediction, lpm_ipi_prediction, bool, 0664);
|
||||
|
||||
struct lpm_history {
|
||||
uint32_t resi[MAXSAMPLES];
|
||||
int mode[MAXSAMPLES];
|
||||
@ -92,8 +95,14 @@ struct lpm_history {
|
||||
int64_t stime;
|
||||
};
|
||||
|
||||
static DEFINE_PER_CPU(struct lpm_history, hist);
|
||||
struct ipi_history {
|
||||
uint32_t interval[MAXSAMPLES];
|
||||
uint32_t current_ptr;
|
||||
ktime_t cpu_idle_resched_ts;
|
||||
};
|
||||
|
||||
static DEFINE_PER_CPU(struct lpm_history, hist);
|
||||
static DEFINE_PER_CPU(struct ipi_history, cpu_ipi_history);
|
||||
static DEFINE_PER_CPU(struct lpm_cpu*, cpu_lpm);
|
||||
static bool suspend_in_progress;
|
||||
static struct hrtimer lpm_hrtimer;
|
||||
@ -453,14 +462,66 @@ static void biastimer_start(uint32_t time_ns)
|
||||
hrtimer_start(cpu_biastimer, bias_ktime, HRTIMER_MODE_REL_PINNED);
|
||||
}
|
||||
|
||||
static uint64_t lpm_cpuidle_predict(struct cpuidle_device *dev,
|
||||
struct lpm_cpu *cpu, int *idx_restrict,
|
||||
uint32_t *idx_restrict_time)
|
||||
static uint64_t find_deviation(int *interval, uint32_t ref_stddev,
|
||||
int64_t *stime)
|
||||
{
|
||||
int i, j, divisor;
|
||||
int divisor, i;
|
||||
uint64_t max, avg, stddev;
|
||||
int64_t thresh = LLONG_MAX;
|
||||
|
||||
if (lpm_ipi_prediction)
|
||||
ref_stddev += DEFAULT_IPI_STDDEV;
|
||||
|
||||
do {
|
||||
max = avg = divisor = stddev = 0;
|
||||
for (i = 0; i < MAXSAMPLES; i++) {
|
||||
int64_t value = interval[i];
|
||||
|
||||
if (value <= thresh) {
|
||||
avg += value;
|
||||
divisor++;
|
||||
if (value > max)
|
||||
max = value;
|
||||
}
|
||||
}
|
||||
do_div(avg, divisor);
|
||||
|
||||
for (i = 0; i < MAXSAMPLES; i++) {
|
||||
int64_t value = interval[i];
|
||||
|
||||
if (value <= thresh) {
|
||||
int64_t diff = value - avg;
|
||||
|
||||
stddev += diff * diff;
|
||||
}
|
||||
}
|
||||
do_div(stddev, divisor);
|
||||
stddev = int_sqrt(stddev);
|
||||
|
||||
/*
|
||||
* If the deviation is less, return the average, else
|
||||
* ignore one maximum sample and retry
|
||||
*/
|
||||
if (((avg > stddev * 6) && (divisor >= (MAXSAMPLES - 1)))
|
||||
|| stddev <= ref_stddev) {
|
||||
*stime = ktime_to_us(ktime_get()) + avg;
|
||||
return avg;
|
||||
}
|
||||
thresh = max - 1;
|
||||
|
||||
} while (divisor > (MAXSAMPLES - 1));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static uint64_t lpm_cpuidle_predict(struct cpuidle_device *dev,
|
||||
struct lpm_cpu *cpu, int *idx_restrict,
|
||||
uint32_t *idx_restrict_time, uint32_t *ipi_predicted)
|
||||
{
|
||||
int i, j;
|
||||
uint64_t avg;
|
||||
struct lpm_history *history = &per_cpu(hist, dev->cpu);
|
||||
struct ipi_history *ipi_history = &per_cpu(cpu_ipi_history, dev->cpu);
|
||||
|
||||
if (!lpm_prediction || !cpu->lpm_prediction)
|
||||
return 0;
|
||||
@ -491,44 +552,9 @@ static uint64_t lpm_cpuidle_predict(struct cpuidle_device *dev,
|
||||
* that mode.
|
||||
*/
|
||||
|
||||
again:
|
||||
max = avg = divisor = stddev = 0;
|
||||
for (i = 0; i < MAXSAMPLES; i++) {
|
||||
int64_t value = history->resi[i];
|
||||
|
||||
if (value <= thresh) {
|
||||
avg += value;
|
||||
divisor++;
|
||||
if (value > max)
|
||||
max = value;
|
||||
}
|
||||
}
|
||||
do_div(avg, divisor);
|
||||
|
||||
for (i = 0; i < MAXSAMPLES; i++) {
|
||||
int64_t value = history->resi[i];
|
||||
|
||||
if (value <= thresh) {
|
||||
int64_t diff = value - avg;
|
||||
|
||||
stddev += diff * diff;
|
||||
}
|
||||
}
|
||||
do_div(stddev, divisor);
|
||||
stddev = int_sqrt(stddev);
|
||||
|
||||
/*
|
||||
* If the deviation is less, return the average, else
|
||||
* ignore one maximum sample and retry
|
||||
*/
|
||||
if (((avg > stddev * 6) && (divisor >= (MAXSAMPLES - 1)))
|
||||
|| stddev <= cpu->ref_stddev) {
|
||||
history->stime = ktime_to_us(ktime_get()) + avg;
|
||||
avg = find_deviation(history->resi, cpu->ref_stddev, &(history->stime));
|
||||
if (avg)
|
||||
return avg;
|
||||
} else if (divisor > (MAXSAMPLES - 1)) {
|
||||
thresh = max - 1;
|
||||
goto again;
|
||||
}
|
||||
|
||||
/*
|
||||
* Find the number of premature exits for each of the mode,
|
||||
@ -571,6 +597,17 @@ again:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (*idx_restrict_time || !cpu->ipi_prediction || !lpm_ipi_prediction)
|
||||
return 0;
|
||||
|
||||
avg = find_deviation(ipi_history->interval, cpu->ref_stddev,
|
||||
&(history->stime));
|
||||
if (avg) {
|
||||
*ipi_predicted = 1;
|
||||
return avg;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -664,7 +701,7 @@ static int cpu_power_select(struct cpuidle_device *dev,
|
||||
int i, idx_restrict;
|
||||
uint32_t lvl_latency_us = 0;
|
||||
uint64_t predicted = 0;
|
||||
uint32_t htime = 0, idx_restrict_time = 0;
|
||||
uint32_t htime = 0, idx_restrict_time = 0, ipi_predicted = 0;
|
||||
uint32_t next_wakeup_us = (uint32_t)sleep_us;
|
||||
uint32_t min_residency, max_residency;
|
||||
struct power_params *pwr_params;
|
||||
@ -698,7 +735,8 @@ static int cpu_power_select(struct cpuidle_device *dev,
|
||||
*/
|
||||
if (next_wakeup_us > max_residency) {
|
||||
predicted = lpm_cpuidle_predict(dev, cpu,
|
||||
&idx_restrict, &idx_restrict_time);
|
||||
&idx_restrict, &idx_restrict_time,
|
||||
&ipi_predicted);
|
||||
if (predicted && (predicted < min_residency))
|
||||
predicted = min_residency;
|
||||
} else
|
||||
@ -735,7 +773,9 @@ static int cpu_power_select(struct cpuidle_device *dev,
|
||||
if ((predicted || (idx_restrict != cpu->nlevels + 1)) &&
|
||||
(best_level < (cpu->nlevels-1))) {
|
||||
htime = predicted + cpu->tmr_add;
|
||||
if (htime == cpu->tmr_add)
|
||||
if (lpm_ipi_prediction && cpu->ipi_prediction)
|
||||
htime += DEFAULT_IPI_TIMER_ADD;
|
||||
if (!predicted)
|
||||
htime = idx_restrict_time;
|
||||
else if (htime > max_residency)
|
||||
htime = max_residency;
|
||||
@ -748,8 +788,8 @@ static int cpu_power_select(struct cpuidle_device *dev,
|
||||
done_select:
|
||||
trace_cpu_power_select(best_level, sleep_us, latency_us, next_event_us);
|
||||
|
||||
trace_cpu_pred_select(idx_restrict_time ? 2 : (predicted ? 1 : 0),
|
||||
predicted, htime);
|
||||
trace_cpu_pred_select(idx_restrict_time ? 2 : (ipi_predicted ?
|
||||
3 : (predicted ? 1 : 0)), predicted, htime);
|
||||
|
||||
return best_level;
|
||||
}
|
||||
@ -1375,6 +1415,20 @@ static int lpm_cpuidle_select(struct cpuidle_driver *drv,
|
||||
return cpu_power_select(dev, cpu);
|
||||
}
|
||||
|
||||
void update_ipi_history(int cpu)
|
||||
{
|
||||
struct ipi_history *history = &per_cpu(cpu_ipi_history, cpu);
|
||||
ktime_t now = ktime_get();
|
||||
|
||||
history->interval[history->current_ptr] =
|
||||
ktime_to_us(ktime_sub(now,
|
||||
history->cpu_idle_resched_ts));
|
||||
(history->current_ptr)++;
|
||||
if (history->current_ptr >= MAXSAMPLES)
|
||||
history->current_ptr = 0;
|
||||
history->cpu_idle_resched_ts = now;
|
||||
}
|
||||
|
||||
static void update_history(struct cpuidle_device *dev, int idx)
|
||||
{
|
||||
struct lpm_history *history = &per_cpu(hist, dev->cpu);
|
||||
|
||||
@ -11,7 +11,9 @@
|
||||
#define CLUST_SMPL_INVLD_TIME 40000
|
||||
#define DEFAULT_PREMATURE_CNT 3
|
||||
#define DEFAULT_STDDEV 100
|
||||
#define DEFAULT_IPI_STDDEV 400
|
||||
#define DEFAULT_TIMER_ADD 100
|
||||
#define DEFAULT_IPI_TIMER_ADD 900
|
||||
#define TIMER_ADD_LOW 100
|
||||
#define TIMER_ADD_HIGH 1500
|
||||
#define STDDEV_LOW 100
|
||||
@ -46,6 +48,7 @@ struct lpm_cpu {
|
||||
uint32_t ref_premature_cnt;
|
||||
uint32_t tmr_add;
|
||||
bool lpm_prediction;
|
||||
bool ipi_prediction;
|
||||
uint64_t bias;
|
||||
struct cpuidle_driver *drv;
|
||||
struct lpm_cluster *parent;
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0-only */
|
||||
|
||||
/*
|
||||
* Copyright (c) 2018, The Linux Foundation. All rights reserved.
|
||||
* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef __SOC_QCOM_LPM_LEVEL_H__
|
||||
@ -16,9 +16,11 @@ struct system_pm_ops {
|
||||
|
||||
#ifdef CONFIG_MSM_PM
|
||||
uint32_t register_system_pm_ops(struct system_pm_ops *pm_ops);
|
||||
void update_ipi_history(int cpu);
|
||||
#else
|
||||
static inline uint32_t register_system_pm_ops(struct system_pm_ops *pm_ops)
|
||||
{ return -ENODEV; }
|
||||
static inline void update_ipi_history(int cpu) {}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
Loading…
Reference in New Issue
Block a user