From mboxrd@z Thu Jan 1 00:00:00 1970 From: =?UTF-8?q?B=C3=A1lint=20Czobor?= Subject: [PATCH 55/70] cpufreq: Interactive: Implement per policy instances of governor Date: Tue, 27 Oct 2015 18:30:43 +0100 Message-ID: <1445967059-6897-55-git-send-email-czoborbalint@gmail.com> References: <1445967059-6897-1-git-send-email-czoborbalint@gmail.com> Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: QUOTED-PRINTABLE Return-path: In-Reply-To: <1445967059-6897-1-git-send-email-czoborbalint@gmail.com> Sender: linux-kernel-owner@vger.kernel.org To: "Rafael J. Wysocki" , Viresh Kumar Cc: linux-kernel@vger.kernel.org, linux-pm@vger.kernel.org, =?UTF-8?q?B=C3=A1lint=20Czobor?= List-Id: linux-pm@vger.kernel.org =46rom: Viresh Kumar If we have a multi-package system, where we have multiple instances of = struct policy (per package), currently we can't have multiple instances of sam= e governor. i.e. We can't have multiple instances of Interactive governor= for multiple packages. This is a bottleneck for multicluster system, where we want different p= ackages to use Interactive governor, but with different tunables. This patch uses the infrastructure provided by earlier patches pushed i= n Mainline in v3.10-rc1/rc2 and implements per policy instances of Intera= ctive governor. Change-Id: I70436d4a5a45c6cb6edf37f3e46d0b9fbc930982 [toddpoynor@google.com: merge with later code, minor changes] Signed-off-by: Viresh Kumar Signed-off-by: B=C3=A1lint Czobor --- drivers/cpufreq/cpufreq_interactive.c | 629 ++++++++++++++++++++-----= -------- 1 file changed, 383 insertions(+), 246 deletions(-) diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cp= ufreq_interactive.c index b0685c7..66c096d 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -36,8 +36,6 @@ #define CREATE_TRACE_POINTS #include =20 -static int active_count; - struct cpufreq_interactive_cpuinfo { struct timer_list cpu_timer; struct timer_list cpu_slack_timer; @@ -64,58 +62,62 @@ static cpumask_t speedchange_cpumask; static spinlock_t speedchange_cpumask_lock; static struct mutex gov_lock; =20 -/* Hi speed to bump to from lo speed when load burst (default max) */ -static unsigned int hispeed_freq; - -/* Go to hi speed when CPU load at or above this value. */ -#define DEFAULT_GO_HISPEED_LOAD 99 -static unsigned long go_hispeed_load =3D DEFAULT_GO_HISPEED_LOAD; - /* Target load. Lower values result in higher CPU speeds. */ #define DEFAULT_TARGET_LOAD 90 static unsigned int default_target_loads[] =3D {DEFAULT_TARGET_LOAD}; -static spinlock_t target_loads_lock; -static unsigned int *target_loads =3D default_target_loads; -static int ntarget_loads =3D ARRAY_SIZE(default_target_loads); - -/* - * The minimum amount of time to spend at a frequency before we can ra= mp down. - */ -#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC) -static unsigned long min_sample_time =3D DEFAULT_MIN_SAMPLE_TIME; =20 -/* - * The sample rate of the timer used to increase frequency - */ #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC) -static unsigned long timer_rate =3D DEFAULT_TIMER_RATE; - -/* - * Wait this long before raising speed above hispeed, by default a sin= gle - * timer interval. - */ #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE static unsigned int default_above_hispeed_delay[] =3D { DEFAULT_ABOVE_HISPEED_DELAY }; -static spinlock_t above_hispeed_delay_lock; -static unsigned int *above_hispeed_delay =3D default_above_hispeed_del= ay; -static int nabove_hispeed_delay =3D ARRAY_SIZE(default_above_hispeed_d= elay); - -/* Non-zero means indefinite speed boost active */ -static int boost_val; -/* Duration of a boot pulse in usecs */ -static int boostpulse_duration_val =3D DEFAULT_MIN_SAMPLE_TIME; -/* End time of boost pulse in ktime converted to usecs */ -static u64 boostpulse_endtime; =20 -/* - * Max additional time to wait in idle, beyond timer_rate, at speeds a= bove - * minimum before wakeup to reduce speed, or -1 if unnecessary. - */ +struct cpufreq_interactive_tunables { + int usage_count; + /* Hi speed to bump to from lo speed when load burst (default max) */ + unsigned int hispeed_freq; + /* Go to hi speed when CPU load at or above this value. */ +#define DEFAULT_GO_HISPEED_LOAD 99 + unsigned long go_hispeed_load; + /* Target load. Lower values result in higher CPU speeds. */ + spinlock_t target_loads_lock; + unsigned int *target_loads; + int ntarget_loads; + /* + * The minimum amount of time to spend at a frequency before we can r= amp + * down. + */ +#define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC) + unsigned long min_sample_time; + /* + * The sample rate of the timer used to increase frequency + */ + unsigned long timer_rate; + /* + * Wait this long before raising speed above hispeed, by default a + * single timer interval. + */ + spinlock_t above_hispeed_delay_lock; + unsigned int *above_hispeed_delay; + int nabove_hispeed_delay; + /* Non-zero means indefinite speed boost active */ + int boost_val; + /* Duration of a boot pulse in usecs */ + int boostpulse_duration_val; + /* End time of boost pulse in ktime converted to usecs */ + u64 boostpulse_endtime; + /* + * Max additional time to wait in idle, beyond timer_rate, at speeds + * above minimum before wakeup to reduce speed, or -1 if unnecessary. + */ #define DEFAULT_TIMER_SLACK (4 * DEFAULT_TIMER_RATE) -static int timer_slack_val =3D DEFAULT_TIMER_SLACK; + int timer_slack_val; + bool io_is_busy; +}; + +/* For cases where we have single governor instance for system */ +struct cpufreq_interactive_tunables *common_tunables; =20 -static bool io_is_busy; +static struct attribute_group *get_sysfs_attr(void); =20 static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu, cputime64_t *wall) @@ -140,8 +142,10 @@ static inline cputime64_t get_cpu_idle_time_jiffy(= unsigned int cpu, return jiffies_to_usecs(idle_time); } =20 -static inline cputime64_t get_cpu_idle_time(unsigned int cpu, - cputime64_t *wall) +static inline cputime64_t get_cpu_idle_time( + unsigned int cpu, + cputime64_t *wall, + bool io_is_busy) { u64 idle_time =3D get_cpu_idle_time_us(cpu, wall); =20 @@ -156,20 +160,24 @@ static inline cputime64_t get_cpu_idle_time(unsig= ned int cpu, static void cpufreq_interactive_timer_resched( struct cpufreq_interactive_cpuinfo *pcpu) { + struct cpufreq_interactive_tunables *tunables =3D + pcpu->policy->governor_data; unsigned long expires; unsigned long flags; =20 spin_lock_irqsave(&pcpu->load_lock, flags); pcpu->time_in_idle =3D get_cpu_idle_time(smp_processor_id(), - &pcpu->time_in_idle_timestamp); + &pcpu->time_in_idle_timestamp, + tunables->io_is_busy); pcpu->cputime_speedadj =3D 0; pcpu->cputime_speedadj_timestamp =3D pcpu->time_in_idle_timestamp; - expires =3D jiffies + usecs_to_jiffies(timer_rate); + expires =3D jiffies + usecs_to_jiffies(tunables->timer_rate); mod_timer_pinned(&pcpu->cpu_timer, expires); =20 - if (timer_slack_val >=3D 0 && pcpu->target_freq > pcpu->policy->min) = { - expires +=3D usecs_to_jiffies(timer_slack_val); + if (tunables->timer_slack_val >=3D 0 && + pcpu->target_freq > pcpu->policy->min) { + expires +=3D usecs_to_jiffies(tunables->timer_slack_val); mod_timer_pinned(&pcpu->cpu_slack_timer, expires); } =20 @@ -180,58 +188,66 @@ static void cpufreq_interactive_timer_resched( * The cpu_timer and cpu_slack_timer must be deactivated when calling = this * function. */ -static void cpufreq_interactive_timer_start(int cpu) +static void cpufreq_interactive_timer_start( + struct cpufreq_interactive_tunables *tunables, int cpu) { struct cpufreq_interactive_cpuinfo *pcpu =3D &per_cpu(cpuinfo, cpu); - unsigned long expires =3D jiffies + usecs_to_jiffies(timer_rate); + unsigned long expires =3D jiffies + + usecs_to_jiffies(tunables->timer_rate); unsigned long flags; =20 pcpu->cpu_timer.expires =3D expires; add_timer_on(&pcpu->cpu_timer, cpu); - if (timer_slack_val >=3D 0 && pcpu->target_freq > pcpu->policy->min) = { - expires +=3D usecs_to_jiffies(timer_slack_val); + if (tunables->timer_slack_val >=3D 0 && + pcpu->target_freq > pcpu->policy->min) { + expires +=3D usecs_to_jiffies(tunables->timer_slack_val); pcpu->cpu_slack_timer.expires =3D expires; add_timer_on(&pcpu->cpu_slack_timer, cpu); } =20 spin_lock_irqsave(&pcpu->load_lock, flags); pcpu->time_in_idle =3D - get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp); + get_cpu_idle_time(cpu, &pcpu->time_in_idle_timestamp, + tunables->io_is_busy); pcpu->cputime_speedadj =3D 0; pcpu->cputime_speedadj_timestamp =3D pcpu->time_in_idle_timestamp; spin_unlock_irqrestore(&pcpu->load_lock, flags); } =20 -static unsigned int freq_to_above_hispeed_delay(unsigned int freq) +static unsigned int freq_to_above_hispeed_delay( + struct cpufreq_interactive_tunables *tunables, + unsigned int freq) { int i; unsigned int ret; unsigned long flags; =20 - spin_lock_irqsave(&above_hispeed_delay_lock, flags); + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); =20 - for (i =3D 0; i < nabove_hispeed_delay - 1 && - freq >=3D above_hispeed_delay[i+1]; i +=3D 2) + for (i =3D 0; i < tunables->nabove_hispeed_delay - 1 && + freq >=3D tunables->above_hispeed_delay[i+1]; i +=3D 2) ; =20 - ret =3D above_hispeed_delay[i]; - spin_unlock_irqrestore(&above_hispeed_delay_lock, flags); + ret =3D tunables->above_hispeed_delay[i]; + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); return ret; } =20 -static unsigned int freq_to_targetload(unsigned int freq) +static unsigned int freq_to_targetload( + struct cpufreq_interactive_tunables *tunables, unsigned int freq) { int i; unsigned int ret; unsigned long flags; =20 - spin_lock_irqsave(&target_loads_lock, flags); + spin_lock_irqsave(&tunables->target_loads_lock, flags); =20 - for (i =3D 0; i < ntarget_loads - 1 && freq >=3D target_loads[i+1]; i= +=3D 2) + for (i =3D 0; i < tunables->ntarget_loads - 1 && + freq >=3D tunables->target_loads[i+1]; i +=3D 2) ; =20 - ret =3D target_loads[i]; - spin_unlock_irqrestore(&target_loads_lock, flags); + ret =3D tunables->target_loads[i]; + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); return ret; } =20 @@ -240,9 +256,8 @@ static unsigned int freq_to_targetload(unsigned int= freq) * choose_freq() will find the minimum frequency that does not exceed = its * target load given the current load. */ - -static unsigned int choose_freq( - struct cpufreq_interactive_cpuinfo *pcpu, unsigned int loadadjfreq) +static unsigned int choose_freq(struct cpufreq_interactive_cpuinfo *pc= pu, + unsigned int loadadjfreq) { unsigned int freq =3D pcpu->policy->cur; unsigned int prevfreq, freqmin, freqmax; @@ -254,7 +269,7 @@ static unsigned int choose_freq( =20 do { prevfreq =3D freq; - tl =3D freq_to_targetload(freq); + tl =3D freq_to_targetload(pcpu->policy->governor_data, freq); =20 /* * Find the lowest frequency where the computed load is less @@ -329,13 +344,15 @@ static unsigned int choose_freq( static u64 update_load(int cpu) { struct cpufreq_interactive_cpuinfo *pcpu =3D &per_cpu(cpuinfo, cpu); + struct cpufreq_interactive_tunables *tunables =3D + pcpu->policy->governor_data; u64 now; u64 now_idle; unsigned int delta_idle; unsigned int delta_time; u64 active_time; =20 - now_idle =3D get_cpu_idle_time(cpu, &now); + now_idle =3D get_cpu_idle_time(cpu, &now, tunables->io_is_busy); delta_idle =3D (unsigned int)(now_idle - pcpu->time_in_idle); delta_time =3D (unsigned int)(now - pcpu->time_in_idle_timestamp); =20 @@ -359,6 +376,8 @@ static void cpufreq_interactive_timer(unsigned long= data) int cpu_load; struct cpufreq_interactive_cpuinfo *pcpu =3D &per_cpu(cpuinfo, data); + struct cpufreq_interactive_tunables *tunables =3D + pcpu->policy->governor_data; unsigned int new_freq; unsigned int loadadjfreq; unsigned int index; @@ -382,25 +401,25 @@ static void cpufreq_interactive_timer(unsigned lo= ng data) do_div(cputime_speedadj, delta_time); loadadjfreq =3D (unsigned int)cputime_speedadj * 100; cpu_load =3D loadadjfreq / pcpu->target_freq; - boosted =3D boost_val || now < boostpulse_endtime; + boosted =3D tunables->boost_val || now < tunables->boostpulse_endtime= ; =20 - if (cpu_load >=3D go_hispeed_load || boosted) { - if (pcpu->target_freq < hispeed_freq) { - new_freq =3D hispeed_freq; + if (cpu_load >=3D tunables->go_hispeed_load || boosted) { + if (pcpu->target_freq < tunables->hispeed_freq) { + new_freq =3D tunables->hispeed_freq; } else { new_freq =3D choose_freq(pcpu, loadadjfreq); =20 - if (new_freq < hispeed_freq) - new_freq =3D hispeed_freq; + if (new_freq < tunables->hispeed_freq) + new_freq =3D tunables->hispeed_freq; } } else { new_freq =3D choose_freq(pcpu, loadadjfreq); } =20 - if (pcpu->target_freq >=3D hispeed_freq && + if (pcpu->target_freq >=3D tunables->hispeed_freq && new_freq > pcpu->target_freq && now - pcpu->hispeed_validate_time < - freq_to_above_hispeed_delay(pcpu->target_freq)) { + freq_to_above_hispeed_delay(tunables, pcpu->target_freq)) { trace_cpufreq_interactive_notyet( data, cpu_load, pcpu->target_freq, pcpu->policy->cur, new_freq); @@ -421,7 +440,8 @@ static void cpufreq_interactive_timer(unsigned long= data) * floor frequency for the minimum sample time since last validated. */ if (new_freq < pcpu->floor_freq) { - if (now - pcpu->floor_validate_time < min_sample_time) { + if (now - pcpu->floor_validate_time < + tunables->min_sample_time) { trace_cpufreq_interactive_notyet( data, cpu_load, pcpu->target_freq, pcpu->policy->cur, new_freq); @@ -437,7 +457,7 @@ static void cpufreq_interactive_timer(unsigned long= data) * (or the indefinite boost is turned off). */ =20 - if (!boosted || new_freq > hispeed_freq) { + if (!boosted || new_freq > tunables->hispeed_freq) { pcpu->floor_freq =3D new_freq; pcpu->floor_validate_time =3D now; } @@ -598,14 +618,16 @@ static void cpufreq_interactive_boost(void) int anyboost =3D 0; unsigned long flags; struct cpufreq_interactive_cpuinfo *pcpu; + struct cpufreq_interactive_tunables *tunables; =20 spin_lock_irqsave(&speedchange_cpumask_lock, flags); =20 for_each_online_cpu(i) { pcpu =3D &per_cpu(cpuinfo, i); + tunables =3D pcpu->policy->governor_data; =20 - if (pcpu->target_freq < hispeed_freq) { - pcpu->target_freq =3D hispeed_freq; + if (pcpu->target_freq < tunables->hispeed_freq) { + pcpu->target_freq =3D tunables->hispeed_freq; cpumask_set_cpu(i, &speedchange_cpumask); pcpu->hispeed_validate_time =3D ktime_to_us(ktime_get()); @@ -617,7 +639,7 @@ static void cpufreq_interactive_boost(void) * validated. */ =20 - pcpu->floor_freq =3D hispeed_freq; + pcpu->floor_freq =3D tunables->hispeed_freq; pcpu->floor_validate_time =3D ktime_to_us(ktime_get()); } =20 @@ -717,26 +739,27 @@ err: } =20 static ssize_t show_target_loads( - struct kobject *kobj, struct attribute *attr, char *buf) + struct cpufreq_interactive_tunables *tunables, + char *buf) { int i; ssize_t ret =3D 0; unsigned long flags; =20 - spin_lock_irqsave(&target_loads_lock, flags); + spin_lock_irqsave(&tunables->target_loads_lock, flags); =20 - for (i =3D 0; i < ntarget_loads; i++) - ret +=3D sprintf(buf + ret, "%u%s", target_loads[i], + for (i =3D 0; i < tunables->ntarget_loads; i++) + ret +=3D sprintf(buf + ret, "%u%s", tunables->target_loads[i], i & 0x1 ? ":" : " "); =20 ret +=3D sprintf(buf + --ret, "\n"); - spin_unlock_irqrestore(&target_loads_lock, flags); + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); return ret; } =20 static ssize_t store_target_loads( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) + struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ntokens; unsigned int *new_target_loads =3D NULL; @@ -746,40 +769,37 @@ static ssize_t store_target_loads( if (IS_ERR(new_target_loads)) return PTR_RET(new_target_loads); =20 - spin_lock_irqsave(&target_loads_lock, flags); - if (target_loads !=3D default_target_loads) - kfree(target_loads); - target_loads =3D new_target_loads; - ntarget_loads =3D ntokens; - spin_unlock_irqrestore(&target_loads_lock, flags); + spin_lock_irqsave(&tunables->target_loads_lock, flags); + if (tunables->target_loads !=3D default_target_loads) + kfree(tunables->target_loads); + tunables->target_loads =3D new_target_loads; + tunables->ntarget_loads =3D ntokens; + spin_unlock_irqrestore(&tunables->target_loads_lock, flags); return count; } =20 -static struct global_attr target_loads_attr =3D - __ATTR(target_loads, S_IRUGO | S_IWUSR, - show_target_loads, store_target_loads); - static ssize_t show_above_hispeed_delay( - struct kobject *kobj, struct attribute *attr, char *buf) + struct cpufreq_interactive_tunables *tunables, char *buf) { int i; ssize_t ret =3D 0; unsigned long flags; =20 - spin_lock_irqsave(&above_hispeed_delay_lock, flags); + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); =20 - for (i =3D 0; i < nabove_hispeed_delay; i++) - ret +=3D sprintf(buf + ret, "%u%s", above_hispeed_delay[i], + for (i =3D 0; i < tunables->nabove_hispeed_delay; i++) + ret +=3D sprintf(buf + ret, "%u%s", + tunables->above_hispeed_delay[i], i & 0x1 ? ":" : " "); =20 ret +=3D sprintf(buf + --ret, "\n"); - spin_unlock_irqrestore(&above_hispeed_delay_lock, flags); + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); return ret; } =20 static ssize_t store_above_hispeed_delay( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) + struct cpufreq_interactive_tunables *tunables, + const char *buf, size_t count) { int ntokens; unsigned int *new_above_hispeed_delay =3D NULL; @@ -789,29 +809,24 @@ static ssize_t store_above_hispeed_delay( if (IS_ERR(new_above_hispeed_delay)) return PTR_RET(new_above_hispeed_delay); =20 - spin_lock_irqsave(&above_hispeed_delay_lock, flags); - if (above_hispeed_delay !=3D default_above_hispeed_delay) - kfree(above_hispeed_delay); - above_hispeed_delay =3D new_above_hispeed_delay; - nabove_hispeed_delay =3D ntokens; - spin_unlock_irqrestore(&above_hispeed_delay_lock, flags); + spin_lock_irqsave(&tunables->above_hispeed_delay_lock, flags); + if (tunables->above_hispeed_delay !=3D default_above_hispeed_delay) + kfree(tunables->above_hispeed_delay); + tunables->above_hispeed_delay =3D new_above_hispeed_delay; + tunables->nabove_hispeed_delay =3D ntokens; + spin_unlock_irqrestore(&tunables->above_hispeed_delay_lock, flags); return count; =20 } =20 -static struct global_attr above_hispeed_delay_attr =3D - __ATTR(above_hispeed_delay, S_IRUGO | S_IWUSR, - show_above_hispeed_delay, store_above_hispeed_delay); - -static ssize_t show_hispeed_freq(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_hispeed_freq(struct cpufreq_interactive_tunables *= tunables, + char *buf) { - return sprintf(buf, "%u\n", hispeed_freq); + return sprintf(buf, "%u\n", tunables->hispeed_freq); } =20 -static ssize_t store_hispeed_freq(struct kobject *kobj, - struct attribute *attr, const char *buf, - size_t count) +static ssize_t store_hispeed_freq(struct cpufreq_interactive_tunables = *tunables, + const char *buf, size_t count) { int ret; long unsigned int val; @@ -819,22 +834,18 @@ static ssize_t store_hispeed_freq(struct kobject = *kobj, ret =3D strict_strtoul(buf, 0, &val); if (ret < 0) return ret; - hispeed_freq =3D val; + tunables->hispeed_freq =3D val; return count; } =20 -static struct global_attr hispeed_freq_attr =3D __ATTR(hispeed_freq, 0= 644, - show_hispeed_freq, store_hispeed_freq); - - -static ssize_t show_go_hispeed_load(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_go_hispeed_load(struct cpufreq_interactive_tunable= s + *tunables, char *buf) { - return sprintf(buf, "%lu\n", go_hispeed_load); + return sprintf(buf, "%lu\n", tunables->go_hispeed_load); } =20 -static ssize_t store_go_hispeed_load(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_go_hispeed_load(struct cpufreq_interactive_tunabl= es + *tunables, const char *buf, size_t count) { int ret; unsigned long val; @@ -842,21 +853,18 @@ static ssize_t store_go_hispeed_load(struct kobje= ct *kobj, ret =3D strict_strtoul(buf, 0, &val); if (ret < 0) return ret; - go_hispeed_load =3D val; + tunables->go_hispeed_load =3D val; return count; } =20 -static struct global_attr go_hispeed_load_attr =3D __ATTR(go_hispeed_l= oad, 0644, - show_go_hispeed_load, store_go_hispeed_load); - -static ssize_t show_min_sample_time(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_min_sample_time(struct cpufreq_interactive_tunable= s + *tunables, char *buf) { - return sprintf(buf, "%lu\n", min_sample_time); + return sprintf(buf, "%lu\n", tunables->min_sample_time); } =20 -static ssize_t store_min_sample_time(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_min_sample_time(struct cpufreq_interactive_tunabl= es + *tunables, const char *buf, size_t count) { int ret; unsigned long val; @@ -864,21 +872,18 @@ static ssize_t store_min_sample_time(struct kobje= ct *kobj, ret =3D strict_strtoul(buf, 0, &val); if (ret < 0) return ret; - min_sample_time =3D val; + tunables->min_sample_time =3D val; return count; } =20 -static struct global_attr min_sample_time_attr =3D __ATTR(min_sample_t= ime, 0644, - show_min_sample_time, store_min_sample_time); - -static ssize_t show_timer_rate(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_timer_rate(struct cpufreq_interactive_tunables *tu= nables, + char *buf) { - return sprintf(buf, "%lu\n", timer_rate); + return sprintf(buf, "%lu\n", tunables->timer_rate); } =20 -static ssize_t store_timer_rate(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_timer_rate(struct cpufreq_interactive_tunables *t= unables, + const char *buf, size_t count) { int ret; unsigned long val; @@ -886,22 +891,18 @@ static ssize_t store_timer_rate(struct kobject *k= obj, ret =3D strict_strtoul(buf, 0, &val); if (ret < 0) return ret; - timer_rate =3D val; + tunables->timer_rate =3D val; return count; } =20 -static struct global_attr timer_rate_attr =3D __ATTR(timer_rate, 0644, - show_timer_rate, store_timer_rate); - -static ssize_t show_timer_slack( - struct kobject *kobj, struct attribute *attr, char *buf) +static ssize_t show_timer_slack(struct cpufreq_interactive_tunables *t= unables, + char *buf) { - return sprintf(buf, "%d\n", timer_slack_val); + return sprintf(buf, "%d\n", tunables->timer_slack_val); } =20 -static ssize_t store_timer_slack( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) +static ssize_t store_timer_slack(struct cpufreq_interactive_tunables *= tunables, + const char *buf, size_t count) { int ret; unsigned long val; @@ -910,19 +911,17 @@ static ssize_t store_timer_slack( if (ret < 0) return ret; =20 - timer_slack_val =3D val; + tunables->timer_slack_val =3D val; return count; } =20 -define_one_global_rw(timer_slack); - -static ssize_t show_boost(struct kobject *kobj, struct attribute *attr= , +static ssize_t show_boost(struct cpufreq_interactive_tunables *tunable= s, char *buf) { - return sprintf(buf, "%d\n", boost_val); + return sprintf(buf, "%d\n", tunables->boost_val); } =20 -static ssize_t store_boost(struct kobject *kobj, struct attribute *att= r, +static ssize_t store_boost(struct cpufreq_interactive_tunables *tunabl= es, const char *buf, size_t count) { int ret; @@ -932,9 +931,9 @@ static ssize_t store_boost(struct kobject *kobj, st= ruct attribute *attr, if (ret < 0) return ret; =20 - boost_val =3D val; + tunables->boost_val =3D val; =20 - if (boost_val) { + if (tunables->boost_val) { trace_cpufreq_interactive_boost("on"); cpufreq_interactive_boost(); } else { @@ -944,9 +943,7 @@ static ssize_t store_boost(struct kobject *kobj, st= ruct attribute *attr, return count; } =20 -define_one_global_rw(boost); - -static ssize_t store_boostpulse(struct kobject *kobj, struct attribute= *attr, +static ssize_t store_boostpulse(struct cpufreq_interactive_tunables *t= unables, const char *buf, size_t count) { int ret; @@ -956,24 +953,21 @@ static ssize_t store_boostpulse(struct kobject *k= obj, struct attribute *attr, if (ret < 0) return ret; =20 - boostpulse_endtime =3D ktime_to_us(ktime_get()) + boostpulse_duration= _val; + tunables->boostpulse_endtime =3D ktime_to_us(ktime_get()) + + tunables->boostpulse_duration_val; trace_cpufreq_interactive_boost("pulse"); cpufreq_interactive_boost(); return count; } =20 -static struct global_attr boostpulse =3D - __ATTR(boostpulse, 0200, NULL, store_boostpulse); - -static ssize_t show_boostpulse_duration( - struct kobject *kobj, struct attribute *attr, char *buf) +static ssize_t show_boostpulse_duration(struct cpufreq_interactive_tun= ables + *tunables, char *buf) { - return sprintf(buf, "%d\n", boostpulse_duration_val); + return sprintf(buf, "%d\n", tunables->boostpulse_duration_val); } =20 -static ssize_t store_boostpulse_duration( - struct kobject *kobj, struct attribute *attr, const char *buf, - size_t count) +static ssize_t store_boostpulse_duration(struct cpufreq_interactive_tu= nables + *tunables, const char *buf, size_t count) { int ret; unsigned long val; @@ -982,20 +976,18 @@ static ssize_t store_boostpulse_duration( if (ret < 0) return ret; =20 - boostpulse_duration_val =3D val; + tunables->boostpulse_duration_val =3D val; return count; } =20 -define_one_global_rw(boostpulse_duration); - -static ssize_t show_io_is_busy(struct kobject *kobj, - struct attribute *attr, char *buf) +static ssize_t show_io_is_busy(struct cpufreq_interactive_tunables *tu= nables, + char *buf) { - return sprintf(buf, "%u\n", io_is_busy); + return sprintf(buf, "%u\n", tunables->io_is_busy); } =20 -static ssize_t store_io_is_busy(struct kobject *kobj, - struct attribute *attr, const char *buf, size_t count) +static ssize_t store_io_is_busy(struct cpufreq_interactive_tunables *t= unables, + const char *buf, size_t count) { int ret; unsigned long val; @@ -1003,33 +995,137 @@ static ssize_t store_io_is_busy(struct kobject = *kobj, ret =3D kstrtoul(buf, 0, &val); if (ret < 0) return ret; - io_is_busy =3D val; + tunables->io_is_busy =3D val; return count; } =20 -static struct global_attr io_is_busy_attr =3D __ATTR(io_is_busy, 0644, - show_io_is_busy, store_io_is_busy); - -static struct attribute *interactive_attributes[] =3D { - &target_loads_attr.attr, - &above_hispeed_delay_attr.attr, - &hispeed_freq_attr.attr, - &go_hispeed_load_attr.attr, - &min_sample_time_attr.attr, - &timer_rate_attr.attr, - &timer_slack.attr, - &boost.attr, - &boostpulse.attr, - &boostpulse_duration.attr, - &io_is_busy_attr.attr, +/* + * Create show/store routines + * - sys: One governor instance for complete SYSTEM + * - pol: One governor instance per struct cpufreq_policy + */ +#define show_gov_pol_sys(file_name) \ +static ssize_t show_##file_name##_gov_sys \ +(struct kobject *kobj, struct attribute *attr, char *buf) \ +{ \ + return show_##file_name(common_tunables, buf); \ +} \ + \ +static ssize_t show_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, char *buf) \ +{ \ + return show_##file_name(policy->governor_data, buf); \ +} + +#define store_gov_pol_sys(file_name) \ +static ssize_t store_##file_name##_gov_sys \ +(struct kobject *kobj, struct attribute *attr, const char *buf, \ + size_t count) \ +{ \ + return store_##file_name(common_tunables, buf, count); \ +} \ + \ +static ssize_t store_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, const char *buf, size_t count) \ +{ \ + return store_##file_name(policy->governor_data, buf, count); \ +} + +#define show_store_gov_pol_sys(file_name) \ +show_gov_pol_sys(file_name); \ +store_gov_pol_sys(file_name) + +show_store_gov_pol_sys(target_loads); +show_store_gov_pol_sys(above_hispeed_delay); +show_store_gov_pol_sys(hispeed_freq); +show_store_gov_pol_sys(go_hispeed_load); +show_store_gov_pol_sys(min_sample_time); +show_store_gov_pol_sys(timer_rate); +show_store_gov_pol_sys(timer_slack); +show_store_gov_pol_sys(boost); +store_gov_pol_sys(boostpulse); +show_store_gov_pol_sys(boostpulse_duration); +show_store_gov_pol_sys(io_is_busy); + +#define gov_sys_attr_rw(_name) \ +static struct global_attr _name##_gov_sys =3D \ +__ATTR(_name, 0644, show_##_name##_gov_sys, store_##_name##_gov_sys) + +#define gov_pol_attr_rw(_name) \ +static struct freq_attr _name##_gov_pol =3D \ +__ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol) + +#define gov_sys_pol_attr_rw(_name) \ + gov_sys_attr_rw(_name); \ + gov_pol_attr_rw(_name) + +gov_sys_pol_attr_rw(target_loads); +gov_sys_pol_attr_rw(above_hispeed_delay); +gov_sys_pol_attr_rw(hispeed_freq); +gov_sys_pol_attr_rw(go_hispeed_load); +gov_sys_pol_attr_rw(min_sample_time); +gov_sys_pol_attr_rw(timer_rate); +gov_sys_pol_attr_rw(timer_slack); +gov_sys_pol_attr_rw(boost); +gov_sys_pol_attr_rw(boostpulse_duration); +gov_sys_pol_attr_rw(io_is_busy); + +static struct global_attr boostpulse_gov_sys =3D + __ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_sys); + +static struct freq_attr boostpulse_gov_pol =3D + __ATTR(boostpulse, 0200, NULL, store_boostpulse_gov_pol); + +/* One Governor instance for entire system */ +static struct attribute *interactive_attributes_gov_sys[] =3D { + &target_loads_gov_sys.attr, + &above_hispeed_delay_gov_sys.attr, + &hispeed_freq_gov_sys.attr, + &go_hispeed_load_gov_sys.attr, + &min_sample_time_gov_sys.attr, + &timer_rate_gov_sys.attr, + &timer_slack_gov_sys.attr, + &boost_gov_sys.attr, + &boostpulse_gov_sys.attr, + &boostpulse_duration_gov_sys.attr, + &io_is_busy_gov_sys.attr, NULL, }; =20 -static struct attribute_group interactive_attr_group =3D { - .attrs =3D interactive_attributes, +static struct attribute_group interactive_attr_group_gov_sys =3D { + .attrs =3D interactive_attributes_gov_sys, .name =3D "interactive", }; =20 +/* Per policy governor instance */ +static struct attribute *interactive_attributes_gov_pol[] =3D { + &target_loads_gov_pol.attr, + &above_hispeed_delay_gov_pol.attr, + &hispeed_freq_gov_pol.attr, + &go_hispeed_load_gov_pol.attr, + &min_sample_time_gov_pol.attr, + &timer_rate_gov_pol.attr, + &timer_slack_gov_pol.attr, + &boost_gov_pol.attr, + &boostpulse_gov_pol.attr, + &boostpulse_duration_gov_pol.attr, + &io_is_busy_gov_pol.attr, + NULL, +}; + +static struct attribute_group interactive_attr_group_gov_pol =3D { + .attrs =3D interactive_attributes_gov_pol, + .name =3D "interactive", +}; + +static struct attribute_group *get_sysfs_attr(void) +{ + if (have_governor_per_policy()) + return &interactive_attr_group_gov_pol; + else + return &interactive_attr_group_gov_sys; +} + static int cpufreq_interactive_idle_notifier(struct notifier_block *nb= , unsigned long val, void *data) @@ -1057,15 +1153,88 @@ static int cpufreq_governor_interactive(struct = cpufreq_policy *policy, unsigned int j; struct cpufreq_interactive_cpuinfo *pcpu; struct cpufreq_frequency_table *freq_table; + struct cpufreq_interactive_tunables *tunables; + + if (have_governor_per_policy()) + tunables =3D policy->governor_data; + else + tunables =3D common_tunables; + + WARN_ON(!tunables && (event !=3D CPUFREQ_GOV_POLICY_INIT)); =20 switch (event) { + case CPUFREQ_GOV_POLICY_INIT: + if (have_governor_per_policy()) { + WARN_ON(tunables); + } else if (tunables) { + tunables->usage_count++; + policy->governor_data =3D tunables; + return 0; + } + + tunables =3D kzalloc(sizeof(*tunables), GFP_KERNEL); + if (!tunables) { + pr_err("%s: POLICY_INIT: kzalloc failed\n", __func__); + return -ENOMEM; + } + + rc =3D sysfs_create_group(get_governor_parent_kobj(policy), + get_sysfs_attr()); + if (rc) { + kfree(tunables); + return rc; + } + + tunables->usage_count =3D 1; + tunables->above_hispeed_delay =3D default_above_hispeed_delay; + tunables->nabove_hispeed_delay =3D + ARRAY_SIZE(default_above_hispeed_delay); + tunables->go_hispeed_load =3D DEFAULT_GO_HISPEED_LOAD; + tunables->target_loads =3D default_target_loads; + tunables->ntarget_loads =3D ARRAY_SIZE(default_target_loads); + tunables->min_sample_time =3D DEFAULT_MIN_SAMPLE_TIME; + tunables->timer_rate =3D DEFAULT_TIMER_RATE; + tunables->boostpulse_duration_val =3D DEFAULT_MIN_SAMPLE_TIME; + tunables->timer_slack_val =3D DEFAULT_TIMER_SLACK; + + spin_lock_init(&tunables->target_loads_lock); + spin_lock_init(&tunables->above_hispeed_delay_lock); + + if (!policy->governor->initialized) { + idle_notifier_register(&cpufreq_interactive_idle_nb); + cpufreq_register_notifier(&cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + } + + policy->governor_data =3D tunables; + if (!have_governor_per_policy()) + common_tunables =3D tunables; + + break; + + case CPUFREQ_GOV_POLICY_EXIT: + if (!--tunables->usage_count) { + if (policy->governor->initialized =3D=3D 1) { + cpufreq_unregister_notifier(&cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + idle_notifier_unregister(&cpufreq_interactive_idle_nb); + } + + sysfs_remove_group(get_governor_parent_kobj(policy), + get_sysfs_attr()); + kfree(tunables); + common_tunables =3D NULL; + } + + policy->governor_data =3D NULL; + break; + case CPUFREQ_GOV_START: mutex_lock(&gov_lock); =20 - freq_table =3D - cpufreq_frequency_get_table(policy->cpu); - if (!hispeed_freq) - hispeed_freq =3D policy->max; + freq_table =3D cpufreq_frequency_get_table(policy->cpu); + if (!tunables->hispeed_freq) + tunables->hispeed_freq =3D policy->max; =20 for_each_cpu(j, policy->cpus) { pcpu =3D &per_cpu(cpuinfo, j); @@ -1078,30 +1247,11 @@ static int cpufreq_governor_interactive(struct = cpufreq_policy *policy, pcpu->hispeed_validate_time =3D pcpu->floor_validate_time; down_write(&pcpu->enable_sem); - cpufreq_interactive_timer_start(j); + cpufreq_interactive_timer_start(tunables, j); pcpu->governor_enabled =3D 1; up_write(&pcpu->enable_sem); } =20 - /* - * Do not register the idle hook and create sysfs - * entries if we have already done so. - */ - if (++active_count > 1) { - mutex_unlock(&gov_lock); - return 0; - } - - rc =3D sysfs_create_group(cpufreq_global_kobject, - &interactive_attr_group); - if (rc) { - mutex_unlock(&gov_lock); - return rc; - } - - idle_notifier_register(&cpufreq_interactive_idle_nb); - cpufreq_register_notifier( - &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER); mutex_unlock(&gov_lock); break; =20 @@ -1116,18 +1266,7 @@ static int cpufreq_governor_interactive(struct c= pufreq_policy *policy, up_write(&pcpu->enable_sem); } =20 - if (--active_count > 0) { - mutex_unlock(&gov_lock); - return 0; - } - - cpufreq_unregister_notifier( - &cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER); - idle_notifier_unregister(&cpufreq_interactive_idle_nb); - sysfs_remove_group(cpufreq_global_kobject, - &interactive_attr_group); mutex_unlock(&gov_lock); - break; =20 case CPUFREQ_GOV_LIMITS: @@ -1161,7 +1300,7 @@ static int cpufreq_governor_interactive(struct cp= ufreq_policy *policy, */ del_timer_sync(&pcpu->cpu_timer); del_timer_sync(&pcpu->cpu_slack_timer); - cpufreq_interactive_timer_start(j); + cpufreq_interactive_timer_start(tunables, j); up_write(&pcpu->enable_sem); } break; @@ -1201,9 +1340,7 @@ static int __init cpufreq_interactive_init(void) init_rwsem(&pcpu->enable_sem); } =20 - spin_lock_init(&target_loads_lock); spin_lock_init(&speedchange_cpumask_lock); - spin_lock_init(&above_hispeed_delay_lock); mutex_init(&gov_lock); speedchange_task =3D kthread_create(cpufreq_interactive_speedchange_task, NULL, --=20 1.7.9.5