joy/android_kernel_sony_sdm845
Watch
1
Fork
You've already forked android_kernel_sony_sdm845
 0
Code Issues Pull requests Projects Releases Wiki Activity

sched: Introduce HMP scheduling

Browse source 
Add a snapshot of all changes associated with the HMP scheduler
ported over to the latest kernel version. This will allow proper big
Little scheduling for targets that will use this kernel version.
The msm-4-4 snapshot is taken at commitcc60f0790f27 (sched: constrain
HMP scheduler tunable range with in better way).

Please note that this is not an exact snapshot; certain HMP related
code has been removed. This includes redundant code, useless CONFIG
options, rarely taken error code paths, runtime enable/disable support
for the HMP scheduler.

Change-Id: I923f35904008f1fb74ac9c1a39481810cdea5790
Signed-off-by: Syed Rameez Mustafa <rameezmustafa@codeaurora.org>
This commit is contained in:
Syed Rameez Mustafa 2016-09-07 16:18:27 -07:00
commit dddcab78bd
23 changed files with 9438 additions and 94 deletions
Download patch file Download diff file Expand all files Collapse all files

1432
Documentation/scheduler/sched-hmp.txt Normal file
View file

File diff suppressed because it is too large Load diff

15
include/linux/cpu.h
View file

@ -28,6 +28,21 @@ struct cpu {
struct device dev;
};
struct cpu_pstate_pwr {
unsigned int freq;
uint32_t power;
};
struct cpu_pwr_stats {
int cpu;
long temp;
struct cpu_pstate_pwr *ptable;
bool throttling;
int len;
};
extern struct cpu_pwr_stats *get_cpu_pwr_stats(void);
extern void boot_cpu_init(void);
extern void boot_cpu_state_init(void);

185
include/linux/sched.h
View file

@ -174,6 +174,9 @@ extern unsigned long nr_iowait(void);
extern unsigned long nr_iowait_cpu(int cpu);
extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);
extern void sched_update_nr_prod(int cpu, long delta, bool inc);
extern void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg);
extern void calc_global_load(unsigned long ticks);
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
@ -315,6 +318,25 @@ extern char ___assert_task_state[1 - 2*!!(
/* Task command name length */
#define TASK_COMM_LEN 16
extern const char *sched_window_reset_reasons[];
enum task_event {
PUT_PREV_TASK = 0,
PICK_NEXT_TASK = 1,
TASK_WAKE = 2,
TASK_MIGRATE = 3,
TASK_UPDATE = 4,
IRQ_UPDATE = 5,
};
/* Note: this need to be in sync with migrate_type_names array */
enum migrate_types {
GROUP_TO_RQ,
RQ_TO_GROUP,
RQ_TO_RQ,
GROUP_TO_GROUP,
};
#include <linux/spinlock.h>
/*
@ -1335,6 +1357,47 @@ struct sched_statistics {
};
#endif
#define RAVG_HIST_SIZE_MAX 5
#define NUM_BUSY_BUCKETS 10
/* ravg represents frequency scaled cpu-demand of tasks */
struct ravg {
/*
* 'mark_start' marks the beginning of an event (task waking up, task
* starting to execute, task being preempted) within a window
*
* 'sum' represents how runnable a task has been within current
* window. It incorporates both running time and wait time and is
* frequency scaled.
*
* 'sum_history' keeps track of history of 'sum' seen over previous
* RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are
* ignored.
*
* 'demand' represents maximum sum seen over previous
* sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency
* demand for tasks.
*
* 'curr_window' represents task's contribution to cpu busy time
* statistics (rq->curr_runnable_sum) in current window
*
* 'prev_window' represents task's contribution to cpu busy time
* statistics (rq->prev_runnable_sum) in previous window
*
* 'pred_demand' represents task's current predicted cpu busy time
*
* 'busy_buckets' groups historical busy time into different buckets
* used for prediction
*/
u64 mark_start;
u32 sum, demand;
u32 sum_history[RAVG_HIST_SIZE_MAX];
u32 curr_window, prev_window;
u16 active_windows;
u32 pred_demand;
u8 busy_buckets[NUM_BUSY_BUCKETS];
};
struct sched_entity {
struct load_weight load; /* for load-balancing */
struct rb_node run_node;
@ -1505,6 +1568,20 @@ struct task_struct {
const struct sched_class *sched_class;
struct sched_entity se;
struct sched_rt_entity rt;
#ifdef CONFIG_SCHED_HMP
struct ravg ravg;
/*
* 'init_load_pct' represents the initial task load assigned to children
* of this task
*/
u32 init_load_pct;
u64 last_wake_ts;
u64 last_switch_out_ts;
u64 last_cpu_selected_ts;
struct related_thread_group *grp;
struct list_head grp_list;
u64 cpu_cycles;
#endif
#ifdef CONFIG_CGROUP_SCHED
struct task_group *sched_task_group;
#endif
@ -2254,6 +2331,7 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut,
/*
* Per process flags
*/
#define PF_WAKE_UP_IDLE 0x00000002 /* try to wake up on an idle CPU */
#define PF_EXITING 0x00000004 /* getting shut down */
#define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */
#define PF_VCPU 0x00000010 /* I'm a virtual CPU */
@ -2436,6 +2514,93 @@ static inline int set_cpus_allowed_ptr(struct task_struct *p,
}
#endif
struct sched_load {
unsigned long prev_load;
unsigned long new_task_load;
unsigned long predicted_load;
};
extern int sched_set_wake_up_idle(struct task_struct *p, int wake_up_idle);
extern u32 sched_get_wake_up_idle(struct task_struct *p);
struct cpu_cycle_counter_cb {
u64 (*get_cpu_cycle_counter)(int cpu);
};
#ifdef CONFIG_SCHED_HMP
extern int sched_set_window(u64 window_start, unsigned int window_size);
extern unsigned long sched_get_busy(int cpu);
extern void sched_get_cpus_busy(struct sched_load *busy,
const struct cpumask *query_cpus);
extern void sched_set_io_is_busy(int val);
extern int sched_set_boost(int enable);
extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct);
extern u32 sched_get_init_task_load(struct task_struct *p);
extern int sched_set_static_cpu_pwr_cost(int cpu, unsigned int cost);
extern unsigned int sched_get_static_cpu_pwr_cost(int cpu);
extern int sched_set_static_cluster_pwr_cost(int cpu, unsigned int cost);
extern unsigned int sched_get_static_cluster_pwr_cost(int cpu);
extern int sched_update_freq_max_load(const cpumask_t *cpumask);
extern void sched_update_cpu_freq_min_max(const cpumask_t *cpus,
u32 fmin, u32 fmax);
extern void sched_set_cpu_cstate(int cpu, int cstate,
int wakeup_energy, int wakeup_latency);
extern void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, int dstate,
int wakeup_energy, int wakeup_latency);
extern int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb);
extern u64 sched_ktime_clock(void);
extern int sched_set_group_id(struct task_struct *p, unsigned int group_id);
extern unsigned int sched_get_group_id(struct task_struct *p);
#else /* CONFIG_SCHED_HMP */
static inline u64 sched_ktime_clock(void)
{
return 0;
}
static inline int
register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb)
{
return 0;
}
static inline int sched_set_window(u64 window_start, unsigned int window_size)
{
return -EINVAL;
}
static inline unsigned long sched_get_busy(int cpu)
{
return 0;
}
static inline void sched_get_cpus_busy(struct sched_load *busy,
const struct cpumask *query_cpus) {};
static inline void sched_set_io_is_busy(int val) {};
static inline int sched_set_boost(int enable)
{
return -EINVAL;
}
static inline int sched_update_freq_max_load(const cpumask_t *cpumask)
{
return 0;
}
static inline void sched_update_cpu_freq_min_max(const cpumask_t *cpus,
u32 fmin, u32 fmax) { }
static inline void
sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency)
{
}
static inline void sched_set_cluster_dstate(const cpumask_t *cluster_cpus,
int dstate, int wakeup_energy, int wakeup_latency)
{
}
#endif /* CONFIG_SCHED_HMP */
#ifdef CONFIG_NO_HZ_COMMON
void calc_load_enter_idle(void);
void calc_load_exit_idle(void);
@ -2444,6 +2609,14 @@ static inline void calc_load_enter_idle(void) { }
static inline void calc_load_exit_idle(void) { }
#endif /* CONFIG_NO_HZ_COMMON */
static inline void set_wake_up_idle(bool enabled)
{
if (enabled)
current->flags |= PF_WAKE_UP_IDLE;
else
current->flags &= ~PF_WAKE_UP_IDLE;
}
/*
* Do not use outside of architecture code which knows its limitations.
*
@ -2461,6 +2634,7 @@ extern u64 sched_clock_cpu(int cpu);
extern void sched_clock_init(void);
extern int sched_clock_initialized(void);
#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
static inline void sched_clock_tick(void)
@ -2537,7 +2711,7 @@ extern unsigned long long
task_sched_runtime(struct task_struct *task);
/* sched_exec is called by processes performing an exec */
#ifdef CONFIG_SMP
#if defined(CONFIG_SMP) && !defined(CONFIG_SCHED_HMP)
extern void sched_exec(void);
#else
#define sched_exec() {}
@ -2670,6 +2844,7 @@ extern void xtime_update(unsigned long ticks);
extern int wake_up_state(struct task_struct *tsk, unsigned int state);
extern int wake_up_process(struct task_struct *tsk);
extern int wake_up_process_no_notif(struct task_struct *tsk);
extern void wake_up_new_task(struct task_struct *tsk);
#ifdef CONFIG_SMP
extern void kick_process(struct task_struct *tsk);
@ -2678,6 +2853,12 @@ extern void wake_up_new_task(struct task_struct *tsk);
#endif
extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
extern void sched_dead(struct task_struct *p);
#ifdef CONFIG_SCHED_HMP
extern void sched_exit(struct task_struct *p);
#else
static inline void sched_exit(struct task_struct *p) { }
#endif
extern void proc_caches_init(void);
extern void flush_signals(struct task_struct *);
@ -3506,6 +3687,8 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
#endif /* CONFIG_SMP */
extern struct atomic_notifier_head load_alert_notifier_head;
extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
extern long sched_getaffinity(pid_t pid, struct cpumask *mask);

38
include/linux/sched/sysctl.h
View file

@ -18,6 +18,32 @@ extern unsigned int sysctl_sched_latency;
extern unsigned int sysctl_sched_min_granularity;
extern unsigned int sysctl_sched_wakeup_granularity;
extern unsigned int sysctl_sched_child_runs_first;
extern unsigned int sysctl_sched_wake_to_idle;
#ifdef CONFIG_SCHED_HMP
extern int sysctl_sched_freq_inc_notify;
extern int sysctl_sched_freq_dec_notify;
extern unsigned int sysctl_sched_window_stats_policy;
extern unsigned int sysctl_sched_ravg_hist_size;
extern unsigned int sysctl_sched_cpu_high_irqload;
extern unsigned int sysctl_sched_init_task_load_pct;
extern unsigned int sysctl_sched_spill_nr_run;
extern unsigned int sysctl_sched_spill_load_pct;
extern unsigned int sysctl_sched_upmigrate_pct;
extern unsigned int sysctl_sched_downmigrate_pct;
extern unsigned int sysctl_early_detection_duration;
extern unsigned int sysctl_sched_boost;
extern unsigned int sysctl_sched_small_wakee_task_load_pct;
extern unsigned int sysctl_sched_big_waker_task_load_pct;
extern unsigned int sysctl_sched_select_prev_cpu_us;
extern unsigned int sysctl_sched_enable_colocation;
extern unsigned int sysctl_sched_restrict_cluster_spill;
extern unsigned int sysctl_sched_new_task_windows;
extern unsigned int sysctl_sched_pred_alert_freq;
extern unsigned int sysctl_sched_freq_aggregate;
extern unsigned int sysctl_sched_enable_thread_grouping;
extern unsigned int sysctl_sched_freq_aggregate_threshold_pct;
#endif /* CONFIG_SCHED_HMP */
enum sched_tunable_scaling {
SCHED_TUNABLESCALING_NONE,
@ -43,6 +69,18 @@ int sched_proc_update_handler(struct ctl_table *table, int write,
loff_t *ppos);
#endif
extern int sched_migrate_notify_proc_handler(struct ctl_table *table,
int write, void __user *buffer, size_t *lenp, loff_t *ppos);
extern int sched_hmp_proc_update_handler(struct ctl_table *table,
int write, void __user *buffer, size_t *lenp, loff_t *ppos);
extern int sched_boost_handler(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos);
extern int sched_window_update_handler(struct ctl_table *table,
int write, void __user *buffer, size_t *lenp, loff_t *ppos);
/*
* control realtime throttling:
*

2
include/linux/tick.h
View file

@ -27,6 +27,8 @@ static inline void tick_handover_do_timer(void) { }
static inline void tick_cleanup_dead_cpu(int cpu) { }
#endif /* !CONFIG_GENERIC_CLOCKEVENTS */
extern u64 jiffy_to_ktime_ns(u64 *now, u64 *jiffy_ktime_ns);
#if defined(CONFIG_GENERIC_CLOCKEVENTS) && defined(CONFIG_SUSPEND)
extern void tick_freeze(void);
extern void tick_unfreeze(void);

664
include/trace/events/sched.h
View file

@ -8,6 +8,8 @@
#include <linux/tracepoint.h>
#include <linux/binfmts.h>
struct rq;
/*
* Tracepoint for calling kthread_stop, performed to end a kthread:
*/
@ -50,6 +52,561 @@ TRACE_EVENT(sched_kthread_stop_ret,
TP_printk("ret=%d", __entry->ret)
);
/*
* Tracepoint for task enqueue/dequeue:
*/
TRACE_EVENT(sched_enq_deq_task,
TP_PROTO(struct task_struct *p, bool enqueue, unsigned int cpus_allowed),
TP_ARGS(p, enqueue, cpus_allowed),
TP_STRUCT__entry(
__array( char, comm, TASK_COMM_LEN )
__field( pid_t, pid )
__field( int, prio )
__field( int, cpu )
__field( bool, enqueue )
__field(unsigned int, nr_running )
__field(unsigned long, cpu_load )
__field(unsigned int, rt_nr_running )
__field(unsigned int, cpus_allowed )
#ifdef CONFIG_SCHED_HMP
__field(unsigned int, demand )
__field(unsigned int, pred_demand )
#endif
),
TP_fast_assign(
memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
__entry->pid = p->pid;
__entry->prio = p->prio;
__entry->cpu = task_cpu(p);
__entry->enqueue = enqueue;
__entry->nr_running = task_rq(p)->nr_running;
__entry->cpu_load = task_rq(p)->cpu_load[0];
__entry->rt_nr_running = task_rq(p)->rt.rt_nr_running;
__entry->cpus_allowed = cpus_allowed;
#ifdef CONFIG_SCHED_HMP
__entry->demand = p->ravg.demand;
__entry->pred_demand = p->ravg.pred_demand;
#endif
),
TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u cpu_load=%lu rt_nr_running=%u affine=%x"
#ifdef CONFIG_SCHED_HMP
" demand=%u pred_demand=%u"
#endif
, __entry->cpu,
__entry->enqueue ? "enqueue" : "dequeue",
__entry->comm, __entry->pid,
__entry->prio, __entry->nr_running,
__entry->cpu_load, __entry->rt_nr_running, __entry->cpus_allowed
#ifdef CONFIG_SCHED_HMP
, __entry->demand, __entry->pred_demand
#endif
)
);
#ifdef CONFIG_SCHED_HMP
struct group_cpu_time;
struct migration_sum_data;
extern const char *task_event_names[];
extern const char *migrate_type_names[];
TRACE_EVENT(sched_task_load,
TP_PROTO(struct task_struct *p, bool boost, int reason,
bool sync, bool need_idle, u32 flags, int best_cpu),
TP_ARGS(p, boost, reason, sync, need_idle, flags, best_cpu),
TP_STRUCT__entry(
__array( char, comm, TASK_COMM_LEN )
__field( pid_t, pid )
__field(unsigned int, demand )
__field( bool, boost )
__field( int, reason )
__field( bool, sync )
__field( bool, need_idle )
__field( u32, flags )
__field( int, best_cpu )
__field( u64, latency )
),
TP_fast_assign(
memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
__entry->pid = p->pid;
__entry->demand = p->ravg.demand;
__entry->boost = boost;
__entry->reason = reason;
__entry->sync = sync;
__entry->need_idle = need_idle;
__entry->flags = flags;
__entry->best_cpu = best_cpu;
__entry->latency = p->state == TASK_WAKING ?
sched_ktime_clock() -
p->ravg.mark_start : 0;
),
TP_printk("%d (%s): demand=%u boost=%d reason=%d sync=%d need_idle=%d flags=%x best_cpu=%d latency=%llu",
__entry->pid, __entry->comm, __entry->demand,
__entry->boost, __entry->reason, __entry->sync,
__entry->need_idle, __entry->flags,
__entry->best_cpu, __entry->latency)
);
TRACE_EVENT(sched_set_preferred_cluster,
TP_PROTO(struct related_thread_group *grp, u64 total_demand),
TP_ARGS(grp, total_demand),
TP_STRUCT__entry(
__field( int, id )
__field( u64, demand )
__field( int, cluster_first_cpu )
),
TP_fast_assign(
__entry->id = grp->id;
__entry->demand = total_demand;
__entry->cluster_first_cpu = grp->preferred_cluster ?
cluster_first_cpu(grp->preferred_cluster)
: -1;
),
TP_printk("group_id %d total_demand %llu preferred_cluster_first_cpu %d",
__entry->id, __entry->demand,
__entry->cluster_first_cpu)
);
DECLARE_EVENT_CLASS(sched_cpu_load,
TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp),
TP_ARGS(rq, idle, irqload, power_cost, temp),
TP_STRUCT__entry(
__field(unsigned int, cpu )
__field(unsigned int, idle )
__field(unsigned int, nr_running )
__field(unsigned int, nr_big_tasks )
__field(unsigned int, load_scale_factor )
__field(unsigned int, capacity )
__field( u64, cumulative_runnable_avg )
__field( u64, irqload )
__field(unsigned int, max_freq )
__field(unsigned int, power_cost )
__field( int, cstate )
__field( int, dstate )
__field( int, temp )
),
TP_fast_assign(
__entry->cpu = rq->cpu;
__entry->idle = idle;
__entry->nr_running = rq->nr_running;
__entry->nr_big_tasks = rq->hmp_stats.nr_big_tasks;
__entry->load_scale_factor = cpu_load_scale_factor(rq->cpu);
__entry->capacity = cpu_capacity(rq->cpu);
__entry->cumulative_runnable_avg = rq->hmp_stats.cumulative_runnable_avg;
__entry->irqload = irqload;
__entry->max_freq = cpu_max_freq(rq->cpu);
__entry->power_cost = power_cost;
__entry->cstate = rq->cstate;
__entry->dstate = rq->cluster->dstate;
__entry->temp = temp;
),
TP_printk("cpu %u idle %d nr_run %u nr_big %u lsf %u capacity %u cr_avg %llu irqload %llu fmax %u power_cost %u cstate %d dstate %d temp %d",
__entry->cpu, __entry->idle, __entry->nr_running, __entry->nr_big_tasks,
__entry->load_scale_factor, __entry->capacity,
__entry->cumulative_runnable_avg, __entry->irqload,
__entry->max_freq, __entry->power_cost, __entry->cstate,
__entry->dstate, __entry->temp)
);
DEFINE_EVENT(sched_cpu_load, sched_cpu_load_wakeup,
TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp),
TP_ARGS(rq, idle, irqload, power_cost, temp)
);
DEFINE_EVENT(sched_cpu_load, sched_cpu_load_lb,
TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp),
TP_ARGS(rq, idle, irqload, power_cost, temp)
);
DEFINE_EVENT(sched_cpu_load, sched_cpu_load_cgroup,
TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp),
TP_ARGS(rq, idle, irqload, power_cost, temp)
);
TRACE_EVENT(sched_set_boost,
TP_PROTO(int ref_count),
TP_ARGS(ref_count),
TP_STRUCT__entry(
__field(unsigned int, ref_count )
),
TP_fast_assign(
__entry->ref_count = ref_count;
),
TP_printk("ref_count=%d", __entry->ref_count)
);
TRACE_EVENT(sched_update_task_ravg,
TP_PROTO(struct task_struct *p, struct rq *rq, enum task_event evt,
u64 wallclock, u64 irqtime, u64 cycles, u64 exec_time,
struct group_cpu_time *cpu_time),
TP_ARGS(p, rq, evt, wallclock, irqtime, cycles, exec_time, cpu_time),
TP_STRUCT__entry(
__array( char, comm, TASK_COMM_LEN )
__field( pid_t, pid )
__field( pid_t, cur_pid )
__field(unsigned int, cur_freq )
__field( u64, wallclock )
__field( u64, mark_start )
__field( u64, delta_m )
__field( u64, win_start )
__field( u64, delta )
__field( u64, irqtime )
__field(enum task_event, evt )
__field(unsigned int, demand )
__field(unsigned int, sum )
__field( int, cpu )
__field(unsigned int, pred_demand )
__field( u64, rq_cs )
__field( u64, rq_ps )
__field( u64, grp_cs )
__field( u64, grp_ps )
__field( u64, grp_nt_cs )
__field( u64, grp_nt_ps )
__field( u32, curr_window )
__field( u32, prev_window )
__field( u64, nt_cs )
__field( u64, nt_ps )
__field( u32, active_windows )
),
TP_fast_assign(
__entry->wallclock = wallclock;
__entry->win_start = rq->window_start;
__entry->delta = (wallclock - rq->window_start);
__entry->evt = evt;
__entry->cpu = rq->cpu;
__entry->cur_pid = rq->curr->pid;
__entry->cur_freq = cpu_cycles_to_freq(cycles, exec_time);
memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
__entry->pid = p->pid;
__entry->mark_start = p->ravg.mark_start;
__entry->delta_m = (wallclock - p->ravg.mark_start);
__entry->demand = p->ravg.demand;
__entry->sum = p->ravg.sum;
__entry->irqtime = irqtime;
__entry->pred_demand = p->ravg.pred_demand;
__entry->rq_cs = rq->curr_runnable_sum;
__entry->rq_ps = rq->prev_runnable_sum;
__entry->grp_cs = cpu_time ? cpu_time->curr_runnable_sum : 0;
__entry->grp_ps = cpu_time ? cpu_time->prev_runnable_sum : 0;
__entry->grp_nt_cs = cpu_time ? cpu_time->nt_curr_runnable_sum : 0;
__entry->grp_nt_ps = cpu_time ? cpu_time->nt_prev_runnable_sum : 0;
__entry->curr_window = p->ravg.curr_window;
__entry->prev_window = p->ravg.prev_window;
__entry->nt_cs = rq->nt_curr_runnable_sum;
__entry->nt_ps = rq->nt_prev_runnable_sum;
__entry->active_windows = p->ravg.active_windows;
),
TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u sum %u irqtime %llu pred_demand %u rq_cs %llu rq_ps %llu cur_window %u prev_window %u nt_cs %llu nt_ps %llu active_wins %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu"
, __entry->wallclock, __entry->win_start, __entry->delta,
task_event_names[__entry->evt], __entry->cpu,
__entry->cur_freq, __entry->cur_pid,
__entry->pid, __entry->comm, __entry->mark_start,
__entry->delta_m, __entry->demand,
__entry->sum, __entry->irqtime, __entry->pred_demand,
__entry->rq_cs, __entry->rq_ps, __entry->curr_window,
__entry->prev_window, __entry->nt_cs, __entry->nt_ps,
__entry->active_windows, __entry->grp_cs,
__entry->grp_ps, __entry->grp_nt_cs, __entry->grp_nt_ps)
);
TRACE_EVENT(sched_get_task_cpu_cycles,
TP_PROTO(int cpu, int event, u64 cycles, u64 exec_time),
TP_ARGS(cpu, event, cycles, exec_time),
TP_STRUCT__entry(
__field(int, cpu )
__field(int, event )
__field(u64, cycles )
__field(u64, exec_time )
__field(u32, freq )
__field(u32, legacy_freq )
),
TP_fast_assign(
__entry->cpu = cpu;
__entry->event = event;
__entry->cycles = cycles;
__entry->exec_time = exec_time;
__entry->freq = cpu_cycles_to_freq(cycles, exec_time);
__entry->legacy_freq = cpu_cur_freq(cpu);
),
TP_printk("cpu=%d event=%d cycles=%llu exec_time=%llu freq=%u legacy_freq=%u",
__entry->cpu, __entry->event, __entry->cycles,
__entry->exec_time, __entry->freq, __entry->legacy_freq)
);
TRACE_EVENT(sched_update_history,
TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int samples,
enum task_event evt),
TP_ARGS(rq, p, runtime, samples, evt),
TP_STRUCT__entry(
__array( char, comm, TASK_COMM_LEN )
__field( pid_t, pid )
__field(unsigned int, runtime )
__field( int, samples )
__field(enum task_event, evt )
__field(unsigned int, demand )
__field(unsigned int, pred_demand )
__array( u32, hist, RAVG_HIST_SIZE_MAX)
__field(unsigned int, nr_big_tasks )
__field( int, cpu )
),
TP_fast_assign(
memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
__entry->pid = p->pid;
__entry->runtime = runtime;
__entry->samples = samples;
__entry->evt = evt;
__entry->demand = p->ravg.demand;
__entry->pred_demand = p->ravg.pred_demand;
memcpy(__entry->hist, p->ravg.sum_history,
RAVG_HIST_SIZE_MAX * sizeof(u32));
__entry->nr_big_tasks = rq->hmp_stats.nr_big_tasks;
__entry->cpu = rq->cpu;
),
TP_printk("%d (%s): runtime %u samples %d event %s demand %u pred_demand %u"
" (hist: %u %u %u %u %u) cpu %d nr_big %u",
__entry->pid, __entry->comm,
__entry->runtime, __entry->samples,
task_event_names[__entry->evt],
__entry->demand, __entry->pred_demand,
__entry->hist[0], __entry->hist[1],
__entry->hist[2], __entry->hist[3],
__entry->hist[4], __entry->cpu, __entry->nr_big_tasks)
);
TRACE_EVENT(sched_reset_all_window_stats,
TP_PROTO(u64 window_start, u64 window_size, u64 time_taken,
int reason, unsigned int old_val, unsigned int new_val),
TP_ARGS(window_start, window_size, time_taken,
reason, old_val, new_val),
TP_STRUCT__entry(
__field( u64, window_start )
__field( u64, window_size )
__field( u64, time_taken )
__field( int, reason )
__field(unsigned int, old_val )
__field(unsigned int, new_val )
),
TP_fast_assign(
__entry->window_start = window_start;
__entry->window_size = window_size;
__entry->time_taken = time_taken;
__entry->reason = reason;
__entry->old_val = old_val;
__entry->new_val = new_val;
),
TP_printk("time_taken %llu window_start %llu window_size %llu reason %s old_val %u new_val %u",
__entry->time_taken, __entry->window_start,
__entry->window_size,
sched_window_reset_reasons[__entry->reason],
__entry->old_val, __entry->new_val)
);
TRACE_EVENT(sched_update_pred_demand,
TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int pct,
unsigned int pred_demand),
TP_ARGS(rq, p, runtime, pct, pred_demand),
TP_STRUCT__entry(
__array( char, comm, TASK_COMM_LEN )
__field( pid_t, pid )
__field(unsigned int, runtime )
__field( int, pct )
__field(unsigned int, pred_demand )
__array( u8, bucket, NUM_BUSY_BUCKETS)
__field( int, cpu )
),
TP_fast_assign(
memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
__entry->pid = p->pid;
__entry->runtime = runtime;
__entry->pct = pct;
__entry->pred_demand = pred_demand;
memcpy(__entry->bucket, p->ravg.busy_buckets,
NUM_BUSY_BUCKETS * sizeof(u8));
__entry->cpu = rq->cpu;
),
TP_printk("%d (%s): runtime %u pct %d cpu %d pred_demand %u (buckets: %u %u %u %u %u %u %u %u %u %u)",
__entry->pid, __entry->comm,
__entry->runtime, __entry->pct, __entry->cpu,
__entry->pred_demand, __entry->bucket[0], __entry->bucket[1],
__entry->bucket[2], __entry->bucket[3] ,__entry->bucket[4],
__entry->bucket[5], __entry->bucket[6], __entry->bucket[7],
__entry->bucket[8], __entry->bucket[9])
);
TRACE_EVENT(sched_migration_update_sum,
TP_PROTO(struct task_struct *p, enum migrate_types migrate_type, struct migration_sum_data *d),
TP_ARGS(p, migrate_type, d),
TP_STRUCT__entry(
__field(int, tcpu )
__field(int, pid )
__field( u64, cs )
__field( u64, ps )
__field( s64, nt_cs )
__field( s64, nt_ps )
__field(enum migrate_types, migrate_type )
__field( s64, src_cs )
__field( s64, src_ps )
__field( s64, dst_cs )
__field( s64, dst_ps )
__field( s64, src_nt_cs )
__field( s64, src_nt_ps )
__field( s64, dst_nt_cs )
__field( s64, dst_nt_ps )
),
TP_fast_assign(
__entry->tcpu = task_cpu(p);
__entry->pid = p->pid;
__entry->migrate_type = migrate_type;
__entry->src_cs = d->src_rq ?
d->src_rq->curr_runnable_sum :
d->src_cpu_time->curr_runnable_sum;
__entry->src_ps = d->src_rq ?
d->src_rq->prev_runnable_sum :
d->src_cpu_time->prev_runnable_sum;
__entry->dst_cs = d->dst_rq ?
d->dst_rq->curr_runnable_sum :
d->dst_cpu_time->curr_runnable_sum;
__entry->dst_ps = d->dst_rq ?
d->dst_rq->prev_runnable_sum :
d->dst_cpu_time->prev_runnable_sum;
__entry->src_nt_cs = d->src_rq ?
d->src_rq->nt_curr_runnable_sum :
d->src_cpu_time->nt_curr_runnable_sum;
__entry->src_nt_ps = d->src_rq ?
d->src_rq->nt_prev_runnable_sum :
d->src_cpu_time->nt_prev_runnable_sum;
__entry->dst_nt_cs = d->dst_rq ?
d->dst_rq->nt_curr_runnable_sum :
d->dst_cpu_time->nt_curr_runnable_sum;
__entry->dst_nt_ps = d->dst_rq ?
d->dst_rq->nt_prev_runnable_sum :
d->dst_cpu_time->nt_prev_runnable_sum;
),
TP_printk("pid %d task_cpu %d migrate_type %s src_cs %llu src_ps %llu dst_cs %lld dst_ps %lld src_nt_cs %llu src_nt_ps %llu dst_nt_cs %lld dst_nt_ps %lld",
__entry->pid, __entry->tcpu, migrate_type_names[__entry->migrate_type],
__entry->src_cs, __entry->src_ps, __entry->dst_cs, __entry->dst_ps,
__entry->src_nt_cs, __entry->src_nt_ps, __entry->dst_nt_cs, __entry->dst_nt_ps)
);
TRACE_EVENT(sched_get_busy,
TP_PROTO(int cpu, u64 load, u64 nload, u64 pload, int early),
TP_ARGS(cpu, load, nload, pload, early),
TP_STRUCT__entry(
__field( int, cpu )
__field( u64, load )
__field( u64, nload )
__field( u64, pload )
__field( int, early )
),
TP_fast_assign(
__entry->cpu = cpu;
__entry->load = load;
__entry->nload = nload;
__entry->pload = pload;
__entry->early = early;
),
TP_printk("cpu %d load %lld new_task_load %lld predicted_load %lld early %d",
__entry->cpu, __entry->load, __entry->nload,
__entry->pload, __entry->early)
);
TRACE_EVENT(sched_freq_alert,
TP_PROTO(int cpu, int pd_notif, int check_groups, struct rq *rq,
u64 new_load),
TP_ARGS(cpu, pd_notif, check_groups, rq, new_load),
TP_STRUCT__entry(
__field( int, cpu )
__field( int, pd_notif )
__field( int, check_groups )
__field( u64, old_busy_time )
__field( u64, ps )
__field( u64, new_load )
__field( u64, old_pred )
__field( u64, new_pred )
),
TP_fast_assign(
__entry->cpu = cpu;
__entry->pd_notif = pd_notif;
__entry->check_groups = check_groups;
__entry->old_busy_time = rq->old_busy_time;
__entry->ps = rq->prev_runnable_sum;
__entry->new_load = new_load;
__entry->old_pred = rq->old_estimated_time;
__entry->new_pred = rq->hmp_stats.pred_demands_sum;
),
TP_printk("cpu %d pd_notif=%d check_groups %d old_busy_time=%llu prev_sum=%lld new_load=%llu old_pred=%llu new_pred=%llu",
__entry->cpu, __entry->pd_notif, __entry->check_groups,
__entry->old_busy_time, __entry->ps, __entry->new_load,
__entry->old_pred, __entry->new_pred)
);
#endif /* CONFIG_SCHED_HMP */
/*
* Tracepoint for waking up a task:
*/
@ -166,14 +723,15 @@ TRACE_EVENT(sched_switch,
*/
TRACE_EVENT(sched_migrate_task,
TP_PROTO(struct task_struct *p, int dest_cpu),
TP_PROTO(struct task_struct *p, int dest_cpu, unsigned int load),
TP_ARGS(p, dest_cpu),
TP_ARGS(p, dest_cpu, load),
TP_STRUCT__entry(
__array( char, comm, TASK_COMM_LEN )
__field( pid_t, pid )
__field( int, prio )
__field(unsigned int, load )
__field( int, orig_cpu )
__field( int, dest_cpu )
),
@ -182,15 +740,91 @@ TRACE_EVENT(sched_migrate_task,
memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
__entry->pid = p->pid;
__entry->prio = p->prio;
__entry->load = load;
__entry->orig_cpu = task_cpu(p);
__entry->dest_cpu = dest_cpu;
),
TP_printk("comm=%s pid=%d prio=%d orig_cpu=%d dest_cpu=%d",
__entry->comm, __entry->pid, __entry->prio,
TP_printk("comm=%s pid=%d prio=%d load=%d orig_cpu=%d dest_cpu=%d",
__entry->comm, __entry->pid, __entry->prio, __entry->load,
__entry->orig_cpu, __entry->dest_cpu)
);
/*
* Tracepoint for a CPU going offline/online:
*/
TRACE_EVENT(sched_cpu_hotplug,
TP_PROTO(int affected_cpu, int error, int status),
TP_ARGS(affected_cpu, error, status),
TP_STRUCT__entry(
__field( int, affected_cpu )
__field( int, error )
__field( int, status )
),
TP_fast_assign(
__entry->affected_cpu = affected_cpu;
__entry->error = error;
__entry->status = status;
),
TP_printk("cpu %d %s error=%d", __entry->affected_cpu,
__entry->status ? "online" : "offline", __entry->error)
);
/*
* Tracepoint for load balancing:
*/
#if NR_CPUS > 32
#error "Unsupported NR_CPUS for lb tracepoint."
#endif
TRACE_EVENT(sched_load_balance,
TP_PROTO(int cpu, enum cpu_idle_type idle, int balance,
unsigned long group_mask, int busiest_nr_running,
unsigned long imbalance, unsigned int env_flags, int ld_moved,
unsigned int balance_interval),
TP_ARGS(cpu, idle, balance, group_mask, busiest_nr_running,
imbalance, env_flags, ld_moved, balance_interval),
TP_STRUCT__entry(
__field( int, cpu)
__field( enum cpu_idle_type, idle)
__field( int, balance)
__field( unsigned long, group_mask)
__field( int, busiest_nr_running)
__field( unsigned long, imbalance)
__field( unsigned int, env_flags)
__field( int, ld_moved)
__field( unsigned int, balance_interval)
),
TP_fast_assign(
__entry->cpu = cpu;
__entry->idle = idle;
__entry->balance = balance;
__entry->group_mask = group_mask;
__entry->busiest_nr_running = busiest_nr_running;
__entry->imbalance = imbalance;
__entry->env_flags = env_flags;
__entry->ld_moved = ld_moved;
__entry->balance_interval = balance_interval;
),
TP_printk("cpu=%d state=%s balance=%d group=%#lx busy_nr=%d imbalance=%ld flags=%#x ld_moved=%d bal_int=%d",
__entry->cpu,
__entry->idle == CPU_IDLE ? "idle" :
(__entry->idle == CPU_NEWLY_IDLE ? "newly_idle" : "busy"),
__entry->balance,
__entry->group_mask, __entry->busiest_nr_running,
__entry->imbalance, __entry->env_flags, __entry->ld_moved,
__entry->balance_interval)
);
DECLARE_EVENT_CLASS(sched_process_template,
TP_PROTO(struct task_struct *p),
@ -586,6 +1220,28 @@ TRACE_EVENT(sched_wake_idle_without_ipi,
TP_printk("cpu=%d", __entry->cpu)
);
TRACE_EVENT(sched_get_nr_running_avg,
TP_PROTO(int avg, int big_avg, int iowait_avg),
TP_ARGS(avg, big_avg, iowait_avg),
TP_STRUCT__entry(
__field( int, avg )
__field( int, big_avg )
__field( int, iowait_avg )
),
TP_fast_assign(
__entry->avg = avg;
__entry->big_avg = big_avg;
__entry->iowait_avg = iowait_avg;
),
TP_printk("avg=%d big_avg=%d iowait_avg=%d",
__entry->avg, __entry->big_avg, __entry->iowait_avg)
);
#endif /* _TRACE_SCHED_H */
/* This part must be outside protection */

17
init/Kconfig
View file

@ -1165,6 +1165,23 @@ config CGROUP_DEBUG
endif # CGROUPS
config SCHED_HMP
bool "Scheduler support for heterogenous multi-processor systems"
depends on SMP && FAIR_GROUP_SCHED
help
This feature will let the scheduler optimize task placement on
systems made of heterogeneous cpus i.e cpus that differ either
in their instructions per-cycle capability or the maximum
frequency they can attain.
config SCHED_HMP_CSTATE_AWARE
bool "CPU C-state aware scheduler"
depends on SCHED_HMP
help
This feature will let the HMP scheduler optimize task placement
with CPUs C-state. If this is enabled, scheduler places tasks
onto the shallowest C-state CPU among the most power efficient CPUs.
config CHECKPOINT_RESTORE
bool "Checkpoint/restore support" if EXPERT
select PROC_CHILDREN

3
kernel/sched/Makefile
View file

@ -17,8 +17,9 @@ endif
obj-y += core.o loadavg.o clock.o cputime.o
obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o
obj-y += wait.o swait.o completion.o idle.o
obj-y += wait.o swait.o completion.o idle.o sched_avg.o
obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o
obj-$(CONFIG_SCHED_HMP) += hmp.o
obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o
obj-$(CONFIG_SCHEDSTATS) += stats.o
obj-$(CONFIG_SCHED_DEBUG) += debug.o

277
kernel/sched/core.c
View file

@ -91,6 +91,8 @@
#define CREATE_TRACE_POINTS
#include <trace/events/sched.h>
ATOMIC_NOTIFIER_HEAD(load_alert_notifier_head);
DEFINE_MUTEX(sched_domains_mutex);
DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
@ -758,6 +760,7 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
if (!(flags & ENQUEUE_RESTORE))
sched_info_queued(rq, p);
p->sched_class->enqueue_task(rq, p, flags);
trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_allowed)[0]);
}
static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
@ -766,6 +769,7 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
if (!(flags & DEQUEUE_SAVE))
sched_info_dequeued(rq, p);
p->sched_class->dequeue_task(rq, p, flags);
trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_allowed)[0]);
}
void activate_task(struct rq *rq, struct task_struct *p, int flags)
@ -998,8 +1002,9 @@ static struct rq *move_queued_task(struct rq *rq, struct task_struct *p, int new
p->on_rq = TASK_ON_RQ_MIGRATING;
dequeue_task(rq, p, 0);
double_lock_balance(rq, cpu_rq(new_cpu));
set_task_cpu(p, new_cpu);
raw_spin_unlock(&rq->lock);
double_rq_unlock(cpu_rq(new_cpu), rq);
rq = cpu_rq(new_cpu);
@ -1050,6 +1055,8 @@ static int migration_cpu_stop(void *data)
struct migration_arg *arg = data;
struct task_struct *p = arg->task;
struct rq *rq = this_rq();
int src_cpu = cpu_of(rq);
bool moved = false;
/*
* The original target cpu might have gone down and we might
@ -1071,15 +1078,21 @@ static int migration_cpu_stop(void *data)
* we're holding p->pi_lock.
*/
if (task_rq(p) == rq) {
if (task_on_rq_queued(p))
if (task_on_rq_queued(p)) {
rq = __migrate_task(rq, p, arg->dest_cpu);
else
moved = true;
} else {
p->wake_cpu = arg->dest_cpu;
}
}
raw_spin_unlock(&rq->lock);
raw_spin_unlock(&p->pi_lock);
local_irq_enable();
if (moved)
notify_migration(src_cpu, arg->dest_cpu, false, p);
return 0;
}
@ -1161,7 +1174,8 @@ static int __set_cpus_allowed_ptr(struct task_struct *p,
if (cpumask_equal(&p->cpus_allowed, new_mask))
goto out;
if (!cpumask_intersects(new_mask, cpu_valid_mask)) {
dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask);
if (dest_cpu >= nr_cpu_ids) {
ret = -EINVAL;
goto out;
}
@ -1246,13 +1260,15 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
#endif
#endif
trace_sched_migrate_task(p, new_cpu);
trace_sched_migrate_task(p, new_cpu, pct_task_load(p));
if (task_cpu(p) != new_cpu) {
if (p->sched_class->migrate_task_rq)
p->sched_class->migrate_task_rq(p);
p->se.nr_migrations++;
perf_event_task_migrate(p);
fixup_busy_time(p, new_cpu);
}
__set_task_cpu(p, new_cpu);
@ -1456,7 +1472,7 @@ unsigned long wait_task_inactive(struct task_struct *p, long match_state)
* yield - it could be a while.
*/
if (unlikely(queued)) {
ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);
ktime_t to = ktime_set(0, NSEC_PER_SEC);
set_current_state(TASK_UNINTERRUPTIBLE);
schedule_hrtimeout(&to, HRTIMER_MODE_REL);
@ -1798,6 +1814,8 @@ void sched_ttwu_pending(void)
void scheduler_ipi(void)
{
int cpu = smp_processor_id();
/*
* Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
* TIF_NEED_RESCHED remotely (for the first time) will also send
@ -1805,9 +1823,18 @@ void scheduler_ipi(void)
*/
preempt_fold_need_resched();
if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick() &&
!got_boost_kick())
return;
if (got_boost_kick()) {
struct rq *rq = cpu_rq(cpu);
if (rq->curr->sched_class == &fair_sched_class)
check_for_migration(rq, rq->curr);
clear_boost_kick(cpu);
}
/*
* Not all reschedule IPI handlers call irq_enter/irq_exit, since
* traditionally all their work was done from the interrupt return
@ -2009,6 +2036,18 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
{
unsigned long flags;
int cpu, success = 0;
#ifdef CONFIG_SMP
unsigned int old_load;
struct rq *rq;
u64 wallclock;
struct related_thread_group *grp = NULL;
int src_cpu;
bool notif_required = false;
#endif
bool freq_notif_allowed = !(wake_flags & WF_NO_NOTIFIER);
bool check_group = false;
wake_flags &= ~WF_NO_NOTIFIER;
/*
* If we are going to wake up a thread waiting for CONDITION we
@ -2082,14 +2121,33 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
*/
smp_cond_load_acquire(&p->on_cpu, !VAL);
rq = cpu_rq(task_cpu(p));
raw_spin_lock(&rq->lock);
old_load = task_load(p);
wallclock = sched_ktime_clock();
update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);
update_task_ravg(p, rq, TASK_WAKE, wallclock, 0);
raw_spin_unlock(&rq->lock);
rcu_read_lock();
grp = task_related_thread_group(p);
if (update_preferred_cluster(grp, p, old_load))
set_preferred_cluster(grp);
rcu_read_unlock();
check_group = grp != NULL;
p->sched_contributes_to_load = !!task_contributes_to_load(p);
p->state = TASK_WAKING;
cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
if (task_cpu(p) != cpu) {
src_cpu = task_cpu(p);
if (src_cpu != cpu) {
wake_flags |= WF_MIGRATED;
set_task_cpu(p, cpu);
notif_required = true;
}
set_task_last_wake(p, wallclock);
#endif /* CONFIG_SMP */
ttwu_queue(p, cpu, wake_flags);
@ -2098,6 +2156,17 @@ stat:
out:
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
if (freq_notif_allowed) {
if (notif_required && !same_freq_domain(src_cpu, cpu)) {
check_for_freq_change(cpu_rq(cpu),
false, check_group);
check_for_freq_change(cpu_rq(src_cpu),
false, check_group);
} else if (success) {
check_for_freq_change(cpu_rq(cpu), true, false);
}
}
return success;
}
@ -2114,9 +2183,13 @@ static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie cookie
{
struct rq *rq = task_rq(p);
if (WARN_ON_ONCE(rq != this_rq()) ||
WARN_ON_ONCE(p == current))
if (rq != this_rq() || p == current) {
printk_deferred("%s: Failed to wakeup task %d (%s), rq = %p,"
" this_rq = %p, p = %p, current = %p\n", __func__,
task_pid_nr(p), p->comm, rq, this_rq(), p, current);
return;
}
lockdep_assert_held(&rq->lock);
@ -2139,8 +2212,14 @@ static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie cookie
trace_sched_waking(p);
if (!task_on_rq_queued(p))
if (!task_on_rq_queued(p)) {
u64 wallclock = sched_ktime_clock();
update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);
update_task_ravg(p, rq, TASK_WAKE, wallclock, 0);
ttwu_activate(rq, p, ENQUEUE_WAKEUP);
set_task_last_wake(p, wallclock);
}
ttwu_do_wakeup(rq, p, 0, cookie);
ttwu_stat(p, smp_processor_id(), 0);
@ -2166,6 +2245,26 @@ int wake_up_process(struct task_struct *p)
}
EXPORT_SYMBOL(wake_up_process);
/**
* wake_up_process_no_notif - Wake up a specific process without notifying
* governor
* @p: The process to be woken up.
*
* Attempt to wake up the nominated process and move it to the set of runnable
* processes.
*
* Return: 1 if the process was woken up, 0 if it was already running.
*
* It may be assumed that this function implies a write memory barrier before
* changing the task state if and only if any tasks are woken up.
*/
int wake_up_process_no_notif(struct task_struct *p)
{
WARN_ON(task_is_stopped_or_traced(p));
return try_to_wake_up(p, TASK_NORMAL, WF_NO_NOTIFIER);
}
EXPORT_SYMBOL(wake_up_process_no_notif);
int wake_up_state(struct task_struct *p, unsigned int state)
{
return try_to_wake_up(p, state, 0);
@ -2188,6 +2287,44 @@ void __dl_clear_params(struct task_struct *p)
dl_se->dl_yielded = 0;
}
#ifdef CONFIG_SCHED_HMP
/*
* sched_exit() - Set EXITING_TASK_MARKER in task's ravg.demand field
*
* Stop accounting (exiting) task's future cpu usage
*
* We need this so that reset_all_windows_stats() can function correctly.
* reset_all_window_stats() depends on do_each_thread/for_each_thread task
* iterators to reset *all* task's statistics. Exiting tasks however become
* invisible to those iterators. sched_exit() is called on a exiting task prior
* to being removed from task_list, which will let reset_all_window_stats()
* function correctly.
*/
void sched_exit(struct task_struct *p)
{
unsigned long flags;
int cpu = get_cpu();
struct rq *rq = cpu_rq(cpu);
u64 wallclock;
sched_set_group_id(p, 0);
raw_spin_lock_irqsave(&rq->lock, flags);
/* rq->curr == p */
wallclock = sched_ktime_clock();
update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);
dequeue_task(rq, p, 0);
reset_task_stats(p);
p->ravg.mark_start = wallclock;
p->ravg.sum_history[0] = EXITING_TASK_MARKER;
enqueue_task(rq, p, 0);
clear_ed_task(p, rq);
raw_spin_unlock_irqrestore(&rq->lock, flags);
put_cpu();
}
#endif /* CONFIG_SCHED_HMP */
/*
* Perform scheduler related setup for a newly forked process p.
* p is forked by current.
@ -2564,6 +2701,8 @@ void wake_up_new_task(struct task_struct *p)
struct rq *rq;
raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
init_new_task_load(p);
add_new_task_to_grp(p);
p->state = TASK_RUNNING;
#ifdef CONFIG_SMP
/*
@ -2579,6 +2718,7 @@ void wake_up_new_task(struct task_struct *p)
rq = __task_rq_lock(p, &rf);
post_init_entity_util_avg(&p->se);
mark_task_starting(p);
activate_task(rq, p, 0);
p->on_rq = TASK_ON_RQ_QUEUED;
trace_sched_wakeup_new(p);
@ -2971,7 +3111,7 @@ void get_iowait_load(unsigned long *nr_waiters, unsigned long *load)
*load = rq->load.weight;
}
#ifdef CONFIG_SMP
#if defined(CONFIG_SMP) && !defined(CONFIG_SCHED_HMP)
/*
* sched_exec - execve() is a valuable balancing opportunity, because at
@ -3077,16 +3217,32 @@ void scheduler_tick(void)
int cpu = smp_processor_id();
struct rq *rq = cpu_rq(cpu);
struct task_struct *curr = rq->curr;
u64 wallclock;
bool early_notif;
u32 old_load;
struct related_thread_group *grp;
sched_clock_tick();
raw_spin_lock(&rq->lock);
old_load = task_load(curr);
set_window_start(rq);
update_rq_clock(rq);
curr->sched_class->task_tick(rq, curr, 0);
cpu_load_update_active(rq);
calc_global_load_tick(rq);
wallclock = sched_ktime_clock();
update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0);
early_notif = early_detection_notify(rq, wallclock);
raw_spin_unlock(&rq->lock);
if (early_notif)
atomic_notifier_call_chain(&load_alert_notifier_head,
0, (void *)(long)cpu);
perf_event_task_tick();
#ifdef CONFIG_SMP
@ -3094,6 +3250,15 @@ void scheduler_tick(void)
trigger_load_balance(rq);
#endif
rq_last_tick_reset(rq);
rcu_read_lock();
grp = task_related_thread_group(curr);
if (update_preferred_cluster(grp, curr, old_load))
set_preferred_cluster(grp);
rcu_read_unlock();
if (curr->sched_class == &fair_sched_class)
check_for_migration(rq, curr);
}
#ifdef CONFIG_NO_HZ_FULL
@ -3227,6 +3392,9 @@ static noinline void __schedule_bug(struct task_struct *prev)
if (panic_on_warn)
panic("scheduling while atomic\n");
#ifdef CONFIG_PANIC_ON_SCHED_BUG
BUG();
#endif
dump_stack();
add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
}
@ -3337,6 +3505,7 @@ static void __sched notrace __schedule(bool preempt)
struct pin_cookie cookie;
struct rq *rq;
int cpu;
u64 wallclock;
cpu = smp_processor_id();
rq = cpu_rq(cpu);
@ -3389,6 +3558,11 @@ static void __sched notrace __schedule(bool preempt)
update_rq_clock(rq);
next = pick_next_task(rq, prev, cookie);
wallclock = sched_ktime_clock();
update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0);
update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0);
clear_tsk_need_resched(prev);
clear_preempt_need_resched();
rq->clock_skip_update = 0;
@ -3398,6 +3572,8 @@ static void __sched notrace __schedule(bool preempt)
rq->curr = next;
++*switch_count;
set_task_last_switch_out(prev, wallclock);
trace_sched_switch(preempt, prev, next);
rq = context_switch(rq, prev, next, cookie); /* unlocks the rq */
} else {
@ -5273,10 +5449,11 @@ void init_idle(struct task_struct *idle, int cpu)
struct rq *rq = cpu_rq(cpu);
unsigned long flags;
__sched_fork(0, idle);
raw_spin_lock_irqsave(&idle->pi_lock, flags);
raw_spin_lock(&rq->lock);
__sched_fork(0, idle);
idle->state = TASK_RUNNING;
idle->se.exec_start = sched_clock();
@ -5586,7 +5763,10 @@ static void migrate_tasks(struct rq *dead_rq)
rq = __migrate_task(rq, next, dest_cpu);
if (rq != dead_rq) {
raw_spin_unlock(&rq->lock);
raw_spin_unlock(&next->pi_lock);
notify_migration(dead_rq->cpu, dest_cpu, true, next);
rq = dead_rq;
raw_spin_lock(&next->pi_lock);
raw_spin_lock(&rq->lock);
}
raw_spin_unlock(&next->pi_lock);
@ -6030,6 +6210,7 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
{
struct rq *rq = cpu_rq(cpu);
struct sched_domain *tmp;
unsigned long next_balance = rq->next_balance;
/* Remove the sched domains which do not contribute to scheduling. */
for (tmp = sd; tmp; ) {
@ -6061,6 +6242,17 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
sd->child = NULL;
}
for (tmp = sd; tmp; ) {
unsigned long interval;
interval = msecs_to_jiffies(tmp->balance_interval);
if (time_after(next_balance, tmp->last_balance + interval))
next_balance = tmp->last_balance + interval;
tmp = tmp->parent;
}
rq->next_balance = next_balance;
sched_domain_debug(sd, cpu);
rq_attach_root(rq, rd);
@ -7386,7 +7578,11 @@ int sched_cpu_deactivate(unsigned int cpu)
static void sched_rq_cpu_starting(unsigned int cpu)
{
struct rq *rq = cpu_rq(cpu);
unsigned long flags;
raw_spin_lock_irqsave(&rq->lock, flags);
set_window_start(rq);
raw_spin_unlock_irqrestore(&rq->lock, flags);
rq->calc_load_update = calc_load_update;
update_max_interval();
}
@ -7407,6 +7603,8 @@ int sched_cpu_dying(unsigned int cpu)
/* Handle pending wakeups and then migrate everything off */
sched_ttwu_pending();
raw_spin_lock_irqsave(&rq->lock, flags);
migrate_sync_cpu(cpu);
if (rq->rd) {
BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
set_rq_offline(rq);
@ -7414,6 +7612,9 @@ int sched_cpu_dying(unsigned int cpu)
migrate_tasks(rq);
BUG_ON(rq->nr_running != 1);
raw_spin_unlock_irqrestore(&rq->lock, flags);
clear_hmp_request(cpu);
calc_load_migrate(rq);
update_max_interval();
nohz_balance_exit_idle(cpu);
@ -7459,6 +7660,8 @@ void __init sched_init_smp(void)
cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
mutex_unlock(&sched_domains_mutex);
update_cluster_topology();
/* Move init over to a non-isolated CPU */
if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
BUG();
@ -7530,6 +7733,11 @@ void __init sched_init(void)
for (i = 0; i < WAIT_TABLE_SIZE; i++)
init_waitqueue_head(bit_wait_table + i);
#ifdef CONFIG_SCHED_HMP
pr_info("HMP scheduling enabled.\n");
init_clusters();
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
alloc_size += 2 * nr_cpu_ids * sizeof(void **);
#endif
@ -7646,7 +7854,36 @@ void __init sched_init(void)
rq->idle_stamp = 0;
rq->avg_idle = 2*sysctl_sched_migration_cost;
rq->max_idle_balance_cost = sysctl_sched_migration_cost;
rq->push_task = NULL;
#ifdef CONFIG_SCHED_HMP
cpumask_set_cpu(i, &rq->freq_domain_cpumask);
rq->hmp_stats.cumulative_runnable_avg = 0;
rq->window_start = 0;
rq->hmp_stats.nr_big_tasks = 0;
rq->hmp_flags = 0;
rq->cur_irqload = 0;
rq->avg_irqload = 0;
rq->irqload_ts = 0;
rq->static_cpu_pwr_cost = 0;
rq->cc.cycles = 1;
rq->cc.time = 1;
rq->cstate = 0;
rq->wakeup_latency = 0;
rq->wakeup_energy = 0;
/*
* All cpus part of same cluster by default. This avoids the
* need to check for rq->cluster being non-NULL in hot-paths
* like select_best_cpu()
*/
rq->cluster = &init_cluster;
rq->curr_runnable_sum = rq->prev_runnable_sum = 0;
rq->nt_curr_runnable_sum = rq->nt_prev_runnable_sum = 0;
rq->old_busy_time = 0;
rq->old_estimated_time = 0;
rq->old_busy_time_group = 0;
rq->hmp_stats.pred_demands_sum = 0;
#endif
INIT_LIST_HEAD(&rq->cfs_tasks);
rq_attach_root(rq, &def_root_domain);
@ -7662,6 +7899,8 @@ void __init sched_init(void)
atomic_set(&rq->nr_iowait, 0);
}
set_hmp_defaults();
set_load_weight(&init_task);
/*
@ -7768,6 +8007,9 @@ void ___might_sleep(const char *file, int line, int preempt_offset)
print_ip_sym(preempt_disable_ip);
pr_cont("\n");
}
#ifdef CONFIG_PANIC_ON_SCHED_BUG
BUG();
#endif
dump_stack();
add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
}
@ -8369,7 +8611,7 @@ int sched_rr_handler(struct ctl_table *table, int write,
#ifdef CONFIG_CGROUP_SCHED
static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
inline struct task_group *css_tg(struct cgroup_subsys_state *css)
{
return css ? container_of(css, struct task_group, css) : NULL;
}
@ -8756,6 +8998,13 @@ static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
#endif /* CONFIG_RT_GROUP_SCHED */
static struct cftype cpu_files[] = {
#ifdef CONFIG_SCHED_HMP
{
.name = "upmigrate_discourage",
.read_u64 = cpu_upmigrate_discourage_read_u64,
.write_u64 = cpu_upmigrate_discourage_write_u64,
},
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
{
.name = "shares",

12
kernel/sched/cputime.c
View file

@ -46,12 +46,15 @@ void irqtime_account_irq(struct task_struct *curr)
struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
s64 delta;
int cpu;
u64 wallclock;
bool account = true;
if (!sched_clock_irqtime)
return;
cpu = smp_processor_id();
delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
wallclock = sched_clock_cpu(cpu);
delta = wallclock - irqtime->irq_start_time;
irqtime->irq_start_time += delta;
u64_stats_update_begin(&irqtime->sync);
@ -65,6 +68,13 @@ void irqtime_account_irq(struct task_struct *curr)
irqtime->hardirq_time += delta;
else if (in_serving_softirq() && curr != this_cpu_ksoftirqd())
irqtime->softirq_time += delta;
else
account = false;
if (account)
sched_account_irqtime(cpu, curr, delta, wallclock);
else if (curr != this_cpu_ksoftirqd())
sched_account_irqstart(cpu, curr, wallclock);
u64_stats_update_end(&irqtime->sync);
}

44
kernel/sched/deadline.c
View file

@ -820,6 +820,41 @@ static inline void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {}
#endif /* CONFIG_SMP */
#ifdef CONFIG_SCHED_HMP
static void
inc_hmp_sched_stats_dl(struct rq *rq, struct task_struct *p)
{
inc_cumulative_runnable_avg(&rq->hmp_stats, p);
}
static void
dec_hmp_sched_stats_dl(struct rq *rq, struct task_struct *p)
{
dec_cumulative_runnable_avg(&rq->hmp_stats, p);
}
static void
fixup_hmp_sched_stats_dl(struct rq *rq, struct task_struct *p,
u32 new_task_load, u32 new_pred_demand)
{
s64 task_load_delta = (s64)new_task_load - task_load(p);
s64 pred_demand_delta = PRED_DEMAND_DELTA;
fixup_cumulative_runnable_avg(&rq->hmp_stats, p, task_load_delta,
pred_demand_delta);
}
#else /* CONFIG_SCHED_HMP */
static inline void
inc_hmp_sched_stats_dl(struct rq *rq, struct task_struct *p) { }
static inline void
dec_hmp_sched_stats_dl(struct rq *rq, struct task_struct *p) { }
#endif /* CONFIG_SCHED_HMP */
static inline
void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
{
@ -829,6 +864,7 @@ void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
WARN_ON(!dl_prio(prio));
dl_rq->dl_nr_running++;
add_nr_running(rq_of_dl_rq(dl_rq), 1);
inc_hmp_sched_stats_dl(rq_of_dl_rq(dl_rq), dl_task_of(dl_se));
inc_dl_deadline(dl_rq, deadline);
inc_dl_migration(dl_se, dl_rq);
@ -843,6 +879,7 @@ void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
WARN_ON(!dl_rq->dl_nr_running);
dl_rq->dl_nr_running--;
sub_nr_running(rq_of_dl_rq(dl_rq), 1);
dec_hmp_sched_stats_dl(rq_of_dl_rq(dl_rq), dl_task_of(dl_se));
dec_dl_deadline(dl_rq, dl_se->deadline);
dec_dl_migration(dl_se, dl_rq);
@ -1500,9 +1537,11 @@ retry:
goto retry;
}
next_task->on_rq = TASK_ON_RQ_MIGRATING;
deactivate_task(rq, next_task, 0);
set_task_cpu(next_task, later_rq->cpu);
activate_task(later_rq, next_task, 0);
next_task->on_rq = TASK_ON_RQ_QUEUED;
ret = 1;
resched_curr(later_rq);
@ -1588,9 +1627,11 @@ static void pull_dl_task(struct rq *this_rq)
resched = true;
p->on_rq = TASK_ON_RQ_MIGRATING;
deactivate_task(src_rq, p, 0);
set_task_cpu(p, this_cpu);
activate_task(this_rq, p, 0);
p->on_rq = TASK_ON_RQ_QUEUED;
dmin = p->dl.deadline;
/* Is there any other task even earlier? */
@ -1803,6 +1844,9 @@ const struct sched_class dl_sched_class = {
.switched_to = switched_to_dl,
.update_curr = update_curr_dl,
#ifdef CONFIG_SCHED_HMP
.fixup_hmp_sched_stats = fixup_hmp_sched_stats_dl,
#endif
};
#ifdef CONFIG_SCHED_DEBUG

43
kernel/sched/debug.c
View file

@ -534,6 +534,14 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
cfs_rq->throttled);
SEQ_printf(m, " .%-30s: %d\n", "throttle_count",
cfs_rq->throttle_count);
SEQ_printf(m, " .%-30s: %d\n", "runtime_enabled",
cfs_rq->runtime_enabled);
#ifdef CONFIG_SCHED_HMP
SEQ_printf(m, " .%-30s: %d\n", "nr_big_tasks",
cfs_rq->hmp_stats.nr_big_tasks);
SEQ_printf(m, " .%-30s: %llu\n", "cumulative_runnable_avg",
cfs_rq->hmp_stats.cumulative_runnable_avg);
#endif
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
@ -622,6 +630,23 @@ do { \
P(cpu_load[2]);
P(cpu_load[3]);
P(cpu_load[4]);
#ifdef CONFIG_SMP
P(cpu_capacity);
#endif
#ifdef CONFIG_SCHED_HMP
P(static_cpu_pwr_cost);
P(cluster->static_cluster_pwr_cost);
P(cluster->load_scale_factor);
P(cluster->capacity);
P(cluster->max_possible_capacity);
P(cluster->efficiency);
P(cluster->cur_freq);
P(cluster->max_freq);
P(cluster->exec_scale_factor);
P(hmp_stats.nr_big_tasks);
SEQ_printf(m, " .%-30s: %llu\n", "hmp_stats.cumulative_runnable_avg",
rq->hmp_stats.cumulative_runnable_avg);
#endif
#undef P
#undef PN
@ -700,6 +725,14 @@ static void sched_debug_header(struct seq_file *m)
PN(sysctl_sched_wakeup_granularity);
P(sysctl_sched_child_runs_first);
P(sysctl_sched_features);
#ifdef CONFIG_SCHED_HMP
P(sched_upmigrate);
P(sched_downmigrate);
P(sched_init_task_load_windows);
P(min_capacity);
P(max_capacity);
P(sched_ravg_window);
#endif
#undef PN
#undef P
@ -861,6 +894,9 @@ static void sched_show_numa(struct task_struct *p, struct seq_file *m)
void proc_sched_show_task(struct task_struct *p, struct seq_file *m)
{
unsigned long nr_switches;
unsigned int load_avg;
load_avg = pct_task_load(p);
SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr(p),
get_nr_threads(p));
@ -918,7 +954,12 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m)
P_SCHEDSTAT(se.statistics.nr_wakeups_affine_attempts);
P_SCHEDSTAT(se.statistics.nr_wakeups_passive);
P_SCHEDSTAT(se.statistics.nr_wakeups_idle);
#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED)
__P(load_avg);
#ifdef CONFIG_SCHED_HMP
P(ravg.demand);
#endif
#endif
avg_atom = p->se.sum_exec_runtime;
if (nr_switches)
avg_atom = div64_ul(avg_atom, nr_switches);

1515
kernel/sched/fair.c
View file

File diff suppressed because it is too large Load diff

2
kernel/sched/features.h
View file

@ -49,7 +49,7 @@ SCHED_FEAT(NONTASK_CAPACITY, true)
* Queue remote wakeups on the target CPU and process them
* using the scheduler IPI. Reduces rq->lock contention/bounces.
*/
SCHED_FEAT(TTWU_QUEUE, true)
SCHED_FEAT(TTWU_QUEUE, false)
#ifdef HAVE_RT_PUSH_IPI
/*

3975
kernel/sched/hmp.c Normal file
View file

File diff suppressed because it is too large Load diff

11
kernel/sched/idle_task.c
View file

@ -78,6 +78,14 @@ static void update_curr_idle(struct rq *rq)
{
}
#ifdef CONFIG_SCHED_HMP
static void
fixup_hmp_sched_stats_idle(struct rq *rq, struct task_struct *p,
u32 new_task_load, u32 new_pred_demand)
{
}
#endif
/*
* Simple, special scheduling class for the per-CPU idle tasks:
*/
@ -106,4 +114,7 @@ const struct sched_class idle_sched_class = {
.prio_changed = prio_changed_idle,
.switched_to = switched_to_idle,
.update_curr = update_curr_idle,
#ifdef CONFIG_SCHED_HMP
.fixup_hmp_sched_stats = fixup_hmp_sched_stats_idle,
#endif
};

181
kernel/sched/rt.c
View file

@ -7,6 +7,7 @@
#include <linux/slab.h>
#include <linux/irq_work.h>
#include <trace/events/sched.h>
int sched_rr_timeslice = RR_TIMESLICE;
@ -897,6 +898,51 @@ static inline int rt_se_prio(struct sched_rt_entity *rt_se)
return rt_task_of(rt_se)->prio;
}
static void dump_throttled_rt_tasks(struct rt_rq *rt_rq)
{
struct rt_prio_array *array = &rt_rq->active;
struct sched_rt_entity *rt_se;
char buf[500];
char *pos = buf;
char *end = buf + sizeof(buf);
int idx;
pos += snprintf(pos, sizeof(buf),
"sched: RT throttling activated for rt_rq %p (cpu %d)\n",
rt_rq, cpu_of(rq_of_rt_rq(rt_rq)));
if (bitmap_empty(array->bitmap, MAX_RT_PRIO))
goto out;
pos += snprintf(pos, end - pos, "potential CPU hogs:\n");
idx = sched_find_first_bit(array->bitmap);
while (idx < MAX_RT_PRIO) {
list_for_each_entry(rt_se, array->queue + idx, run_list) {
struct task_struct *p;
if (!rt_entity_is_task(rt_se))
continue;
p = rt_task_of(rt_se);
if (pos < end)
pos += snprintf(pos, end - pos, "\t%s (%d)\n",
p->comm, p->pid);
}
idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx + 1);
}
out:
#ifdef CONFIG_PANIC_ON_RT_THROTTLING
/*
* Use pr_err() in the BUG() case since printk_sched() will
* not get flushed and deadlock is not a concern.
*/
pr_err("%s", buf);
BUG();
#else
printk_deferred("%s", buf);
#endif
}
static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
{
u64 runtime = sched_rt_runtime(rt_rq);
@ -920,8 +966,14 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
* but accrue some time due to boosting.
*/
if (likely(rt_b->rt_runtime)) {
static bool once;
rt_rq->rt_throttled = 1;
printk_deferred_once("sched: RT throttling activated\n");
if (!once) {
once = true;
dump_throttled_rt_tasks(rt_rq);
}
} else {
/*
* In case we did anyway, make it go away,
@ -1141,6 +1193,41 @@ void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}
#endif /* CONFIG_RT_GROUP_SCHED */
#ifdef CONFIG_SCHED_HMP
static void
inc_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p)
{
inc_cumulative_runnable_avg(&rq->hmp_stats, p);
}
static void
dec_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p)
{
dec_cumulative_runnable_avg(&rq->hmp_stats, p);
}
static void
fixup_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p,
u32 new_task_load, u32 new_pred_demand)
{
s64 task_load_delta = (s64)new_task_load - task_load(p);
s64 pred_demand_delta = PRED_DEMAND_DELTA;
fixup_cumulative_runnable_avg(&rq->hmp_stats, p, task_load_delta,
pred_demand_delta);
}
#else /* CONFIG_SCHED_HMP */
static inline void
inc_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) { }
static inline void
dec_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) { }
#endif /* CONFIG_SCHED_HMP */
static inline
unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se)
{
@ -1322,6 +1409,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
rt_se->timeout = 0;
enqueue_rt_entity(rt_se, flags);
inc_hmp_sched_stats_rt(rq, p);
if (!task_current(rq, p) && tsk_nr_cpus_allowed(p) > 1)
enqueue_pushable_task(rq, p);
@ -1333,6 +1421,7 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
update_curr_rt(rq);
dequeue_rt_entity(rt_se, flags);
dec_hmp_sched_stats_rt(rq, p);
dequeue_pushable_task(rq, p);
}
@ -1374,12 +1463,32 @@ static void yield_task_rt(struct rq *rq)
#ifdef CONFIG_SMP
static int find_lowest_rq(struct task_struct *task);
#ifdef CONFIG_SCHED_HMP
static int
select_task_rq_rt_hmp(struct task_struct *p, int cpu, int sd_flag, int flags)
{
int target;
rcu_read_lock();
target = find_lowest_rq(p);
if (target != -1)
cpu = target;
rcu_read_unlock();
return cpu;
}
#endif /* CONFIG_SCHED_HMP */
static int
select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags)
{
struct task_struct *curr;
struct rq *rq;
#ifdef CONFIG_SCHED_HMP
return select_task_rq_rt_hmp(p, cpu, sd_flag, flags);
#endif
/* For anything but wake ups, just return the task_cpu */
if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK)
goto out;
@ -1617,6 +1726,65 @@ static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
#ifdef CONFIG_SCHED_HMP
static int find_lowest_rq_hmp(struct task_struct *task)
{
struct cpumask *lowest_mask = *this_cpu_ptr(&local_cpu_mask);
struct cpumask candidate_mask = CPU_MASK_NONE;
struct sched_cluster *cluster;
int best_cpu = -1;
int prev_cpu = task_cpu(task);
u64 cpu_load, min_load = ULLONG_MAX;
int i;
int restrict_cluster = sched_boost() ? 0 :
sysctl_sched_restrict_cluster_spill;
/* Make sure the mask is initialized first */
if (unlikely(!lowest_mask))
return best_cpu;
if (task->nr_cpus_allowed == 1)
return best_cpu; /* No other targets possible */
if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask))
return best_cpu; /* No targets found */
/*
* At this point we have built a mask of cpus representing the
* lowest priority tasks in the system. Now we want to elect
* the best one based on our affinity and topology.
*/
for_each_sched_cluster(cluster) {
cpumask_and(&candidate_mask, &cluster->cpus, lowest_mask);
if (cpumask_empty(&candidate_mask))
continue;
for_each_cpu(i, &candidate_mask) {
if (sched_cpu_high_irqload(i))
continue;
cpu_load = cpu_rq(i)->hmp_stats.cumulative_runnable_avg;
if (!restrict_cluster)
cpu_load = scale_load_to_cpu(cpu_load, i);
if (cpu_load < min_load ||
(cpu_load == min_load &&
(i == prev_cpu || (best_cpu != prev_cpu &&
cpus_share_cache(prev_cpu, i))))) {
min_load = cpu_load;
best_cpu = i;
}
}
if (restrict_cluster && best_cpu != -1)
break;
}
return best_cpu;
}
#endif /* CONFIG_SCHED_HMP */
static int find_lowest_rq(struct task_struct *task)
{
struct sched_domain *sd;
@ -1624,6 +1792,10 @@ static int find_lowest_rq(struct task_struct *task)
int this_cpu = smp_processor_id();
int cpu = task_cpu(task);
#ifdef CONFIG_SCHED_HMP
return find_lowest_rq_hmp(task);
#endif
/* Make sure the mask is initialized first */
if (unlikely(!lowest_mask))
return -1;
@ -1841,9 +2013,11 @@ retry:
goto retry;
}
next_task->on_rq = TASK_ON_RQ_MIGRATING;
deactivate_task(rq, next_task, 0);
set_task_cpu(next_task, lowest_rq->cpu);
activate_task(lowest_rq, next_task, 0);
next_task->on_rq = TASK_ON_RQ_QUEUED;
ret = 1;
resched_curr(lowest_rq);
@ -2095,9 +2269,11 @@ static void pull_rt_task(struct rq *this_rq)
resched = true;
p->on_rq = TASK_ON_RQ_MIGRATING;
deactivate_task(src_rq, p, 0);
set_task_cpu(p, this_cpu);
activate_task(this_rq, p, 0);
p->on_rq = TASK_ON_RQ_QUEUED;
/*
* We continue with the search, just in
* case there's an even higher prio task
@ -2352,6 +2528,9 @@ const struct sched_class rt_sched_class = {
.switched_to = switched_to_rt,
.update_curr = update_curr_rt,
#ifdef CONFIG_SCHED_HMP
.fixup_hmp_sched_stats = fixup_hmp_sched_stats_rt,
#endif
};
#ifdef CONFIG_SCHED_DEBUG

690
kernel/sched/sched.h
View file

@ -255,6 +255,10 @@ struct cfs_bandwidth {
struct task_group {
struct cgroup_subsys_state css;
#ifdef CONFIG_SCHED_HMP
bool upmigrate_discouraged;
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
/* schedulable entities of this group on each cpu */
struct sched_entity **se;
@ -365,12 +369,83 @@ static inline void set_task_rq_fair(struct sched_entity *se,
#endif /* CONFIG_SMP */
#endif /* CONFIG_FAIR_GROUP_SCHED */
extern struct task_group *css_tg(struct cgroup_subsys_state *css);
#else /* CONFIG_CGROUP_SCHED */
struct cfs_bandwidth { };
#endif /* CONFIG_CGROUP_SCHED */
#ifdef CONFIG_SCHED_HMP
struct hmp_sched_stats {
int nr_big_tasks;
u64 cumulative_runnable_avg;
u64 pred_demands_sum;
};
struct sched_cluster {
struct list_head list;
struct cpumask cpus;
int id;
int max_power_cost;
int min_power_cost;
int max_possible_capacity;
int capacity;
int efficiency; /* Differentiate cpus with different IPC capability */
int load_scale_factor;
unsigned int exec_scale_factor;
/*
* max_freq = user maximum
* max_mitigated_freq = thermal defined maximum
* max_possible_freq = maximum supported by hardware
*/
unsigned int cur_freq, max_freq, max_mitigated_freq, min_freq;
unsigned int max_possible_freq;
bool freq_init_done;
int dstate, dstate_wakeup_latency, dstate_wakeup_energy;
unsigned int static_cluster_pwr_cost;
int notifier_sent;
};
extern unsigned long all_cluster_ids[];
static inline int cluster_first_cpu(struct sched_cluster *cluster)
{
return cpumask_first(&cluster->cpus);
}
struct related_thread_group {
int id;
raw_spinlock_t lock;
struct list_head tasks;
struct list_head list;
struct sched_cluster *preferred_cluster;
struct rcu_head rcu;
u64 last_update;
struct group_cpu_time __percpu *cpu_time; /* one per cluster */
};
struct migration_sum_data {
struct rq *src_rq, *dst_rq;
struct group_cpu_time *src_cpu_time, *dst_cpu_time;
};
extern struct list_head cluster_head;
extern int num_clusters;
extern struct sched_cluster *sched_cluster[NR_CPUS];
struct cpu_cycle {
u64 cycles;
u64 time;
};
#define for_each_sched_cluster(cluster) \
list_for_each_entry_rcu(cluster, &cluster_head, list)
#endif /* CONFIG_SCHED_HMP */
/* CFS-related fields in a runqueue */
struct cfs_rq {
struct load_weight load;
@ -439,6 +514,10 @@ struct cfs_rq {
struct task_group *tg; /* group that "owns" this runqueue */
#ifdef CONFIG_CFS_BANDWIDTH
#ifdef CONFIG_SCHED_HMP
struct hmp_sched_stats hmp_stats;
#endif
int runtime_enabled;
u64 runtime_expires;
s64 runtime_remaining;
@ -656,6 +735,7 @@ struct rq {
/* For active balancing */
int active_balance;
int push_cpu;
struct task_struct *push_task;
struct cpu_stop_work active_balance_work;
/* cpu of this runqueue: */
int cpu;
@ -672,6 +752,29 @@ struct rq {
u64 max_idle_balance_cost;
#endif
#ifdef CONFIG_SCHED_HMP
struct sched_cluster *cluster;
struct cpumask freq_domain_cpumask;
struct hmp_sched_stats hmp_stats;
int cstate, wakeup_latency, wakeup_energy;
u64 window_start;
unsigned long hmp_flags;
u64 cur_irqload;
u64 avg_irqload;
u64 irqload_ts;
unsigned int static_cpu_pwr_cost;
struct task_struct *ed_task;
struct cpu_cycle cc;
u64 old_busy_time, old_busy_time_group;
u64 old_estimated_time;
u64 curr_runnable_sum;
u64 prev_runnable_sum;
u64 nt_curr_runnable_sum;
u64 nt_prev_runnable_sum;
#endif
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
u64 prev_irq_time;
#endif
@ -962,6 +1065,583 @@ static inline void sched_ttwu_pending(void) { }
#include "stats.h"
#include "auto_group.h"
#ifdef CONFIG_SCHED_HMP
#define WINDOW_STATS_RECENT 0
#define WINDOW_STATS_MAX 1
#define WINDOW_STATS_MAX_RECENT_AVG 2
#define WINDOW_STATS_AVG 3
#define WINDOW_STATS_INVALID_POLICY 4
#define MAJOR_TASK_PCT 85
#define SCHED_UPMIGRATE_MIN_NICE 15
#define EXITING_TASK_MARKER 0xdeaddead
#define UP_MIGRATION 1
#define DOWN_MIGRATION 2
#define IRQLOAD_MIGRATION 3
extern struct mutex policy_mutex;
extern unsigned int sched_ravg_window;
extern unsigned int sched_disable_window_stats;
extern unsigned int max_possible_freq;
extern unsigned int min_max_freq;
extern unsigned int pct_task_load(struct task_struct *p);
extern unsigned int max_possible_efficiency;
extern unsigned int min_possible_efficiency;
extern unsigned int max_capacity;
extern unsigned int min_capacity;
extern unsigned int max_load_scale_factor;
extern unsigned int max_possible_capacity;
extern unsigned int min_max_possible_capacity;
extern unsigned int sched_upmigrate;
extern unsigned int sched_downmigrate;
extern unsigned int sched_init_task_load_windows;
extern unsigned int up_down_migrate_scale_factor;
extern unsigned int sysctl_sched_restrict_cluster_spill;
extern unsigned int sched_pred_alert_load;
extern unsigned int sched_major_task_runtime;
extern struct sched_cluster init_cluster;
extern unsigned int __read_mostly sched_short_sleep_task_threshold;
extern unsigned int __read_mostly sched_long_cpu_selection_threshold;
extern unsigned int __read_mostly sched_big_waker_task_load;
extern unsigned int __read_mostly sched_small_wakee_task_load;
extern unsigned int __read_mostly sched_spill_load;
extern unsigned int __read_mostly sched_upmigrate;
extern unsigned int __read_mostly sched_downmigrate;
extern unsigned int __read_mostly sysctl_sched_spill_nr_run;
extern void init_new_task_load(struct task_struct *p);
extern u64 sched_ktime_clock(void);
extern int got_boost_kick(void);
extern int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb);
extern void update_task_ravg(struct task_struct *p, struct rq *rq, int event,
u64 wallclock, u64 irqtime);
extern bool early_detection_notify(struct rq *rq, u64 wallclock);
extern void clear_ed_task(struct task_struct *p, struct rq *rq);
extern void fixup_busy_time(struct task_struct *p, int new_cpu);
extern void clear_boost_kick(int cpu);
extern void clear_hmp_request(int cpu);
extern void mark_task_starting(struct task_struct *p);
extern void set_window_start(struct rq *rq);
extern void migrate_sync_cpu(int cpu);
extern void update_cluster_topology(void);
extern void set_task_last_wake(struct task_struct *p, u64 wallclock);
extern void set_task_last_switch_out(struct task_struct *p, u64 wallclock);
extern void init_clusters(void);
extern void reset_cpu_hmp_stats(int cpu, int reset_cra);
extern unsigned int max_task_load(void);
extern void sched_account_irqtime(int cpu, struct task_struct *curr,
u64 delta, u64 wallclock);
extern void sched_account_irqstart(int cpu, struct task_struct *curr,
u64 wallclock);
extern unsigned int cpu_temp(int cpu);
extern unsigned int nr_eligible_big_tasks(int cpu);
extern void update_up_down_migrate(void);
extern int update_preferred_cluster(struct related_thread_group *grp,
struct task_struct *p, u32 old_load);
extern void set_preferred_cluster(struct related_thread_group *grp);
extern void add_new_task_to_grp(struct task_struct *new);
enum sched_boost_type {
SCHED_BOOST_NONE,
SCHED_BOOST_ON_BIG,
SCHED_BOOST_ON_ALL,
};
static inline struct sched_cluster *cpu_cluster(int cpu)
{
return cpu_rq(cpu)->cluster;
}
static inline int cpu_capacity(int cpu)
{
return cpu_rq(cpu)->cluster->capacity;
}
static inline int cpu_max_possible_capacity(int cpu)
{
return cpu_rq(cpu)->cluster->max_possible_capacity;
}
static inline int cpu_load_scale_factor(int cpu)
{
return cpu_rq(cpu)->cluster->load_scale_factor;
}
static inline int cpu_efficiency(int cpu)
{
return cpu_rq(cpu)->cluster->efficiency;
}
static inline unsigned int cpu_cur_freq(int cpu)
{
return cpu_rq(cpu)->cluster->cur_freq;
}
static inline unsigned int cpu_min_freq(int cpu)
{
return cpu_rq(cpu)->cluster->min_freq;
}
static inline unsigned int cluster_max_freq(struct sched_cluster *cluster)
{
/*
* Governor and thermal driver don't know the other party's mitigation
* voting. So struct cluster saves both and return min() for current
* cluster fmax.
*/
return min(cluster->max_mitigated_freq, cluster->max_freq);
}
static inline unsigned int cpu_max_freq(int cpu)
{
return cluster_max_freq(cpu_rq(cpu)->cluster);
}
static inline unsigned int cpu_max_possible_freq(int cpu)
{
return cpu_rq(cpu)->cluster->max_possible_freq;
}
static inline int same_cluster(int src_cpu, int dst_cpu)
{
return cpu_rq(src_cpu)->cluster == cpu_rq(dst_cpu)->cluster;
}
static inline int cpu_max_power_cost(int cpu)
{
return cpu_rq(cpu)->cluster->max_power_cost;
}
static inline u32 cpu_cycles_to_freq(u64 cycles, u32 period)
{
return div64_u64(cycles, period);
}
static inline bool hmp_capable(void)
{
return max_possible_capacity != min_max_possible_capacity;
}
/*
* 'load' is in reference to "best cpu" at its best frequency.
* Scale that in reference to a given cpu, accounting for how bad it is
* in reference to "best cpu".
*/
static inline u64 scale_load_to_cpu(u64 task_load, int cpu)
{
u64 lsf = cpu_load_scale_factor(cpu);
if (lsf != 1024) {
task_load *= lsf;
task_load /= 1024;
}
return task_load;
}
static inline unsigned int task_load(struct task_struct *p)
{
return p->ravg.demand;
}
static inline void
inc_cumulative_runnable_avg(struct hmp_sched_stats *stats,
struct task_struct *p)
{
u32 task_load;
if (sched_disable_window_stats)
return;
task_load = sched_disable_window_stats ? 0 : p->ravg.demand;
stats->cumulative_runnable_avg += task_load;
stats->pred_demands_sum += p->ravg.pred_demand;
}
static inline void
dec_cumulative_runnable_avg(struct hmp_sched_stats *stats,
struct task_struct *p)
{
u32 task_load;
if (sched_disable_window_stats)
return;
task_load = sched_disable_window_stats ? 0 : p->ravg.demand;
stats->cumulative_runnable_avg -= task_load;
BUG_ON((s64)stats->cumulative_runnable_avg < 0);
stats->pred_demands_sum -= p->ravg.pred_demand;
BUG_ON((s64)stats->pred_demands_sum < 0);
}
static inline void
fixup_cumulative_runnable_avg(struct hmp_sched_stats *stats,
struct task_struct *p, s64 task_load_delta,
s64 pred_demand_delta)
{
if (sched_disable_window_stats)
return;
stats->cumulative_runnable_avg += task_load_delta;
BUG_ON((s64)stats->cumulative_runnable_avg < 0);
stats->pred_demands_sum += pred_demand_delta;
BUG_ON((s64)stats->pred_demands_sum < 0);
}
#define pct_to_real(tunable) \
(div64_u64((u64)tunable * (u64)max_task_load(), 100))
#define real_to_pct(tunable) \
(div64_u64((u64)tunable * (u64)100, (u64)max_task_load()))
#define SCHED_HIGH_IRQ_TIMEOUT 3
static inline u64 sched_irqload(int cpu)
{
struct rq *rq = cpu_rq(cpu);
s64 delta;
delta = get_jiffies_64() - rq->irqload_ts;
/*
* Current context can be preempted by irq and rq->irqload_ts can be
* updated by irq context so that delta can be negative.
* But this is okay and we can safely return as this means there
* was recent irq occurrence.
*/
if (delta < SCHED_HIGH_IRQ_TIMEOUT)
return rq->avg_irqload;
else
return 0;
}
static inline int sched_cpu_high_irqload(int cpu)
{
return sched_irqload(cpu) >= sysctl_sched_cpu_high_irqload;
}
static inline bool task_in_related_thread_group(struct task_struct *p)
{
return !!(rcu_access_pointer(p->grp) != NULL);
}
static inline
struct related_thread_group *task_related_thread_group(struct task_struct *p)
{
return rcu_dereference(p->grp);
}
#define PRED_DEMAND_DELTA ((s64)new_pred_demand - p->ravg.pred_demand)
extern void
check_for_freq_change(struct rq *rq, bool check_pred, bool check_groups);
extern void notify_migration(int src_cpu, int dest_cpu,
bool src_cpu_dead, struct task_struct *p);
struct group_cpu_time {
u64 curr_runnable_sum;
u64 prev_runnable_sum;
u64 nt_curr_runnable_sum;
u64 nt_prev_runnable_sum;
u64 window_start;
};
/* Is frequency of two cpus synchronized with each other? */
static inline int same_freq_domain(int src_cpu, int dst_cpu)
{
struct rq *rq = cpu_rq(src_cpu);
if (src_cpu == dst_cpu)
return 1;
return cpumask_test_cpu(dst_cpu, &rq->freq_domain_cpumask);
}
#define BOOST_KICK 0
#define CPU_RESERVED 1
static inline int is_reserved(int cpu)
{
struct rq *rq = cpu_rq(cpu);
return test_bit(CPU_RESERVED, &rq->hmp_flags);
}
static inline int mark_reserved(int cpu)
{
struct rq *rq = cpu_rq(cpu);
/* Name boost_flags as hmp_flags? */
return test_and_set_bit(CPU_RESERVED, &rq->hmp_flags);
}
static inline void clear_reserved(int cpu)
{
struct rq *rq = cpu_rq(cpu);
clear_bit(CPU_RESERVED, &rq->hmp_flags);
}
static inline u64 cpu_cravg_sync(int cpu, int sync)
{
struct rq *rq = cpu_rq(cpu);
u64 load;
load = rq->hmp_stats.cumulative_runnable_avg;
/*
* If load is being checked in a sync wakeup environment,
* we may want to discount the load of the currently running
* task.
*/
if (sync && cpu == smp_processor_id()) {
if (load > rq->curr->ravg.demand)
load -= rq->curr->ravg.demand;
else
load = 0;
}
return load;
}
extern void check_for_migration(struct rq *rq, struct task_struct *p);
extern void pre_big_task_count_change(const struct cpumask *cpus);
extern void post_big_task_count_change(const struct cpumask *cpus);
extern void set_hmp_defaults(void);
extern int power_delta_exceeded(unsigned int cpu_cost, unsigned int base_cost);
extern unsigned int power_cost(int cpu, u64 demand);
extern void reset_all_window_stats(u64 window_start, unsigned int window_size);
extern void boost_kick(int cpu);
extern int sched_boost(void);
extern int task_load_will_fit(struct task_struct *p, u64 task_load, int cpu,
enum sched_boost_type boost_type);
extern enum sched_boost_type sched_boost_type(void);
extern int task_will_fit(struct task_struct *p, int cpu);
extern int group_will_fit(struct sched_cluster *cluster,
struct related_thread_group *grp, u64 demand);
extern u64 cpu_load(int cpu);
extern u64 cpu_load_sync(int cpu, int sync);
extern int preferred_cluster(struct sched_cluster *cluster,
struct task_struct *p);
extern void inc_nr_big_task(struct hmp_sched_stats *stats,
struct task_struct *p);
extern void dec_nr_big_task(struct hmp_sched_stats *stats,
struct task_struct *p);
extern void inc_rq_hmp_stats(struct rq *rq,
struct task_struct *p, int change_cra);
extern void dec_rq_hmp_stats(struct rq *rq,
struct task_struct *p, int change_cra);
extern int is_big_task(struct task_struct *p);
extern int upmigrate_discouraged(struct task_struct *p);
extern struct sched_cluster *rq_cluster(struct rq *rq);
extern int nr_big_tasks(struct rq *rq);
extern void fixup_nr_big_tasks(struct hmp_sched_stats *stats,
struct task_struct *p, s64 delta);
extern void reset_task_stats(struct task_struct *p);
extern void reset_cfs_rq_hmp_stats(int cpu, int reset_cra);
extern void inc_hmp_sched_stats_fair(struct rq *rq,
struct task_struct *p, int change_cra);
extern u64 cpu_upmigrate_discourage_read_u64(struct cgroup_subsys_state *css,
struct cftype *cft);
extern int cpu_upmigrate_discourage_write_u64(struct cgroup_subsys_state *css,
struct cftype *cft, u64 upmigrate_discourage);
#else /* CONFIG_SCHED_HMP */
struct hmp_sched_stats;
struct related_thread_group;
struct sched_cluster;
static inline int got_boost_kick(void)
{
return 0;
}
static inline void update_task_ravg(struct task_struct *p, struct rq *rq,
int event, u64 wallclock, u64 irqtime) { }
static inline bool early_detection_notify(struct rq *rq, u64 wallclock)
{
return 0;
}
static inline void clear_ed_task(struct task_struct *p, struct rq *rq) { }
static inline void fixup_busy_time(struct task_struct *p, int new_cpu) { }
static inline void clear_boost_kick(int cpu) { }
static inline void clear_hmp_request(int cpu) { }
static inline void mark_task_starting(struct task_struct *p) { }
static inline void set_window_start(struct rq *rq) { }
static inline void migrate_sync_cpu(int cpu) { }
static inline void update_cluster_topology(void) { }
static inline void set_task_last_wake(struct task_struct *p, u64 wallclock) { }
static inline void set_task_last_switch_out(struct task_struct *p,
u64 wallclock) { }
static inline int task_will_fit(struct task_struct *p, int cpu)
{
return 1;
}
static inline unsigned int power_cost(int cpu, u64 demand)
{
return SCHED_CAPACITY_SCALE;
}
static inline int sched_boost(void)
{
return 0;
}
static inline int is_big_task(struct task_struct *p)
{
return 0;
}
static inline int nr_big_tasks(struct rq *rq)
{
return 0;
}
static inline int is_cpu_throttling_imminent(int cpu)
{
return 0;
}
static inline int is_task_migration_throttled(struct task_struct *p)
{
return 0;
}
static inline unsigned int cpu_temp(int cpu)
{
return 0;
}
static inline void
inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { }
static inline void
dec_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { }
static inline int
preferred_cluster(struct sched_cluster *cluster, struct task_struct *p)
{
return 1;
}
static inline struct sched_cluster *rq_cluster(struct rq *rq)
{
return NULL;
}
static inline void init_new_task_load(struct task_struct *p) { }
static inline u64 scale_load_to_cpu(u64 load, int cpu)
{
return load;
}
static inline unsigned int nr_eligible_big_tasks(int cpu)
{
return 0;
}
static inline int pct_task_load(struct task_struct *p) { return 0; }
static inline int cpu_capacity(int cpu)
{
return SCHED_CAPACITY_SCALE;
}
static inline int same_cluster(int src_cpu, int dst_cpu) { return 1; }
static inline void inc_cumulative_runnable_avg(struct hmp_sched_stats *stats,
struct task_struct *p)
{
}
static inline void dec_cumulative_runnable_avg(struct hmp_sched_stats *stats,
struct task_struct *p)
{
}
static inline void sched_account_irqtime(int cpu, struct task_struct *curr,
u64 delta, u64 wallclock)
{
}
static inline void sched_account_irqstart(int cpu, struct task_struct *curr,
u64 wallclock)
{
}
static inline int sched_cpu_high_irqload(int cpu) { return 0; }
static inline void set_preferred_cluster(struct related_thread_group *grp) { }
static inline bool task_in_related_thread_group(struct task_struct *p)
{
return false;
}
static inline
struct related_thread_group *task_related_thread_group(struct task_struct *p)
{
return NULL;
}
static inline u32 task_load(struct task_struct *p) { return 0; }
static inline int update_preferred_cluster(struct related_thread_group *grp,
struct task_struct *p, u32 old_load)
{
return 0;
}
static inline void add_new_task_to_grp(struct task_struct *new) {}
#define sched_freq_legacy_mode 1
#define sched_migration_fixup 0
#define PRED_DEMAND_DELTA (0)
static inline void
check_for_freq_change(struct rq *rq, bool check_pred, bool check_groups) { }
static inline void notify_migration(int src_cpu, int dest_cpu,
bool src_cpu_dead, struct task_struct *p) { }
static inline int same_freq_domain(int src_cpu, int dst_cpu)
{
return 1;
}
static inline void check_for_migration(struct rq *rq, struct task_struct *p) { }
static inline void pre_big_task_count_change(void) { }
static inline void post_big_task_count_change(void) { }
static inline void set_hmp_defaults(void) { }
static inline void clear_reserved(int cpu) { }
#define trace_sched_cpu_load(...)
#define trace_sched_cpu_load_lb(...)
#define trace_sched_cpu_load_cgroup(...)
#define trace_sched_cpu_load_wakeup(...)
#endif /* CONFIG_SCHED_HMP */
/*
* Returns the rq capacity of any rq in a group. This does not play
* well with groups where rq capacity can change independently.
*/
#define group_rq_capacity(group) cpu_capacity(group_first_cpu(group))
#ifdef CONFIG_CGROUP_SCHED
/*
@ -1163,6 +1843,7 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
#define WF_SYNC 0x01 /* waker goes to sleep after wakeup */
#define WF_FORK 0x02 /* child wakeup after fork */
#define WF_MIGRATED 0x4 /* internal use, task got migrated */
#define WF_NO_NOTIFIER 0x08 /* do not notify governor */
/*
* To aid in avoiding the subversion of "niceness" due to uneven distribution
@ -1278,6 +1959,10 @@ struct sched_class {
#ifdef CONFIG_FAIR_GROUP_SCHED
void (*task_change_group) (struct task_struct *p, int type);
#endif
#ifdef CONFIG_SCHED_HMP
void (*fixup_hmp_sched_stats)(struct rq *rq, struct task_struct *p,
u32 new_task_load, u32 new_pred_demand);
#endif
};
static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
@ -1391,6 +2076,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count)
{
unsigned prev_nr = rq->nr_running;
sched_update_nr_prod(cpu_of(rq), count, true);
rq->nr_running = prev_nr + count;
if (prev_nr < 2 && rq->nr_running >= 2) {
@ -1405,6 +2091,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count)
static inline void sub_nr_running(struct rq *rq, unsigned count)
{
sched_update_nr_prod(cpu_of(rq), count, false);
rq->nr_running -= count;
/* Check if we still need preemption */
sched_update_tick_dependency(rq);
@ -1732,6 +2419,9 @@ enum rq_nohz_flag_bits {
NOHZ_BALANCE_KICK,
};
#define NOHZ_KICK_ANY 0
#define NOHZ_KICK_RESTRICT 1
#define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags)
extern void nohz_balance_exit_idle(unsigned int cpu);

128
kernel/sched/sched_avg.c Normal file
View file

@ -0,0 +1,128 @@
/* Copyright (c) 2012, 2015, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
* only version 2 as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*/
/*
* Scheduler hook for average runqueue determination
*/
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/hrtimer.h>
#include <linux/sched.h>
#include <linux/math64.h>
#include "sched.h"
#include <trace/events/sched.h>
static DEFINE_PER_CPU(u64, nr_prod_sum);
static DEFINE_PER_CPU(u64, last_time);
static DEFINE_PER_CPU(u64, nr_big_prod_sum);
static DEFINE_PER_CPU(u64, nr);
static DEFINE_PER_CPU(unsigned long, iowait_prod_sum);
static DEFINE_PER_CPU(spinlock_t, nr_lock) = __SPIN_LOCK_UNLOCKED(nr_lock);
static s64 last_get_time;
/**
* sched_get_nr_running_avg
* @return: Average nr_running, iowait and nr_big_tasks value since last poll.
* Returns the avg * 100 to return up to two decimal points
* of accuracy.
*
* Obtains the average nr_running value since the last poll.
* This function may not be called concurrently with itself
*/
void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg)
{
int cpu;
u64 curr_time = sched_clock();
u64 diff = curr_time - last_get_time;
u64 tmp_avg = 0, tmp_iowait = 0, tmp_big_avg = 0;
*avg = 0;
*iowait_avg = 0;
*big_avg = 0;
if (!diff)
return;
/* read and reset nr_running counts */
for_each_possible_cpu(cpu) {
unsigned long flags;
spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags);
curr_time = sched_clock();
tmp_avg += per_cpu(nr_prod_sum, cpu);
tmp_avg += per_cpu(nr, cpu) *
(curr_time - per_cpu(last_time, cpu));
tmp_big_avg += per_cpu(nr_big_prod_sum, cpu);
tmp_big_avg += nr_eligible_big_tasks(cpu) *
(curr_time - per_cpu(last_time, cpu));
tmp_iowait += per_cpu(iowait_prod_sum, cpu);
tmp_iowait += nr_iowait_cpu(cpu) *
(curr_time - per_cpu(last_time, cpu));
per_cpu(last_time, cpu) = curr_time;
per_cpu(nr_prod_sum, cpu) = 0;
per_cpu(nr_big_prod_sum, cpu) = 0;
per_cpu(iowait_prod_sum, cpu) = 0;
spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags);
}
diff = curr_time - last_get_time;
last_get_time = curr_time;
*avg = (int)div64_u64(tmp_avg * 100, diff);
*big_avg = (int)div64_u64(tmp_big_avg * 100, diff);
*iowait_avg = (int)div64_u64(tmp_iowait * 100, diff);
trace_sched_get_nr_running_avg(*avg, *big_avg, *iowait_avg);
BUG_ON(*avg < 0 || *big_avg < 0 || *iowait_avg < 0);
pr_debug("%s - avg:%d big_avg:%d iowait_avg:%d\n",
__func__, *avg, *big_avg, *iowait_avg);
}
EXPORT_SYMBOL(sched_get_nr_running_avg);
/**
* sched_update_nr_prod
* @cpu: The core id of the nr running driver.
* @delta: Adjust nr by 'delta' amount
* @inc: Whether we are increasing or decreasing the count
* @return: N/A
*
* Update average with latest nr_running value for CPU
*/
void sched_update_nr_prod(int cpu, long delta, bool inc)
{
int diff;
s64 curr_time;
unsigned long flags, nr_running;
spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags);
nr_running = per_cpu(nr, cpu);
curr_time = sched_clock();
diff = curr_time - per_cpu(last_time, cpu);
per_cpu(last_time, cpu) = curr_time;
per_cpu(nr, cpu) = nr_running + (inc ? delta : -delta);
BUG_ON((s64)per_cpu(nr, cpu) < 0);
per_cpu(nr_prod_sum, cpu) += nr_running * diff;
per_cpu(nr_big_prod_sum, cpu) += nr_eligible_big_tasks(cpu) * diff;
per_cpu(iowait_prod_sum, cpu) += nr_iowait_cpu(cpu) * diff;
spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags);
}
EXPORT_SYMBOL(sched_update_nr_prod);

40
kernel/sched/stop_task.c
View file

@ -17,6 +17,41 @@ select_task_rq_stop(struct task_struct *p, int cpu, int sd_flag, int flags)
}
#endif /* CONFIG_SMP */
#ifdef CONFIG_SCHED_HMP
static void
inc_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p)
{
inc_cumulative_runnable_avg(&rq->hmp_stats, p);
}
static void
dec_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p)
{
dec_cumulative_runnable_avg(&rq->hmp_stats, p);
}
static void
fixup_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p,
u32 new_task_load, u32 new_pred_demand)
{
s64 task_load_delta = (s64)new_task_load - task_load(p);
s64 pred_demand_delta = PRED_DEMAND_DELTA;
fixup_cumulative_runnable_avg(&rq->hmp_stats, p, task_load_delta,
pred_demand_delta);
}
#else /* CONFIG_SCHED_HMP */
static inline void
inc_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) { }
static inline void
dec_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) { }
#endif /* CONFIG_SCHED_HMP */
static void
check_preempt_curr_stop(struct rq *rq, struct task_struct *p, int flags)
{
@ -42,12 +77,14 @@ static void
enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags)
{
add_nr_running(rq, 1);
inc_hmp_sched_stats_stop(rq, p);
}
static void
dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags)
{
sub_nr_running(rq, 1);
dec_hmp_sched_stats_stop(rq, p);
}
static void yield_task_stop(struct rq *rq)
@ -134,4 +171,7 @@ const struct sched_class stop_sched_class = {
.prio_changed = prio_changed_stop,
.switched_to = switched_to_stop,
.update_curr = update_curr_stop,
#ifdef CONFIG_SCHED_HMP
.fixup_hmp_sched_stats = fixup_hmp_sched_stats_stop,
#endif
};

Some files were not shown because too many files have changed in this diff Show more