sched: Import Sony Changes
Signed-off-by: Sorayukii <sorayukii69@gmail.com>
This commit is contained in:
Sorayukii
parent
84b9e8c0d8
commit
e41f8e75a2
10 changed files with 730 additions and 90 deletions
|
|
@ -2526,8 +2526,10 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut,
|
|||
#define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */
|
||||
#define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */
|
||||
#define PF_MEMSTALL 0x01000000 /* Stalled due to lack of memory */
|
||||
#define PF_MIGRATE_ED_TASK 0x02000000 /* early detection task migration */
|
||||
#define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_allowed */
|
||||
#define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */
|
||||
#define PF_MIGRATE_CUM_ADJ_TASK 0x10000000 /* cumulative task adjustment should be done */
|
||||
#define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */
|
||||
#define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */
|
||||
#define PF_SUSPEND_TASK 0x80000000 /* this thread called freeze_processes and should not be frozen */
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ static void boost_kick_cpus(void)
|
|||
{
|
||||
int i;
|
||||
struct cpumask kick_mask;
|
||||
u32 nr_running;
|
||||
|
||||
if (boost_policy != SCHED_BOOST_ON_BIG)
|
||||
return;
|
||||
|
|
@ -47,8 +48,20 @@ static void boost_kick_cpus(void)
|
|||
cpumask_andnot(&kick_mask, cpu_online_mask, cpu_isolated_mask);
|
||||
|
||||
for_each_cpu(i, &kick_mask) {
|
||||
if (cpu_capacity(i) != max_capacity)
|
||||
boost_kick(i);
|
||||
/*
|
||||
* kick only "small" cluster
|
||||
*/
|
||||
if (cpu_capacity(i) != max_capacity) {
|
||||
nr_running = ACCESS_ONCE(cpu_rq(i)->nr_running);
|
||||
|
||||
/*
|
||||
* make sense to interrupt CPU if its run-queue
|
||||
* has something running in order to check for
|
||||
* migration afterwards, otherwise skip it.
|
||||
*/
|
||||
if (nr_running)
|
||||
boost_kick(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -165,6 +165,10 @@ int sysctl_sched_rt_runtime = 950000;
|
|||
/* cpus with isolated domains */
|
||||
cpumask_var_t cpu_isolated_map;
|
||||
|
||||
#ifdef CONFIG_NO_HZ_COMMON
|
||||
DEFINE_PER_CPU(atomic_t, claim_wake_up_cpu) = ATOMIC_INIT(0);
|
||||
#endif
|
||||
|
||||
struct rq *
|
||||
lock_rq_of(struct task_struct *p, struct rq_flags *flags)
|
||||
{
|
||||
|
|
@ -1014,14 +1018,15 @@ static struct rq *move_queued_task(struct rq *rq, struct task_struct *p, int new
|
|||
|
||||
p->on_rq = TASK_ON_RQ_MIGRATING;
|
||||
dequeue_task(rq, p, 0);
|
||||
double_lock_balance(rq, cpu_rq(new_cpu));
|
||||
walt_prepare_migrate(p, cpu_of(rq), new_cpu, true);
|
||||
set_task_cpu(p, new_cpu);
|
||||
double_rq_unlock(cpu_rq(new_cpu), rq);
|
||||
raw_spin_unlock(&rq->lock);
|
||||
|
||||
rq = cpu_rq(new_cpu);
|
||||
|
||||
raw_spin_lock(&rq->lock);
|
||||
BUG_ON(task_cpu(p) != new_cpu);
|
||||
walt_finish_migrate(p, cpu_of(rq), new_cpu, true);
|
||||
enqueue_task(rq, p, 0);
|
||||
p->on_rq = TASK_ON_RQ_QUEUED;
|
||||
check_preempt_curr(rq, p, 0);
|
||||
|
|
@ -1288,8 +1293,6 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
|
|||
p->sched_class->migrate_task_rq(p);
|
||||
p->se.nr_migrations++;
|
||||
perf_event_task_migrate(p);
|
||||
|
||||
fixup_busy_time(p, new_cpu);
|
||||
}
|
||||
|
||||
__set_task_cpu(p, new_cpu);
|
||||
|
|
@ -1305,7 +1308,9 @@ static void __migrate_swap_task(struct task_struct *p, int cpu)
|
|||
|
||||
p->on_rq = TASK_ON_RQ_MIGRATING;
|
||||
deactivate_task(src_rq, p, 0);
|
||||
walt_prepare_migrate(p, cpu_of(src_rq), cpu, true);
|
||||
set_task_cpu(p, cpu);
|
||||
walt_finish_migrate(p, cpu_of(src_rq), cpu, true);
|
||||
activate_task(dst_rq, p, 0);
|
||||
p->on_rq = TASK_ON_RQ_QUEUED;
|
||||
check_preempt_curr(dst_rq, p, 0);
|
||||
|
|
@ -1676,6 +1681,10 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
|
|||
(cpu_isolated(cpu) && !allow_isolated))
|
||||
cpu = select_fallback_rq(task_cpu(p), p, allow_isolated);
|
||||
|
||||
#ifdef CONFIG_NO_HZ_COMMON
|
||||
if (unlikely(!atomic_read(&per_cpu(claim_wake_up_cpu, cpu))))
|
||||
atomic_set(&per_cpu(claim_wake_up_cpu, cpu), 1);
|
||||
#endif
|
||||
return cpu;
|
||||
}
|
||||
|
||||
|
|
@ -1950,13 +1959,16 @@ bool cpus_share_cache(int this_cpu, int that_cpu)
|
|||
}
|
||||
#endif /* CONFIG_SMP */
|
||||
|
||||
static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
|
||||
static void ttwu_queue(struct task_struct *p, int cpu, int prev_cpu, int wake_flags)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
struct rq_flags rf;
|
||||
|
||||
#if defined(CONFIG_SMP)
|
||||
if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
|
||||
if (prev_cpu != cpu)
|
||||
walt_finish_migrate(p, prev_cpu, cpu, false);
|
||||
|
||||
sched_clock_cpu(cpu); /* sync clocks x-cpu */
|
||||
ttwu_queue_remote(p, cpu, wake_flags);
|
||||
return;
|
||||
|
|
@ -1965,6 +1977,10 @@ static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
|
|||
|
||||
raw_spin_lock(&rq->lock);
|
||||
rq_pin_lock(rq, &rf);
|
||||
|
||||
if (prev_cpu != cpu)
|
||||
walt_finish_migrate(p, prev_cpu, cpu, true);
|
||||
|
||||
ttwu_do_activate(rq, p, wake_flags, &rf);
|
||||
rq_unpin_lock(rq, &rf);
|
||||
raw_spin_unlock(&rq->lock);
|
||||
|
|
@ -2186,14 +2202,16 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
|
|||
if (src_cpu != cpu) {
|
||||
wake_flags |= WF_MIGRATED;
|
||||
psi_ttwu_dequeue(p);
|
||||
walt_prepare_migrate(p, src_cpu, cpu, false);
|
||||
set_task_cpu(p, cpu);
|
||||
/* walt_finish_migrate is done in ttwu_queue */
|
||||
notif_required = true;
|
||||
}
|
||||
|
||||
note_task_waking(p, wallclock);
|
||||
#endif /* CONFIG_SMP */
|
||||
|
||||
ttwu_queue(p, cpu, wake_flags);
|
||||
ttwu_queue(p, cpu, src_cpu, wake_flags);
|
||||
stat:
|
||||
ttwu_stat(p, cpu, wake_flags);
|
||||
out:
|
||||
|
|
|
|||
|
|
@ -272,7 +272,9 @@ static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p
|
|||
double_lock_balance(rq, later_rq);
|
||||
}
|
||||
|
||||
walt_prepare_migrate(p, cpu_of(rq), cpu_of(later_rq), true);
|
||||
set_task_cpu(p, later_rq->cpu);
|
||||
walt_finish_migrate(p, cpu_of(rq), cpu_of(later_rq), true);
|
||||
double_unlock_balance(later_rq, rq);
|
||||
|
||||
return later_rq;
|
||||
|
|
@ -1632,7 +1634,9 @@ retry:
|
|||
|
||||
next_task->on_rq = TASK_ON_RQ_MIGRATING;
|
||||
deactivate_task(rq, next_task, 0);
|
||||
walt_prepare_migrate(next_task, cpu_of(rq), cpu_of(later_rq), true);
|
||||
set_task_cpu(next_task, later_rq->cpu);
|
||||
walt_finish_migrate(next_task, cpu_of(rq), cpu_of(later_rq), true);
|
||||
activate_task(later_rq, next_task, 0);
|
||||
next_task->on_rq = TASK_ON_RQ_QUEUED;
|
||||
ret = 1;
|
||||
|
|
@ -1722,7 +1726,9 @@ static void pull_dl_task(struct rq *this_rq)
|
|||
|
||||
p->on_rq = TASK_ON_RQ_MIGRATING;
|
||||
deactivate_task(src_rq, p, 0);
|
||||
walt_prepare_migrate(p, cpu_of(src_rq), cpu_of(this_rq), true);
|
||||
set_task_cpu(p, this_cpu);
|
||||
walt_finish_migrate(p, cpu_of(src_rq), cpu_of(this_rq), true);
|
||||
activate_task(this_rq, p, 0);
|
||||
p->on_rq = TASK_ON_RQ_QUEUED;
|
||||
dmin = p->dl.deadline;
|
||||
|
|
|
|||
|
|
@ -5051,6 +5051,10 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
|
|||
|
||||
if (!se) {
|
||||
add_nr_running(rq, 1);
|
||||
|
||||
if (unlikely(p->nr_cpus_allowed == 1))
|
||||
rq->nr_pinned_tasks++;
|
||||
|
||||
inc_rq_walt_stats(rq, p);
|
||||
}
|
||||
|
||||
|
|
@ -5144,6 +5148,10 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
|
|||
|
||||
if (!se) {
|
||||
sub_nr_running(rq, 1);
|
||||
|
||||
if (unlikely(p->nr_cpus_allowed == 1))
|
||||
rq->nr_pinned_tasks--;
|
||||
|
||||
dec_rq_walt_stats(rq, p);
|
||||
}
|
||||
|
||||
|
|
@ -8096,12 +8104,7 @@ again:
|
|||
set_next_entity(cfs_rq, se);
|
||||
}
|
||||
|
||||
if (hrtick_enabled(rq))
|
||||
hrtick_start_fair(rq, p);
|
||||
|
||||
rq->misfit_task = !task_fits_max(p, rq->cpu);
|
||||
|
||||
return p;
|
||||
goto done;
|
||||
simple:
|
||||
cfs_rq = &rq->cfs;
|
||||
#endif
|
||||
|
|
@ -8119,6 +8122,16 @@ simple:
|
|||
|
||||
p = task_of(se);
|
||||
|
||||
done: __maybe_unused
|
||||
#ifdef CONFIG_SMP
|
||||
/*
|
||||
* Move the next running task to the front of
|
||||
* the list, so our cfs_tasks list becomes MRU
|
||||
* one.
|
||||
*/
|
||||
list_move(&p->se.group_node, &rq->cfs_tasks);
|
||||
#endif
|
||||
|
||||
if (hrtick_enabled(rq))
|
||||
hrtick_start_fair(rq, p);
|
||||
|
||||
|
|
@ -8530,14 +8543,10 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
|
|||
if (energy_aware() && !env->dst_rq->rd->overutilized &&
|
||||
env->idle == CPU_NEWLY_IDLE &&
|
||||
!task_in_related_thread_group(p)) {
|
||||
long util_cum_dst, util_cum_src;
|
||||
unsigned long demand;
|
||||
long new_util_cum = cpu_util_cum(env->dst_cpu, task_util(p));
|
||||
|
||||
demand = task_util(p);
|
||||
util_cum_dst = cpu_util_cum(env->dst_cpu, 0) + demand;
|
||||
util_cum_src = cpu_util_cum(env->src_cpu, 0) - demand;
|
||||
|
||||
if (util_cum_dst > util_cum_src)
|
||||
if (add_capacity_margin(new_util_cum, env->dst_cpu) >
|
||||
capacity_curr_of(env->dst_cpu))
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -8590,11 +8599,10 @@ static void detach_task(struct task_struct *p, struct lb_env *env)
|
|||
|
||||
p->on_rq = TASK_ON_RQ_MIGRATING;
|
||||
deactivate_task(env->src_rq, p, 0);
|
||||
double_lock_balance(env->src_rq, env->dst_rq);
|
||||
walt_prepare_migrate(p, env->src_cpu, env->dst_cpu, true);
|
||||
set_task_cpu(p, env->dst_cpu);
|
||||
if (task_in_related_thread_group(p))
|
||||
env->flags |= LBF_MOVED_RELATED_THREAD_GROUP_TASK;
|
||||
double_unlock_balance(env->src_rq, env->dst_rq);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -8605,11 +8613,12 @@ static void detach_task(struct task_struct *p, struct lb_env *env)
|
|||
*/
|
||||
static struct task_struct *detach_one_task(struct lb_env *env)
|
||||
{
|
||||
struct task_struct *p, *n;
|
||||
struct task_struct *p;
|
||||
|
||||
lockdep_assert_held(&env->src_rq->lock);
|
||||
|
||||
list_for_each_entry_safe(p, n, &env->src_rq->cfs_tasks, se.group_node) {
|
||||
list_for_each_entry_reverse(p,
|
||||
&env->src_rq->cfs_tasks, se.group_node) {
|
||||
if (!can_migrate_task(p, env))
|
||||
continue;
|
||||
|
||||
|
|
@ -8663,7 +8672,15 @@ redo:
|
|||
if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1)
|
||||
break;
|
||||
|
||||
p = list_first_entry(tasks, struct task_struct, se.group_node);
|
||||
/*
|
||||
* Another CPU can place tasks, since we do not hold dst_rq lock
|
||||
* while doing balancing. If newly idle CPU already got something,
|
||||
* give up to reduce a latency.
|
||||
*/
|
||||
if (env->idle == CPU_NEWLY_IDLE && env->dst_rq->nr_running > 0)
|
||||
break;
|
||||
|
||||
p = list_last_entry(tasks, struct task_struct, se.group_node);
|
||||
|
||||
env->loop++;
|
||||
/* We've more or less seen every task there is, call it quits */
|
||||
|
|
@ -8703,6 +8720,7 @@ redo:
|
|||
*/
|
||||
if (((cpu_rq(env->src_cpu)->nr_running > 2) ||
|
||||
(env->flags & LBF_IGNORE_BIG_TASKS)) &&
|
||||
(env->idle != CPU_NEWLY_IDLE) &&
|
||||
((load / 2) > env->imbalance))
|
||||
goto next;
|
||||
|
||||
|
|
@ -8731,7 +8749,7 @@ redo:
|
|||
|
||||
continue;
|
||||
next:
|
||||
list_move_tail(&p->se.group_node, tasks);
|
||||
list_move(&p->se.group_node, tasks);
|
||||
}
|
||||
|
||||
if (env->flags & (LBF_IGNORE_BIG_TASKS |
|
||||
|
|
@ -8756,11 +8774,13 @@ next:
|
|||
/*
|
||||
* attach_task() -- attach the task detached by detach_task() to its new rq.
|
||||
*/
|
||||
static void attach_task(struct rq *rq, struct task_struct *p)
|
||||
static void attach_task(struct rq *rq,
|
||||
struct task_struct *p, struct lb_env *env)
|
||||
{
|
||||
lockdep_assert_held(&rq->lock);
|
||||
|
||||
BUG_ON(task_rq(p) != rq);
|
||||
walt_finish_migrate(p, env->src_cpu, env->dst_cpu, true);
|
||||
activate_task(rq, p, 0);
|
||||
p->on_rq = TASK_ON_RQ_QUEUED;
|
||||
check_preempt_curr(rq, p, 0);
|
||||
|
|
@ -8770,10 +8790,11 @@ static void attach_task(struct rq *rq, struct task_struct *p)
|
|||
* attach_one_task() -- attaches the task returned from detach_one_task() to
|
||||
* its new rq.
|
||||
*/
|
||||
static void attach_one_task(struct rq *rq, struct task_struct *p)
|
||||
static void attach_one_task(struct rq *rq,
|
||||
struct task_struct *p, struct lb_env *env)
|
||||
{
|
||||
raw_spin_lock(&rq->lock);
|
||||
attach_task(rq, p);
|
||||
attach_task(rq, p, env);
|
||||
raw_spin_unlock(&rq->lock);
|
||||
}
|
||||
|
||||
|
|
@ -8781,21 +8802,23 @@ static void attach_one_task(struct rq *rq, struct task_struct *p)
|
|||
* attach_tasks() -- attaches all tasks detached by detach_tasks() to their
|
||||
* new rq.
|
||||
*/
|
||||
static void attach_tasks(struct lb_env *env)
|
||||
static void attach_tasks(struct lb_env *env, bool dst_locked)
|
||||
{
|
||||
struct list_head *tasks = &env->tasks;
|
||||
struct task_struct *p;
|
||||
|
||||
raw_spin_lock(&env->dst_rq->lock);
|
||||
if (!dst_locked)
|
||||
raw_spin_lock(&env->dst_rq->lock);
|
||||
|
||||
while (!list_empty(tasks)) {
|
||||
p = list_first_entry(tasks, struct task_struct, se.group_node);
|
||||
list_del_init(&p->se.group_node);
|
||||
|
||||
attach_task(env->dst_rq, p);
|
||||
attach_task(env->dst_rq, p, env);
|
||||
}
|
||||
|
||||
raw_spin_unlock(&env->dst_rq->lock);
|
||||
if (!dst_locked)
|
||||
raw_spin_unlock(&env->dst_rq->lock);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_FAIR_GROUP_SCHED
|
||||
|
|
@ -9447,7 +9470,7 @@ static bool update_sd_pick_busiest(struct lb_env *env,
|
|||
* per-cpu capacity. No reason to pull tasks to less capable cpus.
|
||||
*/
|
||||
if (sgs->sum_nr_running <= sgs->group_weight &&
|
||||
group_smaller_cpu_capacity(sds->local, sg))
|
||||
!group_smaller_cpu_capacity(sg, sds->local))
|
||||
return false;
|
||||
|
||||
asym_packing:
|
||||
|
|
@ -10249,7 +10272,6 @@ redo:
|
|||
* correctly treated as an imbalance.
|
||||
*/
|
||||
env.flags |= LBF_ALL_PINNED;
|
||||
env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);
|
||||
|
||||
more_balance:
|
||||
raw_spin_lock_irqsave(&busiest->lock, flags);
|
||||
|
|
@ -10261,11 +10283,26 @@ more_balance:
|
|||
goto no_move;
|
||||
}
|
||||
|
||||
/*
|
||||
* Set loop_max when rq's lock is taken to prevent a race.
|
||||
*/
|
||||
env.loop_max = min(sysctl_sched_nr_migrate,
|
||||
busiest->cfs.h_nr_running);
|
||||
|
||||
/*
|
||||
* cur_ld_moved - load moved in current iteration
|
||||
* ld_moved - cumulative load moved across iterations
|
||||
*/
|
||||
cur_ld_moved = detach_tasks(&env);
|
||||
if (cur_ld_moved) {
|
||||
if (same_freq_domain(env.src_cpu, env.dst_cpu))
|
||||
/*
|
||||
* If we move tasks within the same freq domain, it is
|
||||
* important to acquire dst_rq before releasing src_rq
|
||||
* to prevent potential load drop.
|
||||
*/
|
||||
double_lock_balance(env.src_rq, env.dst_rq);
|
||||
}
|
||||
|
||||
/*
|
||||
* We've detached some tasks from busiest_rq. Every
|
||||
|
|
@ -10278,7 +10315,13 @@ more_balance:
|
|||
raw_spin_unlock(&busiest->lock);
|
||||
|
||||
if (cur_ld_moved) {
|
||||
attach_tasks(&env);
|
||||
if (same_freq_domain(env.src_cpu, env.dst_cpu)) {
|
||||
attach_tasks(&env, true);
|
||||
raw_spin_unlock(&env.dst_rq->lock);
|
||||
} else {
|
||||
attach_tasks(&env, false);
|
||||
}
|
||||
|
||||
ld_moved += cur_ld_moved;
|
||||
}
|
||||
|
||||
|
|
@ -10746,12 +10789,12 @@ out_unlock:
|
|||
|
||||
if (push_task) {
|
||||
if (push_task_detached)
|
||||
attach_one_task(target_rq, push_task);
|
||||
attach_one_task(target_rq, push_task, &env);
|
||||
put_task_struct(push_task);
|
||||
}
|
||||
|
||||
if (p)
|
||||
attach_one_task(target_rq, p);
|
||||
attach_one_task(target_rq, p, &env);
|
||||
|
||||
local_irq_enable();
|
||||
|
||||
|
|
@ -10770,6 +10813,31 @@ static inline int on_null_domain(struct rq *rq)
|
|||
* needed, they will kick the idle load balancer, which then does idle
|
||||
* load balancing for all the idle CPUs.
|
||||
*/
|
||||
static inline int
|
||||
get_first_not_claimed_cpu(cpumask_t *cpumask)
|
||||
{
|
||||
int ilb;
|
||||
|
||||
/*
|
||||
* If a CPU is claimed it means that TIF_NEED_RESCHED
|
||||
* is on its way or is already in place. In first case
|
||||
* a nohz_idle_balance work will most likely be stopped
|
||||
* or even canceled earlier, because of pending task.
|
||||
* In second one a CPU is not kicked for doing a load
|
||||
* balancing.
|
||||
*
|
||||
* Therefore exclude CPUs (among nohz.idle_cpus_mask)
|
||||
* which have already been claimed for waking up a task
|
||||
* on, preferring other CPUs to do NO_HZ balancing.
|
||||
*/
|
||||
for_each_cpu(ilb, cpumask) {
|
||||
if (!atomic_read(&per_cpu(claim_wake_up_cpu, ilb)))
|
||||
break;
|
||||
}
|
||||
|
||||
return ilb;
|
||||
}
|
||||
|
||||
static inline int find_new_ilb(int type)
|
||||
{
|
||||
int ilb = nr_cpu_ids;
|
||||
|
|
@ -10789,7 +10857,8 @@ static inline int find_new_ilb(int type)
|
|||
sched_domain_span(sd));
|
||||
cpumask_andnot(&cpumask, &cpumask,
|
||||
cpu_isolated_mask);
|
||||
ilb = cpumask_first(&cpumask);
|
||||
|
||||
ilb = get_first_not_claimed_cpu(&cpumask);
|
||||
}
|
||||
rcu_read_unlock();
|
||||
|
||||
|
|
@ -10800,7 +10869,8 @@ static inline int find_new_ilb(int type)
|
|||
(cpu_overutilized(cpu) && rq->nr_running > 1))) {
|
||||
cpumask_andnot(&cpumask, nohz.idle_cpus_mask,
|
||||
cpu_isolated_mask);
|
||||
ilb = cpumask_first(&cpumask);
|
||||
|
||||
ilb = get_first_not_claimed_cpu(&cpumask);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -10898,6 +10968,8 @@ void nohz_balance_enter_idle(int cpu)
|
|||
if (!cpu_active(cpu))
|
||||
return;
|
||||
|
||||
atomic_set(&per_cpu(claim_wake_up_cpu, cpu), 0);
|
||||
|
||||
if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu)))
|
||||
return;
|
||||
|
||||
|
|
@ -11150,6 +11222,22 @@ static inline bool nohz_kick_needed(struct rq *rq, int *type)
|
|||
return true;
|
||||
|
||||
/* Do idle load balance if there have misfit task */
|
||||
if (unlikely(rq->nr_pinned_tasks > 0)) {
|
||||
int delta = rq->nr_running - rq->nr_pinned_tasks;
|
||||
|
||||
/*
|
||||
* Check if it is possible to "unload" this CPU in case
|
||||
* of having pinned/affine tasks. Do not disturb idle
|
||||
* core if one of the below condition is true:
|
||||
*
|
||||
* - there is one pinned task and it is not "current"
|
||||
* - all tasks are pinned to this CPU
|
||||
*/
|
||||
if (delta < 2)
|
||||
if (current->nr_cpus_allowed > 1 || !delta)
|
||||
return false;
|
||||
}
|
||||
|
||||
if (energy_aware())
|
||||
return rq->misfit_task;
|
||||
|
||||
|
|
|
|||
|
|
@ -2117,7 +2117,9 @@ retry:
|
|||
|
||||
next_task->on_rq = TASK_ON_RQ_MIGRATING;
|
||||
deactivate_task(rq, next_task, 0);
|
||||
walt_prepare_migrate(next_task, cpu_of(rq), cpu_of(lowest_rq), true);
|
||||
set_task_cpu(next_task, lowest_rq->cpu);
|
||||
walt_finish_migrate(next_task, cpu_of(rq), cpu_of(lowest_rq), true);
|
||||
activate_task(lowest_rq, next_task, 0);
|
||||
next_task->on_rq = TASK_ON_RQ_QUEUED;
|
||||
ret = 1;
|
||||
|
|
@ -2391,7 +2393,9 @@ static void pull_rt_task(struct rq *this_rq)
|
|||
|
||||
p->on_rq = TASK_ON_RQ_MIGRATING;
|
||||
deactivate_task(src_rq, p, 0);
|
||||
walt_prepare_migrate(p, cpu_of(src_rq), this_cpu, true);
|
||||
set_task_cpu(p, this_cpu);
|
||||
walt_finish_migrate(p, cpu_of(src_rq), this_cpu, true);
|
||||
activate_task(this_rq, p, 0);
|
||||
p->on_rq = TASK_ON_RQ_QUEUED;
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -722,6 +722,7 @@ struct rq {
|
|||
#endif
|
||||
#define CPU_LOAD_IDX_MAX 5
|
||||
unsigned long cpu_load[CPU_LOAD_IDX_MAX];
|
||||
unsigned int nr_pinned_tasks;
|
||||
unsigned int misfit_task;
|
||||
#ifdef CONFIG_NO_HZ_COMMON
|
||||
#ifdef CONFIG_SMP
|
||||
|
|
@ -2303,6 +2304,7 @@ enum rq_nohz_flag_bits {
|
|||
|
||||
#define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags)
|
||||
|
||||
DECLARE_PER_CPU(atomic_t, claim_wake_up_cpu);
|
||||
extern void nohz_balance_exit_idle(unsigned int cpu);
|
||||
#else
|
||||
static inline void nohz_balance_exit_idle(unsigned int cpu) { }
|
||||
|
|
|
|||
|
|
@ -191,20 +191,28 @@ unsigned int sched_get_cpu_util(int cpu)
|
|||
unsigned long capacity, flags;
|
||||
unsigned int busy;
|
||||
|
||||
raw_spin_lock_irqsave(&rq->lock, flags);
|
||||
|
||||
util = rq->cfs.avg.util_avg;
|
||||
capacity = capacity_orig_of(cpu);
|
||||
|
||||
#ifdef CONFIG_SCHED_WALT
|
||||
if (!walt_disabled && sysctl_sched_use_walt_cpu_util) {
|
||||
util = rq->prev_runnable_sum + rq->grp_time.prev_runnable_sum;
|
||||
util = div64_u64(util,
|
||||
sched_ravg_window >> SCHED_CAPACITY_SHIFT);
|
||||
u32 prev_run_sum, group_run_sum;
|
||||
|
||||
util = walt_get_prev_group_run_sum(rq);
|
||||
group_run_sum = (u32) (util >> 32);
|
||||
prev_run_sum = (u32) util;
|
||||
|
||||
util = prev_run_sum + group_run_sum;
|
||||
util = div64_u64(util, sched_ravg_window >>
|
||||
SCHED_CAPACITY_SHIFT);
|
||||
|
||||
goto done;
|
||||
}
|
||||
#endif
|
||||
|
||||
raw_spin_lock_irqsave(&rq->lock, flags);
|
||||
util = rq->cfs.avg.util_avg;
|
||||
raw_spin_unlock_irqrestore(&rq->lock, flags);
|
||||
|
||||
done: __maybe_unused
|
||||
capacity = capacity_orig_of(cpu);
|
||||
util = (util >= capacity) ? capacity : util;
|
||||
busy = div64_ul((util * 100), capacity);
|
||||
return busy;
|
||||
|
|
|
|||
|
|
@ -52,6 +52,28 @@ u64 walt_load_reported_window;
|
|||
|
||||
static struct irq_work walt_cpufreq_irq_work;
|
||||
static struct irq_work walt_migration_irq_work;
|
||||
static DEFINE_RAW_SPINLOCK(speedchange_cpumask_lock);
|
||||
static cpumask_t speedchange_cpumask = CPU_MASK_NONE;
|
||||
|
||||
static DEFINE_PER_CPU(atomic64_t, cycles) = ATOMIC64_INIT(0);
|
||||
static DEFINE_PER_CPU(atomic64_t, last_cc_update) = ATOMIC64_INIT(0);
|
||||
static DEFINE_PER_CPU(atomic64_t, prev_group_runnable_sum) = ATOMIC64_INIT(0);
|
||||
|
||||
static inline void
|
||||
walt_commit_prev_group_run_sum(struct rq *rq)
|
||||
{
|
||||
u64 val;
|
||||
|
||||
val = rq->grp_time.prev_runnable_sum;
|
||||
val = (val << 32) | rq->prev_runnable_sum;
|
||||
atomic64_set(&per_cpu(prev_group_runnable_sum, cpu_of(rq)), val);
|
||||
}
|
||||
|
||||
u64
|
||||
walt_get_prev_group_run_sum(struct rq *rq)
|
||||
{
|
||||
return (u64) atomic64_read(&per_cpu(prev_group_runnable_sum, cpu_of(rq)));
|
||||
}
|
||||
|
||||
void
|
||||
walt_fixup_cumulative_runnable_avg(struct rq *rq,
|
||||
|
|
@ -356,20 +378,24 @@ int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb)
|
|||
return 0;
|
||||
}
|
||||
|
||||
#define THRESH_CC_UPDATE (2 * NSEC_PER_USEC)
|
||||
|
||||
/*
|
||||
* Assumes rq_lock is held and wallclock was recorded in the same critical
|
||||
* section as this function's invocation.
|
||||
*/
|
||||
static inline u64 read_cycle_counter(int cpu, u64 wallclock)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
u64 delta;
|
||||
|
||||
if (rq->last_cc_update != wallclock) {
|
||||
rq->cycles = cpu_cycle_counter_cb.get_cpu_cycle_counter(cpu);
|
||||
rq->last_cc_update = wallclock;
|
||||
delta = wallclock - atomic64_read(&per_cpu(last_cc_update, cpu));
|
||||
if (delta > THRESH_CC_UPDATE) {
|
||||
atomic64_set(&per_cpu(cycles, cpu),
|
||||
cpu_cycle_counter_cb.get_cpu_cycle_counter(cpu));
|
||||
atomic64_set(&per_cpu(last_cc_update, cpu), wallclock);
|
||||
}
|
||||
|
||||
return rq->cycles;
|
||||
return atomic64_read(&per_cpu(cycles, cpu));
|
||||
}
|
||||
|
||||
static void update_task_cpu_cycles(struct task_struct *p, int cpu,
|
||||
|
|
@ -607,6 +633,8 @@ static inline void account_load_subtractions(struct rq *rq)
|
|||
ls[i].new_subs = 0;
|
||||
}
|
||||
|
||||
walt_commit_prev_group_run_sum(rq);
|
||||
|
||||
BUG_ON((s64)rq->prev_runnable_sum < 0);
|
||||
BUG_ON((s64)rq->curr_runnable_sum < 0);
|
||||
BUG_ON((s64)rq->nt_prev_runnable_sum < 0);
|
||||
|
|
@ -921,6 +949,371 @@ done:
|
|||
double_rq_unlock(src_rq, dest_rq);
|
||||
}
|
||||
|
||||
static void
|
||||
migrate_prepare_top_tasks(struct task_struct *p, struct rq *src_rq)
|
||||
{
|
||||
int index;
|
||||
int top_index;
|
||||
u32 curr_window = p->ravg.curr_window;
|
||||
u32 prev_window = p->ravg.prev_window;
|
||||
u8 src = src_rq->curr_table;
|
||||
u8 *src_table;
|
||||
|
||||
if (curr_window) {
|
||||
src_table = src_rq->top_tasks[src];
|
||||
index = load_to_index(curr_window);
|
||||
src_table[index] -= 1;
|
||||
|
||||
if (!src_table[index])
|
||||
__clear_bit(NUM_LOAD_INDICES - index - 1,
|
||||
src_rq->top_tasks_bitmap[src]);
|
||||
|
||||
|
||||
top_index = src_rq->curr_top;
|
||||
if (index == top_index && !src_table[index])
|
||||
src_rq->curr_top = get_top_index(
|
||||
src_rq->top_tasks_bitmap[src], top_index);
|
||||
}
|
||||
|
||||
if (prev_window) {
|
||||
src = 1 - src;
|
||||
src_table = src_rq->top_tasks[src];
|
||||
index = load_to_index(prev_window);
|
||||
src_table[index] -= 1;
|
||||
|
||||
if (!src_table[index])
|
||||
__clear_bit(NUM_LOAD_INDICES - index - 1,
|
||||
src_rq->top_tasks_bitmap[src]);
|
||||
|
||||
top_index = src_rq->prev_top;
|
||||
if (index == top_index && !src_table[index])
|
||||
src_rq->prev_top = get_top_index(
|
||||
src_rq->top_tasks_bitmap[src], top_index);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
migrate_finish_top_tasks(struct task_struct *p, struct rq *dst_rq)
|
||||
{
|
||||
int index;
|
||||
u32 curr_window = p->ravg.curr_window;
|
||||
u32 prev_window = p->ravg.prev_window;
|
||||
u8 dst = dst_rq->curr_table;
|
||||
u8 *dst_table;
|
||||
|
||||
if (curr_window) {
|
||||
dst_table = dst_rq->top_tasks[dst];
|
||||
index = load_to_index(curr_window);
|
||||
dst_table[index] += 1;
|
||||
|
||||
if (dst_table[index] == 1)
|
||||
__set_bit(NUM_LOAD_INDICES - index - 1,
|
||||
dst_rq->top_tasks_bitmap[dst]);
|
||||
|
||||
if (index > dst_rq->curr_top)
|
||||
dst_rq->curr_top = index;
|
||||
}
|
||||
|
||||
if (prev_window) {
|
||||
dst = 1 - dst;
|
||||
dst_table = dst_rq->top_tasks[dst];
|
||||
index = load_to_index(prev_window);
|
||||
dst_table[index] += 1;
|
||||
|
||||
if (dst_table[index] == 1)
|
||||
__set_bit(NUM_LOAD_INDICES - index - 1,
|
||||
dst_rq->top_tasks_bitmap[dst]);
|
||||
|
||||
if (index > dst_rq->prev_top)
|
||||
dst_rq->prev_top = index;
|
||||
}
|
||||
}
|
||||
|
||||
void walt_prepare_migrate(struct task_struct *p,
|
||||
int src_cpu, int new_cpu, bool locked)
|
||||
{
|
||||
struct rq *src_rq = cpu_rq(src_cpu);
|
||||
struct related_thread_group *grp;
|
||||
u64 *src_curr_runnable_sum;
|
||||
u64 *src_prev_runnable_sum;
|
||||
u64 *src_nt_curr_runnable_sum;
|
||||
u64 *src_nt_prev_runnable_sum;
|
||||
u64 wallclock;
|
||||
bool new_task;
|
||||
|
||||
if (sched_disable_window_stats)
|
||||
return;
|
||||
|
||||
if (!p->on_rq && p->state != TASK_WAKING)
|
||||
return;
|
||||
|
||||
if (exiting_task(p)) {
|
||||
clear_ed_task(p, src_rq);
|
||||
return;
|
||||
}
|
||||
|
||||
BUG_ON(!irqs_disabled());
|
||||
|
||||
if (p->state == TASK_WAKING) {
|
||||
/*
|
||||
* Is there any contribution? zero? If yes, get rid
|
||||
* of extra overhead since there is nothing to move
|
||||
* between CPUs.
|
||||
*
|
||||
* In that case what we need is to update a task with
|
||||
* a new CPU's cycle counter and nothing more. An update
|
||||
* is done only if a "wake-up migration" occurs between
|
||||
* two not the same frequency domains, because:
|
||||
* - their counter cycles (CC) are far different
|
||||
* from each other.
|
||||
*
|
||||
* If a frequency domain is the same we skip an update
|
||||
* because:
|
||||
* - CC are synchronized within the same domain;
|
||||
* - to get rid of quite expensive CC reading.
|
||||
*/
|
||||
if (!p->ravg.curr_window && !p->ravg.prev_window) {
|
||||
if (!same_freq_domain(src_cpu, new_cpu))
|
||||
update_task_cpu_cycles(p, new_cpu, ktime_get_ns());
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!locked)
|
||||
raw_spin_lock(&src_rq->lock);
|
||||
|
||||
lockdep_assert_held(&src_rq->lock);
|
||||
wallclock = ktime_get_ns();
|
||||
|
||||
update_task_ravg(task_rq(p)->curr, task_rq(p),
|
||||
TASK_UPDATE, wallclock, 0);
|
||||
update_task_ravg(p, task_rq(p), TASK_MIGRATE,
|
||||
wallclock, 0);
|
||||
|
||||
/*
|
||||
* See a comment above when an update is done
|
||||
*/
|
||||
if (!same_freq_domain(src_cpu, new_cpu))
|
||||
update_task_cpu_cycles(p, new_cpu, wallclock);
|
||||
|
||||
/*
|
||||
* When a task is migrating during the wakeup, adjust
|
||||
* the task's contribution towards cumulative window
|
||||
* demand.
|
||||
*/
|
||||
if (p->state == TASK_WAKING &&
|
||||
p->last_sleep_ts >= src_rq->window_start) {
|
||||
walt_fixup_cum_window_demand(src_rq, -(s64)p->ravg.demand);
|
||||
p->flags |= PF_MIGRATE_CUM_ADJ_TASK;
|
||||
}
|
||||
|
||||
new_task = is_new_task(p);
|
||||
/* Protected by rq_lock */
|
||||
grp = p->grp;
|
||||
|
||||
/*
|
||||
* For frequency aggregation, we continue to do migration fixups
|
||||
* even for intra cluster migrations. This is because, the aggregated
|
||||
* load has to reported on a single CPU regardless.
|
||||
*/
|
||||
if (grp) {
|
||||
struct group_cpu_time *cpu_time;
|
||||
|
||||
cpu_time = &src_rq->grp_time;
|
||||
src_curr_runnable_sum = &cpu_time->curr_runnable_sum;
|
||||
src_prev_runnable_sum = &cpu_time->prev_runnable_sum;
|
||||
src_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum;
|
||||
src_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum;
|
||||
|
||||
if (p->ravg.curr_window) {
|
||||
*src_curr_runnable_sum -= p->ravg.curr_window;
|
||||
if (new_task)
|
||||
*src_nt_curr_runnable_sum -=
|
||||
p->ravg.curr_window;
|
||||
}
|
||||
|
||||
if (p->ravg.prev_window) {
|
||||
*src_prev_runnable_sum -= p->ravg.prev_window;
|
||||
if (new_task)
|
||||
*src_nt_prev_runnable_sum -=
|
||||
p->ravg.prev_window;
|
||||
}
|
||||
} else if (!same_freq_domain(src_cpu, new_cpu)) {
|
||||
src_rq->curr_runnable_sum -=
|
||||
p->ravg.curr_window_cpu[src_cpu];
|
||||
src_rq->prev_runnable_sum -=
|
||||
p->ravg.prev_window_cpu[src_cpu];
|
||||
|
||||
if (new_task) {
|
||||
src_rq->nt_curr_runnable_sum -=
|
||||
p->ravg.curr_window_cpu[src_cpu];
|
||||
src_rq->nt_prev_runnable_sum -=
|
||||
p->ravg.prev_window_cpu[src_cpu];
|
||||
}
|
||||
|
||||
p->ravg.curr_window_cpu[src_cpu] = 0;
|
||||
p->ravg.prev_window_cpu[src_cpu] = 0;
|
||||
|
||||
update_cluster_load_subtractions(p, src_cpu,
|
||||
src_rq->window_start, new_task);
|
||||
}
|
||||
|
||||
if ((s64) src_rq->prev_runnable_sum < 0) {
|
||||
src_rq->prev_runnable_sum = 0;
|
||||
WARN_ON(1);
|
||||
}
|
||||
|
||||
if ((s64) src_rq->curr_runnable_sum < 0) {
|
||||
src_rq->curr_runnable_sum = 0;
|
||||
WARN_ON(1);
|
||||
}
|
||||
|
||||
if ((s64) src_rq->nt_prev_runnable_sum < 0) {
|
||||
src_rq->nt_prev_runnable_sum = 0;
|
||||
WARN_ON(1);
|
||||
}
|
||||
|
||||
if ((s64) src_rq->nt_curr_runnable_sum < 0) {
|
||||
src_rq->nt_curr_runnable_sum = 0;
|
||||
WARN_ON(1);
|
||||
}
|
||||
|
||||
walt_commit_prev_group_run_sum(src_rq);
|
||||
migrate_prepare_top_tasks(p, src_rq);
|
||||
|
||||
if (src_rq->ed_task == p) {
|
||||
p->flags |= PF_MIGRATE_ED_TASK;
|
||||
src_rq->ed_task = NULL;
|
||||
}
|
||||
|
||||
if (!locked)
|
||||
raw_spin_unlock(&src_rq->lock);
|
||||
}
|
||||
|
||||
void walt_finish_migrate(struct task_struct *p,
|
||||
int src_cpu, int new_cpu, bool locked)
|
||||
{
|
||||
struct rq *dest_rq = cpu_rq(new_cpu);
|
||||
struct rq *src_rq = cpu_rq(src_cpu);
|
||||
struct related_thread_group *grp;
|
||||
u64 *dst_curr_runnable_sum;
|
||||
u64 *dst_prev_runnable_sum;
|
||||
u64 *dst_nt_curr_runnable_sum;
|
||||
u64 *dst_nt_prev_runnable_sum;
|
||||
u64 wallclock;
|
||||
bool new_task;
|
||||
|
||||
if (sched_disable_window_stats)
|
||||
return;
|
||||
|
||||
if (!p->on_rq && p->state != TASK_WAKING)
|
||||
return;
|
||||
|
||||
if (exiting_task(p))
|
||||
return;
|
||||
|
||||
BUG_ON(!irqs_disabled());
|
||||
|
||||
if (!locked)
|
||||
raw_spin_lock(&dest_rq->lock);
|
||||
|
||||
lockdep_assert_held(&dest_rq->lock);
|
||||
wallclock = ktime_get_ns();
|
||||
|
||||
update_task_ravg(dest_rq->curr, dest_rq,
|
||||
TASK_UPDATE, wallclock, 0);
|
||||
update_task_ravg(p, dest_rq, TASK_MIGRATE,
|
||||
wallclock, 0);
|
||||
|
||||
/*
|
||||
* When a task is migrating during the wakeup, adjust
|
||||
* the task's contribution towards cumulative window
|
||||
* demand.
|
||||
*/
|
||||
if (p->state == TASK_WAKING &&
|
||||
p->flags & PF_MIGRATE_CUM_ADJ_TASK) {
|
||||
walt_fixup_cum_window_demand(dest_rq, p->ravg.demand);
|
||||
p->flags &= ~PF_MIGRATE_CUM_ADJ_TASK;
|
||||
}
|
||||
|
||||
new_task = is_new_task(p);
|
||||
/* Protected by rq_lock */
|
||||
grp = p->grp;
|
||||
|
||||
/*
|
||||
* For frequency aggregation, we continue to do migration fixups
|
||||
* even for intra cluster migrations. This is because, the aggregated
|
||||
* load has to reported on a single CPU regardless.
|
||||
*/
|
||||
if (grp) {
|
||||
struct group_cpu_time *cpu_time;
|
||||
|
||||
cpu_time = &dest_rq->grp_time;
|
||||
dst_curr_runnable_sum = &cpu_time->curr_runnable_sum;
|
||||
dst_prev_runnable_sum = &cpu_time->prev_runnable_sum;
|
||||
dst_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum;
|
||||
dst_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum;
|
||||
|
||||
if (p->ravg.curr_window) {
|
||||
*dst_curr_runnable_sum += p->ravg.curr_window;
|
||||
if (new_task)
|
||||
*dst_nt_curr_runnable_sum +=
|
||||
p->ravg.curr_window;
|
||||
}
|
||||
|
||||
if (p->ravg.prev_window) {
|
||||
*dst_prev_runnable_sum += p->ravg.prev_window;
|
||||
if (new_task)
|
||||
*dst_nt_prev_runnable_sum +=
|
||||
p->ravg.prev_window;
|
||||
}
|
||||
} else if (!same_freq_domain(src_cpu, new_cpu)) {
|
||||
p->ravg.curr_window_cpu[new_cpu] = p->ravg.curr_window;
|
||||
p->ravg.prev_window_cpu[new_cpu] = p->ravg.prev_window;
|
||||
|
||||
dest_rq->curr_runnable_sum += p->ravg.curr_window;
|
||||
dest_rq->prev_runnable_sum += p->ravg.prev_window;
|
||||
|
||||
if (new_task) {
|
||||
dest_rq->nt_curr_runnable_sum +=
|
||||
p->ravg.curr_window;
|
||||
dest_rq->nt_prev_runnable_sum +=
|
||||
p->ravg.prev_window;
|
||||
}
|
||||
}
|
||||
|
||||
walt_commit_prev_group_run_sum(dest_rq);
|
||||
migrate_finish_top_tasks(p, dest_rq);
|
||||
|
||||
if (!same_freq_domain(src_cpu, new_cpu)) {
|
||||
/*
|
||||
* If walt_cpufreq_irq_work has been recently queued
|
||||
* and is in a pending state, do not push a migration
|
||||
* job, because all magic will have been done anyway.
|
||||
*/
|
||||
if (!(smp_load_acquire(&walt_cpufreq_irq_work.flags) &
|
||||
IRQ_WORK_PENDING)) {
|
||||
raw_spin_lock(&speedchange_cpumask_lock);
|
||||
src_rq->notif_pending = true;
|
||||
dest_rq->notif_pending = true;
|
||||
cpumask_set_cpu(src_cpu, &speedchange_cpumask);
|
||||
cpumask_set_cpu(new_cpu, &speedchange_cpumask);
|
||||
raw_spin_unlock(&speedchange_cpumask_lock);
|
||||
irq_work_queue(&walt_migration_irq_work);
|
||||
}
|
||||
}
|
||||
|
||||
if (p->flags & PF_MIGRATE_ED_TASK) {
|
||||
if (!dest_rq->ed_task)
|
||||
dest_rq->ed_task = p;
|
||||
|
||||
p->flags &= ~PF_MIGRATE_ED_TASK;
|
||||
}
|
||||
|
||||
if (!locked)
|
||||
raw_spin_unlock(&dest_rq->lock);
|
||||
}
|
||||
|
||||
void set_window_start(struct rq *rq)
|
||||
{
|
||||
static int sync_cpu_available;
|
||||
|
|
@ -1346,6 +1739,7 @@ static inline u64 scale_exec_time(u64 delta, struct rq *rq)
|
|||
u32 freq;
|
||||
|
||||
freq = cpu_cycles_to_freq(rq->cc.cycles, rq->cc.time);
|
||||
|
||||
delta = DIV64_U64_ROUNDUP(delta * freq, max_possible_freq);
|
||||
delta *= rq->cluster->exec_scale_factor;
|
||||
delta >>= 10;
|
||||
|
|
@ -1626,7 +2020,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
|
|||
*/
|
||||
if (mark_start > window_start) {
|
||||
*curr_runnable_sum = scale_exec_time(irqtime, rq);
|
||||
return;
|
||||
goto done;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1642,11 +2036,11 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq,
|
|||
/* Process the remaining IRQ busy time in the current window. */
|
||||
delta = wallclock - window_start;
|
||||
rq->curr_runnable_sum = scale_exec_time(delta, rq);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
done:
|
||||
walt_commit_prev_group_run_sum(rq);
|
||||
|
||||
if (!is_idle_task(p) && !exiting_task(p))
|
||||
update_top_tasks(p, rq, old_curr_window,
|
||||
new_window, full_window);
|
||||
|
|
@ -1954,6 +2348,26 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event,
|
|||
else
|
||||
rq->cc.time = wallclock - p->ravg.mark_start;
|
||||
BUG_ON((s64)rq->cc.time < 0);
|
||||
|
||||
/*
|
||||
* It can happen when a time between two updates
|
||||
* (for example TASK_UPDATE) is very short. The reason
|
||||
* is a read_cycle_counter function now can return
|
||||
* a previous/same CC value if a last read was within
|
||||
* a THRESH_CC_UPDATE threshold.
|
||||
*
|
||||
* In that particular scenario use current CPU OPP
|
||||
* to scale such task's delta contributions which
|
||||
* are smaller than THRESH_CC_UPDATE interval.
|
||||
*
|
||||
* The aim of using a current frequency is because:
|
||||
* - an estimated one can be zero;
|
||||
* - we do not want to lose samples due to that.
|
||||
*/
|
||||
if (unlikely(!rq->cc.cycles)) {
|
||||
rq->cc.cycles = cpu_cur_freq(cpu);
|
||||
rq->cc.time = 1;
|
||||
}
|
||||
}
|
||||
|
||||
p->cpu_cycles = cur_cycles;
|
||||
|
|
@ -1961,17 +2375,50 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event,
|
|||
trace_sched_get_task_cpu_cycles(cpu, event, rq->cc.cycles, rq->cc.time, p);
|
||||
}
|
||||
|
||||
static inline void run_walt_irq_work(u64 old_window_start, struct rq *rq)
|
||||
static inline void
|
||||
run_walt_irq_work(u64 old_window_start, struct rq *rq, u64 wallclock)
|
||||
{
|
||||
u64 result;
|
||||
|
||||
if (old_window_start == rq->window_start)
|
||||
if (likely(old_window_start == rq->window_start))
|
||||
return;
|
||||
|
||||
result = atomic64_cmpxchg(&walt_irq_work_lastq_ws, old_window_start,
|
||||
rq->window_start);
|
||||
if (result == old_window_start)
|
||||
if (result == old_window_start) {
|
||||
/*
|
||||
* Usually a window update is done each sched_ravg_window
|
||||
* interval, but there are exceptions. If a CPU has not been
|
||||
* updated during more than one full window (most of the cases
|
||||
* when it was idle) we end up in double update within the
|
||||
* short period of time.
|
||||
*
|
||||
* ws_old(a) ws(b) wc(c)
|
||||
* | | |
|
||||
* V V V
|
||||
* ---|-------|-------|-------|------ |
|
||||
* | |
|
||||
* |<---- nr_full_win ---->|
|
||||
*
|
||||
* According to the picture, the window was updated in point (b)
|
||||
* as soon as 3 full windows have passed. At the same time the
|
||||
* second roll-over is going to be done quite soon, since the
|
||||
* wallclock is about to cross and close the current window.
|
||||
*
|
||||
* If a remaining time is less than sched_ravg_window / 2 we do
|
||||
* not trigger walt_cpufreq_irq_work because:
|
||||
*
|
||||
* - it most likely spurious decision after been idle;
|
||||
* - we could ignore that decision, but walt_irq_work_roll_over
|
||||
* is expansive and affects the preemption off latency;
|
||||
* - there will be one more roll-over.
|
||||
*/
|
||||
result = sched_ravg_window - (wallclock % sched_ravg_window);
|
||||
if (unlikely(result < (sched_ravg_window / 2)))
|
||||
return;
|
||||
|
||||
sched_irq_work_queue(&walt_cpufreq_irq_work);
|
||||
}
|
||||
}
|
||||
|
||||
/* Reflect task activity on its demand and cpu's busy time statistics */
|
||||
|
|
@ -2009,7 +2456,7 @@ void update_task_ravg(struct task_struct *p, struct rq *rq, int event,
|
|||
done:
|
||||
p->ravg.mark_start = wallclock;
|
||||
|
||||
run_walt_irq_work(old_window_start, rq);
|
||||
run_walt_irq_work(old_window_start, rq, wallclock);
|
||||
}
|
||||
|
||||
u32 sched_get_init_task_load(struct task_struct *p)
|
||||
|
|
@ -3169,6 +3616,7 @@ static void transfer_busy_time(struct rq *rq, struct related_thread_group *grp,
|
|||
p->ravg.curr_window_cpu[cpu] = p->ravg.curr_window;
|
||||
p->ravg.prev_window_cpu[cpu] = p->ravg.prev_window;
|
||||
|
||||
walt_commit_prev_group_run_sum(rq);
|
||||
trace_sched_migration_update_sum(p, migrate_type, rq);
|
||||
|
||||
BUG_ON((s64)*src_curr_runnable_sum < 0);
|
||||
|
|
@ -3241,24 +3689,77 @@ done:
|
|||
return ret;
|
||||
}
|
||||
|
||||
static inline void
|
||||
walt_irq_work_migration(struct irq_work *irq_work)
|
||||
{
|
||||
struct sched_cluster *cluster;
|
||||
cpumask_t tmp_mask;
|
||||
int cpu;
|
||||
|
||||
raw_spin_lock(&speedchange_cpumask_lock);
|
||||
tmp_mask = speedchange_cpumask;
|
||||
cpumask_clear(&speedchange_cpumask);
|
||||
raw_spin_unlock(&speedchange_cpumask_lock);
|
||||
|
||||
/*
|
||||
* walt_irq_work_migration can be run simultaneously
|
||||
* on different cores. That is why a tmp_mask may become
|
||||
* empty for one of them. Thus, if there are no CPUs
|
||||
* to check than we are done.
|
||||
*/
|
||||
if (!cpumask_weight(&tmp_mask))
|
||||
return;
|
||||
|
||||
for_each_sched_cluster(cluster) {
|
||||
u64 aggr_grp_load = 0;
|
||||
u64 wc;
|
||||
|
||||
for_each_cpu(cpu, &cluster->cpus)
|
||||
raw_spin_lock(&cpu_rq(cpu)->lock);
|
||||
|
||||
wc = ktime_get_ns();
|
||||
for_each_cpu(cpu, &cluster->cpus)
|
||||
update_task_ravg(cpu_rq(cpu)->curr,
|
||||
cpu_rq(cpu), TASK_UPDATE, wc, 0);
|
||||
|
||||
raw_spin_lock(&cluster->load_lock);
|
||||
for_each_cpu(cpu, &cluster->cpus)
|
||||
aggr_grp_load += cpu_rq(cpu)->grp_time.prev_runnable_sum;
|
||||
|
||||
cluster->aggr_grp_load = aggr_grp_load;
|
||||
raw_spin_unlock(&cluster->load_lock);
|
||||
|
||||
for_each_cpu(cpu, &cluster->cpus) {
|
||||
int nflag = SCHED_CPUFREQ_WALT;
|
||||
|
||||
struct rq* rq = cpu_rq(cpu);
|
||||
|
||||
if (rq->notif_pending) {
|
||||
nflag |= SCHED_CPUFREQ_INTERCLUSTER_MIG;
|
||||
rq->notif_pending = false;
|
||||
} else {
|
||||
nflag |= SCHED_CPUFREQ_FORCE_UPDATE;
|
||||
}
|
||||
|
||||
cpufreq_update_util(rq, nflag);
|
||||
}
|
||||
for_each_cpu(cpu, &cluster->cpus)
|
||||
raw_spin_unlock(&cpu_rq(cpu)->lock);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Runs in hard-irq context. This should ideally run just after the latest
|
||||
* window roll-over.
|
||||
*/
|
||||
void walt_irq_work(struct irq_work *irq_work)
|
||||
void walt_irq_work_roll_over(struct irq_work *irq_work)
|
||||
{
|
||||
struct sched_cluster *cluster;
|
||||
struct rq *rq;
|
||||
int cpu;
|
||||
u64 wc, total_grp_load = 0;
|
||||
int flag = SCHED_CPUFREQ_WALT;
|
||||
bool is_migration = false;
|
||||
int level = 0;
|
||||
|
||||
/* Am I the window rollover work or the migration work? */
|
||||
if (irq_work == &walt_migration_irq_work)
|
||||
is_migration = true;
|
||||
|
||||
for_each_cpu(cpu, cpu_possible_mask) {
|
||||
if (level == 0)
|
||||
raw_spin_lock(&cpu_rq(cpu)->lock);
|
||||
|
|
@ -3295,30 +3796,14 @@ void walt_irq_work(struct irq_work *irq_work)
|
|||
if (total_grp_load)
|
||||
walt_update_coloc_boost_load();
|
||||
|
||||
for_each_sched_cluster(cluster) {
|
||||
for_each_cpu(cpu, &cluster->cpus) {
|
||||
int nflag = flag;
|
||||
|
||||
rq = cpu_rq(cpu);
|
||||
|
||||
if (is_migration) {
|
||||
if (rq->notif_pending) {
|
||||
nflag |= SCHED_CPUFREQ_INTERCLUSTER_MIG;
|
||||
rq->notif_pending = false;
|
||||
} else {
|
||||
nflag |= SCHED_CPUFREQ_FORCE_UPDATE;
|
||||
}
|
||||
}
|
||||
|
||||
cpufreq_update_util(rq, nflag);
|
||||
}
|
||||
}
|
||||
for_each_sched_cluster(cluster)
|
||||
for_each_cpu(cpu, &cluster->cpus)
|
||||
cpufreq_update_util(cpu_rq(cpu), SCHED_CPUFREQ_WALT);
|
||||
|
||||
for_each_cpu(cpu, cpu_possible_mask)
|
||||
raw_spin_unlock(&cpu_rq(cpu)->lock);
|
||||
|
||||
if (!is_migration)
|
||||
core_ctl_check(this_rq()->window_start);
|
||||
core_ctl_check(this_rq()->window_start);
|
||||
}
|
||||
|
||||
void walt_rotation_checkpoint(int nr_big)
|
||||
|
|
@ -3367,8 +3852,8 @@ void walt_sched_init(struct rq *rq)
|
|||
int j;
|
||||
|
||||
cpumask_set_cpu(cpu_of(rq), &rq->freq_domain_cpumask);
|
||||
init_irq_work(&walt_migration_irq_work, walt_irq_work);
|
||||
init_irq_work(&walt_cpufreq_irq_work, walt_irq_work);
|
||||
init_irq_work(&walt_migration_irq_work, walt_irq_work_migration);
|
||||
init_irq_work(&walt_cpufreq_irq_work, walt_irq_work_roll_over);
|
||||
walt_rotate_work_init();
|
||||
|
||||
rq->walt_stats.cumulative_runnable_avg = 0;
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ extern void update_task_ravg(struct task_struct *p, struct rq *rq, int event,
|
|||
u64 wallclock, u64 irqtime);
|
||||
|
||||
extern unsigned int nr_eligible_big_tasks(int cpu);
|
||||
extern u64 walt_get_prev_group_run_sum(struct rq *rq);
|
||||
|
||||
static inline void
|
||||
inc_nr_big_task(struct walt_sched_stats *stats, struct task_struct *p)
|
||||
|
|
@ -150,6 +151,10 @@ extern void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p,
|
|||
extern void inc_rq_walt_stats(struct rq *rq, struct task_struct *p);
|
||||
extern void dec_rq_walt_stats(struct rq *rq, struct task_struct *p);
|
||||
extern void fixup_busy_time(struct task_struct *p, int new_cpu);
|
||||
extern void walt_prepare_migrate(struct task_struct *p,
|
||||
int src_cpu, int new_cpu, bool locked);
|
||||
extern void walt_finish_migrate(struct task_struct *p,
|
||||
int src_cpu, int new_cpu, bool locked);
|
||||
extern void init_new_task_load(struct task_struct *p);
|
||||
extern void mark_task_starting(struct task_struct *p);
|
||||
extern void set_window_start(struct rq *rq);
|
||||
|
|
@ -325,6 +330,11 @@ static inline unsigned int nr_eligible_big_tasks(int cpu)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static inline u64 walt_get_prev_group_run_sum(struct rq *rq)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void walt_adjust_nr_big_tasks(struct rq *rq,
|
||||
int delta, bool inc)
|
||||
{
|
||||
|
|
@ -345,6 +355,10 @@ static inline void walt_dec_cumulative_runnable_avg(struct rq *rq,
|
|||
}
|
||||
|
||||
static inline void fixup_busy_time(struct task_struct *p, int new_cpu) { }
|
||||
static inline void walt_prepare_migrate(struct task_struct *p,
|
||||
int src_cpu, int new_cpu, bool locked) { }
|
||||
static inline void walt_finish_migrate(struct task_struct *p,
|
||||
int src_cpu, int new_cpu, bool locked) { }
|
||||
static inline void init_new_task_load(struct task_struct *p)
|
||||
{
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue