From e41f8e75a261cae458273c49fc18a295b6b6a295 Mon Sep 17 00:00:00 2001 From: Sorayukii Date: Sun, 15 Feb 2026 07:06:47 +0700 Subject: [PATCH] sched: Import Sony Changes Signed-off-by: Sorayukii --- include/linux/sched.h | 2 + kernel/sched/boost.c | 17 +- kernel/sched/core.c | 30 ++- kernel/sched/deadline.c | 6 + kernel/sched/fair.c | 154 ++++++++--- kernel/sched/rt.c | 4 + kernel/sched/sched.h | 2 + kernel/sched/sched_avg.c | 24 +- kernel/sched/walt.c | 567 ++++++++++++++++++++++++++++++++++++--- kernel/sched/walt.h | 14 + 10 files changed, 730 insertions(+), 90 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index c6875c615d7d..bdfb79317bdb 100755 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2526,8 +2526,10 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, #define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */ #define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */ #define PF_MEMSTALL 0x01000000 /* Stalled due to lack of memory */ +#define PF_MIGRATE_ED_TASK 0x02000000 /* early detection task migration */ #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_allowed */ #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ +#define PF_MIGRATE_CUM_ADJ_TASK 0x10000000 /* cumulative task adjustment should be done */ #define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */ #define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */ #define PF_SUSPEND_TASK 0x80000000 /* this thread called freeze_processes and should not be frozen */ diff --git a/kernel/sched/boost.c b/kernel/sched/boost.c index 9c2269fbd6ae..0ca2ad392719 100644 --- a/kernel/sched/boost.c +++ b/kernel/sched/boost.c @@ -40,6 +40,7 @@ static void boost_kick_cpus(void) { int i; struct cpumask kick_mask; + u32 nr_running; if (boost_policy != SCHED_BOOST_ON_BIG) return; @@ -47,8 +48,20 @@ static void boost_kick_cpus(void) cpumask_andnot(&kick_mask, cpu_online_mask, cpu_isolated_mask); for_each_cpu(i, &kick_mask) { - if (cpu_capacity(i) != max_capacity) - boost_kick(i); + /* + * kick only "small" cluster + */ + if (cpu_capacity(i) != max_capacity) { + nr_running = ACCESS_ONCE(cpu_rq(i)->nr_running); + + /* + * make sense to interrupt CPU if its run-queue + * has something running in order to check for + * migration afterwards, otherwise skip it. + */ + if (nr_running) + boost_kick(i); + } } } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 1e7203e61b22..2e83549c96d8 100755 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -165,6 +165,10 @@ int sysctl_sched_rt_runtime = 950000; /* cpus with isolated domains */ cpumask_var_t cpu_isolated_map; +#ifdef CONFIG_NO_HZ_COMMON +DEFINE_PER_CPU(atomic_t, claim_wake_up_cpu) = ATOMIC_INIT(0); +#endif + struct rq * lock_rq_of(struct task_struct *p, struct rq_flags *flags) { @@ -1014,14 +1018,15 @@ static struct rq *move_queued_task(struct rq *rq, struct task_struct *p, int new p->on_rq = TASK_ON_RQ_MIGRATING; dequeue_task(rq, p, 0); - double_lock_balance(rq, cpu_rq(new_cpu)); + walt_prepare_migrate(p, cpu_of(rq), new_cpu, true); set_task_cpu(p, new_cpu); - double_rq_unlock(cpu_rq(new_cpu), rq); + raw_spin_unlock(&rq->lock); rq = cpu_rq(new_cpu); raw_spin_lock(&rq->lock); BUG_ON(task_cpu(p) != new_cpu); + walt_finish_migrate(p, cpu_of(rq), new_cpu, true); enqueue_task(rq, p, 0); p->on_rq = TASK_ON_RQ_QUEUED; check_preempt_curr(rq, p, 0); @@ -1288,8 +1293,6 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) p->sched_class->migrate_task_rq(p); p->se.nr_migrations++; perf_event_task_migrate(p); - - fixup_busy_time(p, new_cpu); } __set_task_cpu(p, new_cpu); @@ -1305,7 +1308,9 @@ static void __migrate_swap_task(struct task_struct *p, int cpu) p->on_rq = TASK_ON_RQ_MIGRATING; deactivate_task(src_rq, p, 0); + walt_prepare_migrate(p, cpu_of(src_rq), cpu, true); set_task_cpu(p, cpu); + walt_finish_migrate(p, cpu_of(src_rq), cpu, true); activate_task(dst_rq, p, 0); p->on_rq = TASK_ON_RQ_QUEUED; check_preempt_curr(dst_rq, p, 0); @@ -1676,6 +1681,10 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) (cpu_isolated(cpu) && !allow_isolated)) cpu = select_fallback_rq(task_cpu(p), p, allow_isolated); +#ifdef CONFIG_NO_HZ_COMMON + if (unlikely(!atomic_read(&per_cpu(claim_wake_up_cpu, cpu)))) + atomic_set(&per_cpu(claim_wake_up_cpu, cpu), 1); +#endif return cpu; } @@ -1950,13 +1959,16 @@ bool cpus_share_cache(int this_cpu, int that_cpu) } #endif /* CONFIG_SMP */ -static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) +static void ttwu_queue(struct task_struct *p, int cpu, int prev_cpu, int wake_flags) { struct rq *rq = cpu_rq(cpu); struct rq_flags rf; #if defined(CONFIG_SMP) if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { + if (prev_cpu != cpu) + walt_finish_migrate(p, prev_cpu, cpu, false); + sched_clock_cpu(cpu); /* sync clocks x-cpu */ ttwu_queue_remote(p, cpu, wake_flags); return; @@ -1965,6 +1977,10 @@ static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) raw_spin_lock(&rq->lock); rq_pin_lock(rq, &rf); + + if (prev_cpu != cpu) + walt_finish_migrate(p, prev_cpu, cpu, true); + ttwu_do_activate(rq, p, wake_flags, &rf); rq_unpin_lock(rq, &rf); raw_spin_unlock(&rq->lock); @@ -2186,14 +2202,16 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) if (src_cpu != cpu) { wake_flags |= WF_MIGRATED; psi_ttwu_dequeue(p); + walt_prepare_migrate(p, src_cpu, cpu, false); set_task_cpu(p, cpu); + /* walt_finish_migrate is done in ttwu_queue */ notif_required = true; } note_task_waking(p, wallclock); #endif /* CONFIG_SMP */ - ttwu_queue(p, cpu, wake_flags); + ttwu_queue(p, cpu, src_cpu, wake_flags); stat: ttwu_stat(p, cpu, wake_flags); out: diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 26ea5c8130ec..6c76c995ae42 100755 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -272,7 +272,9 @@ static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p double_lock_balance(rq, later_rq); } + walt_prepare_migrate(p, cpu_of(rq), cpu_of(later_rq), true); set_task_cpu(p, later_rq->cpu); + walt_finish_migrate(p, cpu_of(rq), cpu_of(later_rq), true); double_unlock_balance(later_rq, rq); return later_rq; @@ -1632,7 +1634,9 @@ retry: next_task->on_rq = TASK_ON_RQ_MIGRATING; deactivate_task(rq, next_task, 0); + walt_prepare_migrate(next_task, cpu_of(rq), cpu_of(later_rq), true); set_task_cpu(next_task, later_rq->cpu); + walt_finish_migrate(next_task, cpu_of(rq), cpu_of(later_rq), true); activate_task(later_rq, next_task, 0); next_task->on_rq = TASK_ON_RQ_QUEUED; ret = 1; @@ -1722,7 +1726,9 @@ static void pull_dl_task(struct rq *this_rq) p->on_rq = TASK_ON_RQ_MIGRATING; deactivate_task(src_rq, p, 0); + walt_prepare_migrate(p, cpu_of(src_rq), cpu_of(this_rq), true); set_task_cpu(p, this_cpu); + walt_finish_migrate(p, cpu_of(src_rq), cpu_of(this_rq), true); activate_task(this_rq, p, 0); p->on_rq = TASK_ON_RQ_QUEUED; dmin = p->dl.deadline; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 73bb0233e6a7..7a5afdf4da6e 100755 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5051,6 +5051,10 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!se) { add_nr_running(rq, 1); + + if (unlikely(p->nr_cpus_allowed == 1)) + rq->nr_pinned_tasks++; + inc_rq_walt_stats(rq, p); } @@ -5144,6 +5148,10 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!se) { sub_nr_running(rq, 1); + + if (unlikely(p->nr_cpus_allowed == 1)) + rq->nr_pinned_tasks--; + dec_rq_walt_stats(rq, p); } @@ -8096,12 +8104,7 @@ again: set_next_entity(cfs_rq, se); } - if (hrtick_enabled(rq)) - hrtick_start_fair(rq, p); - - rq->misfit_task = !task_fits_max(p, rq->cpu); - - return p; + goto done; simple: cfs_rq = &rq->cfs; #endif @@ -8119,6 +8122,16 @@ simple: p = task_of(se); +done: __maybe_unused +#ifdef CONFIG_SMP + /* + * Move the next running task to the front of + * the list, so our cfs_tasks list becomes MRU + * one. + */ + list_move(&p->se.group_node, &rq->cfs_tasks); +#endif + if (hrtick_enabled(rq)) hrtick_start_fair(rq, p); @@ -8530,14 +8543,10 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) if (energy_aware() && !env->dst_rq->rd->overutilized && env->idle == CPU_NEWLY_IDLE && !task_in_related_thread_group(p)) { - long util_cum_dst, util_cum_src; - unsigned long demand; + long new_util_cum = cpu_util_cum(env->dst_cpu, task_util(p)); - demand = task_util(p); - util_cum_dst = cpu_util_cum(env->dst_cpu, 0) + demand; - util_cum_src = cpu_util_cum(env->src_cpu, 0) - demand; - - if (util_cum_dst > util_cum_src) + if (add_capacity_margin(new_util_cum, env->dst_cpu) > + capacity_curr_of(env->dst_cpu)) return 0; } @@ -8590,11 +8599,10 @@ static void detach_task(struct task_struct *p, struct lb_env *env) p->on_rq = TASK_ON_RQ_MIGRATING; deactivate_task(env->src_rq, p, 0); - double_lock_balance(env->src_rq, env->dst_rq); + walt_prepare_migrate(p, env->src_cpu, env->dst_cpu, true); set_task_cpu(p, env->dst_cpu); if (task_in_related_thread_group(p)) env->flags |= LBF_MOVED_RELATED_THREAD_GROUP_TASK; - double_unlock_balance(env->src_rq, env->dst_rq); } /* @@ -8605,11 +8613,12 @@ static void detach_task(struct task_struct *p, struct lb_env *env) */ static struct task_struct *detach_one_task(struct lb_env *env) { - struct task_struct *p, *n; + struct task_struct *p; lockdep_assert_held(&env->src_rq->lock); - list_for_each_entry_safe(p, n, &env->src_rq->cfs_tasks, se.group_node) { + list_for_each_entry_reverse(p, + &env->src_rq->cfs_tasks, se.group_node) { if (!can_migrate_task(p, env)) continue; @@ -8663,7 +8672,15 @@ redo: if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) break; - p = list_first_entry(tasks, struct task_struct, se.group_node); + /* + * Another CPU can place tasks, since we do not hold dst_rq lock + * while doing balancing. If newly idle CPU already got something, + * give up to reduce a latency. + */ + if (env->idle == CPU_NEWLY_IDLE && env->dst_rq->nr_running > 0) + break; + + p = list_last_entry(tasks, struct task_struct, se.group_node); env->loop++; /* We've more or less seen every task there is, call it quits */ @@ -8703,6 +8720,7 @@ redo: */ if (((cpu_rq(env->src_cpu)->nr_running > 2) || (env->flags & LBF_IGNORE_BIG_TASKS)) && + (env->idle != CPU_NEWLY_IDLE) && ((load / 2) > env->imbalance)) goto next; @@ -8731,7 +8749,7 @@ redo: continue; next: - list_move_tail(&p->se.group_node, tasks); + list_move(&p->se.group_node, tasks); } if (env->flags & (LBF_IGNORE_BIG_TASKS | @@ -8756,11 +8774,13 @@ next: /* * attach_task() -- attach the task detached by detach_task() to its new rq. */ -static void attach_task(struct rq *rq, struct task_struct *p) +static void attach_task(struct rq *rq, + struct task_struct *p, struct lb_env *env) { lockdep_assert_held(&rq->lock); BUG_ON(task_rq(p) != rq); + walt_finish_migrate(p, env->src_cpu, env->dst_cpu, true); activate_task(rq, p, 0); p->on_rq = TASK_ON_RQ_QUEUED; check_preempt_curr(rq, p, 0); @@ -8770,10 +8790,11 @@ static void attach_task(struct rq *rq, struct task_struct *p) * attach_one_task() -- attaches the task returned from detach_one_task() to * its new rq. */ -static void attach_one_task(struct rq *rq, struct task_struct *p) +static void attach_one_task(struct rq *rq, + struct task_struct *p, struct lb_env *env) { raw_spin_lock(&rq->lock); - attach_task(rq, p); + attach_task(rq, p, env); raw_spin_unlock(&rq->lock); } @@ -8781,21 +8802,23 @@ static void attach_one_task(struct rq *rq, struct task_struct *p) * attach_tasks() -- attaches all tasks detached by detach_tasks() to their * new rq. */ -static void attach_tasks(struct lb_env *env) +static void attach_tasks(struct lb_env *env, bool dst_locked) { struct list_head *tasks = &env->tasks; struct task_struct *p; - raw_spin_lock(&env->dst_rq->lock); + if (!dst_locked) + raw_spin_lock(&env->dst_rq->lock); while (!list_empty(tasks)) { p = list_first_entry(tasks, struct task_struct, se.group_node); list_del_init(&p->se.group_node); - attach_task(env->dst_rq, p); + attach_task(env->dst_rq, p, env); } - raw_spin_unlock(&env->dst_rq->lock); + if (!dst_locked) + raw_spin_unlock(&env->dst_rq->lock); } #ifdef CONFIG_FAIR_GROUP_SCHED @@ -9447,7 +9470,7 @@ static bool update_sd_pick_busiest(struct lb_env *env, * per-cpu capacity. No reason to pull tasks to less capable cpus. */ if (sgs->sum_nr_running <= sgs->group_weight && - group_smaller_cpu_capacity(sds->local, sg)) + !group_smaller_cpu_capacity(sg, sds->local)) return false; asym_packing: @@ -10249,7 +10272,6 @@ redo: * correctly treated as an imbalance. */ env.flags |= LBF_ALL_PINNED; - env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); more_balance: raw_spin_lock_irqsave(&busiest->lock, flags); @@ -10261,11 +10283,26 @@ more_balance: goto no_move; } + /* + * Set loop_max when rq's lock is taken to prevent a race. + */ + env.loop_max = min(sysctl_sched_nr_migrate, + busiest->cfs.h_nr_running); + /* * cur_ld_moved - load moved in current iteration * ld_moved - cumulative load moved across iterations */ cur_ld_moved = detach_tasks(&env); + if (cur_ld_moved) { + if (same_freq_domain(env.src_cpu, env.dst_cpu)) + /* + * If we move tasks within the same freq domain, it is + * important to acquire dst_rq before releasing src_rq + * to prevent potential load drop. + */ + double_lock_balance(env.src_rq, env.dst_rq); + } /* * We've detached some tasks from busiest_rq. Every @@ -10278,7 +10315,13 @@ more_balance: raw_spin_unlock(&busiest->lock); if (cur_ld_moved) { - attach_tasks(&env); + if (same_freq_domain(env.src_cpu, env.dst_cpu)) { + attach_tasks(&env, true); + raw_spin_unlock(&env.dst_rq->lock); + } else { + attach_tasks(&env, false); + } + ld_moved += cur_ld_moved; } @@ -10746,12 +10789,12 @@ out_unlock: if (push_task) { if (push_task_detached) - attach_one_task(target_rq, push_task); + attach_one_task(target_rq, push_task, &env); put_task_struct(push_task); } if (p) - attach_one_task(target_rq, p); + attach_one_task(target_rq, p, &env); local_irq_enable(); @@ -10770,6 +10813,31 @@ static inline int on_null_domain(struct rq *rq) * needed, they will kick the idle load balancer, which then does idle * load balancing for all the idle CPUs. */ +static inline int +get_first_not_claimed_cpu(cpumask_t *cpumask) +{ + int ilb; + + /* + * If a CPU is claimed it means that TIF_NEED_RESCHED + * is on its way or is already in place. In first case + * a nohz_idle_balance work will most likely be stopped + * or even canceled earlier, because of pending task. + * In second one a CPU is not kicked for doing a load + * balancing. + * + * Therefore exclude CPUs (among nohz.idle_cpus_mask) + * which have already been claimed for waking up a task + * on, preferring other CPUs to do NO_HZ balancing. + */ + for_each_cpu(ilb, cpumask) { + if (!atomic_read(&per_cpu(claim_wake_up_cpu, ilb))) + break; + } + + return ilb; +} + static inline int find_new_ilb(int type) { int ilb = nr_cpu_ids; @@ -10789,7 +10857,8 @@ static inline int find_new_ilb(int type) sched_domain_span(sd)); cpumask_andnot(&cpumask, &cpumask, cpu_isolated_mask); - ilb = cpumask_first(&cpumask); + + ilb = get_first_not_claimed_cpu(&cpumask); } rcu_read_unlock(); @@ -10800,7 +10869,8 @@ static inline int find_new_ilb(int type) (cpu_overutilized(cpu) && rq->nr_running > 1))) { cpumask_andnot(&cpumask, nohz.idle_cpus_mask, cpu_isolated_mask); - ilb = cpumask_first(&cpumask); + + ilb = get_first_not_claimed_cpu(&cpumask); } } @@ -10898,6 +10968,8 @@ void nohz_balance_enter_idle(int cpu) if (!cpu_active(cpu)) return; + atomic_set(&per_cpu(claim_wake_up_cpu, cpu), 0); + if (test_bit(NOHZ_TICK_STOPPED, nohz_flags(cpu))) return; @@ -11150,6 +11222,22 @@ static inline bool nohz_kick_needed(struct rq *rq, int *type) return true; /* Do idle load balance if there have misfit task */ + if (unlikely(rq->nr_pinned_tasks > 0)) { + int delta = rq->nr_running - rq->nr_pinned_tasks; + + /* + * Check if it is possible to "unload" this CPU in case + * of having pinned/affine tasks. Do not disturb idle + * core if one of the below condition is true: + * + * - there is one pinned task and it is not "current" + * - all tasks are pinned to this CPU + */ + if (delta < 2) + if (current->nr_cpus_allowed > 1 || !delta) + return false; + } + if (energy_aware()) return rq->misfit_task; diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 026a471bf59a..0e90e364fca7 100755 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -2117,7 +2117,9 @@ retry: next_task->on_rq = TASK_ON_RQ_MIGRATING; deactivate_task(rq, next_task, 0); + walt_prepare_migrate(next_task, cpu_of(rq), cpu_of(lowest_rq), true); set_task_cpu(next_task, lowest_rq->cpu); + walt_finish_migrate(next_task, cpu_of(rq), cpu_of(lowest_rq), true); activate_task(lowest_rq, next_task, 0); next_task->on_rq = TASK_ON_RQ_QUEUED; ret = 1; @@ -2391,7 +2393,9 @@ static void pull_rt_task(struct rq *this_rq) p->on_rq = TASK_ON_RQ_MIGRATING; deactivate_task(src_rq, p, 0); + walt_prepare_migrate(p, cpu_of(src_rq), this_cpu, true); set_task_cpu(p, this_cpu); + walt_finish_migrate(p, cpu_of(src_rq), this_cpu, true); activate_task(this_rq, p, 0); p->on_rq = TASK_ON_RQ_QUEUED; /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index a9db959ab2ae..396ca60576c9 100755 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -722,6 +722,7 @@ struct rq { #endif #define CPU_LOAD_IDX_MAX 5 unsigned long cpu_load[CPU_LOAD_IDX_MAX]; + unsigned int nr_pinned_tasks; unsigned int misfit_task; #ifdef CONFIG_NO_HZ_COMMON #ifdef CONFIG_SMP @@ -2303,6 +2304,7 @@ enum rq_nohz_flag_bits { #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags) +DECLARE_PER_CPU(atomic_t, claim_wake_up_cpu); extern void nohz_balance_exit_idle(unsigned int cpu); #else static inline void nohz_balance_exit_idle(unsigned int cpu) { } diff --git a/kernel/sched/sched_avg.c b/kernel/sched/sched_avg.c index 10d7f1ba3b6e..a1c256c0f0f8 100644 --- a/kernel/sched/sched_avg.c +++ b/kernel/sched/sched_avg.c @@ -191,20 +191,28 @@ unsigned int sched_get_cpu_util(int cpu) unsigned long capacity, flags; unsigned int busy; - raw_spin_lock_irqsave(&rq->lock, flags); - - util = rq->cfs.avg.util_avg; - capacity = capacity_orig_of(cpu); - #ifdef CONFIG_SCHED_WALT if (!walt_disabled && sysctl_sched_use_walt_cpu_util) { - util = rq->prev_runnable_sum + rq->grp_time.prev_runnable_sum; - util = div64_u64(util, - sched_ravg_window >> SCHED_CAPACITY_SHIFT); + u32 prev_run_sum, group_run_sum; + + util = walt_get_prev_group_run_sum(rq); + group_run_sum = (u32) (util >> 32); + prev_run_sum = (u32) util; + + util = prev_run_sum + group_run_sum; + util = div64_u64(util, sched_ravg_window >> + SCHED_CAPACITY_SHIFT); + + goto done; } #endif + + raw_spin_lock_irqsave(&rq->lock, flags); + util = rq->cfs.avg.util_avg; raw_spin_unlock_irqrestore(&rq->lock, flags); +done: __maybe_unused + capacity = capacity_orig_of(cpu); util = (util >= capacity) ? capacity : util; busy = div64_ul((util * 100), capacity); return busy; diff --git a/kernel/sched/walt.c b/kernel/sched/walt.c index 578e4c48a777..5023380dcb9f 100755 --- a/kernel/sched/walt.c +++ b/kernel/sched/walt.c @@ -52,6 +52,28 @@ u64 walt_load_reported_window; static struct irq_work walt_cpufreq_irq_work; static struct irq_work walt_migration_irq_work; +static DEFINE_RAW_SPINLOCK(speedchange_cpumask_lock); +static cpumask_t speedchange_cpumask = CPU_MASK_NONE; + +static DEFINE_PER_CPU(atomic64_t, cycles) = ATOMIC64_INIT(0); +static DEFINE_PER_CPU(atomic64_t, last_cc_update) = ATOMIC64_INIT(0); +static DEFINE_PER_CPU(atomic64_t, prev_group_runnable_sum) = ATOMIC64_INIT(0); + +static inline void +walt_commit_prev_group_run_sum(struct rq *rq) +{ + u64 val; + + val = rq->grp_time.prev_runnable_sum; + val = (val << 32) | rq->prev_runnable_sum; + atomic64_set(&per_cpu(prev_group_runnable_sum, cpu_of(rq)), val); +} + +u64 +walt_get_prev_group_run_sum(struct rq *rq) +{ + return (u64) atomic64_read(&per_cpu(prev_group_runnable_sum, cpu_of(rq))); +} void walt_fixup_cumulative_runnable_avg(struct rq *rq, @@ -356,20 +378,24 @@ int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb) return 0; } +#define THRESH_CC_UPDATE (2 * NSEC_PER_USEC) + /* * Assumes rq_lock is held and wallclock was recorded in the same critical * section as this function's invocation. */ static inline u64 read_cycle_counter(int cpu, u64 wallclock) { - struct rq *rq = cpu_rq(cpu); + u64 delta; - if (rq->last_cc_update != wallclock) { - rq->cycles = cpu_cycle_counter_cb.get_cpu_cycle_counter(cpu); - rq->last_cc_update = wallclock; + delta = wallclock - atomic64_read(&per_cpu(last_cc_update, cpu)); + if (delta > THRESH_CC_UPDATE) { + atomic64_set(&per_cpu(cycles, cpu), + cpu_cycle_counter_cb.get_cpu_cycle_counter(cpu)); + atomic64_set(&per_cpu(last_cc_update, cpu), wallclock); } - return rq->cycles; + return atomic64_read(&per_cpu(cycles, cpu)); } static void update_task_cpu_cycles(struct task_struct *p, int cpu, @@ -607,6 +633,8 @@ static inline void account_load_subtractions(struct rq *rq) ls[i].new_subs = 0; } + walt_commit_prev_group_run_sum(rq); + BUG_ON((s64)rq->prev_runnable_sum < 0); BUG_ON((s64)rq->curr_runnable_sum < 0); BUG_ON((s64)rq->nt_prev_runnable_sum < 0); @@ -921,6 +949,371 @@ done: double_rq_unlock(src_rq, dest_rq); } +static void +migrate_prepare_top_tasks(struct task_struct *p, struct rq *src_rq) +{ + int index; + int top_index; + u32 curr_window = p->ravg.curr_window; + u32 prev_window = p->ravg.prev_window; + u8 src = src_rq->curr_table; + u8 *src_table; + + if (curr_window) { + src_table = src_rq->top_tasks[src]; + index = load_to_index(curr_window); + src_table[index] -= 1; + + if (!src_table[index]) + __clear_bit(NUM_LOAD_INDICES - index - 1, + src_rq->top_tasks_bitmap[src]); + + + top_index = src_rq->curr_top; + if (index == top_index && !src_table[index]) + src_rq->curr_top = get_top_index( + src_rq->top_tasks_bitmap[src], top_index); + } + + if (prev_window) { + src = 1 - src; + src_table = src_rq->top_tasks[src]; + index = load_to_index(prev_window); + src_table[index] -= 1; + + if (!src_table[index]) + __clear_bit(NUM_LOAD_INDICES - index - 1, + src_rq->top_tasks_bitmap[src]); + + top_index = src_rq->prev_top; + if (index == top_index && !src_table[index]) + src_rq->prev_top = get_top_index( + src_rq->top_tasks_bitmap[src], top_index); + } +} + +static void +migrate_finish_top_tasks(struct task_struct *p, struct rq *dst_rq) +{ + int index; + u32 curr_window = p->ravg.curr_window; + u32 prev_window = p->ravg.prev_window; + u8 dst = dst_rq->curr_table; + u8 *dst_table; + + if (curr_window) { + dst_table = dst_rq->top_tasks[dst]; + index = load_to_index(curr_window); + dst_table[index] += 1; + + if (dst_table[index] == 1) + __set_bit(NUM_LOAD_INDICES - index - 1, + dst_rq->top_tasks_bitmap[dst]); + + if (index > dst_rq->curr_top) + dst_rq->curr_top = index; + } + + if (prev_window) { + dst = 1 - dst; + dst_table = dst_rq->top_tasks[dst]; + index = load_to_index(prev_window); + dst_table[index] += 1; + + if (dst_table[index] == 1) + __set_bit(NUM_LOAD_INDICES - index - 1, + dst_rq->top_tasks_bitmap[dst]); + + if (index > dst_rq->prev_top) + dst_rq->prev_top = index; + } +} + +void walt_prepare_migrate(struct task_struct *p, + int src_cpu, int new_cpu, bool locked) +{ + struct rq *src_rq = cpu_rq(src_cpu); + struct related_thread_group *grp; + u64 *src_curr_runnable_sum; + u64 *src_prev_runnable_sum; + u64 *src_nt_curr_runnable_sum; + u64 *src_nt_prev_runnable_sum; + u64 wallclock; + bool new_task; + + if (sched_disable_window_stats) + return; + + if (!p->on_rq && p->state != TASK_WAKING) + return; + + if (exiting_task(p)) { + clear_ed_task(p, src_rq); + return; + } + + BUG_ON(!irqs_disabled()); + + if (p->state == TASK_WAKING) { + /* + * Is there any contribution? zero? If yes, get rid + * of extra overhead since there is nothing to move + * between CPUs. + * + * In that case what we need is to update a task with + * a new CPU's cycle counter and nothing more. An update + * is done only if a "wake-up migration" occurs between + * two not the same frequency domains, because: + * - their counter cycles (CC) are far different + * from each other. + * + * If a frequency domain is the same we skip an update + * because: + * - CC are synchronized within the same domain; + * - to get rid of quite expensive CC reading. + */ + if (!p->ravg.curr_window && !p->ravg.prev_window) { + if (!same_freq_domain(src_cpu, new_cpu)) + update_task_cpu_cycles(p, new_cpu, ktime_get_ns()); + return; + } + } + + if (!locked) + raw_spin_lock(&src_rq->lock); + + lockdep_assert_held(&src_rq->lock); + wallclock = ktime_get_ns(); + + update_task_ravg(task_rq(p)->curr, task_rq(p), + TASK_UPDATE, wallclock, 0); + update_task_ravg(p, task_rq(p), TASK_MIGRATE, + wallclock, 0); + + /* + * See a comment above when an update is done + */ + if (!same_freq_domain(src_cpu, new_cpu)) + update_task_cpu_cycles(p, new_cpu, wallclock); + + /* + * When a task is migrating during the wakeup, adjust + * the task's contribution towards cumulative window + * demand. + */ + if (p->state == TASK_WAKING && + p->last_sleep_ts >= src_rq->window_start) { + walt_fixup_cum_window_demand(src_rq, -(s64)p->ravg.demand); + p->flags |= PF_MIGRATE_CUM_ADJ_TASK; + } + + new_task = is_new_task(p); + /* Protected by rq_lock */ + grp = p->grp; + + /* + * For frequency aggregation, we continue to do migration fixups + * even for intra cluster migrations. This is because, the aggregated + * load has to reported on a single CPU regardless. + */ + if (grp) { + struct group_cpu_time *cpu_time; + + cpu_time = &src_rq->grp_time; + src_curr_runnable_sum = &cpu_time->curr_runnable_sum; + src_prev_runnable_sum = &cpu_time->prev_runnable_sum; + src_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + src_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + + if (p->ravg.curr_window) { + *src_curr_runnable_sum -= p->ravg.curr_window; + if (new_task) + *src_nt_curr_runnable_sum -= + p->ravg.curr_window; + } + + if (p->ravg.prev_window) { + *src_prev_runnable_sum -= p->ravg.prev_window; + if (new_task) + *src_nt_prev_runnable_sum -= + p->ravg.prev_window; + } + } else if (!same_freq_domain(src_cpu, new_cpu)) { + src_rq->curr_runnable_sum -= + p->ravg.curr_window_cpu[src_cpu]; + src_rq->prev_runnable_sum -= + p->ravg.prev_window_cpu[src_cpu]; + + if (new_task) { + src_rq->nt_curr_runnable_sum -= + p->ravg.curr_window_cpu[src_cpu]; + src_rq->nt_prev_runnable_sum -= + p->ravg.prev_window_cpu[src_cpu]; + } + + p->ravg.curr_window_cpu[src_cpu] = 0; + p->ravg.prev_window_cpu[src_cpu] = 0; + + update_cluster_load_subtractions(p, src_cpu, + src_rq->window_start, new_task); + } + + if ((s64) src_rq->prev_runnable_sum < 0) { + src_rq->prev_runnable_sum = 0; + WARN_ON(1); + } + + if ((s64) src_rq->curr_runnable_sum < 0) { + src_rq->curr_runnable_sum = 0; + WARN_ON(1); + } + + if ((s64) src_rq->nt_prev_runnable_sum < 0) { + src_rq->nt_prev_runnable_sum = 0; + WARN_ON(1); + } + + if ((s64) src_rq->nt_curr_runnable_sum < 0) { + src_rq->nt_curr_runnable_sum = 0; + WARN_ON(1); + } + + walt_commit_prev_group_run_sum(src_rq); + migrate_prepare_top_tasks(p, src_rq); + + if (src_rq->ed_task == p) { + p->flags |= PF_MIGRATE_ED_TASK; + src_rq->ed_task = NULL; + } + + if (!locked) + raw_spin_unlock(&src_rq->lock); +} + +void walt_finish_migrate(struct task_struct *p, + int src_cpu, int new_cpu, bool locked) +{ + struct rq *dest_rq = cpu_rq(new_cpu); + struct rq *src_rq = cpu_rq(src_cpu); + struct related_thread_group *grp; + u64 *dst_curr_runnable_sum; + u64 *dst_prev_runnable_sum; + u64 *dst_nt_curr_runnable_sum; + u64 *dst_nt_prev_runnable_sum; + u64 wallclock; + bool new_task; + + if (sched_disable_window_stats) + return; + + if (!p->on_rq && p->state != TASK_WAKING) + return; + + if (exiting_task(p)) + return; + + BUG_ON(!irqs_disabled()); + + if (!locked) + raw_spin_lock(&dest_rq->lock); + + lockdep_assert_held(&dest_rq->lock); + wallclock = ktime_get_ns(); + + update_task_ravg(dest_rq->curr, dest_rq, + TASK_UPDATE, wallclock, 0); + update_task_ravg(p, dest_rq, TASK_MIGRATE, + wallclock, 0); + + /* + * When a task is migrating during the wakeup, adjust + * the task's contribution towards cumulative window + * demand. + */ + if (p->state == TASK_WAKING && + p->flags & PF_MIGRATE_CUM_ADJ_TASK) { + walt_fixup_cum_window_demand(dest_rq, p->ravg.demand); + p->flags &= ~PF_MIGRATE_CUM_ADJ_TASK; + } + + new_task = is_new_task(p); + /* Protected by rq_lock */ + grp = p->grp; + + /* + * For frequency aggregation, we continue to do migration fixups + * even for intra cluster migrations. This is because, the aggregated + * load has to reported on a single CPU regardless. + */ + if (grp) { + struct group_cpu_time *cpu_time; + + cpu_time = &dest_rq->grp_time; + dst_curr_runnable_sum = &cpu_time->curr_runnable_sum; + dst_prev_runnable_sum = &cpu_time->prev_runnable_sum; + dst_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + dst_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + + if (p->ravg.curr_window) { + *dst_curr_runnable_sum += p->ravg.curr_window; + if (new_task) + *dst_nt_curr_runnable_sum += + p->ravg.curr_window; + } + + if (p->ravg.prev_window) { + *dst_prev_runnable_sum += p->ravg.prev_window; + if (new_task) + *dst_nt_prev_runnable_sum += + p->ravg.prev_window; + } + } else if (!same_freq_domain(src_cpu, new_cpu)) { + p->ravg.curr_window_cpu[new_cpu] = p->ravg.curr_window; + p->ravg.prev_window_cpu[new_cpu] = p->ravg.prev_window; + + dest_rq->curr_runnable_sum += p->ravg.curr_window; + dest_rq->prev_runnable_sum += p->ravg.prev_window; + + if (new_task) { + dest_rq->nt_curr_runnable_sum += + p->ravg.curr_window; + dest_rq->nt_prev_runnable_sum += + p->ravg.prev_window; + } + } + + walt_commit_prev_group_run_sum(dest_rq); + migrate_finish_top_tasks(p, dest_rq); + + if (!same_freq_domain(src_cpu, new_cpu)) { + /* + * If walt_cpufreq_irq_work has been recently queued + * and is in a pending state, do not push a migration + * job, because all magic will have been done anyway. + */ + if (!(smp_load_acquire(&walt_cpufreq_irq_work.flags) & + IRQ_WORK_PENDING)) { + raw_spin_lock(&speedchange_cpumask_lock); + src_rq->notif_pending = true; + dest_rq->notif_pending = true; + cpumask_set_cpu(src_cpu, &speedchange_cpumask); + cpumask_set_cpu(new_cpu, &speedchange_cpumask); + raw_spin_unlock(&speedchange_cpumask_lock); + irq_work_queue(&walt_migration_irq_work); + } + } + + if (p->flags & PF_MIGRATE_ED_TASK) { + if (!dest_rq->ed_task) + dest_rq->ed_task = p; + + p->flags &= ~PF_MIGRATE_ED_TASK; + } + + if (!locked) + raw_spin_unlock(&dest_rq->lock); +} + void set_window_start(struct rq *rq) { static int sync_cpu_available; @@ -1346,6 +1739,7 @@ static inline u64 scale_exec_time(u64 delta, struct rq *rq) u32 freq; freq = cpu_cycles_to_freq(rq->cc.cycles, rq->cc.time); + delta = DIV64_U64_ROUNDUP(delta * freq, max_possible_freq); delta *= rq->cluster->exec_scale_factor; delta >>= 10; @@ -1626,7 +2020,7 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, */ if (mark_start > window_start) { *curr_runnable_sum = scale_exec_time(irqtime, rq); - return; + goto done; } /* @@ -1642,11 +2036,11 @@ static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, /* Process the remaining IRQ busy time in the current window. */ delta = wallclock - window_start; rq->curr_runnable_sum = scale_exec_time(delta, rq); - - return; } done: + walt_commit_prev_group_run_sum(rq); + if (!is_idle_task(p) && !exiting_task(p)) update_top_tasks(p, rq, old_curr_window, new_window, full_window); @@ -1954,6 +2348,26 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, else rq->cc.time = wallclock - p->ravg.mark_start; BUG_ON((s64)rq->cc.time < 0); + + /* + * It can happen when a time between two updates + * (for example TASK_UPDATE) is very short. The reason + * is a read_cycle_counter function now can return + * a previous/same CC value if a last read was within + * a THRESH_CC_UPDATE threshold. + * + * In that particular scenario use current CPU OPP + * to scale such task's delta contributions which + * are smaller than THRESH_CC_UPDATE interval. + * + * The aim of using a current frequency is because: + * - an estimated one can be zero; + * - we do not want to lose samples due to that. + */ + if (unlikely(!rq->cc.cycles)) { + rq->cc.cycles = cpu_cur_freq(cpu); + rq->cc.time = 1; + } } p->cpu_cycles = cur_cycles; @@ -1961,17 +2375,50 @@ update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, trace_sched_get_task_cpu_cycles(cpu, event, rq->cc.cycles, rq->cc.time, p); } -static inline void run_walt_irq_work(u64 old_window_start, struct rq *rq) +static inline void +run_walt_irq_work(u64 old_window_start, struct rq *rq, u64 wallclock) { u64 result; - if (old_window_start == rq->window_start) + if (likely(old_window_start == rq->window_start)) return; result = atomic64_cmpxchg(&walt_irq_work_lastq_ws, old_window_start, rq->window_start); - if (result == old_window_start) + if (result == old_window_start) { + /* + * Usually a window update is done each sched_ravg_window + * interval, but there are exceptions. If a CPU has not been + * updated during more than one full window (most of the cases + * when it was idle) we end up in double update within the + * short period of time. + * + * ws_old(a) ws(b) wc(c) + * | | | + * V V V + * ---|-------|-------|-------|------ | + * | | + * |<---- nr_full_win ---->| + * + * According to the picture, the window was updated in point (b) + * as soon as 3 full windows have passed. At the same time the + * second roll-over is going to be done quite soon, since the + * wallclock is about to cross and close the current window. + * + * If a remaining time is less than sched_ravg_window / 2 we do + * not trigger walt_cpufreq_irq_work because: + * + * - it most likely spurious decision after been idle; + * - we could ignore that decision, but walt_irq_work_roll_over + * is expansive and affects the preemption off latency; + * - there will be one more roll-over. + */ + result = sched_ravg_window - (wallclock % sched_ravg_window); + if (unlikely(result < (sched_ravg_window / 2))) + return; + sched_irq_work_queue(&walt_cpufreq_irq_work); + } } /* Reflect task activity on its demand and cpu's busy time statistics */ @@ -2009,7 +2456,7 @@ void update_task_ravg(struct task_struct *p, struct rq *rq, int event, done: p->ravg.mark_start = wallclock; - run_walt_irq_work(old_window_start, rq); + run_walt_irq_work(old_window_start, rq, wallclock); } u32 sched_get_init_task_load(struct task_struct *p) @@ -3169,6 +3616,7 @@ static void transfer_busy_time(struct rq *rq, struct related_thread_group *grp, p->ravg.curr_window_cpu[cpu] = p->ravg.curr_window; p->ravg.prev_window_cpu[cpu] = p->ravg.prev_window; + walt_commit_prev_group_run_sum(rq); trace_sched_migration_update_sum(p, migrate_type, rq); BUG_ON((s64)*src_curr_runnable_sum < 0); @@ -3241,24 +3689,77 @@ done: return ret; } +static inline void +walt_irq_work_migration(struct irq_work *irq_work) +{ + struct sched_cluster *cluster; + cpumask_t tmp_mask; + int cpu; + + raw_spin_lock(&speedchange_cpumask_lock); + tmp_mask = speedchange_cpumask; + cpumask_clear(&speedchange_cpumask); + raw_spin_unlock(&speedchange_cpumask_lock); + + /* + * walt_irq_work_migration can be run simultaneously + * on different cores. That is why a tmp_mask may become + * empty for one of them. Thus, if there are no CPUs + * to check than we are done. + */ + if (!cpumask_weight(&tmp_mask)) + return; + + for_each_sched_cluster(cluster) { + u64 aggr_grp_load = 0; + u64 wc; + + for_each_cpu(cpu, &cluster->cpus) + raw_spin_lock(&cpu_rq(cpu)->lock); + + wc = ktime_get_ns(); + for_each_cpu(cpu, &cluster->cpus) + update_task_ravg(cpu_rq(cpu)->curr, + cpu_rq(cpu), TASK_UPDATE, wc, 0); + + raw_spin_lock(&cluster->load_lock); + for_each_cpu(cpu, &cluster->cpus) + aggr_grp_load += cpu_rq(cpu)->grp_time.prev_runnable_sum; + + cluster->aggr_grp_load = aggr_grp_load; + raw_spin_unlock(&cluster->load_lock); + + for_each_cpu(cpu, &cluster->cpus) { + int nflag = SCHED_CPUFREQ_WALT; + + struct rq* rq = cpu_rq(cpu); + + if (rq->notif_pending) { + nflag |= SCHED_CPUFREQ_INTERCLUSTER_MIG; + rq->notif_pending = false; + } else { + nflag |= SCHED_CPUFREQ_FORCE_UPDATE; + } + + cpufreq_update_util(rq, nflag); + } + for_each_cpu(cpu, &cluster->cpus) + raw_spin_unlock(&cpu_rq(cpu)->lock); + } +} + /* * Runs in hard-irq context. This should ideally run just after the latest * window roll-over. */ -void walt_irq_work(struct irq_work *irq_work) +void walt_irq_work_roll_over(struct irq_work *irq_work) { struct sched_cluster *cluster; struct rq *rq; int cpu; u64 wc, total_grp_load = 0; - int flag = SCHED_CPUFREQ_WALT; - bool is_migration = false; int level = 0; - /* Am I the window rollover work or the migration work? */ - if (irq_work == &walt_migration_irq_work) - is_migration = true; - for_each_cpu(cpu, cpu_possible_mask) { if (level == 0) raw_spin_lock(&cpu_rq(cpu)->lock); @@ -3295,30 +3796,14 @@ void walt_irq_work(struct irq_work *irq_work) if (total_grp_load) walt_update_coloc_boost_load(); - for_each_sched_cluster(cluster) { - for_each_cpu(cpu, &cluster->cpus) { - int nflag = flag; - - rq = cpu_rq(cpu); - - if (is_migration) { - if (rq->notif_pending) { - nflag |= SCHED_CPUFREQ_INTERCLUSTER_MIG; - rq->notif_pending = false; - } else { - nflag |= SCHED_CPUFREQ_FORCE_UPDATE; - } - } - - cpufreq_update_util(rq, nflag); - } - } + for_each_sched_cluster(cluster) + for_each_cpu(cpu, &cluster->cpus) + cpufreq_update_util(cpu_rq(cpu), SCHED_CPUFREQ_WALT); for_each_cpu(cpu, cpu_possible_mask) raw_spin_unlock(&cpu_rq(cpu)->lock); - if (!is_migration) - core_ctl_check(this_rq()->window_start); + core_ctl_check(this_rq()->window_start); } void walt_rotation_checkpoint(int nr_big) @@ -3367,8 +3852,8 @@ void walt_sched_init(struct rq *rq) int j; cpumask_set_cpu(cpu_of(rq), &rq->freq_domain_cpumask); - init_irq_work(&walt_migration_irq_work, walt_irq_work); - init_irq_work(&walt_cpufreq_irq_work, walt_irq_work); + init_irq_work(&walt_migration_irq_work, walt_irq_work_migration); + init_irq_work(&walt_cpufreq_irq_work, walt_irq_work_roll_over); walt_rotate_work_init(); rq->walt_stats.cumulative_runnable_avg = 0; diff --git a/kernel/sched/walt.h b/kernel/sched/walt.h index 9671e56b24d9..428891e1090b 100755 --- a/kernel/sched/walt.h +++ b/kernel/sched/walt.h @@ -57,6 +57,7 @@ extern void update_task_ravg(struct task_struct *p, struct rq *rq, int event, u64 wallclock, u64 irqtime); extern unsigned int nr_eligible_big_tasks(int cpu); +extern u64 walt_get_prev_group_run_sum(struct rq *rq); static inline void inc_nr_big_task(struct walt_sched_stats *stats, struct task_struct *p) @@ -150,6 +151,10 @@ extern void fixup_walt_sched_stats_common(struct rq *rq, struct task_struct *p, extern void inc_rq_walt_stats(struct rq *rq, struct task_struct *p); extern void dec_rq_walt_stats(struct rq *rq, struct task_struct *p); extern void fixup_busy_time(struct task_struct *p, int new_cpu); +extern void walt_prepare_migrate(struct task_struct *p, + int src_cpu, int new_cpu, bool locked); +extern void walt_finish_migrate(struct task_struct *p, + int src_cpu, int new_cpu, bool locked); extern void init_new_task_load(struct task_struct *p); extern void mark_task_starting(struct task_struct *p); extern void set_window_start(struct rq *rq); @@ -325,6 +330,11 @@ static inline unsigned int nr_eligible_big_tasks(int cpu) return 0; } +static inline u64 walt_get_prev_group_run_sum(struct rq *rq) +{ + return 0; +} + static inline void walt_adjust_nr_big_tasks(struct rq *rq, int delta, bool inc) { @@ -345,6 +355,10 @@ static inline void walt_dec_cumulative_runnable_avg(struct rq *rq, } static inline void fixup_busy_time(struct task_struct *p, int new_cpu) { } +static inline void walt_prepare_migrate(struct task_struct *p, + int src_cpu, int new_cpu, bool locked) { } +static inline void walt_finish_migrate(struct task_struct *p, + int src_cpu, int new_cpu, bool locked) { } static inline void init_new_task_load(struct task_struct *p) { }