diff --git a/Documentation/scheduler/sched-hmp.txt b/Documentation/scheduler/sched-hmp.txt new file mode 100644 index 000000000000..22449aec5558 --- /dev/null +++ b/Documentation/scheduler/sched-hmp.txt @@ -0,0 +1,1432 @@ +CONTENTS + +1. Introduction + 1.1 Heterogeneous Systems + 1.2 CPU Frequency Guidance +2. Window-Based Load Tracking Scheme + 2.1 Synchronized Windows + 2.2 struct ravg + 2.3 Scaling Load Statistics + 2.4 sched_window_stats_policy + 2.5 Task Events + 2.6 update_task_ravg() + 2.7 update_history() + 2.8 Per-task 'initial task load' +3. CPU Capacity + 3.1 Load scale factor + 3.2 CPU Power +4. CPU Power +5. HMP Scheduler + 5.1 Classification of Tasks and CPUs + 5.2 select_best_cpu() + 5.2.1 sched_boost + 5.2.2 task_will_fit() + 5.2.3 Tunables affecting select_best_cpu() + 5.2.4 Wakeup Logic + 5.3 Scheduler Tick + 5.4 Load Balancer + 5.5 Real Time Tasks + 5.6 Task packing +6. Frequency Guidance + 6.1 Per-CPU Window-Based Stats + 6.2 Per-task Window-Based Stats + 6.3 Effect of various task events +7. Tunables +8. HMP Scheduler Trace Points + 8.1 sched_enq_deq_task + 8.2 sched_task_load + 8.3 sched_cpu_load_* + 8.4 sched_update_task_ravg + 8.5 sched_update_history + 8.6 sched_reset_all_windows_stats + 8.7 sched_migration_update_sum + 8.8 sched_get_busy + 8.9 sched_freq_alert + 8.10 sched_set_boost + +=============== +1. INTRODUCTION +=============== + +Scheduler extensions described in this document serves two goals: + +1) handle heterogeneous multi-processor (HMP) systems +2) guide cpufreq governor on proactive changes to cpu frequency + +*** 1.1 Heterogeneous systems + +Heterogeneous systems have cpus that differ with regard to their performance and +power characteristics. Some cpus could offer peak performance better than +others, although at cost of consuming more power. We shall refer such cpus as +"high performance" or "performance efficient" cpus. Other cpus that offer lesser +peak performance are referred to as "power efficient". + +In this situation the scheduler is tasked with the responsibility of assigning +tasks to run on the right cpus where their performance requirements can be met +at the least expense of power. + +Achieving that goal is made complicated by the fact that the scheduler has +little clue about performance requirements of tasks and how they may change by +running on power or performance efficient cpus! One simplifying assumption here +could be that a task's desire for more performance is expressed by its cpu +utilization. A task demanding high cpu utilization on a power-efficient cpu +would likely improve in its performance by running on a performance-efficient +cpu. This idea forms the basis for HMP-related scheduler extensions. + +Key inputs required by the HMP scheduler for its task placement decisions are: + +a) task load - this reflects cpu utilization or demand of tasks +b) CPU capacity - this reflects peak performance offered by cpus +c) CPU power - this reflects power or energy cost of cpus + +Once all 3 pieces of information are available, the HMP scheduler can place +tasks on the lowest power cpus where their demand can be satisfied. + +*** 1.2 CPU Frequency guidance + +A somewhat separate but related goal of the scheduler extensions described here +is to provide guidance to the cpufreq governor on the need to change cpu +frequency. Most governors that control cpu frequency work on a reactive basis. +CPU utilization is sampled at regular intervals, based on which the need to +change frequency is determined. Higher utilization leads to a frequency increase +and vice-versa. There are several problems with this approach that scheduler +can help resolve. + +a) latency + + Reactive nature introduces latency for cpus to ramp up to desired speed + which can hurt application performance. This is inevitable as cpufreq + governors can only track cpu utilization as a whole and not tasks which + are driving that demand. Scheduler can however keep track of individual + task demand and can alert the governor on changing task activity. For + example, request raise in frequency when tasks activity is increasing on + a cpu because of wakeup or migration or request frequency to be lowered + when task activity is decreasing because of sleep/exit or migration. + +b) part-picture + + Most governors track utilization of each CPU independently. When a task + migrates from one cpu to another the task's execution time is split + across the two cpus. The governor can fail to see the full picture of + task demand in this case and thus the need for increasing frequency, + affecting the task's performance. Scheduler can keep track of task + migrations, fix up busy time upon migration and report per-cpu busy time + to the governor that reflects task demand accurately. + +The rest of this document explains key enhancements made to the scheduler to +accomplish both of the aforementioned goals. + +==================================== +2. WINDOW-BASED LOAD TRACKING SCHEME +==================================== + +As mentioned in the introduction section, knowledge of the CPU demand exerted by +a task is a prerequisite to knowing where to best place the task in an HMP +system. The per-entity load tracking (PELT) scheme, present in Linux kernel +since v3.7, has some perceived shortcomings when used to place tasks on HMP +systems or provide recommendations on CPU frequency. + +Per-entity load tracking does not make a distinction between the ramp up +vs ramp down time of task load. It also decays task load without exception when +a task sleeps. As an example, a cpu bound task at its peak load (LOAD_AVG_MAX or +47742) can see its load decay to 0 after a sleep of just 213ms! A cpu-bound task +running on a performance-efficient cpu could thus get re-classified as not +requiring such a cpu after a short sleep. In the case of mobile workloads, tasks +could go to sleep due to a lack of user input. When they wakeup it is very +likely their cpu utilization pattern repeats. Resetting their load across sleep +and incurring latency to reclassify them as requiring a high performance cpu can +hurt application performance. + +The window-based load tracking scheme described in this document avoids these +drawbacks. It keeps track of N windows of execution for every task. Windows +where a task had no activity are ignored and not recorded. N can be tuned at +compile time (RAVG_HIST_SIZE defined in include/linux/sched.h) or at runtime +(/proc/sys/kernel/sched_ravg_hist_size). The window size, W, is common for all +tasks and currently defaults to 10ms ('sched_ravg_window' defined in +kernel/sched/core.c). The window size can be tuned at boot time via the +sched_ravg_window=W argument to kernel. Alternately it can be tuned after boot +via tunables provided by the interactive governor. More on this later. + +Based on the N samples available per-task, a per-task "demand" attribute is +calculated which represents the cpu demand of that task. The demand attribute is +used to classify tasks as to whether or not they need a performance-efficient +CPU and also serves to provide inputs on frequency to the cpufreq governor. More +on this later. The 'sched_window_stats_policy' tunable (defined in +kernel/sched/core.c) controls how the demand field for a task is derived from +its N past samples. + +*** 2.1 Synchronized windows + +Windows of observation for task activity are synchronized across cpus. This +greatly aids in the scheduler's frequency guidance feature. Scheduler currently +relies on a synchronized clock (sched_clock()) for this feature to work. It may +be possible to extend this feature to work on systems having an unsynchronized +sched_clock(). + +struct rq { + + .. + + u64 window_start; + + .. +}; + +The 'window_start' attribute represents the time when current window began on a +cpu. It is updated when key task events such as wakeup or context-switch call +update_task_ravg() to record task activity. The window_start value is expected +to be the same for all cpus, although it could be behind on some cpus where it +has not yet been updated because update_task_ravg() has not been recently +called. For example, when a cpu is idle for a long time its window_start could +be stale. The window_start value for such cpus is rolled forward upon +occurrence of a task event resulting in a call to update_task_ravg(). + +*** 2.2 struct ravg + +The ravg struct contains information tracked per-task. + +struct ravg { + u64 mark_start; + u32 sum, demand; + u32 sum_history[RAVG_HIST_SIZE]; +}; + +struct task_struct { + + .. + + struct ravg ravg; + + .. +}; + +sum_history[] - stores cpu utilization samples from N previous windows + where task had activity + +sum - stores cpu utilization of the task in its most recently + tracked window. Once the corresponding window terminates, + 'sum' will be pushed into the sum_history[] array and is then + reset to 0. It is possible that the window corresponding to + sum is not the current window being tracked on a cpu. For + example, a task could go to sleep in window X and wakeup in + window Y (Y > X). In this case, sum would correspond to the + task's activity seen in window X. When update_task_ravg() is + called during the task's wakeup event it will be seen that + window X has elapsed. The sum value will be pushed to + 'sum_history[]' array before being reset to 0. + +demand - represents task's cpu demand and is derived from the + elements in sum_history[]. The section on + 'sched_window_stats_policy' provides more details on how + 'demand' is derived from elements in sum_history[] array + +mark_start - records timestamp of the beginning of the most recent task + event. See section on 'Task events' for possible events that + update 'mark_start' + +curr_window - this is described in the section on 'Frequency guidance' + +prev_window - this is described in the section on 'Frequency guidance' + + +*** 2.3 Scaling load statistics + +Time required for a task to complete its work (and hence its load) depends on, +among various other factors, cpu frequency and its efficiency. In a HMP system, +some cpus are more performance efficient than others. Performance efficiency of +a cpu can be described by its "instructions-per-cycle" (IPC) attribute. History +of task execution could involve task having run at different frequencies and on +cpus with different IPC attributes. To avoid ambiguity of how task load relates +to the frequency and IPC of cpus on which a task has run, task load is captured +in a scaled form, with scaling being done in reference to an "ideal" cpu that +has best possible IPC and frequency. Such an "ideal" cpu, having the best +possible frequency and IPC, may or may not exist in system. + +As an example, consider a HMP system, with two types of cpus, A53 and A57. A53 +has IPC count of 1024 and can run at maximum frequency of 1 GHz, while A57 has +IPC count of 2048 and can run at maximum frequency of 2 GHz. Ideal cpu in this +case is A57 running at 2 GHz. + +A unit of work that takes 100ms to finish on A53 running at 100MHz would get +done in 10ms on A53 running at 1GHz, in 5 ms running on A57 at 1 GHz and 2.5ms +on A57 running at 2 GHz. Thus a load of 100ms can be expressed as 2.5ms in +reference to ideal cpu of A57 running at 2 GHz. + +In order to understand how much load a task will consume on a given cpu, its +scaled load needs to be multiplied by a factor (load scale factor). In above +example, scaled load of 2.5ms needs to be multiplied by a factor of 4 in order +to estimate the load of task on A53 running at 1 GHz. + +/proc/sched_debug provides IPC attribute and load scale factor for every cpu. + +In summary, task load information stored in a task's sum_history[] array is +scaled for both frequency and efficiency. If a task runs for X ms, then the +value stored in its 'sum' field is derived as: + + X_s = X * (f_cur / max_possible_freq) * + (efficiency / max_possible_efficiency) + +where: + +X = cpu utilization that needs to be accounted +X_s = Scaled derivative of X +f_cur = current frequency of the cpu where the task was + running +max_possible_freq = maximum possible frequency (across all cpus) +efficiency = instructions per cycle (IPC) of cpu where task was + running +max_possible_efficiency = maximum IPC offered by any cpu in system + + +*** 2.4 sched_window_stats_policy + +sched_window_stats_policy controls how the 'demand' attribute for a task is +derived from elements in its 'sum_history[]' array. + +WINDOW_STATS_RECENT (0) + demand = recent + +WINDOW_STATS_MAX (1) + demand = max + +WINDOW_STATS_MAX_RECENT_AVG (2) + demand = maximum(average, recent) + +WINDOW_STATS_AVG (3) + demand = average + +where: + M = history size specified by + /proc/sys/kernel/sched_ravg_hist_size + average = average of first M samples found in the sum_history[] array + max = maximum value of first M samples found in the sum_history[] + array + recent = most recent sample (sum_history[0]) + demand = demand attribute found in 'struct ravg' + +This policy can be changed at runtime via +/proc/sys/kernel/sched_window_stats_policy. For example, the command +below would select WINDOW_STATS_USE_MAX policy + +echo 1 > /proc/sys/kernel/sched_window_stats_policy + +*** 2.5 Task events + +A number of events results in the window-based stats of a task being +updated. These are: + +PICK_NEXT_TASK - the task is about to start running on a cpu +PUT_PREV_TASK - the task stopped running on a cpu +TASK_WAKE - the task is waking from sleep +TASK_MIGRATE - the task is migrating from one cpu to another +TASK_UPDATE - this event is invoked on a currently running task to + update the task's window-stats and also the cpu's + window-stats such as 'window_start' +IRQ_UPDATE - event to record the busy time spent by an idle cpu + processing interrupts + +*** 2.6 update_task_ravg() + +update_task_ravg() is called to mark the beginning of an event for a task or a +cpu. It serves to accomplish these functions: + +a. Update a cpu's window_start value +b. Update a task's window-stats (sum, sum_history[], demand and mark_start) + +In addition update_task_ravg() updates the busy time information for the given +cpu, which is used for frequency guidance. This is described further in section +6. + +*** 2.7 update_history() + +update_history() is called on a task to record its activity in an elapsed +window. 'sum', which represents task's cpu demand in its elapsed window is +pushed onto sum_history[] array and its 'demand' attribute is updated based on +the sched_window_stats_policy in effect. + +*** 2.8 Initial task load attribute for a task (init_load_pct) + +In some cases, it may be desirable for children of a task to be assigned a +"high" load so that they can start running on best capacity cluster. By default, +newly created tasks are assigned a load defined by tunable sched_init_task_load +(Sec 7.8). Some specialized tasks may need a higher value than the global +default for their child tasks. This will let child tasks run on cpus with best +capacity. This is accomplished by setting the 'initial task load' attribute +(init_load_pct) for a task. Child tasks starting load (ravg.demand and +ravg.sum_history[]) is initialized from their parent's 'initial task load' +attribute. Note that child task's 'initial task load' attribute itself will be 0 +by default (i.e it is not inherited from parent). + +A task's 'initial task load' attribute can be set in two ways: + +**** /proc interface + +/proc/[pid]/sched_init_task_load can be written to for setting a task's 'initial +task load' attribute. A numeric value between 0 - 100 (in percent scale) is +accepted for task's 'initial task load' attribute. + +Reading /proc/[pid]/sched_init_task_load returns the 'initial task load' +attribute for the given task. + +**** kernel API + +Following kernel APIs are provided to set or retrieve a given task's 'initial +task load' attribute: + +int sched_set_init_task_load(struct task_struct *p, int init_load_pct); +int sched_get_init_task_load(struct task_struct *p); + + +=============== +3. CPU CAPACITY +=============== + +CPU capacity reflects peak performance offered by a cpu. It is defined both by +maximum frequency at which cpu can run and its efficiency attribute. Capacity of +a cpu is defined in reference to "least" performing cpu such that "least" +performing cpu has capacity of 1024. + + capacity = 1024 * (fmax_cur * / min_max_freq) * + (efficiency / min_possible_efficiency) + +where: + + fmax_cur = maximum frequency at which cpu is currently + allowed to run at + efficiency = IPC of cpu + min_max_freq = max frequency at which "least" performing cpu + can run + min_possible_efficiency = IPC of "least" performing cpu + +'fmax_cur' reflects the fact that a cpu may be constrained at runtime to run at +a maximum frequency less than what is supported. This may be a constraint placed +by user or drivers such as thermal that intends to reduce temperature of a cpu +by restricting its maximum frequency. + +'max_possible_capacity' reflects the maximum capacity of a cpu based on the +maximum frequency it supports. + +max_possible_capacity = 1024 * (fmax * / min_max_freq) * + (efficiency / min_possible_efficiency) + +where: + fmax = maximum frequency supported by a cpu + +/proc/sched_debug lists capacity and maximum_capacity information for a cpu. + +In the example HMP system quoted in Sec 2.3, "least" performing CPU is A53 and +thus min_max_freq = 1GHz and min_possible_efficiency = 1024. + +Capacity of A57 = 1024 * (2GHz / 1GHz) * (2048 / 1024) = 4096 +Capacity of A53 = 1024 * (1GHz / 1GHz) * (1024 / 1024) = 1024 + +Capacity of A57 when constrained to run at maximum frequency of 500MHz can be +calculated as: + +Capacity of A57 = 1024 * (500MHz / 1GHz) * (2048 / 1024) = 1024 + +*** 3.1 load_scale_factor + +'lsf' or load scale factor attribute of a cpu is used to estimate load of a task +on that cpu when running at its fmax_cur frequency. 'lsf' is defined in +reference to "best" performing cpu such that it's lsf is 1024. 'lsf' for a cpu +is defined as: + + lsf = 1024 * (max_possible_freq / fmax_cur) * + (max_possible_efficiency / ipc) + +where: + fmax_cur = maximum frequency at which cpu is currently + allowed to run at + ipc = IPC of cpu + max_possible_freq = max frequency at which "best" performing cpu + can run + max_possible_efficiency = IPC of "best" performing cpu + +In the example HMP system quoted in Sec 2.3, "best" performing CPU is A57 and +thus max_possible_freq = 2 GHz, max_possible_efficiency = 2048 + +lsf of A57 = 1024 * (2GHz / 2GHz) * (2048 / 2048) = 1024 +lsf of A53 = 1024 * (2GHz / 1 GHz) * (2048 / 1024) = 4096 + +lsf of A57 constrained to run at maximum frequency of 500MHz can be calculated +as: + +lsf of A57 = 1024 * (2GHz / 500Mhz) * (2048 / 2048) = 4096 + +To estimate load of a task on a given cpu running at its fmax_cur: + + load = scaled_load * lsf / 1024 + +A task with scaled load of 20% would thus be estimated to consume 80% bandwidth +of A53 running at 1GHz. The same task with scaled load of 20% would be estimated +to consume 160% bandwidth on A53 constrained to run at maximum frequency of +500MHz. + +load_scale_factor, thus, is very useful to estimate load of a task on a given +cpu and thus to decide whether it can fit in a cpu or not. + +*** 3.2 cpu_power + +A metric 'cpu_power' related to 'capacity' is also listed in /proc/sched_debug. +'cpu_power' is ideally same for all cpus (1024) when they are idle and running +at the same frequency. 'cpu_power' of a cpu can be scaled down from its ideal +value to reflect reduced frequency it is operating at and also to reflect the +amount of cpu bandwidth consumed by real-time tasks executing on it. +'cpu_power' metric is used by scheduler to decide task load distribution among +cpus. CPUs with low 'cpu_power' will be assigned less task load compared to cpus +with higher 'cpu_power' + +============ +4. CPU POWER +============ + +The HMP scheduler extensions currently depend on an architecture-specific driver +to provide runtime information on cpu power. In the absence of an +architecture-specific driver, the scheduler will resort to using the +max_possible_capacity metric of a cpu as a measure of its power. + +================ +5. HMP SCHEDULER +================ + +For normal (SCHED_OTHER/fair class) tasks there are three paths in the +scheduler which these HMP extensions affect. The task wakeup path, the +load balancer, and the scheduler tick are each modified. + +Real-time and stop-class tasks are served by different code +paths. These will be discussed separately. + +Prior to delving further into the algorithm and implementation however +some definitions are required. + +*** 5.1 Classification of Tasks and CPUs + +With the extensions described thus far, the following information is +available to the HMP scheduler: + +- per-task CPU demand information from either Per-Entity Load Tracking + (PELT) or the window-based algorithm described above + +- a power value for each frequency supported by each CPU via the API + described in section 4 + +- current CPU frequency, maximum CPU frequency (may be throttled by at + runtime due to thermal conditions), maximum possible CPU frequency supported + by hardware + +- data previously maintained within the scheduler such as the number + of currently runnable tasks on each CPU + +Combined with tunable parameters, this information can be used to classify +both tasks and CPUs to aid in the placement of tasks. + +- big task + + A big task is one that exerts a CPU demand too high for a particular + CPU to satisfy. The scheduler will attempt to find a CPU with more + capacity for such a task. + + The definition of "big" is specific to a task *and* a CPU. A task + may be considered big on one CPU in the system and not big on + another if the first CPU has less capacity than the second. + + What task demand is "too high" for a particular CPU? One obvious + answer would be a task demand which, as measured by PELT or + window-based load tracking, matches or exceeds the capacity of that + CPU. A task which runs on a CPU for a long time, for example, might + meet this criteria as it would report 100% demand of that CPU. It + may be desirable however to classify tasks which use less than 100% + of a particular CPU as big so that the task has some "headroom" to grow + without its CPU bandwidth getting capped and its performance requirements + not being met. This task demand is therefore a tunable parameter: + + /proc/sys/kernel/sched_upmigrate + + This value is a percentage. If a task consumes more than this much of a + particular CPU, that CPU will be considered too small for the task. The task + will thus be seen as a "big" task on the cpu and will reflect in nr_big_tasks + statistics maintained for that cpu. Note that certain tasks (whose nice + value exceeds SCHED_UPMIGRATE_MIN_NICE value or those that belong to a cgroup + whose upmigrate_discourage flag is set) will never be classified as big tasks + despite their high demand. + + As the load scale factor is calculated against current fmax, it gets boosted + when a lower capacity CPU is restricted to run at lower fmax. The task + demand is inflated in this scenario and the task upmigrates early to the + maximum capacity CPU. Hence this threshold is auto-adjusted by a factor + equal to max_possible_frequency/current_frequency of a lower capacity CPU. + This adjustment happens only when the lower capacity CPU frequency is + restricted. The same adjustment is applied to the downmigrate threshold + as well. + + When the frequency restriction is relaxed, the previous values are restored. + sched_up_down_migrate_auto_update macro defined in kernel/sched/core.c + controls this auto-adjustment behavior and it is enabled by default. + + If the adjusted upmigrate threshold exceeds the window size, it is clipped to + the window size. If the adjusted downmigrate threshold decreases the difference + between the upmigrate and downmigrate, it is clipped to a value such that the + difference between the modified and the original thresholds is same. + +- spill threshold + + Tasks will normally be placed on lowest power-cost cluster where they can fit. + This could result in power-efficient cluster becoming overcrowded when there + are "too" many low-demand tasks. Spill threshold provides a spill over + criteria, wherein low-demand task are allowed to be placed on idle or + busy cpus in high-performance cluster. + + Scheduler will avoid placing a task on a cpu if it can result in cpu exceeding + its spill threshold, which is defined by two tunables: + + /proc/sys/kernel/sched_spill_nr_run (default: 10) + /proc/sys/kernel/sched_spill_load (default : 100%) + + A cpu is considered to be above its spill level if it already has 10 tasks or + if the sum of task load (scaled in reference to given cpu) and + rq->cumulative_runnable_avg exceeds 'sched_spill_load'. + +- power band + + The scheduler may be faced with a tradeoff between power and performance when + placing a task. If the scheduler sees two CPUs which can accommodate a task: + + CPU 1, power cost of 20, load of 10 + CPU 2, power cost of 10, load of 15 + + It is not clear what the right choice of CPU is. The HMP scheduler + offers the sched_powerband_limit tunable to determine how this + situation should be handled. When the power delta between two CPUs + is less than sched_powerband_limit_pct, load will be prioritized as + the deciding factor as to which CPU is selected. If the power delta + between two CPUs exceeds that, the lower power CPU is considered to + be in a different "band" and it is selected, despite perhaps having + a higher current task load. + +*** 5.2 select_best_cpu() + +CPU placement decisions for a task at its wakeup or creation time are the +most important decisions made by the HMP scheduler. This section will describe +the call flow and algorithm used in detail. + +The primary entry point for a task wakeup operation is try_to_wake_up(), +located in kernel/sched/core.c. This function relies on select_task_rq() to +determine the target CPU for the waking task. For fair-class (SCHED_OTHER) +tasks, that request will be routed to select_task_rq_fair() in +kernel/sched/fair.c. As part of these scheduler extensions a hook has been +inserted into the top of that function. If HMP scheduling is enabled the normal +scheduling behavior will be replaced by a call to select_best_cpu(). This +function, select_best_cpu(), represents the heart of the HMP scheduling +algorithm described in this document. Note that select_best_cpu() is also +invoked for a task being created. + +The behavior of select_best_cpu() depends on several factors such as boost +setting, choice of several tunables and on task demand. + +**** 5.2.1 Boost + +The task placement policy changes signifincantly when scheduler boost is in +effect. When boost is in effect the scheduler ignores the power cost of +placing tasks on CPUs. Instead it figures out the load on each CPU and then +places task on the least loaded CPU. If the load of two or more CPUs is the +same (generally when CPUs are idle) the task prefers to go highest capacity +CPU in the system. + +A further enhancement during boost is the scheduler' early detection feature. +While boost is in effect the scheduler checks for the precence of tasks that +have been runnable for over some period of time within the tick. For such +tasks the scheduler informs the governor of imminent need for high frequency. +If there exists a task on the runqueue at the tick that has been runnable +for greater than SCHED_EARLY_DETECTION_DURATION amount of time, it notifies +the governor with a fabricated load of the full window at the highest +frequency. The fabricated load is maintained until the task is no longer +runnable or until the next tick. + +Boost can be set via either /proc/sys/kernel/sched_boost or by invoking +kernel API sched_set_boost(). + + int sched_set_boost(int enable); + +Once turned on, boost will remain in effect until it is explicitly turned off. +To allow for boost to be controlled by multiple external entities (application +or kernel module) at same time, boost setting is reference counted. This means +that two applications can turn on boost and the effect of boost is eliminated +only after both applications have turned off boost. boost_refcount variable +represents this reference count. + +**** 5.2.2 task_will_fit() + +The overall goal of select_best_cpu() is to place a task on the least power +cluster where it can "fit" i.e where its cpu usage shall be below the capacity +offered by cluster. Criteria for a task to be considered as fitting in a cluster +is: + + i) A low-priority task, whose nice value is greater than + SCHED_UPMIGRATE_MIN_NICE or whose cgroup has its + upmigrate_discourage flag set, is considered to be fitting in all clusters, + irrespective of their capacity and task's cpu demand. + + ii) All tasks are considered to fit in highest capacity cluster. + + iii) Task demand scaled in reference to the given cluster should be less than a + threshold. See section on load_scale_factor to know more about how task + demand is scaled in reference to a given cpu (cluster). The threshold used + is normally sched_upmigrate. Its possible for a task's demand to exceed + sched_upmigrate threshold in reference to a cluster when its upmigrated to + higher capacity cluster. To prevent it from coming back immediately to + lower capacity cluster, the task is not considered to "fit" on its earlier + cluster until its demand has dropped below sched_downmigrate in reference + to that earlier cluster. sched_downmigrate thus provides for some + hysteresis control. + + +**** 5.2.3 Factors affecting select_best_cpu() + +Behavior of select_best_cpu() is further controlled by several tunables and +synchronous nature of wakeup. + +a. /proc/sys/kernel/sched_cpu_high_irqload + A cpu whose irq load is greater than this threshold will not be + considered eligible for placement. This threshold value in expressed in + nanoseconds scale, with default threshold being 10000000 (10ms). See + notes on sched_cpu_high_irqload tunable to understand how irq load on a + cpu is measured. + +b. Synchronous nature of wakeup + Synchronous wakeup is a hint to scheduler that the task issuing wakeup + (i.e task currently running on cpu where wakeup is being processed by + scheduler) will "soon" relinquish CPU. A simple example is two tasks + communicating with each other using a pipe structure. When reader task + blocks waiting for data, its woken by writer task after it has written + data to pipe. Writer task usually blocks waiting for reader task to + consume data in pipe (which may not have any more room for writes). + + Synchronous wakeup is accounted for by adjusting load of a cpu to not + include load of currently running task. As a result, a cpu that has only + one runnable task and which is currently processing synchronous wakeup + will be considered idle. + +c. PF_WAKE_UP_IDLE + Any task with this flag set will be woken up to an idle cpu (if one is + available) independent of sched_prefer_idle flag setting, its demand and + synchronous nature of wakeup. Similarly idle cpu is preferred during + wakeup for any task that does not have this flag set but is being woken + by a task with PF_WAKE_UP_IDLE flag set. For simplicity, we will use the + term "PF_WAKE_UP_IDLE wakeup" to signify wakeups involving a task with + PF_WAKE_UP_IDLE flag set. + +d. /proc/sys/kernel/sched_select_prev_cpu_us + This threshold controls whether task placement goes through fast path or + not. If task's wakeup time since last sleep is short there are high + chances that it's better to place the task on its previous CPU. This + reduces task placement latency, cache miss and number of migrations. + Default value of sched_select_prev_cpu_us is 2000 (2ms). This can be + turned off by setting it to 0. + +**** 5.2.4 Wakeup Logic for Task "p" + +Wakeup task placement logic is as follows: + +1) Eliminate CPUs with high irq load based on sched_cpu_high_irqload tunable. + +2) Eliminate CPUs where either the task does not fit or CPUs where placement +will result in exceeding the spill threshold tunables. CPUs elimiated at this +stage will be considered as backup choices incase none of the CPUs get past +this stage. + +3) Find out and return the least power CPU that satisfies all conditions above. + +4) If two or more CPUs are projected to have the same power, break ties in the +following preference order: + a) The CPU is the task's previous CPU. + b) The CPU is in the same cluster as the task's previous CPU. + c) The CPU has the least load + +The placement logic described above does not apply when PF_WAKE_UP_IDLE is set +for either the waker task or the wakee task. Instead the scheduler chooses the +most power efficient idle CPU. + +5) If no CPU is found after step 2, resort to backup CPU selection logic +whereby the CPU with highest amount of spare capacity is selected. + +6) If none of the CPUs have any spare capacity, return the task's previous +CPU. + +*** 5.3 Scheduler Tick + +Every CPU is interrupted periodically to let kernel update various statistics +and possibly preempt the currently running task in favor of a waiting task. This +periodicity, determined by CONFIG_HZ value, is set at 10ms. There are various +optimizations by which a CPU however can skip taking these interrupts (ticks). +A cpu going idle for considerable time in one such case. + +HMP scheduler extensions brings in a change in processing of tick +(scheduler_tick()) that can result in task migration. In case the currently +running task on a cpu belongs to fair_sched class, a check is made if it needs +to be migrated. Possible reasons for migrating task could be: + +a) A big task is running on a power-efficient cpu and a high-performance cpu is +available (idle) to service it + +b) A task is starving on a CPU with high irq load. + +c) A task with upmigration discouraged is running on a performance cluster. +See notes on 'cpu.upmigrate_discourage'. + +In case the test for migration turns out positive (which is expected to be rare +event), a candidate cpu is identified for task migration. To avoid multiple task +migrations to the same candidate cpu(s), identification of candidate cpu is +serialized via global spinlock (migration_lock). + +*** 5.4 Load Balancer + +Load balance is a key functionality of scheduler that strives to distribute task +across available cpus in a "fair" manner. Most of the complexity associated with +this feature involves balancing fair_sched class tasks. Changes made to load +balance code serve these goals: + +1. Restrict flow of tasks from power-efficient cpus to high-performance cpu. + Provide a spill-over threshold, defined in terms of number of tasks + (sched_spill_nr_run) and cpu demand (sched_spill_load), beyond which tasks + can spill over from power-efficient cpu to high-performance cpus. + +2. Allow idle power-efficient cpus to pick up extra load from over-loaded + performance-efficient cpu + +3. Allow idle high-performance cpu to pick up big tasks from power-efficient cpu + +*** 5.5 Real Time Tasks + +Minimal changes introduced in treatment of real-time tasks by HMP scheduler +aims at preferring scheduling of real-time tasks on cpus with low load on +a power efficient cluster. + +Prior to HMP scheduler, the fast-path cpu selection for placing a real-time task +(at wakeup) is its previous cpu, provided the currently running task on its +previous cpu is not a real-time task or a real-time task with lower priority. +Failing this, cpu selection in slow-path involves building a list of candidate +cpus where the waking real-time task will be of highest priority and thus can be +run immediately. The first cpu from this candidate list is chosen for the waking +real-time task. Much of the premise for this simple approach is the assumption +that real-time tasks often execute for very short intervals and thus the focus +is to place them on a cpu where they can be run immediately. + +HMP scheduler brings in a change which avoids fast-path and always resorts to +slow-path. Further cpu with lowest load in a power efficient cluster from +candidate list of cpus is chosen as cpu for placing waking real-time task. + +- PF_WAKE_UP_IDLE + +Idle cpu is preferred for any waking task that has this flag set in its +'task_struct.flags' field. Further idle cpu is preferred for any task woken by +such tasks. PF_WAKE_UP_IDLE flag of a task is inherited by it's children. It can +be modified for a task in two ways: + + > kernel-space interface + set_wake_up_idle() needs to be called in the context of a task + to set or clear its PF_WAKE_UP_IDLE flag. + + > user-space interface + /proc/[pid]/sched_wake_up_idle file needs to be written to for + setting or clearing PF_WAKE_UP_IDLE flag for a given task + +===================== +6. FREQUENCY GUIDANCE +===================== + +As mentioned in the introduction section the scheduler is in a unique +position to assist with the determination of CPU frequency. Because +the scheduler now maintains an estimate of per-task CPU demand, task +activity can be tracked, aggregated and provided to the CPUfreq +governor as a replacement for simple CPU busy time. + +Two of the most popular CPUfreq governors, interactive and ondemand, +utilize a window-based approach for measuring CPU busy time. This +works well with the window-based load tracking scheme previously +described. The following APIs are provided to allow the CPUfreq +governor to query busy time from the scheduler instead of using the +basic CPU busy time value derived via get_cpu_idle_time_us() and +get_cpu_iowait_time_us() APIs. + + int sched_set_window(u64 window_start, unsigned int window_size) + + This API is invoked by governor at initialization time or whenever + window size is changed. 'window_size' argument (in jiffy units) + indicates the size of window to be used. The first window of size + 'window_size' is set to begin at jiffy 'window_start' + + -EINVAL is returned if per-entity load tracking is in use rather + than window-based load tracking, otherwise a success value of 0 + is returned. + + int sched_get_busy(int cpu) + + Returns the busy time for the given CPU in the most recent + complete window. The value returned is microseconds of busy + time at fmax of given CPU. + +The values returned by sched_get_busy() take a bit of explanation, +both in what they mean and also how they are derived. + +*** 6.1 Per-CPU Window-Based Stats + +In addition to the per-task window-based demand, the HMP scheduler +extensions also track the aggregate demand seen on each CPU. This is +done using the same windows that the task demand is tracked with +(which is in turn set by the governor when frequency guidance is in +use). There are four quantities maintained for each CPU by the HMP scheduler: + + curr_runnable_sum: aggregate demand from all tasks which executed during + the current (not yet completed) window + + prev_runnable_sum: aggregate demand from all tasks which executed during + the most recent completed window + + nt_curr_runnable_sum: aggregate demand from all 'new' tasks which executed + during the current (not yet completed) window + + nt_prev_runnable_sum: aggregate demand from all 'new' tasks which executed + during the most recent completed window. + +When the scheduler is updating a task's window-based stats it also +updates these values. Like per-task window-based demand these +quantities are normalized against the max possible frequency and max +efficiency (instructions per cycle) in the system. If an update occurs +and a window rollover is observed, curr_runnable_sum is copied into +prev_runnable_sum before being reset to 0. The sched_get_busy() API +returns prev_runnable_sum, scaled to the efficiency and fmax of given +CPU. The same applies to nt_curr_runnable_sum and nt_prev_runnable_sum. + +A 'new' task is defined as a task whose number of active windows since fork is +less than sysctl_sched_new_task_windows. An active window is defined as a window +where a task was observed to be runnable. + +*** 6.2 Per-task window-based stats + +Corresponding to curr_runnable_sum and prev_runnable_sum, two counters are +maintained per-task + +curr_window - represents cpu demand of task in its most recently tracked + window +prev_window - represents cpu demand of task in the window prior to the one + being tracked by curr_window + +The above counters are resued for nt_curr_runnable_sum and +nt_prev_runnable_sum. + +"cpu demand" of a task includes its execution time and can also include its +wait time. 'SCHED_FREQ_ACCOUNT_WAIT_TIME' controls whether task's wait +time is included in its 'curr_window' and 'prev_window' counters or not. + +Needless to say, curr_runnable_sum counter of a cpu is derived from curr_window +counter of various tasks that ran on it in its most recent window. + +*** 6.3 Effect of various task events + +We now consider various events and how they affect above mentioned counters. + +PICK_NEXT_TASK + This represents beginning of execution for a task. Provided the task + refers to a non-idle task, a portion of task's wait time that + corresponds to the current window being tracked on a cpu is added to + task's curr_window counter, provided SCHED_FREQ_ACCOUNT_WAIT_TIME is + set. The same quantum is also added to cpu's curr_runnable_sum counter. + The remaining portion, which corresponds to task's wait time in previous + window is added to task's prev_window and cpu's prev_runnable_sum + counters. + +PUT_PREV_TASK + This represents end of execution of a time-slice for a task, where the + task could refer to a cpu's idle task also. In case the task is non-idle + or (in case of task being idle with cpu having non-zero rq->nr_iowait + count and sched_io_is_busy =1), a portion of task's execution time, that + corresponds to current window being tracked on a cpu is added to task's + curr_window_counter and also to cpu's curr_runnable_sum counter. Portion + of task's execution that corresponds to the previous window is added to + task's prev_window and cpu's prev_runnable_sum counters. + +TASK_UPDATE + This event is called on a cpu's currently running task and hence + behaves effectively as PUT_PREV_TASK. Task continues executing after + this event, until PUT_PREV_TASK event occurs on the task (during + context switch). + +TASK_WAKE + This event signifies a task waking from sleep. Since many windows + could have elapsed since the task went to sleep, its curr_window + and prev_window are updated to reflect task's demand in the most + recent and its previous window that is being tracked on a cpu. + +TASK_MIGRATE + This event signifies task migration across cpus. It is invoked on the + task prior to being moved. Thus at the time of this event, the task + can be considered to be in "waiting" state on src_cpu. In that way + this event reflects actions taken under PICK_NEXT_TASK (i.e its + wait time is added to task's curr/prev_window counters as well + as src_cpu's curr/prev_runnable_sum counters, provided + SCHED_FREQ_ACCOUNT_WAIT_TIME is non-zero). After that update, + src_cpu's curr_runnable_sum is reduced by task's curr_window value + and dst_cpu's curr_runnable_sum is increased by task's curr_window + value. Similarly, src_cpu's prev_runnable_sum is reduced by task's + prev_window value and dst_cpu's prev_runnable_sum is increased by + task's prev_window value. + +IRQ_UPDATE + This event signifies end of execution of an interrupt handler. This + event results in update of cpu's busy time counters, curr_runnable_sum + and prev_runnable_sum, provided cpu was idle. + When sched_io_is_busy = 0, only the interrupt handling time is added + to cpu's curr_runnable_sum and prev_runnable_sum counters. When + sched_io_is_busy = 1, the event mirrors actions taken under + TASK_UPDATED event i.e time since last accounting of idle task's cpu + usage is added to cpu's curr_runnable_sum and prev_runnable_sum + counters. + +=========== +7. TUNABLES +=========== + +*** 7.1 sched_spill_load + +Appears at: /proc/sys/kernel/sched_spill_load + +Default value: 100 + +CPU selection criteria for fair-sched class tasks is the lowest power cpu where +they can fit. When the most power-efficient cpu where a task can fit is +overloaded (aggregate demand of tasks currently queued on it exceeds +sched_spill_load), a task can be placed on a higher-performance cpu, even though +the task strictly doesn't need one. + +*** 7.2 sched_spill_nr_run + +Appears at: /proc/sys/kernel/sched_spill_nr_run + +Default value: 10 + +The intent of this tunable is similar to sched_spill_load, except it applies to +nr_running count of a cpu. A task can spill over to a higher-performance cpu +when the most power-efficient cpu where it can normally fit has more tasks than +sched_spill_nr_run. + +*** 7.3 sched_upmigrate + +Appears at: /proc/sys/kernel/sched_upmigrate + +Default value: 80 + +This tunable is a percentage. If a task consumes more than this much +of a CPU, the CPU is considered too small for the task and the +scheduler will try to find a bigger CPU to place the task on. + +*** 7.4 sched_init_task_load + +Appears at: /proc/sys/kernel/sched_init_task_load + +Default value: 15 + +This tunable is a percentage. When a task is first created it has no +history, so the task load tracking mechanism cannot determine a +historical load value to assign to it. This tunable specifies the +initial load value for newly created tasks. Also see Sec 2.8 on per-task +'initial task load' attribute. + +*** 7.5 sched_ravg_hist_size + +Appears at: /proc/sys/kernel/sched_ravg_hist_size + +Default value: 5 + +This tunable controls the number of samples used from task's sum_history[] +array for determination of its demand. + +*** 7.6 sched_window_stats_policy + +Appears at: /proc/sys/kernel/sched_window_stats_policy + +Default value: 2 + +This tunable controls the policy in how window-based load tracking +calculates an overall demand value based on the windows of CPU +utilization it has collected for a task. + +Possible values for this tunable are: +0: Just use the most recent window sample of task activity when calculating + task demand. +1: Use the maximum value of first M samples found in task's cpu demand + history (sum_history[] array), where M = sysctl_sched_ravg_hist_size +2: Use the maximum of (the most recent window sample, average of first M + samples), where M = sysctl_sched_ravg_hist_size +3. Use average of first M samples, where M = sysctl_sched_ravg_hist_size + +*** 7.7 sched_ravg_window + +Appears at: kernel command line argument + +Default value: 10000000 (10ms, units of tunable are nanoseconds) + +This specifies the duration of each window in window-based load +tracking. By default each window is 10ms long. This quantity must +currently be set at boot time on the kernel command line (or the +default value of 10ms can be used). + +*** 7.8 RAVG_HIST_SIZE + +Appears at: compile time only (see RAVG_HIST_SIZE in include/linux/sched.h) + +Default value: 5 + +This macro specifies the number of windows the window-based load +tracking mechanism maintains per task. If default values are used for +both this and sched_ravg_window then a total of 50ms of task history +would be maintained in 5 10ms windows. + +*** 7.9 sched_freq_inc_notify + +Appears at: /proc/sys/kernel/sched_freq_inc_notify + +Default value: 10 * 1024 * 1024 (10 Ghz) + +When scheduler detects that cur_freq of a cluster is insufficient to meet +demand, it sends notification to governor, provided (freq_required - cur_freq) +exceeds sched_freq_inc_notify, where freq_required is the frequency calculated +by scheduler to meet current task demand. Note that sched_freq_inc_notify is +specified in kHz units. + +*** 7.10 sched_freq_dec_notify + +Appears at: /proc/sys/kernel/sched_freq_dec_notify + +Default value: 10 * 1024 * 1024 (10 Ghz) + +When scheduler detects that cur_freq of a cluster is far greater than what is +needed to serve current task demand, it will send notification to governor. +More specifically, notification is sent when (cur_freq - freq_required) +exceeds sched_freq_dec_notify, where freq_required is the frequency calculated +by scheduler to meet current task demand. Note that sched_freq_dec_notify is +specified in kHz units. + +*** 7.11 sched_cpu_high_irqload + +Appears at: /proc/sys/kernel/sched_cpu_high_irqload + +Default value: 10000000 (10ms) + +The scheduler keeps a decaying average of the amount of irq and softirq activity +seen on each CPU within a ten millisecond window. Note that this "irqload" +(reported in the sched_cpu_load_* tracepoint) will be higher than the typical load +in a single window since every time the window rolls over, the value is decayed +by some fraction and then added to the irq/softirq time spent in the next +window. + +When the irqload on a CPU exceeds the value of this tunable, the CPU is no +longer eligible for placement. This will affect the task placement logic +described above, causing the scheduler to try and steer tasks away from +the CPU. + +*** 7.12 cpu.upmigrate_discourage + +Default value : 0 + +This is a cgroup attribute supported by the cpu resource controller. It normally +appears at [root_cpu]/[name1]/../[name2]/cpu.upmigrate_discourage. Here +"root_cpu" is the mount point for cgroup (cpu resource control) filesystem +and name1, name2 etc are names of cgroups that form a hierarchy. + +Setting this flag to 1 discourages upmigration for all tasks of a cgroup. High +demand tasks of such a cgroup will never be classified as big tasks and hence +not upmigrated. Any task of the cgroup is allowed to upmigrate only under +overcommitted scenario. See notes on sched_spill_nr_run and sched_spill_load for +how overcommitment threshold is defined. + +*** 7.13 sched_static_cpu_pwr_cost + +Default value: 0 + +Appears at /sys/devices/system/cpu/cpu/sched_static_cpu_pwr_cost + +This is the power cost associated with bringing an idle CPU out of low power +mode. It ignores the actual C-state that a CPU may be in and assumes the +worst case power cost of the highest C-state. It is means of biasing task +placement away from idle CPUs when necessary. It can be defined per CPU, +however, a more appropriate usage to define the same value for every CPU +within a cluster and possibly have differing value between clusters as +needed. + + +*** 7.14 sched_static_cluster_pwr_cost + +Default value: 0 + +Appears at /sys/devices/system/cpu/cpu/sched_static_cluster_pwr_cost + +This is the power cost associated with bringing an idle cluster out of low +power mode. It ignores the actual D-state that a cluster may be in and assumes +the worst case power cost of the highest D-state. It is means of biasing task +placement away from idle clusters when necessary. + +*** 7.15 sched_restrict_cluster_spill + +Default value: 0 + +Appears at /proc/sys/kernel/sched_restrict_cluster_spill + +This tunable can be used to restrict tasks spilling to the higher capacity +(higher power) cluster. When this tunable is enabled, + +- Restrict the higher capacity cluster pulling tasks from the lower capacity +cluster in the load balance path. The restriction is lifted if all of the CPUS +in the lower capacity cluster are above spill. The power cost is used to break +the ties if the capacity of clusters are same for applying this restriction. + +- The current CPU selection algorithm for RT tasks looks for the least loaded +CPU across all clusters. When this tunable is enabled, the RT tasks are +restricted to the lowest possible power cluster. + + +*** 7.16 sched_downmigrate + +Appears at: /proc/sys/kernel/sched_downmigrate + +Default value: 60 + +This tunable is a percentage. It exists to control hysteresis. Lets say a task +migrated to a high-performance cpu when it crossed 80% demand on a +power-efficient cpu. We don't let it come back to a power-efficient cpu until +its demand *in reference to the power-efficient cpu* drops less than 60% +(sched_downmigrate). + + +*** 7.17 sched_small_wakee_task_load + +Appears at: /proc/sys/kernel/sched_small_wakee_task_load + +Default value: 10 + +This tunable is a percentage. Configure the maximum demand of small wakee task. +Sync wakee tasks which have demand less than sched_small_wakee_task_load are +categorized as small wakee tasks. Scheduler places small wakee tasks on the +waker's cluster. + + +*** 7.18 sched_big_waker_task_load + +Appears at: /proc/sys/kernel/sched_big_waker_task_load + +Default value: 25 + +This tunable is a percentage. Configure the minimum demand of big sync waker +task. Scheduler places small wakee tasks woken up by big sync waker on the +waker's cluster. + +========================= +8. HMP SCHEDULER TRACE POINTS +========================= + +*** 8.1 sched_enq_deq_task + +Logged when a task is either enqueued or dequeued on a CPU's run queue. + + -0 [004] d.h4 12700.711665: sched_enq_deq_task: cpu=4 enqueue comm=powertop pid=13227 prio=120 nr_running=1 cpu_load=0 rt_nr_running=0 affine=ff demand=13364423 + +- cpu: the CPU that the task is being enqueued on to or dequeued off of +- enqueue/dequeue: whether this was an enqueue or dequeue event +- comm: name of task +- pid: PID of task +- prio: priority of task +- nr_running: number of runnable tasks on this CPU +- cpu_load: current priority-weighted load on the CPU (note, this is *not* + the same as CPU utilization or a metric tracked by PELT/window-based tracking) +- rt_nr_running: number of real-time processes running on this CPU +- affine: CPU affinity mask in hex for this task (so ff is a task eligible to + run on CPUs 0-7) +- demand: window-based task demand computed based on selected policy (recent, + max, or average) (ns) + +*** 8.2 sched_task_load + +Logged when selecting the best CPU to run the task (select_best_cpu()). + +sched_task_load: 4004 (adbd): demand=698425 boost=0 reason=0 sync=0 need_idle=0 best_cpu=0 latency=103177 + +- demand: window-based task demand computed based on selected policy (recent, + max, or average) (ns) +- boost: whether boost is in effect +- reason: reason we are picking a new CPU: + 0: no migration - selecting a CPU for a wakeup or new task wakeup + 1: move to big CPU (migration) + 2: move to little CPU (migration) + 3: move to low irq load CPU (migration) +- sync: is the nature synchronous in nature +- need_idle: is an idle CPU required for this task based on PF_WAKE_UP_IDLE +- best_cpu: The CPU selected by the select_best_cpu() function for placement +- latency: The execution time of the function select_best_cpu() + +*** 8.3 sched_cpu_load_* + +Logged when selecting the best CPU to run a task (select_best_cpu() for fair +class tasks, find_lowest_rq_hmp() for RT tasks) and load balancing +(update_sg_lb_stats()). + +-0 [004] d.h3 12700.711541: sched_cpu_load_*: cpu 0 idle 1 nr_run 0 nr_big 0 lsf 1119 capacity 1024 cr_avg 0 irqload 3301121 fcur 729600 fmax 1459200 power_cost 5 cstate 2 temp 38 + +- cpu: the CPU being described +- idle: boolean indicating whether the CPU is idle +- nr_run: number of tasks running on CPU +- nr_big: number of BIG tasks running on CPU +- lsf: load scale factor - multiply normalized load by this factor to determine + how much load task will exert on CPU +- capacity: capacity of CPU (based on max possible frequency and efficiency) +- cr_avg: cumulative runnable average, instantaneous sum of the demand (either + PELT or window-based) of all the runnable task on a CPU (ns) +- irqload: decaying average of irq activity on CPU (ns) +- fcur: current CPU frequency (Khz) +- fmax: max CPU frequency (but not maximum _possible_ frequency) (KHz) +- power_cost: cost of running this CPU at the current frequency +- cstate: current cstate of CPU +- temp: current temperature of the CPU + +The power_cost value above differs in how it is calculated depending on the +callsite of this tracepoint. The select_best_cpu() call to this tracepoint +finds the minimum frequency required to satisfy the existing load on the CPU +as well as the task being placed, and returns the power cost of that frequency. +The load balance and real time task placement paths used a fixed frequency +(highest frequency common to all CPUs for load balancing, minimum +frequency of the CPU for real time task placement). + +*** 8.4 sched_update_task_ravg + +Logged when window-based stats are updated for a task. The update may happen +for a variety of reasons, see section 2.5, "Task Events." + +-0 [004] d.h4 12700.711513: sched_update_task_ravg: wc 12700711473496 ws 12700691772135 delta 19701361 event TASK_WAKE cpu 4 cur_freq 199200 cur_pid 0 task 13227 (powertop) ms 12640648272532 delta 60063200964 demand 13364423 sum 0 irqtime 0 cs 0 ps 495018 cur_window 0 prev_window 0 + +- wc: wallclock, output of sched_clock(), monotonically increasing time since + boot (will roll over in 585 years) (ns) +- ws: window start, time when the current window started (ns) +- delta: time since the window started (wc - ws) (ns) +- event: What event caused this trace event to occur (see section 2.5 for more + details) +- cpu: which CPU the task is running on +- cur_freq: CPU's current frequency in KHz +- curr_pid: PID of the current running task (current) +- task: PID and name of task being updated +- ms: mark start - timestamp of the beginning of a segment of task activity, + either sleeping or runnable/running (ns) +- delta: time since last event within the window (wc - ms) (ns) +- demand: task demand computed based on selected policy (recent, max, or + average) (ns) +- sum: the task's run time during current window scaled by frequency and + efficiency (ns) +- irqtime: length of interrupt activity (ns). A non-zero irqtime is seen + when an idle cpu handles interrupts, the time for which needs to be + accounted as cpu busy time +- cs: curr_runnable_sum of cpu (ns). See section 6.1 for more details of this + counter. +- ps: prev_runnable_sum of cpu (ns). See section 6.1 for more details of this + counter. +- cur_window: cpu demand of task in its most recently tracked window (ns) +- prev_window: cpu demand of task in the window prior to the one being tracked + by cur_window + +*** 8.5 sched_update_history + +Logged when update_task_ravg() is accounting task activity into one or +more windows that have completed. This may occur more than once for a +single call into update_task_ravg(). A task that ran for 24ms spanning +four 10ms windows (the last 2ms of window 1, all of windows 2 and 3, +and the first 2ms of window 4) would result in two calls into +update_history() from update_task_ravg(). The first call would record activity +in completed window 1 and second call would record activity for windows 2 and 3 +together (samples will be 2 in second call). + +-0 [004] d.h4 12700.711489: sched_update_history: 13227 (powertop): runtime 13364423 samples 1 event TASK_WAKE demand 13364423 (hist: 13364423 9871252 2236009 6162476 10282078) cpu 4 nr_big 0 + +- runtime: task cpu demand in recently completed window(s). This value is scaled + to max_possible_freq and max_possible_efficiency. This value is pushed into + task's demand history array. The number of windows to which runtime applies is + provided by samples field. +- samples: Number of samples (windows), each having value of runtime, that is + recorded in task's demand history array. +- event: What event caused this trace event to occur (see section 2.5 for more + details) - PUT_PREV_TASK, PICK_NEXT_TASK, TASK_WAKE, TASK_MIGRATE, + TASK_UPDATE +- demand: task demand computed based on selected policy (recent, max, or + average) (ns) +- hist: last 5 windows of history for the task with the most recent window + listed first +- cpu: CPU the task is associated with +- nr_big: number of big tasks on the CPU + +*** 8.6 sched_reset_all_windows_stats + +Logged when key parameters controlling window-based statistics collection are +changed. This event signifies that all window-based statistics for tasks and +cpus are being reset. Changes to below attributes result in such a reset: + +* sched_ravg_window (See Sec 2) +* sched_window_stats_policy (See Sec 2.4) +* sched_ravg_hist_size (See Sec 7.11) + +-0 [004] d.h4 12700.711489: sched_reset_all_windows_stats: time_taken 1123 window_start 0 window_size 0 reason POLICY_CHANGE old_val 0 new_val 1 + +- time_taken: time taken for the reset function to complete (ns) +- window_start: Beginning of first window following change to window size (ns) +- window_size: Size of window. Non-zero if window-size is changing (in ticks) +- reason: Reason for reset of statistics. +- old_val: Old value of variable, change of which is triggering reset +- new_val: New value of variable, change of which is triggering reset + +*** 8.7 sched_migration_update_sum + +Logged when a task is migrating to another cpu. + +-0 [000] d..8 5020.404137: sched_migration_update_sum: cpu 0: cs 471278 ps 902463 nt_cs 0 nt_ps 0 pid 2645 + +- cpu: cpu, away from which or to which, task is migrating +- cs: curr_runnable_sum of cpu (ns). See Sec 6.1 for more details of this + counter. +- ps: prev_runnable_sum of cpu (ns). See Sec 6.1 for more details of this + counter. +- nt_cs: nt_curr_runnable_sum of cpu (ns). See Sec 6.1 for more details of + this counter. +- nt_ps: nt_prev_runnable_sum of cpu (ns). See Sec 6.1 for more details of + this counter +- pid: PID of migrating task + +*** 8.8 sched_get_busy + +Logged when scheduler is returning busy time statistics for a cpu. + +<...>-4331 [003] d.s3 313.700108: sched_get_busy: cpu 3 load 19076 new_task_load 0 early 0 + + +- cpu: cpu, for which busy time statistic (prev_runnable_sum) is being + returned (ns) +- load: corresponds to prev_runnable_sum (ns), scaled to fmax of cpu +- new_task_load: corresponds to nt_prev_runnable_sum to fmax of cpu +- early: A flag indicating whether the scheduler is passing regular load or early detection load + 0 - regular load + 1 - early detection load + +*** 8.9 sched_freq_alert + +Logged when scheduler is alerting cpufreq governor about need to change +frequency + +-0 [004] d.h4 12700.711489: sched_freq_alert: cpu 0 old_load=XXX new_load=YYY + +- cpu: cpu in cluster that has highest load (prev_runnable_sum) +- old_load: cpu busy time last reported to governor. This is load scaled in + reference to max_possible_freq and max_possible_efficiency. +- new_load: recent cpu busy time. This is load scaled in + reference to max_possible_freq and max_possible_efficiency. + +*** 8.10 sched_set_boost + +Logged when boost settings are being changed + +-0 [004] d.h4 12700.711489: sched_set_boost: ref_count=1 + +- ref_count: A non-zero value indicates boost is in effect diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 9f237b657018..aae03c4077b4 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -28,6 +28,21 @@ struct cpu { struct device dev; }; +struct cpu_pstate_pwr { + unsigned int freq; + uint32_t power; +}; + +struct cpu_pwr_stats { + int cpu; + long temp; + struct cpu_pstate_pwr *ptable; + bool throttling; + int len; +}; + +extern struct cpu_pwr_stats *get_cpu_pwr_stats(void); + extern void boot_cpu_init(void); extern void boot_cpu_state_init(void); diff --git a/include/linux/sched.h b/include/linux/sched.h index 348f51b0ec92..cc6f436e8a48 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -174,6 +174,9 @@ extern unsigned long nr_iowait(void); extern unsigned long nr_iowait_cpu(int cpu); extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load); +extern void sched_update_nr_prod(int cpu, long delta, bool inc); +extern void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg); + extern void calc_global_load(unsigned long ticks); #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) @@ -315,6 +318,25 @@ extern char ___assert_task_state[1 - 2*!!( /* Task command name length */ #define TASK_COMM_LEN 16 +extern const char *sched_window_reset_reasons[]; + +enum task_event { + PUT_PREV_TASK = 0, + PICK_NEXT_TASK = 1, + TASK_WAKE = 2, + TASK_MIGRATE = 3, + TASK_UPDATE = 4, + IRQ_UPDATE = 5, +}; + +/* Note: this need to be in sync with migrate_type_names array */ +enum migrate_types { + GROUP_TO_RQ, + RQ_TO_GROUP, + RQ_TO_RQ, + GROUP_TO_GROUP, +}; + #include /* @@ -1335,6 +1357,47 @@ struct sched_statistics { }; #endif +#define RAVG_HIST_SIZE_MAX 5 +#define NUM_BUSY_BUCKETS 10 + +/* ravg represents frequency scaled cpu-demand of tasks */ +struct ravg { + /* + * 'mark_start' marks the beginning of an event (task waking up, task + * starting to execute, task being preempted) within a window + * + * 'sum' represents how runnable a task has been within current + * window. It incorporates both running time and wait time and is + * frequency scaled. + * + * 'sum_history' keeps track of history of 'sum' seen over previous + * RAVG_HIST_SIZE windows. Windows where task was entirely sleeping are + * ignored. + * + * 'demand' represents maximum sum seen over previous + * sysctl_sched_ravg_hist_size windows. 'demand' could drive frequency + * demand for tasks. + * + * 'curr_window' represents task's contribution to cpu busy time + * statistics (rq->curr_runnable_sum) in current window + * + * 'prev_window' represents task's contribution to cpu busy time + * statistics (rq->prev_runnable_sum) in previous window + * + * 'pred_demand' represents task's current predicted cpu busy time + * + * 'busy_buckets' groups historical busy time into different buckets + * used for prediction + */ + u64 mark_start; + u32 sum, demand; + u32 sum_history[RAVG_HIST_SIZE_MAX]; + u32 curr_window, prev_window; + u16 active_windows; + u32 pred_demand; + u8 busy_buckets[NUM_BUSY_BUCKETS]; +}; + struct sched_entity { struct load_weight load; /* for load-balancing */ struct rb_node run_node; @@ -1505,6 +1568,20 @@ struct task_struct { const struct sched_class *sched_class; struct sched_entity se; struct sched_rt_entity rt; +#ifdef CONFIG_SCHED_HMP + struct ravg ravg; + /* + * 'init_load_pct' represents the initial task load assigned to children + * of this task + */ + u32 init_load_pct; + u64 last_wake_ts; + u64 last_switch_out_ts; + u64 last_cpu_selected_ts; + struct related_thread_group *grp; + struct list_head grp_list; + u64 cpu_cycles; +#endif #ifdef CONFIG_CGROUP_SCHED struct task_group *sched_task_group; #endif @@ -2254,6 +2331,7 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, /* * Per process flags */ +#define PF_WAKE_UP_IDLE 0x00000002 /* try to wake up on an idle CPU */ #define PF_EXITING 0x00000004 /* getting shut down */ #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */ #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ @@ -2436,6 +2514,93 @@ static inline int set_cpus_allowed_ptr(struct task_struct *p, } #endif +struct sched_load { + unsigned long prev_load; + unsigned long new_task_load; + unsigned long predicted_load; +}; + +extern int sched_set_wake_up_idle(struct task_struct *p, int wake_up_idle); +extern u32 sched_get_wake_up_idle(struct task_struct *p); + +struct cpu_cycle_counter_cb { + u64 (*get_cpu_cycle_counter)(int cpu); +}; + +#ifdef CONFIG_SCHED_HMP +extern int sched_set_window(u64 window_start, unsigned int window_size); +extern unsigned long sched_get_busy(int cpu); +extern void sched_get_cpus_busy(struct sched_load *busy, + const struct cpumask *query_cpus); +extern void sched_set_io_is_busy(int val); +extern int sched_set_boost(int enable); +extern int sched_set_init_task_load(struct task_struct *p, int init_load_pct); +extern u32 sched_get_init_task_load(struct task_struct *p); +extern int sched_set_static_cpu_pwr_cost(int cpu, unsigned int cost); +extern unsigned int sched_get_static_cpu_pwr_cost(int cpu); +extern int sched_set_static_cluster_pwr_cost(int cpu, unsigned int cost); +extern unsigned int sched_get_static_cluster_pwr_cost(int cpu); +extern int sched_update_freq_max_load(const cpumask_t *cpumask); +extern void sched_update_cpu_freq_min_max(const cpumask_t *cpus, + u32 fmin, u32 fmax); +extern void sched_set_cpu_cstate(int cpu, int cstate, + int wakeup_energy, int wakeup_latency); +extern void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, int dstate, + int wakeup_energy, int wakeup_latency); +extern int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb); +extern u64 sched_ktime_clock(void); +extern int sched_set_group_id(struct task_struct *p, unsigned int group_id); +extern unsigned int sched_get_group_id(struct task_struct *p); + +#else /* CONFIG_SCHED_HMP */ +static inline u64 sched_ktime_clock(void) +{ + return 0; +} + +static inline int +register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb) +{ + return 0; +} + +static inline int sched_set_window(u64 window_start, unsigned int window_size) +{ + return -EINVAL; +} +static inline unsigned long sched_get_busy(int cpu) +{ + return 0; +} +static inline void sched_get_cpus_busy(struct sched_load *busy, + const struct cpumask *query_cpus) {}; + +static inline void sched_set_io_is_busy(int val) {}; + +static inline int sched_set_boost(int enable) +{ + return -EINVAL; +} + +static inline int sched_update_freq_max_load(const cpumask_t *cpumask) +{ + return 0; +} + +static inline void sched_update_cpu_freq_min_max(const cpumask_t *cpus, + u32 fmin, u32 fmax) { } + +static inline void +sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) +{ +} + +static inline void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, + int dstate, int wakeup_energy, int wakeup_latency) +{ +} +#endif /* CONFIG_SCHED_HMP */ + #ifdef CONFIG_NO_HZ_COMMON void calc_load_enter_idle(void); void calc_load_exit_idle(void); @@ -2444,6 +2609,14 @@ static inline void calc_load_enter_idle(void) { } static inline void calc_load_exit_idle(void) { } #endif /* CONFIG_NO_HZ_COMMON */ +static inline void set_wake_up_idle(bool enabled) +{ + if (enabled) + current->flags |= PF_WAKE_UP_IDLE; + else + current->flags &= ~PF_WAKE_UP_IDLE; +} + /* * Do not use outside of architecture code which knows its limitations. * @@ -2461,6 +2634,7 @@ extern u64 sched_clock_cpu(int cpu); extern void sched_clock_init(void); +extern int sched_clock_initialized(void); #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK static inline void sched_clock_tick(void) @@ -2537,7 +2711,7 @@ extern unsigned long long task_sched_runtime(struct task_struct *task); /* sched_exec is called by processes performing an exec */ -#ifdef CONFIG_SMP +#if defined(CONFIG_SMP) && !defined(CONFIG_SCHED_HMP) extern void sched_exec(void); #else #define sched_exec() {} @@ -2670,6 +2844,7 @@ extern void xtime_update(unsigned long ticks); extern int wake_up_state(struct task_struct *tsk, unsigned int state); extern int wake_up_process(struct task_struct *tsk); +extern int wake_up_process_no_notif(struct task_struct *tsk); extern void wake_up_new_task(struct task_struct *tsk); #ifdef CONFIG_SMP extern void kick_process(struct task_struct *tsk); @@ -2678,6 +2853,12 @@ extern void wake_up_new_task(struct task_struct *tsk); #endif extern int sched_fork(unsigned long clone_flags, struct task_struct *p); extern void sched_dead(struct task_struct *p); +#ifdef CONFIG_SCHED_HMP +extern void sched_exit(struct task_struct *p); +#else +static inline void sched_exit(struct task_struct *p) { } +#endif + extern void proc_caches_init(void); extern void flush_signals(struct task_struct *); @@ -3506,6 +3687,8 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) #endif /* CONFIG_SMP */ +extern struct atomic_notifier_head load_alert_notifier_head; + extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); extern long sched_getaffinity(pid_t pid, struct cpumask *mask); diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 22db1e63707e..6726f05ee4b7 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -18,6 +18,32 @@ extern unsigned int sysctl_sched_latency; extern unsigned int sysctl_sched_min_granularity; extern unsigned int sysctl_sched_wakeup_granularity; extern unsigned int sysctl_sched_child_runs_first; +extern unsigned int sysctl_sched_wake_to_idle; + +#ifdef CONFIG_SCHED_HMP +extern int sysctl_sched_freq_inc_notify; +extern int sysctl_sched_freq_dec_notify; +extern unsigned int sysctl_sched_window_stats_policy; +extern unsigned int sysctl_sched_ravg_hist_size; +extern unsigned int sysctl_sched_cpu_high_irqload; +extern unsigned int sysctl_sched_init_task_load_pct; +extern unsigned int sysctl_sched_spill_nr_run; +extern unsigned int sysctl_sched_spill_load_pct; +extern unsigned int sysctl_sched_upmigrate_pct; +extern unsigned int sysctl_sched_downmigrate_pct; +extern unsigned int sysctl_early_detection_duration; +extern unsigned int sysctl_sched_boost; +extern unsigned int sysctl_sched_small_wakee_task_load_pct; +extern unsigned int sysctl_sched_big_waker_task_load_pct; +extern unsigned int sysctl_sched_select_prev_cpu_us; +extern unsigned int sysctl_sched_enable_colocation; +extern unsigned int sysctl_sched_restrict_cluster_spill; +extern unsigned int sysctl_sched_new_task_windows; +extern unsigned int sysctl_sched_pred_alert_freq; +extern unsigned int sysctl_sched_freq_aggregate; +extern unsigned int sysctl_sched_enable_thread_grouping; +extern unsigned int sysctl_sched_freq_aggregate_threshold_pct; +#endif /* CONFIG_SCHED_HMP */ enum sched_tunable_scaling { SCHED_TUNABLESCALING_NONE, @@ -43,6 +69,18 @@ int sched_proc_update_handler(struct ctl_table *table, int write, loff_t *ppos); #endif +extern int sched_migrate_notify_proc_handler(struct ctl_table *table, + int write, void __user *buffer, size_t *lenp, loff_t *ppos); + +extern int sched_hmp_proc_update_handler(struct ctl_table *table, + int write, void __user *buffer, size_t *lenp, loff_t *ppos); + +extern int sched_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos); + +extern int sched_window_update_handler(struct ctl_table *table, + int write, void __user *buffer, size_t *lenp, loff_t *ppos); + /* * control realtime throttling: * diff --git a/include/linux/tick.h b/include/linux/tick.h index 62be0786d6d0..9e207e30a699 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -27,6 +27,8 @@ static inline void tick_handover_do_timer(void) { } static inline void tick_cleanup_dead_cpu(int cpu) { } #endif /* !CONFIG_GENERIC_CLOCKEVENTS */ +extern u64 jiffy_to_ktime_ns(u64 *now, u64 *jiffy_ktime_ns); + #if defined(CONFIG_GENERIC_CLOCKEVENTS) && defined(CONFIG_SUSPEND) extern void tick_freeze(void); extern void tick_unfreeze(void); diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 3211890ee7d5..a52c343ecf42 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -8,6 +8,8 @@ #include #include +struct rq; + /* * Tracepoint for calling kthread_stop, performed to end a kthread: */ @@ -50,6 +52,561 @@ TRACE_EVENT(sched_kthread_stop_ret, TP_printk("ret=%d", __entry->ret) ); +/* + * Tracepoint for task enqueue/dequeue: + */ +TRACE_EVENT(sched_enq_deq_task, + + TP_PROTO(struct task_struct *p, bool enqueue, unsigned int cpus_allowed), + + TP_ARGS(p, enqueue, cpus_allowed), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( int, prio ) + __field( int, cpu ) + __field( bool, enqueue ) + __field(unsigned int, nr_running ) + __field(unsigned long, cpu_load ) + __field(unsigned int, rt_nr_running ) + __field(unsigned int, cpus_allowed ) +#ifdef CONFIG_SCHED_HMP + __field(unsigned int, demand ) + __field(unsigned int, pred_demand ) +#endif + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->prio = p->prio; + __entry->cpu = task_cpu(p); + __entry->enqueue = enqueue; + __entry->nr_running = task_rq(p)->nr_running; + __entry->cpu_load = task_rq(p)->cpu_load[0]; + __entry->rt_nr_running = task_rq(p)->rt.rt_nr_running; + __entry->cpus_allowed = cpus_allowed; +#ifdef CONFIG_SCHED_HMP + __entry->demand = p->ravg.demand; + __entry->pred_demand = p->ravg.pred_demand; +#endif + ), + + TP_printk("cpu=%d %s comm=%s pid=%d prio=%d nr_running=%u cpu_load=%lu rt_nr_running=%u affine=%x" +#ifdef CONFIG_SCHED_HMP + " demand=%u pred_demand=%u" +#endif + , __entry->cpu, + __entry->enqueue ? "enqueue" : "dequeue", + __entry->comm, __entry->pid, + __entry->prio, __entry->nr_running, + __entry->cpu_load, __entry->rt_nr_running, __entry->cpus_allowed +#ifdef CONFIG_SCHED_HMP + , __entry->demand, __entry->pred_demand +#endif + ) +); + +#ifdef CONFIG_SCHED_HMP + +struct group_cpu_time; +struct migration_sum_data; +extern const char *task_event_names[]; +extern const char *migrate_type_names[]; + +TRACE_EVENT(sched_task_load, + + TP_PROTO(struct task_struct *p, bool boost, int reason, + bool sync, bool need_idle, u32 flags, int best_cpu), + + TP_ARGS(p, boost, reason, sync, need_idle, flags, best_cpu), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, demand ) + __field( bool, boost ) + __field( int, reason ) + __field( bool, sync ) + __field( bool, need_idle ) + __field( u32, flags ) + __field( int, best_cpu ) + __field( u64, latency ) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->demand = p->ravg.demand; + __entry->boost = boost; + __entry->reason = reason; + __entry->sync = sync; + __entry->need_idle = need_idle; + __entry->flags = flags; + __entry->best_cpu = best_cpu; + __entry->latency = p->state == TASK_WAKING ? + sched_ktime_clock() - + p->ravg.mark_start : 0; + ), + + TP_printk("%d (%s): demand=%u boost=%d reason=%d sync=%d need_idle=%d flags=%x best_cpu=%d latency=%llu", + __entry->pid, __entry->comm, __entry->demand, + __entry->boost, __entry->reason, __entry->sync, + __entry->need_idle, __entry->flags, + __entry->best_cpu, __entry->latency) +); + +TRACE_EVENT(sched_set_preferred_cluster, + + TP_PROTO(struct related_thread_group *grp, u64 total_demand), + + TP_ARGS(grp, total_demand), + + TP_STRUCT__entry( + __field( int, id ) + __field( u64, demand ) + __field( int, cluster_first_cpu ) + ), + + TP_fast_assign( + __entry->id = grp->id; + __entry->demand = total_demand; + __entry->cluster_first_cpu = grp->preferred_cluster ? + cluster_first_cpu(grp->preferred_cluster) + : -1; + ), + + TP_printk("group_id %d total_demand %llu preferred_cluster_first_cpu %d", + __entry->id, __entry->demand, + __entry->cluster_first_cpu) +); + +DECLARE_EVENT_CLASS(sched_cpu_load, + + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + + TP_ARGS(rq, idle, irqload, power_cost, temp), + + TP_STRUCT__entry( + __field(unsigned int, cpu ) + __field(unsigned int, idle ) + __field(unsigned int, nr_running ) + __field(unsigned int, nr_big_tasks ) + __field(unsigned int, load_scale_factor ) + __field(unsigned int, capacity ) + __field( u64, cumulative_runnable_avg ) + __field( u64, irqload ) + __field(unsigned int, max_freq ) + __field(unsigned int, power_cost ) + __field( int, cstate ) + __field( int, dstate ) + __field( int, temp ) + ), + + TP_fast_assign( + __entry->cpu = rq->cpu; + __entry->idle = idle; + __entry->nr_running = rq->nr_running; + __entry->nr_big_tasks = rq->hmp_stats.nr_big_tasks; + __entry->load_scale_factor = cpu_load_scale_factor(rq->cpu); + __entry->capacity = cpu_capacity(rq->cpu); + __entry->cumulative_runnable_avg = rq->hmp_stats.cumulative_runnable_avg; + __entry->irqload = irqload; + __entry->max_freq = cpu_max_freq(rq->cpu); + __entry->power_cost = power_cost; + __entry->cstate = rq->cstate; + __entry->dstate = rq->cluster->dstate; + __entry->temp = temp; + ), + + TP_printk("cpu %u idle %d nr_run %u nr_big %u lsf %u capacity %u cr_avg %llu irqload %llu fmax %u power_cost %u cstate %d dstate %d temp %d", + __entry->cpu, __entry->idle, __entry->nr_running, __entry->nr_big_tasks, + __entry->load_scale_factor, __entry->capacity, + __entry->cumulative_runnable_avg, __entry->irqload, + __entry->max_freq, __entry->power_cost, __entry->cstate, + __entry->dstate, __entry->temp) +); + +DEFINE_EVENT(sched_cpu_load, sched_cpu_load_wakeup, + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + TP_ARGS(rq, idle, irqload, power_cost, temp) +); + +DEFINE_EVENT(sched_cpu_load, sched_cpu_load_lb, + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + TP_ARGS(rq, idle, irqload, power_cost, temp) +); + +DEFINE_EVENT(sched_cpu_load, sched_cpu_load_cgroup, + TP_PROTO(struct rq *rq, int idle, u64 irqload, unsigned int power_cost, int temp), + TP_ARGS(rq, idle, irqload, power_cost, temp) +); + +TRACE_EVENT(sched_set_boost, + + TP_PROTO(int ref_count), + + TP_ARGS(ref_count), + + TP_STRUCT__entry( + __field(unsigned int, ref_count ) + ), + + TP_fast_assign( + __entry->ref_count = ref_count; + ), + + TP_printk("ref_count=%d", __entry->ref_count) +); + +TRACE_EVENT(sched_update_task_ravg, + + TP_PROTO(struct task_struct *p, struct rq *rq, enum task_event evt, + u64 wallclock, u64 irqtime, u64 cycles, u64 exec_time, + struct group_cpu_time *cpu_time), + + TP_ARGS(p, rq, evt, wallclock, irqtime, cycles, exec_time, cpu_time), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field( pid_t, cur_pid ) + __field(unsigned int, cur_freq ) + __field( u64, wallclock ) + __field( u64, mark_start ) + __field( u64, delta_m ) + __field( u64, win_start ) + __field( u64, delta ) + __field( u64, irqtime ) + __field(enum task_event, evt ) + __field(unsigned int, demand ) + __field(unsigned int, sum ) + __field( int, cpu ) + __field(unsigned int, pred_demand ) + __field( u64, rq_cs ) + __field( u64, rq_ps ) + __field( u64, grp_cs ) + __field( u64, grp_ps ) + __field( u64, grp_nt_cs ) + __field( u64, grp_nt_ps ) + __field( u32, curr_window ) + __field( u32, prev_window ) + __field( u64, nt_cs ) + __field( u64, nt_ps ) + __field( u32, active_windows ) + ), + + TP_fast_assign( + __entry->wallclock = wallclock; + __entry->win_start = rq->window_start; + __entry->delta = (wallclock - rq->window_start); + __entry->evt = evt; + __entry->cpu = rq->cpu; + __entry->cur_pid = rq->curr->pid; + __entry->cur_freq = cpu_cycles_to_freq(cycles, exec_time); + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->mark_start = p->ravg.mark_start; + __entry->delta_m = (wallclock - p->ravg.mark_start); + __entry->demand = p->ravg.demand; + __entry->sum = p->ravg.sum; + __entry->irqtime = irqtime; + __entry->pred_demand = p->ravg.pred_demand; + __entry->rq_cs = rq->curr_runnable_sum; + __entry->rq_ps = rq->prev_runnable_sum; + __entry->grp_cs = cpu_time ? cpu_time->curr_runnable_sum : 0; + __entry->grp_ps = cpu_time ? cpu_time->prev_runnable_sum : 0; + __entry->grp_nt_cs = cpu_time ? cpu_time->nt_curr_runnable_sum : 0; + __entry->grp_nt_ps = cpu_time ? cpu_time->nt_prev_runnable_sum : 0; + __entry->curr_window = p->ravg.curr_window; + __entry->prev_window = p->ravg.prev_window; + __entry->nt_cs = rq->nt_curr_runnable_sum; + __entry->nt_ps = rq->nt_prev_runnable_sum; + __entry->active_windows = p->ravg.active_windows; + ), + + TP_printk("wc %llu ws %llu delta %llu event %s cpu %d cur_freq %u cur_pid %d task %d (%s) ms %llu delta %llu demand %u sum %u irqtime %llu pred_demand %u rq_cs %llu rq_ps %llu cur_window %u prev_window %u nt_cs %llu nt_ps %llu active_wins %u grp_cs %lld grp_ps %lld, grp_nt_cs %llu, grp_nt_ps: %llu" + , __entry->wallclock, __entry->win_start, __entry->delta, + task_event_names[__entry->evt], __entry->cpu, + __entry->cur_freq, __entry->cur_pid, + __entry->pid, __entry->comm, __entry->mark_start, + __entry->delta_m, __entry->demand, + __entry->sum, __entry->irqtime, __entry->pred_demand, + __entry->rq_cs, __entry->rq_ps, __entry->curr_window, + __entry->prev_window, __entry->nt_cs, __entry->nt_ps, + __entry->active_windows, __entry->grp_cs, + __entry->grp_ps, __entry->grp_nt_cs, __entry->grp_nt_ps) +); + +TRACE_EVENT(sched_get_task_cpu_cycles, + + TP_PROTO(int cpu, int event, u64 cycles, u64 exec_time), + + TP_ARGS(cpu, event, cycles, exec_time), + + TP_STRUCT__entry( + __field(int, cpu ) + __field(int, event ) + __field(u64, cycles ) + __field(u64, exec_time ) + __field(u32, freq ) + __field(u32, legacy_freq ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->event = event; + __entry->cycles = cycles; + __entry->exec_time = exec_time; + __entry->freq = cpu_cycles_to_freq(cycles, exec_time); + __entry->legacy_freq = cpu_cur_freq(cpu); + ), + + TP_printk("cpu=%d event=%d cycles=%llu exec_time=%llu freq=%u legacy_freq=%u", + __entry->cpu, __entry->event, __entry->cycles, + __entry->exec_time, __entry->freq, __entry->legacy_freq) +); + +TRACE_EVENT(sched_update_history, + + TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int samples, + enum task_event evt), + + TP_ARGS(rq, p, runtime, samples, evt), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, runtime ) + __field( int, samples ) + __field(enum task_event, evt ) + __field(unsigned int, demand ) + __field(unsigned int, pred_demand ) + __array( u32, hist, RAVG_HIST_SIZE_MAX) + __field(unsigned int, nr_big_tasks ) + __field( int, cpu ) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->runtime = runtime; + __entry->samples = samples; + __entry->evt = evt; + __entry->demand = p->ravg.demand; + __entry->pred_demand = p->ravg.pred_demand; + memcpy(__entry->hist, p->ravg.sum_history, + RAVG_HIST_SIZE_MAX * sizeof(u32)); + __entry->nr_big_tasks = rq->hmp_stats.nr_big_tasks; + __entry->cpu = rq->cpu; + ), + + TP_printk("%d (%s): runtime %u samples %d event %s demand %u pred_demand %u" + " (hist: %u %u %u %u %u) cpu %d nr_big %u", + __entry->pid, __entry->comm, + __entry->runtime, __entry->samples, + task_event_names[__entry->evt], + __entry->demand, __entry->pred_demand, + __entry->hist[0], __entry->hist[1], + __entry->hist[2], __entry->hist[3], + __entry->hist[4], __entry->cpu, __entry->nr_big_tasks) +); + +TRACE_EVENT(sched_reset_all_window_stats, + + TP_PROTO(u64 window_start, u64 window_size, u64 time_taken, + int reason, unsigned int old_val, unsigned int new_val), + + TP_ARGS(window_start, window_size, time_taken, + reason, old_val, new_val), + + TP_STRUCT__entry( + __field( u64, window_start ) + __field( u64, window_size ) + __field( u64, time_taken ) + __field( int, reason ) + __field(unsigned int, old_val ) + __field(unsigned int, new_val ) + ), + + TP_fast_assign( + __entry->window_start = window_start; + __entry->window_size = window_size; + __entry->time_taken = time_taken; + __entry->reason = reason; + __entry->old_val = old_val; + __entry->new_val = new_val; + ), + + TP_printk("time_taken %llu window_start %llu window_size %llu reason %s old_val %u new_val %u", + __entry->time_taken, __entry->window_start, + __entry->window_size, + sched_window_reset_reasons[__entry->reason], + __entry->old_val, __entry->new_val) +); + +TRACE_EVENT(sched_update_pred_demand, + + TP_PROTO(struct rq *rq, struct task_struct *p, u32 runtime, int pct, + unsigned int pred_demand), + + TP_ARGS(rq, p, runtime, pct, pred_demand), + + TP_STRUCT__entry( + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + __field(unsigned int, runtime ) + __field( int, pct ) + __field(unsigned int, pred_demand ) + __array( u8, bucket, NUM_BUSY_BUCKETS) + __field( int, cpu ) + ), + + TP_fast_assign( + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + __entry->runtime = runtime; + __entry->pct = pct; + __entry->pred_demand = pred_demand; + memcpy(__entry->bucket, p->ravg.busy_buckets, + NUM_BUSY_BUCKETS * sizeof(u8)); + __entry->cpu = rq->cpu; + ), + + TP_printk("%d (%s): runtime %u pct %d cpu %d pred_demand %u (buckets: %u %u %u %u %u %u %u %u %u %u)", + __entry->pid, __entry->comm, + __entry->runtime, __entry->pct, __entry->cpu, + __entry->pred_demand, __entry->bucket[0], __entry->bucket[1], + __entry->bucket[2], __entry->bucket[3] ,__entry->bucket[4], + __entry->bucket[5], __entry->bucket[6], __entry->bucket[7], + __entry->bucket[8], __entry->bucket[9]) +); + +TRACE_EVENT(sched_migration_update_sum, + + TP_PROTO(struct task_struct *p, enum migrate_types migrate_type, struct migration_sum_data *d), + + TP_ARGS(p, migrate_type, d), + + TP_STRUCT__entry( + __field(int, tcpu ) + __field(int, pid ) + __field( u64, cs ) + __field( u64, ps ) + __field( s64, nt_cs ) + __field( s64, nt_ps ) + __field(enum migrate_types, migrate_type ) + __field( s64, src_cs ) + __field( s64, src_ps ) + __field( s64, dst_cs ) + __field( s64, dst_ps ) + __field( s64, src_nt_cs ) + __field( s64, src_nt_ps ) + __field( s64, dst_nt_cs ) + __field( s64, dst_nt_ps ) + ), + + TP_fast_assign( + __entry->tcpu = task_cpu(p); + __entry->pid = p->pid; + __entry->migrate_type = migrate_type; + __entry->src_cs = d->src_rq ? + d->src_rq->curr_runnable_sum : + d->src_cpu_time->curr_runnable_sum; + __entry->src_ps = d->src_rq ? + d->src_rq->prev_runnable_sum : + d->src_cpu_time->prev_runnable_sum; + __entry->dst_cs = d->dst_rq ? + d->dst_rq->curr_runnable_sum : + d->dst_cpu_time->curr_runnable_sum; + __entry->dst_ps = d->dst_rq ? + d->dst_rq->prev_runnable_sum : + d->dst_cpu_time->prev_runnable_sum; + __entry->src_nt_cs = d->src_rq ? + d->src_rq->nt_curr_runnable_sum : + d->src_cpu_time->nt_curr_runnable_sum; + __entry->src_nt_ps = d->src_rq ? + d->src_rq->nt_prev_runnable_sum : + d->src_cpu_time->nt_prev_runnable_sum; + __entry->dst_nt_cs = d->dst_rq ? + d->dst_rq->nt_curr_runnable_sum : + d->dst_cpu_time->nt_curr_runnable_sum; + __entry->dst_nt_ps = d->dst_rq ? + d->dst_rq->nt_prev_runnable_sum : + d->dst_cpu_time->nt_prev_runnable_sum; + ), + + TP_printk("pid %d task_cpu %d migrate_type %s src_cs %llu src_ps %llu dst_cs %lld dst_ps %lld src_nt_cs %llu src_nt_ps %llu dst_nt_cs %lld dst_nt_ps %lld", + __entry->pid, __entry->tcpu, migrate_type_names[__entry->migrate_type], + __entry->src_cs, __entry->src_ps, __entry->dst_cs, __entry->dst_ps, + __entry->src_nt_cs, __entry->src_nt_ps, __entry->dst_nt_cs, __entry->dst_nt_ps) +); + +TRACE_EVENT(sched_get_busy, + + TP_PROTO(int cpu, u64 load, u64 nload, u64 pload, int early), + + TP_ARGS(cpu, load, nload, pload, early), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( u64, load ) + __field( u64, nload ) + __field( u64, pload ) + __field( int, early ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->load = load; + __entry->nload = nload; + __entry->pload = pload; + __entry->early = early; + ), + + TP_printk("cpu %d load %lld new_task_load %lld predicted_load %lld early %d", + __entry->cpu, __entry->load, __entry->nload, + __entry->pload, __entry->early) +); + +TRACE_EVENT(sched_freq_alert, + + TP_PROTO(int cpu, int pd_notif, int check_groups, struct rq *rq, + u64 new_load), + + TP_ARGS(cpu, pd_notif, check_groups, rq, new_load), + + TP_STRUCT__entry( + __field( int, cpu ) + __field( int, pd_notif ) + __field( int, check_groups ) + __field( u64, old_busy_time ) + __field( u64, ps ) + __field( u64, new_load ) + __field( u64, old_pred ) + __field( u64, new_pred ) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->pd_notif = pd_notif; + __entry->check_groups = check_groups; + __entry->old_busy_time = rq->old_busy_time; + __entry->ps = rq->prev_runnable_sum; + __entry->new_load = new_load; + __entry->old_pred = rq->old_estimated_time; + __entry->new_pred = rq->hmp_stats.pred_demands_sum; + ), + + TP_printk("cpu %d pd_notif=%d check_groups %d old_busy_time=%llu prev_sum=%lld new_load=%llu old_pred=%llu new_pred=%llu", + __entry->cpu, __entry->pd_notif, __entry->check_groups, + __entry->old_busy_time, __entry->ps, __entry->new_load, + __entry->old_pred, __entry->new_pred) +); + +#endif /* CONFIG_SCHED_HMP */ + /* * Tracepoint for waking up a task: */ @@ -166,14 +723,15 @@ TRACE_EVENT(sched_switch, */ TRACE_EVENT(sched_migrate_task, - TP_PROTO(struct task_struct *p, int dest_cpu), + TP_PROTO(struct task_struct *p, int dest_cpu, unsigned int load), - TP_ARGS(p, dest_cpu), + TP_ARGS(p, dest_cpu, load), TP_STRUCT__entry( __array( char, comm, TASK_COMM_LEN ) __field( pid_t, pid ) __field( int, prio ) + __field(unsigned int, load ) __field( int, orig_cpu ) __field( int, dest_cpu ) ), @@ -182,15 +740,91 @@ TRACE_EVENT(sched_migrate_task, memcpy(__entry->comm, p->comm, TASK_COMM_LEN); __entry->pid = p->pid; __entry->prio = p->prio; + __entry->load = load; __entry->orig_cpu = task_cpu(p); __entry->dest_cpu = dest_cpu; ), - TP_printk("comm=%s pid=%d prio=%d orig_cpu=%d dest_cpu=%d", - __entry->comm, __entry->pid, __entry->prio, + TP_printk("comm=%s pid=%d prio=%d load=%d orig_cpu=%d dest_cpu=%d", + __entry->comm, __entry->pid, __entry->prio, __entry->load, __entry->orig_cpu, __entry->dest_cpu) ); +/* + * Tracepoint for a CPU going offline/online: + */ +TRACE_EVENT(sched_cpu_hotplug, + + TP_PROTO(int affected_cpu, int error, int status), + + TP_ARGS(affected_cpu, error, status), + + TP_STRUCT__entry( + __field( int, affected_cpu ) + __field( int, error ) + __field( int, status ) + ), + + TP_fast_assign( + __entry->affected_cpu = affected_cpu; + __entry->error = error; + __entry->status = status; + ), + + TP_printk("cpu %d %s error=%d", __entry->affected_cpu, + __entry->status ? "online" : "offline", __entry->error) +); + +/* + * Tracepoint for load balancing: + */ +#if NR_CPUS > 32 +#error "Unsupported NR_CPUS for lb tracepoint." +#endif +TRACE_EVENT(sched_load_balance, + + TP_PROTO(int cpu, enum cpu_idle_type idle, int balance, + unsigned long group_mask, int busiest_nr_running, + unsigned long imbalance, unsigned int env_flags, int ld_moved, + unsigned int balance_interval), + + TP_ARGS(cpu, idle, balance, group_mask, busiest_nr_running, + imbalance, env_flags, ld_moved, balance_interval), + + TP_STRUCT__entry( + __field( int, cpu) + __field( enum cpu_idle_type, idle) + __field( int, balance) + __field( unsigned long, group_mask) + __field( int, busiest_nr_running) + __field( unsigned long, imbalance) + __field( unsigned int, env_flags) + __field( int, ld_moved) + __field( unsigned int, balance_interval) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->idle = idle; + __entry->balance = balance; + __entry->group_mask = group_mask; + __entry->busiest_nr_running = busiest_nr_running; + __entry->imbalance = imbalance; + __entry->env_flags = env_flags; + __entry->ld_moved = ld_moved; + __entry->balance_interval = balance_interval; + ), + + TP_printk("cpu=%d state=%s balance=%d group=%#lx busy_nr=%d imbalance=%ld flags=%#x ld_moved=%d bal_int=%d", + __entry->cpu, + __entry->idle == CPU_IDLE ? "idle" : + (__entry->idle == CPU_NEWLY_IDLE ? "newly_idle" : "busy"), + __entry->balance, + __entry->group_mask, __entry->busiest_nr_running, + __entry->imbalance, __entry->env_flags, __entry->ld_moved, + __entry->balance_interval) +); + DECLARE_EVENT_CLASS(sched_process_template, TP_PROTO(struct task_struct *p), @@ -586,6 +1220,28 @@ TRACE_EVENT(sched_wake_idle_without_ipi, TP_printk("cpu=%d", __entry->cpu) ); + +TRACE_EVENT(sched_get_nr_running_avg, + + TP_PROTO(int avg, int big_avg, int iowait_avg), + + TP_ARGS(avg, big_avg, iowait_avg), + + TP_STRUCT__entry( + __field( int, avg ) + __field( int, big_avg ) + __field( int, iowait_avg ) + ), + + TP_fast_assign( + __entry->avg = avg; + __entry->big_avg = big_avg; + __entry->iowait_avg = iowait_avg; + ), + + TP_printk("avg=%d big_avg=%d iowait_avg=%d", + __entry->avg, __entry->big_avg, __entry->iowait_avg) +); #endif /* _TRACE_SCHED_H */ /* This part must be outside protection */ diff --git a/init/Kconfig b/init/Kconfig index 34407f15e6d3..f595c26258fd 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1165,6 +1165,23 @@ config CGROUP_DEBUG endif # CGROUPS +config SCHED_HMP + bool "Scheduler support for heterogenous multi-processor systems" + depends on SMP && FAIR_GROUP_SCHED + help + This feature will let the scheduler optimize task placement on + systems made of heterogeneous cpus i.e cpus that differ either + in their instructions per-cycle capability or the maximum + frequency they can attain. + +config SCHED_HMP_CSTATE_AWARE + bool "CPU C-state aware scheduler" + depends on SCHED_HMP + help + This feature will let the HMP scheduler optimize task placement + with CPUs C-state. If this is enabled, scheduler places tasks + onto the shallowest C-state CPU among the most power efficient CPUs. + config CHECKPOINT_RESTORE bool "Checkpoint/restore support" if EXPERT select PROC_CHILDREN diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 5e59b832ae2b..1302fffe0d50 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -17,8 +17,9 @@ endif obj-y += core.o loadavg.o clock.o cputime.o obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o -obj-y += wait.o swait.o completion.o idle.o +obj-y += wait.o swait.o completion.o idle.o sched_avg.o obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o +obj-$(CONFIG_SCHED_HMP) += hmp.o obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o obj-$(CONFIG_SCHEDSTATS) += stats.o obj-$(CONFIG_SCHED_DEBUG) += debug.o diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ae4c93ffe84e..a662a7f753b2 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -91,6 +91,8 @@ #define CREATE_TRACE_POINTS #include +ATOMIC_NOTIFIER_HEAD(load_alert_notifier_head); + DEFINE_MUTEX(sched_domains_mutex); DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); @@ -758,6 +760,7 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) if (!(flags & ENQUEUE_RESTORE)) sched_info_queued(rq, p); p->sched_class->enqueue_task(rq, p, flags); + trace_sched_enq_deq_task(p, 1, cpumask_bits(&p->cpus_allowed)[0]); } static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) @@ -766,6 +769,7 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) if (!(flags & DEQUEUE_SAVE)) sched_info_dequeued(rq, p); p->sched_class->dequeue_task(rq, p, flags); + trace_sched_enq_deq_task(p, 0, cpumask_bits(&p->cpus_allowed)[0]); } void activate_task(struct rq *rq, struct task_struct *p, int flags) @@ -998,8 +1002,9 @@ static struct rq *move_queued_task(struct rq *rq, struct task_struct *p, int new p->on_rq = TASK_ON_RQ_MIGRATING; dequeue_task(rq, p, 0); + double_lock_balance(rq, cpu_rq(new_cpu)); set_task_cpu(p, new_cpu); - raw_spin_unlock(&rq->lock); + double_rq_unlock(cpu_rq(new_cpu), rq); rq = cpu_rq(new_cpu); @@ -1050,6 +1055,8 @@ static int migration_cpu_stop(void *data) struct migration_arg *arg = data; struct task_struct *p = arg->task; struct rq *rq = this_rq(); + int src_cpu = cpu_of(rq); + bool moved = false; /* * The original target cpu might have gone down and we might @@ -1071,15 +1078,21 @@ static int migration_cpu_stop(void *data) * we're holding p->pi_lock. */ if (task_rq(p) == rq) { - if (task_on_rq_queued(p)) + if (task_on_rq_queued(p)) { rq = __migrate_task(rq, p, arg->dest_cpu); - else + moved = true; + } else { p->wake_cpu = arg->dest_cpu; + } } raw_spin_unlock(&rq->lock); raw_spin_unlock(&p->pi_lock); local_irq_enable(); + + if (moved) + notify_migration(src_cpu, arg->dest_cpu, false, p); + return 0; } @@ -1161,7 +1174,8 @@ static int __set_cpus_allowed_ptr(struct task_struct *p, if (cpumask_equal(&p->cpus_allowed, new_mask)) goto out; - if (!cpumask_intersects(new_mask, cpu_valid_mask)) { + dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask); + if (dest_cpu >= nr_cpu_ids) { ret = -EINVAL; goto out; } @@ -1246,13 +1260,15 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) #endif #endif - trace_sched_migrate_task(p, new_cpu); + trace_sched_migrate_task(p, new_cpu, pct_task_load(p)); if (task_cpu(p) != new_cpu) { if (p->sched_class->migrate_task_rq) p->sched_class->migrate_task_rq(p); p->se.nr_migrations++; perf_event_task_migrate(p); + + fixup_busy_time(p, new_cpu); } __set_task_cpu(p, new_cpu); @@ -1456,7 +1472,7 @@ unsigned long wait_task_inactive(struct task_struct *p, long match_state) * yield - it could be a while. */ if (unlikely(queued)) { - ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ); + ktime_t to = ktime_set(0, NSEC_PER_SEC); set_current_state(TASK_UNINTERRUPTIBLE); schedule_hrtimeout(&to, HRTIMER_MODE_REL); @@ -1798,6 +1814,8 @@ void sched_ttwu_pending(void) void scheduler_ipi(void) { + int cpu = smp_processor_id(); + /* * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting * TIF_NEED_RESCHED remotely (for the first time) will also send @@ -1805,9 +1823,18 @@ void scheduler_ipi(void) */ preempt_fold_need_resched(); - if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) + if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick() && + !got_boost_kick()) return; + if (got_boost_kick()) { + struct rq *rq = cpu_rq(cpu); + + if (rq->curr->sched_class == &fair_sched_class) + check_for_migration(rq, rq->curr); + clear_boost_kick(cpu); + } + /* * Not all reschedule IPI handlers call irq_enter/irq_exit, since * traditionally all their work was done from the interrupt return @@ -2009,6 +2036,18 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) { unsigned long flags; int cpu, success = 0; +#ifdef CONFIG_SMP + unsigned int old_load; + struct rq *rq; + u64 wallclock; + struct related_thread_group *grp = NULL; + int src_cpu; + bool notif_required = false; +#endif + bool freq_notif_allowed = !(wake_flags & WF_NO_NOTIFIER); + bool check_group = false; + + wake_flags &= ~WF_NO_NOTIFIER; /* * If we are going to wake up a thread waiting for CONDITION we @@ -2082,14 +2121,33 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_cond_load_acquire(&p->on_cpu, !VAL); + rq = cpu_rq(task_cpu(p)); + raw_spin_lock(&rq->lock); + old_load = task_load(p); + wallclock = sched_ktime_clock(); + update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); + raw_spin_unlock(&rq->lock); + + rcu_read_lock(); + grp = task_related_thread_group(p); + if (update_preferred_cluster(grp, p, old_load)) + set_preferred_cluster(grp); + rcu_read_unlock(); + check_group = grp != NULL; + p->sched_contributes_to_load = !!task_contributes_to_load(p); p->state = TASK_WAKING; cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); - if (task_cpu(p) != cpu) { + src_cpu = task_cpu(p); + if (src_cpu != cpu) { wake_flags |= WF_MIGRATED; set_task_cpu(p, cpu); + notif_required = true; } + + set_task_last_wake(p, wallclock); #endif /* CONFIG_SMP */ ttwu_queue(p, cpu, wake_flags); @@ -2098,6 +2156,17 @@ stat: out: raw_spin_unlock_irqrestore(&p->pi_lock, flags); + if (freq_notif_allowed) { + if (notif_required && !same_freq_domain(src_cpu, cpu)) { + check_for_freq_change(cpu_rq(cpu), + false, check_group); + check_for_freq_change(cpu_rq(src_cpu), + false, check_group); + } else if (success) { + check_for_freq_change(cpu_rq(cpu), true, false); + } + } + return success; } @@ -2114,9 +2183,13 @@ static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie cookie { struct rq *rq = task_rq(p); - if (WARN_ON_ONCE(rq != this_rq()) || - WARN_ON_ONCE(p == current)) + if (rq != this_rq() || p == current) { + printk_deferred("%s: Failed to wakeup task %d (%s), rq = %p," + " this_rq = %p, p = %p, current = %p\n", __func__, + task_pid_nr(p), p->comm, rq, this_rq(), p, current); + return; + } lockdep_assert_held(&rq->lock); @@ -2139,8 +2212,14 @@ static void try_to_wake_up_local(struct task_struct *p, struct pin_cookie cookie trace_sched_waking(p); - if (!task_on_rq_queued(p)) + if (!task_on_rq_queued(p)) { + u64 wallclock = sched_ktime_clock(); + + update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + update_task_ravg(p, rq, TASK_WAKE, wallclock, 0); ttwu_activate(rq, p, ENQUEUE_WAKEUP); + set_task_last_wake(p, wallclock); + } ttwu_do_wakeup(rq, p, 0, cookie); ttwu_stat(p, smp_processor_id(), 0); @@ -2166,6 +2245,26 @@ int wake_up_process(struct task_struct *p) } EXPORT_SYMBOL(wake_up_process); +/** + * wake_up_process_no_notif - Wake up a specific process without notifying + * governor + * @p: The process to be woken up. + * + * Attempt to wake up the nominated process and move it to the set of runnable + * processes. + * + * Return: 1 if the process was woken up, 0 if it was already running. + * + * It may be assumed that this function implies a write memory barrier before + * changing the task state if and only if any tasks are woken up. + */ +int wake_up_process_no_notif(struct task_struct *p) +{ + WARN_ON(task_is_stopped_or_traced(p)); + return try_to_wake_up(p, TASK_NORMAL, WF_NO_NOTIFIER); +} +EXPORT_SYMBOL(wake_up_process_no_notif); + int wake_up_state(struct task_struct *p, unsigned int state) { return try_to_wake_up(p, state, 0); @@ -2188,6 +2287,44 @@ void __dl_clear_params(struct task_struct *p) dl_se->dl_yielded = 0; } +#ifdef CONFIG_SCHED_HMP +/* + * sched_exit() - Set EXITING_TASK_MARKER in task's ravg.demand field + * + * Stop accounting (exiting) task's future cpu usage + * + * We need this so that reset_all_windows_stats() can function correctly. + * reset_all_window_stats() depends on do_each_thread/for_each_thread task + * iterators to reset *all* task's statistics. Exiting tasks however become + * invisible to those iterators. sched_exit() is called on a exiting task prior + * to being removed from task_list, which will let reset_all_window_stats() + * function correctly. + */ +void sched_exit(struct task_struct *p) +{ + unsigned long flags; + int cpu = get_cpu(); + struct rq *rq = cpu_rq(cpu); + u64 wallclock; + + sched_set_group_id(p, 0); + + raw_spin_lock_irqsave(&rq->lock, flags); + /* rq->curr == p */ + wallclock = sched_ktime_clock(); + update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + dequeue_task(rq, p, 0); + reset_task_stats(p); + p->ravg.mark_start = wallclock; + p->ravg.sum_history[0] = EXITING_TASK_MARKER; + enqueue_task(rq, p, 0); + clear_ed_task(p, rq); + raw_spin_unlock_irqrestore(&rq->lock, flags); + + put_cpu(); +} +#endif /* CONFIG_SCHED_HMP */ + /* * Perform scheduler related setup for a newly forked process p. * p is forked by current. @@ -2564,6 +2701,8 @@ void wake_up_new_task(struct task_struct *p) struct rq *rq; raw_spin_lock_irqsave(&p->pi_lock, rf.flags); + init_new_task_load(p); + add_new_task_to_grp(p); p->state = TASK_RUNNING; #ifdef CONFIG_SMP /* @@ -2579,6 +2718,7 @@ void wake_up_new_task(struct task_struct *p) rq = __task_rq_lock(p, &rf); post_init_entity_util_avg(&p->se); + mark_task_starting(p); activate_task(rq, p, 0); p->on_rq = TASK_ON_RQ_QUEUED; trace_sched_wakeup_new(p); @@ -2971,7 +3111,7 @@ void get_iowait_load(unsigned long *nr_waiters, unsigned long *load) *load = rq->load.weight; } -#ifdef CONFIG_SMP +#if defined(CONFIG_SMP) && !defined(CONFIG_SCHED_HMP) /* * sched_exec - execve() is a valuable balancing opportunity, because at @@ -3077,16 +3217,32 @@ void scheduler_tick(void) int cpu = smp_processor_id(); struct rq *rq = cpu_rq(cpu); struct task_struct *curr = rq->curr; + u64 wallclock; + bool early_notif; + u32 old_load; + struct related_thread_group *grp; sched_clock_tick(); raw_spin_lock(&rq->lock); + + old_load = task_load(curr); + set_window_start(rq); + update_rq_clock(rq); curr->sched_class->task_tick(rq, curr, 0); cpu_load_update_active(rq); calc_global_load_tick(rq); + + wallclock = sched_ktime_clock(); + update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + early_notif = early_detection_notify(rq, wallclock); + raw_spin_unlock(&rq->lock); + if (early_notif) + atomic_notifier_call_chain(&load_alert_notifier_head, + 0, (void *)(long)cpu); perf_event_task_tick(); #ifdef CONFIG_SMP @@ -3094,6 +3250,15 @@ void scheduler_tick(void) trigger_load_balance(rq); #endif rq_last_tick_reset(rq); + + rcu_read_lock(); + grp = task_related_thread_group(curr); + if (update_preferred_cluster(grp, curr, old_load)) + set_preferred_cluster(grp); + rcu_read_unlock(); + + if (curr->sched_class == &fair_sched_class) + check_for_migration(rq, curr); } #ifdef CONFIG_NO_HZ_FULL @@ -3227,6 +3392,9 @@ static noinline void __schedule_bug(struct task_struct *prev) if (panic_on_warn) panic("scheduling while atomic\n"); +#ifdef CONFIG_PANIC_ON_SCHED_BUG + BUG(); +#endif dump_stack(); add_taint(TAINT_WARN, LOCKDEP_STILL_OK); } @@ -3337,6 +3505,7 @@ static void __sched notrace __schedule(bool preempt) struct pin_cookie cookie; struct rq *rq; int cpu; + u64 wallclock; cpu = smp_processor_id(); rq = cpu_rq(cpu); @@ -3389,6 +3558,11 @@ static void __sched notrace __schedule(bool preempt) update_rq_clock(rq); next = pick_next_task(rq, prev, cookie); + + wallclock = sched_ktime_clock(); + update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0); + update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0); + clear_tsk_need_resched(prev); clear_preempt_need_resched(); rq->clock_skip_update = 0; @@ -3398,6 +3572,8 @@ static void __sched notrace __schedule(bool preempt) rq->curr = next; ++*switch_count; + set_task_last_switch_out(prev, wallclock); + trace_sched_switch(preempt, prev, next); rq = context_switch(rq, prev, next, cookie); /* unlocks the rq */ } else { @@ -5273,10 +5449,11 @@ void init_idle(struct task_struct *idle, int cpu) struct rq *rq = cpu_rq(cpu); unsigned long flags; + __sched_fork(0, idle); + raw_spin_lock_irqsave(&idle->pi_lock, flags); raw_spin_lock(&rq->lock); - __sched_fork(0, idle); idle->state = TASK_RUNNING; idle->se.exec_start = sched_clock(); @@ -5586,7 +5763,10 @@ static void migrate_tasks(struct rq *dead_rq) rq = __migrate_task(rq, next, dest_cpu); if (rq != dead_rq) { raw_spin_unlock(&rq->lock); + raw_spin_unlock(&next->pi_lock); + notify_migration(dead_rq->cpu, dest_cpu, true, next); rq = dead_rq; + raw_spin_lock(&next->pi_lock); raw_spin_lock(&rq->lock); } raw_spin_unlock(&next->pi_lock); @@ -6030,6 +6210,7 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) { struct rq *rq = cpu_rq(cpu); struct sched_domain *tmp; + unsigned long next_balance = rq->next_balance; /* Remove the sched domains which do not contribute to scheduling. */ for (tmp = sd; tmp; ) { @@ -6061,6 +6242,17 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) sd->child = NULL; } + for (tmp = sd; tmp; ) { + unsigned long interval; + + interval = msecs_to_jiffies(tmp->balance_interval); + if (time_after(next_balance, tmp->last_balance + interval)) + next_balance = tmp->last_balance + interval; + + tmp = tmp->parent; + } + rq->next_balance = next_balance; + sched_domain_debug(sd, cpu); rq_attach_root(rq, rd); @@ -7386,7 +7578,11 @@ int sched_cpu_deactivate(unsigned int cpu) static void sched_rq_cpu_starting(unsigned int cpu) { struct rq *rq = cpu_rq(cpu); + unsigned long flags; + raw_spin_lock_irqsave(&rq->lock, flags); + set_window_start(rq); + raw_spin_unlock_irqrestore(&rq->lock, flags); rq->calc_load_update = calc_load_update; update_max_interval(); } @@ -7407,6 +7603,8 @@ int sched_cpu_dying(unsigned int cpu) /* Handle pending wakeups and then migrate everything off */ sched_ttwu_pending(); raw_spin_lock_irqsave(&rq->lock, flags); + migrate_sync_cpu(cpu); + if (rq->rd) { BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); set_rq_offline(rq); @@ -7414,6 +7612,9 @@ int sched_cpu_dying(unsigned int cpu) migrate_tasks(rq); BUG_ON(rq->nr_running != 1); raw_spin_unlock_irqrestore(&rq->lock, flags); + + clear_hmp_request(cpu); + calc_load_migrate(rq); update_max_interval(); nohz_balance_exit_idle(cpu); @@ -7459,6 +7660,8 @@ void __init sched_init_smp(void) cpumask_set_cpu(smp_processor_id(), non_isolated_cpus); mutex_unlock(&sched_domains_mutex); + update_cluster_topology(); + /* Move init over to a non-isolated CPU */ if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0) BUG(); @@ -7530,6 +7733,11 @@ void __init sched_init(void) for (i = 0; i < WAIT_TABLE_SIZE; i++) init_waitqueue_head(bit_wait_table + i); +#ifdef CONFIG_SCHED_HMP + pr_info("HMP scheduling enabled.\n"); + init_clusters(); +#endif + #ifdef CONFIG_FAIR_GROUP_SCHED alloc_size += 2 * nr_cpu_ids * sizeof(void **); #endif @@ -7646,7 +7854,36 @@ void __init sched_init(void) rq->idle_stamp = 0; rq->avg_idle = 2*sysctl_sched_migration_cost; rq->max_idle_balance_cost = sysctl_sched_migration_cost; + rq->push_task = NULL; +#ifdef CONFIG_SCHED_HMP + cpumask_set_cpu(i, &rq->freq_domain_cpumask); + rq->hmp_stats.cumulative_runnable_avg = 0; + rq->window_start = 0; + rq->hmp_stats.nr_big_tasks = 0; + rq->hmp_flags = 0; + rq->cur_irqload = 0; + rq->avg_irqload = 0; + rq->irqload_ts = 0; + rq->static_cpu_pwr_cost = 0; + rq->cc.cycles = 1; + rq->cc.time = 1; + rq->cstate = 0; + rq->wakeup_latency = 0; + rq->wakeup_energy = 0; + /* + * All cpus part of same cluster by default. This avoids the + * need to check for rq->cluster being non-NULL in hot-paths + * like select_best_cpu() + */ + rq->cluster = &init_cluster; + rq->curr_runnable_sum = rq->prev_runnable_sum = 0; + rq->nt_curr_runnable_sum = rq->nt_prev_runnable_sum = 0; + rq->old_busy_time = 0; + rq->old_estimated_time = 0; + rq->old_busy_time_group = 0; + rq->hmp_stats.pred_demands_sum = 0; +#endif INIT_LIST_HEAD(&rq->cfs_tasks); rq_attach_root(rq, &def_root_domain); @@ -7662,6 +7899,8 @@ void __init sched_init(void) atomic_set(&rq->nr_iowait, 0); } + set_hmp_defaults(); + set_load_weight(&init_task); /* @@ -7768,6 +8007,9 @@ void ___might_sleep(const char *file, int line, int preempt_offset) print_ip_sym(preempt_disable_ip); pr_cont("\n"); } +#ifdef CONFIG_PANIC_ON_SCHED_BUG + BUG(); +#endif dump_stack(); add_taint(TAINT_WARN, LOCKDEP_STILL_OK); } @@ -8369,7 +8611,7 @@ int sched_rr_handler(struct ctl_table *table, int write, #ifdef CONFIG_CGROUP_SCHED -static inline struct task_group *css_tg(struct cgroup_subsys_state *css) +inline struct task_group *css_tg(struct cgroup_subsys_state *css) { return css ? container_of(css, struct task_group, css) : NULL; } @@ -8756,6 +8998,13 @@ static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, #endif /* CONFIG_RT_GROUP_SCHED */ static struct cftype cpu_files[] = { +#ifdef CONFIG_SCHED_HMP + { + .name = "upmigrate_discourage", + .read_u64 = cpu_upmigrate_discourage_read_u64, + .write_u64 = cpu_upmigrate_discourage_write_u64, + }, +#endif #ifdef CONFIG_FAIR_GROUP_SCHED { .name = "shares", diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 5ebee3164e64..6ce492fbedcf 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -46,12 +46,15 @@ void irqtime_account_irq(struct task_struct *curr) struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime); s64 delta; int cpu; + u64 wallclock; + bool account = true; if (!sched_clock_irqtime) return; cpu = smp_processor_id(); - delta = sched_clock_cpu(cpu) - irqtime->irq_start_time; + wallclock = sched_clock_cpu(cpu); + delta = wallclock - irqtime->irq_start_time; irqtime->irq_start_time += delta; u64_stats_update_begin(&irqtime->sync); @@ -65,6 +68,13 @@ void irqtime_account_irq(struct task_struct *curr) irqtime->hardirq_time += delta; else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) irqtime->softirq_time += delta; + else + account = false; + + if (account) + sched_account_irqtime(cpu, curr, delta, wallclock); + else if (curr != this_cpu_ksoftirqd()) + sched_account_irqstart(cpu, curr, wallclock); u64_stats_update_end(&irqtime->sync); } diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 37e2449186c4..ffca4787dac2 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -820,6 +820,41 @@ static inline void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {} #endif /* CONFIG_SMP */ +#ifdef CONFIG_SCHED_HMP + +static void +inc_hmp_sched_stats_dl(struct rq *rq, struct task_struct *p) +{ + inc_cumulative_runnable_avg(&rq->hmp_stats, p); +} + +static void +dec_hmp_sched_stats_dl(struct rq *rq, struct task_struct *p) +{ + dec_cumulative_runnable_avg(&rq->hmp_stats, p); +} + +static void +fixup_hmp_sched_stats_dl(struct rq *rq, struct task_struct *p, + u32 new_task_load, u32 new_pred_demand) +{ + s64 task_load_delta = (s64)new_task_load - task_load(p); + s64 pred_demand_delta = PRED_DEMAND_DELTA; + + fixup_cumulative_runnable_avg(&rq->hmp_stats, p, task_load_delta, + pred_demand_delta); +} + +#else /* CONFIG_SCHED_HMP */ + +static inline void +inc_hmp_sched_stats_dl(struct rq *rq, struct task_struct *p) { } + +static inline void +dec_hmp_sched_stats_dl(struct rq *rq, struct task_struct *p) { } + +#endif /* CONFIG_SCHED_HMP */ + static inline void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) { @@ -829,6 +864,7 @@ void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) WARN_ON(!dl_prio(prio)); dl_rq->dl_nr_running++; add_nr_running(rq_of_dl_rq(dl_rq), 1); + inc_hmp_sched_stats_dl(rq_of_dl_rq(dl_rq), dl_task_of(dl_se)); inc_dl_deadline(dl_rq, deadline); inc_dl_migration(dl_se, dl_rq); @@ -843,6 +879,7 @@ void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) WARN_ON(!dl_rq->dl_nr_running); dl_rq->dl_nr_running--; sub_nr_running(rq_of_dl_rq(dl_rq), 1); + dec_hmp_sched_stats_dl(rq_of_dl_rq(dl_rq), dl_task_of(dl_se)); dec_dl_deadline(dl_rq, dl_se->deadline); dec_dl_migration(dl_se, dl_rq); @@ -1500,9 +1537,11 @@ retry: goto retry; } + next_task->on_rq = TASK_ON_RQ_MIGRATING; deactivate_task(rq, next_task, 0); set_task_cpu(next_task, later_rq->cpu); activate_task(later_rq, next_task, 0); + next_task->on_rq = TASK_ON_RQ_QUEUED; ret = 1; resched_curr(later_rq); @@ -1588,9 +1627,11 @@ static void pull_dl_task(struct rq *this_rq) resched = true; + p->on_rq = TASK_ON_RQ_MIGRATING; deactivate_task(src_rq, p, 0); set_task_cpu(p, this_cpu); activate_task(this_rq, p, 0); + p->on_rq = TASK_ON_RQ_QUEUED; dmin = p->dl.deadline; /* Is there any other task even earlier? */ @@ -1803,6 +1844,9 @@ const struct sched_class dl_sched_class = { .switched_to = switched_to_dl, .update_curr = update_curr_dl, +#ifdef CONFIG_SCHED_HMP + .fixup_hmp_sched_stats = fixup_hmp_sched_stats_dl, +#endif }; #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index fa178b62ea79..5671f2606702 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -534,6 +534,14 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->throttled); SEQ_printf(m, " .%-30s: %d\n", "throttle_count", cfs_rq->throttle_count); + SEQ_printf(m, " .%-30s: %d\n", "runtime_enabled", + cfs_rq->runtime_enabled); +#ifdef CONFIG_SCHED_HMP + SEQ_printf(m, " .%-30s: %d\n", "nr_big_tasks", + cfs_rq->hmp_stats.nr_big_tasks); + SEQ_printf(m, " .%-30s: %llu\n", "cumulative_runnable_avg", + cfs_rq->hmp_stats.cumulative_runnable_avg); +#endif #endif #ifdef CONFIG_FAIR_GROUP_SCHED @@ -622,6 +630,23 @@ do { \ P(cpu_load[2]); P(cpu_load[3]); P(cpu_load[4]); +#ifdef CONFIG_SMP + P(cpu_capacity); +#endif +#ifdef CONFIG_SCHED_HMP + P(static_cpu_pwr_cost); + P(cluster->static_cluster_pwr_cost); + P(cluster->load_scale_factor); + P(cluster->capacity); + P(cluster->max_possible_capacity); + P(cluster->efficiency); + P(cluster->cur_freq); + P(cluster->max_freq); + P(cluster->exec_scale_factor); + P(hmp_stats.nr_big_tasks); + SEQ_printf(m, " .%-30s: %llu\n", "hmp_stats.cumulative_runnable_avg", + rq->hmp_stats.cumulative_runnable_avg); +#endif #undef P #undef PN @@ -700,6 +725,14 @@ static void sched_debug_header(struct seq_file *m) PN(sysctl_sched_wakeup_granularity); P(sysctl_sched_child_runs_first); P(sysctl_sched_features); +#ifdef CONFIG_SCHED_HMP + P(sched_upmigrate); + P(sched_downmigrate); + P(sched_init_task_load_windows); + P(min_capacity); + P(max_capacity); + P(sched_ravg_window); +#endif #undef PN #undef P @@ -861,6 +894,9 @@ static void sched_show_numa(struct task_struct *p, struct seq_file *m) void proc_sched_show_task(struct task_struct *p, struct seq_file *m) { unsigned long nr_switches; + unsigned int load_avg; + + load_avg = pct_task_load(p); SEQ_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr(p), get_nr_threads(p)); @@ -918,7 +954,12 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) P_SCHEDSTAT(se.statistics.nr_wakeups_affine_attempts); P_SCHEDSTAT(se.statistics.nr_wakeups_passive); P_SCHEDSTAT(se.statistics.nr_wakeups_idle); - +#if defined(CONFIG_SMP) && defined(CONFIG_FAIR_GROUP_SCHED) + __P(load_avg); +#ifdef CONFIG_SCHED_HMP + P(ravg.demand); +#endif +#endif avg_atom = p->se.sum_exec_runtime; if (nr_switches) avg_atom = div64_ul(avg_atom, nr_switches); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index bdb03b0104b7..e84ec63a3974 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -31,9 +31,8 @@ #include #include -#include - #include "sched.h" +#include /* * Targeted preemption latency for CPU-bound tasks: @@ -80,6 +79,14 @@ static unsigned int sched_nr_latency = 8; */ unsigned int sysctl_sched_child_runs_first __read_mostly; +/* + * Controls whether, when SD_SHARE_PKG_RESOURCES is on, if all + * tasks go to idle CPUs when woken. If this is off, note that the + * per-task flag PF_WAKE_UP_IDLE can still cause a task to go to an + * idle CPU upon being woken. + */ +unsigned int __read_mostly sysctl_sched_wake_to_idle; + /* * SCHED_OTHER wake-up granularity. * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) @@ -242,6 +249,9 @@ static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight return mul_u64_u32_shr(delta_exec, fact, shift); } +#ifdef CONFIG_SMP +static int active_load_balance_cpu_stop(void *data); +#endif const struct sched_class fair_sched_class; @@ -2671,6 +2681,25 @@ static inline void update_cfs_shares(struct cfs_rq *cfs_rq) #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_SMP +u32 sched_get_wake_up_idle(struct task_struct *p) +{ + u32 enabled = p->flags & PF_WAKE_UP_IDLE; + + return !!enabled; +} + +int sched_set_wake_up_idle(struct task_struct *p, int wake_up_idle) +{ + int enable = !!wake_up_idle; + + if (enable) + p->flags |= PF_WAKE_UP_IDLE; + else + p->flags &= ~PF_WAKE_UP_IDLE; + + return 0; +} + /* Precomputed fixed inverse multiplies for multiplication by y^n */ static const u32 runnable_avg_yN_inv[] = { 0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6, @@ -2756,6 +2785,920 @@ static u32 __compute_runnable_contrib(u64 n) return contrib + runnable_avg_yN_sum[n]; } +#ifdef CONFIG_SCHED_HMP + +/* CPU selection flag */ +#define SBC_FLAG_PREV_CPU 0x1 +#define SBC_FLAG_BEST_CAP_CPU 0x2 +#define SBC_FLAG_CPU_COST 0x4 +#define SBC_FLAG_MIN_COST 0x8 +#define SBC_FLAG_IDLE_LEAST_LOADED 0x10 +#define SBC_FLAG_IDLE_CSTATE 0x20 +#define SBC_FLAG_COST_CSTATE_TIE_BREAKER 0x40 +#define SBC_FLAG_COST_CSTATE_PREV_CPU_TIE_BREAKER 0x80 +#define SBC_FLAG_CSTATE_LOAD 0x100 +#define SBC_FLAG_BEST_SIBLING 0x200 + +/* Cluster selection flag */ +#define SBC_FLAG_COLOC_CLUSTER 0x10000 +#define SBC_FLAG_WAKER_CLUSTER 0x20000 +#define SBC_FLAG_BACKUP_CLUSTER 0x40000 + +struct cpu_select_env { + struct task_struct *p; + struct related_thread_group *rtg; + u8 reason; + u8 need_idle:1; + u8 need_waker_cluster:1; + u8 sync:1; + u8 ignore_prev_cpu:1; + enum sched_boost_type boost_type; + int prev_cpu; + DECLARE_BITMAP(candidate_list, NR_CPUS); + DECLARE_BITMAP(backup_list, NR_CPUS); + u64 task_load; + u64 cpu_load; + u32 sbc_best_flag; + u32 sbc_best_cluster_flag; +}; + +struct cluster_cpu_stats { + int best_idle_cpu, least_loaded_cpu; + int best_capacity_cpu, best_cpu, best_sibling_cpu; + int min_cost, best_sibling_cpu_cost; + int best_cpu_cstate; + u64 min_load, best_load, best_sibling_cpu_load; + s64 highest_spare_capacity; +}; + +static int spill_threshold_crossed(struct cpu_select_env *env, struct rq *rq) +{ + u64 total_load; + + total_load = env->task_load + env->cpu_load; + + if (total_load > sched_spill_load || + (rq->nr_running + 1) > sysctl_sched_spill_nr_run) + return 1; + + return 0; +} + +static int skip_cpu(int cpu, struct cpu_select_env *env) +{ + int tcpu = task_cpu(env->p); + int skip = 0; + + if (!env->reason) + return 0; + + if (is_reserved(cpu)) + return 1; + + switch (env->reason) { + case UP_MIGRATION: + skip = !idle_cpu(cpu); + break; + case IRQLOAD_MIGRATION: + /* Purposely fall through */ + default: + skip = (cpu == tcpu); + break; + } + + return skip; +} + +static inline int +acceptable_capacity(struct sched_cluster *cluster, struct cpu_select_env *env) +{ + int tcpu; + + if (!env->reason) + return 1; + + tcpu = task_cpu(env->p); + switch (env->reason) { + case UP_MIGRATION: + return cluster->capacity > cpu_capacity(tcpu); + + case DOWN_MIGRATION: + return cluster->capacity < cpu_capacity(tcpu); + + default: + break; + } + + return 1; +} + +static int +skip_cluster(struct sched_cluster *cluster, struct cpu_select_env *env) +{ + if (!test_bit(cluster->id, env->candidate_list)) + return 1; + + if (!acceptable_capacity(cluster, env)) { + __clear_bit(cluster->id, env->candidate_list); + return 1; + } + + return 0; +} + +static struct sched_cluster * +select_least_power_cluster(struct cpu_select_env *env) +{ + struct sched_cluster *cluster; + + if (env->rtg) { + env->task_load = scale_load_to_cpu(task_load(env->p), + cluster_first_cpu(env->rtg->preferred_cluster)); + env->sbc_best_cluster_flag |= SBC_FLAG_COLOC_CLUSTER; + return env->rtg->preferred_cluster; + } + + for_each_sched_cluster(cluster) { + if (!skip_cluster(cluster, env)) { + int cpu = cluster_first_cpu(cluster); + + env->task_load = scale_load_to_cpu(task_load(env->p), + cpu); + if (task_load_will_fit(env->p, env->task_load, cpu, + env->boost_type)) + return cluster; + + __set_bit(cluster->id, env->backup_list); + __clear_bit(cluster->id, env->candidate_list); + } + } + + return NULL; +} + +static struct sched_cluster * +next_candidate(const unsigned long *list, int start, int end) +{ + int cluster_id; + + cluster_id = find_next_bit(list, end, start - 1 + 1); + if (cluster_id >= end) + return NULL; + + return sched_cluster[cluster_id]; +} + +static void +update_spare_capacity(struct cluster_cpu_stats *stats, + struct cpu_select_env *env, int cpu, int capacity, + u64 cpu_load) +{ + s64 spare_capacity = sched_ravg_window - cpu_load; + + if (spare_capacity > 0 && + (spare_capacity > stats->highest_spare_capacity || + (spare_capacity == stats->highest_spare_capacity && + ((!env->need_waker_cluster && + capacity > cpu_capacity(stats->best_capacity_cpu)) || + (env->need_waker_cluster && + cpu_rq(cpu)->nr_running < + cpu_rq(stats->best_capacity_cpu)->nr_running))))) { + /* + * If sync waker is the only runnable of CPU, cr_avg of the + * CPU is 0 so we have high chance to place the wakee on the + * waker's CPU which likely causes preemtion of the waker. + * This can lead migration of preempted waker. Place the + * wakee on the real idle CPU when it's possible by checking + * nr_running to avoid such preemption. + */ + stats->highest_spare_capacity = spare_capacity; + stats->best_capacity_cpu = cpu; + } +} + +static inline void find_backup_cluster( +struct cpu_select_env *env, struct cluster_cpu_stats *stats) +{ + struct sched_cluster *next = NULL; + int i; + + while (!bitmap_empty(env->backup_list, num_clusters)) { + next = next_candidate(env->backup_list, 0, num_clusters); + __clear_bit(next->id, env->backup_list); + for_each_cpu_and(i, &env->p->cpus_allowed, &next->cpus) { + trace_sched_cpu_load_wakeup(cpu_rq(i), idle_cpu(i), + sched_irqload(i), power_cost(i, task_load(env->p) + + cpu_cravg_sync(i, env->sync)), 0); + + update_spare_capacity(stats, env, i, next->capacity, + cpu_load_sync(i, env->sync)); + } + env->sbc_best_cluster_flag = SBC_FLAG_BACKUP_CLUSTER; + } +} + +struct sched_cluster * +next_best_cluster(struct sched_cluster *cluster, struct cpu_select_env *env, + struct cluster_cpu_stats *stats) +{ + struct sched_cluster *next = NULL; + + __clear_bit(cluster->id, env->candidate_list); + + if (env->rtg && preferred_cluster(cluster, env->p)) + return NULL; + + do { + if (bitmap_empty(env->candidate_list, num_clusters)) + return NULL; + + next = next_candidate(env->candidate_list, 0, num_clusters); + if (next) { + if (next->min_power_cost > stats->min_cost) { + clear_bit(next->id, env->candidate_list); + next = NULL; + continue; + } + + if (skip_cluster(next, env)) + next = NULL; + } + } while (!next); + + env->task_load = scale_load_to_cpu(task_load(env->p), + cluster_first_cpu(next)); + return next; +} + +#ifdef CONFIG_SCHED_HMP_CSTATE_AWARE +static void __update_cluster_stats(int cpu, struct cluster_cpu_stats *stats, + struct cpu_select_env *env, int cpu_cost) +{ + int cpu_cstate; + int prev_cpu = env->prev_cpu; + + cpu_cstate = cpu_rq(cpu)->cstate; + + if (env->need_idle) { + stats->min_cost = cpu_cost; + if (idle_cpu(cpu)) { + if (cpu_cstate < stats->best_cpu_cstate || + (cpu_cstate == stats->best_cpu_cstate && + cpu == prev_cpu)) { + stats->best_idle_cpu = cpu; + stats->best_cpu_cstate = cpu_cstate; + } + } else { + if (env->cpu_load < stats->min_load || + (env->cpu_load == stats->min_load && + cpu == prev_cpu)) { + stats->least_loaded_cpu = cpu; + stats->min_load = env->cpu_load; + } + } + + return; + } + + if (cpu_cost < stats->min_cost) { + stats->min_cost = cpu_cost; + stats->best_cpu_cstate = cpu_cstate; + stats->best_load = env->cpu_load; + stats->best_cpu = cpu; + env->sbc_best_flag = SBC_FLAG_CPU_COST; + return; + } + + /* CPU cost is the same. Start breaking the tie by C-state */ + + if (cpu_cstate > stats->best_cpu_cstate) + return; + + if (cpu_cstate < stats->best_cpu_cstate) { + stats->best_cpu_cstate = cpu_cstate; + stats->best_load = env->cpu_load; + stats->best_cpu = cpu; + env->sbc_best_flag = SBC_FLAG_COST_CSTATE_TIE_BREAKER; + return; + } + + /* C-state is the same. Use prev CPU to break the tie */ + if (cpu == prev_cpu) { + stats->best_cpu = cpu; + env->sbc_best_flag = SBC_FLAG_COST_CSTATE_PREV_CPU_TIE_BREAKER; + return; + } + + if (stats->best_cpu != prev_cpu && + ((cpu_cstate == 0 && env->cpu_load < stats->best_load) || + (cpu_cstate > 0 && env->cpu_load > stats->best_load))) { + stats->best_load = env->cpu_load; + stats->best_cpu = cpu; + env->sbc_best_flag = SBC_FLAG_CSTATE_LOAD; + } +} +#else /* CONFIG_SCHED_HMP_CSTATE_AWARE */ +static void __update_cluster_stats(int cpu, struct cluster_cpu_stats *stats, + struct cpu_select_env *env, int cpu_cost) +{ + int prev_cpu = env->prev_cpu; + + if (cpu != prev_cpu && cpus_share_cache(prev_cpu, cpu)) { + if (stats->best_sibling_cpu_cost > cpu_cost || + (stats->best_sibling_cpu_cost == cpu_cost && + stats->best_sibling_cpu_load > env->cpu_load)) { + stats->best_sibling_cpu_cost = cpu_cost; + stats->best_sibling_cpu_load = env->cpu_load; + stats->best_sibling_cpu = cpu; + } + } + + if ((cpu_cost < stats->min_cost) || + ((stats->best_cpu != prev_cpu && + stats->min_load > env->cpu_load) || cpu == prev_cpu)) { + if (env->need_idle) { + if (idle_cpu(cpu)) { + stats->min_cost = cpu_cost; + stats->best_idle_cpu = cpu; + } + } else { + stats->min_cost = cpu_cost; + stats->min_load = env->cpu_load; + stats->best_cpu = cpu; + env->sbc_best_flag = SBC_FLAG_MIN_COST; + } + } +} +#endif /* CONFIG_SCHED_HMP_CSTATE_AWARE */ + +static void update_cluster_stats(int cpu, struct cluster_cpu_stats *stats, + struct cpu_select_env *env) +{ + int cpu_cost; + + cpu_cost = power_cost(cpu, task_load(env->p) + + cpu_cravg_sync(cpu, env->sync)); + if (cpu_cost <= stats->min_cost) + __update_cluster_stats(cpu, stats, env, cpu_cost); +} + +static void find_best_cpu_in_cluster(struct sched_cluster *c, + struct cpu_select_env *env, struct cluster_cpu_stats *stats) +{ + int i; + struct cpumask search_cpus; + + cpumask_and(&search_cpus, tsk_cpus_allowed(env->p), &c->cpus); + if (env->ignore_prev_cpu) + cpumask_clear_cpu(env->prev_cpu, &search_cpus); + + for_each_cpu(i, &search_cpus) { + env->cpu_load = cpu_load_sync(i, env->sync); + + trace_sched_cpu_load_wakeup(cpu_rq(i), idle_cpu(i), + sched_irqload(i), + power_cost(i, task_load(env->p) + + cpu_cravg_sync(i, env->sync)), 0); + + if (unlikely(!cpu_active(i)) || skip_cpu(i, env)) + continue; + + update_spare_capacity(stats, env, i, c->capacity, + env->cpu_load); + + if (env->boost_type == SCHED_BOOST_ON_ALL || + env->need_waker_cluster || + sched_cpu_high_irqload(i) || + spill_threshold_crossed(env, cpu_rq(i))) + continue; + + update_cluster_stats(i, stats, env); + } +} + +static inline void init_cluster_cpu_stats(struct cluster_cpu_stats *stats) +{ + stats->best_cpu = stats->best_idle_cpu = -1; + stats->best_capacity_cpu = stats->best_sibling_cpu = -1; + stats->min_cost = stats->best_sibling_cpu_cost = INT_MAX; + stats->min_load = stats->best_sibling_cpu_load = ULLONG_MAX; + stats->highest_spare_capacity = 0; + stats->least_loaded_cpu = -1; + stats->best_cpu_cstate = INT_MAX; + /* No need to initialize stats->best_load */ +} + +/* + * Should task be woken to any available idle cpu? + * + * Waking tasks to idle cpu has mixed implications on both performance and + * power. In many cases, scheduler can't estimate correctly impact of using idle + * cpus on either performance or power. PF_WAKE_UP_IDLE allows external kernel + * module to pass a strong hint to scheduler that the task in question should be + * woken to idle cpu, generally to improve performance. + */ +static inline int wake_to_idle(struct task_struct *p) +{ + return (current->flags & PF_WAKE_UP_IDLE) || + (p->flags & PF_WAKE_UP_IDLE) || sysctl_sched_wake_to_idle; +} + +static inline bool +bias_to_prev_cpu(struct cpu_select_env *env, struct cluster_cpu_stats *stats) +{ + int prev_cpu; + struct task_struct *task = env->p; + struct sched_cluster *cluster; + + if (env->boost_type != SCHED_BOOST_NONE || env->reason || + !task->ravg.mark_start || + env->need_idle || !sched_short_sleep_task_threshold) + return false; + + prev_cpu = env->prev_cpu; + if (!cpumask_test_cpu(prev_cpu, tsk_cpus_allowed(task)) || + unlikely(!cpu_active(prev_cpu))) + return false; + + if (task->ravg.mark_start - task->last_cpu_selected_ts >= + sched_long_cpu_selection_threshold) + return false; + + /* + * This function should be used by task wake up path only as it's + * assuming p->last_switch_out_ts as last sleep time. + * p->last_switch_out_ts can denote last preemption time as well as + * last sleep time. + */ + if (task->ravg.mark_start - task->last_switch_out_ts >= + sched_short_sleep_task_threshold) + return false; + + env->task_load = scale_load_to_cpu(task_load(task), prev_cpu); + cluster = cpu_rq(prev_cpu)->cluster; + + if (!task_load_will_fit(task, env->task_load, prev_cpu, + sched_boost_type())) { + + __set_bit(cluster->id, env->backup_list); + __clear_bit(cluster->id, env->candidate_list); + return false; + } + + env->cpu_load = cpu_load_sync(prev_cpu, env->sync); + if (sched_cpu_high_irqload(prev_cpu) || + spill_threshold_crossed(env, cpu_rq(prev_cpu))) { + update_spare_capacity(stats, env, prev_cpu, + cluster->capacity, env->cpu_load); + env->ignore_prev_cpu = 1; + return false; + } + + return true; +} + +static inline bool +wake_to_waker_cluster(struct cpu_select_env *env) +{ + return !env->need_idle && !env->reason && env->sync && + task_load(current) > sched_big_waker_task_load && + task_load(env->p) < sched_small_wakee_task_load; +} + +static inline int +cluster_allowed(struct task_struct *p, struct sched_cluster *cluster) +{ + cpumask_t tmp_mask; + + cpumask_and(&tmp_mask, &cluster->cpus, cpu_active_mask); + cpumask_and(&tmp_mask, &tmp_mask, &p->cpus_allowed); + + return !cpumask_empty(&tmp_mask); +} + + +/* return cheapest cpu that can fit this task */ +static int select_best_cpu(struct task_struct *p, int target, int reason, + int sync) +{ + struct sched_cluster *cluster, *pref_cluster = NULL; + struct cluster_cpu_stats stats; + struct related_thread_group *grp; + unsigned int sbc_flag = 0; + + struct cpu_select_env env = { + .p = p, + .reason = reason, + .need_idle = wake_to_idle(p), + .need_waker_cluster = 0, + .boost_type = sched_boost_type(), + .sync = sync, + .prev_cpu = target, + .ignore_prev_cpu = 0, + .rtg = NULL, + .sbc_best_flag = 0, + .sbc_best_cluster_flag = 0, + }; + + bitmap_copy(env.candidate_list, all_cluster_ids, NR_CPUS); + bitmap_zero(env.backup_list, NR_CPUS); + + init_cluster_cpu_stats(&stats); + + rcu_read_lock(); + + grp = task_related_thread_group(p); + + if (grp && grp->preferred_cluster) { + pref_cluster = grp->preferred_cluster; + if (!cluster_allowed(p, pref_cluster)) + clear_bit(pref_cluster->id, env.candidate_list); + else + env.rtg = grp; + } else { + cluster = cpu_rq(smp_processor_id())->cluster; + if (wake_to_waker_cluster(&env) && + cluster_allowed(p, cluster)) { + env.need_waker_cluster = 1; + bitmap_zero(env.candidate_list, NR_CPUS); + __set_bit(cluster->id, env.candidate_list); + env.sbc_best_cluster_flag = SBC_FLAG_WAKER_CLUSTER; + + } else if (bias_to_prev_cpu(&env, &stats)) { + sbc_flag = SBC_FLAG_PREV_CPU; + goto out; + } + } + +retry: + cluster = select_least_power_cluster(&env); + + if (!cluster) + goto out; + + /* + * 'cluster' now points to the minimum power cluster which can satisfy + * task's perf goals. Walk down the cluster list starting with that + * cluster. For non-small tasks, skip clusters that don't have + * mostly_idle/idle cpus + */ + + do { + find_best_cpu_in_cluster(cluster, &env, &stats); + + } while ((cluster = next_best_cluster(cluster, &env, &stats))); + + if (env.need_idle) { + if (stats.best_idle_cpu >= 0) { + target = stats.best_idle_cpu; + sbc_flag |= SBC_FLAG_IDLE_CSTATE; + } else if (stats.least_loaded_cpu >= 0) { + target = stats.least_loaded_cpu; + sbc_flag |= SBC_FLAG_IDLE_LEAST_LOADED; + } + } else if (stats.best_cpu >= 0) { + if (stats.best_cpu != task_cpu(p) && + stats.min_cost == stats.best_sibling_cpu_cost) { + stats.best_cpu = stats.best_sibling_cpu; + sbc_flag |= SBC_FLAG_BEST_SIBLING; + } + sbc_flag |= env.sbc_best_flag; + target = stats.best_cpu; + } else { + if (env.rtg) { + env.rtg = NULL; + goto retry; + } + + find_backup_cluster(&env, &stats); + if (stats.best_capacity_cpu >= 0) { + target = stats.best_capacity_cpu; + sbc_flag |= SBC_FLAG_BEST_CAP_CPU; + } + } + p->last_cpu_selected_ts = sched_ktime_clock(); + sbc_flag |= env.sbc_best_cluster_flag; +out: + rcu_read_unlock(); + trace_sched_task_load(p, sched_boost(), env.reason, env.sync, + env.need_idle, sbc_flag, target); + return target; +} + +#ifdef CONFIG_CFS_BANDWIDTH + +static inline struct task_group *next_task_group(struct task_group *tg) +{ + tg = list_entry_rcu(tg->list.next, typeof(struct task_group), list); + + return (&tg->list == &task_groups) ? NULL : tg; +} + +/* Iterate over all cfs_rq in a cpu */ +#define for_each_cfs_rq(cfs_rq, tg, cpu) \ + for (tg = container_of(&task_groups, struct task_group, list); \ + ((tg = next_task_group(tg)) && (cfs_rq = tg->cfs_rq[cpu]));) + +void reset_cfs_rq_hmp_stats(int cpu, int reset_cra) +{ + struct task_group *tg; + struct cfs_rq *cfs_rq; + + rcu_read_lock(); + + for_each_cfs_rq(cfs_rq, tg, cpu) + reset_hmp_stats(&cfs_rq->hmp_stats, reset_cra); + + rcu_read_unlock(); +} + +static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); + +static void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, + struct task_struct *p, int change_cra); +static void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, + struct task_struct *p, int change_cra); + +/* Add task's contribution to a cpu' HMP statistics */ +void inc_hmp_sched_stats_fair(struct rq *rq, + struct task_struct *p, int change_cra) +{ + struct cfs_rq *cfs_rq; + struct sched_entity *se = &p->se; + + /* + * Although below check is not strictly required (as + * inc/dec_nr_big_task and inc/dec_cumulative_runnable_avg called + * from inc_cfs_rq_hmp_stats() have similar checks), we gain a bit on + * efficiency by short-circuiting for_each_sched_entity() loop when + * sched_disable_window_stats + */ + if (sched_disable_window_stats) + return; + + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + inc_cfs_rq_hmp_stats(cfs_rq, p, change_cra); + if (cfs_rq_throttled(cfs_rq)) + break; + } + + /* Update rq->hmp_stats only if we didn't find any throttled cfs_rq */ + if (!se) + inc_rq_hmp_stats(rq, p, change_cra); +} + +static void fixup_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, + u32 new_task_load, u32 new_pred_demand) +{ + struct cfs_rq *cfs_rq; + struct sched_entity *se = &p->se; + s64 task_load_delta = (s64)new_task_load - task_load(p); + s64 pred_demand_delta = PRED_DEMAND_DELTA; + + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + + fixup_cumulative_runnable_avg(&cfs_rq->hmp_stats, p, + task_load_delta, + pred_demand_delta); + fixup_nr_big_tasks(&cfs_rq->hmp_stats, p, task_load_delta); + if (cfs_rq_throttled(cfs_rq)) + break; + } + + /* Fix up rq->hmp_stats only if we didn't find any throttled cfs_rq */ + if (!se) { + fixup_cumulative_runnable_avg(&rq->hmp_stats, p, + task_load_delta, + pred_demand_delta); + fixup_nr_big_tasks(&rq->hmp_stats, p, task_load_delta); + } +} + +static int task_will_be_throttled(struct task_struct *p); + +#else /* CONFIG_CFS_BANDWIDTH */ + +inline void reset_cfs_rq_hmp_stats(int cpu, int reset_cra) { } + +static void +fixup_hmp_sched_stats_fair(struct rq *rq, struct task_struct *p, + u32 new_task_load, u32 new_pred_demand) +{ + s64 task_load_delta = (s64)new_task_load - task_load(p); + s64 pred_demand_delta = PRED_DEMAND_DELTA; + + fixup_cumulative_runnable_avg(&rq->hmp_stats, p, task_load_delta, + pred_demand_delta); + fixup_nr_big_tasks(&rq->hmp_stats, p, task_load_delta); +} + +static inline int task_will_be_throttled(struct task_struct *p) +{ + return 0; +} + +void inc_hmp_sched_stats_fair(struct rq *rq, + struct task_struct *p, int change_cra) +{ + inc_nr_big_task(&rq->hmp_stats, p); +} + +#endif /* CONFIG_CFS_BANDWIDTH */ + +/* + * Reset balance_interval at all sched_domain levels of given cpu, so that it + * honors kick. + */ +static inline void reset_balance_interval(int cpu) +{ + struct sched_domain *sd; + + if (cpu >= nr_cpu_ids) + return; + + rcu_read_lock(); + for_each_domain(cpu, sd) + sd->balance_interval = 0; + rcu_read_unlock(); +} + +/* + * Check if a task is on the "wrong" cpu (i.e its current cpu is not the ideal + * cpu as per its demand or priority) + * + * Returns reason why task needs to be migrated + */ +static inline int migration_needed(struct task_struct *p, int cpu) +{ + int nice; + struct related_thread_group *grp; + + if (p->state != TASK_RUNNING || p->nr_cpus_allowed == 1) + return 0; + + /* No need to migrate task that is about to be throttled */ + if (task_will_be_throttled(p)) + return 0; + + if (sched_boost_type() == SCHED_BOOST_ON_BIG) { + if (cpu_capacity(cpu) != max_capacity) + return UP_MIGRATION; + return 0; + } + + if (sched_cpu_high_irqload(cpu)) + return IRQLOAD_MIGRATION; + + nice = task_nice(p); + rcu_read_lock(); + grp = task_related_thread_group(p); + if (!grp && (nice > SCHED_UPMIGRATE_MIN_NICE || + upmigrate_discouraged(p)) && cpu_capacity(cpu) > min_capacity) { + rcu_read_unlock(); + return DOWN_MIGRATION; + } + + if (!grp && !task_will_fit(p, cpu)) { + rcu_read_unlock(); + return UP_MIGRATION; + } + rcu_read_unlock(); + + return 0; +} + +static inline int +kick_active_balance(struct rq *rq, struct task_struct *p, int new_cpu) +{ + unsigned long flags; + int rc = 0; + + /* Invoke active balance to force migrate currently running task */ + raw_spin_lock_irqsave(&rq->lock, flags); + if (!rq->active_balance) { + rq->active_balance = 1; + rq->push_cpu = new_cpu; + get_task_struct(p); + rq->push_task = p; + rc = 1; + } + raw_spin_unlock_irqrestore(&rq->lock, flags); + + return rc; +} + +static DEFINE_RAW_SPINLOCK(migration_lock); + +/* + * Check if currently running task should be migrated to a better cpu. + * + * Todo: Effect this via changes to nohz_balancer_kick() and load balance? + */ +void check_for_migration(struct rq *rq, struct task_struct *p) +{ + int cpu = cpu_of(rq), new_cpu; + int active_balance = 0, reason; + + reason = migration_needed(p, cpu); + if (!reason) + return; + + raw_spin_lock(&migration_lock); + new_cpu = select_best_cpu(p, cpu, reason, 0); + + if (new_cpu != cpu) { + active_balance = kick_active_balance(rq, p, new_cpu); + if (active_balance) + mark_reserved(new_cpu); + } + + raw_spin_unlock(&migration_lock); + + if (active_balance) + stop_one_cpu_nowait(cpu, active_load_balance_cpu_stop, rq, + &rq->active_balance_work); +} + +#ifdef CONFIG_CFS_BANDWIDTH + +static void init_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq) +{ + cfs_rq->hmp_stats.nr_big_tasks = 0; + cfs_rq->hmp_stats.cumulative_runnable_avg = 0; + cfs_rq->hmp_stats.pred_demands_sum = 0; +} + +static void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, + struct task_struct *p, int change_cra) +{ + inc_nr_big_task(&cfs_rq->hmp_stats, p); + if (change_cra) + inc_cumulative_runnable_avg(&cfs_rq->hmp_stats, p); +} + +static void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, + struct task_struct *p, int change_cra) +{ + dec_nr_big_task(&cfs_rq->hmp_stats, p); + if (change_cra) + dec_cumulative_runnable_avg(&cfs_rq->hmp_stats, p); +} + +static void inc_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, + struct cfs_rq *cfs_rq) +{ + stats->nr_big_tasks += cfs_rq->hmp_stats.nr_big_tasks; + stats->cumulative_runnable_avg += + cfs_rq->hmp_stats.cumulative_runnable_avg; + stats->pred_demands_sum += cfs_rq->hmp_stats.pred_demands_sum; +} + +static void dec_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, + struct cfs_rq *cfs_rq) +{ + stats->nr_big_tasks -= cfs_rq->hmp_stats.nr_big_tasks; + stats->cumulative_runnable_avg -= + cfs_rq->hmp_stats.cumulative_runnable_avg; + stats->pred_demands_sum -= cfs_rq->hmp_stats.pred_demands_sum; + + BUG_ON(stats->nr_big_tasks < 0 || + (s64)stats->cumulative_runnable_avg < 0); + verify_pred_demands_sum(stats); +} + +#else /* CONFIG_CFS_BANDWIDTH */ + +static inline void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, + struct task_struct *p, int change_cra) { } + +static inline void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, + struct task_struct *p, int change_cra) { } + +#endif /* CONFIG_CFS_BANDWIDTH */ + +#else /* CONFIG_SCHED_HMP */ + +static inline void init_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq) { } + +static inline void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, + struct task_struct *p, int change_cra) { } + +static inline void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, + struct task_struct *p, int change_cra) { } + +static inline void inc_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, + struct cfs_rq *cfs_rq) +{ +} + +static inline void dec_throttled_cfs_rq_hmp_stats(struct hmp_sched_stats *stats, + struct cfs_rq *cfs_rq) +{ +} +#endif /* CONFIG_SCHED_HMP */ + #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT) /* @@ -3273,6 +4216,12 @@ static inline int idle_balance(struct rq *rq) return 0; } +static inline void inc_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, + struct task_struct *p, int change_cra) { } + +static inline void dec_cfs_rq_hmp_stats(struct cfs_rq *cfs_rq, + struct task_struct *p, int change_cra) { } + #endif /* CONFIG_SMP */ static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se) @@ -3884,6 +4833,33 @@ static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) return cfs_bandwidth_used() && cfs_rq->throttled; } +/* + * Check if task is part of a hierarchy where some cfs_rq does not have any + * runtime left. + * + * We can't rely on throttled_hierarchy() to do this test, as + * cfs_rq->throttle_count will not be updated yet when this function is called + * from scheduler_tick() + */ +static int task_will_be_throttled(struct task_struct *p) +{ + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq; + + if (!cfs_bandwidth_used()) + return 0; + + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + if (!cfs_rq->runtime_enabled) + continue; + if (cfs_rq->runtime_remaining <= 0) + return 1; + } + + return 0; +} + /* check whether cfs_rq, or any parent, is throttled */ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) { @@ -3961,13 +4937,20 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) if (dequeue) dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); qcfs_rq->h_nr_running -= task_delta; +#ifdef CONFIG_SCHED_HMP + dec_throttled_cfs_rq_hmp_stats(&qcfs_rq->hmp_stats, cfs_rq); +#endif if (qcfs_rq->load.weight) dequeue = 0; } - if (!se) + if (!se) { sub_nr_running(rq, task_delta); +#ifdef CONFIG_SCHED_HMP + dec_throttled_cfs_rq_hmp_stats(&rq->hmp_stats, cfs_rq); +#endif + } cfs_rq->throttled = 1; cfs_rq->throttled_clock = rq_clock(rq); @@ -3988,6 +4971,12 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) start_cfs_bandwidth(cfs_b); raw_spin_unlock(&cfs_b->lock); + + /* Log effect on hmp stats after throttling */ + trace_sched_cpu_load_cgroup(rq, idle_cpu(cpu_of(rq)), + sched_irqload(cpu_of(rq)), + power_cost(cpu_of(rq), 0), + cpu_temp(cpu_of(rq))); } void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) @@ -3997,6 +4986,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) struct sched_entity *se; int enqueue = 1; long task_delta; + struct cfs_rq *tcfs_rq = cfs_rq; se = cfs_rq->tg->se[cpu_of(rq)]; @@ -4024,17 +5014,30 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) if (enqueue) enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); cfs_rq->h_nr_running += task_delta; +#ifdef CONFIG_SCHED_HMP + inc_throttled_cfs_rq_hmp_stats(&cfs_rq->hmp_stats, tcfs_rq); +#endif if (cfs_rq_throttled(cfs_rq)) break; } - if (!se) + if (!se) { add_nr_running(rq, task_delta); +#ifdef CONFIG_SCHED_HMP + inc_throttled_cfs_rq_hmp_stats(&rq->hmp_stats, tcfs_rq); +#endif + } /* determine whether we need to wake up potentially idle cpu */ if (rq->curr == rq->idle && rq->cfs.nr_running) resched_curr(rq); + + /* Log effect on hmp stats after un-throttling */ + trace_sched_cpu_load_cgroup(rq, idle_cpu(cpu_of(rq)), + sched_irqload(cpu_of(rq)), + power_cost(cpu_of(rq), 0), + cpu_temp(cpu_of(rq))); } static u64 distribute_cfs_runtime(struct cfs_bandwidth *cfs_b, @@ -4372,6 +5375,7 @@ static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) { cfs_rq->runtime_enabled = 0; INIT_LIST_HEAD(&cfs_rq->throttled_list); + init_cfs_rq_hmp_stats(cfs_rq); } void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) @@ -4562,6 +5566,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (cfs_rq_throttled(cfs_rq)) break; cfs_rq->h_nr_running++; + inc_cfs_rq_hmp_stats(cfs_rq, p, 1); flags = ENQUEUE_WAKEUP; } @@ -4569,6 +5574,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running++; + inc_cfs_rq_hmp_stats(cfs_rq, p, 1); if (cfs_rq_throttled(cfs_rq)) break; @@ -4577,8 +5583,10 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) update_cfs_shares(cfs_rq); } - if (!se) + if (!se) { add_nr_running(rq, 1); + inc_rq_hmp_stats(rq, p, 1); + } hrtick_update(rq); } @@ -4609,6 +5617,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (cfs_rq_throttled(cfs_rq)) break; cfs_rq->h_nr_running--; + dec_cfs_rq_hmp_stats(cfs_rq, p, 1); /* Don't dequeue parent if it has other entities besides us */ if (cfs_rq->load.weight) { @@ -4628,6 +5637,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running--; + dec_cfs_rq_hmp_stats(cfs_rq, p, 1); if (cfs_rq_throttled(cfs_rq)) break; @@ -4636,8 +5646,10 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) update_cfs_shares(cfs_rq); } - if (!se) + if (!se) { sub_nr_running(rq, 1); + dec_rq_hmp_stats(rq, p, 1); + } hrtick_update(rq); } @@ -5632,6 +6644,10 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f int want_affine = 0; int sync = wake_flags & WF_SYNC; +#ifdef CONFIG_SCHED_HMP + return select_best_cpu(p, prev_cpu, 0, sync); +#endif + if (sd_flag & SD_BALANCE_WAKE) { record_wakee(p); want_affine = !wake_wide(p) && !wake_cap(p, cpu, prev_cpu) @@ -6247,6 +7263,13 @@ enum fbq_type { regular, remote, all }; #define LBF_NEED_BREAK 0x02 #define LBF_DST_PINNED 0x04 #define LBF_SOME_PINNED 0x08 +#define LBF_SCHED_BOOST_ACTIVE_BALANCE 0x40 +#define LBF_BIG_TASK_ACTIVE_BALANCE 0x80 +#define LBF_HMP_ACTIVE_BALANCE (LBF_SCHED_BOOST_ACTIVE_BALANCE | \ + LBF_BIG_TASK_ACTIVE_BALANCE) +#define LBF_IGNORE_BIG_TASKS 0x100 +#define LBF_IGNORE_PREFERRED_CLUSTER_TASKS 0x200 +#define LBF_MOVED_RELATED_THREAD_GROUP_TASK 0x400 struct lb_env { struct sched_domain *sd; @@ -6263,6 +7286,8 @@ struct lb_env { long imbalance; /* The set of CPUs under consideration for load-balancing */ struct cpumask *cpus; + unsigned int busiest_grp_capacity; + unsigned int busiest_nr_running; unsigned int flags; @@ -6369,6 +7394,7 @@ static int can_migrate_task(struct task_struct *p, struct lb_env *env) { int tsk_cache_hot; + int twf, group_cpus; lockdep_assert_held(&env->src_rq->lock); @@ -6415,6 +7441,34 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) /* Record that we found atleast one task that could run on dst_cpu */ env->flags &= ~LBF_ALL_PINNED; + if (cpu_capacity(env->dst_cpu) > cpu_capacity(env->src_cpu) && + nr_big_tasks(env->src_rq) && !is_big_task(p)) + return 0; + + twf = task_will_fit(p, env->dst_cpu); + + /* + * Attempt to not pull tasks that don't fit. We may get lucky and find + * one that actually fits. + */ + if (env->flags & LBF_IGNORE_BIG_TASKS && !twf) + return 0; + + if (env->flags & LBF_IGNORE_PREFERRED_CLUSTER_TASKS && + !preferred_cluster(rq_cluster(cpu_rq(env->dst_cpu)), p)) + return 0; + + /* + * Group imbalance can sometimes cause work to be pulled across groups + * even though the group could have managed the imbalance on its own. + * Prevent inter-cluster migrations for big tasks when the number of + * tasks is lower than the capacity of the group. + */ + group_cpus = DIV_ROUND_UP(env->busiest_grp_capacity, + SCHED_CAPACITY_SCALE); + if (!twf && env->busiest_nr_running <= group_cpus) + return 0; + if (task_running(env->src_rq, p)) { schedstat_inc(p->se.statistics.nr_failed_migrations_running); return 0; @@ -6422,15 +7476,16 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) /* * Aggressive migration if: - * 1) destination numa is preferred - * 2) task is cache cold, or - * 3) too many balance attempts have failed. + * 1) IDLE or NEWLY_IDLE balance. + * 2) destination numa is preferred + * 3) task is cache cold, or + * 4) too many balance attempts have failed. */ tsk_cache_hot = migrate_degrades_locality(p, env); if (tsk_cache_hot == -1) tsk_cache_hot = task_hot(p, env); - if (tsk_cache_hot <= 0 || + if (env->idle != CPU_NOT_IDLE || tsk_cache_hot <= 0 || env->sd->nr_balance_failed > env->sd->cache_nice_tries) { if (tsk_cache_hot == 1) { schedstat_inc(env->sd->lb_hot_gained[env->idle]); @@ -6452,7 +7507,11 @@ static void detach_task(struct task_struct *p, struct lb_env *env) p->on_rq = TASK_ON_RQ_MIGRATING; deactivate_task(env->src_rq, p, 0); + double_lock_balance(env->src_rq, env->dst_rq); set_task_cpu(p, env->dst_cpu); + if (task_in_related_thread_group(p)) + env->flags |= LBF_MOVED_RELATED_THREAD_GROUP_TASK; + double_unlock_balance(env->src_rq, env->dst_rq); } /* @@ -6499,12 +7558,20 @@ static int detach_tasks(struct lb_env *env) struct task_struct *p; unsigned long load; int detached = 0; + int orig_loop = env->loop; lockdep_assert_held(&env->src_rq->lock); if (env->imbalance <= 0) return 0; + if (cpu_capacity(env->dst_cpu) < cpu_capacity(env->src_cpu) && + !sched_boost()) + env->flags |= LBF_IGNORE_BIG_TASKS; + else if (!same_cluster(env->dst_cpu, env->src_cpu)) + env->flags |= LBF_IGNORE_PREFERRED_CLUSTER_TASKS; + +redo: while (!list_empty(tasks)) { /* * We don't want to steal all, otherwise we may be treated likewise, @@ -6566,6 +7633,15 @@ next: list_move_tail(&p->se.group_node, tasks); } + if (env->flags & (LBF_IGNORE_BIG_TASKS | + LBF_IGNORE_PREFERRED_CLUSTER_TASKS) && !detached) { + tasks = &env->src_rq->cfs_tasks; + env->flags &= ~(LBF_IGNORE_BIG_TASKS | + LBF_IGNORE_PREFERRED_CLUSTER_TASKS); + env->loop = orig_loop; + goto redo; + } + /* * Right now, this is one of only two places we collect this stat * so we can safely collect detach_one_task() stats here rather @@ -6730,6 +7806,10 @@ struct sg_lb_stats { unsigned long group_capacity; unsigned long group_util; /* Total utilization of the group */ unsigned int sum_nr_running; /* Nr tasks running in the group */ +#ifdef CONFIG_SCHED_HMP + unsigned long sum_nr_big_tasks; + u64 group_cpu_load; /* Scaled load of all CPUs of the group */ +#endif unsigned int idle_cpus; unsigned int group_weight; enum group_type group_type; @@ -6772,10 +7852,64 @@ static inline void init_sd_lb_stats(struct sd_lb_stats *sds) .avg_load = 0UL, .sum_nr_running = 0, .group_type = group_other, +#ifdef CONFIG_SCHED_HMP + .sum_nr_big_tasks = 0UL, + .group_cpu_load = 0ULL, +#endif }, }; } +#ifdef CONFIG_SCHED_HMP + +static int +bail_inter_cluster_balance(struct lb_env *env, struct sd_lb_stats *sds) +{ + int local_cpu, busiest_cpu; + int local_capacity, busiest_capacity; + int local_pwr_cost, busiest_pwr_cost; + int nr_cpus; + + if (!sysctl_sched_restrict_cluster_spill || sched_boost()) + return 0; + + local_cpu = group_first_cpu(sds->local); + busiest_cpu = group_first_cpu(sds->busiest); + + local_capacity = cpu_max_possible_capacity(local_cpu); + busiest_capacity = cpu_max_possible_capacity(busiest_cpu); + + local_pwr_cost = cpu_max_power_cost(local_cpu); + busiest_pwr_cost = cpu_max_power_cost(busiest_cpu); + + if (local_capacity < busiest_capacity || + (local_capacity == busiest_capacity && + local_pwr_cost <= busiest_pwr_cost)) + return 0; + + if (local_capacity > busiest_capacity && + sds->busiest_stat.sum_nr_big_tasks) + return 0; + + nr_cpus = cpumask_weight(sched_group_cpus(sds->busiest)); + if ((sds->busiest_stat.group_cpu_load < nr_cpus * sched_spill_load) && + (sds->busiest_stat.sum_nr_running < + nr_cpus * sysctl_sched_spill_nr_run)) + return 1; + + return 0; +} + +#else /* CONFIG_SCHED_HMP */ + +static inline int +bail_inter_cluster_balance(struct lb_env *env, struct sd_lb_stats *sds) +{ + return 0; +} + +#endif /* CONFIG_SCHED_HMP */ + /** * get_sd_load_idx - Obtain the load index for a given sched domain. * @sd: The sched_domain whose load_idx is to be obtained. @@ -7038,6 +8172,11 @@ static inline void update_sg_lb_stats(struct lb_env *env, for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { struct rq *rq = cpu_rq(i); + trace_sched_cpu_load_lb(cpu_rq(i), idle_cpu(i), + sched_irqload(i), + power_cost(i, 0), + cpu_temp(i)); + /* Bias balancing toward cpus of our domain */ if (local_group) load = target_load(i, load_idx); @@ -7052,6 +8191,11 @@ static inline void update_sg_lb_stats(struct lb_env *env, if (nr_running > 1) *overload = true; +#ifdef CONFIG_SCHED_HMP + sgs->sum_nr_big_tasks += rq->hmp_stats.nr_big_tasks; + sgs->group_cpu_load += cpu_load(i); +#endif + #ifdef CONFIG_NUMA_BALANCING sgs->nr_numa_running += rq->nr_numa_running; sgs->nr_preferred_running += rq->nr_preferred_running; @@ -7077,6 +8221,38 @@ static inline void update_sg_lb_stats(struct lb_env *env, sgs->group_type = group_classify(group, sgs); } +#ifdef CONFIG_SCHED_HMP +static bool update_sd_pick_busiest_active_balance(struct lb_env *env, + struct sd_lb_stats *sds, + struct sched_group *sg, + struct sg_lb_stats *sgs) +{ + if (env->idle != CPU_NOT_IDLE && + cpu_capacity(env->dst_cpu) > group_rq_capacity(sg)) { + if (sched_boost() && !sds->busiest && sgs->sum_nr_running) { + env->flags |= LBF_SCHED_BOOST_ACTIVE_BALANCE; + return true; + } + + if (sgs->sum_nr_big_tasks > + sds->busiest_stat.sum_nr_big_tasks) { + env->flags |= LBF_BIG_TASK_ACTIVE_BALANCE; + return true; + } + } + + return false; +} +#else +static bool update_sd_pick_busiest_active_balance(struct lb_env *env, + struct sd_lb_stats *sds, + struct sched_group *sg, + struct sg_lb_stats *sgs) +{ + return false; +} +#endif + /** * update_sd_pick_busiest - return 1 on busiest group * @env: The load balancing environment. @@ -7097,6 +8273,9 @@ static bool update_sd_pick_busiest(struct lb_env *env, { struct sg_lb_stats *busiest = &sds->busiest_stat; + if (update_sd_pick_busiest_active_balance(env, sds, sg, sgs)) + return true; + if (sgs->group_type > busiest->group_type) return true; @@ -7218,6 +8397,8 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd if (update_sd_pick_busiest(env, sds, sg, sgs)) { sds->busiest = sg; sds->busiest_stat = *sgs; + env->busiest_nr_running = sgs->sum_nr_running; + env->busiest_grp_capacity = sgs->group_capacity; } next_group: @@ -7466,6 +8647,12 @@ static struct sched_group *find_busiest_group(struct lb_env *env) if (!sds.busiest || busiest->sum_nr_running == 0) goto out_balanced; + if (env->flags & LBF_HMP_ACTIVE_BALANCE) + goto force_balance; + + if (bail_inter_cluster_balance(env, &sds)) + goto out_balanced; + sds.avg_load = (SCHED_CAPACITY_SCALE * sds.total_load) / sds.total_capacity; @@ -7527,6 +8714,57 @@ out_balanced: return NULL; } +#ifdef CONFIG_SCHED_HMP +static struct rq *find_busiest_queue_hmp(struct lb_env *env, + struct sched_group *group) +{ + struct rq *busiest = NULL, *busiest_big = NULL; + u64 max_runnable_avg = 0, max_runnable_avg_big = 0; + int max_nr_big = 0, nr_big; + bool find_big = !!(env->flags & LBF_BIG_TASK_ACTIVE_BALANCE); + int i; + + for_each_cpu(i, sched_group_cpus(group)) { + struct rq *rq = cpu_rq(i); + u64 cumulative_runnable_avg = + rq->hmp_stats.cumulative_runnable_avg; + + if (!cpumask_test_cpu(i, env->cpus)) + continue; + + + if (find_big) { + nr_big = nr_big_tasks(rq); + if (nr_big > max_nr_big || + (nr_big > 0 && nr_big == max_nr_big && + cumulative_runnable_avg > max_runnable_avg_big)) { + max_runnable_avg_big = cumulative_runnable_avg; + busiest_big = rq; + max_nr_big = nr_big; + continue; + } + } + + if (cumulative_runnable_avg > max_runnable_avg) { + max_runnable_avg = cumulative_runnable_avg; + busiest = rq; + } + } + + if (busiest_big) + return busiest_big; + + env->flags &= ~LBF_BIG_TASK_ACTIVE_BALANCE; + return busiest; +} +#else +static inline struct rq *find_busiest_queue_hmp(struct lb_env *env, + struct sched_group *group) +{ + return NULL; +} +#endif + /* * find_busiest_queue - find the busiest runqueue among the cpus in group. */ @@ -7537,6 +8775,10 @@ static struct rq *find_busiest_queue(struct lb_env *env, unsigned long busiest_load = 0, busiest_capacity = 1; int i; +#ifdef CONFIG_SCHED_HMP + return find_busiest_queue_hmp(env, group); +#endif + for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { unsigned long capacity, wl; enum fbq_type rt; @@ -7604,12 +8846,16 @@ static struct rq *find_busiest_queue(struct lb_env *env, * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but * so long as it is large enough. */ -#define MAX_PINNED_INTERVAL 512 +#define MAX_PINNED_INTERVAL 16 +#define NEED_ACTIVE_BALANCE_THRESHOLD 10 static int need_active_balance(struct lb_env *env) { struct sched_domain *sd = env->sd; + if (env->flags & LBF_HMP_ACTIVE_BALANCE) + return 1; + if (env->idle == CPU_NEWLY_IDLE) { /* @@ -7634,7 +8880,8 @@ static int need_active_balance(struct lb_env *env) return 1; } - return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2); + return unlikely(sd->nr_balance_failed > + sd->cache_nice_tries + NEED_ACTIVE_BALANCE_THRESHOLD); } static int active_load_balance_cpu_stop(void *data); @@ -7681,10 +8928,10 @@ static int load_balance(int this_cpu, struct rq *this_rq, struct sched_domain *sd, enum cpu_idle_type idle, int *continue_balancing) { - int ld_moved, cur_ld_moved, active_balance = 0; + int ld_moved = 0, cur_ld_moved, active_balance = 0; struct sched_domain *sd_parent = sd->parent; - struct sched_group *group; - struct rq *busiest; + struct sched_group *group = NULL; + struct rq *busiest = NULL; unsigned long flags; struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); @@ -7698,6 +8945,11 @@ static int load_balance(int this_cpu, struct rq *this_rq, .cpus = cpus, .fbq_type = all, .tasks = LIST_HEAD_INIT(env.tasks), + .imbalance = 0, + .flags = 0, + .loop = 0, + .busiest_nr_running = 0, + .busiest_grp_capacity = 0, }; /* @@ -7750,6 +9002,13 @@ redo: more_balance: raw_spin_lock_irqsave(&busiest->lock, flags); + /* The world might have changed. Validate assumptions */ + if (busiest->nr_running <= 1) { + raw_spin_unlock_irqrestore(&busiest->lock, flags); + env.flags &= ~LBF_ALL_PINNED; + goto no_move; + } + /* * cur_ld_moved - load moved in current iteration * ld_moved - cumulative load moved across iterations @@ -7837,15 +9096,18 @@ more_balance: } } +no_move: if (!ld_moved) { - schedstat_inc(sd->lb_failed[idle]); + if (!(env.flags & LBF_HMP_ACTIVE_BALANCE)) + schedstat_inc(sd->lb_failed[idle]); /* * Increment the failure counter only on periodic balance. * We do not want newidle balance, which can be very * frequent, pollute the failure counter causing * excessive cache_hot migrations and active balances. */ - if (idle != CPU_NEWLY_IDLE) + if (idle != CPU_NEWLY_IDLE && + !(env.flags & LBF_HMP_ACTIVE_BALANCE)) sd->nr_balance_failed++; if (need_active_balance(&env)) { @@ -7879,14 +9141,28 @@ more_balance: stop_one_cpu_nowait(cpu_of(busiest), active_load_balance_cpu_stop, busiest, &busiest->active_balance_work); + *continue_balancing = 0; } /* We've kicked active balancing, force task migration. */ - sd->nr_balance_failed = sd->cache_nice_tries+1; + sd->nr_balance_failed = sd->cache_nice_tries + + NEED_ACTIVE_BALANCE_THRESHOLD - 1; } - } else + } else { sd->nr_balance_failed = 0; + /* Assumes one 'busiest' cpu that we pulled tasks from */ + if (!same_freq_domain(this_cpu, cpu_of(busiest))) { + int check_groups = !!(env.flags & + LBF_MOVED_RELATED_THREAD_GROUP_TASK); + + check_for_freq_change(this_rq, false, check_groups); + check_for_freq_change(busiest, false, check_groups); + } else { + check_for_freq_change(this_rq, true, false); + } + } + if (likely(!active_balance)) { /* We were unbalanced, so reset the balancing interval */ sd->balance_interval = sd->min_interval; @@ -7934,6 +9210,11 @@ out_one_pinned: ld_moved = 0; out: + trace_sched_load_balance(this_cpu, idle, *continue_balancing, + group ? group->cpumask[0] : 0, + busiest ? busiest->nr_running : 0, + env.imbalance, env.flags, ld_moved, + sd->balance_interval); return ld_moved; } @@ -8028,9 +9309,12 @@ static int idle_balance(struct rq *this_rq) /* * Stop searching for tasks to pull if there are - * now runnable tasks on this rq. + * now runnable tasks on the balance rq or if + * continue_balancing has been unset (only possible + * due to active migration). */ - if (pulled_task || this_rq->nr_running > 0) + if (pulled_task || this_rq->nr_running > 0 || + !continue_balancing) break; } rcu_read_unlock(); @@ -8075,8 +9359,23 @@ static int active_load_balance_cpu_stop(void *data) int busiest_cpu = cpu_of(busiest_rq); int target_cpu = busiest_rq->push_cpu; struct rq *target_rq = cpu_rq(target_cpu); - struct sched_domain *sd; + struct sched_domain *sd = NULL; struct task_struct *p = NULL; + struct task_struct *push_task; + int push_task_detached = 0; + struct lb_env env = { + .sd = sd, + .dst_cpu = target_cpu, + .dst_rq = target_rq, + .src_cpu = busiest_rq->cpu, + .src_rq = busiest_rq, + .idle = CPU_IDLE, + .busiest_nr_running = 0, + .busiest_grp_capacity = 0, + .flags = 0, + .loop = 0, + }; + bool moved = false; raw_spin_lock_irq(&busiest_rq->lock); @@ -8096,6 +9395,20 @@ static int active_load_balance_cpu_stop(void *data) */ BUG_ON(busiest_rq == target_rq); + push_task = busiest_rq->push_task; + target_cpu = busiest_rq->push_cpu; + if (push_task) { + if (task_on_rq_queued(push_task) && + push_task->state == TASK_RUNNING && + task_cpu(push_task) == busiest_cpu && + cpu_online(target_cpu)) { + detach_task(push_task, &env); + push_task_detached = 1; + moved = true; + } + goto out_unlock; + } + /* Search for an sd spanning us and the target CPU. */ rcu_read_lock(); for_each_domain(target_cpu, sd) { @@ -8105,15 +9418,7 @@ static int active_load_balance_cpu_stop(void *data) } if (likely(sd)) { - struct lb_env env = { - .sd = sd, - .dst_cpu = target_cpu, - .dst_rq = target_rq, - .src_cpu = busiest_rq->cpu, - .src_rq = busiest_rq, - .idle = CPU_IDLE, - }; - + env.sd = sd; schedstat_inc(sd->alb_count); p = detach_one_task(&env); @@ -8121,6 +9426,7 @@ static int active_load_balance_cpu_stop(void *data) schedstat_inc(sd->alb_pushed); /* Active balancing done, reset the failure counter. */ sd->nr_balance_failed = 0; + moved = true; } else { schedstat_inc(sd->alb_failed); } @@ -8128,6 +9434,12 @@ static int active_load_balance_cpu_stop(void *data) rcu_read_unlock(); out_unlock: busiest_rq->active_balance = 0; + push_task = busiest_rq->push_task; + target_cpu = busiest_rq->push_cpu; + + if (push_task) + busiest_rq->push_task = NULL; + raw_spin_unlock(&busiest_rq->lock); if (p) @@ -8135,6 +9447,15 @@ out_unlock: local_irq_enable(); + if (moved && !same_freq_domain(busiest_cpu, target_cpu)) { + int check_groups = !!(env.flags & + LBF_MOVED_RELATED_THREAD_GROUP_TASK); + check_for_freq_change(busiest_rq, false, check_groups); + check_for_freq_change(target_rq, false, check_groups); + } else if (moved) { + check_for_freq_change(target_rq, true, false); + } + return 0; } @@ -8156,10 +9477,45 @@ static struct { unsigned long next_balance; /* in jiffy units */ } nohz ____cacheline_aligned; -static inline int find_new_ilb(void) +#ifdef CONFIG_SCHED_HMP +static inline int find_new_hmp_ilb(int type) +{ + int call_cpu = raw_smp_processor_id(); + struct sched_domain *sd; + int ilb; + + rcu_read_lock(); + + /* Pick an idle cpu "closest" to call_cpu */ + for_each_domain(call_cpu, sd) { + for_each_cpu_and(ilb, nohz.idle_cpus_mask, + sched_domain_span(sd)) { + if (idle_cpu(ilb) && (type != NOHZ_KICK_RESTRICT || + (hmp_capable() && + cpu_max_possible_capacity(ilb) <= + cpu_max_possible_capacity(call_cpu)) || + cpu_max_power_cost(ilb) <= + cpu_max_power_cost(call_cpu))) { + rcu_read_unlock(); + reset_balance_interval(ilb); + return ilb; + } + } + } + + rcu_read_unlock(); + return nr_cpu_ids; +} +#endif /* CONFIG_SCHED_HMP */ + +static inline int find_new_ilb(int type) { int ilb = cpumask_first(nohz.idle_cpus_mask); +#ifdef CONFIG_SCHED_HMP + return find_new_hmp_ilb(type); +#endif + if (ilb < nr_cpu_ids && idle_cpu(ilb)) return ilb; @@ -8171,13 +9527,13 @@ static inline int find_new_ilb(void) * nohz_load_balancer CPU (if there is one) otherwise fallback to any idle * CPU (if there is one). */ -static void nohz_balancer_kick(void) +static void nohz_balancer_kick(int type) { int ilb_cpu; nohz.next_balance++; - ilb_cpu = find_new_ilb(); + ilb_cpu = find_new_ilb(type); if (ilb_cpu >= nr_cpu_ids) return; @@ -8450,6 +9806,66 @@ end: clear_bit(NOHZ_BALANCE_KICK, nohz_flags(this_cpu)); } +#ifdef CONFIG_SCHED_HMP +static inline int _nohz_kick_needed_hmp(struct rq *rq, int cpu, int *type) +{ + struct sched_domain *sd; + int i; + + if (rq->nr_running < 2) + return 0; + + if (!sysctl_sched_restrict_cluster_spill || sched_boost()) + return 1; + + if (hmp_capable() && cpu_max_possible_capacity(cpu) == + max_possible_capacity) + return 1; + + rcu_read_lock(); + sd = rcu_dereference_check_sched_domain(rq->sd); + if (!sd) { + rcu_read_unlock(); + return 0; + } + + for_each_cpu(i, sched_domain_span(sd)) { + if (cpu_load(i) < sched_spill_load && + cpu_rq(i)->nr_running < + sysctl_sched_spill_nr_run) { + /* Change the kick type to limit to CPUs that + * are of equal or lower capacity. + */ + *type = NOHZ_KICK_RESTRICT; + break; + } + } + rcu_read_unlock(); + return 1; +} +#endif /* CONFIG_SCHED_HMP */ + +static inline int _nohz_kick_needed(struct rq *rq, int cpu, int *type) +{ + unsigned long now = jiffies; + + /* + * None are in tickless mode and hence no need for NOHZ idle load + * balancing. + */ + if (likely(!atomic_read(&nohz.nr_cpus))) + return 0; + +#ifdef CONFIG_SCHED_HMP + return _nohz_kick_needed_hmp(rq, cpu, type); +#endif + + if (time_before(now, nohz.next_balance)) + return 0; + + return (rq->nr_running >= 2); +} + /* * Current heuristic for kicking the idle load balancer in the presence * of an idle cpu in the system. @@ -8461,12 +9877,14 @@ end: * - For SD_ASYM_PACKING, if the lower numbered cpu's in the scheduler * domain span are idle. */ -static inline bool nohz_kick_needed(struct rq *rq) +static inline bool nohz_kick_needed(struct rq *rq, int *type) { - unsigned long now = jiffies; +#ifndef CONFIG_SCHED_HMP struct sched_domain_shared *sds; struct sched_domain *sd; - int nr_busy, cpu = rq->cpu; + int nr_busy; +#endif + int cpu = rq->cpu; bool kick = false; if (unlikely(rq->idle_balance)) @@ -8479,19 +9897,10 @@ static inline bool nohz_kick_needed(struct rq *rq) set_cpu_sd_state_busy(); nohz_balance_exit_idle(cpu); - /* - * None are in tickless mode and hence no need for NOHZ idle load - * balancing. - */ - if (likely(!atomic_read(&nohz.nr_cpus))) - return false; - - if (time_before(now, nohz.next_balance)) - return false; - - if (rq->nr_running >= 2) + if (_nohz_kick_needed(rq, cpu, type)) return true; +#ifndef CONFIG_SCHED_HMP rcu_read_lock(); sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); if (sds) { @@ -8525,6 +9934,7 @@ static inline bool nohz_kick_needed(struct rq *rq) unlock: rcu_read_unlock(); +#endif return kick; } #else @@ -8558,6 +9968,8 @@ static __latent_entropy void run_rebalance_domains(struct softirq_action *h) */ void trigger_load_balance(struct rq *rq) { + int type = NOHZ_KICK_ANY; + /* Don't need to rebalance while attached to NULL domain */ if (unlikely(on_null_domain(rq))) return; @@ -8565,8 +9977,8 @@ void trigger_load_balance(struct rq *rq) if (time_after_eq(jiffies, rq->next_balance)) raise_softirq(SCHED_SOFTIRQ); #ifdef CONFIG_NO_HZ_COMMON - if (nohz_kick_needed(rq)) - nohz_balancer_kick(); + if (nohz_kick_needed(rq, &type)) + nohz_balancer_kick(type); #endif } @@ -9061,6 +10473,9 @@ const struct sched_class fair_sched_class = { #ifdef CONFIG_FAIR_GROUP_SCHED .task_change_group = task_change_group_fair, #endif +#ifdef CONFIG_SCHED_HMP + .fixup_hmp_sched_stats = fixup_hmp_sched_stats_fair, +#endif }; #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 69631fa46c2f..6651a37ba057 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -49,7 +49,7 @@ SCHED_FEAT(NONTASK_CAPACITY, true) * Queue remote wakeups on the target CPU and process them * using the scheduler IPI. Reduces rq->lock contention/bounces. */ -SCHED_FEAT(TTWU_QUEUE, true) +SCHED_FEAT(TTWU_QUEUE, false) #ifdef HAVE_RT_PUSH_IPI /* diff --git a/kernel/sched/hmp.c b/kernel/sched/hmp.c new file mode 100644 index 000000000000..9bc1bacd52e5 --- /dev/null +++ b/kernel/sched/hmp.c @@ -0,0 +1,3975 @@ +/* Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Implementation credits: Srivatsa Vaddagiri, Steve Muckle + * Syed Rameez Mustafa, Olav haugan, Joonwoo Park, Pavan Kumar Kondeti + * and Vikram Mulukutla + */ + +#include +#include +#include + +#include "sched.h" + +#include + +const char *task_event_names[] = {"PUT_PREV_TASK", "PICK_NEXT_TASK", + "TASK_WAKE", "TASK_MIGRATE", "TASK_UPDATE", "IRQ_UPDATE"}; + +const char *migrate_type_names[] = {"GROUP_TO_RQ", "RQ_TO_GROUP", + "RQ_TO_RQ", "GROUP_TO_GROUP"}; + + +static ktime_t ktime_last; +static bool sched_ktime_suspended; + +static bool use_cycle_counter; +static struct cpu_cycle_counter_cb cpu_cycle_counter_cb; + +u64 sched_ktime_clock(void) +{ + if (unlikely(sched_ktime_suspended)) + return ktime_to_ns(ktime_last); + return ktime_get_ns(); +} + +static void sched_resume(void) +{ + sched_ktime_suspended = false; +} + +static int sched_suspend(void) +{ + ktime_last = ktime_get(); + sched_ktime_suspended = true; + return 0; +} + +static struct syscore_ops sched_syscore_ops = { + .resume = sched_resume, + .suspend = sched_suspend +}; + +static int __init sched_init_ops(void) +{ + register_syscore_ops(&sched_syscore_ops); + return 0; +} +late_initcall(sched_init_ops); + +inline void clear_ed_task(struct task_struct *p, struct rq *rq) +{ + if (p == rq->ed_task) + rq->ed_task = NULL; +} + +inline void set_task_last_wake(struct task_struct *p, u64 wallclock) +{ + p->last_wake_ts = wallclock; +} + +inline void set_task_last_switch_out(struct task_struct *p, u64 wallclock) +{ + p->last_switch_out_ts = wallclock; +} + +/* + * Note C-state for (idle) cpus. + * + * @cstate = cstate index, 0 -> active state + * @wakeup_energy = energy spent in waking up cpu + * @wakeup_latency = latency to wakeup from cstate + * + */ +void +sched_set_cpu_cstate(int cpu, int cstate, int wakeup_energy, int wakeup_latency) +{ + struct rq *rq = cpu_rq(cpu); + + rq->cstate = cstate; /* C1, C2 etc */ + rq->wakeup_energy = wakeup_energy; + rq->wakeup_latency = wakeup_latency; +} + +/* + * Note D-state for (idle) cluster. + * + * @dstate = dstate index, 0 -> active state + * @wakeup_energy = energy spent in waking up cluster + * @wakeup_latency = latency to wakeup from cluster + * + */ +void sched_set_cluster_dstate(const cpumask_t *cluster_cpus, int dstate, + int wakeup_energy, int wakeup_latency) +{ + struct sched_cluster *cluster = + cpu_rq(cpumask_first(cluster_cpus))->cluster; + cluster->dstate = dstate; + cluster->dstate_wakeup_energy = wakeup_energy; + cluster->dstate_wakeup_latency = wakeup_latency; +} + +u32 __weak get_freq_max_load(int cpu, u32 freq) +{ + /* 100% by default */ + return 100; +} + +struct freq_max_load_entry { + /* The maximum load which has accounted governor's headroom. */ + u64 hdemand; +}; + +struct freq_max_load { + struct rcu_head rcu; + int length; + struct freq_max_load_entry freqs[0]; +}; + +static DEFINE_PER_CPU(struct freq_max_load *, freq_max_load); +static DEFINE_SPINLOCK(freq_max_load_lock); + +struct cpu_pwr_stats __weak *get_cpu_pwr_stats(void) +{ + return NULL; +} + +int sched_update_freq_max_load(const cpumask_t *cpumask) +{ + int i, cpu, ret; + unsigned int freq; + struct cpu_pstate_pwr *costs; + struct cpu_pwr_stats *per_cpu_info = get_cpu_pwr_stats(); + struct freq_max_load *max_load, *old_max_load; + struct freq_max_load_entry *entry; + u64 max_demand_capacity, max_demand; + unsigned long flags; + u32 hfreq; + int hpct; + + if (!per_cpu_info) + return 0; + + spin_lock_irqsave(&freq_max_load_lock, flags); + max_demand_capacity = div64_u64(max_task_load(), max_possible_capacity); + for_each_cpu(cpu, cpumask) { + if (!per_cpu_info[cpu].ptable) { + ret = -EINVAL; + goto fail; + } + + old_max_load = rcu_dereference(per_cpu(freq_max_load, cpu)); + + /* + * allocate len + 1 and leave the last power cost as 0 for + * power_cost() can stop iterating index when + * per_cpu_info[cpu].len > len of max_load due to race between + * cpu power stats update and get_cpu_pwr_stats(). + */ + max_load = kzalloc(sizeof(struct freq_max_load) + + sizeof(struct freq_max_load_entry) * + (per_cpu_info[cpu].len + 1), GFP_ATOMIC); + if (unlikely(!max_load)) { + ret = -ENOMEM; + goto fail; + } + + max_load->length = per_cpu_info[cpu].len; + + max_demand = max_demand_capacity * + cpu_max_possible_capacity(cpu); + + i = 0; + costs = per_cpu_info[cpu].ptable; + while (costs[i].freq) { + entry = &max_load->freqs[i]; + freq = costs[i].freq; + hpct = get_freq_max_load(cpu, freq); + if (hpct <= 0 && hpct > 100) + hpct = 100; + hfreq = div64_u64((u64)freq * hpct, 100); + entry->hdemand = + div64_u64(max_demand * hfreq, + cpu_max_possible_freq(cpu)); + i++; + } + + rcu_assign_pointer(per_cpu(freq_max_load, cpu), max_load); + if (old_max_load) + kfree_rcu(old_max_load, rcu); + } + + spin_unlock_irqrestore(&freq_max_load_lock, flags); + return 0; + +fail: + for_each_cpu(cpu, cpumask) { + max_load = rcu_dereference(per_cpu(freq_max_load, cpu)); + if (max_load) { + rcu_assign_pointer(per_cpu(freq_max_load, cpu), NULL); + kfree_rcu(max_load, rcu); + } + } + + spin_unlock_irqrestore(&freq_max_load_lock, flags); + return ret; +} + +unsigned int max_possible_efficiency = 1; +unsigned int min_possible_efficiency = UINT_MAX; + +unsigned long __weak arch_get_cpu_efficiency(int cpu) +{ + return SCHED_CAPACITY_SCALE; +} + +/* Keep track of max/min capacity possible across CPUs "currently" */ +static void __update_min_max_capacity(void) +{ + int i; + int max_cap = 0, min_cap = INT_MAX; + + for_each_online_cpu(i) { + max_cap = max(max_cap, cpu_capacity(i)); + min_cap = min(min_cap, cpu_capacity(i)); + } + + max_capacity = max_cap; + min_capacity = min_cap; +} + +static void update_min_max_capacity(void) +{ + unsigned long flags; + int i; + + local_irq_save(flags); + for_each_possible_cpu(i) + raw_spin_lock(&cpu_rq(i)->lock); + + __update_min_max_capacity(); + + for_each_possible_cpu(i) + raw_spin_unlock(&cpu_rq(i)->lock); + local_irq_restore(flags); +} + +/* + * Return 'capacity' of a cpu in reference to "least" efficient cpu, such that + * least efficient cpu gets capacity of 1024 + */ +static unsigned long +capacity_scale_cpu_efficiency(struct sched_cluster *cluster) +{ + return (1024 * cluster->efficiency) / min_possible_efficiency; +} + +/* + * Return 'capacity' of a cpu in reference to cpu with lowest max_freq + * (min_max_freq), such that one with lowest max_freq gets capacity of 1024. + */ +static unsigned long capacity_scale_cpu_freq(struct sched_cluster *cluster) +{ + return (1024 * cluster_max_freq(cluster)) / min_max_freq; +} + +/* + * Return load_scale_factor of a cpu in reference to "most" efficient cpu, so + * that "most" efficient cpu gets a load_scale_factor of 1 + */ +static inline unsigned long +load_scale_cpu_efficiency(struct sched_cluster *cluster) +{ + return DIV_ROUND_UP(1024 * max_possible_efficiency, + cluster->efficiency); +} + +/* + * Return load_scale_factor of a cpu in reference to cpu with best max_freq + * (max_possible_freq), so that one with best max_freq gets a load_scale_factor + * of 1. + */ +static inline unsigned long load_scale_cpu_freq(struct sched_cluster *cluster) +{ + return DIV_ROUND_UP(1024 * max_possible_freq, + cluster_max_freq(cluster)); +} + +static int compute_capacity(struct sched_cluster *cluster) +{ + int capacity = 1024; + + capacity *= capacity_scale_cpu_efficiency(cluster); + capacity >>= 10; + + capacity *= capacity_scale_cpu_freq(cluster); + capacity >>= 10; + + return capacity; +} + +static int compute_max_possible_capacity(struct sched_cluster *cluster) +{ + int capacity = 1024; + + capacity *= capacity_scale_cpu_efficiency(cluster); + capacity >>= 10; + + capacity *= (1024 * cluster->max_possible_freq) / min_max_freq; + capacity >>= 10; + + return capacity; +} + +static int compute_load_scale_factor(struct sched_cluster *cluster) +{ + int load_scale = 1024; + + /* + * load_scale_factor accounts for the fact that task load + * is in reference to "best" performing cpu. Task's load will need to be + * scaled (up) by a factor to determine suitability to be placed on a + * (little) cpu. + */ + load_scale *= load_scale_cpu_efficiency(cluster); + load_scale >>= 10; + + load_scale *= load_scale_cpu_freq(cluster); + load_scale >>= 10; + + return load_scale; +} + +struct list_head cluster_head; +static DEFINE_MUTEX(cluster_lock); +static cpumask_t all_cluster_cpus = CPU_MASK_NONE; +DECLARE_BITMAP(all_cluster_ids, NR_CPUS); +struct sched_cluster *sched_cluster[NR_CPUS]; +int num_clusters; + +struct sched_cluster init_cluster = { + .list = LIST_HEAD_INIT(init_cluster.list), + .id = 0, + .max_power_cost = 1, + .min_power_cost = 1, + .capacity = 1024, + .max_possible_capacity = 1024, + .efficiency = 1, + .load_scale_factor = 1024, + .cur_freq = 1, + .max_freq = 1, + .max_mitigated_freq = UINT_MAX, + .min_freq = 1, + .max_possible_freq = 1, + .dstate = 0, + .dstate_wakeup_energy = 0, + .dstate_wakeup_latency = 0, + .exec_scale_factor = 1024, + .notifier_sent = 0, +}; + +static void update_all_clusters_stats(void) +{ + struct sched_cluster *cluster; + u64 highest_mpc = 0, lowest_mpc = U64_MAX; + + pre_big_task_count_change(cpu_possible_mask); + + for_each_sched_cluster(cluster) { + u64 mpc; + + cluster->capacity = compute_capacity(cluster); + mpc = cluster->max_possible_capacity = + compute_max_possible_capacity(cluster); + cluster->load_scale_factor = compute_load_scale_factor(cluster); + + cluster->exec_scale_factor = + DIV_ROUND_UP(cluster->efficiency * 1024, + max_possible_efficiency); + + if (mpc > highest_mpc) + highest_mpc = mpc; + + if (mpc < lowest_mpc) + lowest_mpc = mpc; + } + + max_possible_capacity = highest_mpc; + min_max_possible_capacity = lowest_mpc; + + __update_min_max_capacity(); + sched_update_freq_max_load(cpu_possible_mask); + post_big_task_count_change(cpu_possible_mask); +} + +static void assign_cluster_ids(struct list_head *head) +{ + struct sched_cluster *cluster; + int pos = 0; + + list_for_each_entry(cluster, head, list) { + cluster->id = pos; + sched_cluster[pos++] = cluster; + } +} + +static void +move_list(struct list_head *dst, struct list_head *src, bool sync_rcu) +{ + struct list_head *first, *last; + + first = src->next; + last = src->prev; + + if (sync_rcu) { + INIT_LIST_HEAD_RCU(src); + synchronize_rcu(); + } + + first->prev = dst; + dst->prev = last; + last->next = dst; + + /* Ensure list sanity before making the head visible to all CPUs. */ + smp_mb(); + dst->next = first; +} + +static int +compare_clusters(void *priv, struct list_head *a, struct list_head *b) +{ + struct sched_cluster *cluster1, *cluster2; + int ret; + + cluster1 = container_of(a, struct sched_cluster, list); + cluster2 = container_of(b, struct sched_cluster, list); + + ret = cluster1->max_power_cost > cluster2->max_power_cost || + (cluster1->max_power_cost == cluster2->max_power_cost && + cluster1->max_possible_capacity < + cluster2->max_possible_capacity); + + return ret; +} + +static void sort_clusters(void) +{ + struct sched_cluster *cluster; + struct list_head new_head; + + INIT_LIST_HEAD(&new_head); + + for_each_sched_cluster(cluster) { + cluster->max_power_cost = power_cost(cluster_first_cpu(cluster), + max_task_load()); + cluster->min_power_cost = power_cost(cluster_first_cpu(cluster), + 0); + } + + move_list(&new_head, &cluster_head, true); + + list_sort(NULL, &new_head, compare_clusters); + assign_cluster_ids(&new_head); + + /* + * Ensure cluster ids are visible to all CPUs before making + * cluster_head visible. + */ + move_list(&cluster_head, &new_head, false); +} + +static void +insert_cluster(struct sched_cluster *cluster, struct list_head *head) +{ + struct sched_cluster *tmp; + struct list_head *iter = head; + + list_for_each_entry(tmp, head, list) { + if (cluster->max_power_cost < tmp->max_power_cost) + break; + iter = &tmp->list; + } + + list_add(&cluster->list, iter); +} + +static struct sched_cluster *alloc_new_cluster(const struct cpumask *cpus) +{ + struct sched_cluster *cluster = NULL; + + cluster = kzalloc(sizeof(struct sched_cluster), GFP_ATOMIC); + if (!cluster) { + __WARN_printf("Cluster allocation failed. \ + Possible bad scheduling\n"); + return NULL; + } + + INIT_LIST_HEAD(&cluster->list); + cluster->max_power_cost = 1; + cluster->min_power_cost = 1; + cluster->capacity = 1024; + cluster->max_possible_capacity = 1024; + cluster->efficiency = 1; + cluster->load_scale_factor = 1024; + cluster->cur_freq = 1; + cluster->max_freq = 1; + cluster->max_mitigated_freq = UINT_MAX; + cluster->min_freq = 1; + cluster->max_possible_freq = 1; + cluster->dstate = 0; + cluster->dstate_wakeup_energy = 0; + cluster->dstate_wakeup_latency = 0; + cluster->freq_init_done = false; + + cluster->cpus = *cpus; + cluster->efficiency = arch_get_cpu_efficiency(cpumask_first(cpus)); + + if (cluster->efficiency > max_possible_efficiency) + max_possible_efficiency = cluster->efficiency; + if (cluster->efficiency < min_possible_efficiency) + min_possible_efficiency = cluster->efficiency; + + cluster->notifier_sent = 0; + return cluster; +} + +static void add_cluster(const struct cpumask *cpus, struct list_head *head) +{ + struct sched_cluster *cluster = alloc_new_cluster(cpus); + int i; + + if (!cluster) + return; + + for_each_cpu(i, cpus) + cpu_rq(i)->cluster = cluster; + + insert_cluster(cluster, head); + set_bit(num_clusters, all_cluster_ids); + num_clusters++; +} + +void update_cluster_topology(void) +{ + struct cpumask cpus = *cpu_possible_mask; + const struct cpumask *cluster_cpus; + struct list_head new_head; + int i; + + INIT_LIST_HEAD(&new_head); + + for_each_cpu(i, &cpus) { + cluster_cpus = cpu_coregroup_mask(i); + cpumask_or(&all_cluster_cpus, &all_cluster_cpus, cluster_cpus); + cpumask_andnot(&cpus, &cpus, cluster_cpus); + add_cluster(cluster_cpus, &new_head); + } + + assign_cluster_ids(&new_head); + + /* + * Ensure cluster ids are visible to all CPUs before making + * cluster_head visible. + */ + move_list(&cluster_head, &new_head, false); +} + +void init_clusters(void) +{ + bitmap_clear(all_cluster_ids, 0, NR_CPUS); + init_cluster.cpus = *cpu_possible_mask; + INIT_LIST_HEAD(&cluster_head); +} + +int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb) +{ + mutex_lock(&cluster_lock); + if (!cb->get_cpu_cycle_counter) { + mutex_unlock(&cluster_lock); + return -EINVAL; + } + + cpu_cycle_counter_cb = *cb; + use_cycle_counter = true; + mutex_unlock(&cluster_lock); + + return 0; +} + +int got_boost_kick(void) +{ + int cpu = smp_processor_id(); + struct rq *rq = cpu_rq(cpu); + + return test_bit(BOOST_KICK, &rq->hmp_flags); +} + +inline void clear_boost_kick(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + clear_bit(BOOST_KICK, &rq->hmp_flags); +} + +inline void boost_kick(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + if (!test_and_set_bit(BOOST_KICK, &rq->hmp_flags)) + smp_send_reschedule(cpu); +} + +/* Clear any HMP scheduler related requests pending from or on cpu */ +void clear_hmp_request(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + unsigned long flags; + + clear_boost_kick(cpu); + clear_reserved(cpu); + if (rq->push_task) { + raw_spin_lock_irqsave(&rq->lock, flags); + if (rq->push_task) { + clear_reserved(rq->push_cpu); + put_task_struct(rq->push_task); + rq->push_task = NULL; + } + rq->active_balance = 0; + raw_spin_unlock_irqrestore(&rq->lock, flags); + } +} + +int sched_set_static_cpu_pwr_cost(int cpu, unsigned int cost) +{ + struct rq *rq = cpu_rq(cpu); + + rq->static_cpu_pwr_cost = cost; + return 0; +} + +unsigned int sched_get_static_cpu_pwr_cost(int cpu) +{ + return cpu_rq(cpu)->static_cpu_pwr_cost; +} + +int sched_set_static_cluster_pwr_cost(int cpu, unsigned int cost) +{ + struct sched_cluster *cluster = cpu_rq(cpu)->cluster; + + cluster->static_cluster_pwr_cost = cost; + return 0; +} + +unsigned int sched_get_static_cluster_pwr_cost(int cpu) +{ + return cpu_rq(cpu)->cluster->static_cluster_pwr_cost; +} + +/* + * sched_window_stats_policy and sched_ravg_hist_size have a 'sysctl' copy + * associated with them. This is required for atomic update of those variables + * when being modifed via sysctl interface. + * + * IMPORTANT: Initialize both copies to same value!! + */ + +/* + * Tasks that are runnable continuously for a period greather than + * EARLY_DETECTION_DURATION can be flagged early as potential + * high load tasks. + */ +#define EARLY_DETECTION_DURATION 9500000 + +static __read_mostly unsigned int sched_ravg_hist_size = 5; +__read_mostly unsigned int sysctl_sched_ravg_hist_size = 5; + +static __read_mostly unsigned int sched_window_stats_policy = + WINDOW_STATS_MAX_RECENT_AVG; +__read_mostly unsigned int sysctl_sched_window_stats_policy = + WINDOW_STATS_MAX_RECENT_AVG; + +#define SCHED_ACCOUNT_WAIT_TIME 1 + +__read_mostly unsigned int sysctl_sched_cpu_high_irqload = (10 * NSEC_PER_MSEC); + +unsigned int __read_mostly sysctl_sched_enable_colocation = 1; + +/* + * Enable colocation and frequency aggregation for all threads in a process. + * The children inherits the group id from the parent. + */ +unsigned int __read_mostly sysctl_sched_enable_thread_grouping; + + +__read_mostly unsigned int sysctl_sched_new_task_windows = 5; + +#define SCHED_FREQ_ACCOUNT_WAIT_TIME 0 + +/* + * For increase, send notification if + * freq_required - cur_freq > sysctl_sched_freq_inc_notify + */ +__read_mostly int sysctl_sched_freq_inc_notify = 10 * 1024 * 1024; /* + 10GHz */ + +/* + * For decrease, send notification if + * cur_freq - freq_required > sysctl_sched_freq_dec_notify + */ +__read_mostly int sysctl_sched_freq_dec_notify = 10 * 1024 * 1024; /* - 10GHz */ + +static __read_mostly unsigned int sched_io_is_busy; + +__read_mostly unsigned int sysctl_sched_pred_alert_freq = 10 * 1024 * 1024; + +/* + * Maximum possible frequency across all cpus. Task demand and cpu + * capacity (cpu_power) metrics are scaled in reference to it. + */ +unsigned int max_possible_freq = 1; + +/* + * Minimum possible max_freq across all cpus. This will be same as + * max_possible_freq on homogeneous systems and could be different from + * max_possible_freq on heterogenous systems. min_max_freq is used to derive + * capacity (cpu_power) of cpus. + */ +unsigned int min_max_freq = 1; + +unsigned int max_capacity = 1024; /* max(rq->capacity) */ +unsigned int min_capacity = 1024; /* min(rq->capacity) */ +unsigned int max_possible_capacity = 1024; /* max(rq->max_possible_capacity) */ +unsigned int +min_max_possible_capacity = 1024; /* min(rq->max_possible_capacity) */ + +/* Window size (in ns) */ +__read_mostly unsigned int sched_ravg_window = 10000000; + +/* Min window size (in ns) = 10ms */ +#define MIN_SCHED_RAVG_WINDOW 10000000 + +/* Max window size (in ns) = 1s */ +#define MAX_SCHED_RAVG_WINDOW 1000000000 + +/* Temporarily disable window-stats activity on all cpus */ +unsigned int __read_mostly sched_disable_window_stats; + +/* + * Major task runtime. If a task runs for more than sched_major_task_runtime + * in a window, it's considered to be generating majority of workload + * for this window. Prediction could be adjusted for such tasks. + */ +__read_mostly unsigned int sched_major_task_runtime = 10000000; + +static unsigned int sync_cpu; + +static LIST_HEAD(related_thread_groups); +static DEFINE_RWLOCK(related_thread_group_lock); + +#define for_each_related_thread_group(grp) \ + list_for_each_entry(grp, &related_thread_groups, list) + +/* + * Demand aggregation for frequency purpose: + * + * 'sched_freq_aggregate' controls aggregation of cpu demand of related threads + * for frequency determination purpose. This aggregation is done per-cluster. + * + * CPU demand of tasks from various related groups is aggregated per-cluster and + * added to the "max_busy_cpu" in that cluster, where max_busy_cpu is determined + * by just rq->prev_runnable_sum. + * + * Some examples follow, which assume: + * Cluster0 = CPU0-3, Cluster1 = CPU4-7 + * One related thread group A that has tasks A0, A1, A2 + * + * A->cpu_time[X].curr/prev_sum = counters in which cpu execution stats of + * tasks belonging to group A are accumulated when they run on cpu X. + * + * CX->curr/prev_sum = counters in which cpu execution stats of all tasks + * not belonging to group A are accumulated when they run on cpu X + * + * Lets say the stats for window M was as below: + * + * C0->prev_sum = 1ms, A->cpu_time[0].prev_sum = 5ms + * Task A0 ran 5ms on CPU0 + * Task B0 ran 1ms on CPU0 + * + * C1->prev_sum = 5ms, A->cpu_time[1].prev_sum = 6ms + * Task A1 ran 4ms on CPU1 + * Task A2 ran 2ms on CPU1 + * Task B1 ran 5ms on CPU1 + * + * C2->prev_sum = 0ms, A->cpu_time[2].prev_sum = 0 + * CPU2 idle + * + * C3->prev_sum = 0ms, A->cpu_time[3].prev_sum = 0 + * CPU3 idle + * + * In this case, CPU1 was most busy going by just its prev_sum counter. Demand + * from all group A tasks are added to CPU1. IOW, at end of window M, cpu busy + * time reported to governor will be: + * + * + * C0 busy time = 1ms + * C1 busy time = 5 + 5 + 6 = 16ms + * + */ +static __read_mostly unsigned int sched_freq_aggregate; +__read_mostly unsigned int sysctl_sched_freq_aggregate; + +unsigned int __read_mostly sysctl_sched_freq_aggregate_threshold_pct; +static unsigned int __read_mostly sched_freq_aggregate_threshold; + +/* Initial task load. Newly created tasks are assigned this load. */ +unsigned int __read_mostly sched_init_task_load_windows; +unsigned int __read_mostly sysctl_sched_init_task_load_pct = 15; + +unsigned int max_task_load(void) +{ + return sched_ravg_window; +} + +/* + * Scheduler boost is a mechanism to temporarily place tasks on CPUs + * with higher capacity than those where a task would have normally + * ended up with their load characteristics. Any entity enabling + * boost is responsible for disabling it as well. + */ +unsigned int sysctl_sched_boost; + +/* A cpu can no longer accommodate more tasks if: + * + * rq->nr_running > sysctl_sched_spill_nr_run || + * rq->hmp_stats.cumulative_runnable_avg > sched_spill_load + */ +unsigned int __read_mostly sysctl_sched_spill_nr_run = 10; + +/* + * Place sync wakee tasks those have less than configured demand to the waker's + * cluster. + */ +unsigned int __read_mostly sched_small_wakee_task_load; +unsigned int __read_mostly sysctl_sched_small_wakee_task_load_pct = 10; + +unsigned int __read_mostly sched_big_waker_task_load; +unsigned int __read_mostly sysctl_sched_big_waker_task_load_pct = 25; + +/* + * CPUs with load greater than the sched_spill_load_threshold are not + * eligible for task placement. When all CPUs in a cluster achieve a + * load higher than this level, tasks becomes eligible for inter + * cluster migration. + */ +unsigned int __read_mostly sched_spill_load; +unsigned int __read_mostly sysctl_sched_spill_load_pct = 100; + +/* + * Tasks whose bandwidth consumption on a cpu is more than + * sched_upmigrate are considered "big" tasks. Big tasks will be + * considered for "up" migration, i.e migrating to a cpu with better + * capacity. + */ +unsigned int __read_mostly sched_upmigrate; +unsigned int __read_mostly sysctl_sched_upmigrate_pct = 80; + +/* + * Big tasks, once migrated, will need to drop their bandwidth + * consumption to less than sched_downmigrate before they are "down" + * migrated. + */ +unsigned int __read_mostly sched_downmigrate; +unsigned int __read_mostly sysctl_sched_downmigrate_pct = 60; + +/* + * The load scale factor of a CPU gets boosted when its max frequency + * is restricted due to which the tasks are migrating to higher capacity + * CPUs early. The sched_upmigrate threshold is auto-upgraded by + * rq->max_possible_freq/rq->max_freq of a lower capacity CPU. + */ +unsigned int up_down_migrate_scale_factor = 1024; + +/* + * Scheduler selects and places task to its previous CPU if sleep time is + * less than sysctl_sched_select_prev_cpu_us. + */ +unsigned int __read_mostly +sched_short_sleep_task_threshold = 2000 * NSEC_PER_USEC; + +unsigned int __read_mostly sysctl_sched_select_prev_cpu_us = 2000; + +unsigned int __read_mostly +sched_long_cpu_selection_threshold = 100 * NSEC_PER_MSEC; + +unsigned int __read_mostly sysctl_sched_restrict_cluster_spill; + +void update_up_down_migrate(void) +{ + unsigned int up_migrate = pct_to_real(sysctl_sched_upmigrate_pct); + unsigned int down_migrate = pct_to_real(sysctl_sched_downmigrate_pct); + unsigned int delta; + + if (up_down_migrate_scale_factor == 1024) + goto done; + + delta = up_migrate - down_migrate; + + up_migrate /= NSEC_PER_USEC; + up_migrate *= up_down_migrate_scale_factor; + up_migrate >>= 10; + up_migrate *= NSEC_PER_USEC; + + up_migrate = min(up_migrate, sched_ravg_window); + + down_migrate /= NSEC_PER_USEC; + down_migrate *= up_down_migrate_scale_factor; + down_migrate >>= 10; + down_migrate *= NSEC_PER_USEC; + + down_migrate = min(down_migrate, up_migrate - delta); +done: + sched_upmigrate = up_migrate; + sched_downmigrate = down_migrate; +} + +void set_hmp_defaults(void) +{ + sched_spill_load = + pct_to_real(sysctl_sched_spill_load_pct); + + update_up_down_migrate(); + + sched_major_task_runtime = + mult_frac(sched_ravg_window, MAJOR_TASK_PCT, 100); + + sched_init_task_load_windows = + div64_u64((u64)sysctl_sched_init_task_load_pct * + (u64)sched_ravg_window, 100); + + sched_short_sleep_task_threshold = sysctl_sched_select_prev_cpu_us * + NSEC_PER_USEC; + + sched_small_wakee_task_load = + div64_u64((u64)sysctl_sched_small_wakee_task_load_pct * + (u64)sched_ravg_window, 100); + + sched_big_waker_task_load = + div64_u64((u64)sysctl_sched_big_waker_task_load_pct * + (u64)sched_ravg_window, 100); + + sched_freq_aggregate_threshold = + pct_to_real(sysctl_sched_freq_aggregate_threshold_pct); +} + +u32 sched_get_init_task_load(struct task_struct *p) +{ + return p->init_load_pct; +} + +int sched_set_init_task_load(struct task_struct *p, int init_load_pct) +{ + if (init_load_pct < 0 || init_load_pct > 100) + return -EINVAL; + + p->init_load_pct = init_load_pct; + + return 0; +} + +#ifdef CONFIG_CGROUP_SCHED + +int upmigrate_discouraged(struct task_struct *p) +{ + return task_group(p)->upmigrate_discouraged; +} + +#else + +static inline int upmigrate_discouraged(struct task_struct *p) +{ + return 0; +} + +#endif + +/* Is a task "big" on its current cpu */ +static inline int __is_big_task(struct task_struct *p, u64 scaled_load) +{ + int nice = task_nice(p); + + if (nice > SCHED_UPMIGRATE_MIN_NICE || upmigrate_discouraged(p)) + return 0; + + return scaled_load > sched_upmigrate; +} + +int is_big_task(struct task_struct *p) +{ + return __is_big_task(p, scale_load_to_cpu(task_load(p), task_cpu(p))); +} + +u64 cpu_load(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + return scale_load_to_cpu(rq->hmp_stats.cumulative_runnable_avg, cpu); +} + +u64 cpu_load_sync(int cpu, int sync) +{ + return scale_load_to_cpu(cpu_cravg_sync(cpu, sync), cpu); +} + +static int boost_refcount; +static DEFINE_SPINLOCK(boost_lock); +static DEFINE_MUTEX(boost_mutex); + +static void boost_kick_cpus(void) +{ + int i; + + for_each_online_cpu(i) { + if (cpu_capacity(i) != max_capacity) + boost_kick(i); + } +} + +int sched_boost(void) +{ + return boost_refcount > 0; +} + +int sched_set_boost(int enable) +{ + unsigned long flags; + int ret = 0; + int old_refcount; + + spin_lock_irqsave(&boost_lock, flags); + + old_refcount = boost_refcount; + + if (enable == 1) { + boost_refcount++; + } else if (!enable) { + if (boost_refcount >= 1) + boost_refcount--; + else + ret = -EINVAL; + } else { + ret = -EINVAL; + } + + if (!old_refcount && boost_refcount) + boost_kick_cpus(); + + trace_sched_set_boost(boost_refcount); + spin_unlock_irqrestore(&boost_lock, flags); + + return ret; +} + +int sched_boost_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + + mutex_lock(&boost_mutex); + if (!write) + sysctl_sched_boost = sched_boost(); + + ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (ret || !write) + goto done; + + ret = (sysctl_sched_boost <= 1) ? + sched_set_boost(sysctl_sched_boost) : -EINVAL; + +done: + mutex_unlock(&boost_mutex); + return ret; +} + +/* + * Task will fit on a cpu if it's bandwidth consumption on that cpu + * will be less than sched_upmigrate. A big task that was previously + * "up" migrated will be considered fitting on "little" cpu if its + * bandwidth consumption on "little" cpu will be less than + * sched_downmigrate. This will help avoid frequenty migrations for + * tasks with load close to the upmigrate threshold + */ +int task_load_will_fit(struct task_struct *p, u64 task_load, int cpu, + enum sched_boost_type boost_type) +{ + int upmigrate; + + if (cpu_capacity(cpu) == max_capacity) + return 1; + + if (boost_type != SCHED_BOOST_ON_BIG) { + if (task_nice(p) > SCHED_UPMIGRATE_MIN_NICE || + upmigrate_discouraged(p)) + return 1; + + upmigrate = sched_upmigrate; + if (cpu_capacity(task_cpu(p)) > cpu_capacity(cpu)) + upmigrate = sched_downmigrate; + + if (task_load < upmigrate) + return 1; + } + + return 0; +} + +enum sched_boost_type sched_boost_type(void) +{ + if (sched_boost()) { + if (min_possible_efficiency != max_possible_efficiency) + return SCHED_BOOST_ON_BIG; + else + return SCHED_BOOST_ON_ALL; + } + return SCHED_BOOST_NONE; +} + +int task_will_fit(struct task_struct *p, int cpu) +{ + u64 tload = scale_load_to_cpu(task_load(p), cpu); + + return task_load_will_fit(p, tload, cpu, sched_boost_type()); +} + +int group_will_fit(struct sched_cluster *cluster, + struct related_thread_group *grp, u64 demand) +{ + int cpu = cluster_first_cpu(cluster); + int prev_capacity = 0; + unsigned int threshold = sched_upmigrate; + u64 load; + + if (cluster->capacity == max_capacity) + return 1; + + if (grp->preferred_cluster) + prev_capacity = grp->preferred_cluster->capacity; + + if (cluster->capacity < prev_capacity) + threshold = sched_downmigrate; + + load = scale_load_to_cpu(demand, cpu); + if (load < threshold) + return 1; + + return 0; +} + +/* + * Return the cost of running task p on CPU cpu. This function + * currently assumes that task p is the only task which will run on + * the CPU. + */ +unsigned int power_cost(int cpu, u64 demand) +{ + int first, mid, last; + struct cpu_pwr_stats *per_cpu_info = get_cpu_pwr_stats(); + struct cpu_pstate_pwr *costs; + struct freq_max_load *max_load; + int total_static_pwr_cost = 0; + struct rq *rq = cpu_rq(cpu); + unsigned int pc; + + if (!per_cpu_info || !per_cpu_info[cpu].ptable) + /* + * When power aware scheduling is not in use, or CPU + * power data is not available, just use the CPU + * capacity as a rough stand-in for real CPU power + * numbers, assuming bigger CPUs are more power + * hungry. + */ + return cpu_max_possible_capacity(cpu); + + rcu_read_lock(); + max_load = rcu_dereference(per_cpu(freq_max_load, cpu)); + if (!max_load) { + pc = cpu_max_possible_capacity(cpu); + goto unlock; + } + + costs = per_cpu_info[cpu].ptable; + + if (demand <= max_load->freqs[0].hdemand) { + pc = costs[0].power; + goto unlock; + } else if (demand > max_load->freqs[max_load->length - 1].hdemand) { + pc = costs[max_load->length - 1].power; + goto unlock; + } + + first = 0; + last = max_load->length - 1; + mid = (last - first) >> 1; + while (1) { + if (demand <= max_load->freqs[mid].hdemand) + last = mid; + else + first = mid; + + if (last - first == 1) + break; + mid = first + ((last - first) >> 1); + } + + pc = costs[last].power; + +unlock: + rcu_read_unlock(); + + if (idle_cpu(cpu) && rq->cstate) { + total_static_pwr_cost += rq->static_cpu_pwr_cost; + if (rq->cluster->dstate) + total_static_pwr_cost += + rq->cluster->static_cluster_pwr_cost; + } + + return pc + total_static_pwr_cost; + +} + +void inc_nr_big_task(struct hmp_sched_stats *stats, struct task_struct *p) +{ + if (sched_disable_window_stats) + return; + + if (is_big_task(p)) + stats->nr_big_tasks++; +} + +void dec_nr_big_task(struct hmp_sched_stats *stats, struct task_struct *p) +{ + if (sched_disable_window_stats) + return; + + if (is_big_task(p)) + stats->nr_big_tasks--; + + BUG_ON(stats->nr_big_tasks < 0); +} + +void inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) +{ + inc_nr_big_task(&rq->hmp_stats, p); + if (change_cra) + inc_cumulative_runnable_avg(&rq->hmp_stats, p); +} + +void dec_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) +{ + dec_nr_big_task(&rq->hmp_stats, p); + if (change_cra) + dec_cumulative_runnable_avg(&rq->hmp_stats, p); +} + +static void reset_hmp_stats(struct hmp_sched_stats *stats, int reset_cra) +{ + stats->nr_big_tasks = 0; + if (reset_cra) { + stats->cumulative_runnable_avg = 0; + stats->pred_demands_sum = 0; + } +} + +/* + * Invoked from three places: + * 1) try_to_wake_up() -> ... -> select_best_cpu() + * 2) scheduler_tick() -> ... -> migration_needed() -> select_best_cpu() + * 3) can_migrate_task() + * + * Its safe to de-reference p->grp in first case (since p->pi_lock is held) + * but not in other cases. p->grp is hence freed after a RCU grace period and + * accessed under rcu_read_lock() + */ +int preferred_cluster(struct sched_cluster *cluster, struct task_struct *p) +{ + struct related_thread_group *grp; + int rc = 0; + + rcu_read_lock(); + + grp = task_related_thread_group(p); + if (!grp || !sysctl_sched_enable_colocation) + rc = 1; + else + rc = (grp->preferred_cluster == cluster); + + rcu_read_unlock(); + return rc; +} + +struct sched_cluster *rq_cluster(struct rq *rq) +{ + return rq->cluster; +} + +/* + * reset_cpu_hmp_stats - reset HMP stats for a cpu + * nr_big_tasks + * cumulative_runnable_avg (iff reset_cra is true) + */ +void reset_cpu_hmp_stats(int cpu, int reset_cra) +{ + reset_cfs_rq_hmp_stats(cpu, reset_cra); + reset_hmp_stats(&cpu_rq(cpu)->hmp_stats, reset_cra); +} + +void fixup_nr_big_tasks(struct hmp_sched_stats *stats, + struct task_struct *p, s64 delta) +{ + u64 new_task_load; + u64 old_task_load; + + if (sched_disable_window_stats) + return; + + old_task_load = scale_load_to_cpu(task_load(p), task_cpu(p)); + new_task_load = scale_load_to_cpu(delta + task_load(p), task_cpu(p)); + + if (__is_big_task(p, old_task_load) && !__is_big_task(p, new_task_load)) + stats->nr_big_tasks--; + else if (!__is_big_task(p, old_task_load) && + __is_big_task(p, new_task_load)) + stats->nr_big_tasks++; + + BUG_ON(stats->nr_big_tasks < 0); +} + +/* + * Walk runqueue of cpu and re-initialize 'nr_big_tasks' counters. + */ +static void update_nr_big_tasks(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + struct task_struct *p; + + /* Do not reset cumulative_runnable_avg */ + reset_cpu_hmp_stats(cpu, 0); + + list_for_each_entry(p, &rq->cfs_tasks, se.group_node) + inc_hmp_sched_stats_fair(rq, p, 0); +} + +/* Disable interrupts and grab runqueue lock of all cpus listed in @cpus */ +void pre_big_task_count_change(const struct cpumask *cpus) +{ + int i; + + local_irq_disable(); + + for_each_cpu(i, cpus) + raw_spin_lock(&cpu_rq(i)->lock); +} + +/* + * Reinitialize 'nr_big_tasks' counters on all affected cpus + */ +void post_big_task_count_change(const struct cpumask *cpus) +{ + int i; + + /* Assumes local_irq_disable() keeps online cpumap stable */ + for_each_cpu(i, cpus) + update_nr_big_tasks(i); + + for_each_cpu(i, cpus) + raw_spin_unlock(&cpu_rq(i)->lock); + + local_irq_enable(); +} + +DEFINE_MUTEX(policy_mutex); + +static inline int invalid_value_freq_input(unsigned int *data) +{ + if (data == &sysctl_sched_freq_aggregate) + return !(*data == 0 || *data == 1); + + return 0; +} + +static inline int invalid_value(unsigned int *data) +{ + unsigned int val = *data; + + if (data == &sysctl_sched_ravg_hist_size) + return (val < 2 || val > RAVG_HIST_SIZE_MAX); + + if (data == &sysctl_sched_window_stats_policy) + return val >= WINDOW_STATS_INVALID_POLICY; + + return invalid_value_freq_input(data); +} + +/* + * Handle "atomic" update of sysctl_sched_window_stats_policy, + * sysctl_sched_ravg_hist_size and sched_freq_legacy_mode variables. + */ +int sched_window_update_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + unsigned int *data = (unsigned int *)table->data; + unsigned int old_val; + + mutex_lock(&policy_mutex); + + old_val = *data; + + ret = proc_dointvec(table, write, buffer, lenp, ppos); + if (ret || !write || (write && (old_val == *data))) + goto done; + + if (invalid_value(data)) { + *data = old_val; + ret = -EINVAL; + goto done; + } + + reset_all_window_stats(0, 0); + +done: + mutex_unlock(&policy_mutex); + + return ret; +} + +/* + * Convert percentage value into absolute form. This will avoid div() operation + * in fast path, to convert task load in percentage scale. + */ +int sched_hmp_proc_update_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret; + unsigned int old_val; + unsigned int *data = (unsigned int *)table->data; + int update_min_nice = 0; + + mutex_lock(&policy_mutex); + + old_val = *data; + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + + if (ret || !write) + goto done; + + if (write && (old_val == *data)) + goto done; + + if (sysctl_sched_downmigrate_pct > sysctl_sched_upmigrate_pct) { + *data = old_val; + ret = -EINVAL; + goto done; + } + + /* + * Big task tunable change will need to re-classify tasks on + * runqueue as big and set their counters appropriately. + * sysctl interface affects secondary variables (*_pct), which is then + * "atomically" carried over to the primary variables. Atomic change + * includes taking runqueue lock of all online cpus and re-initiatizing + * their big counter values based on changed criteria. + */ + if ((data == &sysctl_sched_upmigrate_pct || update_min_nice)) { + get_online_cpus(); + pre_big_task_count_change(cpu_online_mask); + } + + set_hmp_defaults(); + + if ((data == &sysctl_sched_upmigrate_pct || update_min_nice)) { + post_big_task_count_change(cpu_online_mask); + put_online_cpus(); + } + +done: + mutex_unlock(&policy_mutex); + return ret; +} + +inline int nr_big_tasks(struct rq *rq) +{ + return rq->hmp_stats.nr_big_tasks; +} + +unsigned int cpu_temp(int cpu) +{ + struct cpu_pwr_stats *per_cpu_info = get_cpu_pwr_stats(); + + if (per_cpu_info) + return per_cpu_info[cpu].temp; + else + return 0; +} + +void init_new_task_load(struct task_struct *p) +{ + int i; + u32 init_load_windows = sched_init_task_load_windows; + u32 init_load_pct = current->init_load_pct; + + p->init_load_pct = 0; + rcu_assign_pointer(p->grp, NULL); + INIT_LIST_HEAD(&p->grp_list); + memset(&p->ravg, 0, sizeof(struct ravg)); + p->cpu_cycles = 0; + + if (init_load_pct) + init_load_windows = div64_u64((u64)init_load_pct * + (u64)sched_ravg_window, 100); + + p->ravg.demand = init_load_windows; + p->ravg.pred_demand = 0; + for (i = 0; i < RAVG_HIST_SIZE_MAX; ++i) + p->ravg.sum_history[i] = init_load_windows; +} + +/* Return task demand in percentage scale */ +unsigned int pct_task_load(struct task_struct *p) +{ + unsigned int load; + + load = div64_u64((u64)task_load(p) * 100, (u64)max_task_load()); + + return load; +} + +/* + * Return total number of tasks "eligible" to run on highest capacity cpu + * + * This is simply nr_big_tasks for cpus which are not of max_capacity and + * nr_running for cpus of max_capacity + */ +unsigned int nr_eligible_big_tasks(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + int nr_big = rq->hmp_stats.nr_big_tasks; + int nr = rq->nr_running; + + if (cpu_max_possible_capacity(cpu) != max_possible_capacity) + return nr_big; + + return nr; +} + +static inline int exiting_task(struct task_struct *p) +{ + return (p->ravg.sum_history[0] == EXITING_TASK_MARKER); +} + +static int __init set_sched_ravg_window(char *str) +{ + unsigned int window_size; + + get_option(&str, &window_size); + + if (window_size < MIN_SCHED_RAVG_WINDOW || + window_size > MAX_SCHED_RAVG_WINDOW) { + WARN_ON(1); + return -EINVAL; + } + + sched_ravg_window = window_size; + return 0; +} + +early_param("sched_ravg_window", set_sched_ravg_window); + +static inline void +update_window_start(struct rq *rq, u64 wallclock) +{ + s64 delta; + int nr_windows; + + delta = wallclock - rq->window_start; + BUG_ON(delta < 0); + if (delta < sched_ravg_window) + return; + + nr_windows = div64_u64(delta, sched_ravg_window); + rq->window_start += (u64)nr_windows * (u64)sched_ravg_window; +} + +#define DIV64_U64_ROUNDUP(X, Y) div64_u64((X) + (Y - 1), Y) + +static inline u64 scale_exec_time(u64 delta, struct rq *rq) +{ + u32 freq; + + freq = cpu_cycles_to_freq(rq->cc.cycles, rq->cc.time); + delta = DIV64_U64_ROUNDUP(delta * freq, max_possible_freq); + delta *= rq->cluster->exec_scale_factor; + delta >>= 10; + + return delta; +} + +static inline int cpu_is_waiting_on_io(struct rq *rq) +{ + if (!sched_io_is_busy) + return 0; + + return atomic_read(&rq->nr_iowait); +} + +/* Does freq_required sufficiently exceed or fall behind cur_freq? */ +static inline int +nearly_same_freq(unsigned int cur_freq, unsigned int freq_required) +{ + int delta = freq_required - cur_freq; + + if (freq_required > cur_freq) + return delta < sysctl_sched_freq_inc_notify; + + delta = -delta; + + return delta < sysctl_sched_freq_dec_notify; +} + +/* Convert busy time to frequency equivalent */ +static inline unsigned int load_to_freq(struct rq *rq, u64 load) +{ + unsigned int freq; + + load = scale_load_to_cpu(load, cpu_of(rq)); + load *= 128; + load = div64_u64(load, max_task_load()); + + freq = load * cpu_max_possible_freq(cpu_of(rq)); + freq /= 128; + + return freq; +} + +static inline struct group_cpu_time * +_group_cpu_time(struct related_thread_group *grp, int cpu); + +/* + * Return load from all related group in given cpu. + * Caller must ensure that related_thread_group_lock is held. + */ +static void _group_load_in_cpu(int cpu, u64 *grp_load, u64 *new_grp_load) +{ + struct related_thread_group *grp; + + for_each_related_thread_group(grp) { + struct group_cpu_time *cpu_time; + + cpu_time = _group_cpu_time(grp, cpu); + *grp_load += cpu_time->prev_runnable_sum; + if (new_grp_load) + *new_grp_load += cpu_time->nt_prev_runnable_sum; + } +} + +/* + * Return load from all related groups in given frequency domain. + * Caller must ensure that related_thread_group_lock is held. + */ +static void group_load_in_freq_domain(struct cpumask *cpus, + u64 *grp_load, u64 *new_grp_load) +{ + struct related_thread_group *grp; + int j; + + for_each_related_thread_group(grp) { + for_each_cpu(j, cpus) { + struct group_cpu_time *cpu_time; + + cpu_time = _group_cpu_time(grp, j); + *grp_load += cpu_time->prev_runnable_sum; + *new_grp_load += cpu_time->nt_prev_runnable_sum; + } + } +} + +/* + * Should scheduler alert governor for changing frequency? + * + * @check_pred - evaluate frequency based on the predictive demand + * @check_groups - add load from all related groups on given cpu + * + * check_groups is set to 1 if a "related" task movement/wakeup is triggering + * the notification check. To avoid "re-aggregation" of demand in such cases, + * we check whether the migrated/woken tasks demand (along with demand from + * existing tasks on the cpu) can be met on target cpu + * + */ + +static int send_notification(struct rq *rq, int check_pred, int check_groups) +{ + unsigned int cur_freq, freq_required; + unsigned long flags; + int rc = 0; + u64 group_load = 0, new_load = 0; + + if (check_pred) { + u64 prev = rq->old_busy_time; + u64 predicted = rq->hmp_stats.pred_demands_sum; + + if (rq->cluster->cur_freq == cpu_max_freq(cpu_of(rq))) + return 0; + + prev = max(prev, rq->old_estimated_time); + if (prev > predicted) + return 0; + + cur_freq = load_to_freq(rq, prev); + freq_required = load_to_freq(rq, predicted); + + if (freq_required < cur_freq + sysctl_sched_pred_alert_freq) + return 0; + } else { + read_lock(&related_thread_group_lock); + /* + * Protect from concurrent update of rq->prev_runnable_sum and + * group cpu load + */ + raw_spin_lock_irqsave(&rq->lock, flags); + if (check_groups) + _group_load_in_cpu(cpu_of(rq), &group_load, NULL); + + new_load = rq->prev_runnable_sum + group_load; + + raw_spin_unlock_irqrestore(&rq->lock, flags); + read_unlock(&related_thread_group_lock); + + cur_freq = load_to_freq(rq, rq->old_busy_time); + freq_required = load_to_freq(rq, new_load); + + if (nearly_same_freq(cur_freq, freq_required)) + return 0; + } + + raw_spin_lock_irqsave(&rq->lock, flags); + if (!rq->cluster->notifier_sent) { + rq->cluster->notifier_sent = 1; + rc = 1; + trace_sched_freq_alert(cpu_of(rq), check_pred, check_groups, rq, + new_load); + } + raw_spin_unlock_irqrestore(&rq->lock, flags); + + return rc; +} + +/* Alert governor if there is a need to change frequency */ +void check_for_freq_change(struct rq *rq, bool check_pred, bool check_groups) +{ + int cpu = cpu_of(rq); + + if (!send_notification(rq, check_pred, check_groups)) + return; + + atomic_notifier_call_chain( + &load_alert_notifier_head, 0, + (void *)(long)cpu); +} + +void notify_migration(int src_cpu, int dest_cpu, bool src_cpu_dead, + struct task_struct *p) +{ + bool check_groups; + + rcu_read_lock(); + check_groups = task_in_related_thread_group(p); + rcu_read_unlock(); + + if (!same_freq_domain(src_cpu, dest_cpu)) { + if (!src_cpu_dead) + check_for_freq_change(cpu_rq(src_cpu), false, + check_groups); + check_for_freq_change(cpu_rq(dest_cpu), false, check_groups); + } else { + check_for_freq_change(cpu_rq(dest_cpu), true, check_groups); + } +} + +static int account_busy_for_cpu_time(struct rq *rq, struct task_struct *p, + u64 irqtime, int event) +{ + if (is_idle_task(p)) { + /* TASK_WAKE && TASK_MIGRATE is not possible on idle task! */ + if (event == PICK_NEXT_TASK) + return 0; + + /* PUT_PREV_TASK, TASK_UPDATE && IRQ_UPDATE are left */ + return irqtime || cpu_is_waiting_on_io(rq); + } + + if (event == TASK_WAKE) + return 0; + + if (event == PUT_PREV_TASK || event == IRQ_UPDATE) + return 1; + + /* + * TASK_UPDATE can be called on sleeping task, when its moved between + * related groups + */ + if (event == TASK_UPDATE) { + if (rq->curr == p) + return 1; + + return p->on_rq ? SCHED_FREQ_ACCOUNT_WAIT_TIME : 0; + } + + /* TASK_MIGRATE, PICK_NEXT_TASK left */ + return SCHED_FREQ_ACCOUNT_WAIT_TIME; +} + +static inline bool is_new_task(struct task_struct *p) +{ + return p->ravg.active_windows < sysctl_sched_new_task_windows; +} + +#define INC_STEP 8 +#define DEC_STEP 2 +#define CONSISTENT_THRES 16 +#define INC_STEP_BIG 16 +/* + * bucket_increase - update the count of all buckets + * + * @buckets: array of buckets tracking busy time of a task + * @idx: the index of bucket to be incremented + * + * Each time a complete window finishes, count of bucket that runtime + * falls in (@idx) is incremented. Counts of all other buckets are + * decayed. The rate of increase and decay could be different based + * on current count in the bucket. + */ +static inline void bucket_increase(u8 *buckets, int idx) +{ + int i, step; + + for (i = 0; i < NUM_BUSY_BUCKETS; i++) { + if (idx != i) { + if (buckets[i] > DEC_STEP) + buckets[i] -= DEC_STEP; + else + buckets[i] = 0; + } else { + step = buckets[i] >= CONSISTENT_THRES ? + INC_STEP_BIG : INC_STEP; + if (buckets[i] > U8_MAX - step) + buckets[i] = U8_MAX; + else + buckets[i] += step; + } + } +} + +static inline int busy_to_bucket(u32 normalized_rt) +{ + int bidx; + + bidx = mult_frac(normalized_rt, NUM_BUSY_BUCKETS, max_task_load()); + bidx = min(bidx, NUM_BUSY_BUCKETS - 1); + + /* + * Combine lowest two buckets. The lowest frequency falls into + * 2nd bucket and thus keep predicting lowest bucket is not + * useful. + */ + if (!bidx) + bidx++; + + return bidx; +} + +static inline u64 +scale_load_to_freq(u64 load, unsigned int src_freq, unsigned int dst_freq) +{ + return div64_u64(load * (u64)src_freq, (u64)dst_freq); +} + +#define HEAVY_TASK_SKIP 2 +#define HEAVY_TASK_SKIP_LIMIT 4 +/* + * get_pred_busy - calculate predicted demand for a task on runqueue + * + * @rq: runqueue of task p + * @p: task whose prediction is being updated + * @start: starting bucket. returned prediction should not be lower than + * this bucket. + * @runtime: runtime of the task. returned prediction should not be lower + * than this runtime. + * Note: @start can be derived from @runtime. It's passed in only to + * avoid duplicated calculation in some cases. + * + * A new predicted busy time is returned for task @p based on @runtime + * passed in. The function searches through buckets that represent busy + * time equal to or bigger than @runtime and attempts to find the bucket to + * to use for prediction. Once found, it searches through historical busy + * time and returns the latest that falls into the bucket. If no such busy + * time exists, it returns the medium of that bucket. + */ +static u32 get_pred_busy(struct rq *rq, struct task_struct *p, + int start, u32 runtime) +{ + int i; + u8 *buckets = p->ravg.busy_buckets; + u32 *hist = p->ravg.sum_history; + u32 dmin, dmax; + u64 cur_freq_runtime = 0; + int first = NUM_BUSY_BUCKETS, final, skip_to; + u32 ret = runtime; + + /* skip prediction for new tasks due to lack of history */ + if (unlikely(is_new_task(p))) + goto out; + + /* find minimal bucket index to pick */ + for (i = start; i < NUM_BUSY_BUCKETS; i++) { + if (buckets[i]) { + first = i; + break; + } + } + /* if no higher buckets are filled, predict runtime */ + if (first >= NUM_BUSY_BUCKETS) + goto out; + + /* compute the bucket for prediction */ + final = first; + if (first < HEAVY_TASK_SKIP_LIMIT) { + /* compute runtime at current CPU frequency */ + cur_freq_runtime = mult_frac(runtime, max_possible_efficiency, + rq->cluster->efficiency); + cur_freq_runtime = scale_load_to_freq(cur_freq_runtime, + max_possible_freq, rq->cluster->cur_freq); + /* + * if the task runs for majority of the window, try to + * pick higher buckets. + */ + if (cur_freq_runtime >= sched_major_task_runtime) { + int next = NUM_BUSY_BUCKETS; + /* + * if there is a higher bucket that's consistently + * hit, don't jump beyond that. + */ + for (i = start + 1; i <= HEAVY_TASK_SKIP_LIMIT && + i < NUM_BUSY_BUCKETS; i++) { + if (buckets[i] > CONSISTENT_THRES) { + next = i; + break; + } + } + skip_to = min(next, start + HEAVY_TASK_SKIP); + /* don't jump beyond HEAVY_TASK_SKIP_LIMIT */ + skip_to = min(HEAVY_TASK_SKIP_LIMIT, skip_to); + /* don't go below first non-empty bucket, if any */ + final = max(first, skip_to); + } + } + + /* determine demand range for the predicted bucket */ + if (final < 2) { + /* lowest two buckets are combined */ + dmin = 0; + final = 1; + } else { + dmin = mult_frac(final, max_task_load(), NUM_BUSY_BUCKETS); + } + dmax = mult_frac(final + 1, max_task_load(), NUM_BUSY_BUCKETS); + + /* + * search through runtime history and return first runtime that falls + * into the range of predicted bucket. + */ + for (i = 0; i < sched_ravg_hist_size; i++) { + if (hist[i] >= dmin && hist[i] < dmax) { + ret = hist[i]; + break; + } + } + /* no historical runtime within bucket found, use average of the bin */ + if (ret < dmin) + ret = (dmin + dmax) / 2; + /* + * when updating in middle of a window, runtime could be higher + * than all recorded history. Always predict at least runtime. + */ + ret = max(runtime, ret); +out: + trace_sched_update_pred_demand(rq, p, runtime, + mult_frac((unsigned int)cur_freq_runtime, 100, + sched_ravg_window), ret); + return ret; +} + +static inline u32 calc_pred_demand(struct rq *rq, struct task_struct *p) +{ + if (p->ravg.pred_demand >= p->ravg.curr_window) + return p->ravg.pred_demand; + + return get_pred_busy(rq, p, busy_to_bucket(p->ravg.curr_window), + p->ravg.curr_window); +} + +/* + * predictive demand of a task is calculated at the window roll-over. + * if the task current window busy time exceeds the predicted + * demand, update it here to reflect the task needs. + */ +void update_task_pred_demand(struct rq *rq, struct task_struct *p, int event) +{ + u32 new, old; + + if (is_idle_task(p) || exiting_task(p)) + return; + + if (event != PUT_PREV_TASK && event != TASK_UPDATE && + (!SCHED_FREQ_ACCOUNT_WAIT_TIME || + (event != TASK_MIGRATE && + event != PICK_NEXT_TASK))) + return; + + /* + * TASK_UPDATE can be called on sleeping task, when its moved between + * related groups + */ + if (event == TASK_UPDATE) { + if (!p->on_rq && !SCHED_FREQ_ACCOUNT_WAIT_TIME) + return; + } + + new = calc_pred_demand(rq, p); + old = p->ravg.pred_demand; + + if (old >= new) + return; + + if (task_on_rq_queued(p) && (!task_has_dl_policy(p) || + !p->dl.dl_throttled)) + p->sched_class->fixup_hmp_sched_stats(rq, p, + p->ravg.demand, + new); + + p->ravg.pred_demand = new; +} + +/* + * Account cpu activity in its busy time counters (rq->curr/prev_runnable_sum) + */ +static void update_cpu_busy_time(struct task_struct *p, struct rq *rq, + int event, u64 wallclock, u64 irqtime) +{ + int new_window, full_window = 0; + int p_is_curr_task = (p == rq->curr); + u64 mark_start = p->ravg.mark_start; + u64 window_start = rq->window_start; + u32 window_size = sched_ravg_window; + u64 delta; + u64 *curr_runnable_sum = &rq->curr_runnable_sum; + u64 *prev_runnable_sum = &rq->prev_runnable_sum; + u64 *nt_curr_runnable_sum = &rq->nt_curr_runnable_sum; + u64 *nt_prev_runnable_sum = &rq->nt_prev_runnable_sum; + int flip_counters = 0; + int prev_sum_reset = 0; + bool new_task; + struct related_thread_group *grp; + + new_window = mark_start < window_start; + if (new_window) { + full_window = (window_start - mark_start) >= window_size; + if (p->ravg.active_windows < USHRT_MAX) + p->ravg.active_windows++; + } + + new_task = is_new_task(p); + + grp = p->grp; + if (grp && sched_freq_aggregate) { + /* cpu_time protected by rq_lock */ + struct group_cpu_time *cpu_time = + _group_cpu_time(grp, cpu_of(rq)); + + curr_runnable_sum = &cpu_time->curr_runnable_sum; + prev_runnable_sum = &cpu_time->prev_runnable_sum; + + nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + + if (cpu_time->window_start != rq->window_start) { + int nr_windows; + + delta = rq->window_start - cpu_time->window_start; + nr_windows = div64_u64(delta, window_size); + if (nr_windows > 1) + prev_sum_reset = 1; + + cpu_time->window_start = rq->window_start; + flip_counters = 1; + } + + if (p_is_curr_task && new_window) { + u64 curr_sum = rq->curr_runnable_sum; + u64 nt_curr_sum = rq->nt_curr_runnable_sum; + + if (full_window) + curr_sum = nt_curr_sum = 0; + + rq->prev_runnable_sum = curr_sum; + rq->nt_prev_runnable_sum = nt_curr_sum; + + rq->curr_runnable_sum = 0; + rq->nt_curr_runnable_sum = 0; + } + } else { + if (p_is_curr_task && new_window) { + flip_counters = 1; + if (full_window) + prev_sum_reset = 1; + } + } + + /* + * Handle per-task window rollover. We don't care about the idle + * task or exiting tasks. + */ + if (new_window && !is_idle_task(p) && !exiting_task(p)) { + u32 curr_window = 0; + + if (!full_window) + curr_window = p->ravg.curr_window; + + p->ravg.prev_window = curr_window; + p->ravg.curr_window = 0; + } + + if (flip_counters) { + u64 curr_sum = *curr_runnable_sum; + u64 nt_curr_sum = *nt_curr_runnable_sum; + + if (prev_sum_reset) + curr_sum = nt_curr_sum = 0; + + *prev_runnable_sum = curr_sum; + *nt_prev_runnable_sum = nt_curr_sum; + + *curr_runnable_sum = 0; + *nt_curr_runnable_sum = 0; + } + + if (!account_busy_for_cpu_time(rq, p, irqtime, event)) { + /* + * account_busy_for_cpu_time() = 0, so no update to the + * task's current window needs to be made. This could be + * for example + * + * - a wakeup event on a task within the current + * window (!new_window below, no action required), + * - switching to a new task from idle (PICK_NEXT_TASK) + * in a new window where irqtime is 0 and we aren't + * waiting on IO + */ + + if (!new_window) + return; + + /* + * A new window has started. The RQ demand must be rolled + * over if p is the current task. + */ + if (p_is_curr_task) { + /* p is idle task */ + BUG_ON(p != rq->idle); + } + + return; + } + + if (!new_window) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. No rollover + * since we didn't start a new window. An example of this is + * when a task starts execution and then sleeps within the + * same window. + */ + + if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) + delta = wallclock - mark_start; + else + delta = irqtime; + delta = scale_exec_time(delta, rq); + *curr_runnable_sum += delta; + if (new_task) + *nt_curr_runnable_sum += delta; + + if (!is_idle_task(p) && !exiting_task(p)) + p->ravg.curr_window += delta; + + return; + } + + if (!p_is_curr_task) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * has also started, but p is not the current task, so the + * window is not rolled over - just split up and account + * as necessary into curr and prev. The window is only + * rolled over when a new window is processed for the current + * task. + * + * Irqtime can't be accounted by a task that isn't the + * currently running task. + */ + + if (!full_window) { + /* + * A full window hasn't elapsed, account partial + * contribution to previous completed window. + */ + delta = scale_exec_time(window_start - mark_start, rq); + if (!exiting_task(p)) + p->ravg.prev_window += delta; + } else { + /* + * Since at least one full window has elapsed, + * the contribution to the previous window is the + * full window (window_size). + */ + delta = scale_exec_time(window_size, rq); + if (!exiting_task(p)) + p->ravg.prev_window = delta; + } + + *prev_runnable_sum += delta; + if (new_task) + *nt_prev_runnable_sum += delta; + + /* Account piece of busy time in the current window. */ + delta = scale_exec_time(wallclock - window_start, rq); + *curr_runnable_sum += delta; + if (new_task) + *nt_curr_runnable_sum += delta; + + if (!exiting_task(p)) + p->ravg.curr_window = delta; + + return; + } + + if (!irqtime || !is_idle_task(p) || cpu_is_waiting_on_io(rq)) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * has started and p is the current task so rollover is + * needed. If any of these three above conditions are true + * then this busy time can't be accounted as irqtime. + * + * Busy time for the idle task or exiting tasks need not + * be accounted. + * + * An example of this would be a task that starts execution + * and then sleeps once a new window has begun. + */ + + if (!full_window) { + /* + * A full window hasn't elapsed, account partial + * contribution to previous completed window. + */ + delta = scale_exec_time(window_start - mark_start, rq); + if (!is_idle_task(p) && !exiting_task(p)) + p->ravg.prev_window += delta; + } else { + /* + * Since at least one full window has elapsed, + * the contribution to the previous window is the + * full window (window_size). + */ + delta = scale_exec_time(window_size, rq); + if (!is_idle_task(p) && !exiting_task(p)) + p->ravg.prev_window = delta; + } + + /* + * Rollover is done here by overwriting the values in + * prev_runnable_sum and curr_runnable_sum. + */ + *prev_runnable_sum += delta; + if (new_task) + *nt_prev_runnable_sum += delta; + + /* Account piece of busy time in the current window. */ + delta = scale_exec_time(wallclock - window_start, rq); + *curr_runnable_sum += delta; + if (new_task) + *nt_curr_runnable_sum += delta; + + if (!is_idle_task(p) && !exiting_task(p)) + p->ravg.curr_window = delta; + + return; + } + + if (irqtime) { + /* + * account_busy_for_cpu_time() = 1 so busy time needs + * to be accounted to the current window. A new window + * has started and p is the current task so rollover is + * needed. The current task must be the idle task because + * irqtime is not accounted for any other task. + * + * Irqtime will be accounted each time we process IRQ activity + * after a period of idleness, so we know the IRQ busy time + * started at wallclock - irqtime. + */ + + BUG_ON(!is_idle_task(p)); + mark_start = wallclock - irqtime; + + /* + * Roll window over. If IRQ busy time was just in the current + * window then that is all that need be accounted. + */ + if (mark_start > window_start) { + *curr_runnable_sum = scale_exec_time(irqtime, rq); + return; + } + + /* + * The IRQ busy time spanned multiple windows. Process the + * busy time preceding the current window start first. + */ + delta = window_start - mark_start; + if (delta > window_size) + delta = window_size; + delta = scale_exec_time(delta, rq); + *prev_runnable_sum += delta; + + /* Process the remaining IRQ busy time in the current window. */ + delta = wallclock - window_start; + rq->curr_runnable_sum = scale_exec_time(delta, rq); + + return; + } + + BUG(); +} + +static inline u32 predict_and_update_buckets(struct rq *rq, + struct task_struct *p, u32 runtime) { + + int bidx; + u32 pred_demand; + + bidx = busy_to_bucket(runtime); + pred_demand = get_pred_busy(rq, p, bidx, runtime); + bucket_increase(p->ravg.busy_buckets, bidx); + + return pred_demand; +} + +static void update_task_cpu_cycles(struct task_struct *p, int cpu) +{ + if (use_cycle_counter) + p->cpu_cycles = cpu_cycle_counter_cb.get_cpu_cycle_counter(cpu); +} + +static void +update_task_rq_cpu_cycles(struct task_struct *p, struct rq *rq, int event, + u64 wallclock, u64 irqtime) +{ + u64 cur_cycles; + int cpu = cpu_of(rq); + + lockdep_assert_held(&rq->lock); + + if (!use_cycle_counter) { + rq->cc.cycles = cpu_cur_freq(cpu); + rq->cc.time = 1; + return; + } + + cur_cycles = cpu_cycle_counter_cb.get_cpu_cycle_counter(cpu); + + /* + * If current task is idle task and irqtime == 0 CPU was + * indeed idle and probably its cycle counter was not + * increasing. We still need estimatied CPU frequency + * for IO wait time accounting. Use the previously + * calculated frequency in such a case. + */ + if (!is_idle_task(rq->curr) || irqtime) { + if (unlikely(cur_cycles < p->cpu_cycles)) + rq->cc.cycles = cur_cycles + (U64_MAX - p->cpu_cycles); + else + rq->cc.cycles = cur_cycles - p->cpu_cycles; + rq->cc.cycles = rq->cc.cycles * NSEC_PER_MSEC; + + if (event == IRQ_UPDATE && is_idle_task(p)) + /* + * Time between mark_start of idle task and IRQ handler + * entry time is CPU cycle counter stall period. + * Upon IRQ handler entry sched_account_irqstart() + * replenishes idle task's cpu cycle counter so + * rq->cc.cycles now represents increased cycles during + * IRQ handler rather than time between idle entry and + * IRQ exit. Thus use irqtime as time delta. + */ + rq->cc.time = irqtime; + else + rq->cc.time = wallclock - p->ravg.mark_start; + BUG_ON((s64)rq->cc.time < 0); + } + + p->cpu_cycles = cur_cycles; + + trace_sched_get_task_cpu_cycles(cpu, event, rq->cc.cycles, rq->cc.time); +} + +static int account_busy_for_task_demand(struct task_struct *p, int event) +{ + /* + * No need to bother updating task demand for exiting tasks + * or the idle task. + */ + if (exiting_task(p) || is_idle_task(p)) + return 0; + + /* + * When a task is waking up it is completing a segment of non-busy + * time. Likewise, if wait time is not treated as busy time, then + * when a task begins to run or is migrated, it is not running and + * is completing a segment of non-busy time. + */ + if (event == TASK_WAKE || (!SCHED_ACCOUNT_WAIT_TIME && + (event == PICK_NEXT_TASK || event == TASK_MIGRATE))) + return 0; + + return 1; +} + +/* + * Called when new window is starting for a task, to record cpu usage over + * recently concluded window(s). Normally 'samples' should be 1. It can be > 1 + * when, say, a real-time task runs without preemption for several windows at a + * stretch. + */ +static void update_history(struct rq *rq, struct task_struct *p, + u32 runtime, int samples, int event) +{ + u32 *hist = &p->ravg.sum_history[0]; + int ridx, widx; + u32 max = 0, avg, demand, pred_demand; + u64 sum = 0; + + /* Ignore windows where task had no activity */ + if (!runtime || is_idle_task(p) || exiting_task(p) || !samples) + goto done; + + /* Push new 'runtime' value onto stack */ + widx = sched_ravg_hist_size - 1; + ridx = widx - samples; + for (; ridx >= 0; --widx, --ridx) { + hist[widx] = hist[ridx]; + sum += hist[widx]; + if (hist[widx] > max) + max = hist[widx]; + } + + for (widx = 0; widx < samples && widx < sched_ravg_hist_size; widx++) { + hist[widx] = runtime; + sum += hist[widx]; + if (hist[widx] > max) + max = hist[widx]; + } + + p->ravg.sum = 0; + + if (sched_window_stats_policy == WINDOW_STATS_RECENT) { + demand = runtime; + } else if (sched_window_stats_policy == WINDOW_STATS_MAX) { + demand = max; + } else { + avg = div64_u64(sum, sched_ravg_hist_size); + if (sched_window_stats_policy == WINDOW_STATS_AVG) + demand = avg; + else + demand = max(avg, runtime); + } + pred_demand = predict_and_update_buckets(rq, p, runtime); + + /* + * A throttled deadline sched class task gets dequeued without + * changing p->on_rq. Since the dequeue decrements hmp stats + * avoid decrementing it here again. + */ + if (task_on_rq_queued(p) && (!task_has_dl_policy(p) || + !p->dl.dl_throttled)) + p->sched_class->fixup_hmp_sched_stats(rq, p, demand, + pred_demand); + + p->ravg.demand = demand; + p->ravg.pred_demand = pred_demand; + +done: + trace_sched_update_history(rq, p, runtime, samples, event); +} + +static void add_to_task_demand(struct rq *rq, struct task_struct *p, u64 delta) +{ + delta = scale_exec_time(delta, rq); + p->ravg.sum += delta; + if (unlikely(p->ravg.sum > sched_ravg_window)) + p->ravg.sum = sched_ravg_window; +} + +/* + * Account cpu demand of task and/or update task's cpu demand history + * + * ms = p->ravg.mark_start; + * wc = wallclock + * ws = rq->window_start + * + * Three possibilities: + * + * a) Task event is contained within one window. + * window_start < mark_start < wallclock + * + * ws ms wc + * | | | + * V V V + * |---------------| + * + * In this case, p->ravg.sum is updated *iff* event is appropriate + * (ex: event == PUT_PREV_TASK) + * + * b) Task event spans two windows. + * mark_start < window_start < wallclock + * + * ms ws wc + * | | | + * V V V + * -----|------------------- + * + * In this case, p->ravg.sum is updated with (ws - ms) *iff* event + * is appropriate, then a new window sample is recorded followed + * by p->ravg.sum being set to (wc - ws) *iff* event is appropriate. + * + * c) Task event spans more than two windows. + * + * ms ws_tmp ws wc + * | | | | + * V V V V + * ---|-------|-------|-------|-------|------ + * | | + * |<------ nr_full_windows ------>| + * + * In this case, p->ravg.sum is updated with (ws_tmp - ms) first *iff* + * event is appropriate, window sample of p->ravg.sum is recorded, + * 'nr_full_window' samples of window_size is also recorded *iff* + * event is appropriate and finally p->ravg.sum is set to (wc - ws) + * *iff* event is appropriate. + * + * IMPORTANT : Leave p->ravg.mark_start unchanged, as update_cpu_busy_time() + * depends on it! + */ +static void update_task_demand(struct task_struct *p, struct rq *rq, + int event, u64 wallclock) +{ + u64 mark_start = p->ravg.mark_start; + u64 delta, window_start = rq->window_start; + int new_window, nr_full_windows; + u32 window_size = sched_ravg_window; + + new_window = mark_start < window_start; + if (!account_busy_for_task_demand(p, event)) { + if (new_window) + /* + * If the time accounted isn't being accounted as + * busy time, and a new window started, only the + * previous window need be closed out with the + * pre-existing demand. Multiple windows may have + * elapsed, but since empty windows are dropped, + * it is not necessary to account those. + */ + update_history(rq, p, p->ravg.sum, 1, event); + return; + } + + if (!new_window) { + /* + * The simple case - busy time contained within the existing + * window. + */ + add_to_task_demand(rq, p, wallclock - mark_start); + return; + } + + /* + * Busy time spans at least two windows. Temporarily rewind + * window_start to first window boundary after mark_start. + */ + delta = window_start - mark_start; + nr_full_windows = div64_u64(delta, window_size); + window_start -= (u64)nr_full_windows * (u64)window_size; + + /* Process (window_start - mark_start) first */ + add_to_task_demand(rq, p, window_start - mark_start); + + /* Push new sample(s) into task's demand history */ + update_history(rq, p, p->ravg.sum, 1, event); + if (nr_full_windows) + update_history(rq, p, scale_exec_time(window_size, rq), + nr_full_windows, event); + + /* + * Roll window_start back to current to process any remainder + * in current window. + */ + window_start += (u64)nr_full_windows * (u64)window_size; + + /* Process (wallclock - window_start) next */ + mark_start = window_start; + add_to_task_demand(rq, p, wallclock - mark_start); +} + +/* Reflect task activity on its demand and cpu's busy time statistics */ +void update_task_ravg(struct task_struct *p, struct rq *rq, int event, + u64 wallclock, u64 irqtime) +{ + if (!rq->window_start || sched_disable_window_stats) + return; + + lockdep_assert_held(&rq->lock); + + update_window_start(rq, wallclock); + + if (!p->ravg.mark_start) { + update_task_cpu_cycles(p, cpu_of(rq)); + goto done; + } + + update_task_rq_cpu_cycles(p, rq, event, wallclock, irqtime); + update_task_demand(p, rq, event, wallclock); + update_cpu_busy_time(p, rq, event, wallclock, irqtime); + update_task_pred_demand(rq, p, event); +done: + trace_sched_update_task_ravg(p, rq, event, wallclock, irqtime, + rq->cc.cycles, rq->cc.time, + _group_cpu_time(p->grp, cpu_of(rq))); + + p->ravg.mark_start = wallclock; +} + +void sched_account_irqtime(int cpu, struct task_struct *curr, + u64 delta, u64 wallclock) +{ + struct rq *rq = cpu_rq(cpu); + unsigned long flags, nr_windows; + u64 cur_jiffies_ts; + + raw_spin_lock_irqsave(&rq->lock, flags); + + /* + * cputime (wallclock) uses sched_clock so use the same here for + * consistency. + */ + delta += sched_clock() - wallclock; + cur_jiffies_ts = get_jiffies_64(); + + if (is_idle_task(curr)) + update_task_ravg(curr, rq, IRQ_UPDATE, sched_ktime_clock(), + delta); + + nr_windows = cur_jiffies_ts - rq->irqload_ts; + + if (nr_windows) { + if (nr_windows < 10) { + /* Decay CPU's irqload by 3/4 for each window. */ + rq->avg_irqload *= (3 * nr_windows); + rq->avg_irqload = div64_u64(rq->avg_irqload, + 4 * nr_windows); + } else { + rq->avg_irqload = 0; + } + rq->avg_irqload += rq->cur_irqload; + rq->cur_irqload = 0; + } + + rq->cur_irqload += delta; + rq->irqload_ts = cur_jiffies_ts; + raw_spin_unlock_irqrestore(&rq->lock, flags); +} + +void sched_account_irqstart(int cpu, struct task_struct *curr, u64 wallclock) +{ + struct rq *rq = cpu_rq(cpu); + + if (!rq->window_start || sched_disable_window_stats) + return; + + if (is_idle_task(curr)) { + /* We're here without rq->lock held, IRQ disabled */ + raw_spin_lock(&rq->lock); + update_task_cpu_cycles(curr, cpu); + raw_spin_unlock(&rq->lock); + } +} + +void reset_task_stats(struct task_struct *p) +{ + u32 sum = 0; + + if (exiting_task(p)) + sum = EXITING_TASK_MARKER; + + memset(&p->ravg, 0, sizeof(struct ravg)); + /* Retain EXITING_TASK marker */ + p->ravg.sum_history[0] = sum; +} + +void mark_task_starting(struct task_struct *p) +{ + u64 wallclock; + struct rq *rq = task_rq(p); + + if (!rq->window_start || sched_disable_window_stats) { + reset_task_stats(p); + return; + } + + wallclock = sched_ktime_clock(); + p->ravg.mark_start = p->last_wake_ts = wallclock; + p->last_cpu_selected_ts = wallclock; + p->last_switch_out_ts = 0; + update_task_cpu_cycles(p, cpu_of(rq)); +} + +void set_window_start(struct rq *rq) +{ + int cpu = cpu_of(rq); + struct rq *sync_rq = cpu_rq(sync_cpu); + + if (rq->window_start) + return; + + if (cpu == sync_cpu) { + rq->window_start = sched_ktime_clock(); + } else { + raw_spin_unlock(&rq->lock); + double_rq_lock(rq, sync_rq); + rq->window_start = cpu_rq(sync_cpu)->window_start; + rq->curr_runnable_sum = rq->prev_runnable_sum = 0; + rq->nt_curr_runnable_sum = rq->nt_prev_runnable_sum = 0; + raw_spin_unlock(&sync_rq->lock); + } + + rq->curr->ravg.mark_start = rq->window_start; +} + +void migrate_sync_cpu(int cpu) +{ + if (cpu == sync_cpu) + sync_cpu = smp_processor_id(); +} + +static void reset_all_task_stats(void) +{ + struct task_struct *g, *p; + + read_lock(&tasklist_lock); + do_each_thread(g, p) { + reset_task_stats(p); + } while_each_thread(g, p); + read_unlock(&tasklist_lock); +} + +static void disable_window_stats(void) +{ + unsigned long flags; + int i; + + local_irq_save(flags); + for_each_possible_cpu(i) + raw_spin_lock(&cpu_rq(i)->lock); + + sched_disable_window_stats = 1; + + for_each_possible_cpu(i) + raw_spin_unlock(&cpu_rq(i)->lock); + + local_irq_restore(flags); +} + +/* Called with all cpu's rq->lock held */ +static void enable_window_stats(void) +{ + sched_disable_window_stats = 0; + +} + +enum reset_reason_code { + WINDOW_CHANGE, + POLICY_CHANGE, + HIST_SIZE_CHANGE, + FREQ_AGGREGATE_CHANGE, +}; + +const char *sched_window_reset_reasons[] = { + "WINDOW_CHANGE", + "POLICY_CHANGE", + "HIST_SIZE_CHANGE", +}; + +/* Called with IRQs enabled */ +void reset_all_window_stats(u64 window_start, unsigned int window_size) +{ + int cpu; + unsigned long flags; + u64 start_ts = sched_ktime_clock(); + int reason = WINDOW_CHANGE; + unsigned int old = 0, new = 0; + struct related_thread_group *grp; + + disable_window_stats(); + + reset_all_task_stats(); + + local_irq_save(flags); + + read_lock(&related_thread_group_lock); + + for_each_possible_cpu(cpu) + raw_spin_lock(&cpu_rq(cpu)->lock); + + list_for_each_entry(grp, &related_thread_groups, list) { + int j; + + for_each_possible_cpu(j) { + struct group_cpu_time *cpu_time; + /* Protected by rq lock */ + cpu_time = _group_cpu_time(grp, j); + memset(cpu_time, 0, sizeof(struct group_cpu_time)); + if (window_start) + cpu_time->window_start = window_start; + } + } + + if (window_size) { + sched_ravg_window = window_size * TICK_NSEC; + set_hmp_defaults(); + } + + enable_window_stats(); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + if (window_start) + rq->window_start = window_start; + rq->curr_runnable_sum = rq->prev_runnable_sum = 0; + rq->nt_curr_runnable_sum = rq->nt_prev_runnable_sum = 0; + reset_cpu_hmp_stats(cpu, 1); + } + + if (sched_window_stats_policy != sysctl_sched_window_stats_policy) { + reason = POLICY_CHANGE; + old = sched_window_stats_policy; + new = sysctl_sched_window_stats_policy; + sched_window_stats_policy = sysctl_sched_window_stats_policy; + } else if (sched_ravg_hist_size != sysctl_sched_ravg_hist_size) { + reason = HIST_SIZE_CHANGE; + old = sched_ravg_hist_size; + new = sysctl_sched_ravg_hist_size; + sched_ravg_hist_size = sysctl_sched_ravg_hist_size; + } else if (sched_freq_aggregate != + sysctl_sched_freq_aggregate) { + reason = FREQ_AGGREGATE_CHANGE; + old = sched_freq_aggregate; + new = sysctl_sched_freq_aggregate; + sched_freq_aggregate = sysctl_sched_freq_aggregate; + } + + for_each_possible_cpu(cpu) + raw_spin_unlock(&cpu_rq(cpu)->lock); + + read_unlock(&related_thread_group_lock); + + local_irq_restore(flags); + + trace_sched_reset_all_window_stats(window_start, window_size, + sched_ktime_clock() - start_ts, reason, old, new); +} + +static inline void +sync_window_start(struct rq *rq, struct group_cpu_time *cpu_time); + +void sched_get_cpus_busy(struct sched_load *busy, + const struct cpumask *query_cpus) +{ + unsigned long flags; + struct rq *rq; + const int cpus = cpumask_weight(query_cpus); + u64 load[cpus], group_load[cpus]; + u64 nload[cpus], ngload[cpus]; + u64 pload[cpus]; + unsigned int cur_freq[cpus], max_freq[cpus]; + int notifier_sent = 0; + int early_detection[cpus]; + int cpu, i = 0; + unsigned int window_size; + u64 max_prev_sum = 0; + int max_busy_cpu = cpumask_first(query_cpus); + struct related_thread_group *grp; + u64 total_group_load = 0, total_ngload = 0; + bool aggregate_load = false; + + if (unlikely(cpus == 0)) + return; + + /* + * This function could be called in timer context, and the + * current task may have been executing for a long time. Ensure + * that the window stats are current by doing an update. + */ + read_lock(&related_thread_group_lock); + + local_irq_save(flags); + for_each_cpu(cpu, query_cpus) + raw_spin_lock(&cpu_rq(cpu)->lock); + + window_size = sched_ravg_window; + + for_each_cpu(cpu, query_cpus) { + rq = cpu_rq(cpu); + + update_task_ravg(rq->curr, rq, TASK_UPDATE, sched_ktime_clock(), + 0); + cur_freq[i] = cpu_cycles_to_freq(rq->cc.cycles, rq->cc.time); + + load[i] = rq->old_busy_time = rq->prev_runnable_sum; + nload[i] = rq->nt_prev_runnable_sum; + pload[i] = rq->hmp_stats.pred_demands_sum; + rq->old_estimated_time = pload[i]; + + if (load[i] > max_prev_sum) { + max_prev_sum = load[i]; + max_busy_cpu = cpu; + } + + /* + * sched_get_cpus_busy() is called for all CPUs in a + * frequency domain. So the notifier_sent flag per + * cluster works even when a frequency domain spans + * more than 1 cluster. + */ + if (rq->cluster->notifier_sent) { + notifier_sent = 1; + rq->cluster->notifier_sent = 0; + } + early_detection[i] = (rq->ed_task != NULL); + cur_freq[i] = cpu_cur_freq(cpu); + max_freq[i] = cpu_max_freq(cpu); + i++; + } + + for_each_related_thread_group(grp) { + for_each_cpu(cpu, query_cpus) { + /* Protected by rq_lock */ + struct group_cpu_time *cpu_time = + _group_cpu_time(grp, cpu); + sync_window_start(cpu_rq(cpu), cpu_time); + } + } + + group_load_in_freq_domain( + &cpu_rq(max_busy_cpu)->freq_domain_cpumask, + &total_group_load, &total_ngload); + aggregate_load = !!(total_group_load > sched_freq_aggregate_threshold); + + i = 0; + for_each_cpu(cpu, query_cpus) { + group_load[i] = 0; + ngload[i] = 0; + + if (early_detection[i]) + goto skip_early; + + rq = cpu_rq(cpu); + if (aggregate_load) { + if (cpu == max_busy_cpu) { + group_load[i] = total_group_load; + ngload[i] = total_ngload; + } + } else { + _group_load_in_cpu(cpu, &group_load[i], &ngload[i]); + } + + load[i] += group_load[i]; + nload[i] += ngload[i]; + /* + * Scale load in reference to cluster max_possible_freq. + * + * Note that scale_load_to_cpu() scales load in reference to + * the cluster max_freq. + */ + load[i] = scale_load_to_cpu(load[i], cpu); + nload[i] = scale_load_to_cpu(nload[i], cpu); + pload[i] = scale_load_to_cpu(pload[i], cpu); +skip_early: + i++; + } + + for_each_cpu(cpu, query_cpus) + raw_spin_unlock(&(cpu_rq(cpu))->lock); + local_irq_restore(flags); + + read_unlock(&related_thread_group_lock); + + i = 0; + for_each_cpu(cpu, query_cpus) { + rq = cpu_rq(cpu); + + if (early_detection[i]) { + busy[i].prev_load = div64_u64(sched_ravg_window, + NSEC_PER_USEC); + busy[i].new_task_load = 0; + goto exit_early; + } + + /* + * When the load aggregation is controlled by + * sched_freq_aggregate_threshold, allow reporting loads + * greater than 100 @ Fcur to ramp up the frequency + * faster. + */ + if (notifier_sent || (aggregate_load && + sched_freq_aggregate_threshold)) { + load[i] = scale_load_to_freq(load[i], max_freq[i], + cpu_max_possible_freq(cpu)); + nload[i] = scale_load_to_freq(nload[i], max_freq[i], + cpu_max_possible_freq(cpu)); + } else { + load[i] = scale_load_to_freq(load[i], max_freq[i], + cur_freq[i]); + nload[i] = scale_load_to_freq(nload[i], max_freq[i], + cur_freq[i]); + if (load[i] > window_size) + load[i] = window_size; + if (nload[i] > window_size) + nload[i] = window_size; + + load[i] = scale_load_to_freq(load[i], cur_freq[i], + cpu_max_possible_freq(cpu)); + nload[i] = scale_load_to_freq(nload[i], cur_freq[i], + cpu_max_possible_freq(cpu)); + } + pload[i] = scale_load_to_freq(pload[i], max_freq[i], + rq->cluster->max_possible_freq); + + busy[i].prev_load = div64_u64(load[i], NSEC_PER_USEC); + busy[i].new_task_load = div64_u64(nload[i], NSEC_PER_USEC); + busy[i].predicted_load = div64_u64(pload[i], NSEC_PER_USEC); + +exit_early: + trace_sched_get_busy(cpu, busy[i].prev_load, + busy[i].new_task_load, + busy[i].predicted_load, + early_detection[i]); + i++; + } +} + +void sched_set_io_is_busy(int val) +{ + sched_io_is_busy = val; +} + +int sched_set_window(u64 window_start, unsigned int window_size) +{ + u64 now, cur_jiffies, jiffy_ktime_ns; + s64 ws; + unsigned long flags; + + if (window_size * TICK_NSEC < MIN_SCHED_RAVG_WINDOW) + return -EINVAL; + + mutex_lock(&policy_mutex); + + /* + * Get a consistent view of ktime, jiffies, and the time + * since the last jiffy (based on last_jiffies_update). + */ + local_irq_save(flags); + cur_jiffies = jiffy_to_ktime_ns(&now, &jiffy_ktime_ns); + local_irq_restore(flags); + + /* translate window_start from jiffies to nanoseconds */ + ws = (window_start - cur_jiffies); /* jiffy difference */ + ws *= TICK_NSEC; + ws += jiffy_ktime_ns; + + /* + * Roll back calculated window start so that it is in + * the past (window stats must have a current window). + */ + while (ws > now) + ws -= (window_size * TICK_NSEC); + + BUG_ON(sched_ktime_clock() < ws); + + reset_all_window_stats(ws, window_size); + + sched_update_freq_max_load(cpu_possible_mask); + + mutex_unlock(&policy_mutex); + + return 0; +} + +void fixup_busy_time(struct task_struct *p, int new_cpu) +{ + struct rq *src_rq = task_rq(p); + struct rq *dest_rq = cpu_rq(new_cpu); + u64 wallclock; + u64 *src_curr_runnable_sum, *dst_curr_runnable_sum; + u64 *src_prev_runnable_sum, *dst_prev_runnable_sum; + u64 *src_nt_curr_runnable_sum, *dst_nt_curr_runnable_sum; + u64 *src_nt_prev_runnable_sum, *dst_nt_prev_runnable_sum; + int migrate_type; + struct migration_sum_data d; + bool new_task; + struct related_thread_group *grp; + + if (!p->on_rq && p->state != TASK_WAKING) + return; + + if (exiting_task(p)) { + clear_ed_task(p, src_rq); + return; + } + + if (p->state == TASK_WAKING) + double_rq_lock(src_rq, dest_rq); + + if (sched_disable_window_stats) + goto done; + + wallclock = sched_ktime_clock(); + + update_task_ravg(task_rq(p)->curr, task_rq(p), + TASK_UPDATE, + wallclock, 0); + update_task_ravg(dest_rq->curr, dest_rq, + TASK_UPDATE, wallclock, 0); + + update_task_ravg(p, task_rq(p), TASK_MIGRATE, + wallclock, 0); + + update_task_cpu_cycles(p, new_cpu); + + new_task = is_new_task(p); + /* Protected by rq_lock */ + grp = p->grp; + if (grp && sched_freq_aggregate) { + struct group_cpu_time *cpu_time; + + migrate_type = GROUP_TO_GROUP; + /* Protected by rq_lock */ + cpu_time = _group_cpu_time(grp, cpu_of(src_rq)); + d.src_rq = NULL; + d.src_cpu_time = cpu_time; + src_curr_runnable_sum = &cpu_time->curr_runnable_sum; + src_prev_runnable_sum = &cpu_time->prev_runnable_sum; + src_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + src_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + + /* Protected by rq_lock */ + cpu_time = _group_cpu_time(grp, cpu_of(dest_rq)); + d.dst_rq = NULL; + d.dst_cpu_time = cpu_time; + dst_curr_runnable_sum = &cpu_time->curr_runnable_sum; + dst_prev_runnable_sum = &cpu_time->prev_runnable_sum; + dst_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + dst_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + sync_window_start(dest_rq, cpu_time); + } else { + migrate_type = RQ_TO_RQ; + d.src_rq = src_rq; + d.src_cpu_time = NULL; + d.dst_rq = dest_rq; + d.dst_cpu_time = NULL; + src_curr_runnable_sum = &src_rq->curr_runnable_sum; + src_prev_runnable_sum = &src_rq->prev_runnable_sum; + src_nt_curr_runnable_sum = &src_rq->nt_curr_runnable_sum; + src_nt_prev_runnable_sum = &src_rq->nt_prev_runnable_sum; + + dst_curr_runnable_sum = &dest_rq->curr_runnable_sum; + dst_prev_runnable_sum = &dest_rq->prev_runnable_sum; + dst_nt_curr_runnable_sum = &dest_rq->nt_curr_runnable_sum; + dst_nt_prev_runnable_sum = &dest_rq->nt_prev_runnable_sum; + } + + if (p->ravg.curr_window) { + *src_curr_runnable_sum -= p->ravg.curr_window; + *dst_curr_runnable_sum += p->ravg.curr_window; + if (new_task) { + *src_nt_curr_runnable_sum -= p->ravg.curr_window; + *dst_nt_curr_runnable_sum += p->ravg.curr_window; + } + } + + if (p->ravg.prev_window) { + *src_prev_runnable_sum -= p->ravg.prev_window; + *dst_prev_runnable_sum += p->ravg.prev_window; + if (new_task) { + *src_nt_prev_runnable_sum -= p->ravg.prev_window; + *dst_nt_prev_runnable_sum += p->ravg.prev_window; + } + } + + if (p == src_rq->ed_task) { + src_rq->ed_task = NULL; + if (!dest_rq->ed_task) + dest_rq->ed_task = p; + } + + trace_sched_migration_update_sum(p, migrate_type, &d); + BUG_ON((s64)*src_prev_runnable_sum < 0); + BUG_ON((s64)*src_curr_runnable_sum < 0); + BUG_ON((s64)*src_nt_prev_runnable_sum < 0); + BUG_ON((s64)*src_nt_curr_runnable_sum < 0); + +done: + if (p->state == TASK_WAKING) + double_rq_unlock(src_rq, dest_rq); +} + +#define sched_up_down_migrate_auto_update 1 +static void check_for_up_down_migrate_update(const struct cpumask *cpus) +{ + int i = cpumask_first(cpus); + + if (!sched_up_down_migrate_auto_update) + return; + + if (cpu_max_possible_capacity(i) == max_possible_capacity) + return; + + if (cpu_max_possible_freq(i) == cpu_max_freq(i)) + up_down_migrate_scale_factor = 1024; + else + up_down_migrate_scale_factor = (1024 * + cpu_max_possible_freq(i)) / cpu_max_freq(i); + + update_up_down_migrate(); +} + +/* Return cluster which can offer required capacity for group */ +static struct sched_cluster * +best_cluster(struct related_thread_group *grp, u64 total_demand) +{ + struct sched_cluster *cluster = NULL; + + for_each_sched_cluster(cluster) { + if (group_will_fit(cluster, grp, total_demand)) + return cluster; + } + + return NULL; +} + +static void _set_preferred_cluster(struct related_thread_group *grp) +{ + struct task_struct *p; + u64 combined_demand = 0; + + if (!sysctl_sched_enable_colocation) { + grp->last_update = sched_ktime_clock(); + grp->preferred_cluster = NULL; + return; + } + + /* + * wakeup of two or more related tasks could race with each other and + * could result in multiple calls to _set_preferred_cluster being issued + * at same time. Avoid overhead in such cases of rechecking preferred + * cluster + */ + if (sched_ktime_clock() - grp->last_update < sched_ravg_window / 10) + return; + + list_for_each_entry(p, &grp->tasks, grp_list) + combined_demand += p->ravg.demand; + + grp->preferred_cluster = best_cluster(grp, combined_demand); + grp->last_update = sched_ktime_clock(); + trace_sched_set_preferred_cluster(grp, combined_demand); +} + +void set_preferred_cluster(struct related_thread_group *grp) +{ + raw_spin_lock(&grp->lock); + _set_preferred_cluster(grp); + raw_spin_unlock(&grp->lock); +} + +#define ADD_TASK 0 +#define REM_TASK 1 + +static inline void free_group_cputime(struct related_thread_group *grp) +{ + free_percpu(grp->cpu_time); +} + +static int alloc_group_cputime(struct related_thread_group *grp) +{ + int i; + struct group_cpu_time *cpu_time; + int cpu = raw_smp_processor_id(); + struct rq *rq = cpu_rq(cpu); + u64 window_start = rq->window_start; + + grp->cpu_time = alloc_percpu(struct group_cpu_time); + if (!grp->cpu_time) + return -ENOMEM; + + for_each_possible_cpu(i) { + cpu_time = per_cpu_ptr(grp->cpu_time, i); + memset(cpu_time, 0, sizeof(struct group_cpu_time)); + cpu_time->window_start = window_start; + } + + return 0; +} + +/* + * A group's window_start may be behind. When moving it forward, flip prev/curr + * counters. When moving forward > 1 window, prev counter is set to 0 + */ +static inline void +sync_window_start(struct rq *rq, struct group_cpu_time *cpu_time) +{ + u64 delta; + int nr_windows; + u64 curr_sum = cpu_time->curr_runnable_sum; + u64 nt_curr_sum = cpu_time->nt_curr_runnable_sum; + + delta = rq->window_start - cpu_time->window_start; + if (!delta) + return; + + nr_windows = div64_u64(delta, sched_ravg_window); + if (nr_windows > 1) + curr_sum = nt_curr_sum = 0; + + cpu_time->prev_runnable_sum = curr_sum; + cpu_time->curr_runnable_sum = 0; + + cpu_time->nt_prev_runnable_sum = nt_curr_sum; + cpu_time->nt_curr_runnable_sum = 0; + + cpu_time->window_start = rq->window_start; +} + +/* + * Task's cpu usage is accounted in: + * rq->curr/prev_runnable_sum, when its ->grp is NULL + * grp->cpu_time[cpu]->curr/prev_runnable_sum, when its ->grp is !NULL + * + * Transfer task's cpu usage between those counters when transitioning between + * groups + */ +static void transfer_busy_time(struct rq *rq, struct related_thread_group *grp, + struct task_struct *p, int event) +{ + u64 wallclock; + struct group_cpu_time *cpu_time; + u64 *src_curr_runnable_sum, *dst_curr_runnable_sum; + u64 *src_prev_runnable_sum, *dst_prev_runnable_sum; + u64 *src_nt_curr_runnable_sum, *dst_nt_curr_runnable_sum; + u64 *src_nt_prev_runnable_sum, *dst_nt_prev_runnable_sum; + struct migration_sum_data d; + int migrate_type; + + if (!sched_freq_aggregate) + return; + + wallclock = sched_ktime_clock(); + + update_task_ravg(rq->curr, rq, TASK_UPDATE, wallclock, 0); + update_task_ravg(p, rq, TASK_UPDATE, wallclock, 0); + + /* cpu_time protected by related_thread_group_lock, grp->lock rq_lock */ + cpu_time = _group_cpu_time(grp, cpu_of(rq)); + if (event == ADD_TASK) { + sync_window_start(rq, cpu_time); + migrate_type = RQ_TO_GROUP; + d.src_rq = rq; + d.src_cpu_time = NULL; + d.dst_rq = NULL; + d.dst_cpu_time = cpu_time; + src_curr_runnable_sum = &rq->curr_runnable_sum; + dst_curr_runnable_sum = &cpu_time->curr_runnable_sum; + src_prev_runnable_sum = &rq->prev_runnable_sum; + dst_prev_runnable_sum = &cpu_time->prev_runnable_sum; + + src_nt_curr_runnable_sum = &rq->nt_curr_runnable_sum; + dst_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + src_nt_prev_runnable_sum = &rq->nt_prev_runnable_sum; + dst_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + } else { + migrate_type = GROUP_TO_RQ; + d.src_rq = NULL; + d.src_cpu_time = cpu_time; + d.dst_rq = rq; + d.dst_cpu_time = NULL; + + /* + * In case of REM_TASK, cpu_time->window_start would be + * uptodate, because of the update_task_ravg() we called + * above on the moving task. Hence no need for + * sync_window_start() + */ + src_curr_runnable_sum = &cpu_time->curr_runnable_sum; + dst_curr_runnable_sum = &rq->curr_runnable_sum; + src_prev_runnable_sum = &cpu_time->prev_runnable_sum; + dst_prev_runnable_sum = &rq->prev_runnable_sum; + + src_nt_curr_runnable_sum = &cpu_time->nt_curr_runnable_sum; + dst_nt_curr_runnable_sum = &rq->nt_curr_runnable_sum; + src_nt_prev_runnable_sum = &cpu_time->nt_prev_runnable_sum; + dst_nt_prev_runnable_sum = &rq->nt_prev_runnable_sum; + } + + *src_curr_runnable_sum -= p->ravg.curr_window; + *dst_curr_runnable_sum += p->ravg.curr_window; + + *src_prev_runnable_sum -= p->ravg.prev_window; + *dst_prev_runnable_sum += p->ravg.prev_window; + + if (is_new_task(p)) { + *src_nt_curr_runnable_sum -= p->ravg.curr_window; + *dst_nt_curr_runnable_sum += p->ravg.curr_window; + *src_nt_prev_runnable_sum -= p->ravg.prev_window; + *dst_nt_prev_runnable_sum += p->ravg.prev_window; + } + + trace_sched_migration_update_sum(p, migrate_type, &d); + + BUG_ON((s64)*src_curr_runnable_sum < 0); + BUG_ON((s64)*src_prev_runnable_sum < 0); +} + +static inline struct group_cpu_time * +task_group_cpu_time(struct task_struct *p, int cpu) +{ + return _group_cpu_time(rcu_dereference(p->grp), cpu); +} + +static inline struct group_cpu_time * +_group_cpu_time(struct related_thread_group *grp, int cpu) +{ + return grp ? per_cpu_ptr(grp->cpu_time, cpu) : NULL; +} + +struct related_thread_group *alloc_related_thread_group(int group_id) +{ + struct related_thread_group *grp; + + grp = kzalloc(sizeof(*grp), GFP_KERNEL); + if (!grp) + return ERR_PTR(-ENOMEM); + + if (alloc_group_cputime(grp)) { + kfree(grp); + return ERR_PTR(-ENOMEM); + } + + grp->id = group_id; + INIT_LIST_HEAD(&grp->tasks); + INIT_LIST_HEAD(&grp->list); + raw_spin_lock_init(&grp->lock); + + return grp; +} + +struct related_thread_group *lookup_related_thread_group(unsigned int group_id) +{ + struct related_thread_group *grp; + + list_for_each_entry(grp, &related_thread_groups, list) { + if (grp->id == group_id) + return grp; + } + + return NULL; +} + +/* See comments before preferred_cluster() */ +static void free_related_thread_group(struct rcu_head *rcu) +{ + struct related_thread_group *grp = container_of(rcu, struct + related_thread_group, rcu); + + free_group_cputime(grp); + kfree(grp); +} + +static void remove_task_from_group(struct task_struct *p) +{ + struct related_thread_group *grp = p->grp; + struct rq *rq; + int empty_group = 1; + struct rq_flags rf; + + raw_spin_lock(&grp->lock); + + rq = __task_rq_lock(p, &rf); + transfer_busy_time(rq, p->grp, p, REM_TASK); + list_del_init(&p->grp_list); + rcu_assign_pointer(p->grp, NULL); + __task_rq_unlock(rq, &rf); + + if (!list_empty(&grp->tasks)) { + empty_group = 0; + _set_preferred_cluster(grp); + } + + raw_spin_unlock(&grp->lock); + + if (empty_group) { + list_del(&grp->list); + call_rcu(&grp->rcu, free_related_thread_group); + } +} + +static int +add_task_to_group(struct task_struct *p, struct related_thread_group *grp) +{ + struct rq *rq; + struct rq_flags rf; + + raw_spin_lock(&grp->lock); + + /* + * Change p->grp under rq->lock. Will prevent races with read-side + * reference of p->grp in various hot-paths + */ + rq = __task_rq_lock(p, &rf); + transfer_busy_time(rq, grp, p, ADD_TASK); + list_add(&p->grp_list, &grp->tasks); + rcu_assign_pointer(p->grp, grp); + __task_rq_unlock(rq, &rf); + + _set_preferred_cluster(grp); + + raw_spin_unlock(&grp->lock); + + return 0; +} + +void add_new_task_to_grp(struct task_struct *new) +{ + unsigned long flags; + struct related_thread_group *grp; + struct task_struct *parent; + + if (!sysctl_sched_enable_thread_grouping) + return; + + if (thread_group_leader(new)) + return; + + parent = new->group_leader; + + /* + * The parent's pi_lock is required here to protect race + * against the parent task being removed from the + * group. + */ + raw_spin_lock_irqsave(&parent->pi_lock, flags); + + /* protected by pi_lock. */ + grp = task_related_thread_group(parent); + if (!grp) { + raw_spin_unlock_irqrestore(&parent->pi_lock, flags); + return; + } + raw_spin_lock(&grp->lock); + + rcu_assign_pointer(new->grp, grp); + list_add(&new->grp_list, &grp->tasks); + + raw_spin_unlock(&grp->lock); + raw_spin_unlock_irqrestore(&parent->pi_lock, flags); +} + +int sched_set_group_id(struct task_struct *p, unsigned int group_id) +{ + int rc = 0, destroy = 0; + unsigned long flags; + struct related_thread_group *grp = NULL, *new = NULL; + +redo: + raw_spin_lock_irqsave(&p->pi_lock, flags); + + if ((current != p && p->flags & PF_EXITING) || + (!p->grp && !group_id) || + (p->grp && p->grp->id == group_id)) + goto done; + + write_lock(&related_thread_group_lock); + + if (!group_id) { + remove_task_from_group(p); + write_unlock(&related_thread_group_lock); + goto done; + } + + if (p->grp && p->grp->id != group_id) + remove_task_from_group(p); + + grp = lookup_related_thread_group(group_id); + if (!grp && !new) { + /* New group */ + write_unlock(&related_thread_group_lock); + raw_spin_unlock_irqrestore(&p->pi_lock, flags); + new = alloc_related_thread_group(group_id); + if (IS_ERR(new)) + return -ENOMEM; + destroy = 1; + /* Rerun checks (like task exiting), since we dropped pi_lock */ + goto redo; + } else if (!grp && new) { + /* New group - use object allocated before */ + destroy = 0; + list_add(&new->list, &related_thread_groups); + grp = new; + } + + BUG_ON(!grp); + rc = add_task_to_group(p, grp); + write_unlock(&related_thread_group_lock); +done: + raw_spin_unlock_irqrestore(&p->pi_lock, flags); + + if (new && destroy) { + free_group_cputime(new); + kfree(new); + } + + return rc; +} + +unsigned int sched_get_group_id(struct task_struct *p) +{ + unsigned int group_id; + struct related_thread_group *grp; + + rcu_read_lock(); + grp = task_related_thread_group(p); + group_id = grp ? grp->id : 0; + rcu_read_unlock(); + + return group_id; +} + +static void update_cpu_cluster_capacity(const cpumask_t *cpus) +{ + int i; + struct sched_cluster *cluster; + struct cpumask cpumask; + + cpumask_copy(&cpumask, cpus); + pre_big_task_count_change(cpu_possible_mask); + + for_each_cpu(i, &cpumask) { + cluster = cpu_rq(i)->cluster; + cpumask_andnot(&cpumask, &cpumask, &cluster->cpus); + + cluster->capacity = compute_capacity(cluster); + cluster->load_scale_factor = compute_load_scale_factor(cluster); + + /* 'cpus' can contain cpumask more than one cluster */ + check_for_up_down_migrate_update(&cluster->cpus); + } + + __update_min_max_capacity(); + + post_big_task_count_change(cpu_possible_mask); +} + +static DEFINE_SPINLOCK(cpu_freq_min_max_lock); +void sched_update_cpu_freq_min_max(const cpumask_t *cpus, u32 fmin, u32 fmax) +{ + struct cpumask cpumask; + struct sched_cluster *cluster; + int i, update_capacity = 0; + unsigned long flags; + + spin_lock_irqsave(&cpu_freq_min_max_lock, flags); + cpumask_copy(&cpumask, cpus); + for_each_cpu(i, &cpumask) { + cluster = cpu_rq(i)->cluster; + cpumask_andnot(&cpumask, &cpumask, &cluster->cpus); + + update_capacity += (cluster->max_mitigated_freq != fmax); + cluster->max_mitigated_freq = fmax; + } + spin_unlock_irqrestore(&cpu_freq_min_max_lock, flags); + + if (update_capacity) + update_cpu_cluster_capacity(cpus); +} + +static int cpufreq_notifier_policy(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_policy *policy = (struct cpufreq_policy *)data; + struct sched_cluster *cluster = NULL; + struct cpumask policy_cluster = *policy->related_cpus; + unsigned int orig_max_freq = 0; + int i, j, update_capacity = 0; + + if (val != CPUFREQ_NOTIFY && val != CPUFREQ_REMOVE_POLICY && + val != CPUFREQ_CREATE_POLICY) + return 0; + + if (val == CPUFREQ_REMOVE_POLICY || val == CPUFREQ_CREATE_POLICY) { + update_min_max_capacity(); + return 0; + } + + max_possible_freq = max(max_possible_freq, policy->cpuinfo.max_freq); + if (min_max_freq == 1) + min_max_freq = UINT_MAX; + min_max_freq = min(min_max_freq, policy->cpuinfo.max_freq); + BUG_ON(!min_max_freq); + BUG_ON(!policy->max); + + for_each_cpu(i, &policy_cluster) { + cluster = cpu_rq(i)->cluster; + cpumask_andnot(&policy_cluster, &policy_cluster, + &cluster->cpus); + + orig_max_freq = cluster->max_freq; + cluster->min_freq = policy->min; + cluster->max_freq = policy->max; + cluster->cur_freq = policy->cur; + + if (!cluster->freq_init_done) { + mutex_lock(&cluster_lock); + for_each_cpu(j, &cluster->cpus) + cpumask_copy(&cpu_rq(j)->freq_domain_cpumask, + policy->related_cpus); + cluster->max_possible_freq = policy->cpuinfo.max_freq; + cluster->max_possible_capacity = + compute_max_possible_capacity(cluster); + cluster->freq_init_done = true; + + sort_clusters(); + update_all_clusters_stats(); + mutex_unlock(&cluster_lock); + continue; + } + + update_capacity += (orig_max_freq != cluster->max_freq); + } + + if (update_capacity) + update_cpu_cluster_capacity(policy->related_cpus); + + return 0; +} + +static int cpufreq_notifier_trans(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freq = (struct cpufreq_freqs *)data; + unsigned int cpu = freq->cpu, new_freq = freq->new; + unsigned long flags; + struct sched_cluster *cluster; + struct cpumask policy_cpus = cpu_rq(cpu)->freq_domain_cpumask; + int i, j; + + if (val != CPUFREQ_POSTCHANGE) + return 0; + + BUG_ON(!new_freq); + + if (cpu_cur_freq(cpu) == new_freq) + return 0; + + for_each_cpu(i, &policy_cpus) { + cluster = cpu_rq(i)->cluster; + + for_each_cpu(j, &cluster->cpus) { + struct rq *rq = cpu_rq(j); + + raw_spin_lock_irqsave(&rq->lock, flags); + update_task_ravg(rq->curr, rq, TASK_UPDATE, + sched_ktime_clock(), 0); + raw_spin_unlock_irqrestore(&rq->lock, flags); + } + + cluster->cur_freq = new_freq; + cpumask_andnot(&policy_cpus, &policy_cpus, &cluster->cpus); + } + + return 0; +} + +static int pwr_stats_ready_notifier(struct notifier_block *nb, + unsigned long cpu, void *data) +{ + cpumask_t mask = CPU_MASK_NONE; + + cpumask_set_cpu(cpu, &mask); + sched_update_freq_max_load(&mask); + + mutex_lock(&cluster_lock); + sort_clusters(); + mutex_unlock(&cluster_lock); + + return 0; +} + +static struct notifier_block notifier_policy_block = { + .notifier_call = cpufreq_notifier_policy +}; + +static struct notifier_block notifier_trans_block = { + .notifier_call = cpufreq_notifier_trans +}; + +static struct notifier_block notifier_pwr_stats_ready = { + .notifier_call = pwr_stats_ready_notifier +}; + +int __weak register_cpu_pwr_stats_ready_notifier(struct notifier_block *nb) +{ + return -EINVAL; +} + +static int register_sched_callback(void) +{ + int ret; + + ret = cpufreq_register_notifier(¬ifier_policy_block, + CPUFREQ_POLICY_NOTIFIER); + + if (!ret) + ret = cpufreq_register_notifier(¬ifier_trans_block, + CPUFREQ_TRANSITION_NOTIFIER); + + register_cpu_pwr_stats_ready_notifier(¬ifier_pwr_stats_ready); + + return 0; +} + +/* + * cpufreq callbacks can be registered at core_initcall or later time. + * Any registration done prior to that is "forgotten" by cpufreq. See + * initialization of variable init_cpufreq_transition_notifier_list_called + * for further information. + */ +core_initcall(register_sched_callback); + +int update_preferred_cluster(struct related_thread_group *grp, + struct task_struct *p, u32 old_load) +{ + u32 new_load = task_load(p); + + if (!grp) + return 0; + + /* + * Update if task's load has changed significantly or a complete window + * has passed since we last updated preference + */ + if (abs(new_load - old_load) > sched_ravg_window / 4 || + sched_ktime_clock() - grp->last_update > sched_ravg_window) + return 1; + + return 0; +} + +bool early_detection_notify(struct rq *rq, u64 wallclock) +{ + struct task_struct *p; + int loop_max = 10; + + if (!sched_boost() || !rq->cfs.h_nr_running) + return 0; + + rq->ed_task = NULL; + list_for_each_entry(p, &rq->cfs_tasks, se.group_node) { + if (!loop_max) + break; + + if (wallclock - p->last_wake_ts >= EARLY_DETECTION_DURATION) { + rq->ed_task = p; + return 1; + } + + loop_max--; + } + + return 0; +} + +#ifdef CONFIG_CGROUP_SCHED +u64 cpu_upmigrate_discourage_read_u64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct task_group *tg = css_tg(css); + + return tg->upmigrate_discouraged; +} + +int cpu_upmigrate_discourage_write_u64(struct cgroup_subsys_state *css, + struct cftype *cft, u64 upmigrate_discourage) +{ + struct task_group *tg = css_tg(css); + int discourage = upmigrate_discourage > 0; + + if (tg->upmigrate_discouraged == discourage) + return 0; + + /* + * Revisit big-task classification for tasks of this cgroup. It would + * have been efficient to walk tasks of just this cgroup in running + * state, but we don't have easy means to do that. Walk all tasks in + * running state on all cpus instead and re-visit their big task + * classification. + */ + get_online_cpus(); + pre_big_task_count_change(cpu_online_mask); + + tg->upmigrate_discouraged = discourage; + + post_big_task_count_change(cpu_online_mask); + put_online_cpus(); + + return 0; +} +#endif /* CONFIG_CGROUP_SCHED */ diff --git a/kernel/sched/idle_task.c b/kernel/sched/idle_task.c index 5405d3feb112..13c881832f6d 100644 --- a/kernel/sched/idle_task.c +++ b/kernel/sched/idle_task.c @@ -78,6 +78,14 @@ static void update_curr_idle(struct rq *rq) { } +#ifdef CONFIG_SCHED_HMP +static void +fixup_hmp_sched_stats_idle(struct rq *rq, struct task_struct *p, + u32 new_task_load, u32 new_pred_demand) +{ +} +#endif + /* * Simple, special scheduling class for the per-CPU idle tasks: */ @@ -106,4 +114,7 @@ const struct sched_class idle_sched_class = { .prio_changed = prio_changed_idle, .switched_to = switched_to_idle, .update_curr = update_curr_idle, +#ifdef CONFIG_SCHED_HMP + .fixup_hmp_sched_stats = fixup_hmp_sched_stats_idle, +#endif }; diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 2516b8df6dbb..a791486cdb49 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -7,6 +7,7 @@ #include #include +#include int sched_rr_timeslice = RR_TIMESLICE; @@ -897,6 +898,51 @@ static inline int rt_se_prio(struct sched_rt_entity *rt_se) return rt_task_of(rt_se)->prio; } +static void dump_throttled_rt_tasks(struct rt_rq *rt_rq) +{ + struct rt_prio_array *array = &rt_rq->active; + struct sched_rt_entity *rt_se; + char buf[500]; + char *pos = buf; + char *end = buf + sizeof(buf); + int idx; + + pos += snprintf(pos, sizeof(buf), + "sched: RT throttling activated for rt_rq %p (cpu %d)\n", + rt_rq, cpu_of(rq_of_rt_rq(rt_rq))); + + if (bitmap_empty(array->bitmap, MAX_RT_PRIO)) + goto out; + + pos += snprintf(pos, end - pos, "potential CPU hogs:\n"); + idx = sched_find_first_bit(array->bitmap); + while (idx < MAX_RT_PRIO) { + list_for_each_entry(rt_se, array->queue + idx, run_list) { + struct task_struct *p; + + if (!rt_entity_is_task(rt_se)) + continue; + + p = rt_task_of(rt_se); + if (pos < end) + pos += snprintf(pos, end - pos, "\t%s (%d)\n", + p->comm, p->pid); + } + idx = find_next_bit(array->bitmap, MAX_RT_PRIO, idx + 1); + } +out: +#ifdef CONFIG_PANIC_ON_RT_THROTTLING + /* + * Use pr_err() in the BUG() case since printk_sched() will + * not get flushed and deadlock is not a concern. + */ + pr_err("%s", buf); + BUG(); +#else + printk_deferred("%s", buf); +#endif +} + static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) { u64 runtime = sched_rt_runtime(rt_rq); @@ -920,8 +966,14 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq) * but accrue some time due to boosting. */ if (likely(rt_b->rt_runtime)) { + static bool once; + rt_rq->rt_throttled = 1; - printk_deferred_once("sched: RT throttling activated\n"); + + if (!once) { + once = true; + dump_throttled_rt_tasks(rt_rq); + } } else { /* * In case we did anyway, make it go away, @@ -1141,6 +1193,41 @@ void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {} #endif /* CONFIG_RT_GROUP_SCHED */ +#ifdef CONFIG_SCHED_HMP + +static void +inc_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) +{ + inc_cumulative_runnable_avg(&rq->hmp_stats, p); +} + +static void +dec_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) +{ + dec_cumulative_runnable_avg(&rq->hmp_stats, p); +} + +static void +fixup_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p, + u32 new_task_load, u32 new_pred_demand) +{ + s64 task_load_delta = (s64)new_task_load - task_load(p); + s64 pred_demand_delta = PRED_DEMAND_DELTA; + + fixup_cumulative_runnable_avg(&rq->hmp_stats, p, task_load_delta, + pred_demand_delta); +} + +#else /* CONFIG_SCHED_HMP */ + +static inline void +inc_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) { } + +static inline void +dec_hmp_sched_stats_rt(struct rq *rq, struct task_struct *p) { } + +#endif /* CONFIG_SCHED_HMP */ + static inline unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se) { @@ -1322,6 +1409,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags) rt_se->timeout = 0; enqueue_rt_entity(rt_se, flags); + inc_hmp_sched_stats_rt(rq, p); if (!task_current(rq, p) && tsk_nr_cpus_allowed(p) > 1) enqueue_pushable_task(rq, p); @@ -1333,6 +1421,7 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags) update_curr_rt(rq); dequeue_rt_entity(rt_se, flags); + dec_hmp_sched_stats_rt(rq, p); dequeue_pushable_task(rq, p); } @@ -1374,12 +1463,32 @@ static void yield_task_rt(struct rq *rq) #ifdef CONFIG_SMP static int find_lowest_rq(struct task_struct *task); +#ifdef CONFIG_SCHED_HMP +static int +select_task_rq_rt_hmp(struct task_struct *p, int cpu, int sd_flag, int flags) +{ + int target; + + rcu_read_lock(); + target = find_lowest_rq(p); + if (target != -1) + cpu = target; + rcu_read_unlock(); + + return cpu; +} +#endif /* CONFIG_SCHED_HMP */ + static int select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags) { struct task_struct *curr; struct rq *rq; +#ifdef CONFIG_SCHED_HMP + return select_task_rq_rt_hmp(p, cpu, sd_flag, flags); +#endif + /* For anything but wake ups, just return the task_cpu */ if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK) goto out; @@ -1617,6 +1726,65 @@ static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu) static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); +#ifdef CONFIG_SCHED_HMP +static int find_lowest_rq_hmp(struct task_struct *task) +{ + struct cpumask *lowest_mask = *this_cpu_ptr(&local_cpu_mask); + struct cpumask candidate_mask = CPU_MASK_NONE; + struct sched_cluster *cluster; + int best_cpu = -1; + int prev_cpu = task_cpu(task); + u64 cpu_load, min_load = ULLONG_MAX; + int i; + int restrict_cluster = sched_boost() ? 0 : + sysctl_sched_restrict_cluster_spill; + + /* Make sure the mask is initialized first */ + if (unlikely(!lowest_mask)) + return best_cpu; + + if (task->nr_cpus_allowed == 1) + return best_cpu; /* No other targets possible */ + + if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask)) + return best_cpu; /* No targets found */ + + /* + * At this point we have built a mask of cpus representing the + * lowest priority tasks in the system. Now we want to elect + * the best one based on our affinity and topology. + */ + + for_each_sched_cluster(cluster) { + cpumask_and(&candidate_mask, &cluster->cpus, lowest_mask); + + if (cpumask_empty(&candidate_mask)) + continue; + + for_each_cpu(i, &candidate_mask) { + if (sched_cpu_high_irqload(i)) + continue; + + cpu_load = cpu_rq(i)->hmp_stats.cumulative_runnable_avg; + if (!restrict_cluster) + cpu_load = scale_load_to_cpu(cpu_load, i); + + if (cpu_load < min_load || + (cpu_load == min_load && + (i == prev_cpu || (best_cpu != prev_cpu && + cpus_share_cache(prev_cpu, i))))) { + min_load = cpu_load; + best_cpu = i; + } + } + if (restrict_cluster && best_cpu != -1) + break; + } + + return best_cpu; +} +#endif /* CONFIG_SCHED_HMP */ + static int find_lowest_rq(struct task_struct *task) { struct sched_domain *sd; @@ -1624,6 +1792,10 @@ static int find_lowest_rq(struct task_struct *task) int this_cpu = smp_processor_id(); int cpu = task_cpu(task); +#ifdef CONFIG_SCHED_HMP + return find_lowest_rq_hmp(task); +#endif + /* Make sure the mask is initialized first */ if (unlikely(!lowest_mask)) return -1; @@ -1841,9 +2013,11 @@ retry: goto retry; } + next_task->on_rq = TASK_ON_RQ_MIGRATING; deactivate_task(rq, next_task, 0); set_task_cpu(next_task, lowest_rq->cpu); activate_task(lowest_rq, next_task, 0); + next_task->on_rq = TASK_ON_RQ_QUEUED; ret = 1; resched_curr(lowest_rq); @@ -2095,9 +2269,11 @@ static void pull_rt_task(struct rq *this_rq) resched = true; + p->on_rq = TASK_ON_RQ_MIGRATING; deactivate_task(src_rq, p, 0); set_task_cpu(p, this_cpu); activate_task(this_rq, p, 0); + p->on_rq = TASK_ON_RQ_QUEUED; /* * We continue with the search, just in * case there's an even higher prio task @@ -2352,6 +2528,9 @@ const struct sched_class rt_sched_class = { .switched_to = switched_to_rt, .update_curr = update_curr_rt, +#ifdef CONFIG_SCHED_HMP + .fixup_hmp_sched_stats = fixup_hmp_sched_stats_rt, +#endif }; #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 055f935d4421..f6e2bf132d08 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -255,6 +255,10 @@ struct cfs_bandwidth { struct task_group { struct cgroup_subsys_state css; +#ifdef CONFIG_SCHED_HMP + bool upmigrate_discouraged; +#endif + #ifdef CONFIG_FAIR_GROUP_SCHED /* schedulable entities of this group on each cpu */ struct sched_entity **se; @@ -365,12 +369,83 @@ static inline void set_task_rq_fair(struct sched_entity *se, #endif /* CONFIG_SMP */ #endif /* CONFIG_FAIR_GROUP_SCHED */ +extern struct task_group *css_tg(struct cgroup_subsys_state *css); + #else /* CONFIG_CGROUP_SCHED */ struct cfs_bandwidth { }; #endif /* CONFIG_CGROUP_SCHED */ +#ifdef CONFIG_SCHED_HMP + +struct hmp_sched_stats { + int nr_big_tasks; + u64 cumulative_runnable_avg; + u64 pred_demands_sum; +}; + +struct sched_cluster { + struct list_head list; + struct cpumask cpus; + int id; + int max_power_cost; + int min_power_cost; + int max_possible_capacity; + int capacity; + int efficiency; /* Differentiate cpus with different IPC capability */ + int load_scale_factor; + unsigned int exec_scale_factor; + /* + * max_freq = user maximum + * max_mitigated_freq = thermal defined maximum + * max_possible_freq = maximum supported by hardware + */ + unsigned int cur_freq, max_freq, max_mitigated_freq, min_freq; + unsigned int max_possible_freq; + bool freq_init_done; + int dstate, dstate_wakeup_latency, dstate_wakeup_energy; + unsigned int static_cluster_pwr_cost; + int notifier_sent; +}; + +extern unsigned long all_cluster_ids[]; + +static inline int cluster_first_cpu(struct sched_cluster *cluster) +{ + return cpumask_first(&cluster->cpus); +} + +struct related_thread_group { + int id; + raw_spinlock_t lock; + struct list_head tasks; + struct list_head list; + struct sched_cluster *preferred_cluster; + struct rcu_head rcu; + u64 last_update; + struct group_cpu_time __percpu *cpu_time; /* one per cluster */ +}; + +struct migration_sum_data { + struct rq *src_rq, *dst_rq; + struct group_cpu_time *src_cpu_time, *dst_cpu_time; +}; + +extern struct list_head cluster_head; +extern int num_clusters; +extern struct sched_cluster *sched_cluster[NR_CPUS]; + +struct cpu_cycle { + u64 cycles; + u64 time; +}; + +#define for_each_sched_cluster(cluster) \ + list_for_each_entry_rcu(cluster, &cluster_head, list) + +#endif /* CONFIG_SCHED_HMP */ + /* CFS-related fields in a runqueue */ struct cfs_rq { struct load_weight load; @@ -439,6 +514,10 @@ struct cfs_rq { struct task_group *tg; /* group that "owns" this runqueue */ #ifdef CONFIG_CFS_BANDWIDTH +#ifdef CONFIG_SCHED_HMP + struct hmp_sched_stats hmp_stats; +#endif + int runtime_enabled; u64 runtime_expires; s64 runtime_remaining; @@ -656,6 +735,7 @@ struct rq { /* For active balancing */ int active_balance; int push_cpu; + struct task_struct *push_task; struct cpu_stop_work active_balance_work; /* cpu of this runqueue: */ int cpu; @@ -672,6 +752,29 @@ struct rq { u64 max_idle_balance_cost; #endif +#ifdef CONFIG_SCHED_HMP + struct sched_cluster *cluster; + struct cpumask freq_domain_cpumask; + struct hmp_sched_stats hmp_stats; + + int cstate, wakeup_latency, wakeup_energy; + u64 window_start; + unsigned long hmp_flags; + + u64 cur_irqload; + u64 avg_irqload; + u64 irqload_ts; + unsigned int static_cpu_pwr_cost; + struct task_struct *ed_task; + struct cpu_cycle cc; + u64 old_busy_time, old_busy_time_group; + u64 old_estimated_time; + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; +#endif + #ifdef CONFIG_IRQ_TIME_ACCOUNTING u64 prev_irq_time; #endif @@ -962,6 +1065,583 @@ static inline void sched_ttwu_pending(void) { } #include "stats.h" #include "auto_group.h" +#ifdef CONFIG_SCHED_HMP + +#define WINDOW_STATS_RECENT 0 +#define WINDOW_STATS_MAX 1 +#define WINDOW_STATS_MAX_RECENT_AVG 2 +#define WINDOW_STATS_AVG 3 +#define WINDOW_STATS_INVALID_POLICY 4 + +#define MAJOR_TASK_PCT 85 +#define SCHED_UPMIGRATE_MIN_NICE 15 +#define EXITING_TASK_MARKER 0xdeaddead + +#define UP_MIGRATION 1 +#define DOWN_MIGRATION 2 +#define IRQLOAD_MIGRATION 3 + +extern struct mutex policy_mutex; +extern unsigned int sched_ravg_window; +extern unsigned int sched_disable_window_stats; +extern unsigned int max_possible_freq; +extern unsigned int min_max_freq; +extern unsigned int pct_task_load(struct task_struct *p); +extern unsigned int max_possible_efficiency; +extern unsigned int min_possible_efficiency; +extern unsigned int max_capacity; +extern unsigned int min_capacity; +extern unsigned int max_load_scale_factor; +extern unsigned int max_possible_capacity; +extern unsigned int min_max_possible_capacity; +extern unsigned int sched_upmigrate; +extern unsigned int sched_downmigrate; +extern unsigned int sched_init_task_load_windows; +extern unsigned int up_down_migrate_scale_factor; +extern unsigned int sysctl_sched_restrict_cluster_spill; +extern unsigned int sched_pred_alert_load; +extern unsigned int sched_major_task_runtime; +extern struct sched_cluster init_cluster; +extern unsigned int __read_mostly sched_short_sleep_task_threshold; +extern unsigned int __read_mostly sched_long_cpu_selection_threshold; +extern unsigned int __read_mostly sched_big_waker_task_load; +extern unsigned int __read_mostly sched_small_wakee_task_load; +extern unsigned int __read_mostly sched_spill_load; +extern unsigned int __read_mostly sched_upmigrate; +extern unsigned int __read_mostly sched_downmigrate; +extern unsigned int __read_mostly sysctl_sched_spill_nr_run; + +extern void init_new_task_load(struct task_struct *p); +extern u64 sched_ktime_clock(void); +extern int got_boost_kick(void); +extern int register_cpu_cycle_counter_cb(struct cpu_cycle_counter_cb *cb); +extern void update_task_ravg(struct task_struct *p, struct rq *rq, int event, + u64 wallclock, u64 irqtime); +extern bool early_detection_notify(struct rq *rq, u64 wallclock); +extern void clear_ed_task(struct task_struct *p, struct rq *rq); +extern void fixup_busy_time(struct task_struct *p, int new_cpu); +extern void clear_boost_kick(int cpu); +extern void clear_hmp_request(int cpu); +extern void mark_task_starting(struct task_struct *p); +extern void set_window_start(struct rq *rq); +extern void migrate_sync_cpu(int cpu); +extern void update_cluster_topology(void); +extern void set_task_last_wake(struct task_struct *p, u64 wallclock); +extern void set_task_last_switch_out(struct task_struct *p, u64 wallclock); +extern void init_clusters(void); +extern void reset_cpu_hmp_stats(int cpu, int reset_cra); +extern unsigned int max_task_load(void); +extern void sched_account_irqtime(int cpu, struct task_struct *curr, + u64 delta, u64 wallclock); +extern void sched_account_irqstart(int cpu, struct task_struct *curr, + u64 wallclock); +extern unsigned int cpu_temp(int cpu); +extern unsigned int nr_eligible_big_tasks(int cpu); +extern void update_up_down_migrate(void); +extern int update_preferred_cluster(struct related_thread_group *grp, + struct task_struct *p, u32 old_load); +extern void set_preferred_cluster(struct related_thread_group *grp); +extern void add_new_task_to_grp(struct task_struct *new); + +enum sched_boost_type { + SCHED_BOOST_NONE, + SCHED_BOOST_ON_BIG, + SCHED_BOOST_ON_ALL, +}; + +static inline struct sched_cluster *cpu_cluster(int cpu) +{ + return cpu_rq(cpu)->cluster; +} + +static inline int cpu_capacity(int cpu) +{ + return cpu_rq(cpu)->cluster->capacity; +} + +static inline int cpu_max_possible_capacity(int cpu) +{ + return cpu_rq(cpu)->cluster->max_possible_capacity; +} + +static inline int cpu_load_scale_factor(int cpu) +{ + return cpu_rq(cpu)->cluster->load_scale_factor; +} + +static inline int cpu_efficiency(int cpu) +{ + return cpu_rq(cpu)->cluster->efficiency; +} + +static inline unsigned int cpu_cur_freq(int cpu) +{ + return cpu_rq(cpu)->cluster->cur_freq; +} + +static inline unsigned int cpu_min_freq(int cpu) +{ + return cpu_rq(cpu)->cluster->min_freq; +} + +static inline unsigned int cluster_max_freq(struct sched_cluster *cluster) +{ + /* + * Governor and thermal driver don't know the other party's mitigation + * voting. So struct cluster saves both and return min() for current + * cluster fmax. + */ + return min(cluster->max_mitigated_freq, cluster->max_freq); +} + +static inline unsigned int cpu_max_freq(int cpu) +{ + return cluster_max_freq(cpu_rq(cpu)->cluster); +} + +static inline unsigned int cpu_max_possible_freq(int cpu) +{ + return cpu_rq(cpu)->cluster->max_possible_freq; +} + +static inline int same_cluster(int src_cpu, int dst_cpu) +{ + return cpu_rq(src_cpu)->cluster == cpu_rq(dst_cpu)->cluster; +} + +static inline int cpu_max_power_cost(int cpu) +{ + return cpu_rq(cpu)->cluster->max_power_cost; +} + +static inline u32 cpu_cycles_to_freq(u64 cycles, u32 period) +{ + return div64_u64(cycles, period); +} + +static inline bool hmp_capable(void) +{ + return max_possible_capacity != min_max_possible_capacity; +} + +/* + * 'load' is in reference to "best cpu" at its best frequency. + * Scale that in reference to a given cpu, accounting for how bad it is + * in reference to "best cpu". + */ +static inline u64 scale_load_to_cpu(u64 task_load, int cpu) +{ + u64 lsf = cpu_load_scale_factor(cpu); + + if (lsf != 1024) { + task_load *= lsf; + task_load /= 1024; + } + + return task_load; +} + +static inline unsigned int task_load(struct task_struct *p) +{ + return p->ravg.demand; +} + +static inline void +inc_cumulative_runnable_avg(struct hmp_sched_stats *stats, + struct task_struct *p) +{ + u32 task_load; + + if (sched_disable_window_stats) + return; + + task_load = sched_disable_window_stats ? 0 : p->ravg.demand; + + stats->cumulative_runnable_avg += task_load; + stats->pred_demands_sum += p->ravg.pred_demand; +} + +static inline void +dec_cumulative_runnable_avg(struct hmp_sched_stats *stats, + struct task_struct *p) +{ + u32 task_load; + + if (sched_disable_window_stats) + return; + + task_load = sched_disable_window_stats ? 0 : p->ravg.demand; + + stats->cumulative_runnable_avg -= task_load; + + BUG_ON((s64)stats->cumulative_runnable_avg < 0); + + stats->pred_demands_sum -= p->ravg.pred_demand; + BUG_ON((s64)stats->pred_demands_sum < 0); +} + +static inline void +fixup_cumulative_runnable_avg(struct hmp_sched_stats *stats, + struct task_struct *p, s64 task_load_delta, + s64 pred_demand_delta) +{ + if (sched_disable_window_stats) + return; + + stats->cumulative_runnable_avg += task_load_delta; + BUG_ON((s64)stats->cumulative_runnable_avg < 0); + + stats->pred_demands_sum += pred_demand_delta; + BUG_ON((s64)stats->pred_demands_sum < 0); +} + +#define pct_to_real(tunable) \ + (div64_u64((u64)tunable * (u64)max_task_load(), 100)) + +#define real_to_pct(tunable) \ + (div64_u64((u64)tunable * (u64)100, (u64)max_task_load())) + +#define SCHED_HIGH_IRQ_TIMEOUT 3 +static inline u64 sched_irqload(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + s64 delta; + + delta = get_jiffies_64() - rq->irqload_ts; + /* + * Current context can be preempted by irq and rq->irqload_ts can be + * updated by irq context so that delta can be negative. + * But this is okay and we can safely return as this means there + * was recent irq occurrence. + */ + + if (delta < SCHED_HIGH_IRQ_TIMEOUT) + return rq->avg_irqload; + else + return 0; +} + +static inline int sched_cpu_high_irqload(int cpu) +{ + return sched_irqload(cpu) >= sysctl_sched_cpu_high_irqload; +} + +static inline bool task_in_related_thread_group(struct task_struct *p) +{ + return !!(rcu_access_pointer(p->grp) != NULL); +} + +static inline +struct related_thread_group *task_related_thread_group(struct task_struct *p) +{ + return rcu_dereference(p->grp); +} + +#define PRED_DEMAND_DELTA ((s64)new_pred_demand - p->ravg.pred_demand) + +extern void +check_for_freq_change(struct rq *rq, bool check_pred, bool check_groups); + +extern void notify_migration(int src_cpu, int dest_cpu, + bool src_cpu_dead, struct task_struct *p); + +struct group_cpu_time { + u64 curr_runnable_sum; + u64 prev_runnable_sum; + u64 nt_curr_runnable_sum; + u64 nt_prev_runnable_sum; + u64 window_start; +}; + +/* Is frequency of two cpus synchronized with each other? */ +static inline int same_freq_domain(int src_cpu, int dst_cpu) +{ + struct rq *rq = cpu_rq(src_cpu); + + if (src_cpu == dst_cpu) + return 1; + + return cpumask_test_cpu(dst_cpu, &rq->freq_domain_cpumask); +} + +#define BOOST_KICK 0 +#define CPU_RESERVED 1 + +static inline int is_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + return test_bit(CPU_RESERVED, &rq->hmp_flags); +} + +static inline int mark_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + /* Name boost_flags as hmp_flags? */ + return test_and_set_bit(CPU_RESERVED, &rq->hmp_flags); +} + +static inline void clear_reserved(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + clear_bit(CPU_RESERVED, &rq->hmp_flags); +} + +static inline u64 cpu_cravg_sync(int cpu, int sync) +{ + struct rq *rq = cpu_rq(cpu); + u64 load; + + load = rq->hmp_stats.cumulative_runnable_avg; + + /* + * If load is being checked in a sync wakeup environment, + * we may want to discount the load of the currently running + * task. + */ + if (sync && cpu == smp_processor_id()) { + if (load > rq->curr->ravg.demand) + load -= rq->curr->ravg.demand; + else + load = 0; + } + + return load; +} + +extern void check_for_migration(struct rq *rq, struct task_struct *p); +extern void pre_big_task_count_change(const struct cpumask *cpus); +extern void post_big_task_count_change(const struct cpumask *cpus); +extern void set_hmp_defaults(void); +extern int power_delta_exceeded(unsigned int cpu_cost, unsigned int base_cost); +extern unsigned int power_cost(int cpu, u64 demand); +extern void reset_all_window_stats(u64 window_start, unsigned int window_size); +extern void boost_kick(int cpu); +extern int sched_boost(void); +extern int task_load_will_fit(struct task_struct *p, u64 task_load, int cpu, + enum sched_boost_type boost_type); +extern enum sched_boost_type sched_boost_type(void); +extern int task_will_fit(struct task_struct *p, int cpu); +extern int group_will_fit(struct sched_cluster *cluster, + struct related_thread_group *grp, u64 demand); +extern u64 cpu_load(int cpu); +extern u64 cpu_load_sync(int cpu, int sync); +extern int preferred_cluster(struct sched_cluster *cluster, + struct task_struct *p); +extern void inc_nr_big_task(struct hmp_sched_stats *stats, + struct task_struct *p); +extern void dec_nr_big_task(struct hmp_sched_stats *stats, + struct task_struct *p); +extern void inc_rq_hmp_stats(struct rq *rq, + struct task_struct *p, int change_cra); +extern void dec_rq_hmp_stats(struct rq *rq, + struct task_struct *p, int change_cra); +extern int is_big_task(struct task_struct *p); +extern int upmigrate_discouraged(struct task_struct *p); +extern struct sched_cluster *rq_cluster(struct rq *rq); +extern int nr_big_tasks(struct rq *rq); +extern void fixup_nr_big_tasks(struct hmp_sched_stats *stats, + struct task_struct *p, s64 delta); +extern void reset_task_stats(struct task_struct *p); +extern void reset_cfs_rq_hmp_stats(int cpu, int reset_cra); +extern void inc_hmp_sched_stats_fair(struct rq *rq, + struct task_struct *p, int change_cra); +extern u64 cpu_upmigrate_discourage_read_u64(struct cgroup_subsys_state *css, + struct cftype *cft); +extern int cpu_upmigrate_discourage_write_u64(struct cgroup_subsys_state *css, + struct cftype *cft, u64 upmigrate_discourage); + +#else /* CONFIG_SCHED_HMP */ + +struct hmp_sched_stats; +struct related_thread_group; +struct sched_cluster; + +static inline int got_boost_kick(void) +{ + return 0; +} + +static inline void update_task_ravg(struct task_struct *p, struct rq *rq, + int event, u64 wallclock, u64 irqtime) { } + +static inline bool early_detection_notify(struct rq *rq, u64 wallclock) +{ + return 0; +} + +static inline void clear_ed_task(struct task_struct *p, struct rq *rq) { } +static inline void fixup_busy_time(struct task_struct *p, int new_cpu) { } +static inline void clear_boost_kick(int cpu) { } +static inline void clear_hmp_request(int cpu) { } +static inline void mark_task_starting(struct task_struct *p) { } +static inline void set_window_start(struct rq *rq) { } +static inline void migrate_sync_cpu(int cpu) { } +static inline void update_cluster_topology(void) { } +static inline void set_task_last_wake(struct task_struct *p, u64 wallclock) { } +static inline void set_task_last_switch_out(struct task_struct *p, + u64 wallclock) { } + +static inline int task_will_fit(struct task_struct *p, int cpu) +{ + return 1; +} + +static inline unsigned int power_cost(int cpu, u64 demand) +{ + return SCHED_CAPACITY_SCALE; +} + +static inline int sched_boost(void) +{ + return 0; +} + +static inline int is_big_task(struct task_struct *p) +{ + return 0; +} + +static inline int nr_big_tasks(struct rq *rq) +{ + return 0; +} + +static inline int is_cpu_throttling_imminent(int cpu) +{ + return 0; +} + +static inline int is_task_migration_throttled(struct task_struct *p) +{ + return 0; +} + +static inline unsigned int cpu_temp(int cpu) +{ + return 0; +} + +static inline void +inc_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { } + +static inline void +dec_rq_hmp_stats(struct rq *rq, struct task_struct *p, int change_cra) { } + +static inline int +preferred_cluster(struct sched_cluster *cluster, struct task_struct *p) +{ + return 1; +} + +static inline struct sched_cluster *rq_cluster(struct rq *rq) +{ + return NULL; +} + +static inline void init_new_task_load(struct task_struct *p) { } + +static inline u64 scale_load_to_cpu(u64 load, int cpu) +{ + return load; +} + +static inline unsigned int nr_eligible_big_tasks(int cpu) +{ + return 0; +} + +static inline int pct_task_load(struct task_struct *p) { return 0; } + +static inline int cpu_capacity(int cpu) +{ + return SCHED_CAPACITY_SCALE; +} + +static inline int same_cluster(int src_cpu, int dst_cpu) { return 1; } + +static inline void inc_cumulative_runnable_avg(struct hmp_sched_stats *stats, + struct task_struct *p) +{ +} + +static inline void dec_cumulative_runnable_avg(struct hmp_sched_stats *stats, + struct task_struct *p) +{ +} + +static inline void sched_account_irqtime(int cpu, struct task_struct *curr, + u64 delta, u64 wallclock) +{ +} + +static inline void sched_account_irqstart(int cpu, struct task_struct *curr, + u64 wallclock) +{ +} + +static inline int sched_cpu_high_irqload(int cpu) { return 0; } + +static inline void set_preferred_cluster(struct related_thread_group *grp) { } + +static inline bool task_in_related_thread_group(struct task_struct *p) +{ + return false; +} + +static inline +struct related_thread_group *task_related_thread_group(struct task_struct *p) +{ + return NULL; +} + +static inline u32 task_load(struct task_struct *p) { return 0; } + +static inline int update_preferred_cluster(struct related_thread_group *grp, + struct task_struct *p, u32 old_load) +{ + return 0; +} + +static inline void add_new_task_to_grp(struct task_struct *new) {} + +#define sched_freq_legacy_mode 1 +#define sched_migration_fixup 0 +#define PRED_DEMAND_DELTA (0) + +static inline void +check_for_freq_change(struct rq *rq, bool check_pred, bool check_groups) { } + +static inline void notify_migration(int src_cpu, int dest_cpu, + bool src_cpu_dead, struct task_struct *p) { } + +static inline int same_freq_domain(int src_cpu, int dst_cpu) +{ + return 1; +} + +static inline void check_for_migration(struct rq *rq, struct task_struct *p) { } +static inline void pre_big_task_count_change(void) { } +static inline void post_big_task_count_change(void) { } +static inline void set_hmp_defaults(void) { } + +static inline void clear_reserved(int cpu) { } + +#define trace_sched_cpu_load(...) +#define trace_sched_cpu_load_lb(...) +#define trace_sched_cpu_load_cgroup(...) +#define trace_sched_cpu_load_wakeup(...) + +#endif /* CONFIG_SCHED_HMP */ + +/* + * Returns the rq capacity of any rq in a group. This does not play + * well with groups where rq capacity can change independently. + */ +#define group_rq_capacity(group) cpu_capacity(group_first_cpu(group)) + #ifdef CONFIG_CGROUP_SCHED /* @@ -1163,6 +1843,7 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) #define WF_SYNC 0x01 /* waker goes to sleep after wakeup */ #define WF_FORK 0x02 /* child wakeup after fork */ #define WF_MIGRATED 0x4 /* internal use, task got migrated */ +#define WF_NO_NOTIFIER 0x08 /* do not notify governor */ /* * To aid in avoiding the subversion of "niceness" due to uneven distribution @@ -1278,6 +1959,10 @@ struct sched_class { #ifdef CONFIG_FAIR_GROUP_SCHED void (*task_change_group) (struct task_struct *p, int type); #endif +#ifdef CONFIG_SCHED_HMP + void (*fixup_hmp_sched_stats)(struct rq *rq, struct task_struct *p, + u32 new_task_load, u32 new_pred_demand); +#endif }; static inline void put_prev_task(struct rq *rq, struct task_struct *prev) @@ -1391,6 +2076,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count) { unsigned prev_nr = rq->nr_running; + sched_update_nr_prod(cpu_of(rq), count, true); rq->nr_running = prev_nr + count; if (prev_nr < 2 && rq->nr_running >= 2) { @@ -1405,6 +2091,7 @@ static inline void add_nr_running(struct rq *rq, unsigned count) static inline void sub_nr_running(struct rq *rq, unsigned count) { + sched_update_nr_prod(cpu_of(rq), count, false); rq->nr_running -= count; /* Check if we still need preemption */ sched_update_tick_dependency(rq); @@ -1732,6 +2419,9 @@ enum rq_nohz_flag_bits { NOHZ_BALANCE_KICK, }; +#define NOHZ_KICK_ANY 0 +#define NOHZ_KICK_RESTRICT 1 + #define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags) extern void nohz_balance_exit_idle(unsigned int cpu); diff --git a/kernel/sched/sched_avg.c b/kernel/sched/sched_avg.c new file mode 100644 index 000000000000..c70e0466c36c --- /dev/null +++ b/kernel/sched/sched_avg.c @@ -0,0 +1,128 @@ +/* Copyright (c) 2012, 2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/* + * Scheduler hook for average runqueue determination + */ +#include +#include +#include +#include +#include + +#include "sched.h" +#include + +static DEFINE_PER_CPU(u64, nr_prod_sum); +static DEFINE_PER_CPU(u64, last_time); +static DEFINE_PER_CPU(u64, nr_big_prod_sum); +static DEFINE_PER_CPU(u64, nr); + +static DEFINE_PER_CPU(unsigned long, iowait_prod_sum); +static DEFINE_PER_CPU(spinlock_t, nr_lock) = __SPIN_LOCK_UNLOCKED(nr_lock); +static s64 last_get_time; + +/** + * sched_get_nr_running_avg + * @return: Average nr_running, iowait and nr_big_tasks value since last poll. + * Returns the avg * 100 to return up to two decimal points + * of accuracy. + * + * Obtains the average nr_running value since the last poll. + * This function may not be called concurrently with itself + */ +void sched_get_nr_running_avg(int *avg, int *iowait_avg, int *big_avg) +{ + int cpu; + u64 curr_time = sched_clock(); + u64 diff = curr_time - last_get_time; + u64 tmp_avg = 0, tmp_iowait = 0, tmp_big_avg = 0; + + *avg = 0; + *iowait_avg = 0; + *big_avg = 0; + + if (!diff) + return; + + /* read and reset nr_running counts */ + for_each_possible_cpu(cpu) { + unsigned long flags; + + spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags); + curr_time = sched_clock(); + tmp_avg += per_cpu(nr_prod_sum, cpu); + tmp_avg += per_cpu(nr, cpu) * + (curr_time - per_cpu(last_time, cpu)); + + tmp_big_avg += per_cpu(nr_big_prod_sum, cpu); + tmp_big_avg += nr_eligible_big_tasks(cpu) * + (curr_time - per_cpu(last_time, cpu)); + + tmp_iowait += per_cpu(iowait_prod_sum, cpu); + tmp_iowait += nr_iowait_cpu(cpu) * + (curr_time - per_cpu(last_time, cpu)); + + per_cpu(last_time, cpu) = curr_time; + + per_cpu(nr_prod_sum, cpu) = 0; + per_cpu(nr_big_prod_sum, cpu) = 0; + per_cpu(iowait_prod_sum, cpu) = 0; + + spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags); + } + + diff = curr_time - last_get_time; + last_get_time = curr_time; + + *avg = (int)div64_u64(tmp_avg * 100, diff); + *big_avg = (int)div64_u64(tmp_big_avg * 100, diff); + *iowait_avg = (int)div64_u64(tmp_iowait * 100, diff); + + trace_sched_get_nr_running_avg(*avg, *big_avg, *iowait_avg); + + BUG_ON(*avg < 0 || *big_avg < 0 || *iowait_avg < 0); + pr_debug("%s - avg:%d big_avg:%d iowait_avg:%d\n", + __func__, *avg, *big_avg, *iowait_avg); +} +EXPORT_SYMBOL(sched_get_nr_running_avg); + +/** + * sched_update_nr_prod + * @cpu: The core id of the nr running driver. + * @delta: Adjust nr by 'delta' amount + * @inc: Whether we are increasing or decreasing the count + * @return: N/A + * + * Update average with latest nr_running value for CPU + */ +void sched_update_nr_prod(int cpu, long delta, bool inc) +{ + int diff; + s64 curr_time; + unsigned long flags, nr_running; + + spin_lock_irqsave(&per_cpu(nr_lock, cpu), flags); + nr_running = per_cpu(nr, cpu); + curr_time = sched_clock(); + diff = curr_time - per_cpu(last_time, cpu); + per_cpu(last_time, cpu) = curr_time; + per_cpu(nr, cpu) = nr_running + (inc ? delta : -delta); + + BUG_ON((s64)per_cpu(nr, cpu) < 0); + + per_cpu(nr_prod_sum, cpu) += nr_running * diff; + per_cpu(nr_big_prod_sum, cpu) += nr_eligible_big_tasks(cpu) * diff; + per_cpu(iowait_prod_sum, cpu) += nr_iowait_cpu(cpu) * diff; + spin_unlock_irqrestore(&per_cpu(nr_lock, cpu), flags); +} +EXPORT_SYMBOL(sched_update_nr_prod); diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index 604297a08b3a..427bc8f8b88c 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -17,6 +17,41 @@ select_task_rq_stop(struct task_struct *p, int cpu, int sd_flag, int flags) } #endif /* CONFIG_SMP */ +#ifdef CONFIG_SCHED_HMP + +static void +inc_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) +{ + inc_cumulative_runnable_avg(&rq->hmp_stats, p); +} + +static void +dec_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) +{ + dec_cumulative_runnable_avg(&rq->hmp_stats, p); +} + +static void +fixup_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p, + u32 new_task_load, u32 new_pred_demand) +{ + s64 task_load_delta = (s64)new_task_load - task_load(p); + s64 pred_demand_delta = PRED_DEMAND_DELTA; + + fixup_cumulative_runnable_avg(&rq->hmp_stats, p, task_load_delta, + pred_demand_delta); +} + +#else /* CONFIG_SCHED_HMP */ + +static inline void +inc_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) { } + +static inline void +dec_hmp_sched_stats_stop(struct rq *rq, struct task_struct *p) { } + +#endif /* CONFIG_SCHED_HMP */ + static void check_preempt_curr_stop(struct rq *rq, struct task_struct *p, int flags) { @@ -42,12 +77,14 @@ static void enqueue_task_stop(struct rq *rq, struct task_struct *p, int flags) { add_nr_running(rq, 1); + inc_hmp_sched_stats_stop(rq, p); } static void dequeue_task_stop(struct rq *rq, struct task_struct *p, int flags) { sub_nr_running(rq, 1); + dec_hmp_sched_stats_stop(rq, p); } static void yield_task_stop(struct rq *rq) @@ -134,4 +171,7 @@ const struct sched_class stop_sched_class = { .prio_changed = prio_changed_stop, .switched_to = switched_to_stop, .update_curr = update_curr_stop, +#ifdef CONFIG_SCHED_HMP + .fixup_hmp_sched_stats = fixup_hmp_sched_stats_stop, +#endif }; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 706309f9ed84..fe6537188085 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -288,6 +288,190 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "sched_wake_to_idle", + .data = &sysctl_sched_wake_to_idle, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#ifdef CONFIG_SCHED_HMP + { + .procname = "sched_freq_inc_notify", + .data = &sysctl_sched_freq_inc_notify, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &zero, + }, + { + .procname = "sched_freq_dec_notify", + .data = &sysctl_sched_freq_dec_notify, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &zero, + }, + { + .procname = "sched_cpu_high_irqload", + .data = &sysctl_sched_cpu_high_irqload, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_ravg_hist_size", + .data = &sysctl_sched_ravg_hist_size, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_window_update_handler, + }, + { + .procname = "sched_window_stats_policy", + .data = &sysctl_sched_window_stats_policy, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_window_update_handler, + }, + { + .procname = "sched_spill_load", + .data = &sysctl_sched_spill_load_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_hmp_proc_update_handler, + .extra1 = &zero, + .extra2 = &one_hundred, + }, + { + .procname = "sched_spill_nr_run", + .data = &sysctl_sched_spill_nr_run, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &zero, + }, + { + .procname = "sched_upmigrate", + .data = &sysctl_sched_upmigrate_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_hmp_proc_update_handler, + .extra1 = &zero, + .extra2 = &one_hundred, + }, + { + .procname = "sched_downmigrate", + .data = &sysctl_sched_downmigrate_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_hmp_proc_update_handler, + .extra1 = &zero, + .extra2 = &one_hundred, + }, + { + .procname = "sched_init_task_load", + .data = &sysctl_sched_init_task_load_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_hmp_proc_update_handler, + .extra1 = &zero, + .extra2 = &one_hundred, + }, + { + .procname = "sched_select_prev_cpu_us", + .data = &sysctl_sched_select_prev_cpu_us, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_hmp_proc_update_handler, + .extra1 = &zero, + }, + { + .procname = "sched_enable_colocation", + .data = &sysctl_sched_enable_colocation, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + .extra1 = &zero, + .extra2 = &one, + }, + { + .procname = "sched_restrict_cluster_spill", + .data = &sysctl_sched_restrict_cluster_spill, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &zero, + .extra2 = &one, + }, + { + .procname = "sched_small_wakee_task_load", + .data = &sysctl_sched_small_wakee_task_load_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_hmp_proc_update_handler, + .extra1 = &zero, + .extra2 = &one_hundred, + }, + { + .procname = "sched_big_waker_task_load", + .data = &sysctl_sched_big_waker_task_load_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_hmp_proc_update_handler, + .extra1 = &zero, + .extra2 = &one_hundred, + }, + { + .procname = "sched_enable_thread_grouping", + .data = &sysctl_sched_enable_thread_grouping, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_new_task_windows", + .data = &sysctl_sched_new_task_windows, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_window_update_handler, + }, + { + .procname = "sched_pred_alert_freq", + .data = &sysctl_sched_pred_alert_freq, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &zero, + }, + { + .procname = "sched_freq_aggregate", + .data = &sysctl_sched_freq_aggregate, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_window_update_handler, + }, + { + .procname = "sched_freq_aggregate_threshold", + .data = &sysctl_sched_freq_aggregate_threshold_pct, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_hmp_proc_update_handler, + .extra1 = &zero, + /* + * Special handling for sched_freq_aggregate_threshold_pct + * which can be greater than 100. Use 1000 as an upper bound + * value which works for all practical use cases. + */ + .extra2 = &one_thousand, + }, + { + .procname = "sched_boost", + .data = &sysctl_sched_boost, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_boost_handler, + }, +#endif /* CONFIG_SCHED_HMP */ #ifdef CONFIG_SCHED_DEBUG { .procname = "sched_min_granularity_ns", diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 11ce935633f4..189e77b4bb25 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -51,6 +51,21 @@ struct tick_sched *tick_get_tick_sched(int cpu) */ static ktime_t last_jiffies_update; +u64 jiffy_to_ktime_ns(u64 *now, u64 *jiffy_ktime_ns) +{ + u64 cur_jiffies; + unsigned long seq; + + do { + seq = read_seqbegin(&jiffies_lock); + *now = ktime_get_ns(); + *jiffy_ktime_ns = ktime_to_ns(last_jiffies_update); + cur_jiffies = get_jiffies_64(); + } while (read_seqretry(&jiffies_lock, seq)); + + return cur_jiffies; +} + /* * Must be called with interrupts disabled ! */ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 5d663b322a3e..9747e380bf47 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -935,6 +935,25 @@ config SCHED_INFO bool default n +config PANIC_ON_SCHED_BUG + bool "Panic on all bugs encountered by the scheduler" + help + Say Y here to panic on all 'BUG:' conditions encountered by the + scheduler, even potentially-recoverable ones such as scheduling + while atomic, sleeping from invalid context, and detection of + broken arch topologies. + + Say N if unsure. + +config PANIC_ON_RT_THROTTLING + bool "Panic on RT throttling" + help + Say Y here to enable the kernel to panic when a realtime + runqueue is throttled. This may be useful for detecting + and debugging RT throttling issues. + + Say N if unsure. + config SCHEDSTATS bool "Collect scheduler statistics" depends on DEBUG_KERNEL && PROC_FS