@@ -12122,24 +12122,43 @@ void update_max_interval(void)
1212212122 max_load_balance_interval = HZ * num_online_cpus ()/10 ;
1212312123}
1212412124
12125- static inline bool update_newidle_cost (struct sched_domain * sd , u64 cost )
12125+ static inline void update_newidle_stats (struct sched_domain * sd , unsigned int success )
1212612126{
12127+ sd -> newidle_call ++ ;
12128+ sd -> newidle_success += success ;
12129+
12130+ if (sd -> newidle_call >= 1024 ) {
12131+ sd -> newidle_ratio = sd -> newidle_success ;
12132+ sd -> newidle_call /= 2 ;
12133+ sd -> newidle_success /= 2 ;
12134+ }
12135+ }
12136+
12137+ static inline bool
12138+ update_newidle_cost (struct sched_domain * sd , u64 cost , unsigned int success )
12139+ {
12140+ unsigned long next_decay = sd -> last_decay_max_lb_cost + HZ ;
12141+ unsigned long now = jiffies ;
12142+
12143+ if (cost )
12144+ update_newidle_stats (sd , success );
12145+
1212712146 if (cost > sd -> max_newidle_lb_cost ) {
1212812147 /*
1212912148 * Track max cost of a domain to make sure to not delay the
1213012149 * next wakeup on the CPU.
1213112150 */
1213212151 sd -> max_newidle_lb_cost = cost ;
12133- sd -> last_decay_max_lb_cost = jiffies ;
12134- } else if (time_after (jiffies , sd -> last_decay_max_lb_cost + HZ )) {
12152+ sd -> last_decay_max_lb_cost = now ;
12153+
12154+ } else if (time_after (now , next_decay )) {
1213512155 /*
1213612156 * Decay the newidle max times by ~1% per second to ensure that
1213712157 * it is not outdated and the current max cost is actually
1213812158 * shorter.
1213912159 */
1214012160 sd -> max_newidle_lb_cost = (sd -> max_newidle_lb_cost * 253 ) / 256 ;
12141- sd -> last_decay_max_lb_cost = jiffies ;
12142-
12161+ sd -> last_decay_max_lb_cost = now ;
1214312162 return true;
1214412163 }
1214512164
@@ -12171,7 +12190,7 @@ static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle)
1217112190 * Decay the newidle max times here because this is a regular
1217212191 * visit to all the domains.
1217312192 */
12174- need_decay = update_newidle_cost (sd , 0 );
12193+ need_decay = update_newidle_cost (sd , 0 , 0 );
1217512194 max_cost += sd -> max_newidle_lb_cost ;
1217612195
1217712196 /*
@@ -12787,14 +12806,16 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf)
1278712806
1278812807 rcu_read_lock ();
1278912808 sd = rcu_dereference_check_sched_domain (this_rq -> sd );
12809+ if (!sd ) {
12810+ rcu_read_unlock ();
12811+ goto out ;
12812+ }
1279012813
1279112814 if (!get_rd_overloaded (this_rq -> rd ) ||
12792- ( sd && this_rq -> avg_idle < sd -> max_newidle_lb_cost ) ) {
12815+ this_rq -> avg_idle < sd -> max_newidle_lb_cost ) {
1279312816
12794- if (sd )
12795- update_next_balance (sd , & next_balance );
12817+ update_next_balance (sd , & next_balance );
1279612818 rcu_read_unlock ();
12797-
1279812819 goto out ;
1279912820 }
1280012821 rcu_read_unlock ();
@@ -12814,17 +12835,37 @@ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf)
1281412835 break ;
1281512836
1281612837 if (sd -> flags & SD_BALANCE_NEWIDLE ) {
12838+ unsigned int weight = 1 ;
12839+
12840+ if (sched_feat (NI_RANDOM )) {
12841+ /*
12842+ * Throw a 1k sided dice; and only run
12843+ * newidle_balance according to the success
12844+ * rate.
12845+ */
12846+ u32 d1k = sched_rng () % 1024 ;
12847+ weight = 1 + sd -> newidle_ratio ;
12848+ if (d1k > weight ) {
12849+ update_newidle_stats (sd , 0 );
12850+ continue ;
12851+ }
12852+ weight = (1024 + weight /2 ) / weight ;
12853+ }
1281712854
1281812855 pulled_task = sched_balance_rq (this_cpu , this_rq ,
1281912856 sd , CPU_NEWLY_IDLE ,
1282012857 & continue_balancing );
1282112858
1282212859 t1 = sched_clock_cpu (this_cpu );
1282312860 domain_cost = t1 - t0 ;
12824- update_newidle_cost (sd , domain_cost );
12825-
1282612861 curr_cost += domain_cost ;
1282712862 t0 = t1 ;
12863+
12864+ /*
12865+ * Track max cost of a domain to make sure to not delay the
12866+ * next wakeup on the CPU.
12867+ */
12868+ update_newidle_cost (sd , domain_cost , weight * !!pulled_task );
1282812869 }
1282912870
1283012871 /*
0 commit comments