diff options
Diffstat (limited to 'kernel/sched.c')
| -rw-r--r-- | kernel/sched.c | 27 |
1 files changed, 23 insertions, 4 deletions
diff --git a/kernel/sched.c b/kernel/sched.c index e7ec90424de..27d4879e86e 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -511,7 +511,7 @@ struct rq { unsigned long cpu_power; - unsigned char idle_at_tick; + unsigned char idle_balance; /* For active balancing */ int post_schedule; int active_balance; @@ -1273,6 +1273,18 @@ void wake_up_idle_cpu(int cpu) smp_send_reschedule(cpu); } +static inline bool got_nohz_idle_kick(void) +{ + return idle_cpu(smp_processor_id()) && this_rq()->nohz_balance_kick; +} + +#else /* CONFIG_NO_HZ */ + +static inline bool got_nohz_idle_kick(void) +{ + return false; +} + #endif /* CONFIG_NO_HZ */ static u64 sched_avg_period(void) @@ -2592,7 +2604,7 @@ void scheduler_ipi(void) struct rq *rq = this_rq(); struct task_struct *list = xchg(&rq->wake_list, NULL); - if (!list) + if ((!list) && !got_nohz_idle_kick()) return; /* @@ -2610,6 +2622,14 @@ void scheduler_ipi(void) */ irq_enter(); sched_ttwu_do_pending(list); + + /* + * Check if someone kicked us for doing the nohz idle load balance. + */ + if (unlikely(got_nohz_idle_kick() && !need_resched())) { + this_rq()->idle_balance = 1; + raise_softirq_irqoff(SCHED_SOFTIRQ); + } irq_exit(); } @@ -4117,7 +4137,7 @@ void scheduler_tick(void) perf_event_task_tick(); #ifdef CONFIG_SMP - rq->idle_at_tick = idle_cpu(cpu); + rq->idle_balance = idle_cpu(cpu); trigger_load_balance(rq, cpu); #endif } @@ -8134,7 +8154,6 @@ void __init sched_init(void) rq_attach_root(rq, &def_root_domain); #ifdef CONFIG_NO_HZ rq->nohz_balance_kick = 0; - init_sched_softirq_csd(&per_cpu(remote_sched_softirq_cb, i)); #endif #endif init_rq_hrtick(rq); |