diff options
Diffstat (limited to 'kernel/sched.c')
| -rw-r--r-- | kernel/sched.c | 27 |
1 files changed, 23 insertions, 4 deletions
diff --git a/kernel/sched.c b/kernel/sched.c index b50b0f0c9aa..036311afeed 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -510,7 +510,7 @@ struct rq { unsigned long cpu_power; - unsigned char idle_at_tick; + unsigned char idle_balance; /* For active balancing */ int post_schedule; int active_balance; @@ -1272,6 +1272,18 @@ void wake_up_idle_cpu(int cpu) smp_send_reschedule(cpu); } +static inline bool got_nohz_idle_kick(void) +{ + return idle_cpu(smp_processor_id()) && this_rq()->nohz_balance_kick; +} + +#else /* CONFIG_NO_HZ */ + +static inline bool got_nohz_idle_kick(void) +{ + return false; +} + #endif /* CONFIG_NO_HZ */ static u64 sched_avg_period(void) @@ -2591,7 +2603,7 @@ void scheduler_ipi(void) struct rq *rq = this_rq(); struct task_struct *list = xchg(&rq->wake_list, NULL); - if (!list) + if ((!list) && !got_nohz_idle_kick()) return; /* @@ -2609,6 +2621,14 @@ void scheduler_ipi(void) */ irq_enter(); sched_ttwu_do_pending(list); + + /* + * Check if someone kicked us for doing the nohz idle load balance. + */ + if (unlikely(got_nohz_idle_kick() && !need_resched())) { + this_rq()->idle_balance = 1; + raise_softirq_irqoff(SCHED_SOFTIRQ); + } irq_exit(); } @@ -4116,7 +4136,7 @@ void scheduler_tick(void) perf_event_task_tick(); #ifdef CONFIG_SMP - rq->idle_at_tick = idle_cpu(cpu); + rq->idle_balance = idle_cpu(cpu); trigger_load_balance(rq, cpu); #endif } @@ -8133,7 +8153,6 @@ void __init sched_init(void) rq_attach_root(rq, &def_root_domain); #ifdef CONFIG_NO_HZ rq->nohz_balance_kick = 0; - init_sched_softirq_csd(&per_cpu(remote_sched_softirq_cb, i)); #endif #endif init_rq_hrtick(rq); |