diff options
author | Mark Brown <broonie@linaro.org> | 2014-01-22 12:43:08 +0000 |
---|---|---|
committer | Mark Brown <broonie@linaro.org> | 2014-01-22 12:43:08 +0000 |
commit | 4105a61b153ff238387b999be6f8bf97a2843690 (patch) | |
tree | 95a4d0609a5c41dead8b9f2d0b3bf4a97883e730 | |
parent | 750a8dee18ab8b5dc6ddec36016fcacc4293d6d4 (diff) | |
parent | 2305009118181260b45aab82f14bcbb659953ef1 (diff) |
Merge remote-tracking branch 'lsk/v3.10/topic/big.LITTLE' into linux-linaro-lsk
-rw-r--r-- | kernel/irq/irqdesc.c | 8 | ||||
-rw-r--r-- | kernel/sched/core.c | 5 | ||||
-rw-r--r-- | kernel/sched/fair.c | 71 | ||||
-rw-r--r-- | linaro/configs/big-LITTLE-MP.conf | 1 |
4 files changed, 68 insertions, 17 deletions
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index 473b2b6eccb5..20ecfb0984e1 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -36,12 +36,20 @@ static int __init irq_affinity_setup(char *str) } __setup("irqaffinity=", irq_affinity_setup); +extern struct cpumask hmp_slow_cpu_mask; + static void __init init_irq_default_affinity(void) { #ifdef CONFIG_CPUMASK_OFFSTACK if (!irq_default_affinity) zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT); #endif +#ifdef CONFIG_SCHED_HMP + if (!cpumask_empty(&hmp_slow_cpu_mask)) { + cpumask_copy(irq_default_affinity, &hmp_slow_cpu_mask); + return; + } +#endif if (cpumask_empty(irq_default_affinity)) cpumask_setall(irq_default_affinity); } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 134d8158dd8a..277e3557d0e6 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3854,8 +3854,11 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio) p->sched_class = &rt_sched_class; #ifdef CONFIG_SCHED_HMP if (!cpumask_empty(&hmp_slow_cpu_mask)) - if (cpumask_equal(&p->cpus_allowed, cpu_all_mask)) + if (cpumask_equal(&p->cpus_allowed, cpu_all_mask)) { + p->nr_cpus_allowed = + cpumask_weight(&hmp_slow_cpu_mask); do_set_cpus_allowed(p, &hmp_slow_cpu_mask); + } #endif } else diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a771fa6e423a..62a8808e825d 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1460,12 +1460,9 @@ static inline u64 __synchronize_entity_decay(struct sched_entity *se) u64 decays = atomic64_read(&cfs_rq->decay_counter); decays -= se->avg.decay_count; - if (!decays) - return 0; - - se->avg.load_avg_contrib = decay_load(se->avg.load_avg_contrib, decays); + if (decays) + se->avg.load_avg_contrib = decay_load(se->avg.load_avg_contrib, decays); se->avg.decay_count = 0; - return decays; } @@ -1721,13 +1718,7 @@ static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq, } wakeup = 0; } else { - /* - * Task re-woke on same cpu (or else migrate_task_rq_fair() - * would have made count negative); we must be careful to avoid - * double-accounting blocked time after synchronizing decays. - */ - se->avg.last_runnable_update += __synchronize_entity_decay(se) - << 20; + __synchronize_entity_decay(se); } /* migrated tasks did not contribute to our blocked load */ @@ -3712,12 +3703,13 @@ unsigned int hmp_next_up_threshold = 4096; unsigned int hmp_next_down_threshold = 4096; #ifdef CONFIG_SCHED_HMP_LITTLE_PACKING -unsigned int hmp_packing_enabled = 1; #ifndef CONFIG_ARCH_VEXPRESS_TC2 +unsigned int hmp_packing_enabled = 1; unsigned int hmp_full_threshold = (NICE_0_LOAD * 9) / 8; #else /* TC2 has a sharp consumption curve @ around 800Mhz, so we aim to spread the load around that frequency. */ +unsigned int hmp_packing_enabled; unsigned int hmp_full_threshold = 650; /* 80% of the 800Mhz freq * NICE_0_LOAD */ #endif #endif @@ -4382,6 +4374,16 @@ unlock: * load-balance). */ #ifdef CONFIG_FAIR_GROUP_SCHED + +#ifdef CONFIG_NO_HZ_COMMON +static int nohz_test_cpu(int cpu); +#else +static inline int nohz_test_cpu(int cpu) +{ + return 0; +} +#endif + /* * Called immediately before a task is migrated to a new cpu; task_cpu(p) and * cfs_rq_of(p) references at time of call are still valid and identify the @@ -4401,6 +4403,25 @@ migrate_task_rq_fair(struct task_struct *p, int next_cpu) * be negative here since on-rq tasks have decay-count == 0. */ if (se->avg.decay_count) { + /* + * If we migrate a sleeping task away from a CPU + * which has the tick stopped, then both the clock_task + * and decay_counter will be out of date for that CPU + * and we will not decay load correctly. + */ + if (!se->on_rq && nohz_test_cpu(task_cpu(p))) { + struct rq *rq = cpu_rq(task_cpu(p)); + unsigned long flags; + /* + * Current CPU cannot be holding rq->lock in this + * circumstance, but another might be. We must hold + * rq->lock before we go poking around in its clocks + */ + raw_spin_lock_irqsave(&rq->lock, flags); + update_rq_clock(rq); + update_cfs_rq_blocked_load(cfs_rq, 0); + raw_spin_unlock_irqrestore(&rq->lock, flags); + } se->avg.decay_count = -__synchronize_entity_decay(se); atomic64_add(se->avg.load_avg_contrib, &cfs_rq->removed_load); } @@ -6333,6 +6354,18 @@ static struct { unsigned long next_balance; /* in jiffy units */ } nohz ____cacheline_aligned; +/* + * nohz_test_cpu used when load tracking is enabled. FAIR_GROUP_SCHED + * dependency below may be removed when load tracking guards are + * removed. + */ +#ifdef CONFIG_FAIR_GROUP_SCHED +static int nohz_test_cpu(int cpu) +{ + return cpumask_test_cpu(cpu, nohz.idle_cpus_mask); +} +#endif + #ifdef CONFIG_SCHED_HMP_LITTLE_PACKING /* * Decide if the tasks on the busy CPUs in the @@ -6345,6 +6378,10 @@ static int hmp_packing_ilb_needed(int cpu) if (!hmp_cpu_is_slowest(cpu)) return 1; + /* if disabled, use normal ILB behaviour */ + if (!hmp_packing_enabled) + return 1; + hmp = hmp_cpu_domain(cpu); for_each_cpu_and(cpu, &hmp->cpus, nohz.idle_cpus_mask) { /* only idle balance if a CPU is loaded over threshold */ @@ -7017,13 +7054,13 @@ static void hmp_migrate_runnable_task(struct rq *rq) * with the source rq. */ if (src_rq->active_balance) - return; + goto out; if (src_rq->nr_running <= 1) - return; + goto out; if (task_rq(p) != src_rq) - return; + goto out; /* * Not sure if this applies here but one can never * be too cautious @@ -7058,6 +7095,8 @@ static void hmp_migrate_runnable_task(struct rq *rq) rcu_read_unlock(); double_unlock_balance(src_rq, dst_rq); +out: + put_task_struct(p); } static DEFINE_SPINLOCK(hmp_force_migration); diff --git a/linaro/configs/big-LITTLE-MP.conf b/linaro/configs/big-LITTLE-MP.conf index 0bbc603a13e5..ced3cf974f13 100644 --- a/linaro/configs/big-LITTLE-MP.conf +++ b/linaro/configs/big-LITTLE-MP.conf @@ -9,3 +9,4 @@ CONFIG_HMP_FAST_CPU_MASK="" CONFIG_HMP_SLOW_CPU_MASK="" CONFIG_HMP_VARIABLE_SCALE=y CONFIG_HMP_FREQUENCY_INVARIANT_SCALE=y +CONFIG_SCHED_HMP_LITTLE_PACKING=y |