From 86399328f106d657d3b7d4c72d87d6fae03fb1de Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Thu, 19 Nov 2015 23:29:34 +0100 Subject: HACK: change irqstat to be per irq per cpu array. --- include/linux/irqdesc.h | 3 - kernel/irq/internals.h | 9 +- kernel/irq/manage.c | 2 +- kernel/irq/proc.c | 4 +- kernel/irq/timings.c | 224 +++++++++++++++++++----------------------------- 5 files changed, 97 insertions(+), 145 deletions(-) diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h index 5bc2cd7fe19f..ac22297516a9 100644 --- a/include/linux/irqdesc.h +++ b/include/linux/irqdesc.h @@ -48,9 +48,6 @@ struct irq_desc { struct irq_common_data irq_common_data; struct irq_data irq_data; unsigned int __percpu *kstat_irqs; -#ifdef CONFIG_IRQ_TIMINGS - struct irqt_stat *irq_timings; -#endif irq_flow_handler_t handle_irq; #ifdef CONFIG_IRQ_PREFLOW_FASTEOI irq_preflow_handler_t preflow_handler; diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index e29d44f922e5..2fcd3cc3a052 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -105,18 +105,17 @@ static inline void unregister_handler_proc(unsigned int irq, #ifdef CONFIG_IRQ_TIMINGS extern void __init irqt_init(void); -extern void irqt_process(unsigned int irq, struct irqt_stat *s); +extern void irqt_process(unsigned int irq); static inline void irqt_event(int irq, struct irq_desc *desc) { - if (desc->irq_timings) - irqt_process(irq, desc->irq_timings); + irqt_process(irq); } -extern int irqt_register(struct irq_desc *desc); +extern int irqt_register(int irq, struct irq_desc *desc); extern void irqt_unregister(struct irq_desc *desc); #else static inline void irqt_init(void) { } static inline void irqt_event(int irq, struct irq_desc *desc) { } -static inline int irqt_register(struct irq_desc *desc) { return 0; } +static inline int irqt_register(int irq, struct irq_desc *desc) { return 0; } static inline void irqt_unregister(struct irq_desc *desc) { } #endif diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index ef4709c7d6c4..548de1683e43 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -1300,7 +1300,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new) free_cpumask_var(mask); if (new->flags & IRQF_TIMINGS) - irqt_register(desc); + irqt_register(irq, desc); return 0; diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 534ae42e7d4c..5956b583f89f 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -287,7 +287,7 @@ static int irq_timings_proc_show(struct seq_file *m, void *v) { struct irq_desc *desc = irq_to_desc((long) m->private); - seq_printf(m, "%d\n", desc->irq_timings ? 1 : 0); + seq_printf(m, "%d\n", 0); return 0; } @@ -319,7 +319,7 @@ static ssize_t irq_timings_proc_write(struct file *file, goto out; if (enable) { - ret = irqt_register(desc); + ret = irqt_register(irq, desc); } else { unsigned long flags; raw_spin_lock_irqsave(&desc->lock, flags); diff --git a/kernel/irq/timings.c b/kernel/irq/timings.c index d8f11ceae18c..1c875e0d475b 100644 --- a/kernel/irq/timings.c +++ b/kernel/irq/timings.c @@ -20,53 +20,22 @@ #include - -/* - * This is the size of the IRQ interval window used to compute the - * mean interval and its variance. This has to be at least 3 to still - * make sense. Higher values may improve prediction confidence but more - * false negatives are to be expected. - */ -#define IRQT_INTERVAL_WINDOW 3 - - -struct irqt_prediction { - struct list_head node; - ktime_t time; /* expected occurrence time */ - int cpu; /* CPU for which this was queued for */ -}; - -struct irqt_stat { - ktime_t last_time; /* previous IRQ occurrence */ - u64 n_M2; /* IRQ interval variance (n scaled) */ - u32 n_mean; /* IRQ mean interval (n scaled) */ - u32 intervals[IRQT_INTERVAL_WINDOW]; - /* window of recent IRQ intervals */ - unsigned int w_ptr; /* current window pointer */ - u32 predictable; /* # of IRQs that were predictable */ - u32 unpredictable; /* # of IRQs that were not */ - struct irqt_prediction prediction; -}; - -static DEFINE_PER_CPU(struct list_head, irqt_predictions); -static DEFINE_PER_CPU(raw_spinlock_t, irqt_predictions_lock); - #ifdef CONFIG_DEBUG_FS -struct irqt_stats { +struct irqt_stats_debug { atomic_t correct; atomic_t early; atomic_t late; atomic_t total; }; -static DEFINE_PER_CPU(struct irqt_stats, irqt_stats[NR_IRQS]); +static DEFINE_PER_CPU(struct irqt_stats_debug, irqt_stats_debug[NR_IRQS]); static unsigned irqt_registered[NR_IRQS]; static int irqt_debugfs_stats_show(struct seq_file *m, void *v) { int i, cpu; - struct irqt_stats *s; + struct irqt_stats_debug *s; seq_printf(m, "# cpu\t irq\tcorrect\tearly\tlate\ttotal\n"); @@ -77,7 +46,7 @@ static int irqt_debugfs_stats_show(struct seq_file *m, void *v) if (!irqt_registered[i]) continue; - s = &per_cpu(irqt_stats[i], cpu); + s = &per_cpu(irqt_stats_debug[i], cpu); if (!atomic_read(&s->total)) continue; @@ -107,21 +76,21 @@ static const struct file_operations stats_fops = { static void irqt_debugfs_correct_inc(int cpu, int irq) { - struct irqt_stats *s = &per_cpu(irqt_stats[irq], cpu); + struct irqt_stats_debug *s = &per_cpu(irqt_stats_debug[irq], cpu); atomic_inc(&s->correct); atomic_inc(&s->total); } static void irqt_debugfs_early_inc(int cpu, int irq) { - struct irqt_stats *s = &per_cpu(irqt_stats[irq], cpu); + struct irqt_stats_debug *s = &per_cpu(irqt_stats_debug[irq], cpu); atomic_inc(&s->early); atomic_inc(&s->total); } static void irqt_debugfs_late_inc(int cpu, int irq) { - struct irqt_stats *s = &per_cpu(irqt_stats[irq], cpu); + struct irqt_stats_debug *s = &per_cpu(irqt_stats_debug[irq], cpu); atomic_inc(&s->late); atomic_inc(&s->total); } @@ -150,14 +119,36 @@ late_initcall(irqt_debugfs_init); #endif +/* + * This is the size of the IRQ interval window used to compute the + * mean interval and its variance. This has to be at least 3 to still + * make sense. Higher values may improve prediction confidence but more + * false negatives are to be expected. + */ +#define IRQT_INTERVAL_WINDOW 3 + +struct irqt_prediction { + ktime_t time; /* expected occurrence time */ + int cpu; /* CPU for which this was queued for */ +}; + +struct irqt_stat { + struct list_head node; + ktime_t last_time; /* previous IRQ occurrence */ + u64 n_M2; /* IRQ interval variance (n scaled) */ + u32 n_mean; /* IRQ mean interval (n scaled) */ + u32 intervals[IRQT_INTERVAL_WINDOW]; + /* window of recent IRQ intervals */ + unsigned int w_ptr; /* current window pointer */ + u32 predictable; /* # of IRQs that were predictable */ + u32 unpredictable; /* # of IRQs that were not */ + struct irqt_prediction prediction; +}; + +static DEFINE_PER_CPU(struct irqt_stat, *irqt_stats[NR_IRQS]); + void __init irqt_init(void) { - int cpu; - - for_each_possible_cpu(cpu) { - INIT_LIST_HEAD(&per_cpu(irqt_predictions, cpu)); - raw_spin_lock_init(&per_cpu(irqt_predictions_lock, cpu)); - } } /* @@ -166,13 +157,6 @@ void __init irqt_init(void) */ static void irqt_purge(ktime_t now, struct list_head *head) { - struct irqt_prediction *entry, *n; - - list_for_each_entry_safe(entry, n, head, node) { - if (ktime_after(entry->time, now)) - break; - list_del_init(&entry->node); - } } /* @@ -183,30 +167,12 @@ static void irqt_enqueue_prediction(ktime_t now, struct irqt_stat *s) { int this_cpu = raw_smp_processor_id(); int prev_cpu = s->prediction.cpu; - struct list_head *head = &per_cpu(irqt_predictions, this_cpu); u32 predicted_interval = s->n_mean / IRQT_INTERVAL_WINDOW; - struct irqt_prediction *list_entry, *new_entry; - raw_spinlock_t *lock; - - if (unlikely(prev_cpu != this_cpu && prev_cpu != -1)) { - lock = &per_cpu(irqt_predictions_lock, prev_cpu); - raw_spin_lock(lock); - list_del_init(&s->prediction.node); - raw_spin_unlock(lock); - } + + if (unlikely(prev_cpu != this_cpu && prev_cpu != -1)) + WARN_ONCE(1, "CPUs are not the same"); - lock = &per_cpu(irqt_predictions_lock, this_cpu); - raw_spin_lock(lock); - irqt_purge(now, head); - __list_del_entry(&s->prediction.node); - new_entry = &s->prediction; - new_entry->time = ktime_add_us(now, predicted_interval); - new_entry->cpu = this_cpu; - list_for_each_entry(list_entry, head, node) - if (ktime_after(new_entry->time, list_entry->time)) - break; - list_add_tail(&new_entry->node, &list_entry->node); - raw_spin_unlock(lock); + s->prediction.time = ktime_add_us(now, predicted_interval); } /** @@ -221,20 +187,26 @@ static void irqt_enqueue_prediction(ktime_t now, struct irqt_stat *s) */ s64 irqt_get_next_prediction(int cpu) { - raw_spinlock_t *lock = &per_cpu(irqt_predictions_lock, cpu); - struct list_head *head = &per_cpu(irqt_predictions, cpu); - unsigned long flags; - ktime_t now; - struct irqt_prediction *next; - s64 result; - - raw_spin_lock_irqsave(lock, flags); - now = ktime_get(); - irqt_purge(now, head); - next = list_first_entry_or_null(head, struct irqt_prediction, node); - result = next ? ktime_us_delta(next->time, now) : 0; - raw_spin_unlock_irqrestore(lock, flags); - return result; + struct irqt_stat *s; + int i; + ktime_t now = ktime_get(); + s64 value, min = S64_MAX; + + for (i = 0; i < NR_IRQS; i++) { + + s = per_cpu(irqt_stats[i], cpu); + if (!s) + continue; + + value = ktime_us_delta(s->prediction.time, now); + if (value <= 0) + continue; + + if (value < min) + min = value; + } + + return min != S64_MAX ? min : 0; } /* @@ -246,13 +218,18 @@ s64 irqt_get_next_prediction(int cpu) * This is assumed to be called in IRQ context with desc->lock held and * IRQs turned off. */ -void irqt_process(unsigned int irq, struct irqt_stat *s) +void irqt_process(unsigned int irq) { + struct irqt_stat *s = per_cpu(irqt_stats[irq], raw_smp_processor_id()); ktime_t now = ktime_get(); - ktime_t ktime_interval = ktime_sub(now, s->last_time); + ktime_t ktime_interval; u32 oldX, newX, n = IRQT_INTERVAL_WINDOW; s32 delta, n_dold, n_dnew; + if (!s) + return; + + ktime_interval = ktime_sub(now, s->last_time); s->last_time = now; /* An interval needs at least two events */ @@ -376,42 +353,38 @@ void irqt_process(unsigned int irq, struct irqt_stat *s) * Called from __setup_irq() after successful registration of a new action * handler. */ -int irqt_register(struct irq_desc *desc) +int irqt_register(int irq, struct irq_desc *desc) { struct irqt_stat *s; unsigned long flags; - int ret; - - if (desc->irq_timings) - return 0; - - s = kzalloc(sizeof(*s), GFP_KERNEL); - if (!s) - return -ENOMEM; - INIT_LIST_HEAD(&s->prediction.node); - s->prediction.cpu = -1; + int cpu, ret = 0; raw_spin_lock_irqsave(&desc->lock, flags); - if (desc->irq_timings) { - /* someone else raced ahead of us */ - ret = 0; - } else if (!desc->action) { + if (!desc->action) { /* unused IRQ? */ ret = -ENXIO; - } else if (irq_settings_is_per_cpu(desc)) { - /* we're not set for per-CPU accounting */ - pr_warn("IRQ %d: can't do timing stats on per-CPU IRQs\n", - desc->action->irq); - ret = -ENOSYS; - } else { - desc->irq_timings = s; - irqt_registered[desc->action->irq] = 1; - s = NULL; - ret = 0; + } /* else if (irq_settings_is_per_cpu(desc)) { */ + /* /\* we're not set for per-CPU accounting *\/ */ + /* pr_warn("IRQ %d: can't do timing stats on per-CPU IRQs\n", */ + /* desc->action->irq); */ + /* ret = -ENOSYS; */ + /* } */ else { + for_each_possible_cpu(cpu) { + + if (!per_cpu(irqt_stats[irq], cpu)) { + s = kzalloc(sizeof(*s), GFP_KERNEL); + if (!s) + return -ENOMEM; + s->prediction.cpu = cpu; + per_cpu(irqt_stats[irq], cpu) = s; + } + } + + irqt_registered[irq] = 1; } + raw_spin_unlock_irqrestore(&desc->lock, flags); - if (s) - kfree(s); + return ret; } @@ -421,22 +394,5 @@ int irqt_register(struct irq_desc *desc) */ void irqt_unregister(struct irq_desc *desc) { - struct irqt_stat *s; - int cpu; - raw_spinlock_t *lock; - - assert_raw_spin_locked(&desc->lock); - if (!desc->irq_timings) - return; - s = desc->irq_timings; - desc->irq_timings = NULL; - irqt_registered[desc->action->irq] = 0; - cpu = s->prediction.cpu; - if (cpu != -1) { - lock = &per_cpu(irqt_predictions_lock, cpu); - raw_spin_lock(lock); - __list_del_entry(&s->prediction.node); - raw_spin_unlock(lock); - } - kfree(s); + ; } -- cgit v1.2.3