diff options
author | Jon Medhurst <tixy@linaro.org> | 2012-05-10 14:15:56 +0100 |
---|---|---|
committer | Jon Medhurst <tixy@linaro.org> | 2012-05-16 14:22:59 +0100 |
commit | d18974d3f05535eda819f2d0b92a9d49719b0f26 (patch) | |
tree | ee0d02ac702b3802b0f002a0f8edf2171d7e58c3 /driver | |
parent | 970700feed8c3523b06476ae340bf46f6d262550 (diff) |
gator: Version 5.10DS-5.10
New gator release (build 1385) for ARM DS-5 v5.10
Signed-off-by: Jon Medhurst <tixy@linaro.org>
Diffstat (limited to 'driver')
-rw-r--r-- | driver/Makefile | 11 | ||||
-rw-r--r-- | driver/gator.h | 1 | ||||
-rw-r--r-- | driver/gator_annotate.c | 184 | ||||
-rw-r--r-- | driver/gator_backtrace.c | 6 | ||||
-rw-r--r-- | driver/gator_cookies.c | 10 | ||||
-rw-r--r-- | driver/gator_ebs.c | 64 | ||||
-rw-r--r-- | driver/gator_events_armv6.c | 16 | ||||
-rw-r--r-- | driver/gator_events_armv7.c | 15 | ||||
-rw-r--r-- | driver/gator_events_block.c | 6 | ||||
-rw-r--r-- | driver/gator_events_irq.c | 19 | ||||
-rwxr-xr-x[-rw-r--r--] | driver/gator_events_mali.c | 165 | ||||
-rw-r--r-- | driver/gator_events_meminfo.c | 9 | ||||
-rw-r--r-- | driver/gator_events_mmaped.c | 77 | ||||
-rw-r--r-- | driver/gator_events_net.c | 17 | ||||
-rwxr-xr-x[-rw-r--r--] | driver/gator_events_perf_pmu.c | 16 | ||||
-rw-r--r-- | driver/gator_events_power.c | 178 | ||||
-rw-r--r-- | driver/gator_events_scorpion.c | 18 | ||||
-rw-r--r-- | driver/gator_fs.c | 8 | ||||
-rwxr-xr-x[-rw-r--r--] | driver/gator_hrtimer_gator.c | 23 | ||||
-rw-r--r-- | driver/gator_main.c | 471 | ||||
-rwxr-xr-x | driver/gator_marshaling.c | 239 | ||||
-rw-r--r-- | driver/gator_pack.c | 98 | ||||
-rw-r--r-- | driver/gator_trace_gpu.c | 164 | ||||
-rwxr-xr-x | driver/gator_trace_power.c | 160 | ||||
-rw-r--r-- | driver/gator_trace_sched.c | 182 |
25 files changed, 1087 insertions, 1070 deletions
diff --git a/driver/Makefile b/driver/Makefile index 8824f8e..667637e 100644 --- a/driver/Makefile +++ b/driver/Makefile @@ -11,21 +11,10 @@ gator-y := gator_main.o \ gator_events_net.o \ gator_events_block.o \ gator_events_meminfo.o \ - gator_events_power.o \ gator_events_perf_pmu.o gator-y += gator_events_mmaped.o -ifeq ($(GATOR_WITH_MALI_SUPPORT),) -ifeq ($(GATOR_MALI_INCLUDE),) -GATOR_MALI_INCLUDE = $(abspath $(shell find -L . -name "mali_linux_trace.h" | sed -n -e '1s,\(.*\)/linux/mali_linux_trace.h$$,\1,p')) -endif -ifneq ($(GATOR_MALI_INCLUDE),) -GATOR_WITH_MALI_SUPPORT = MALI_400 # for now, assume all devices with Mali have the Mali-400 -EXTRA_CFLAGS += -I$(GATOR_MALI_INCLUDE) -endif -endif - ifneq ($(GATOR_WITH_MALI_SUPPORT),) ifeq ($(GATOR_WITH_MALI_SUPPORT),MALI_T6xx) gator-y += gator_events_mali_t6xx.o diff --git a/driver/gator.h b/driver/gator.h index a7a323c..6b96109 100644 --- a/driver/gator.h +++ b/driver/gator.h @@ -88,6 +88,7 @@ struct gator_interface { struct list_head list; }; +// gator_events_init is used as a search term in gator_events.sh #define gator_events_init(initfn) \ static inline int __gator_events_init_test(void) \ { return initfn(); } diff --git a/driver/gator_annotate.c b/driver/gator_annotate.c index 36a921c..b2288b3 100644 --- a/driver/gator_annotate.c +++ b/driver/gator_annotate.c @@ -15,65 +15,93 @@ #include <asm/current.h> #include <linux/spinlock.h> -#define ANNOTATE_SIZE (16*1024) static DEFINE_SPINLOCK(annotate_lock); -static char *annotateBuf; -static char *annotateBuf0; -static char *annotateBuf1; -static int annotatePos; -static int annotateSel; static bool collect_annotations = false; -static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) +static int annotate_copy(struct file *file, char const __user *buf, size_t count) { - char tempBuffer[512]; - int remaining, size; - uint32_t tid; + int cpu = 0; + int write = per_cpu(gator_buffer_write, cpu)[ANNOTATE_BUF]; + + if (file == NULL) { + // copy from kernel + memcpy(&per_cpu(gator_buffer, cpu)[ANNOTATE_BUF][write], buf, count); + } else { + // copy from user space + if (copy_from_user(&per_cpu(gator_buffer, cpu)[ANNOTATE_BUF][write], buf, count) != 0) + return -1; + } + per_cpu(gator_buffer_write, cpu)[ANNOTATE_BUF] = (write + count) & gator_buffer_mask[ANNOTATE_BUF]; + + return 0; +} + +static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count_orig, loff_t *offset) +{ + int tid, cpu, header_size, available, contiguous, length1, length2, size, count = count_orig & 0x7fffffff; if (*offset) return -EINVAL; - // determine size to capture - size = count < sizeof(tempBuffer) ? count : sizeof(tempBuffer); + if (!collect_annotations) { + return count_orig; + } + + cpu = 0; // Annotation only uses a single per-cpu buffer as the data must be in order to the engine - // note: copy may be for naught if remaining is zero, but better to do the copy outside of the spinlock if (file == NULL) { - // copy from kernel - memcpy(tempBuffer, buf, size); - - // set the thread id to the kernel thread, not the current thread - tid = -1; + tid = -1; // set the thread id to the kernel thread } else { - // copy from user space - if (copy_from_user(tempBuffer, buf, size) != 0) - return -EINVAL; tid = current->pid; } - // synchronize shared variables annotateBuf and annotatePos + // synchronize between cores spin_lock(&annotate_lock); - if (collect_annotations && annotateBuf) { - remaining = ANNOTATE_SIZE - annotatePos - 256; // pad for headers and release - size = size < remaining ? size : remaining; - if (size > 0) { - uint64_t time = gator_get_time(); - uint32_t cpuid = smp_processor_id(); - int pos = annotatePos; - pos += gator_write_packed_int(&annotateBuf[pos], tid); - pos += gator_write_packed_int64(&annotateBuf[pos], time); - pos += gator_write_packed_int(&annotateBuf[pos], cpuid); - pos += gator_write_packed_int(&annotateBuf[pos], size); - memcpy(&annotateBuf[pos], tempBuffer, size); - annotatePos = pos + size; - } - } - spin_unlock(&annotate_lock); + + // determine total size of the payload + header_size = MAXSIZE_PACK32 * 3 + MAXSIZE_PACK64; + available = buffer_bytes_available(cpu, ANNOTATE_BUF) - header_size; + size = count < available ? count : available; if (size <= 0) { - wake_up(&gator_buffer_wait); - return 0; + size = 0; + goto annotate_write_out; } + // synchronize shared variables annotateBuf and annotatePos + if (collect_annotations && per_cpu(gator_buffer, cpu)[ANNOTATE_BUF]) { + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, smp_processor_id()); + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, tid); + gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, gator_get_time()); + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, size); + + // determine the sizes to capture, length1 + length2 will equal size + contiguous = contiguous_space_available(cpu, ANNOTATE_BUF); + if (size < contiguous) { + length1 = size; + length2 = 0; + } else { + length1 = contiguous; + length2 = size - contiguous; + } + + if (annotate_copy(file, buf, length1) != 0) { + size = -EINVAL; + goto annotate_write_out; + } + + if (length2 > 0 && annotate_copy(file, &buf[length1], length2) != 0) { + size = -EINVAL; + goto annotate_write_out; + } + + // Check and commit; commit is set to occur once buffer is 3/4 full + buffer_check(cpu, ANNOTATE_BUF); + } + +annotate_write_out: + spin_unlock(&annotate_lock); + // return the number of bytes written return size; } @@ -82,21 +110,18 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t static int annotate_release(struct inode *inode, struct file *file) { - int remaining = ANNOTATE_SIZE - annotatePos; - if (remaining < 16) { - return -EFAULT; - } + int cpu = 0; + // synchronize between cores spin_lock(&annotate_lock); - if (annotateBuf) { + + if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF] && buffer_check_space(cpu, ANNOTATE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { uint32_t tid = current->pid; - int pos = annotatePos; - pos += gator_write_packed_int(&annotateBuf[pos], tid); - pos += gator_write_packed_int64(&annotateBuf[pos], 0); // time - pos += gator_write_packed_int(&annotateBuf[pos], 0); // cpuid - pos += gator_write_packed_int(&annotateBuf[pos], 0); // size - annotatePos = pos; + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, tid); + gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, 0); // time + gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, 0); // size } + spin_unlock(&annotate_lock); return 0; @@ -109,25 +134,11 @@ static const struct file_operations annotate_fops = { static int gator_annotate_create_files(struct super_block *sb, struct dentry *root) { - annotateBuf = NULL; return gatorfs_create_file_perm(sb, root, "annotate", &annotate_fops, 0666); } -static int gator_annotate_init(void) -{ - annotateBuf0 = kmalloc(ANNOTATE_SIZE, GFP_KERNEL); - annotateBuf1 = kmalloc(ANNOTATE_SIZE, GFP_KERNEL); - if (!annotateBuf0 || !annotateBuf1) - return -1; - return 0; -} - static int gator_annotate_start(void) { - annotateSel = 0; - annotatePos = 1; - annotateBuf = annotateBuf0; - annotateBuf[0] = FRAME_ANNOTATE; collect_annotations = true; return 0; } @@ -136,46 +147,3 @@ static void gator_annotate_stop(void) { collect_annotations = false; } - -static void gator_annotate_shutdown(void) -{ - spin_lock(&annotate_lock); - annotateBuf = NULL; - spin_unlock(&annotate_lock); -} - -static void gator_annotate_exit(void) -{ - spin_lock(&annotate_lock); - kfree(annotateBuf0); - kfree(annotateBuf1); - annotateBuf = annotateBuf0 = annotateBuf1 = NULL; - spin_unlock(&annotate_lock); -} - -static int gator_annotate_ready(void) -{ - return annotatePos > 1 && annotateBuf; -} - -static int gator_annotate_read(char **buffer) -{ - int len; - - if (!gator_annotate_ready()) - return 0; - - annotateSel = !annotateSel; - - if (buffer) - *buffer = annotateBuf; - - spin_lock(&annotate_lock); - len = annotatePos; - annotateBuf = annotateSel ? annotateBuf1 : annotateBuf0; - annotateBuf[0] = FRAME_ANNOTATE; - annotatePos = 1; - spin_unlock(&annotate_lock); - - return len; -} diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c index 26503ef..50783d6 100644 --- a/driver/gator_backtrace.c +++ b/driver/gator_backtrace.c @@ -81,8 +81,7 @@ static int report_trace(struct stackframe *frame, void *d) cookie = get_cookie(cpu, per_cpu(backtrace_buffer, cpu), current, NULL, mod, true); addr = addr - (unsigned long)mod->module_core; } - gator_buffer_write_packed_int(cpu, per_cpu(backtrace_buffer, cpu), addr & ~1); - gator_buffer_write_packed_int(cpu, per_cpu(backtrace_buffer, cpu), cookie); + marshal_backtrace(addr & ~1, cookie); (*depth)--; } @@ -110,7 +109,6 @@ static void kernel_backtrace(int cpu, int buftype, struct pt_regs * const regs) per_cpu(backtrace_buffer, cpu) = buftype; walk_stackframe(&frame, report_trace, &depth); #else - gator_buffer_write_packed_int(cpu, buftype, PC_REG & ~1); - gator_buffer_write_packed_int(cpu, buftype, NO_COOKIE); + marshal_backtrace(PC_REG & ~1, NO_COOKIE); #endif } diff --git a/driver/gator_cookies.c b/driver/gator_cookies.c index 1beb34f..7b50916 100644 --- a/driver/gator_cookies.c +++ b/driver/gator_cookies.c @@ -129,7 +129,7 @@ static void wq_cookie_handler(struct work_struct *unused) while (per_cpu(translate_buffer_read, cpu) != commit) { task = (struct task_struct *)translate_buffer_read_int(cpu); vma = (struct vm_area_struct *)translate_buffer_read_int(cpu); - cookie = get_cookie(cpu, TIMER_BUF, task, vma, NULL, false); + cookie = get_cookie(cpu, BACKTRACE_BUF, task, vma, NULL, false); } } @@ -251,13 +251,10 @@ static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task local_irq_save(flags); cookie = INVALID_COOKIE; - if (buffer_check_space(cpu, buftype, strlen(text) + 2 * MAXSIZE_PACK32)) { + if (marshal_cookie_header(text)) { cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids; cookiemap_add(key, cookie); - - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COOKIE); - gator_buffer_write_packed_int(cpu, buftype, cookie); - gator_buffer_write_string(cpu, buftype, text); + marshal_cookie(cookie, text); } local_irq_restore(flags); @@ -271,6 +268,7 @@ static int get_exec_cookie(int cpu, int buftype, struct task_struct *task) struct mm_struct *mm = task->mm; struct vm_area_struct *vma; + // kernel threads have no address space if (!mm) return cookie; diff --git a/driver/gator_ebs.c b/driver/gator_ebs.c index 8c2997c..1208d69 100644 --- a/driver/gator_ebs.c +++ b/driver/gator_ebs.c @@ -14,6 +14,8 @@ #include <asm/pmu.h> +static DEFINE_MUTEX(perf_mutex); + extern int pmnc_counters; extern int ccnt; extern unsigned long pmnc_enabled[]; @@ -32,59 +34,16 @@ static void ebs_overflow_handler(struct perf_event *event, int unused, struct pe static void ebs_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) #endif { - unsigned int value, delta, cpu = smp_processor_id(), buftype = EVENT_BUF; + int cpu = smp_processor_id(); if (event != per_cpu(pevent, cpu)) return; - if (buffer_check_space(cpu, buftype, 5 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) { - value = local64_read(&event->count); - delta = value - per_cpu(prev_value, cpu); - per_cpu(prev_value, cpu) = value; - - // Counters header - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type - gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time - - // Output counter - gator_buffer_write_packed_int(cpu, buftype, 2); // length - gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key - gator_buffer_write_packed_int(cpu, buftype, delta); // delta - - // End Counters, length of zero - gator_buffer_write_packed_int(cpu, buftype, 0); - } - // Output backtrace - if (buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32)) - gator_add_sample(cpu, buftype, regs); + gator_add_sample(cpu, BACKTRACE_BUF, regs); - // Check and commit; commit is set to occur once buffer is 3/4 full - buffer_check(cpu, buftype); -} - -static void gator_event_sampling_online(void) -{ - int cpu = smp_processor_id(), buftype = EVENT_BUF; - - // read the counter and toss the invalid data, return zero instead - struct perf_event * ev = per_cpu(pevent, cpu); - if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { - ev->pmu->read(ev); - per_cpu(prev_value, cpu) = local64_read(&ev->count); - - // Counters header - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type - gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time - - // Output counter - gator_buffer_write_packed_int(cpu, buftype, 2); // length - gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key - gator_buffer_write_packed_int(cpu, buftype, 0); // delta - zero for initialization - - // End Counters, length of zero - gator_buffer_write_packed_int(cpu, buftype, 0); - } + // Collect counters + collect_counters(); } static void gator_event_sampling_online_dispatch(int cpu) @@ -117,10 +76,18 @@ static void gator_event_sampling_online_dispatch(int cpu) static void gator_event_sampling_offline_dispatch(int cpu) { + struct perf_event * pe = NULL; + + mutex_lock(&perf_mutex); if (per_cpu(pevent, cpu)) { - perf_event_release_kernel(per_cpu(pevent, cpu)); + pe = per_cpu(pevent, cpu); per_cpu(pevent, cpu) = NULL; } + mutex_unlock(&perf_mutex); + + if (pe) { + perf_event_release_kernel(pe); + } } static int gator_event_sampling_start(void) @@ -184,7 +151,6 @@ static void gator_event_sampling_stop(void) } #else -static void gator_event_sampling_online(void) {} static void gator_event_sampling_online_dispatch(int cpu) {} static void gator_event_sampling_offline_dispatch(int cpu) {} static int gator_event_sampling_start(void) {return 0;} diff --git a/driver/gator_events_armv6.c b/driver/gator_events_armv6.c index ef51898..5f989ba 100644 --- a/driver/gator_events_armv6.c +++ b/driver/gator_events_armv6.c @@ -33,7 +33,6 @@ static unsigned long pmnc_enabled[CNTMAX]; static unsigned long pmnc_event[CNTMAX]; static unsigned long pmnc_key[CNTMAX]; -static DEFINE_PER_CPU(int[CNTMAX], perfPrev); static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); static inline void armv6_pmnc_write(u32 val) @@ -111,8 +110,6 @@ static int gator_events_armv6_online(int** buffer) for (pmnc = 0, cnt = PMN0; cnt <= CCNT; cnt++) { unsigned long event; - per_cpu(perfPrev, cpu)[cnt] = 0; - if (!pmnc_enabled[cnt]) continue; @@ -171,6 +168,11 @@ static int gator_events_armv6_read(int **buffer) int cnt, len = 0; int cpu = smp_processor_id(); + // a context switch may occur before the online hotplug event, thus need to check that the pmu is enabled + if (!(armv6_pmnc_read() & PMCR_E)) { + return 0; + } + for (cnt = PMN0; cnt <= CCNT; cnt++) { if (pmnc_enabled[cnt]) { u32 value = 0; @@ -186,11 +188,9 @@ static int gator_events_armv6_read(int **buffer) break; } armv6_pmnc_reset_counter(cnt); - if (value != per_cpu(perfPrev, cpu)[cnt]) { - per_cpu(perfPrev, cpu)[cnt] = value; - per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; - per_cpu(perfCnt, cpu)[len++] = value; - } + + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = value; } } diff --git a/driver/gator_events_armv7.c b/driver/gator_events_armv7.c index cdf450f..590421d 100644 --- a/driver/gator_events_armv7.c +++ b/driver/gator_events_armv7.c @@ -38,7 +38,6 @@ static unsigned long pmnc_enabled[CNTMAX]; static unsigned long pmnc_event[CNTMAX]; static unsigned long pmnc_key[CNTMAX]; -static DEFINE_PER_CPU(int[CNTMAX], perfPrev); static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); inline void armv7_pmnc_write(u32 val) @@ -177,8 +176,6 @@ static int gator_events_armv7_online(int** buffer) for (cnt = CCNT; cnt < CNTMAX; cnt++) { unsigned long event; - per_cpu(perfPrev, cpu)[cnt] = 0; - if (!pmnc_enabled[cnt]) continue; @@ -240,6 +237,11 @@ static int gator_events_armv7_read(int **buffer) int cnt, len = 0; int cpu = smp_processor_id(); + // a context switch may occur before the online hotplug event, thus need to check that the pmu is enabled + if (!(armv7_pmnc_read() & PMNC_E)) { + return 0; + } + for (cnt = 0; cnt < pmnc_counters; cnt++) { if (pmnc_enabled[cnt]) { int value; @@ -248,11 +250,8 @@ static int gator_events_armv7_read(int **buffer) } else { value = armv7_cntn_read(cnt, 0); } - if (value != per_cpu(perfPrev, cpu)[cnt]) { - per_cpu(perfPrev, cpu)[cnt] = value; - per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; - per_cpu(perfCnt, cpu)[len++] = value; - } + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = value; } } diff --git a/driver/gator_events_block.c b/driver/gator_events_block.c index f1bbbc8..a8b8114 100644 --- a/driver/gator_events_block.c +++ b/driver/gator_events_block.c @@ -26,7 +26,7 @@ static ulong block_rq_rd_enabled; static ulong block_rq_wr_key; static ulong block_rq_rd_key; static DEFINE_PER_CPU(int[BLOCK_TOTAL], blockCnt); -static DEFINE_PER_CPU(int[BLOCK_TOTAL * 2], blockGet); +static DEFINE_PER_CPU(int[BLOCK_TOTAL * 4], blockGet); static DEFINE_PER_CPU(bool, new_data_avail); GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct request *rq)) @@ -129,6 +129,8 @@ static int gator_events_block_read(int **buffer) per_cpu(blockCnt, cpu)[BLOCK_RQ_WR] = 0; local_irq_restore(flags); per_cpu(blockGet, cpu)[len++] = block_rq_wr_key; + per_cpu(blockGet, cpu)[len++] = 0; // indicates to Streamline that value bytes were written now, not since the last message + per_cpu(blockGet, cpu)[len++] = block_rq_wr_key; per_cpu(blockGet, cpu)[len++] = value; data += value; } @@ -138,6 +140,8 @@ static int gator_events_block_read(int **buffer) per_cpu(blockCnt, cpu)[BLOCK_RQ_RD] = 0; local_irq_restore(flags); per_cpu(blockGet, cpu)[len++] = block_rq_rd_key; + per_cpu(blockGet, cpu)[len++] = 0; // indicates to Streamline that value bytes were read now, not since the last message + per_cpu(blockGet, cpu)[len++] = block_rq_rd_key; per_cpu(blockGet, cpu)[len++] = value; data += value; } diff --git a/driver/gator_events_irq.c b/driver/gator_events_irq.c index 59461b9..435bc86 100644 --- a/driver/gator_events_irq.c +++ b/driver/gator_events_irq.c @@ -19,7 +19,6 @@ static ulong softirq_enabled; static ulong hardirq_key; static ulong softirq_key; static DEFINE_PER_CPU(int[TOTALIRQ], irqCnt); -static DEFINE_PER_CPU(int[TOTALIRQ], irqPrev); static DEFINE_PER_CPU(int[TOTALIRQ * 2], irqGet); GATOR_DEFINE_PROBE(irq_handler_exit, TP_PROTO(int irq, @@ -82,7 +81,6 @@ static int gator_events_irq_online(int** buffer) local_irq_save(flags); per_cpu(irqCnt, cpu)[HARDIRQ] = 0; local_irq_restore(flags); - per_cpu(irqPrev, cpu)[HARDIRQ] = 0; per_cpu(irqGet, cpu)[len++] = hardirq_key; per_cpu(irqGet, cpu)[len++] = 0; } @@ -91,7 +89,6 @@ static int gator_events_irq_online(int** buffer) local_irq_save(flags); per_cpu(irqCnt, cpu)[SOFTIRQ] = 0; local_irq_restore(flags); - per_cpu(irqPrev, cpu)[SOFTIRQ] = 0; per_cpu(irqGet, cpu)[len++] = softirq_key; per_cpu(irqGet, cpu)[len++] = 0; } @@ -149,11 +146,9 @@ static int gator_events_irq_read(int **buffer) value = per_cpu(irqCnt, cpu)[HARDIRQ]; per_cpu(irqCnt, cpu)[HARDIRQ] = 0; local_irq_restore(flags); - if (value != per_cpu(irqPrev, cpu)[HARDIRQ]) { - per_cpu(irqPrev, cpu)[HARDIRQ] = value; - per_cpu(irqGet, cpu)[len++] = hardirq_key; - per_cpu(irqGet, cpu)[len++] = value; - } + + per_cpu(irqGet, cpu)[len++] = hardirq_key; + per_cpu(irqGet, cpu)[len++] = value; } if (softirq_enabled) { @@ -161,11 +156,9 @@ static int gator_events_irq_read(int **buffer) value = per_cpu(irqCnt, cpu)[SOFTIRQ]; per_cpu(irqCnt, cpu)[SOFTIRQ] = 0; local_irq_restore(flags); - if (value != per_cpu(irqPrev, cpu)[SOFTIRQ]) { - per_cpu(irqPrev, cpu)[SOFTIRQ] = value; - per_cpu(irqGet, cpu)[len++] = softirq_key; - per_cpu(irqGet, cpu)[len++] = value; - } + + per_cpu(irqGet, cpu)[len++] = softirq_key; + per_cpu(irqGet, cpu)[len++] = value; } if (buffer) diff --git a/driver/gator_events_mali.c b/driver/gator_events_mali.c index 31e8f0d..21a6324 100644..100755 --- a/driver/gator_events_mali.c +++ b/driver/gator_events_mali.c @@ -14,17 +14,41 @@ #include "linux/mali_linux_trace.h" -#define ACTIVITY_START 1 -#define ACTIVITY_STOP 2 +/* + * There are (currently) three different variants of the comms between gator and Mali: + * 1 (deprecated): No software counter support + * 2 (deprecated): Tracepoint called for each separate s/w counter value as it appears + * 3 (default): Single tracepoint for all s/w counters in a bundle. + * Interface style 3 is the default if no other is specified. 1 and 2 will be eliminated when + * existing Mali DDKs are upgraded. + */ -#ifndef MALI_SUPPORT -#error MALI_SUPPORT not defined! +#if !defined(GATOR_MALI_INTERFACE_STYLE) +#define GATOR_MALI_INTERFACE_STYLE (3) #endif -#define MALI_200 0x0a07 -#define MALI_300 0x0b06 //This is not actually true; Mali-300 is also 0x0b07 -#define MALI_400 0x0b07 -#define MALI_T6xx 0x0056 +#define MALI_200 (0x0a07) +#define MALI_300 (0x0b06) //This is not actually true; Mali-300 is also 0x0b07 +#define MALI_400 (0x0b07) +#define MALI_T6xx (0x0056) + +/* + * List of possible actions allowing DDK to be controlled by Streamline. + * The following numbers are used by DDK to control the frame buffer dumping. + */ +#define FBDUMP_CONTROL_ENABLE (1) +#define FBDUMP_CONTROL_RATE (2) +#define SW_EVENTS_ENABLE (3) +#define FBDUMP_CONTROL_RESIZE_FACTOR (4) + +/* + * Check that the MALI_SUPPORT define is set to one of the allowable device codes. + */ +#if !defined(MALI_SUPPORT) +#error MALI_SUPPORT not defined! +#elif (MALI_SUPPORT != MALI_200) && (MALI_SUPPORT != MALI_300) && (MALI_SUPPORT != MALI_400) && (MALI_SUPPORT != MALI_T6xx) +#error MALI_SUPPORT set to an invalid device code +#endif static const char *mali_name; @@ -226,12 +250,31 @@ GATOR_DEFINE_PROBE(mali_hw_counter, TP_PROTO(unsigned int event_id, unsigned int } } +#if GATOR_MALI_INTERFACE_STYLE == 2 GATOR_DEFINE_PROBE(mali_sw_counter, TP_PROTO(unsigned int event_id, signed long long value)) { if (is_sw_counter(event_id)) { counter_data[event_id] = scale_sw_counter_value(event_id, value); } } +#endif /* GATOR_MALI_INTERFACE_STYLE == 2 */ + + +#if GATOR_MALI_INTERFACE_STYLE == 3 +GATOR_DEFINE_PROBE(mali_sw_counters, TP_PROTO(pid_t pid, pid_t tid, void * surface_id, unsigned int * counters)) +{ + u32 i; + + /* Copy over the values for those counters which are enabled. */ + for(i=FIRST_SW_COUNTER; i <= LAST_SW_COUNTER; i++) + { + if(counter_enabled[i]) + { + counter_data[i] = (u32)(counters[i - FIRST_SW_COUNTER]); + } + } +} +#endif /* GATOR_MALI_INTERFACE_STYLE == 3 */ //TODO need to work out how many fp units we have u32 gator_mali_get_n_fp(void) { @@ -363,10 +406,30 @@ int gator_events_mali_create_files(struct super_block *sb, struct dentry *root) //TODO void _mali_profiling_set_event(unsigned int, unsigned int); void _mali_osk_fb_control_set(unsigned int, unsigned int); +void _mali_profiling_control(unsigned int, unsigned int); void _mali_profiling_get_counters(unsigned int*, unsigned int*, unsigned int*, unsigned int*); void (*_mali_profiling_get_counters_function_pointer)(unsigned int*, unsigned int*, unsigned int*, unsigned int*); +/* + * Examine list of software counters and determine if any one is enabled. + * Returns 1 if any counter is enabled, 0 if none is. + */ +static int is_any_sw_counter_enabled(void) +{ + unsigned int i; + + for (i = FIRST_SW_COUNTER; i <= LAST_SW_COUNTER; i++) + { + if (counter_enabled[i]) + { + return 1; /* At least one counter is enabled */ + } + } + + return 0; /* No s/w counters enabled */ +} + static void mali_counter_initialize(void) { /* If a Mali driver is present and exporting the appropriate symbol @@ -375,6 +438,7 @@ static void mali_counter_initialize(void) */ void (*set_hw_event)(unsigned int, unsigned int); void (*set_fb_event)(unsigned int, unsigned int); + void (*mali_control)(unsigned int, unsigned int); set_hw_event = symbol_get(_mali_profiling_set_event); @@ -408,6 +472,27 @@ static void mali_counter_initialize(void) printk("gator: mali online _mali_osk_fb_control_set symbol not found\n"); } + /* Generic control interface for Mali DDK. */ + mali_control = symbol_get(_mali_profiling_control); + if (mali_control) { + /* The event attribute in the XML file keeps the actual frame rate. */ + unsigned int rate = counter_event[COUNTER_FILMSTRIP] & 0xff; + unsigned int resize_factor = (counter_event[COUNTER_FILMSTRIP] >> 8) & 0xff; + + pr_debug("gator: mali online _mali_profiling_control symbol @ %p\n", mali_control); + + mali_control(SW_EVENTS_ENABLE, (is_any_sw_counter_enabled()?1:0)); + mali_control(FBDUMP_CONTROL_ENABLE, (counter_enabled[COUNTER_FILMSTRIP]?1:0)); + mali_control(FBDUMP_CONTROL_RATE, rate); + mali_control(FBDUMP_CONTROL_RESIZE_FACTOR, resize_factor); + + pr_debug("gator: sent mali_control enabled=%d, rate=%d\n", (counter_enabled[COUNTER_FILMSTRIP]?1:0), rate); + + symbol_put(_mali_profiling_control); + } else { + printk("gator: mali online _mali_profiling_control symbol not found\n"); + } + _mali_profiling_get_counters_function_pointer = symbol_get(_mali_profiling_get_counters); if (_mali_profiling_get_counters_function_pointer){ pr_debug("gator: mali online _mali_profiling_get_counters symbol @ %p\n", _mali_profiling_get_counters_function_pointer); @@ -417,13 +502,13 @@ static void mali_counter_initialize(void) else{ pr_debug("gator WARNING: mali _mali_profiling_get_counters symbol not defined"); } - } static void mali_counter_deinitialize(void) { void (*set_hw_event)(unsigned int, unsigned int); void (*set_fb_event)(unsigned int, unsigned int); + void (*mali_control)(unsigned int, unsigned int); set_hw_event = symbol_get(_mali_profiling_set_event); @@ -452,6 +537,22 @@ static void mali_counter_deinitialize(void) printk("gator: mali offline _mali_osk_fb_control_set symbol not found\n"); } + /* Generic control interface for Mali DDK. */ + mali_control = symbol_get(_mali_profiling_control); + + if (mali_control) { + pr_debug("gator: mali offline _mali_profiling_control symbol @ %p\n", set_fb_event); + + /* Reset the DDK state - disable counter collection */ + mali_control(SW_EVENTS_ENABLE, 0); + + mali_control(FBDUMP_CONTROL_ENABLE, 0); + + symbol_put(_mali_profiling_control); + } else { + printk("gator: mali offline _mali_profiling_control symbol not found\n"); + } + if (_mali_profiling_get_counters_function_pointer){ symbol_put(_mali_profiling_get_counters); } @@ -465,10 +566,23 @@ static int gator_events_mali_start(void) { return -1; } +#if GATOR_MALI_INTERFACE_STYLE == 1 + /* None. */ +#elif GATOR_MALI_INTERFACE_STYLE == 2 + /* For patched Mali driver. */ if (GATOR_REGISTER_TRACE(mali_sw_counter)) { printk("gator: mali_sw_counter tracepoint failed to activate\n"); return -1; } +#elif GATOR_MALI_INTERFACE_STYLE == 3 +/* For Mali drivers with built-in support. */ + if (GATOR_REGISTER_TRACE(mali_sw_counters)) { + printk("gator: mali_sw_counters tracepoint failed to activate\n"); + return -1; + } +#else +#error Unknown GATOR_MALI_INTERFACE_STYLE option. +#endif trace_registered = 1; @@ -483,8 +597,19 @@ static void gator_events_mali_stop(void) { if (trace_registered) { GATOR_UNREGISTER_TRACE(mali_hw_counter); - GATOR_UNREGISTER_TRACE(mali_sw_counter); - + +#if GATOR_MALI_INTERFACE_STYLE == 1 + /* None. */ +#elif GATOR_MALI_INTERFACE_STYLE == 2 + /* For patched Mali driver. */ + GATOR_UNREGISTER_TRACE(mali_sw_counter); +#elif GATOR_MALI_INTERFACE_STYLE == 3 + /* For Mali drivers with built-in support. */ + GATOR_UNREGISTER_TRACE(mali_sw_counters); +#else +#error Unknown GATOR_MALI_INTERFACE_STYLE option. +#endif + pr_debug("gator: mali timeline tracepoint deactivated\n"); trace_registered = 0; @@ -538,22 +663,10 @@ static int gator_events_mali_read(int **buffer) { // Process other (non-timeline) counters. for (cnt = COUNTER_VP_C0; cnt <= LAST_SW_COUNTER; cnt++) { if (counter_enabled[cnt]) { - u32 value = 0; - - // Determine the current value of the counter. - if( counter_address[cnt] != NULL && 0 ) { // Never true! - value = *counter_address[cnt]; - } else if (counter_data[cnt]!=0) { - value = counter_data[cnt]; - counter_data[cnt] = 0; - } + counter_dump[len++] = counter_key[cnt]; + counter_dump[len++] = counter_data[cnt]; - // Send the counter value only if it differs from last time. - if (value != counter_prev[cnt]) { - counter_prev[cnt] = value; - counter_dump[len++] = counter_key[cnt]; - counter_dump[len++] = value; - } + counter_data[cnt] = 0; } } diff --git a/driver/gator_events_meminfo.c b/driver/gator_events_meminfo.c index c763634..ad552ef 100644 --- a/driver/gator_events_meminfo.c +++ b/driver/gator_events_meminfo.c @@ -10,6 +10,7 @@ #include "gator.h" #include <linux/workqueue.h> #include <trace/events/kmem.h> +#include <linux/hardirq.h> #define MEMINFO_MEMFREE 0 #define MEMINFO_MEMUSED 1 @@ -145,7 +146,7 @@ static void gator_events_meminfo_stop(void) } } -// Must be run in a work queue as the kernel function si_meminfo() can sleep +// Must be run in process context (work queue) as the kernel function si_meminfo() can sleep static void wq_sched_handler(struct work_struct *wsptr) { struct sysinfo info; @@ -189,7 +190,11 @@ static int gator_events_meminfo_read(long long **buffer) if (last_mem_event != mem_event) { last_mem_event = mem_event; - schedule_work(&work); + if (in_interrupt()) { + schedule_work(&work); + } else { + wq_sched_handler(NULL); + } } if (!new_data_avail) diff --git a/driver/gator_events_mmaped.c b/driver/gator_events_mmaped.c index cbb22b1..f81c402 100644 --- a/driver/gator_events_mmaped.c +++ b/driver/gator_events_mmaped.c @@ -14,11 +14,12 @@ * <counter_set name="mmaped_cntX"> * <counter name="mmaped_cnt0"/> * <counter name="mmaped_cnt1"/> + * <counter name="mmaped_cnt2"/> * </counter_set> * <category name="mmaped" counter_set="mmaped_cntX" per_cpu="no"> - * <event event="0x0" title="Simulated" name="Sine" description="Sort-of-sine"/> - * <event event="0x1" title="Simulated" name="Triangle" description="Triangular wave"/> - * <event event="0x2" title="Simulated" name="PWM" description="PWM Signal"/> + * <event event="0x0" title="Simulated" name="Sine" display="maximum" average_selection="yes" description="Sort-of-sine"/> + * <event event="0x1" title="Simulated" name="Triangle" display="maximum" average_selection="yes" description="Triangular wave"/> + * <event event="0x2" title="Simulated" name="PWM" display="maximum" average_selection="yes" description="PWM Signal"/> * </category> */ @@ -42,6 +43,10 @@ static int mmaped_buffer[MMAPED_COUNTERS_NUM * 2]; static void __iomem *mmaped_base; #endif +#ifndef TODO +static s64 prev_time; +#endif + /* Adds mmaped_cntX directories and enabled, event, and key files to /dev/gator/events */ static int gator_events_mmaped_create_files(struct super_block *sb, struct dentry *root) @@ -77,6 +82,12 @@ static int gator_events_mmaped_start(void) writel(ENABLED, COUNTERS_CONTROL_OFFSET); #endif +#ifndef TODO + struct timespec ts; + getnstimeofday(&ts); + prev_time = timespec_to_ns(&ts); +#endif + return 0; } @@ -90,50 +101,60 @@ static void gator_events_mmaped_stop(void) #ifndef TODO /* This function "simulates" counters, generating values of fancy * functions like sine or triangle... */ -static int mmaped_simulate(int counter) +static int mmaped_simulate(int counter, int delta_in_us) { int result = 0; switch (counter) { case 0: /* sort-of-sine */ { - static int t; + static int t = 0; int x; - if (t % 1024 < 512) - x = 512 - (t % 512); + t += delta_in_us; + if (t > 2048000) + t = 0; + + if (t % 1024000 < 512000) + x = 512000 - (t % 512000); else - x = t % 512; + x = t % 512000; - result = 32 * x / 512; + result = 32 * x / 512000; result = result * result; - if (t < 1024) + if (t < 1024000) result = 1922 - result; - - t = (t + 1) % 2048; } break; case 1: /* triangle */ { static int v, d = 1; - v += d; - if (v % 2000 == 0) - d = -d; + v = v + d * delta_in_us; + if (v < 0) { + v = 0; + d = 1; + } else if (v > 1000000) { + v = 1000000; + d = -1; + } result = v; } break; case 2: /* PWM signal */ { - static int t, dc; - - t = (t + 1) % 2000; - if (t % 100) - dc = (dc + 200) % 2000; + static int t, dc, x; + + t += delta_in_us; + if (t > 1000000) + t = 0; + if (x / 1000000 != (x + delta_in_us) / 1000000) + dc = (dc + 100000) % 1000000; + x += delta_in_us; - result = t < dc ? 0 : 2000; + result = t < dc ? 0 : 10; } break; } @@ -146,11 +167,23 @@ static int gator_events_mmaped_read(int **buffer) { int i; int len = 0; +#ifndef TODO + int delta_in_us; + struct timespec ts; + s64 time; +#endif /* System wide counters - read from one core only */ if (smp_processor_id()) return 0; +#ifndef TODO + getnstimeofday(&ts); + time = timespec_to_ns(&ts); + delta_in_us = (int)(time - prev_time) / 1000; + prev_time = time; +#endif + for (i = 0; i < MMAPED_COUNTERS_NUM; i++) { if (mmaped_counters[i].enabled) { mmaped_buffer[len++] = mmaped_counters[i].key; @@ -159,7 +192,7 @@ static int gator_events_mmaped_read(int **buffer) COUNTERS_VALUE_OFFSET[i]); #else mmaped_buffer[len++] = mmaped_simulate( - mmaped_counters[i].event); + mmaped_counters[i].event, delta_in_us); #endif } } diff --git a/driver/gator_events_net.c b/driver/gator_events_net.c index ef1623b..9298905 100644 --- a/driver/gator_events_net.c +++ b/driver/gator_events_net.c @@ -9,6 +9,7 @@ #include "gator.h" #include <linux/netdevice.h> +#include <linux/hardirq.h> #define NETRX 0 #define NETTX 1 @@ -20,7 +21,7 @@ static ulong netrx_key; static ulong nettx_key; static int rx_total, tx_total; static ulong netPrev[TOTALNET]; -static int netGet[TOTALNET * 2]; +static int netGet[TOTALNET * 4]; static void get_network_stats(struct work_struct *wsptr) { int rx = 0, tx = 0; @@ -102,19 +103,31 @@ static int gator_events_net_read(int **buffer) if (smp_processor_id() != 0) return 0; - schedule_work(&wq_get_stats); + if (!netrx_enabled && !nettx_enabled) + return 0; + + if (in_interrupt()){ + schedule_work(&wq_get_stats); + } else { + get_network_stats(NULL); + } + calculate_delta(&rx_delta, &tx_delta); len = 0; if (netrx_enabled && last_rx_delta != rx_delta) { last_rx_delta = rx_delta; netGet[len++] = netrx_key; + netGet[len++] = 0; // indicates to Streamline that rx_delta bytes were transmitted now, not since the last message + netGet[len++] = netrx_key; netGet[len++] = rx_delta; } if (nettx_enabled && last_tx_delta != tx_delta) { last_tx_delta = tx_delta; netGet[len++] = nettx_key; + netGet[len++] = 0; // indicates to Streamline that tx_delta bytes were transmitted now, not since the last message + netGet[len++] = nettx_key; netGet[len++] = tx_delta; } diff --git a/driver/gator_events_perf_pmu.c b/driver/gator_events_perf_pmu.c index 322ebc4..da76a9c 100644..100755 --- a/driver/gator_events_perf_pmu.c +++ b/driver/gator_events_perf_pmu.c @@ -19,6 +19,8 @@ int ccnt = 0; #define CNTMAX (6+1) +static DEFINE_MUTEX(perf_mutex); + unsigned long pmnc_enabled[CNTMAX]; unsigned long pmnc_event[CNTMAX]; unsigned long pmnc_count[CNTMAX]; @@ -122,12 +124,20 @@ static void gator_events_perf_pmu_online_dispatch(int cpu) static void gator_events_perf_pmu_offline_dispatch(int cpu) { int cnt; + struct perf_event * pe; for (cnt = 0; cnt < pmnc_counters; cnt++) { - if (per_cpu(pevent, cpu)[cnt] != NULL) { - perf_event_release_kernel(per_cpu(pevent, cpu)[cnt]); + pe = NULL; + mutex_lock(&perf_mutex); + if (per_cpu(pevent, cpu)[cnt]) { + pe = per_cpu(pevent, cpu)[cnt]; per_cpu(pevent, cpu)[cnt] = NULL; } + mutex_unlock(&perf_mutex); + + if (pe) { + perf_event_release_kernel(pe); + } } } @@ -139,7 +149,7 @@ static int gator_events_perf_pmu_start(void) for_each_present_cpu(cpu) { for (cnt = 0; cnt < pmnc_counters; cnt++) { per_cpu(pevent, cpu)[cnt] = NULL; - if (!pmnc_enabled[cnt] || pmnc_count[cnt] > 0) // Skip disabled counters and EBS counters + if (!pmnc_enabled[cnt]) // Skip disabled counters continue; per_cpu(perfPrev, cpu)[cnt] = 0; diff --git a/driver/gator_events_power.c b/driver/gator_events_power.c deleted file mode 100644 index a0ae684..0000000 --- a/driver/gator_events_power.c +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Copyright (C) ARM Limited 2011-2012. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - */ - -#include "gator.h" -#include <linux/slab.h> -#include <linux/cpufreq.h> -#include <trace/events/power.h> - -// cpu_frequency and cpu_idle trace points were introduced in Linux kernel v2.6.38 -// the now deprecated power_frequency trace point was available prior to 2.6.38, but only for x86 -// CPU Idle is currently disabled in the .xml -#if GATOR_CPU_FREQ_SUPPORT -enum { - POWER_CPU_FREQ, - POWER_CPU_IDLE, - POWER_TOTAL -}; - -static ulong power_cpu_enabled[POWER_TOTAL]; -static ulong power_cpu_key[POWER_TOTAL]; -static DEFINE_PER_CPU(ulong[POWER_TOTAL], power); -static DEFINE_PER_CPU(ulong[POWER_TOTAL], prev); -static DEFINE_PER_CPU(int *, powerGet); - -GATOR_DEFINE_PROBE(cpu_frequency, TP_PROTO(unsigned int frequency, unsigned int cpu)) -{ - per_cpu(power, cpu)[POWER_CPU_FREQ] = frequency * 1000; -} - -GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu)) -{ - per_cpu(power, cpu)[POWER_CPU_IDLE] = state; -} - -static int gator_events_power_create_files(struct super_block *sb, struct dentry *root) -{ - struct dentry *dir; - - // cpu_frequency - dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_freq"); - if (!dir) { - return -1; - } - gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_FREQ]); - gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_FREQ]); - - // cpu_idle - dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_idle"); - if (!dir) { - return -1; - } - gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_IDLE]); - gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_IDLE]); - - return 0; -} - -static int gator_events_power_populate(int cpu, int** buffer) -{ - int i, len = 0; - - for (i = 0; i < POWER_TOTAL; i++) { - if (power_cpu_enabled[i]) { - if (per_cpu(power, cpu)[i] != per_cpu(prev, cpu)[i]) { - per_cpu(prev, cpu)[i] = per_cpu(power, cpu)[i]; - per_cpu(powerGet, cpu)[len++] = power_cpu_key[i]; - per_cpu(powerGet, cpu)[len++] = per_cpu(power, cpu)[i]; - } - } - } - - if (buffer) - *buffer = per_cpu(powerGet, cpu); - - return len; -} - -static int gator_events_power_online(int** buffer) -{ - int i, cpu = smp_processor_id(); - for (i = 0; i < POWER_TOTAL; i++) - per_cpu(prev, cpu)[i] = -1; - per_cpu(power, cpu)[POWER_CPU_FREQ] = cpufreq_quick_get(cpu) * 1000; - return gator_events_power_populate(cpu, buffer); -} - -static int gator_events_power_offline(int** buffer) -{ - int cpu = smp_processor_id(); - // Set frequency to zero on an offline - per_cpu(power, cpu)[POWER_CPU_FREQ] = 0; - return gator_events_power_populate(cpu, buffer); -} - -static int gator_events_power_start(void) -{ - int cpu; - - for_each_present_cpu(cpu) { - per_cpu(powerGet, cpu) = kmalloc(POWER_TOTAL * 2, GFP_KERNEL); - if (!per_cpu(powerGet, cpu)) - return -1; - } - - // register tracepoints - if (power_cpu_enabled[POWER_CPU_FREQ]) - if (GATOR_REGISTER_TRACE(cpu_frequency)) - goto fail_cpu_frequency_exit; - if (power_cpu_enabled[POWER_CPU_IDLE]) - if (GATOR_REGISTER_TRACE(cpu_idle)) - goto fail_cpu_idle_exit; - pr_debug("gator: registered power event tracepoints\n"); - - return 0; - - // unregister tracepoints on error -fail_cpu_idle_exit: - GATOR_UNREGISTER_TRACE(cpu_frequency); -fail_cpu_frequency_exit: - pr_err("gator: power event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); - - return -1; -} - -static void gator_events_power_stop(void) -{ - int i, cpu; - if (power_cpu_enabled[POWER_CPU_FREQ]) - GATOR_UNREGISTER_TRACE(cpu_frequency); - if (power_cpu_enabled[POWER_CPU_IDLE]) - GATOR_UNREGISTER_TRACE(cpu_idle); - pr_debug("gator: unregistered power event tracepoints\n"); - - for (i = 0; i < POWER_TOTAL; i++) { - power_cpu_enabled[i] = 0; - } - - for_each_present_cpu(cpu) { - kfree(per_cpu(powerGet, cpu)); - } -} - -static int gator_events_power_read(int **buffer) -{ - return gator_events_power_populate(smp_processor_id(), buffer); -} - -static struct gator_interface gator_events_power_interface = { - .create_files = gator_events_power_create_files, - .online = gator_events_power_online, - .offline = gator_events_power_offline, - .start = gator_events_power_start, - .stop = gator_events_power_stop, - .read = gator_events_power_read, -}; -#endif - -int gator_events_power_init(void) -{ -#if (GATOR_CPU_FREQ_SUPPORT) - int i; - for (i = 0; i < POWER_TOTAL; i++) { - power_cpu_enabled[i] = 0; - power_cpu_key[i] = gator_events_get_key(); - } - - return gator_events_install(&gator_events_power_interface); -#else - return -1; -#endif -} -gator_events_init(gator_events_power_init); diff --git a/driver/gator_events_scorpion.c b/driver/gator_events_scorpion.c index 477e7c9..ed0d8de 100644 --- a/driver/gator_events_scorpion.c +++ b/driver/gator_events_scorpion.c @@ -34,7 +34,6 @@ static unsigned long pmnc_enabled[CNTMAX]; static unsigned long pmnc_event[CNTMAX]; static unsigned long pmnc_key[CNTMAX]; -static DEFINE_PER_CPU(int[CNTMAX], perfPrev); static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); enum scorpion_perf_types { @@ -538,8 +537,6 @@ static int gator_events_scorpion_online(int** buffer) for (cnt = CCNT; cnt < CNTMAX; cnt++) { unsigned long event; - per_cpu(perfPrev, smp_processor_id())[cnt] = 0; - if (!pmnc_enabled[cnt]) continue; @@ -574,7 +571,7 @@ static int gator_events_scorpion_online(int** buffer) value = 0; } scorpion_pmnc_reset_counter(cnt); - per_cpu(perfPrev, cpu)[cnt] = 0; + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; per_cpu(perfCnt, cpu)[len++] = 0; } @@ -607,6 +604,11 @@ static int gator_events_scorpion_read(int **buffer) int cnt, len = 0; int cpu = smp_processor_id(); + // a context switch may occur before the online hotplug event, thus need to check that the pmu is enabled + if (!(scorpion_pmnc_read() & PMNC_E)) { + return 0; + } + for (cnt = 0; cnt < pmnc_counters; cnt++) { if (pmnc_enabled[cnt]) { int value; @@ -618,11 +620,9 @@ static int gator_events_scorpion_read(int **buffer) value = 0; } scorpion_pmnc_reset_counter(cnt); - if (value != per_cpu(perfPrev, cpu)[cnt]) { - per_cpu(perfPrev, cpu)[cnt] = value; - per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; - per_cpu(perfCnt, cpu)[len++] = value; - } + + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = value; } } diff --git a/driver/gator_fs.c b/driver/gator_fs.c index 8277c3a..39adfbe 100644 --- a/driver/gator_fs.c +++ b/driver/gator_fs.c @@ -233,9 +233,17 @@ static int gatorfs_fill_super(struct super_block *sb, void *data, int silent) return -ENOMEM; root_inode->i_op = &simple_dir_inode_operations; root_inode->i_fop = &simple_dir_operations; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) root_dentry = d_alloc_root(root_inode); +#else + root_dentry = d_make_root(root_inode); +#endif + if (!root_dentry) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0) iput(root_inode); +#endif return -ENOMEM; } diff --git a/driver/gator_hrtimer_gator.c b/driver/gator_hrtimer_gator.c index 5896b3c..846fba4 100644..100755 --- a/driver/gator_hrtimer_gator.c +++ b/driver/gator_hrtimer_gator.c @@ -13,6 +13,7 @@ void (*callback)(void); DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); +DEFINE_PER_CPU(int, hrtimer_is_active); static ktime_t profiling_interval; static void gator_hrtimer_online(int cpu); static void gator_hrtimer_offline(int cpu); @@ -32,11 +33,16 @@ static void gator_hrtimer_switch_cpus_online(void *unused) static void gator_hrtimer_online(int cpu) { struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); + if (cpu != smp_processor_id()) { smp_call_function_single(cpu, gator_hrtimer_switch_cpus_online, NULL, 1); return; } + if (per_cpu(hrtimer_is_active, cpu) || profiling_interval.tv64 == 0) + return; + + per_cpu(hrtimer_is_active, cpu) = 1; hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); hrtimer->function = gator_hrtimer_notify; hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED); @@ -50,20 +56,35 @@ static void gator_hrtimer_switch_cpus_offline(void *unused) static void gator_hrtimer_offline(int cpu) { struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); + if (cpu != smp_processor_id()) { smp_call_function_single(cpu, gator_hrtimer_switch_cpus_offline, NULL, 1); return; } + if (!per_cpu(hrtimer_is_active, cpu)) + return; + + per_cpu(hrtimer_is_active, cpu) = 0; hrtimer_cancel(hrtimer); } static int gator_hrtimer_init(int interval, void (*func)(void)) { + int cpu; + (callback) = (func); + for_each_present_cpu(cpu) { + per_cpu(hrtimer_is_active, cpu) = 0; + } + // calculate profiling interval - profiling_interval = ns_to_ktime(1000000000UL / interval); + if (interval > 0) { + profiling_interval = ns_to_ktime(1000000000UL / interval); + } else { + profiling_interval.tv64 = 0; + } return 0; } diff --git a/driver/gator_main.c b/driver/gator_main.c index fff2d19..7d48812 100644 --- a/driver/gator_main.c +++ b/driver/gator_main.c @@ -7,7 +7,7 @@ * */ -static unsigned long gator_protocol_version = 8; +static unsigned long gator_protocol_version = 9; #include <linux/slab.h> #include <linux/cpu.h> @@ -39,7 +39,7 @@ static unsigned long gator_protocol_version = 8; #endif #ifndef CONFIG_HIGH_RES_TIMERS -#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined +#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling #endif #if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS) @@ -48,37 +48,39 @@ static unsigned long gator_protocol_version = 8; #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT)) #ifndef CONFIG_PERF_EVENTS -#warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters +#error gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters #elif !defined CONFIG_HW_PERF_EVENTS -#warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters +#error gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters #endif #endif -#if (!(GATOR_CPU_FREQ_SUPPORT)) -#warning gator requires kernel version 2.6.38 or greater and CONFIG_CPU_FREQ defined in order to enable the CPU Freq timeline chart -#endif - /****************************************************************************** * DEFINES ******************************************************************************/ -#define TIMER_BUFFER_SIZE_DEFAULT (512*1024) -#define EVENT_BUFFER_SIZE_DEFAULT (128*1024) +#define BACKTRACE_BUFFER_SIZE (128*1024) +#define COUNTER_BUFFER_SIZE (128*1024) +#define ANNOTATE_BUFFER_SIZE (64*1024) // annotate counters have the core as part of the data and the core value in the frame header may be discarded +#define SCHED_TRACE_BUFFER_SIZE (128*1024) +#define GPU_TRACE_BUFFER_SIZE (64*1024) +#define COUNTER2_BUFFER_SIZE (64*1024) // counters2 counters have the core as part of the data and the core value in the frame header may be discarded +#define WFI_BUFFER_SIZE (32*1024) // wfi counters have the core as part of the data and the core value in the frame header may be discarded #define NO_COOKIE 0UL #define INVALID_COOKIE ~0UL -#define FRAME_HRTIMER 1 -#define FRAME_EVENT 2 -#define FRAME_ANNOTATE 3 +#define FRAME_BACKTRACE 1 +#define FRAME_COUNTER 2 +#define FRAME_ANNOTATE 3 +#define FRAME_SCHED_TRACE 4 +#define FRAME_GPU_TRACE 5 +#define FRAME_COUNTER2 6 +#define FRAME_WFI 7 -#define MESSAGE_COOKIE 1 -#define MESSAGE_COUNTERS 3 -#define MESSAGE_START_BACKTRACE 5 -#define MESSAGE_END_BACKTRACE 7 -#define MESSAGE_SCHEDULER_TRACE 9 -#define MESSAGE_PID_NAME 11 -#define MESSAGE_GPU_TRACE 13 -#define MESSAGE_OVERFLOW 127 +#define MESSAGE_COOKIE 1 +#define MESSAGE_START_BACKTRACE 5 +#define MESSAGE_END_BACKTRACE 7 +#define MESSAGE_SUMMARY 9 +#define MESSAGE_PID_NAME 11 #define MAXSIZE_PACK32 5 #define MAXSIZE_PACK64 9 @@ -89,7 +91,7 @@ static unsigned long gator_protocol_version = 8; #define PC_REG regs->ip #endif -enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS}; +enum {BACKTRACE_BUF, COUNTER_BUF, SCHED_TRACE_BUF, GPU_TRACE_BUF, ANNOTATE_BUF, COUNTER2_BUF, WFI_BUF, NUM_GATOR_BUFS}; /****************************************************************************** * Globals @@ -101,25 +103,26 @@ static unsigned long gator_backtrace_depth; static unsigned long gator_started; static unsigned long gator_buffer_opened; static unsigned long gator_timer_count; -static unsigned long gator_streaming; +static unsigned long gator_response_type; static DEFINE_MUTEX(start_mutex); static DEFINE_MUTEX(gator_buffer_mutex); bool event_based_sampling; static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); - -static void buffer_check(int cpu, int buftype); +static LIST_HEAD(gator_events); /****************************************************************************** * Prototypes ******************************************************************************/ +static void buffer_check(int cpu, int buftype); +static int buffer_bytes_available(int cpu, int buftype); static bool buffer_check_space(int cpu, int buftype, int bytes); +static int contiguous_space_available(int cpu, int bufytpe); static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x); static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x); +static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len); static void gator_buffer_write_string(int cpu, int buftype, char *x); -static int gator_write_packed_int(char *buffer, unsigned int x); -static int gator_write_packed_int64(char *buffer, unsigned long long x); static void gator_add_trace(int cpu, int buftype, unsigned int address); static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs); static uint64_t gator_get_time(void); @@ -131,15 +134,16 @@ static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write); static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit); static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available); static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); -static DEFINE_PER_CPU(uint64_t, emit_overflow); /****************************************************************************** * Application Includes ******************************************************************************/ +#include "gator_marshaling.c" #include "gator_hrtimer_perf.c" #include "gator_hrtimer_gator.c" #include "gator_cookies.c" #include "gator_trace_sched.c" +#include "gator_trace_power.c" #include "gator_trace_gpu.c" #include "gator_backtrace.c" #include "gator_annotate.c" @@ -179,10 +183,10 @@ static bool buffer_commit_ready(int* cpu, int* buftype) /****************************************************************************** * Buffer management ******************************************************************************/ -static bool buffer_check_space(int cpu, int buftype, int bytes) +static int buffer_bytes_available(int cpu, int buftype) { int remaining, filled; - + filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype]; if (filled < 0) { filled += gator_buffer_size[buftype]; @@ -198,12 +202,24 @@ static bool buffer_check_space(int cpu, int buftype, int bytes) remaining -= 2000; } + return remaining; +} + +static int contiguous_space_available(int cpu, int buftype) +{ + int remaining = buffer_bytes_available(cpu, buftype); + int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype]; + if (remaining < contiguous) + return remaining; + else + return contiguous; +} + +static bool buffer_check_space(int cpu, int buftype, int bytes) +{ + int remaining = buffer_bytes_available(cpu, buftype); + if (remaining < bytes) { - if (per_cpu(buffer_space_available, cpu)[buftype] == true) { - // overflow packet to be emitted at a later time, as we may be in the middle of writing a message, e.g. counters - per_cpu(emit_overflow, cpu) = gator_get_time(); - pr_err("overflow: remaining = %d\n", gator_buffer_size[buftype] - filled); - } per_cpu(buffer_space_available, cpu)[buftype] = false; } else { per_cpu(buffer_space_available, cpu)[buftype] = true; @@ -238,19 +254,33 @@ static void gator_buffer_header(int cpu, int buftype) { int frame; - if (buftype == TIMER_BUF) - frame = FRAME_HRTIMER; - else if (buftype == EVENT_BUF) - frame = FRAME_EVENT; + if (buftype == BACKTRACE_BUF) + frame = FRAME_BACKTRACE; + else if (buftype == COUNTER_BUF) + frame = FRAME_COUNTER; + else if (buftype == ANNOTATE_BUF) + frame = FRAME_ANNOTATE; + else if (buftype == SCHED_TRACE_BUF) + frame = FRAME_SCHED_TRACE; + else if (buftype == GPU_TRACE_BUF) + frame = FRAME_GPU_TRACE; + else if (buftype == COUNTER2_BUF) + frame = FRAME_COUNTER2; + else if (buftype == WFI_BUF) + frame = FRAME_WFI; else frame = -1; - gator_buffer_write_packed_int(cpu, buftype, frame); - gator_buffer_write_packed_int(cpu, buftype, cpu); + if (per_cpu(gator_buffer, cpu)[buftype]) { + marshal_frame(cpu, buftype, frame); + } } static void gator_commit_buffer(int cpu, int buftype) { + if (!per_cpu(gator_buffer, cpu)[buftype]) + return; + per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype]; gator_buffer_header(cpu, buftype); wake_up(&gator_buffer_wait); @@ -276,8 +306,7 @@ static void gator_add_trace(int cpu, int buftype, unsigned int address) offset = address; } - gator_buffer_write_packed_int(cpu, buftype, offset & ~1); - gator_buffer_write_packed_int(cpu, buftype, cookie); + marshal_backtrace(offset & ~1, cookie); } static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs) @@ -288,12 +317,8 @@ static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs) if (!regs) return; - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE); - gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); - gator_buffer_write_packed_int(cpu, buftype, exec_cookie); - gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid); - gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid); - gator_buffer_write_packed_int(cpu, buftype, inKernel); + if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel)) + return; if (inKernel) { kernel_backtrace(cpu, buftype, regs); @@ -306,142 +331,54 @@ static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs) arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth); } - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE); + marshal_backtrace_footer(); } /****************************************************************************** * hrtimer interrupt processing ******************************************************************************/ -static LIST_HEAD(gator_events); - static void gator_timer_interrupt(void) { struct pt_regs * const regs = get_irq_regs(); int cpu = smp_processor_id(); - int *buffer, len, i, buftype = TIMER_BUF; - long long *buffer64; - struct gator_interface *gi; - - // Output scheduler trace - len = gator_trace_sched_read(&buffer64); - if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE); - gator_buffer_write_packed_int(cpu, buftype, len); - for (i = 0; i < len; i++) { - gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); - } - } - - // Output GPU trace - len = gator_trace_gpu_read(&buffer64); - if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_GPU_TRACE); - gator_buffer_write_packed_int(cpu, buftype, len); - for (i = 0; i < len; i++) { - gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); - } - } - - // Output counters - if (buffer_check_space(cpu, buftype, MAXSIZE_PACK32 * 2 + MAXSIZE_PACK64)) { - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); - gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); - list_for_each_entry(gi, &gator_events, list) { - if (gi->read) { - len = gi->read(&buffer); - if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { - gator_buffer_write_packed_int(cpu, buftype, len); - for (i = 0; i < len; i++) { - gator_buffer_write_packed_int(cpu, buftype, buffer[i]); - } - } - } else if (gi->read64) { - len = gi->read64(&buffer64); - if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + MAXSIZE_PACK32)) { - gator_buffer_write_packed_int(cpu, buftype, len); - for (i = 0; i < len; i++) { - gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); - } - } - } - } - gator_buffer_write_packed_int(cpu, buftype, 0); - } // Output backtrace - if (!event_based_sampling && buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32)) - gator_add_sample(cpu, buftype, regs); - - // Overflow message - if (per_cpu(emit_overflow, cpu)) { - gator_buffer_write_packed_int(cpu, buftype, MESSAGE_OVERFLOW); - gator_buffer_write_packed_int64(cpu, buftype, per_cpu(emit_overflow, cpu)); - per_cpu(emit_overflow, cpu) = 0; - } + gator_add_sample(cpu, BACKTRACE_BUF, regs); - // Check and commit; generally, commit is set to occur once per second - buffer_check(cpu, buftype); + // Collect counters + collect_counters(); } -DEFINE_PER_CPU(int, hrtimer_is_active); -static int hrtimer_running; +static int gator_running; // This function runs in interrupt context and on the appropriate core static void gator_timer_offline(void* unused) { + struct gator_interface *gi; int i, len, cpu = smp_processor_id(); int* buffer; - long long* buffer64; - - if (per_cpu(hrtimer_is_active, cpu)) { - struct gator_interface *gi; - gator_hrtimer_offline(cpu); - per_cpu(hrtimer_is_active, cpu) = 0; - - // Output scheduler trace - len = gator_trace_sched_offline(&buffer64); - if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { - gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_SCHEDULER_TRACE); - gator_buffer_write_packed_int(cpu, TIMER_BUF, len); - for (i = 0; i < len; i++) { - gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]); - } - } - // Output GPU trace - len = gator_trace_gpu_offline(&buffer64); - if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { - gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_GPU_TRACE); - gator_buffer_write_packed_int(cpu, TIMER_BUF, len); - for (i = 0; i < len; i++) { - gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]); - } - } + gator_trace_sched_offline(); + gator_trace_power_offline(); - // offline any events and output counters - gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS); - gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time()); + gator_hrtimer_offline(cpu); + + // Offline any events and output counters + if (marshal_event_header()) { list_for_each_entry(gi, &gator_events, list) { if (gi->offline) { len = gi->offline(&buffer); - if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { - gator_buffer_write_packed_int(cpu, TIMER_BUF, len); - for (i = 0; i < len; i++) - gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]); - } + marshal_event(len, buffer); } } - gator_buffer_write_packed_int(cpu, TIMER_BUF, 0); - - gator_commit_buffer(cpu, TIMER_BUF); } - if (event_based_sampling) { - gator_commit_buffer(cpu, EVENT_BUF); - } + // Flush all buffers on this core + for (i = 0; i < NUM_GATOR_BUFS; i++) + gator_commit_buffer(cpu, i); } -// This function runs in interrupt context and may be running on a core other than core 'cpu' +// This function runs in process context and may be running on a core other than core 'cpu' static void gator_timer_offline_dispatch(int cpu) { struct gator_interface *gi; @@ -457,13 +394,13 @@ static void gator_timer_stop(void) { int cpu; - if (hrtimer_running) { + if (gator_running) { on_each_cpu(gator_timer_offline, NULL, 1); for_each_online_cpu(cpu) { gator_timer_offline_dispatch(cpu); } - hrtimer_running = 0; + gator_running = 0; gator_hrtimer_shutdown(); } } @@ -471,32 +408,23 @@ static void gator_timer_stop(void) // This function runs in interrupt context and on the appropriate core static void gator_timer_online(void* unused) { - int i, len, cpu = smp_processor_id(); + struct gator_interface *gi; + int len, cpu = smp_processor_id(); int* buffer; - if (!per_cpu(hrtimer_is_active, cpu)) { - struct gator_interface *gi; + gator_trace_power_online(); - // online any events and output counters - gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS); - gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time()); + // online any events and output counters + if (marshal_event_header()) { list_for_each_entry(gi, &gator_events, list) { if (gi->online) { len = gi->online(&buffer); - if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { - gator_buffer_write_packed_int(cpu, TIMER_BUF, len); - for (i = 0; i < len; i++) - gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]); - } + marshal_event(len, buffer); } } - gator_buffer_write_packed_int(cpu, TIMER_BUF, 0); - - gator_event_sampling_online(); - - gator_hrtimer_online(cpu); - per_cpu(hrtimer_is_active, cpu) = 1; } + + gator_hrtimer_online(cpu); } // This function runs in interrupt context and may be running on a core other than core 'cpu' @@ -511,21 +439,22 @@ static void gator_timer_online_dispatch(int cpu) gator_event_sampling_online_dispatch(cpu); } -int gator_timer_start(unsigned long setup) +int gator_timer_start(unsigned long sample_rate) { int cpu; - if (!setup) { - pr_err("gator: cannot start due to a system tick value of zero\n"); - return -1; - } else if (hrtimer_running) { - pr_notice("gator: high res timer already running\n"); + if (gator_running) { + pr_notice("gator: already running\n"); return 0; } - hrtimer_running = 1; + gator_running = 1; - if (gator_hrtimer_init(setup, gator_timer_interrupt) == -1) + // event based sampling trumps hr timer based sampling + if (event_based_sampling) + sample_rate = 0; + + if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1) return -1; for_each_online_cpu(cpu) { @@ -550,7 +479,7 @@ static uint64_t gator_get_time(void) /****************************************************************************** * cpu hotplug and pm notifiers ******************************************************************************/ -static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) +static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { long cpu = (long)hcpu; @@ -570,8 +499,8 @@ static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long return NOTIFY_OK; } -static struct notifier_block __refdata gator_cpu_notifier = { - .notifier_call = gator_cpu_notify, +static struct notifier_block __refdata gator_hotcpu_notifier = { + .notifier_call = gator_hotcpu_notify, }; // n.b. calling "on_each_cpu" only runs on those that are online @@ -583,7 +512,7 @@ static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void switch (event) { case PM_HIBERNATION_PREPARE: case PM_SUSPEND_PREPARE: - unregister_hotcpu_notifier(&gator_cpu_notifier); + unregister_hotcpu_notifier(&gator_hotcpu_notifier); unregister_scheduler_tracepoints(); on_each_cpu(gator_timer_offline, NULL, 1); for_each_online_cpu(cpu) { @@ -597,7 +526,7 @@ static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void } on_each_cpu(gator_timer_online, NULL, 1); register_scheduler_tracepoints(); - register_hotcpu_notifier(&gator_cpu_notifier); + register_hotcpu_notifier(&gator_hotcpu_notifier); break; } @@ -611,7 +540,7 @@ static struct notifier_block gator_pm_notifier = { static int gator_notifier_start(void) { int retval; - retval = register_hotcpu_notifier(&gator_cpu_notifier); + retval = register_hotcpu_notifier(&gator_hotcpu_notifier); if (retval == 0) retval = register_pm_notifier(&gator_pm_notifier); return retval; @@ -620,12 +549,30 @@ static int gator_notifier_start(void) static void gator_notifier_stop(void) { unregister_pm_notifier(&gator_pm_notifier); - unregister_hotcpu_notifier(&gator_cpu_notifier); + unregister_hotcpu_notifier(&gator_hotcpu_notifier); } /****************************************************************************** * Main ******************************************************************************/ +static void gator_summary(void) +{ + uint64_t timestamp, uptime = 0; + struct timespec uptime_ts; + void (*m2b)(struct timespec *ts); + + timestamp = gator_get_time(); + + do_posix_clock_monotonic_gettime(&uptime_ts); + m2b = symbol_get(monotonic_to_bootbased); + if (m2b) { + m2b(&uptime_ts); + uptime = (long long)uptime_ts.tv_sec * 1000000000 + uptime_ts.tv_nsec; + } + + marshal_summary(timestamp, uptime); +} + int gator_events_install(struct gator_interface *interface) { list_add_tail(&interface->list, &gator_events); @@ -635,7 +582,8 @@ int gator_events_install(struct gator_interface *interface) int gator_events_get_key(void) { - static int key; + // key of zero is reserved as a timestamp + static int key = 1; return key++; } @@ -644,21 +592,31 @@ static int gator_init(void) { int i; - if (gator_annotate_init()) - return -1; - // events sources (gator_events.h, generated by gator_events.sh) for (i = 0; i < ARRAY_SIZE(gator_events_list); i++) if (gator_events_list[i]) gator_events_list[i](); + gator_trace_power_init(); + return 0; } static int gator_start(void) { + unsigned long cpu, i; struct gator_interface *gi; + // Initialize the buffer with the frame type and core + for_each_present_cpu(cpu) { + for (i = 0; i < NUM_GATOR_BUFS; i++) { + gator_buffer_header(cpu, i); + } + } + + // Capture the start time + gator_summary(); + // start all events list_for_each_entry(gi, &gator_events, list) { if (gi->start && gi->start() != 0) { @@ -683,6 +641,8 @@ static int gator_start(void) goto annotate_failure; if (gator_trace_sched_start()) goto sched_failure; + if (gator_trace_power_start()) + goto power_failure; if (gator_trace_gpu_start()) goto gpu_failure; if (gator_event_sampling_start()) @@ -701,6 +661,8 @@ timer_failure: event_sampling_failure: gator_trace_gpu_stop(); gpu_failure: + gator_trace_power_stop(); +power_failure: gator_trace_sched_stop(); sched_failure: gator_annotate_stop(); @@ -727,6 +689,7 @@ static void gator_stop(void) gator_annotate_stop(); gator_trace_sched_stop(); + gator_trace_power_stop(); gator_trace_gpu_stop(); gator_event_sampling_stop(); @@ -735,11 +698,6 @@ static void gator_stop(void) gator_timer_stop(); } -static void gator_exit(void) -{ - gator_annotate_exit(); -} - /****************************************************************************** * Filesystem ******************************************************************************/ @@ -751,33 +709,52 @@ static int gator_op_setup(void) mutex_lock(&start_mutex); - gator_buffer_size[TIMER_BUF] = userspace_buffer_size; - gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1; + gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE; + gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1; - // must be a power of 2 - if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) { - err = -ENOEXEC; - goto setup_error; - } + gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE; + gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1; + + gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE; + gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1; + + gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE; + gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1; - gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT; - gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1; + gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE; + gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1; + + gator_buffer_size[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE; + gator_buffer_mask[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE - 1; + + gator_buffer_size[WFI_BUF] = WFI_BUFFER_SIZE; + gator_buffer_mask[WFI_BUF] = WFI_BUFFER_SIZE - 1; // Initialize percpu per buffer variables for (i = 0; i < NUM_GATOR_BUFS; i++) { + // Verify buffers are a power of 2 + if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) { + err = -ENOEXEC; + goto setup_error; + } + for_each_present_cpu(cpu) { + per_cpu(gator_buffer_read, cpu)[i] = 0; + per_cpu(gator_buffer_write, cpu)[i] = 0; + per_cpu(gator_buffer_commit, cpu)[i] = 0; + per_cpu(buffer_space_available, cpu)[i] = true; + + // Annotation is a special case that only uses a single buffer + if (cpu > 0 && i == ANNOTATE_BUF) { + per_cpu(gator_buffer, cpu)[i] = NULL; + continue; + } + per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]); if (!per_cpu(gator_buffer, cpu)[i]) { err = -ENOMEM; goto setup_error; } - - per_cpu(gator_buffer_read, cpu)[i] = 0; - per_cpu(gator_buffer_write, cpu)[i] = 0; - per_cpu(gator_buffer_commit, cpu)[i] = 0; - per_cpu(buffer_space_available, cpu)[i] = true; - per_cpu(emit_overflow, cpu) = 0; - gator_buffer_header(cpu, i); } } @@ -829,8 +806,6 @@ static void gator_shutdown(void) mutex_lock(&start_mutex); - gator_annotate_shutdown(); - for_each_present_cpu(cpu) { mutex_lock(&gator_buffer_mutex); for (i = 0; i < NUM_GATOR_BUFS; i++) { @@ -840,7 +815,6 @@ static void gator_shutdown(void) per_cpu(gator_buffer_write, cpu)[i] = 0; per_cpu(gator_buffer_commit, cpu)[i] = 0; per_cpu(buffer_space_available, cpu)[i] = true; - per_cpu(emit_overflow, cpu) = 0; } mutex_unlock(&gator_buffer_mutex); } @@ -932,7 +906,7 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset) { int retval = -EINVAL; - int commit = 0, length1, length2, read; + int commit = 0, length, length1, length2, read, byte, type_length; char *buffer1; char *buffer2 = NULL; int cpu, buftype; @@ -944,7 +918,7 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, // sleep until the condition is true or a signal is received // the condition is checked each time gator_buffer_wait is woken up buftype = cpu = -1; - wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || gator_annotate_ready() || !gator_started); + wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started); if (signal_pending(current)) return -EINTR; @@ -954,33 +928,36 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, mutex_lock(&gator_buffer_mutex); - if (buftype != -1 && cpu != -1) { - read = per_cpu(gator_buffer_read, cpu)[buftype]; - commit = per_cpu(gator_buffer_commit, cpu)[buftype]; + if (buftype == -1 || cpu == -1) { + retval = 0; + goto out; + } - /* May happen if the buffer is freed during pending reads. */ - if (!per_cpu(gator_buffer, cpu)[buftype]) { - retval = -EFAULT; - goto out; - } + read = per_cpu(gator_buffer_read, cpu)[buftype]; + commit = per_cpu(gator_buffer_commit, cpu)[buftype]; - /* determine the size of two halves */ - length1 = commit - read; - buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]); - buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]); - if (length1 < 0) { - length1 = gator_buffer_size[buftype] - read; - length2 = commit; - } - } else if (gator_annotate_ready()) { - length1 = gator_annotate_read(&buffer1); - if (!length1) - goto out; - } else { - retval = 0; + /* May happen if the buffer is freed during pending reads. */ + if (!per_cpu(gator_buffer, cpu)[buftype]) { + retval = -EFAULT; goto out; } + /* determine the size of two halves */ + length1 = commit - read; + buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]); + buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]); + if (length1 < 0) { + length1 = gator_buffer_size[buftype] - read; + length2 = commit; + } + + // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload + type_length = gator_response_type ? 1 : 0; + length = length1 + length2 - type_length - sizeof(int); + for (byte = 0; byte < sizeof(int); byte++) { + per_cpu(gator_buffer, cpu)[buftype][(read + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF; + } + /* start, middle or end */ if (length1 > 0) { if (copy_to_user(&buf[0], buffer1, length1)) { @@ -995,9 +972,7 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf, } } - if (buftype != -1 && cpu != -1) - per_cpu(gator_buffer_read, cpu)[buftype] = commit; - + per_cpu(gator_buffer_read, cpu)[buftype] = commit; retval = length1 + length2; /* kick just in case we've lost an SMP event */ @@ -1055,8 +1030,8 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root) for_each_present_cpu(cpu) { gator_cpu_cores++; } - userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT; - gator_streaming = 1; + userspace_buffer_size = BACKTRACE_BUFFER_SIZE; + gator_response_type = 1; gatorfs_create_file(sb, root, "enable", &enable_fops); gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops); @@ -1064,7 +1039,7 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root) gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores); gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size); gatorfs_create_ulong(sb, root, "tick", &gator_timer_count); - gatorfs_create_ulong(sb, root, "streaming", &gator_streaming); + gatorfs_create_ulong(sb, root, "response_type", &gator_response_type); gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version); // Annotate interface @@ -1075,6 +1050,9 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root) list_for_each_entry(gi, &gator_events, list) if (gi->create_files) gi->create_files(sb, dir); + + // Power interface + gator_trace_power_create_files(sb, dir); } /****************************************************************************** @@ -1098,7 +1076,6 @@ static void __exit gator_module_exit(void) { tracepoint_synchronize_unregister(); gatorfs_unregister(); - gator_exit(); } module_init(gator_module_init); diff --git a/driver/gator_marshaling.c b/driver/gator_marshaling.c new file mode 100755 index 0000000..630d142 --- /dev/null +++ b/driver/gator_marshaling.c @@ -0,0 +1,239 @@ +/** + * Copyright (C) ARM Limited 2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +static void marshal_summary(long long timestamp, long long uptime) { + int cpu = 0; + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_SUMMARY); + gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, timestamp); + gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, uptime); +} + +static bool marshal_cookie_header(char* text) { + int cpu = smp_processor_id(); + return buffer_check_space(cpu, BACKTRACE_BUF, strlen(text) + 2 * MAXSIZE_PACK32); +} + +static void marshal_cookie(int cookie, char* text) { + int cpu = smp_processor_id(); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_COOKIE); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, cookie); + gator_buffer_write_string(cpu, BACKTRACE_BUF, text); +} + +static void marshal_pid_name(int pid, char* name) { + unsigned long flags, cpu; + local_irq_save(flags); + cpu = smp_processor_id(); + if (buffer_check_space(cpu, BACKTRACE_BUF, TASK_COMM_LEN + 2 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) { + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_PID_NAME); + gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, gator_get_time()); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, pid); + gator_buffer_write_string(cpu, BACKTRACE_BUF, name); + } + local_irq_restore(flags); +} + +static bool marshal_backtrace_header(int exec_cookie, int tgid, int pid, int inKernel) { + int cpu = smp_processor_id(); + if (buffer_check_space(cpu, BACKTRACE_BUF, gator_backtrace_depth * 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_START_BACKTRACE); + gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, gator_get_time()); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, exec_cookie); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, tgid); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, pid); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, inKernel); + return true; + } + + // Check and commit; commit is set to occur once buffer is 3/4 full + buffer_check(cpu, BACKTRACE_BUF); + + return false; +} + +static void marshal_backtrace(int address, int cookie) { + int cpu = smp_processor_id(); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, address); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, cookie); +} + +static void marshal_backtrace_footer(void) { + int cpu = smp_processor_id(); + gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_END_BACKTRACE); + + // Check and commit; commit is set to occur once buffer is 3/4 full + buffer_check(cpu, BACKTRACE_BUF); +} + +static bool marshal_event_header(void) { + unsigned long flags, cpu = smp_processor_id(); + bool retval = false; + + local_irq_save(flags); + if (buffer_check_space(cpu, COUNTER_BUF, MAXSIZE_PACK32 + MAXSIZE_PACK64)) { + gator_buffer_write_packed_int(cpu, COUNTER_BUF, 0); // key of zero indicates a timestamp + gator_buffer_write_packed_int64(cpu, COUNTER_BUF, gator_get_time()); + retval = true; + } + local_irq_restore(flags); + + // Check and commit; commit is set to occur once buffer is 3/4 full + buffer_check(cpu, COUNTER_BUF); + + return retval; +} + +static void marshal_event(int len, int* buffer) { + unsigned long i, flags, cpu = smp_processor_id(); + + if (len <= 0) + return; + + // length must be even since all data is a (key, value) pair + if (len & 0x1) { + pr_err("gator: invalid counter data detected and discarded"); + return; + } + + // events must be written in key,value pairs + for (i = 0; i < len; i += 2) { + local_irq_save(flags); + if (!buffer_check_space(cpu, COUNTER_BUF, MAXSIZE_PACK32 * 2)) { + local_irq_restore(flags); + break; + } + gator_buffer_write_packed_int(cpu, COUNTER_BUF, buffer[i]); + gator_buffer_write_packed_int(cpu, COUNTER_BUF, buffer[i + 1]); + local_irq_restore(flags); + } + + // Check and commit; commit is set to occur once buffer is 3/4 full + buffer_check(cpu, COUNTER_BUF); +} + +static void marshal_event64(int len, long long* buffer64) { + unsigned long i, flags, cpu = smp_processor_id(); + + if (len <= 0) + return; + + // length must be even since all data is a (key, value) pair + if (len & 0x1) { + pr_err("gator: invalid counter data detected and discarded"); + return; + } + + // events must be written in key,value pairs + for (i = 0; i < len; i += 2) { + local_irq_save(flags); + if (!buffer_check_space(cpu, COUNTER_BUF, MAXSIZE_PACK64 * 2)) { + local_irq_restore(flags); + break; + } + gator_buffer_write_packed_int64(cpu, COUNTER_BUF, buffer64[i]); + gator_buffer_write_packed_int64(cpu, COUNTER_BUF, buffer64[i + 1]); + local_irq_restore(flags); + } + + // Check and commit; commit is set to occur once buffer is 3/4 full + buffer_check(cpu, COUNTER_BUF); +} + +#if GATOR_CPU_FREQ_SUPPORT +static void marshal_event_single(int core, int key, int value) { + unsigned long flags, cpu; + + local_irq_save(flags); + cpu = smp_processor_id(); + if (buffer_check_space(cpu, COUNTER2_BUF, MAXSIZE_PACK64 + MAXSIZE_PACK32 * 3)) { + gator_buffer_write_packed_int64(cpu, COUNTER2_BUF, gator_get_time()); + gator_buffer_write_packed_int(cpu, COUNTER2_BUF, core); + gator_buffer_write_packed_int(cpu, COUNTER2_BUF, key); + gator_buffer_write_packed_int(cpu, COUNTER2_BUF, value); + } + local_irq_restore(flags); + + // Check and commit; commit is set to occur once buffer is 3/4 full + buffer_check(cpu, COUNTER2_BUF); +} +#endif + +static void marshal_sched_gpu(int type, int unit, int core, int tgid, int pid) { + unsigned long cpu = smp_processor_id(), flags; + + if (!per_cpu(gator_buffer, cpu)[GPU_TRACE_BUF]) + return; + + local_irq_save(flags); + if (buffer_check_space(cpu, GPU_TRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, type); + gator_buffer_write_packed_int64(cpu, GPU_TRACE_BUF, gator_get_time()); + gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, unit); + gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, core); + gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, tgid); + gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, pid); + } + local_irq_restore(flags); + + // Check and commit; commit is set to occur once buffer is 3/4 full + buffer_check(cpu, GPU_TRACE_BUF); +} + +static void marshal_sched_trace(int type, int pid, int tgid, int cookie, int state) { + unsigned long cpu = smp_processor_id(), flags; + + if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF]) + return; + + local_irq_save(flags); + if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, type); + gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, gator_get_time()); + gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid); + gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, tgid); + gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, cookie); + gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, state); + } + local_irq_restore(flags); + + // Check and commit; commit is set to occur once buffer is 3/4 full + buffer_check(cpu, SCHED_TRACE_BUF); +} + +#if GATOR_CPU_FREQ_SUPPORT +static void marshal_wfi(int core, int state) { + unsigned long flags, cpu; + + local_irq_save(flags); + cpu = smp_processor_id(); + if (buffer_check_space(cpu, WFI_BUF, MAXSIZE_PACK64 + MAXSIZE_PACK32 * 2)) { + gator_buffer_write_packed_int64(cpu, WFI_BUF, gator_get_time()); + gator_buffer_write_packed_int(cpu, WFI_BUF, core); + gator_buffer_write_packed_int(cpu, WFI_BUF, state); + } + local_irq_restore(flags); + + // Check and commit; commit is set to occur once buffer is 3/4 full + buffer_check(cpu, WFI_BUF); +} +#endif + +static void marshal_frame(int cpu, int buftype, int frame) { + // add response type + if (gator_response_type > 0) { + gator_buffer_write_packed_int(cpu, buftype, gator_response_type); + } + + // leave space for 4-byte unpacked length + per_cpu(gator_buffer_write, cpu)[buftype] = (per_cpu(gator_buffer_write, cpu)[buftype] + 4) & gator_buffer_mask[buftype]; + + // add frame type and core number + gator_buffer_write_packed_int(cpu, buftype, frame); + gator_buffer_write_packed_int(cpu, buftype, cpu); +} diff --git a/driver/gator_pack.c b/driver/gator_pack.c index 985e960..925469a 100644 --- a/driver/gator_pack.c +++ b/driver/gator_pack.c @@ -162,101 +162,3 @@ static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long per_cpu(gator_buffer_write, cpu)[buftype] = write9; } } - -static int gator_write_packed_int(char *buffer, unsigned int x) -{ - if ((x & 0xffffff80) == 0) { - buffer[0] = x & 0x7f; - return 1; - } else if ((x & 0xffffc000) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) & 0x7f; - return 2; - } else if ((x & 0xffe00000) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) & 0x7f; - return 3; - } else if ((x & 0xf0000000) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) | 0x80; - buffer[3] = (x>>21) & 0x7f; - return 4; - } else { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) | 0x80; - buffer[3] = (x>>21) | 0x80; - buffer[4] = (x>>28) & 0x0f; - return 5; - } -} - -static int gator_write_packed_int64(char *buffer, unsigned long long x) -{ - if ((x & 0xffffffffffffff80LL) == 0) { - buffer[0] = x & 0x7f; - return 1; - } else if ((x & 0xffffffffffffc000LL) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) & 0x7f; - return 2; - } else if ((x & 0xffffffffffe00000LL) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) & 0x7f; - return 3; - } else if ((x & 0xfffffffff0000000LL) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) | 0x80; - buffer[3] = (x>>21) & 0x7f; - return 4; - } else if ((x & 0xfffffff800000000LL) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) | 0x80; - buffer[3] = (x>>21) | 0x80; - buffer[4] = (x>>28) & 0x7f; - return 5; - } else if ((x & 0xfffffc0000000000LL) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) | 0x80; - buffer[3] = (x>>21) | 0x80; - buffer[4] = (x>>28) | 0x80; - buffer[5] = (x>>35) & 0x7f; - return 6; - } else if ((x & 0xfffe000000000000LL) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) | 0x80; - buffer[3] = (x>>21) | 0x80; - buffer[4] = (x>>28) | 0x80; - buffer[5] = (x>>35) | 0x80; - buffer[6] = (x>>42) & 0x7f; - return 7; - } else if ((x & 0xff00000000000000LL) == 0) { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) | 0x80; - buffer[3] = (x>>21) | 0x80; - buffer[4] = (x>>28) | 0x80; - buffer[5] = (x>>35) | 0x80; - buffer[6] = (x>>42) | 0x80; - buffer[7] = (x>>49) & 0x7f; - return 8; - } else { - buffer[0] = x | 0x80; - buffer[1] = (x>>7) | 0x80; - buffer[2] = (x>>14) | 0x80; - buffer[3] = (x>>21) | 0x80; - buffer[4] = (x>>28) | 0x80; - buffer[5] = (x>>35) | 0x80; - buffer[6] = (x>>42) | 0x80; - buffer[7] = (x>>49) | 0x80; - buffer[8] = (x>>56) & 0xff; - return 9; - } -} diff --git a/driver/gator_trace_gpu.c b/driver/gator_trace_gpu.c index bc63995..921932c 100644 --- a/driver/gator_trace_gpu.c +++ b/driver/gator_trace_gpu.c @@ -25,68 +25,12 @@ static int mali_trace_registered; static int gpu_trace_registered; -#define GPU_OVERFLOW -1 #define GPU_START 1 #define GPU_STOP 2 #define GPU_UNIT_VP 1 #define GPU_UNIT_FP 2 - -#define TRACESIZE (8*1024) - -static DEFINE_PER_CPU(uint64_t *[2], theGpuTraceBuf); -static DEFINE_PER_CPU(int, theGpuTraceSel); -static DEFINE_PER_CPU(int, theGpuTracePos); -static DEFINE_PER_CPU(int, theGpuTraceErr); - -int gator_trace_gpu_read(long long **buffer); - -static void probe_gpu_write(int type, int unit, int core, struct task_struct* task) -{ - int tracePos; - unsigned long flags; - uint64_t *traceBuf, time; - int pid, tgid; - int cpu = smp_processor_id(); - - if (!per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]) - return; - - if (task) { - tgid = (int)task->tgid; - pid = (int)task->pid; - } else { - tgid = pid = 0; - } - - // disable interrupts to synchronize with gator_trace_gpu_read(); spinlocks not needed since percpu buffers are used - local_irq_save(flags); - - time = gator_get_time(); - tracePos = per_cpu(theGpuTracePos, cpu); - traceBuf = per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]; - - if (tracePos < (TRACESIZE - 100)) { - // capture - traceBuf[tracePos++] = type; - traceBuf[tracePos++] = time; - traceBuf[tracePos++] = unit; - traceBuf[tracePos++] = core; - traceBuf[tracePos++] = tgid; - traceBuf[tracePos++] = pid; - } else if (!per_cpu(theGpuTraceErr, cpu)) { - per_cpu(theGpuTraceErr, cpu) = 1; - traceBuf[tracePos++] = GPU_OVERFLOW; - traceBuf[tracePos++] = time; - traceBuf[tracePos++] = 0; - traceBuf[tracePos++] = 0; - traceBuf[tracePos++] = 0; - traceBuf[tracePos++] = 0; - pr_debug("gator: gpu trace overflow\n"); - } - per_cpu(theGpuTracePos, cpu) = tracePos; - local_irq_restore(flags); -} +#define GPU_UNIT_CL 3 #ifdef MALI_SUPPORT @@ -105,19 +49,23 @@ enum components { GATOR_DEFINE_PROBE(mali_timeline_event, TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, unsigned int d2, unsigned int d3, unsigned int d4)) { unsigned int component, state; + int tgid = 0, pid = 0; // do as much work as possible before disabling interrupts - component = (event_id >> 16) & 0xF; - state = (event_id >> 24) & 0xF; + component = (event_id >> 16) & 0xFF; // component is an 8-bit field + state = (event_id >> 24) & 0xF; // state is a 4-bit field if ((component == COMPONENT_VP0) || (component >= COMPONENT_FP0 && component <= COMPONENT_FP7)) { if (state == ACTIVITY_START || state == ACTIVITY_STOP) { unsigned int type = (state == ACTIVITY_START) ? GPU_START : GPU_STOP; unsigned int unit = (component < COMPONENT_FP0) ? GPU_UNIT_VP : GPU_UNIT_FP; unsigned int core = (component < COMPONENT_FP0) ? component - COMPONENT_VP0 : component - COMPONENT_FP0; - struct task_struct* task = (state == ACTIVITY_START) ? (struct task_struct*)d2 : NULL; + if (state == ACTIVITY_START) { + tgid = d0; + pid = d1; + } - probe_gpu_write(type, unit, core, task); + marshal_sched_gpu(type, unit, core, tgid, pid); } } } @@ -125,20 +73,18 @@ GATOR_DEFINE_PROBE(mali_timeline_event, TP_PROTO(unsigned int event_id, unsigned GATOR_DEFINE_PROBE(gpu_activity_start, TP_PROTO(int gpu_unit, int gpu_core, struct task_struct *p)) { - probe_gpu_write(GPU_START, gpu_unit, gpu_core, p); + marshal_sched_gpu(GPU_START, gpu_unit, gpu_core, (int)p->tgid, (int)p->pid); } GATOR_DEFINE_PROBE(gpu_activity_stop, TP_PROTO(int gpu_unit, int gpu_core)) { - probe_gpu_write(GPU_STOP, gpu_unit, gpu_core, NULL); + marshal_sched_gpu(GPU_STOP, gpu_unit, gpu_core, 0, 0); } int gator_trace_gpu_start(void) { - int cpu; - /* - * Returns 0 for installation failed + * Returns nonzero for installation failed * Absence of gpu trace points is not an error */ @@ -161,90 +107,20 @@ int gator_trace_gpu_start(void) gpu_trace_registered = 1; } - if (!gpu_trace_registered && !mali_trace_registered) { - return 0; - } - - for_each_present_cpu(cpu) { - per_cpu(theGpuTraceSel, cpu) = 0; - per_cpu(theGpuTracePos, cpu) = 0; - per_cpu(theGpuTraceErr, cpu) = 0; - per_cpu(theGpuTraceBuf, cpu)[0] = kmalloc(TRACESIZE * sizeof(uint64_t), GFP_KERNEL); - per_cpu(theGpuTraceBuf, cpu)[1] = kmalloc(TRACESIZE * sizeof(uint64_t), GFP_KERNEL); - if (!per_cpu(theGpuTraceBuf, cpu)[0] || !per_cpu(theGpuTraceBuf, cpu)[1]) { -#ifdef MALI_SUPPORT - if (mali_trace_registered) { - GATOR_UNREGISTER_TRACE(mali_timeline_event); - } -#endif - if (gpu_trace_registered) { - GATOR_UNREGISTER_TRACE(gpu_activity_stop); - GATOR_UNREGISTER_TRACE(gpu_activity_start); - } - - gpu_trace_registered = mali_trace_registered = 0; - - return -1; - } - } - return 0; } -int gator_trace_gpu_offline(long long **buffer) -{ - return gator_trace_gpu_read(buffer); -} - void gator_trace_gpu_stop(void) { - int cpu; - - if (gpu_trace_registered || mali_trace_registered) { - for_each_present_cpu(cpu) { - kfree(per_cpu(theGpuTraceBuf, cpu)[0]); - kfree(per_cpu(theGpuTraceBuf, cpu)[1]); - per_cpu(theGpuTraceBuf, cpu)[0] = NULL; - per_cpu(theGpuTraceBuf, cpu)[1] = NULL; - } - #ifdef MALI_SUPPORT - if (mali_trace_registered) { - GATOR_UNREGISTER_TRACE(mali_timeline_event); - } + if (mali_trace_registered) { + GATOR_UNREGISTER_TRACE(mali_timeline_event); + } #endif - if (gpu_trace_registered) { - GATOR_UNREGISTER_TRACE(gpu_activity_stop); - GATOR_UNREGISTER_TRACE(gpu_activity_start); - } - - gpu_trace_registered = mali_trace_registered = 0; - } -} - -int gator_trace_gpu_read(long long **buffer) -{ - int cpu = smp_processor_id(); - unsigned long flags; - uint64_t *traceBuf; - int tracePos; - - if (!per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]) - return 0; - - local_irq_save(flags); - - traceBuf = per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]; - tracePos = per_cpu(theGpuTracePos, cpu); - - per_cpu(theGpuTraceSel, cpu) = !per_cpu(theGpuTraceSel, cpu); - per_cpu(theGpuTracePos, cpu) = 0; - per_cpu(theGpuTraceErr, cpu) = 0; - - local_irq_restore(flags); - - if (buffer) - *buffer = traceBuf; + if (gpu_trace_registered) { + GATOR_UNREGISTER_TRACE(gpu_activity_stop); + GATOR_UNREGISTER_TRACE(gpu_activity_start); + } - return tracePos; + gpu_trace_registered = mali_trace_registered = 0; } diff --git a/driver/gator_trace_power.c b/driver/gator_trace_power.c new file mode 100755 index 0000000..ca89b19 --- /dev/null +++ b/driver/gator_trace_power.c @@ -0,0 +1,160 @@ +/** + * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include <linux/cpufreq.h> +#include <trace/events/power.h> + +// cpu_frequency and cpu_idle trace points were introduced in Linux kernel v2.6.38 +// the now deprecated power_frequency trace point was available prior to 2.6.38, but only for x86 +#if GATOR_CPU_FREQ_SUPPORT +enum { + POWER_CPU_FREQ, + POWER_CPU_IDLE, + POWER_TOTAL +}; + +static DEFINE_PER_CPU(ulong, idle_prev_state); +static ulong power_cpu_enabled[POWER_TOTAL]; +static ulong power_cpu_key[POWER_TOTAL]; + +static int gator_trace_power_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + + // cpu_frequency + dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_freq"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_FREQ]); + gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_FREQ]); + + // cpu_idle + dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_idle"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_IDLE]); + gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_IDLE]); + + return 0; +} + +// 'cpu' may not equal smp_processor_id(), i.e. may not be running on the core that is having the freq/idle state change +GATOR_DEFINE_PROBE(cpu_frequency, TP_PROTO(unsigned int frequency, unsigned int cpu)) +{ + marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], frequency * 1000); +} + +#define WFI_ACTIVE_THRESHOLD 2 // may vary on platform/OS +#define WFI_EXIT 0 +#define WFI_ENTER 1 +GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu)) +{ + // the streamline engine treats all counter values as unsigned + if (state & 0x80000000) { + state = 0; + } + + if (state == per_cpu(idle_prev_state, cpu)) { + return; + } + + if (state < WFI_ACTIVE_THRESHOLD && per_cpu(idle_prev_state, cpu) >= WFI_ACTIVE_THRESHOLD) { + // transition from wfi to non-wfi + marshal_wfi(cpu, WFI_EXIT); + } else if (state >= WFI_ACTIVE_THRESHOLD && per_cpu(idle_prev_state, cpu) < WFI_ACTIVE_THRESHOLD) { + // transition from non-wfi to wfi + marshal_wfi(cpu, WFI_ENTER); + } + + per_cpu(idle_prev_state, cpu) = state; + + if (power_cpu_enabled[POWER_CPU_IDLE]) { + marshal_event_single(cpu, power_cpu_key[POWER_CPU_IDLE], state); + } +} + +static void gator_trace_power_online(void) +{ + int cpu = smp_processor_id(); + if (power_cpu_enabled[POWER_CPU_FREQ]) { + marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], cpufreq_quick_get(cpu) * 1000); + } +} + +static void gator_trace_power_offline(void) +{ + // Set frequency to zero on an offline + int cpu = smp_processor_id(); + if (power_cpu_enabled[POWER_CPU_FREQ]) { + marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], 0); + } +} + +static int gator_trace_power_start(void) +{ + int cpu; + + // register tracepoints + if (power_cpu_enabled[POWER_CPU_FREQ]) + if (GATOR_REGISTER_TRACE(cpu_frequency)) + goto fail_cpu_frequency_exit; + + // Always register for cpu:idle for detecting WFI, independent of power_cpu_enabled[POWER_CPU_IDLE] + if (GATOR_REGISTER_TRACE(cpu_idle)) + goto fail_cpu_idle_exit; + pr_debug("gator: registered power event tracepoints\n"); + + for_each_present_cpu(cpu) { + per_cpu(idle_prev_state, cpu) = 0; + } + + return 0; + + // unregister tracepoints on error +fail_cpu_idle_exit: + if (power_cpu_enabled[POWER_CPU_FREQ]) + GATOR_UNREGISTER_TRACE(cpu_frequency); +fail_cpu_frequency_exit: + pr_err("gator: power event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +static void gator_trace_power_stop(void) +{ + int i; + + if (power_cpu_enabled[POWER_CPU_FREQ]) + GATOR_UNREGISTER_TRACE(cpu_frequency); + GATOR_UNREGISTER_TRACE(cpu_idle); + pr_debug("gator: unregistered power event tracepoints\n"); + + for (i = 0; i < POWER_TOTAL; i++) { + power_cpu_enabled[i] = 0; + } +} + +void gator_trace_power_init(void) +{ + int i; + for (i = 0; i < POWER_TOTAL; i++) { + power_cpu_enabled[i] = 0; + power_cpu_key[i] = gator_events_get_key(); + } +} +#else +static int gator_trace_power_create_files(struct super_block *sb, struct dentry *root) {return 0;} +static void gator_trace_power_online(void) {} +static void gator_trace_power_offline(void) {} +static int gator_trace_power_start(void) {return 0;} +static void gator_trace_power_stop(void) {} +void gator_trace_power_init(void) {} +#endif diff --git a/driver/gator_trace_sched.c b/driver/gator_trace_sched.c index dafacb7..08b0270 100644 --- a/driver/gator_trace_sched.c +++ b/driver/gator_trace_sched.c @@ -10,34 +10,26 @@ #include <trace/events/sched.h> #include "gator.h" -#define SCHED_OVERFLOW -1 #define SCHED_SWITCH 1 -#define SCHED_PROCESS_FREE 2 +#define SCHED_PROCESS_EXIT 2 -#define SCHEDSIZE (8*1024) #define TASK_MAP_ENTRIES 1024 /* must be power of 2 */ #define TASK_MAX_COLLISIONS 2 -static DEFINE_PER_CPU(uint64_t *[2], theSchedBuf); -static DEFINE_PER_CPU(int, theSchedSel); -static DEFINE_PER_CPU(int, theSchedPos); -static DEFINE_PER_CPU(int, theSchedErr); static DEFINE_PER_CPU(uint64_t *, taskname_keys); enum { - STATE_CONTENTION = 0, + STATE_WAIT_ON_OTHER = 0, + STATE_CONTENTION, STATE_WAIT_ON_IO, - STATE_WAIT_ON_OTHER + STATE_WAIT_ON_MUTEX, }; -int gator_trace_sched_read(long long **buffer); - void emit_pid_name(struct task_struct* task) { bool found = false; - unsigned long flags; char taskcomm[TASK_COMM_LEN + 3]; - int x, cpu = smp_processor_id(); + unsigned long x, cpu = smp_processor_id(); uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]); uint64_t value; @@ -52,7 +44,7 @@ void emit_pid_name(struct task_struct* task) } } - if (!found && buffer_check_space(cpu, TIMER_BUF, TASK_COMM_LEN + 2 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) { + if (!found) { // shift values, new value always in front uint64_t oldv, newv = value; for (x = 0; x < TASK_MAX_COLLISIONS; x++) { @@ -62,100 +54,70 @@ void emit_pid_name(struct task_struct* task) } // emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions - if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) + if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) { // append ellipses if task->comm has length of TASK_COMM_LEN - 1 strcat(taskcomm, "..."); + } - // disable interrupts to synchronize with hrtimer populating timer buf - local_irq_save(flags); - gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_PID_NAME); - gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time()); - gator_buffer_write_packed_int(cpu, TIMER_BUF, task->pid); - gator_buffer_write_string(cpu, TIMER_BUF, taskcomm); - local_irq_restore(flags); + marshal_pid_name(task->pid, taskcomm); + } +} + + +static void collect_counters(void) +{ + int *buffer, len; + long long *buffer64; + struct gator_interface *gi; + + if (marshal_event_header()) { + list_for_each_entry(gi, &gator_events, list) { + if (gi->read) { + len = gi->read(&buffer); + marshal_event(len, buffer); + } else if (gi->read64) { + len = gi->read64(&buffer64); + marshal_event64(len, buffer64); + } + } } } static void probe_sched_write(int type, struct task_struct* task, struct task_struct* old_task) { - int schedPos, cookie = 0, state = 0; - unsigned long flags; - uint64_t *schedBuf, time; + int cookie = 0, state = 0; int cpu = smp_processor_id(); int pid = task->pid; int tgid = task->tgid; - if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) - return; - if (type == SCHED_SWITCH) { // do as much work as possible before disabling interrupts - cookie = get_exec_cookie(cpu, TIMER_BUF, task); + cookie = get_exec_cookie(cpu, BACKTRACE_BUF, task); emit_pid_name(task); - if (old_task->state == 0) + if (old_task->state == TASK_RUNNING) { state = STATE_CONTENTION; - else if (old_task->in_iowait) + } else if (old_task->in_iowait) { state = STATE_WAIT_ON_IO; - else +#ifdef CONFIG_DEBUG_MUTEXES + } else if (old_task->blocked_on) { + state = STATE_WAIT_ON_MUTEX; +#endif + } else { state = STATE_WAIT_ON_OTHER; - } - - // disable interrupts to synchronize with gator_trace_sched_read(); spinlocks not needed since percpu buffers are used - local_irq_save(flags); - - time = gator_get_time(); - schedPos = per_cpu(theSchedPos, cpu); - schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]; + } - if (schedPos < (SCHEDSIZE - 100)) { - // capture - schedBuf[schedPos++] = type; - schedBuf[schedPos++] = time; - schedBuf[schedPos++] = pid; - schedBuf[schedPos++] = tgid; - schedBuf[schedPos++] = cookie; - schedBuf[schedPos++] = state; - } else if (!per_cpu(theSchedErr, cpu)) { - per_cpu(theSchedErr, cpu) = 1; - schedBuf[schedPos++] = SCHED_OVERFLOW; - schedBuf[schedPos++] = time; - schedBuf[schedPos++] = 0; - schedBuf[schedPos++] = 0; - schedBuf[schedPos++] = 0; - schedBuf[schedPos++] = 0; - pr_debug("gator: tracepoint overflow\n"); + collect_counters(); } - per_cpu(theSchedPos, cpu) = schedPos; - local_irq_restore(flags); + + // marshal_sched_trace() disables interrupts as the free may trigger while switch is writing to the buffer; disabling preemption is not sufficient + // is disable interrupts necessary now that exit is used instead of free? + marshal_sched_trace(type, pid, tgid, cookie, state); } // special case used during a suspend of the system static void trace_sched_insert_idle(void) { - unsigned long flags; - uint64_t *schedBuf; - int schedPos, cpu = smp_processor_id(); - - if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) - return; - - local_irq_save(flags); - - schedPos = per_cpu(theSchedPos, cpu); - schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]; - - if (schedPos < (SCHEDSIZE - (6 * 8))) { - // capture - schedBuf[schedPos++] = SCHED_SWITCH; - schedBuf[schedPos++] = gator_get_time(); - schedBuf[schedPos++] = 0; // idle pid is zero - schedBuf[schedPos++] = 0; // idle tid is zero - schedBuf[schedPos++] = 0; // idle cookie is zero - schedBuf[schedPos++] = STATE_WAIT_ON_OTHER; - } - - per_cpu(theSchedPos, cpu) = schedPos; - local_irq_restore(flags); + marshal_sched_trace(SCHED_SWITCH, 0, 0, 0, 0); } #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) @@ -167,23 +129,23 @@ GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_ probe_sched_write(SCHED_SWITCH, next, prev); } -GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p)) +GATOR_DEFINE_PROBE(sched_process_exit, TP_PROTO(struct task_struct *p)) { - probe_sched_write(SCHED_PROCESS_FREE, p, 0); + probe_sched_write(SCHED_PROCESS_EXIT, p, 0); } static int register_scheduler_tracepoints(void) { // register tracepoints if (GATOR_REGISTER_TRACE(sched_switch)) goto fail_sched_switch; - if (GATOR_REGISTER_TRACE(sched_process_free)) - goto fail_sched_process_free; + if (GATOR_REGISTER_TRACE(sched_process_exit)) + goto fail_sched_process_exit; pr_debug("gator: registered tracepoints\n"); return 0; // unregister tracepoints on error -fail_sched_process_free: +fail_sched_process_exit: GATOR_UNREGISTER_TRACE(sched_switch); fail_sched_switch: pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); @@ -196,14 +158,6 @@ int gator_trace_sched_start(void) int cpu, size; for_each_present_cpu(cpu) { - per_cpu(theSchedSel, cpu) = 0; - per_cpu(theSchedPos, cpu) = 0; - per_cpu(theSchedErr, cpu) = 0; - per_cpu(theSchedBuf, cpu)[0] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL); - per_cpu(theSchedBuf, cpu)[1] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL); - if (!per_cpu(theSchedBuf, cpu)) - return -1; - size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t); per_cpu(taskname_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL); if (!per_cpu(taskname_keys, cpu)) @@ -214,16 +168,15 @@ int gator_trace_sched_start(void) return register_scheduler_tracepoints(); } -int gator_trace_sched_offline(long long **buffer) +void gator_trace_sched_offline(void) { trace_sched_insert_idle(); - return gator_trace_sched_read(buffer); } static void unregister_scheduler_tracepoints(void) { GATOR_UNREGISTER_TRACE(sched_switch); - GATOR_UNREGISTER_TRACE(sched_process_free); + GATOR_UNREGISTER_TRACE(sched_process_exit); pr_debug("gator: unregistered tracepoints\n"); } @@ -233,37 +186,6 @@ void gator_trace_sched_stop(void) unregister_scheduler_tracepoints(); for_each_present_cpu(cpu) { - kfree(per_cpu(theSchedBuf, cpu)[0]); - kfree(per_cpu(theSchedBuf, cpu)[1]); - per_cpu(theSchedBuf, cpu)[0] = NULL; - per_cpu(theSchedBuf, cpu)[1] = NULL; kfree(per_cpu(taskname_keys, cpu)); } } - -int gator_trace_sched_read(long long **buffer) -{ - int cpu = smp_processor_id(); - unsigned long flags; - uint64_t *schedBuf; - int schedPos; - - if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) - return 0; - - local_irq_save(flags); - - schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]; - schedPos = per_cpu(theSchedPos, cpu); - - per_cpu(theSchedSel, cpu) = !per_cpu(theSchedSel, cpu); - per_cpu(theSchedPos, cpu) = 0; - per_cpu(theSchedErr, cpu) = 0; - - local_irq_restore(flags); - - if (buffer) - *buffer = schedBuf; - - return schedPos; -} |