aboutsummaryrefslogtreecommitdiff
path: root/driver
diff options
context:
space:
mode:
authorJon Medhurst <tixy@linaro.org>2012-05-10 14:15:56 +0100
committerJon Medhurst <tixy@linaro.org>2012-05-16 14:22:59 +0100
commitd18974d3f05535eda819f2d0b92a9d49719b0f26 (patch)
treeee0d02ac702b3802b0f002a0f8edf2171d7e58c3 /driver
parent970700feed8c3523b06476ae340bf46f6d262550 (diff)
gator: Version 5.10DS-5.10
New gator release (build 1385) for ARM DS-5 v5.10 Signed-off-by: Jon Medhurst <tixy@linaro.org>
Diffstat (limited to 'driver')
-rw-r--r--driver/Makefile11
-rw-r--r--driver/gator.h1
-rw-r--r--driver/gator_annotate.c184
-rw-r--r--driver/gator_backtrace.c6
-rw-r--r--driver/gator_cookies.c10
-rw-r--r--driver/gator_ebs.c64
-rw-r--r--driver/gator_events_armv6.c16
-rw-r--r--driver/gator_events_armv7.c15
-rw-r--r--driver/gator_events_block.c6
-rw-r--r--driver/gator_events_irq.c19
-rwxr-xr-x[-rw-r--r--]driver/gator_events_mali.c165
-rw-r--r--driver/gator_events_meminfo.c9
-rw-r--r--driver/gator_events_mmaped.c77
-rw-r--r--driver/gator_events_net.c17
-rwxr-xr-x[-rw-r--r--]driver/gator_events_perf_pmu.c16
-rw-r--r--driver/gator_events_power.c178
-rw-r--r--driver/gator_events_scorpion.c18
-rw-r--r--driver/gator_fs.c8
-rwxr-xr-x[-rw-r--r--]driver/gator_hrtimer_gator.c23
-rw-r--r--driver/gator_main.c471
-rwxr-xr-xdriver/gator_marshaling.c239
-rw-r--r--driver/gator_pack.c98
-rw-r--r--driver/gator_trace_gpu.c164
-rwxr-xr-xdriver/gator_trace_power.c160
-rw-r--r--driver/gator_trace_sched.c182
25 files changed, 1087 insertions, 1070 deletions
diff --git a/driver/Makefile b/driver/Makefile
index 8824f8e..667637e 100644
--- a/driver/Makefile
+++ b/driver/Makefile
@@ -11,21 +11,10 @@ gator-y := gator_main.o \
gator_events_net.o \
gator_events_block.o \
gator_events_meminfo.o \
- gator_events_power.o \
gator_events_perf_pmu.o
gator-y += gator_events_mmaped.o
-ifeq ($(GATOR_WITH_MALI_SUPPORT),)
-ifeq ($(GATOR_MALI_INCLUDE),)
-GATOR_MALI_INCLUDE = $(abspath $(shell find -L . -name "mali_linux_trace.h" | sed -n -e '1s,\(.*\)/linux/mali_linux_trace.h$$,\1,p'))
-endif
-ifneq ($(GATOR_MALI_INCLUDE),)
-GATOR_WITH_MALI_SUPPORT = MALI_400 # for now, assume all devices with Mali have the Mali-400
-EXTRA_CFLAGS += -I$(GATOR_MALI_INCLUDE)
-endif
-endif
-
ifneq ($(GATOR_WITH_MALI_SUPPORT),)
ifeq ($(GATOR_WITH_MALI_SUPPORT),MALI_T6xx)
gator-y += gator_events_mali_t6xx.o
diff --git a/driver/gator.h b/driver/gator.h
index a7a323c..6b96109 100644
--- a/driver/gator.h
+++ b/driver/gator.h
@@ -88,6 +88,7 @@ struct gator_interface {
struct list_head list;
};
+// gator_events_init is used as a search term in gator_events.sh
#define gator_events_init(initfn) \
static inline int __gator_events_init_test(void) \
{ return initfn(); }
diff --git a/driver/gator_annotate.c b/driver/gator_annotate.c
index 36a921c..b2288b3 100644
--- a/driver/gator_annotate.c
+++ b/driver/gator_annotate.c
@@ -15,65 +15,93 @@
#include <asm/current.h>
#include <linux/spinlock.h>
-#define ANNOTATE_SIZE (16*1024)
static DEFINE_SPINLOCK(annotate_lock);
-static char *annotateBuf;
-static char *annotateBuf0;
-static char *annotateBuf1;
-static int annotatePos;
-static int annotateSel;
static bool collect_annotations = false;
-static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
+static int annotate_copy(struct file *file, char const __user *buf, size_t count)
{
- char tempBuffer[512];
- int remaining, size;
- uint32_t tid;
+ int cpu = 0;
+ int write = per_cpu(gator_buffer_write, cpu)[ANNOTATE_BUF];
+
+ if (file == NULL) {
+ // copy from kernel
+ memcpy(&per_cpu(gator_buffer, cpu)[ANNOTATE_BUF][write], buf, count);
+ } else {
+ // copy from user space
+ if (copy_from_user(&per_cpu(gator_buffer, cpu)[ANNOTATE_BUF][write], buf, count) != 0)
+ return -1;
+ }
+ per_cpu(gator_buffer_write, cpu)[ANNOTATE_BUF] = (write + count) & gator_buffer_mask[ANNOTATE_BUF];
+
+ return 0;
+}
+
+static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count_orig, loff_t *offset)
+{
+ int tid, cpu, header_size, available, contiguous, length1, length2, size, count = count_orig & 0x7fffffff;
if (*offset)
return -EINVAL;
- // determine size to capture
- size = count < sizeof(tempBuffer) ? count : sizeof(tempBuffer);
+ if (!collect_annotations) {
+ return count_orig;
+ }
+
+ cpu = 0; // Annotation only uses a single per-cpu buffer as the data must be in order to the engine
- // note: copy may be for naught if remaining is zero, but better to do the copy outside of the spinlock
if (file == NULL) {
- // copy from kernel
- memcpy(tempBuffer, buf, size);
-
- // set the thread id to the kernel thread, not the current thread
- tid = -1;
+ tid = -1; // set the thread id to the kernel thread
} else {
- // copy from user space
- if (copy_from_user(tempBuffer, buf, size) != 0)
- return -EINVAL;
tid = current->pid;
}
- // synchronize shared variables annotateBuf and annotatePos
+ // synchronize between cores
spin_lock(&annotate_lock);
- if (collect_annotations && annotateBuf) {
- remaining = ANNOTATE_SIZE - annotatePos - 256; // pad for headers and release
- size = size < remaining ? size : remaining;
- if (size > 0) {
- uint64_t time = gator_get_time();
- uint32_t cpuid = smp_processor_id();
- int pos = annotatePos;
- pos += gator_write_packed_int(&annotateBuf[pos], tid);
- pos += gator_write_packed_int64(&annotateBuf[pos], time);
- pos += gator_write_packed_int(&annotateBuf[pos], cpuid);
- pos += gator_write_packed_int(&annotateBuf[pos], size);
- memcpy(&annotateBuf[pos], tempBuffer, size);
- annotatePos = pos + size;
- }
- }
- spin_unlock(&annotate_lock);
+
+ // determine total size of the payload
+ header_size = MAXSIZE_PACK32 * 3 + MAXSIZE_PACK64;
+ available = buffer_bytes_available(cpu, ANNOTATE_BUF) - header_size;
+ size = count < available ? count : available;
if (size <= 0) {
- wake_up(&gator_buffer_wait);
- return 0;
+ size = 0;
+ goto annotate_write_out;
}
+ // synchronize shared variables annotateBuf and annotatePos
+ if (collect_annotations && per_cpu(gator_buffer, cpu)[ANNOTATE_BUF]) {
+ gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, smp_processor_id());
+ gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, tid);
+ gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, gator_get_time());
+ gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, size);
+
+ // determine the sizes to capture, length1 + length2 will equal size
+ contiguous = contiguous_space_available(cpu, ANNOTATE_BUF);
+ if (size < contiguous) {
+ length1 = size;
+ length2 = 0;
+ } else {
+ length1 = contiguous;
+ length2 = size - contiguous;
+ }
+
+ if (annotate_copy(file, buf, length1) != 0) {
+ size = -EINVAL;
+ goto annotate_write_out;
+ }
+
+ if (length2 > 0 && annotate_copy(file, &buf[length1], length2) != 0) {
+ size = -EINVAL;
+ goto annotate_write_out;
+ }
+
+ // Check and commit; commit is set to occur once buffer is 3/4 full
+ buffer_check(cpu, ANNOTATE_BUF);
+ }
+
+annotate_write_out:
+ spin_unlock(&annotate_lock);
+
// return the number of bytes written
return size;
}
@@ -82,21 +110,18 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t
static int annotate_release(struct inode *inode, struct file *file)
{
- int remaining = ANNOTATE_SIZE - annotatePos;
- if (remaining < 16) {
- return -EFAULT;
- }
+ int cpu = 0;
+ // synchronize between cores
spin_lock(&annotate_lock);
- if (annotateBuf) {
+
+ if (per_cpu(gator_buffer, cpu)[ANNOTATE_BUF] && buffer_check_space(cpu, ANNOTATE_BUF, MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
uint32_t tid = current->pid;
- int pos = annotatePos;
- pos += gator_write_packed_int(&annotateBuf[pos], tid);
- pos += gator_write_packed_int64(&annotateBuf[pos], 0); // time
- pos += gator_write_packed_int(&annotateBuf[pos], 0); // cpuid
- pos += gator_write_packed_int(&annotateBuf[pos], 0); // size
- annotatePos = pos;
+ gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, tid);
+ gator_buffer_write_packed_int64(cpu, ANNOTATE_BUF, 0); // time
+ gator_buffer_write_packed_int(cpu, ANNOTATE_BUF, 0); // size
}
+
spin_unlock(&annotate_lock);
return 0;
@@ -109,25 +134,11 @@ static const struct file_operations annotate_fops = {
static int gator_annotate_create_files(struct super_block *sb, struct dentry *root)
{
- annotateBuf = NULL;
return gatorfs_create_file_perm(sb, root, "annotate", &annotate_fops, 0666);
}
-static int gator_annotate_init(void)
-{
- annotateBuf0 = kmalloc(ANNOTATE_SIZE, GFP_KERNEL);
- annotateBuf1 = kmalloc(ANNOTATE_SIZE, GFP_KERNEL);
- if (!annotateBuf0 || !annotateBuf1)
- return -1;
- return 0;
-}
-
static int gator_annotate_start(void)
{
- annotateSel = 0;
- annotatePos = 1;
- annotateBuf = annotateBuf0;
- annotateBuf[0] = FRAME_ANNOTATE;
collect_annotations = true;
return 0;
}
@@ -136,46 +147,3 @@ static void gator_annotate_stop(void)
{
collect_annotations = false;
}
-
-static void gator_annotate_shutdown(void)
-{
- spin_lock(&annotate_lock);
- annotateBuf = NULL;
- spin_unlock(&annotate_lock);
-}
-
-static void gator_annotate_exit(void)
-{
- spin_lock(&annotate_lock);
- kfree(annotateBuf0);
- kfree(annotateBuf1);
- annotateBuf = annotateBuf0 = annotateBuf1 = NULL;
- spin_unlock(&annotate_lock);
-}
-
-static int gator_annotate_ready(void)
-{
- return annotatePos > 1 && annotateBuf;
-}
-
-static int gator_annotate_read(char **buffer)
-{
- int len;
-
- if (!gator_annotate_ready())
- return 0;
-
- annotateSel = !annotateSel;
-
- if (buffer)
- *buffer = annotateBuf;
-
- spin_lock(&annotate_lock);
- len = annotatePos;
- annotateBuf = annotateSel ? annotateBuf1 : annotateBuf0;
- annotateBuf[0] = FRAME_ANNOTATE;
- annotatePos = 1;
- spin_unlock(&annotate_lock);
-
- return len;
-}
diff --git a/driver/gator_backtrace.c b/driver/gator_backtrace.c
index 26503ef..50783d6 100644
--- a/driver/gator_backtrace.c
+++ b/driver/gator_backtrace.c
@@ -81,8 +81,7 @@ static int report_trace(struct stackframe *frame, void *d)
cookie = get_cookie(cpu, per_cpu(backtrace_buffer, cpu), current, NULL, mod, true);
addr = addr - (unsigned long)mod->module_core;
}
- gator_buffer_write_packed_int(cpu, per_cpu(backtrace_buffer, cpu), addr & ~1);
- gator_buffer_write_packed_int(cpu, per_cpu(backtrace_buffer, cpu), cookie);
+ marshal_backtrace(addr & ~1, cookie);
(*depth)--;
}
@@ -110,7 +109,6 @@ static void kernel_backtrace(int cpu, int buftype, struct pt_regs * const regs)
per_cpu(backtrace_buffer, cpu) = buftype;
walk_stackframe(&frame, report_trace, &depth);
#else
- gator_buffer_write_packed_int(cpu, buftype, PC_REG & ~1);
- gator_buffer_write_packed_int(cpu, buftype, NO_COOKIE);
+ marshal_backtrace(PC_REG & ~1, NO_COOKIE);
#endif
}
diff --git a/driver/gator_cookies.c b/driver/gator_cookies.c
index 1beb34f..7b50916 100644
--- a/driver/gator_cookies.c
+++ b/driver/gator_cookies.c
@@ -129,7 +129,7 @@ static void wq_cookie_handler(struct work_struct *unused)
while (per_cpu(translate_buffer_read, cpu) != commit) {
task = (struct task_struct *)translate_buffer_read_int(cpu);
vma = (struct vm_area_struct *)translate_buffer_read_int(cpu);
- cookie = get_cookie(cpu, TIMER_BUF, task, vma, NULL, false);
+ cookie = get_cookie(cpu, BACKTRACE_BUF, task, vma, NULL, false);
}
}
@@ -251,13 +251,10 @@ static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task
local_irq_save(flags);
cookie = INVALID_COOKIE;
- if (buffer_check_space(cpu, buftype, strlen(text) + 2 * MAXSIZE_PACK32)) {
+ if (marshal_cookie_header(text)) {
cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
cookiemap_add(key, cookie);
-
- gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COOKIE);
- gator_buffer_write_packed_int(cpu, buftype, cookie);
- gator_buffer_write_string(cpu, buftype, text);
+ marshal_cookie(cookie, text);
}
local_irq_restore(flags);
@@ -271,6 +268,7 @@ static int get_exec_cookie(int cpu, int buftype, struct task_struct *task)
struct mm_struct *mm = task->mm;
struct vm_area_struct *vma;
+ // kernel threads have no address space
if (!mm)
return cookie;
diff --git a/driver/gator_ebs.c b/driver/gator_ebs.c
index 8c2997c..1208d69 100644
--- a/driver/gator_ebs.c
+++ b/driver/gator_ebs.c
@@ -14,6 +14,8 @@
#include <asm/pmu.h>
+static DEFINE_MUTEX(perf_mutex);
+
extern int pmnc_counters;
extern int ccnt;
extern unsigned long pmnc_enabled[];
@@ -32,59 +34,16 @@ static void ebs_overflow_handler(struct perf_event *event, int unused, struct pe
static void ebs_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs)
#endif
{
- unsigned int value, delta, cpu = smp_processor_id(), buftype = EVENT_BUF;
+ int cpu = smp_processor_id();
if (event != per_cpu(pevent, cpu))
return;
- if (buffer_check_space(cpu, buftype, 5 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
- value = local64_read(&event->count);
- delta = value - per_cpu(prev_value, cpu);
- per_cpu(prev_value, cpu) = value;
-
- // Counters header
- gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type
- gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time
-
- // Output counter
- gator_buffer_write_packed_int(cpu, buftype, 2); // length
- gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key
- gator_buffer_write_packed_int(cpu, buftype, delta); // delta
-
- // End Counters, length of zero
- gator_buffer_write_packed_int(cpu, buftype, 0);
- }
-
// Output backtrace
- if (buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32))
- gator_add_sample(cpu, buftype, regs);
+ gator_add_sample(cpu, BACKTRACE_BUF, regs);
- // Check and commit; commit is set to occur once buffer is 3/4 full
- buffer_check(cpu, buftype);
-}
-
-static void gator_event_sampling_online(void)
-{
- int cpu = smp_processor_id(), buftype = EVENT_BUF;
-
- // read the counter and toss the invalid data, return zero instead
- struct perf_event * ev = per_cpu(pevent, cpu);
- if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) {
- ev->pmu->read(ev);
- per_cpu(prev_value, cpu) = local64_read(&ev->count);
-
- // Counters header
- gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type
- gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time
-
- // Output counter
- gator_buffer_write_packed_int(cpu, buftype, 2); // length
- gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key
- gator_buffer_write_packed_int(cpu, buftype, 0); // delta - zero for initialization
-
- // End Counters, length of zero
- gator_buffer_write_packed_int(cpu, buftype, 0);
- }
+ // Collect counters
+ collect_counters();
}
static void gator_event_sampling_online_dispatch(int cpu)
@@ -117,10 +76,18 @@ static void gator_event_sampling_online_dispatch(int cpu)
static void gator_event_sampling_offline_dispatch(int cpu)
{
+ struct perf_event * pe = NULL;
+
+ mutex_lock(&perf_mutex);
if (per_cpu(pevent, cpu)) {
- perf_event_release_kernel(per_cpu(pevent, cpu));
+ pe = per_cpu(pevent, cpu);
per_cpu(pevent, cpu) = NULL;
}
+ mutex_unlock(&perf_mutex);
+
+ if (pe) {
+ perf_event_release_kernel(pe);
+ }
}
static int gator_event_sampling_start(void)
@@ -184,7 +151,6 @@ static void gator_event_sampling_stop(void)
}
#else
-static void gator_event_sampling_online(void) {}
static void gator_event_sampling_online_dispatch(int cpu) {}
static void gator_event_sampling_offline_dispatch(int cpu) {}
static int gator_event_sampling_start(void) {return 0;}
diff --git a/driver/gator_events_armv6.c b/driver/gator_events_armv6.c
index ef51898..5f989ba 100644
--- a/driver/gator_events_armv6.c
+++ b/driver/gator_events_armv6.c
@@ -33,7 +33,6 @@ static unsigned long pmnc_enabled[CNTMAX];
static unsigned long pmnc_event[CNTMAX];
static unsigned long pmnc_key[CNTMAX];
-static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt);
static inline void armv6_pmnc_write(u32 val)
@@ -111,8 +110,6 @@ static int gator_events_armv6_online(int** buffer)
for (pmnc = 0, cnt = PMN0; cnt <= CCNT; cnt++) {
unsigned long event;
- per_cpu(perfPrev, cpu)[cnt] = 0;
-
if (!pmnc_enabled[cnt])
continue;
@@ -171,6 +168,11 @@ static int gator_events_armv6_read(int **buffer)
int cnt, len = 0;
int cpu = smp_processor_id();
+ // a context switch may occur before the online hotplug event, thus need to check that the pmu is enabled
+ if (!(armv6_pmnc_read() & PMCR_E)) {
+ return 0;
+ }
+
for (cnt = PMN0; cnt <= CCNT; cnt++) {
if (pmnc_enabled[cnt]) {
u32 value = 0;
@@ -186,11 +188,9 @@ static int gator_events_armv6_read(int **buffer)
break;
}
armv6_pmnc_reset_counter(cnt);
- if (value != per_cpu(perfPrev, cpu)[cnt]) {
- per_cpu(perfPrev, cpu)[cnt] = value;
- per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
- per_cpu(perfCnt, cpu)[len++] = value;
- }
+
+ per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
+ per_cpu(perfCnt, cpu)[len++] = value;
}
}
diff --git a/driver/gator_events_armv7.c b/driver/gator_events_armv7.c
index cdf450f..590421d 100644
--- a/driver/gator_events_armv7.c
+++ b/driver/gator_events_armv7.c
@@ -38,7 +38,6 @@ static unsigned long pmnc_enabled[CNTMAX];
static unsigned long pmnc_event[CNTMAX];
static unsigned long pmnc_key[CNTMAX];
-static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt);
inline void armv7_pmnc_write(u32 val)
@@ -177,8 +176,6 @@ static int gator_events_armv7_online(int** buffer)
for (cnt = CCNT; cnt < CNTMAX; cnt++) {
unsigned long event;
- per_cpu(perfPrev, cpu)[cnt] = 0;
-
if (!pmnc_enabled[cnt])
continue;
@@ -240,6 +237,11 @@ static int gator_events_armv7_read(int **buffer)
int cnt, len = 0;
int cpu = smp_processor_id();
+ // a context switch may occur before the online hotplug event, thus need to check that the pmu is enabled
+ if (!(armv7_pmnc_read() & PMNC_E)) {
+ return 0;
+ }
+
for (cnt = 0; cnt < pmnc_counters; cnt++) {
if (pmnc_enabled[cnt]) {
int value;
@@ -248,11 +250,8 @@ static int gator_events_armv7_read(int **buffer)
} else {
value = armv7_cntn_read(cnt, 0);
}
- if (value != per_cpu(perfPrev, cpu)[cnt]) {
- per_cpu(perfPrev, cpu)[cnt] = value;
- per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
- per_cpu(perfCnt, cpu)[len++] = value;
- }
+ per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
+ per_cpu(perfCnt, cpu)[len++] = value;
}
}
diff --git a/driver/gator_events_block.c b/driver/gator_events_block.c
index f1bbbc8..a8b8114 100644
--- a/driver/gator_events_block.c
+++ b/driver/gator_events_block.c
@@ -26,7 +26,7 @@ static ulong block_rq_rd_enabled;
static ulong block_rq_wr_key;
static ulong block_rq_rd_key;
static DEFINE_PER_CPU(int[BLOCK_TOTAL], blockCnt);
-static DEFINE_PER_CPU(int[BLOCK_TOTAL * 2], blockGet);
+static DEFINE_PER_CPU(int[BLOCK_TOTAL * 4], blockGet);
static DEFINE_PER_CPU(bool, new_data_avail);
GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct request *rq))
@@ -129,6 +129,8 @@ static int gator_events_block_read(int **buffer)
per_cpu(blockCnt, cpu)[BLOCK_RQ_WR] = 0;
local_irq_restore(flags);
per_cpu(blockGet, cpu)[len++] = block_rq_wr_key;
+ per_cpu(blockGet, cpu)[len++] = 0; // indicates to Streamline that value bytes were written now, not since the last message
+ per_cpu(blockGet, cpu)[len++] = block_rq_wr_key;
per_cpu(blockGet, cpu)[len++] = value;
data += value;
}
@@ -138,6 +140,8 @@ static int gator_events_block_read(int **buffer)
per_cpu(blockCnt, cpu)[BLOCK_RQ_RD] = 0;
local_irq_restore(flags);
per_cpu(blockGet, cpu)[len++] = block_rq_rd_key;
+ per_cpu(blockGet, cpu)[len++] = 0; // indicates to Streamline that value bytes were read now, not since the last message
+ per_cpu(blockGet, cpu)[len++] = block_rq_rd_key;
per_cpu(blockGet, cpu)[len++] = value;
data += value;
}
diff --git a/driver/gator_events_irq.c b/driver/gator_events_irq.c
index 59461b9..435bc86 100644
--- a/driver/gator_events_irq.c
+++ b/driver/gator_events_irq.c
@@ -19,7 +19,6 @@ static ulong softirq_enabled;
static ulong hardirq_key;
static ulong softirq_key;
static DEFINE_PER_CPU(int[TOTALIRQ], irqCnt);
-static DEFINE_PER_CPU(int[TOTALIRQ], irqPrev);
static DEFINE_PER_CPU(int[TOTALIRQ * 2], irqGet);
GATOR_DEFINE_PROBE(irq_handler_exit, TP_PROTO(int irq,
@@ -82,7 +81,6 @@ static int gator_events_irq_online(int** buffer)
local_irq_save(flags);
per_cpu(irqCnt, cpu)[HARDIRQ] = 0;
local_irq_restore(flags);
- per_cpu(irqPrev, cpu)[HARDIRQ] = 0;
per_cpu(irqGet, cpu)[len++] = hardirq_key;
per_cpu(irqGet, cpu)[len++] = 0;
}
@@ -91,7 +89,6 @@ static int gator_events_irq_online(int** buffer)
local_irq_save(flags);
per_cpu(irqCnt, cpu)[SOFTIRQ] = 0;
local_irq_restore(flags);
- per_cpu(irqPrev, cpu)[SOFTIRQ] = 0;
per_cpu(irqGet, cpu)[len++] = softirq_key;
per_cpu(irqGet, cpu)[len++] = 0;
}
@@ -149,11 +146,9 @@ static int gator_events_irq_read(int **buffer)
value = per_cpu(irqCnt, cpu)[HARDIRQ];
per_cpu(irqCnt, cpu)[HARDIRQ] = 0;
local_irq_restore(flags);
- if (value != per_cpu(irqPrev, cpu)[HARDIRQ]) {
- per_cpu(irqPrev, cpu)[HARDIRQ] = value;
- per_cpu(irqGet, cpu)[len++] = hardirq_key;
- per_cpu(irqGet, cpu)[len++] = value;
- }
+
+ per_cpu(irqGet, cpu)[len++] = hardirq_key;
+ per_cpu(irqGet, cpu)[len++] = value;
}
if (softirq_enabled) {
@@ -161,11 +156,9 @@ static int gator_events_irq_read(int **buffer)
value = per_cpu(irqCnt, cpu)[SOFTIRQ];
per_cpu(irqCnt, cpu)[SOFTIRQ] = 0;
local_irq_restore(flags);
- if (value != per_cpu(irqPrev, cpu)[SOFTIRQ]) {
- per_cpu(irqPrev, cpu)[SOFTIRQ] = value;
- per_cpu(irqGet, cpu)[len++] = softirq_key;
- per_cpu(irqGet, cpu)[len++] = value;
- }
+
+ per_cpu(irqGet, cpu)[len++] = softirq_key;
+ per_cpu(irqGet, cpu)[len++] = value;
}
if (buffer)
diff --git a/driver/gator_events_mali.c b/driver/gator_events_mali.c
index 31e8f0d..21a6324 100644..100755
--- a/driver/gator_events_mali.c
+++ b/driver/gator_events_mali.c
@@ -14,17 +14,41 @@
#include "linux/mali_linux_trace.h"
-#define ACTIVITY_START 1
-#define ACTIVITY_STOP 2
+/*
+ * There are (currently) three different variants of the comms between gator and Mali:
+ * 1 (deprecated): No software counter support
+ * 2 (deprecated): Tracepoint called for each separate s/w counter value as it appears
+ * 3 (default): Single tracepoint for all s/w counters in a bundle.
+ * Interface style 3 is the default if no other is specified. 1 and 2 will be eliminated when
+ * existing Mali DDKs are upgraded.
+ */
-#ifndef MALI_SUPPORT
-#error MALI_SUPPORT not defined!
+#if !defined(GATOR_MALI_INTERFACE_STYLE)
+#define GATOR_MALI_INTERFACE_STYLE (3)
#endif
-#define MALI_200 0x0a07
-#define MALI_300 0x0b06 //This is not actually true; Mali-300 is also 0x0b07
-#define MALI_400 0x0b07
-#define MALI_T6xx 0x0056
+#define MALI_200 (0x0a07)
+#define MALI_300 (0x0b06) //This is not actually true; Mali-300 is also 0x0b07
+#define MALI_400 (0x0b07)
+#define MALI_T6xx (0x0056)
+
+/*
+ * List of possible actions allowing DDK to be controlled by Streamline.
+ * The following numbers are used by DDK to control the frame buffer dumping.
+ */
+#define FBDUMP_CONTROL_ENABLE (1)
+#define FBDUMP_CONTROL_RATE (2)
+#define SW_EVENTS_ENABLE (3)
+#define FBDUMP_CONTROL_RESIZE_FACTOR (4)
+
+/*
+ * Check that the MALI_SUPPORT define is set to one of the allowable device codes.
+ */
+#if !defined(MALI_SUPPORT)
+#error MALI_SUPPORT not defined!
+#elif (MALI_SUPPORT != MALI_200) && (MALI_SUPPORT != MALI_300) && (MALI_SUPPORT != MALI_400) && (MALI_SUPPORT != MALI_T6xx)
+#error MALI_SUPPORT set to an invalid device code
+#endif
static const char *mali_name;
@@ -226,12 +250,31 @@ GATOR_DEFINE_PROBE(mali_hw_counter, TP_PROTO(unsigned int event_id, unsigned int
}
}
+#if GATOR_MALI_INTERFACE_STYLE == 2
GATOR_DEFINE_PROBE(mali_sw_counter, TP_PROTO(unsigned int event_id, signed long long value))
{
if (is_sw_counter(event_id)) {
counter_data[event_id] = scale_sw_counter_value(event_id, value);
}
}
+#endif /* GATOR_MALI_INTERFACE_STYLE == 2 */
+
+
+#if GATOR_MALI_INTERFACE_STYLE == 3
+GATOR_DEFINE_PROBE(mali_sw_counters, TP_PROTO(pid_t pid, pid_t tid, void * surface_id, unsigned int * counters))
+{
+ u32 i;
+
+ /* Copy over the values for those counters which are enabled. */
+ for(i=FIRST_SW_COUNTER; i <= LAST_SW_COUNTER; i++)
+ {
+ if(counter_enabled[i])
+ {
+ counter_data[i] = (u32)(counters[i - FIRST_SW_COUNTER]);
+ }
+ }
+}
+#endif /* GATOR_MALI_INTERFACE_STYLE == 3 */
//TODO need to work out how many fp units we have
u32 gator_mali_get_n_fp(void) {
@@ -363,10 +406,30 @@ int gator_events_mali_create_files(struct super_block *sb, struct dentry *root)
//TODO
void _mali_profiling_set_event(unsigned int, unsigned int);
void _mali_osk_fb_control_set(unsigned int, unsigned int);
+void _mali_profiling_control(unsigned int, unsigned int);
void _mali_profiling_get_counters(unsigned int*, unsigned int*, unsigned int*, unsigned int*);
void (*_mali_profiling_get_counters_function_pointer)(unsigned int*, unsigned int*, unsigned int*, unsigned int*);
+/*
+ * Examine list of software counters and determine if any one is enabled.
+ * Returns 1 if any counter is enabled, 0 if none is.
+ */
+static int is_any_sw_counter_enabled(void)
+{
+ unsigned int i;
+
+ for (i = FIRST_SW_COUNTER; i <= LAST_SW_COUNTER; i++)
+ {
+ if (counter_enabled[i])
+ {
+ return 1; /* At least one counter is enabled */
+ }
+ }
+
+ return 0; /* No s/w counters enabled */
+}
+
static void mali_counter_initialize(void)
{
/* If a Mali driver is present and exporting the appropriate symbol
@@ -375,6 +438,7 @@ static void mali_counter_initialize(void)
*/
void (*set_hw_event)(unsigned int, unsigned int);
void (*set_fb_event)(unsigned int, unsigned int);
+ void (*mali_control)(unsigned int, unsigned int);
set_hw_event = symbol_get(_mali_profiling_set_event);
@@ -408,6 +472,27 @@ static void mali_counter_initialize(void)
printk("gator: mali online _mali_osk_fb_control_set symbol not found\n");
}
+ /* Generic control interface for Mali DDK. */
+ mali_control = symbol_get(_mali_profiling_control);
+ if (mali_control) {
+ /* The event attribute in the XML file keeps the actual frame rate. */
+ unsigned int rate = counter_event[COUNTER_FILMSTRIP] & 0xff;
+ unsigned int resize_factor = (counter_event[COUNTER_FILMSTRIP] >> 8) & 0xff;
+
+ pr_debug("gator: mali online _mali_profiling_control symbol @ %p\n", mali_control);
+
+ mali_control(SW_EVENTS_ENABLE, (is_any_sw_counter_enabled()?1:0));
+ mali_control(FBDUMP_CONTROL_ENABLE, (counter_enabled[COUNTER_FILMSTRIP]?1:0));
+ mali_control(FBDUMP_CONTROL_RATE, rate);
+ mali_control(FBDUMP_CONTROL_RESIZE_FACTOR, resize_factor);
+
+ pr_debug("gator: sent mali_control enabled=%d, rate=%d\n", (counter_enabled[COUNTER_FILMSTRIP]?1:0), rate);
+
+ symbol_put(_mali_profiling_control);
+ } else {
+ printk("gator: mali online _mali_profiling_control symbol not found\n");
+ }
+
_mali_profiling_get_counters_function_pointer = symbol_get(_mali_profiling_get_counters);
if (_mali_profiling_get_counters_function_pointer){
pr_debug("gator: mali online _mali_profiling_get_counters symbol @ %p\n", _mali_profiling_get_counters_function_pointer);
@@ -417,13 +502,13 @@ static void mali_counter_initialize(void)
else{
pr_debug("gator WARNING: mali _mali_profiling_get_counters symbol not defined");
}
-
}
static void mali_counter_deinitialize(void)
{
void (*set_hw_event)(unsigned int, unsigned int);
void (*set_fb_event)(unsigned int, unsigned int);
+ void (*mali_control)(unsigned int, unsigned int);
set_hw_event = symbol_get(_mali_profiling_set_event);
@@ -452,6 +537,22 @@ static void mali_counter_deinitialize(void)
printk("gator: mali offline _mali_osk_fb_control_set symbol not found\n");
}
+ /* Generic control interface for Mali DDK. */
+ mali_control = symbol_get(_mali_profiling_control);
+
+ if (mali_control) {
+ pr_debug("gator: mali offline _mali_profiling_control symbol @ %p\n", set_fb_event);
+
+ /* Reset the DDK state - disable counter collection */
+ mali_control(SW_EVENTS_ENABLE, 0);
+
+ mali_control(FBDUMP_CONTROL_ENABLE, 0);
+
+ symbol_put(_mali_profiling_control);
+ } else {
+ printk("gator: mali offline _mali_profiling_control symbol not found\n");
+ }
+
if (_mali_profiling_get_counters_function_pointer){
symbol_put(_mali_profiling_get_counters);
}
@@ -465,10 +566,23 @@ static int gator_events_mali_start(void) {
return -1;
}
+#if GATOR_MALI_INTERFACE_STYLE == 1
+ /* None. */
+#elif GATOR_MALI_INTERFACE_STYLE == 2
+ /* For patched Mali driver. */
if (GATOR_REGISTER_TRACE(mali_sw_counter)) {
printk("gator: mali_sw_counter tracepoint failed to activate\n");
return -1;
}
+#elif GATOR_MALI_INTERFACE_STYLE == 3
+/* For Mali drivers with built-in support. */
+ if (GATOR_REGISTER_TRACE(mali_sw_counters)) {
+ printk("gator: mali_sw_counters tracepoint failed to activate\n");
+ return -1;
+ }
+#else
+#error Unknown GATOR_MALI_INTERFACE_STYLE option.
+#endif
trace_registered = 1;
@@ -483,8 +597,19 @@ static void gator_events_mali_stop(void) {
if (trace_registered) {
GATOR_UNREGISTER_TRACE(mali_hw_counter);
- GATOR_UNREGISTER_TRACE(mali_sw_counter);
-
+
+#if GATOR_MALI_INTERFACE_STYLE == 1
+ /* None. */
+#elif GATOR_MALI_INTERFACE_STYLE == 2
+ /* For patched Mali driver. */
+ GATOR_UNREGISTER_TRACE(mali_sw_counter);
+#elif GATOR_MALI_INTERFACE_STYLE == 3
+ /* For Mali drivers with built-in support. */
+ GATOR_UNREGISTER_TRACE(mali_sw_counters);
+#else
+#error Unknown GATOR_MALI_INTERFACE_STYLE option.
+#endif
+
pr_debug("gator: mali timeline tracepoint deactivated\n");
trace_registered = 0;
@@ -538,22 +663,10 @@ static int gator_events_mali_read(int **buffer) {
// Process other (non-timeline) counters.
for (cnt = COUNTER_VP_C0; cnt <= LAST_SW_COUNTER; cnt++) {
if (counter_enabled[cnt]) {
- u32 value = 0;
-
- // Determine the current value of the counter.
- if( counter_address[cnt] != NULL && 0 ) { // Never true!
- value = *counter_address[cnt];
- } else if (counter_data[cnt]!=0) {
- value = counter_data[cnt];
- counter_data[cnt] = 0;
- }
+ counter_dump[len++] = counter_key[cnt];
+ counter_dump[len++] = counter_data[cnt];
- // Send the counter value only if it differs from last time.
- if (value != counter_prev[cnt]) {
- counter_prev[cnt] = value;
- counter_dump[len++] = counter_key[cnt];
- counter_dump[len++] = value;
- }
+ counter_data[cnt] = 0;
}
}
diff --git a/driver/gator_events_meminfo.c b/driver/gator_events_meminfo.c
index c763634..ad552ef 100644
--- a/driver/gator_events_meminfo.c
+++ b/driver/gator_events_meminfo.c
@@ -10,6 +10,7 @@
#include "gator.h"
#include <linux/workqueue.h>
#include <trace/events/kmem.h>
+#include <linux/hardirq.h>
#define MEMINFO_MEMFREE 0
#define MEMINFO_MEMUSED 1
@@ -145,7 +146,7 @@ static void gator_events_meminfo_stop(void)
}
}
-// Must be run in a work queue as the kernel function si_meminfo() can sleep
+// Must be run in process context (work queue) as the kernel function si_meminfo() can sleep
static void wq_sched_handler(struct work_struct *wsptr)
{
struct sysinfo info;
@@ -189,7 +190,11 @@ static int gator_events_meminfo_read(long long **buffer)
if (last_mem_event != mem_event) {
last_mem_event = mem_event;
- schedule_work(&work);
+ if (in_interrupt()) {
+ schedule_work(&work);
+ } else {
+ wq_sched_handler(NULL);
+ }
}
if (!new_data_avail)
diff --git a/driver/gator_events_mmaped.c b/driver/gator_events_mmaped.c
index cbb22b1..f81c402 100644
--- a/driver/gator_events_mmaped.c
+++ b/driver/gator_events_mmaped.c
@@ -14,11 +14,12 @@
* <counter_set name="mmaped_cntX">
* <counter name="mmaped_cnt0"/>
* <counter name="mmaped_cnt1"/>
+ * <counter name="mmaped_cnt2"/>
* </counter_set>
* <category name="mmaped" counter_set="mmaped_cntX" per_cpu="no">
- * <event event="0x0" title="Simulated" name="Sine" description="Sort-of-sine"/>
- * <event event="0x1" title="Simulated" name="Triangle" description="Triangular wave"/>
- * <event event="0x2" title="Simulated" name="PWM" description="PWM Signal"/>
+ * <event event="0x0" title="Simulated" name="Sine" display="maximum" average_selection="yes" description="Sort-of-sine"/>
+ * <event event="0x1" title="Simulated" name="Triangle" display="maximum" average_selection="yes" description="Triangular wave"/>
+ * <event event="0x2" title="Simulated" name="PWM" display="maximum" average_selection="yes" description="PWM Signal"/>
* </category>
*/
@@ -42,6 +43,10 @@ static int mmaped_buffer[MMAPED_COUNTERS_NUM * 2];
static void __iomem *mmaped_base;
#endif
+#ifndef TODO
+static s64 prev_time;
+#endif
+
/* Adds mmaped_cntX directories and enabled, event, and key files to /dev/gator/events */
static int gator_events_mmaped_create_files(struct super_block *sb,
struct dentry *root)
@@ -77,6 +82,12 @@ static int gator_events_mmaped_start(void)
writel(ENABLED, COUNTERS_CONTROL_OFFSET);
#endif
+#ifndef TODO
+ struct timespec ts;
+ getnstimeofday(&ts);
+ prev_time = timespec_to_ns(&ts);
+#endif
+
return 0;
}
@@ -90,50 +101,60 @@ static void gator_events_mmaped_stop(void)
#ifndef TODO
/* This function "simulates" counters, generating values of fancy
* functions like sine or triangle... */
-static int mmaped_simulate(int counter)
+static int mmaped_simulate(int counter, int delta_in_us)
{
int result = 0;
switch (counter) {
case 0: /* sort-of-sine */
{
- static int t;
+ static int t = 0;
int x;
- if (t % 1024 < 512)
- x = 512 - (t % 512);
+ t += delta_in_us;
+ if (t > 2048000)
+ t = 0;
+
+ if (t % 1024000 < 512000)
+ x = 512000 - (t % 512000);
else
- x = t % 512;
+ x = t % 512000;
- result = 32 * x / 512;
+ result = 32 * x / 512000;
result = result * result;
- if (t < 1024)
+ if (t < 1024000)
result = 1922 - result;
-
- t = (t + 1) % 2048;
}
break;
case 1: /* triangle */
{
static int v, d = 1;
- v += d;
- if (v % 2000 == 0)
- d = -d;
+ v = v + d * delta_in_us;
+ if (v < 0) {
+ v = 0;
+ d = 1;
+ } else if (v > 1000000) {
+ v = 1000000;
+ d = -1;
+ }
result = v;
}
break;
case 2: /* PWM signal */
{
- static int t, dc;
-
- t = (t + 1) % 2000;
- if (t % 100)
- dc = (dc + 200) % 2000;
+ static int t, dc, x;
+
+ t += delta_in_us;
+ if (t > 1000000)
+ t = 0;
+ if (x / 1000000 != (x + delta_in_us) / 1000000)
+ dc = (dc + 100000) % 1000000;
+ x += delta_in_us;
- result = t < dc ? 0 : 2000;
+ result = t < dc ? 0 : 10;
}
break;
}
@@ -146,11 +167,23 @@ static int gator_events_mmaped_read(int **buffer)
{
int i;
int len = 0;
+#ifndef TODO
+ int delta_in_us;
+ struct timespec ts;
+ s64 time;
+#endif
/* System wide counters - read from one core only */
if (smp_processor_id())
return 0;
+#ifndef TODO
+ getnstimeofday(&ts);
+ time = timespec_to_ns(&ts);
+ delta_in_us = (int)(time - prev_time) / 1000;
+ prev_time = time;
+#endif
+
for (i = 0; i < MMAPED_COUNTERS_NUM; i++) {
if (mmaped_counters[i].enabled) {
mmaped_buffer[len++] = mmaped_counters[i].key;
@@ -159,7 +192,7 @@ static int gator_events_mmaped_read(int **buffer)
COUNTERS_VALUE_OFFSET[i]);
#else
mmaped_buffer[len++] = mmaped_simulate(
- mmaped_counters[i].event);
+ mmaped_counters[i].event, delta_in_us);
#endif
}
}
diff --git a/driver/gator_events_net.c b/driver/gator_events_net.c
index ef1623b..9298905 100644
--- a/driver/gator_events_net.c
+++ b/driver/gator_events_net.c
@@ -9,6 +9,7 @@
#include "gator.h"
#include <linux/netdevice.h>
+#include <linux/hardirq.h>
#define NETRX 0
#define NETTX 1
@@ -20,7 +21,7 @@ static ulong netrx_key;
static ulong nettx_key;
static int rx_total, tx_total;
static ulong netPrev[TOTALNET];
-static int netGet[TOTALNET * 2];
+static int netGet[TOTALNET * 4];
static void get_network_stats(struct work_struct *wsptr) {
int rx = 0, tx = 0;
@@ -102,19 +103,31 @@ static int gator_events_net_read(int **buffer)
if (smp_processor_id() != 0)
return 0;
- schedule_work(&wq_get_stats);
+ if (!netrx_enabled && !nettx_enabled)
+ return 0;
+
+ if (in_interrupt()){
+ schedule_work(&wq_get_stats);
+ } else {
+ get_network_stats(NULL);
+ }
+
calculate_delta(&rx_delta, &tx_delta);
len = 0;
if (netrx_enabled && last_rx_delta != rx_delta) {
last_rx_delta = rx_delta;
netGet[len++] = netrx_key;
+ netGet[len++] = 0; // indicates to Streamline that rx_delta bytes were transmitted now, not since the last message
+ netGet[len++] = netrx_key;
netGet[len++] = rx_delta;
}
if (nettx_enabled && last_tx_delta != tx_delta) {
last_tx_delta = tx_delta;
netGet[len++] = nettx_key;
+ netGet[len++] = 0; // indicates to Streamline that tx_delta bytes were transmitted now, not since the last message
+ netGet[len++] = nettx_key;
netGet[len++] = tx_delta;
}
diff --git a/driver/gator_events_perf_pmu.c b/driver/gator_events_perf_pmu.c
index 322ebc4..da76a9c 100644..100755
--- a/driver/gator_events_perf_pmu.c
+++ b/driver/gator_events_perf_pmu.c
@@ -19,6 +19,8 @@ int ccnt = 0;
#define CNTMAX (6+1)
+static DEFINE_MUTEX(perf_mutex);
+
unsigned long pmnc_enabled[CNTMAX];
unsigned long pmnc_event[CNTMAX];
unsigned long pmnc_count[CNTMAX];
@@ -122,12 +124,20 @@ static void gator_events_perf_pmu_online_dispatch(int cpu)
static void gator_events_perf_pmu_offline_dispatch(int cpu)
{
int cnt;
+ struct perf_event * pe;
for (cnt = 0; cnt < pmnc_counters; cnt++) {
- if (per_cpu(pevent, cpu)[cnt] != NULL) {
- perf_event_release_kernel(per_cpu(pevent, cpu)[cnt]);
+ pe = NULL;
+ mutex_lock(&perf_mutex);
+ if (per_cpu(pevent, cpu)[cnt]) {
+ pe = per_cpu(pevent, cpu)[cnt];
per_cpu(pevent, cpu)[cnt] = NULL;
}
+ mutex_unlock(&perf_mutex);
+
+ if (pe) {
+ perf_event_release_kernel(pe);
+ }
}
}
@@ -139,7 +149,7 @@ static int gator_events_perf_pmu_start(void)
for_each_present_cpu(cpu) {
for (cnt = 0; cnt < pmnc_counters; cnt++) {
per_cpu(pevent, cpu)[cnt] = NULL;
- if (!pmnc_enabled[cnt] || pmnc_count[cnt] > 0) // Skip disabled counters and EBS counters
+ if (!pmnc_enabled[cnt]) // Skip disabled counters
continue;
per_cpu(perfPrev, cpu)[cnt] = 0;
diff --git a/driver/gator_events_power.c b/driver/gator_events_power.c
deleted file mode 100644
index a0ae684..0000000
--- a/driver/gator_events_power.c
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- * Copyright (C) ARM Limited 2011-2012. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
-
-#include "gator.h"
-#include <linux/slab.h>
-#include <linux/cpufreq.h>
-#include <trace/events/power.h>
-
-// cpu_frequency and cpu_idle trace points were introduced in Linux kernel v2.6.38
-// the now deprecated power_frequency trace point was available prior to 2.6.38, but only for x86
-// CPU Idle is currently disabled in the .xml
-#if GATOR_CPU_FREQ_SUPPORT
-enum {
- POWER_CPU_FREQ,
- POWER_CPU_IDLE,
- POWER_TOTAL
-};
-
-static ulong power_cpu_enabled[POWER_TOTAL];
-static ulong power_cpu_key[POWER_TOTAL];
-static DEFINE_PER_CPU(ulong[POWER_TOTAL], power);
-static DEFINE_PER_CPU(ulong[POWER_TOTAL], prev);
-static DEFINE_PER_CPU(int *, powerGet);
-
-GATOR_DEFINE_PROBE(cpu_frequency, TP_PROTO(unsigned int frequency, unsigned int cpu))
-{
- per_cpu(power, cpu)[POWER_CPU_FREQ] = frequency * 1000;
-}
-
-GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu))
-{
- per_cpu(power, cpu)[POWER_CPU_IDLE] = state;
-}
-
-static int gator_events_power_create_files(struct super_block *sb, struct dentry *root)
-{
- struct dentry *dir;
-
- // cpu_frequency
- dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_freq");
- if (!dir) {
- return -1;
- }
- gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_FREQ]);
- gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_FREQ]);
-
- // cpu_idle
- dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_idle");
- if (!dir) {
- return -1;
- }
- gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_IDLE]);
- gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_IDLE]);
-
- return 0;
-}
-
-static int gator_events_power_populate(int cpu, int** buffer)
-{
- int i, len = 0;
-
- for (i = 0; i < POWER_TOTAL; i++) {
- if (power_cpu_enabled[i]) {
- if (per_cpu(power, cpu)[i] != per_cpu(prev, cpu)[i]) {
- per_cpu(prev, cpu)[i] = per_cpu(power, cpu)[i];
- per_cpu(powerGet, cpu)[len++] = power_cpu_key[i];
- per_cpu(powerGet, cpu)[len++] = per_cpu(power, cpu)[i];
- }
- }
- }
-
- if (buffer)
- *buffer = per_cpu(powerGet, cpu);
-
- return len;
-}
-
-static int gator_events_power_online(int** buffer)
-{
- int i, cpu = smp_processor_id();
- for (i = 0; i < POWER_TOTAL; i++)
- per_cpu(prev, cpu)[i] = -1;
- per_cpu(power, cpu)[POWER_CPU_FREQ] = cpufreq_quick_get(cpu) * 1000;
- return gator_events_power_populate(cpu, buffer);
-}
-
-static int gator_events_power_offline(int** buffer)
-{
- int cpu = smp_processor_id();
- // Set frequency to zero on an offline
- per_cpu(power, cpu)[POWER_CPU_FREQ] = 0;
- return gator_events_power_populate(cpu, buffer);
-}
-
-static int gator_events_power_start(void)
-{
- int cpu;
-
- for_each_present_cpu(cpu) {
- per_cpu(powerGet, cpu) = kmalloc(POWER_TOTAL * 2, GFP_KERNEL);
- if (!per_cpu(powerGet, cpu))
- return -1;
- }
-
- // register tracepoints
- if (power_cpu_enabled[POWER_CPU_FREQ])
- if (GATOR_REGISTER_TRACE(cpu_frequency))
- goto fail_cpu_frequency_exit;
- if (power_cpu_enabled[POWER_CPU_IDLE])
- if (GATOR_REGISTER_TRACE(cpu_idle))
- goto fail_cpu_idle_exit;
- pr_debug("gator: registered power event tracepoints\n");
-
- return 0;
-
- // unregister tracepoints on error
-fail_cpu_idle_exit:
- GATOR_UNREGISTER_TRACE(cpu_frequency);
-fail_cpu_frequency_exit:
- pr_err("gator: power event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");
-
- return -1;
-}
-
-static void gator_events_power_stop(void)
-{
- int i, cpu;
- if (power_cpu_enabled[POWER_CPU_FREQ])
- GATOR_UNREGISTER_TRACE(cpu_frequency);
- if (power_cpu_enabled[POWER_CPU_IDLE])
- GATOR_UNREGISTER_TRACE(cpu_idle);
- pr_debug("gator: unregistered power event tracepoints\n");
-
- for (i = 0; i < POWER_TOTAL; i++) {
- power_cpu_enabled[i] = 0;
- }
-
- for_each_present_cpu(cpu) {
- kfree(per_cpu(powerGet, cpu));
- }
-}
-
-static int gator_events_power_read(int **buffer)
-{
- return gator_events_power_populate(smp_processor_id(), buffer);
-}
-
-static struct gator_interface gator_events_power_interface = {
- .create_files = gator_events_power_create_files,
- .online = gator_events_power_online,
- .offline = gator_events_power_offline,
- .start = gator_events_power_start,
- .stop = gator_events_power_stop,
- .read = gator_events_power_read,
-};
-#endif
-
-int gator_events_power_init(void)
-{
-#if (GATOR_CPU_FREQ_SUPPORT)
- int i;
- for (i = 0; i < POWER_TOTAL; i++) {
- power_cpu_enabled[i] = 0;
- power_cpu_key[i] = gator_events_get_key();
- }
-
- return gator_events_install(&gator_events_power_interface);
-#else
- return -1;
-#endif
-}
-gator_events_init(gator_events_power_init);
diff --git a/driver/gator_events_scorpion.c b/driver/gator_events_scorpion.c
index 477e7c9..ed0d8de 100644
--- a/driver/gator_events_scorpion.c
+++ b/driver/gator_events_scorpion.c
@@ -34,7 +34,6 @@ static unsigned long pmnc_enabled[CNTMAX];
static unsigned long pmnc_event[CNTMAX];
static unsigned long pmnc_key[CNTMAX];
-static DEFINE_PER_CPU(int[CNTMAX], perfPrev);
static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt);
enum scorpion_perf_types {
@@ -538,8 +537,6 @@ static int gator_events_scorpion_online(int** buffer)
for (cnt = CCNT; cnt < CNTMAX; cnt++) {
unsigned long event;
- per_cpu(perfPrev, smp_processor_id())[cnt] = 0;
-
if (!pmnc_enabled[cnt])
continue;
@@ -574,7 +571,7 @@ static int gator_events_scorpion_online(int** buffer)
value = 0;
}
scorpion_pmnc_reset_counter(cnt);
- per_cpu(perfPrev, cpu)[cnt] = 0;
+
per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
per_cpu(perfCnt, cpu)[len++] = 0;
}
@@ -607,6 +604,11 @@ static int gator_events_scorpion_read(int **buffer)
int cnt, len = 0;
int cpu = smp_processor_id();
+ // a context switch may occur before the online hotplug event, thus need to check that the pmu is enabled
+ if (!(scorpion_pmnc_read() & PMNC_E)) {
+ return 0;
+ }
+
for (cnt = 0; cnt < pmnc_counters; cnt++) {
if (pmnc_enabled[cnt]) {
int value;
@@ -618,11 +620,9 @@ static int gator_events_scorpion_read(int **buffer)
value = 0;
}
scorpion_pmnc_reset_counter(cnt);
- if (value != per_cpu(perfPrev, cpu)[cnt]) {
- per_cpu(perfPrev, cpu)[cnt] = value;
- per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
- per_cpu(perfCnt, cpu)[len++] = value;
- }
+
+ per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt];
+ per_cpu(perfCnt, cpu)[len++] = value;
}
}
diff --git a/driver/gator_fs.c b/driver/gator_fs.c
index 8277c3a..39adfbe 100644
--- a/driver/gator_fs.c
+++ b/driver/gator_fs.c
@@ -233,9 +233,17 @@ static int gatorfs_fill_super(struct super_block *sb, void *data, int silent)
return -ENOMEM;
root_inode->i_op = &simple_dir_inode_operations;
root_inode->i_fop = &simple_dir_operations;
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
root_dentry = d_alloc_root(root_inode);
+#else
+ root_dentry = d_make_root(root_inode);
+#endif
+
if (!root_dentry) {
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
iput(root_inode);
+#endif
return -ENOMEM;
}
diff --git a/driver/gator_hrtimer_gator.c b/driver/gator_hrtimer_gator.c
index 5896b3c..846fba4 100644..100755
--- a/driver/gator_hrtimer_gator.c
+++ b/driver/gator_hrtimer_gator.c
@@ -13,6 +13,7 @@
void (*callback)(void);
DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer);
+DEFINE_PER_CPU(int, hrtimer_is_active);
static ktime_t profiling_interval;
static void gator_hrtimer_online(int cpu);
static void gator_hrtimer_offline(int cpu);
@@ -32,11 +33,16 @@ static void gator_hrtimer_switch_cpus_online(void *unused)
static void gator_hrtimer_online(int cpu)
{
struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
+
if (cpu != smp_processor_id()) {
smp_call_function_single(cpu, gator_hrtimer_switch_cpus_online, NULL, 1);
return;
}
+ if (per_cpu(hrtimer_is_active, cpu) || profiling_interval.tv64 == 0)
+ return;
+
+ per_cpu(hrtimer_is_active, cpu) = 1;
hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
hrtimer->function = gator_hrtimer_notify;
hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED);
@@ -50,20 +56,35 @@ static void gator_hrtimer_switch_cpus_offline(void *unused)
static void gator_hrtimer_offline(int cpu)
{
struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu);
+
if (cpu != smp_processor_id()) {
smp_call_function_single(cpu, gator_hrtimer_switch_cpus_offline, NULL, 1);
return;
}
+ if (!per_cpu(hrtimer_is_active, cpu))
+ return;
+
+ per_cpu(hrtimer_is_active, cpu) = 0;
hrtimer_cancel(hrtimer);
}
static int gator_hrtimer_init(int interval, void (*func)(void))
{
+ int cpu;
+
(callback) = (func);
+ for_each_present_cpu(cpu) {
+ per_cpu(hrtimer_is_active, cpu) = 0;
+ }
+
// calculate profiling interval
- profiling_interval = ns_to_ktime(1000000000UL / interval);
+ if (interval > 0) {
+ profiling_interval = ns_to_ktime(1000000000UL / interval);
+ } else {
+ profiling_interval.tv64 = 0;
+ }
return 0;
}
diff --git a/driver/gator_main.c b/driver/gator_main.c
index fff2d19..7d48812 100644
--- a/driver/gator_main.c
+++ b/driver/gator_main.c
@@ -7,7 +7,7 @@
*
*/
-static unsigned long gator_protocol_version = 8;
+static unsigned long gator_protocol_version = 9;
#include <linux/slab.h>
#include <linux/cpu.h>
@@ -39,7 +39,7 @@ static unsigned long gator_protocol_version = 8;
#endif
#ifndef CONFIG_HIGH_RES_TIMERS
-#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
+#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
#endif
#if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
@@ -48,37 +48,39 @@ static unsigned long gator_protocol_version = 8;
#if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
#ifndef CONFIG_PERF_EVENTS
-#warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
+#error gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
#elif !defined CONFIG_HW_PERF_EVENTS
-#warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
+#error gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
#endif
#endif
-#if (!(GATOR_CPU_FREQ_SUPPORT))
-#warning gator requires kernel version 2.6.38 or greater and CONFIG_CPU_FREQ defined in order to enable the CPU Freq timeline chart
-#endif
-
/******************************************************************************
* DEFINES
******************************************************************************/
-#define TIMER_BUFFER_SIZE_DEFAULT (512*1024)
-#define EVENT_BUFFER_SIZE_DEFAULT (128*1024)
+#define BACKTRACE_BUFFER_SIZE (128*1024)
+#define COUNTER_BUFFER_SIZE (128*1024)
+#define ANNOTATE_BUFFER_SIZE (64*1024) // annotate counters have the core as part of the data and the core value in the frame header may be discarded
+#define SCHED_TRACE_BUFFER_SIZE (128*1024)
+#define GPU_TRACE_BUFFER_SIZE (64*1024)
+#define COUNTER2_BUFFER_SIZE (64*1024) // counters2 counters have the core as part of the data and the core value in the frame header may be discarded
+#define WFI_BUFFER_SIZE (32*1024) // wfi counters have the core as part of the data and the core value in the frame header may be discarded
#define NO_COOKIE 0UL
#define INVALID_COOKIE ~0UL
-#define FRAME_HRTIMER 1
-#define FRAME_EVENT 2
-#define FRAME_ANNOTATE 3
+#define FRAME_BACKTRACE 1
+#define FRAME_COUNTER 2
+#define FRAME_ANNOTATE 3
+#define FRAME_SCHED_TRACE 4
+#define FRAME_GPU_TRACE 5
+#define FRAME_COUNTER2 6
+#define FRAME_WFI 7
-#define MESSAGE_COOKIE 1
-#define MESSAGE_COUNTERS 3
-#define MESSAGE_START_BACKTRACE 5
-#define MESSAGE_END_BACKTRACE 7
-#define MESSAGE_SCHEDULER_TRACE 9
-#define MESSAGE_PID_NAME 11
-#define MESSAGE_GPU_TRACE 13
-#define MESSAGE_OVERFLOW 127
+#define MESSAGE_COOKIE 1
+#define MESSAGE_START_BACKTRACE 5
+#define MESSAGE_END_BACKTRACE 7
+#define MESSAGE_SUMMARY 9
+#define MESSAGE_PID_NAME 11
#define MAXSIZE_PACK32 5
#define MAXSIZE_PACK64 9
@@ -89,7 +91,7 @@ static unsigned long gator_protocol_version = 8;
#define PC_REG regs->ip
#endif
-enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS};
+enum {BACKTRACE_BUF, COUNTER_BUF, SCHED_TRACE_BUF, GPU_TRACE_BUF, ANNOTATE_BUF, COUNTER2_BUF, WFI_BUF, NUM_GATOR_BUFS};
/******************************************************************************
* Globals
@@ -101,25 +103,26 @@ static unsigned long gator_backtrace_depth;
static unsigned long gator_started;
static unsigned long gator_buffer_opened;
static unsigned long gator_timer_count;
-static unsigned long gator_streaming;
+static unsigned long gator_response_type;
static DEFINE_MUTEX(start_mutex);
static DEFINE_MUTEX(gator_buffer_mutex);
bool event_based_sampling;
static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
-
-static void buffer_check(int cpu, int buftype);
+static LIST_HEAD(gator_events);
/******************************************************************************
* Prototypes
******************************************************************************/
+static void buffer_check(int cpu, int buftype);
+static int buffer_bytes_available(int cpu, int buftype);
static bool buffer_check_space(int cpu, int buftype, int bytes);
+static int contiguous_space_available(int cpu, int bufytpe);
static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x);
static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x);
+static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len);
static void gator_buffer_write_string(int cpu, int buftype, char *x);
-static int gator_write_packed_int(char *buffer, unsigned int x);
-static int gator_write_packed_int64(char *buffer, unsigned long long x);
static void gator_add_trace(int cpu, int buftype, unsigned int address);
static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs);
static uint64_t gator_get_time(void);
@@ -131,15 +134,16 @@ static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
-static DEFINE_PER_CPU(uint64_t, emit_overflow);
/******************************************************************************
* Application Includes
******************************************************************************/
+#include "gator_marshaling.c"
#include "gator_hrtimer_perf.c"
#include "gator_hrtimer_gator.c"
#include "gator_cookies.c"
#include "gator_trace_sched.c"
+#include "gator_trace_power.c"
#include "gator_trace_gpu.c"
#include "gator_backtrace.c"
#include "gator_annotate.c"
@@ -179,10 +183,10 @@ static bool buffer_commit_ready(int* cpu, int* buftype)
/******************************************************************************
* Buffer management
******************************************************************************/
-static bool buffer_check_space(int cpu, int buftype, int bytes)
+static int buffer_bytes_available(int cpu, int buftype)
{
int remaining, filled;
-
+
filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype];
if (filled < 0) {
filled += gator_buffer_size[buftype];
@@ -198,12 +202,24 @@ static bool buffer_check_space(int cpu, int buftype, int bytes)
remaining -= 2000;
}
+ return remaining;
+}
+
+static int contiguous_space_available(int cpu, int buftype)
+{
+ int remaining = buffer_bytes_available(cpu, buftype);
+ int contiguous = gator_buffer_size[buftype] - per_cpu(gator_buffer_write, cpu)[buftype];
+ if (remaining < contiguous)
+ return remaining;
+ else
+ return contiguous;
+}
+
+static bool buffer_check_space(int cpu, int buftype, int bytes)
+{
+ int remaining = buffer_bytes_available(cpu, buftype);
+
if (remaining < bytes) {
- if (per_cpu(buffer_space_available, cpu)[buftype] == true) {
- // overflow packet to be emitted at a later time, as we may be in the middle of writing a message, e.g. counters
- per_cpu(emit_overflow, cpu) = gator_get_time();
- pr_err("overflow: remaining = %d\n", gator_buffer_size[buftype] - filled);
- }
per_cpu(buffer_space_available, cpu)[buftype] = false;
} else {
per_cpu(buffer_space_available, cpu)[buftype] = true;
@@ -238,19 +254,33 @@ static void gator_buffer_header(int cpu, int buftype)
{
int frame;
- if (buftype == TIMER_BUF)
- frame = FRAME_HRTIMER;
- else if (buftype == EVENT_BUF)
- frame = FRAME_EVENT;
+ if (buftype == BACKTRACE_BUF)
+ frame = FRAME_BACKTRACE;
+ else if (buftype == COUNTER_BUF)
+ frame = FRAME_COUNTER;
+ else if (buftype == ANNOTATE_BUF)
+ frame = FRAME_ANNOTATE;
+ else if (buftype == SCHED_TRACE_BUF)
+ frame = FRAME_SCHED_TRACE;
+ else if (buftype == GPU_TRACE_BUF)
+ frame = FRAME_GPU_TRACE;
+ else if (buftype == COUNTER2_BUF)
+ frame = FRAME_COUNTER2;
+ else if (buftype == WFI_BUF)
+ frame = FRAME_WFI;
else
frame = -1;
- gator_buffer_write_packed_int(cpu, buftype, frame);
- gator_buffer_write_packed_int(cpu, buftype, cpu);
+ if (per_cpu(gator_buffer, cpu)[buftype]) {
+ marshal_frame(cpu, buftype, frame);
+ }
}
static void gator_commit_buffer(int cpu, int buftype)
{
+ if (!per_cpu(gator_buffer, cpu)[buftype])
+ return;
+
per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype];
gator_buffer_header(cpu, buftype);
wake_up(&gator_buffer_wait);
@@ -276,8 +306,7 @@ static void gator_add_trace(int cpu, int buftype, unsigned int address)
offset = address;
}
- gator_buffer_write_packed_int(cpu, buftype, offset & ~1);
- gator_buffer_write_packed_int(cpu, buftype, cookie);
+ marshal_backtrace(offset & ~1, cookie);
}
static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
@@ -288,12 +317,8 @@ static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
if (!regs)
return;
- gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE);
- gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
- gator_buffer_write_packed_int(cpu, buftype, exec_cookie);
- gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid);
- gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid);
- gator_buffer_write_packed_int(cpu, buftype, inKernel);
+ if (!marshal_backtrace_header(exec_cookie, current->tgid, current->pid, inKernel))
+ return;
if (inKernel) {
kernel_backtrace(cpu, buftype, regs);
@@ -306,142 +331,54 @@ static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs)
arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth);
}
- gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE);
+ marshal_backtrace_footer();
}
/******************************************************************************
* hrtimer interrupt processing
******************************************************************************/
-static LIST_HEAD(gator_events);
-
static void gator_timer_interrupt(void)
{
struct pt_regs * const regs = get_irq_regs();
int cpu = smp_processor_id();
- int *buffer, len, i, buftype = TIMER_BUF;
- long long *buffer64;
- struct gator_interface *gi;
-
- // Output scheduler trace
- len = gator_trace_sched_read(&buffer64);
- if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
- gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE);
- gator_buffer_write_packed_int(cpu, buftype, len);
- for (i = 0; i < len; i++) {
- gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
- }
- }
-
- // Output GPU trace
- len = gator_trace_gpu_read(&buffer64);
- if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
- gator_buffer_write_packed_int(cpu, buftype, MESSAGE_GPU_TRACE);
- gator_buffer_write_packed_int(cpu, buftype, len);
- for (i = 0; i < len; i++) {
- gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
- }
- }
-
- // Output counters
- if (buffer_check_space(cpu, buftype, MAXSIZE_PACK32 * 2 + MAXSIZE_PACK64)) {
- gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS);
- gator_buffer_write_packed_int64(cpu, buftype, gator_get_time());
- list_for_each_entry(gi, &gator_events, list) {
- if (gi->read) {
- len = gi->read(&buffer);
- if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
- gator_buffer_write_packed_int(cpu, buftype, len);
- for (i = 0; i < len; i++) {
- gator_buffer_write_packed_int(cpu, buftype, buffer[i]);
- }
- }
- } else if (gi->read64) {
- len = gi->read64(&buffer64);
- if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + MAXSIZE_PACK32)) {
- gator_buffer_write_packed_int(cpu, buftype, len);
- for (i = 0; i < len; i++) {
- gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]);
- }
- }
- }
- }
- gator_buffer_write_packed_int(cpu, buftype, 0);
- }
// Output backtrace
- if (!event_based_sampling && buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32))
- gator_add_sample(cpu, buftype, regs);
-
- // Overflow message
- if (per_cpu(emit_overflow, cpu)) {
- gator_buffer_write_packed_int(cpu, buftype, MESSAGE_OVERFLOW);
- gator_buffer_write_packed_int64(cpu, buftype, per_cpu(emit_overflow, cpu));
- per_cpu(emit_overflow, cpu) = 0;
- }
+ gator_add_sample(cpu, BACKTRACE_BUF, regs);
- // Check and commit; generally, commit is set to occur once per second
- buffer_check(cpu, buftype);
+ // Collect counters
+ collect_counters();
}
-DEFINE_PER_CPU(int, hrtimer_is_active);
-static int hrtimer_running;
+static int gator_running;
// This function runs in interrupt context and on the appropriate core
static void gator_timer_offline(void* unused)
{
+ struct gator_interface *gi;
int i, len, cpu = smp_processor_id();
int* buffer;
- long long* buffer64;
-
- if (per_cpu(hrtimer_is_active, cpu)) {
- struct gator_interface *gi;
- gator_hrtimer_offline(cpu);
- per_cpu(hrtimer_is_active, cpu) = 0;
-
- // Output scheduler trace
- len = gator_trace_sched_offline(&buffer64);
- if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
- gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_SCHEDULER_TRACE);
- gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
- for (i = 0; i < len; i++) {
- gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]);
- }
- }
- // Output GPU trace
- len = gator_trace_gpu_offline(&buffer64);
- if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) {
- gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_GPU_TRACE);
- gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
- for (i = 0; i < len; i++) {
- gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]);
- }
- }
+ gator_trace_sched_offline();
+ gator_trace_power_offline();
- // offline any events and output counters
- gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS);
- gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time());
+ gator_hrtimer_offline(cpu);
+
+ // Offline any events and output counters
+ if (marshal_event_header()) {
list_for_each_entry(gi, &gator_events, list) {
if (gi->offline) {
len = gi->offline(&buffer);
- if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
- gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
- for (i = 0; i < len; i++)
- gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]);
- }
+ marshal_event(len, buffer);
}
}
- gator_buffer_write_packed_int(cpu, TIMER_BUF, 0);
-
- gator_commit_buffer(cpu, TIMER_BUF);
}
- if (event_based_sampling) {
- gator_commit_buffer(cpu, EVENT_BUF);
- }
+ // Flush all buffers on this core
+ for (i = 0; i < NUM_GATOR_BUFS; i++)
+ gator_commit_buffer(cpu, i);
}
-// This function runs in interrupt context and may be running on a core other than core 'cpu'
+// This function runs in process context and may be running on a core other than core 'cpu'
static void gator_timer_offline_dispatch(int cpu)
{
struct gator_interface *gi;
@@ -457,13 +394,13 @@ static void gator_timer_stop(void)
{
int cpu;
- if (hrtimer_running) {
+ if (gator_running) {
on_each_cpu(gator_timer_offline, NULL, 1);
for_each_online_cpu(cpu) {
gator_timer_offline_dispatch(cpu);
}
- hrtimer_running = 0;
+ gator_running = 0;
gator_hrtimer_shutdown();
}
}
@@ -471,32 +408,23 @@ static void gator_timer_stop(void)
// This function runs in interrupt context and on the appropriate core
static void gator_timer_online(void* unused)
{
- int i, len, cpu = smp_processor_id();
+ struct gator_interface *gi;
+ int len, cpu = smp_processor_id();
int* buffer;
- if (!per_cpu(hrtimer_is_active, cpu)) {
- struct gator_interface *gi;
+ gator_trace_power_online();
- // online any events and output counters
- gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS);
- gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time());
+ // online any events and output counters
+ if (marshal_event_header()) {
list_for_each_entry(gi, &gator_events, list) {
if (gi->online) {
len = gi->online(&buffer);
- if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) {
- gator_buffer_write_packed_int(cpu, TIMER_BUF, len);
- for (i = 0; i < len; i++)
- gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]);
- }
+ marshal_event(len, buffer);
}
}
- gator_buffer_write_packed_int(cpu, TIMER_BUF, 0);
-
- gator_event_sampling_online();
-
- gator_hrtimer_online(cpu);
- per_cpu(hrtimer_is_active, cpu) = 1;
}
+
+ gator_hrtimer_online(cpu);
}
// This function runs in interrupt context and may be running on a core other than core 'cpu'
@@ -511,21 +439,22 @@ static void gator_timer_online_dispatch(int cpu)
gator_event_sampling_online_dispatch(cpu);
}
-int gator_timer_start(unsigned long setup)
+int gator_timer_start(unsigned long sample_rate)
{
int cpu;
- if (!setup) {
- pr_err("gator: cannot start due to a system tick value of zero\n");
- return -1;
- } else if (hrtimer_running) {
- pr_notice("gator: high res timer already running\n");
+ if (gator_running) {
+ pr_notice("gator: already running\n");
return 0;
}
- hrtimer_running = 1;
+ gator_running = 1;
- if (gator_hrtimer_init(setup, gator_timer_interrupt) == -1)
+ // event based sampling trumps hr timer based sampling
+ if (event_based_sampling)
+ sample_rate = 0;
+
+ if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
return -1;
for_each_online_cpu(cpu) {
@@ -550,7 +479,7 @@ static uint64_t gator_get_time(void)
/******************************************************************************
* cpu hotplug and pm notifiers
******************************************************************************/
-static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
+static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
long cpu = (long)hcpu;
@@ -570,8 +499,8 @@ static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long
return NOTIFY_OK;
}
-static struct notifier_block __refdata gator_cpu_notifier = {
- .notifier_call = gator_cpu_notify,
+static struct notifier_block __refdata gator_hotcpu_notifier = {
+ .notifier_call = gator_hotcpu_notify,
};
// n.b. calling "on_each_cpu" only runs on those that are online
@@ -583,7 +512,7 @@ static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void
switch (event) {
case PM_HIBERNATION_PREPARE:
case PM_SUSPEND_PREPARE:
- unregister_hotcpu_notifier(&gator_cpu_notifier);
+ unregister_hotcpu_notifier(&gator_hotcpu_notifier);
unregister_scheduler_tracepoints();
on_each_cpu(gator_timer_offline, NULL, 1);
for_each_online_cpu(cpu) {
@@ -597,7 +526,7 @@ static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void
}
on_each_cpu(gator_timer_online, NULL, 1);
register_scheduler_tracepoints();
- register_hotcpu_notifier(&gator_cpu_notifier);
+ register_hotcpu_notifier(&gator_hotcpu_notifier);
break;
}
@@ -611,7 +540,7 @@ static struct notifier_block gator_pm_notifier = {
static int gator_notifier_start(void)
{
int retval;
- retval = register_hotcpu_notifier(&gator_cpu_notifier);
+ retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
if (retval == 0)
retval = register_pm_notifier(&gator_pm_notifier);
return retval;
@@ -620,12 +549,30 @@ static int gator_notifier_start(void)
static void gator_notifier_stop(void)
{
unregister_pm_notifier(&gator_pm_notifier);
- unregister_hotcpu_notifier(&gator_cpu_notifier);
+ unregister_hotcpu_notifier(&gator_hotcpu_notifier);
}
/******************************************************************************
* Main
******************************************************************************/
+static void gator_summary(void)
+{
+ uint64_t timestamp, uptime = 0;
+ struct timespec uptime_ts;
+ void (*m2b)(struct timespec *ts);
+
+ timestamp = gator_get_time();
+
+ do_posix_clock_monotonic_gettime(&uptime_ts);
+ m2b = symbol_get(monotonic_to_bootbased);
+ if (m2b) {
+ m2b(&uptime_ts);
+ uptime = (long long)uptime_ts.tv_sec * 1000000000 + uptime_ts.tv_nsec;
+ }
+
+ marshal_summary(timestamp, uptime);
+}
+
int gator_events_install(struct gator_interface *interface)
{
list_add_tail(&interface->list, &gator_events);
@@ -635,7 +582,8 @@ int gator_events_install(struct gator_interface *interface)
int gator_events_get_key(void)
{
- static int key;
+ // key of zero is reserved as a timestamp
+ static int key = 1;
return key++;
}
@@ -644,21 +592,31 @@ static int gator_init(void)
{
int i;
- if (gator_annotate_init())
- return -1;
-
// events sources (gator_events.h, generated by gator_events.sh)
for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
if (gator_events_list[i])
gator_events_list[i]();
+ gator_trace_power_init();
+
return 0;
}
static int gator_start(void)
{
+ unsigned long cpu, i;
struct gator_interface *gi;
+ // Initialize the buffer with the frame type and core
+ for_each_present_cpu(cpu) {
+ for (i = 0; i < NUM_GATOR_BUFS; i++) {
+ gator_buffer_header(cpu, i);
+ }
+ }
+
+ // Capture the start time
+ gator_summary();
+
// start all events
list_for_each_entry(gi, &gator_events, list) {
if (gi->start && gi->start() != 0) {
@@ -683,6 +641,8 @@ static int gator_start(void)
goto annotate_failure;
if (gator_trace_sched_start())
goto sched_failure;
+ if (gator_trace_power_start())
+ goto power_failure;
if (gator_trace_gpu_start())
goto gpu_failure;
if (gator_event_sampling_start())
@@ -701,6 +661,8 @@ timer_failure:
event_sampling_failure:
gator_trace_gpu_stop();
gpu_failure:
+ gator_trace_power_stop();
+power_failure:
gator_trace_sched_stop();
sched_failure:
gator_annotate_stop();
@@ -727,6 +689,7 @@ static void gator_stop(void)
gator_annotate_stop();
gator_trace_sched_stop();
+ gator_trace_power_stop();
gator_trace_gpu_stop();
gator_event_sampling_stop();
@@ -735,11 +698,6 @@ static void gator_stop(void)
gator_timer_stop();
}
-static void gator_exit(void)
-{
- gator_annotate_exit();
-}
-
/******************************************************************************
* Filesystem
******************************************************************************/
@@ -751,33 +709,52 @@ static int gator_op_setup(void)
mutex_lock(&start_mutex);
- gator_buffer_size[TIMER_BUF] = userspace_buffer_size;
- gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1;
+ gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
+ gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
- // must be a power of 2
- if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) {
- err = -ENOEXEC;
- goto setup_error;
- }
+ gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
+ gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
+
+ gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
+ gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
+
+ gator_buffer_size[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE;
+ gator_buffer_mask[GPU_TRACE_BUF] = GPU_TRACE_BUFFER_SIZE - 1;
- gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT;
- gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1;
+ gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
+ gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
+
+ gator_buffer_size[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE;
+ gator_buffer_mask[COUNTER2_BUF] = COUNTER2_BUFFER_SIZE - 1;
+
+ gator_buffer_size[WFI_BUF] = WFI_BUFFER_SIZE;
+ gator_buffer_mask[WFI_BUF] = WFI_BUFFER_SIZE - 1;
// Initialize percpu per buffer variables
for (i = 0; i < NUM_GATOR_BUFS; i++) {
+ // Verify buffers are a power of 2
+ if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
+ err = -ENOEXEC;
+ goto setup_error;
+ }
+
for_each_present_cpu(cpu) {
+ per_cpu(gator_buffer_read, cpu)[i] = 0;
+ per_cpu(gator_buffer_write, cpu)[i] = 0;
+ per_cpu(gator_buffer_commit, cpu)[i] = 0;
+ per_cpu(buffer_space_available, cpu)[i] = true;
+
+ // Annotation is a special case that only uses a single buffer
+ if (cpu > 0 && i == ANNOTATE_BUF) {
+ per_cpu(gator_buffer, cpu)[i] = NULL;
+ continue;
+ }
+
per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
if (!per_cpu(gator_buffer, cpu)[i]) {
err = -ENOMEM;
goto setup_error;
}
-
- per_cpu(gator_buffer_read, cpu)[i] = 0;
- per_cpu(gator_buffer_write, cpu)[i] = 0;
- per_cpu(gator_buffer_commit, cpu)[i] = 0;
- per_cpu(buffer_space_available, cpu)[i] = true;
- per_cpu(emit_overflow, cpu) = 0;
- gator_buffer_header(cpu, i);
}
}
@@ -829,8 +806,6 @@ static void gator_shutdown(void)
mutex_lock(&start_mutex);
- gator_annotate_shutdown();
-
for_each_present_cpu(cpu) {
mutex_lock(&gator_buffer_mutex);
for (i = 0; i < NUM_GATOR_BUFS; i++) {
@@ -840,7 +815,6 @@ static void gator_shutdown(void)
per_cpu(gator_buffer_write, cpu)[i] = 0;
per_cpu(gator_buffer_commit, cpu)[i] = 0;
per_cpu(buffer_space_available, cpu)[i] = true;
- per_cpu(emit_overflow, cpu) = 0;
}
mutex_unlock(&gator_buffer_mutex);
}
@@ -932,7 +906,7 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
size_t count, loff_t *offset)
{
int retval = -EINVAL;
- int commit = 0, length1, length2, read;
+ int commit = 0, length, length1, length2, read, byte, type_length;
char *buffer1;
char *buffer2 = NULL;
int cpu, buftype;
@@ -944,7 +918,7 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
// sleep until the condition is true or a signal is received
// the condition is checked each time gator_buffer_wait is woken up
buftype = cpu = -1;
- wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || gator_annotate_ready() || !gator_started);
+ wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
if (signal_pending(current))
return -EINTR;
@@ -954,33 +928,36 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
mutex_lock(&gator_buffer_mutex);
- if (buftype != -1 && cpu != -1) {
- read = per_cpu(gator_buffer_read, cpu)[buftype];
- commit = per_cpu(gator_buffer_commit, cpu)[buftype];
+ if (buftype == -1 || cpu == -1) {
+ retval = 0;
+ goto out;
+ }
- /* May happen if the buffer is freed during pending reads. */
- if (!per_cpu(gator_buffer, cpu)[buftype]) {
- retval = -EFAULT;
- goto out;
- }
+ read = per_cpu(gator_buffer_read, cpu)[buftype];
+ commit = per_cpu(gator_buffer_commit, cpu)[buftype];
- /* determine the size of two halves */
- length1 = commit - read;
- buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
- buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
- if (length1 < 0) {
- length1 = gator_buffer_size[buftype] - read;
- length2 = commit;
- }
- } else if (gator_annotate_ready()) {
- length1 = gator_annotate_read(&buffer1);
- if (!length1)
- goto out;
- } else {
- retval = 0;
+ /* May happen if the buffer is freed during pending reads. */
+ if (!per_cpu(gator_buffer, cpu)[buftype]) {
+ retval = -EFAULT;
goto out;
}
+ /* determine the size of two halves */
+ length1 = commit - read;
+ buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
+ buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
+ if (length1 < 0) {
+ length1 = gator_buffer_size[buftype] - read;
+ length2 = commit;
+ }
+
+ // post-populate the length, which does not include the response type length nor the length itself, i.e. only the length of the payload
+ type_length = gator_response_type ? 1 : 0;
+ length = length1 + length2 - type_length - sizeof(int);
+ for (byte = 0; byte < sizeof(int); byte++) {
+ per_cpu(gator_buffer, cpu)[buftype][(read + type_length + byte) & gator_buffer_mask[buftype]] = (length >> byte * 8) & 0xFF;
+ }
+
/* start, middle or end */
if (length1 > 0) {
if (copy_to_user(&buf[0], buffer1, length1)) {
@@ -995,9 +972,7 @@ static ssize_t userspace_buffer_read(struct file *file, char __user *buf,
}
}
- if (buftype != -1 && cpu != -1)
- per_cpu(gator_buffer_read, cpu)[buftype] = commit;
-
+ per_cpu(gator_buffer_read, cpu)[buftype] = commit;
retval = length1 + length2;
/* kick just in case we've lost an SMP event */
@@ -1055,8 +1030,8 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root)
for_each_present_cpu(cpu) {
gator_cpu_cores++;
}
- userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT;
- gator_streaming = 1;
+ userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
+ gator_response_type = 1;
gatorfs_create_file(sb, root, "enable", &enable_fops);
gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
@@ -1064,7 +1039,7 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root)
gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size);
gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
- gatorfs_create_ulong(sb, root, "streaming", &gator_streaming);
+ gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
// Annotate interface
@@ -1075,6 +1050,9 @@ void gator_op_create_files(struct super_block *sb, struct dentry *root)
list_for_each_entry(gi, &gator_events, list)
if (gi->create_files)
gi->create_files(sb, dir);
+
+ // Power interface
+ gator_trace_power_create_files(sb, dir);
}
/******************************************************************************
@@ -1098,7 +1076,6 @@ static void __exit gator_module_exit(void)
{
tracepoint_synchronize_unregister();
gatorfs_unregister();
- gator_exit();
}
module_init(gator_module_init);
diff --git a/driver/gator_marshaling.c b/driver/gator_marshaling.c
new file mode 100755
index 0000000..630d142
--- /dev/null
+++ b/driver/gator_marshaling.c
@@ -0,0 +1,239 @@
+/**
+ * Copyright (C) ARM Limited 2012. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+static void marshal_summary(long long timestamp, long long uptime) {
+ int cpu = 0;
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_SUMMARY);
+ gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, timestamp);
+ gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, uptime);
+}
+
+static bool marshal_cookie_header(char* text) {
+ int cpu = smp_processor_id();
+ return buffer_check_space(cpu, BACKTRACE_BUF, strlen(text) + 2 * MAXSIZE_PACK32);
+}
+
+static void marshal_cookie(int cookie, char* text) {
+ int cpu = smp_processor_id();
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_COOKIE);
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, cookie);
+ gator_buffer_write_string(cpu, BACKTRACE_BUF, text);
+}
+
+static void marshal_pid_name(int pid, char* name) {
+ unsigned long flags, cpu;
+ local_irq_save(flags);
+ cpu = smp_processor_id();
+ if (buffer_check_space(cpu, BACKTRACE_BUF, TASK_COMM_LEN + 2 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_PID_NAME);
+ gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, gator_get_time());
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, pid);
+ gator_buffer_write_string(cpu, BACKTRACE_BUF, name);
+ }
+ local_irq_restore(flags);
+}
+
+static bool marshal_backtrace_header(int exec_cookie, int tgid, int pid, int inKernel) {
+ int cpu = smp_processor_id();
+ if (buffer_check_space(cpu, BACKTRACE_BUF, gator_backtrace_depth * 2 * MAXSIZE_PACK32)) {
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_START_BACKTRACE);
+ gator_buffer_write_packed_int64(cpu, BACKTRACE_BUF, gator_get_time());
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, exec_cookie);
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, tgid);
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, pid);
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, inKernel);
+ return true;
+ }
+
+ // Check and commit; commit is set to occur once buffer is 3/4 full
+ buffer_check(cpu, BACKTRACE_BUF);
+
+ return false;
+}
+
+static void marshal_backtrace(int address, int cookie) {
+ int cpu = smp_processor_id();
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, address);
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, cookie);
+}
+
+static void marshal_backtrace_footer(void) {
+ int cpu = smp_processor_id();
+ gator_buffer_write_packed_int(cpu, BACKTRACE_BUF, MESSAGE_END_BACKTRACE);
+
+ // Check and commit; commit is set to occur once buffer is 3/4 full
+ buffer_check(cpu, BACKTRACE_BUF);
+}
+
+static bool marshal_event_header(void) {
+ unsigned long flags, cpu = smp_processor_id();
+ bool retval = false;
+
+ local_irq_save(flags);
+ if (buffer_check_space(cpu, COUNTER_BUF, MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
+ gator_buffer_write_packed_int(cpu, COUNTER_BUF, 0); // key of zero indicates a timestamp
+ gator_buffer_write_packed_int64(cpu, COUNTER_BUF, gator_get_time());
+ retval = true;
+ }
+ local_irq_restore(flags);
+
+ // Check and commit; commit is set to occur once buffer is 3/4 full
+ buffer_check(cpu, COUNTER_BUF);
+
+ return retval;
+}
+
+static void marshal_event(int len, int* buffer) {
+ unsigned long i, flags, cpu = smp_processor_id();
+
+ if (len <= 0)
+ return;
+
+ // length must be even since all data is a (key, value) pair
+ if (len & 0x1) {
+ pr_err("gator: invalid counter data detected and discarded");
+ return;
+ }
+
+ // events must be written in key,value pairs
+ for (i = 0; i < len; i += 2) {
+ local_irq_save(flags);
+ if (!buffer_check_space(cpu, COUNTER_BUF, MAXSIZE_PACK32 * 2)) {
+ local_irq_restore(flags);
+ break;
+ }
+ gator_buffer_write_packed_int(cpu, COUNTER_BUF, buffer[i]);
+ gator_buffer_write_packed_int(cpu, COUNTER_BUF, buffer[i + 1]);
+ local_irq_restore(flags);
+ }
+
+ // Check and commit; commit is set to occur once buffer is 3/4 full
+ buffer_check(cpu, COUNTER_BUF);
+}
+
+static void marshal_event64(int len, long long* buffer64) {
+ unsigned long i, flags, cpu = smp_processor_id();
+
+ if (len <= 0)
+ return;
+
+ // length must be even since all data is a (key, value) pair
+ if (len & 0x1) {
+ pr_err("gator: invalid counter data detected and discarded");
+ return;
+ }
+
+ // events must be written in key,value pairs
+ for (i = 0; i < len; i += 2) {
+ local_irq_save(flags);
+ if (!buffer_check_space(cpu, COUNTER_BUF, MAXSIZE_PACK64 * 2)) {
+ local_irq_restore(flags);
+ break;
+ }
+ gator_buffer_write_packed_int64(cpu, COUNTER_BUF, buffer64[i]);
+ gator_buffer_write_packed_int64(cpu, COUNTER_BUF, buffer64[i + 1]);
+ local_irq_restore(flags);
+ }
+
+ // Check and commit; commit is set to occur once buffer is 3/4 full
+ buffer_check(cpu, COUNTER_BUF);
+}
+
+#if GATOR_CPU_FREQ_SUPPORT
+static void marshal_event_single(int core, int key, int value) {
+ unsigned long flags, cpu;
+
+ local_irq_save(flags);
+ cpu = smp_processor_id();
+ if (buffer_check_space(cpu, COUNTER2_BUF, MAXSIZE_PACK64 + MAXSIZE_PACK32 * 3)) {
+ gator_buffer_write_packed_int64(cpu, COUNTER2_BUF, gator_get_time());
+ gator_buffer_write_packed_int(cpu, COUNTER2_BUF, core);
+ gator_buffer_write_packed_int(cpu, COUNTER2_BUF, key);
+ gator_buffer_write_packed_int(cpu, COUNTER2_BUF, value);
+ }
+ local_irq_restore(flags);
+
+ // Check and commit; commit is set to occur once buffer is 3/4 full
+ buffer_check(cpu, COUNTER2_BUF);
+}
+#endif
+
+static void marshal_sched_gpu(int type, int unit, int core, int tgid, int pid) {
+ unsigned long cpu = smp_processor_id(), flags;
+
+ if (!per_cpu(gator_buffer, cpu)[GPU_TRACE_BUF])
+ return;
+
+ local_irq_save(flags);
+ if (buffer_check_space(cpu, GPU_TRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) {
+ gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, type);
+ gator_buffer_write_packed_int64(cpu, GPU_TRACE_BUF, gator_get_time());
+ gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, unit);
+ gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, core);
+ gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, tgid);
+ gator_buffer_write_packed_int(cpu, GPU_TRACE_BUF, pid);
+ }
+ local_irq_restore(flags);
+
+ // Check and commit; commit is set to occur once buffer is 3/4 full
+ buffer_check(cpu, GPU_TRACE_BUF);
+}
+
+static void marshal_sched_trace(int type, int pid, int tgid, int cookie, int state) {
+ unsigned long cpu = smp_processor_id(), flags;
+
+ if (!per_cpu(gator_buffer, cpu)[SCHED_TRACE_BUF])
+ return;
+
+ local_irq_save(flags);
+ if (buffer_check_space(cpu, SCHED_TRACE_BUF, MAXSIZE_PACK64 + 5 * MAXSIZE_PACK32)) {
+ gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, type);
+ gator_buffer_write_packed_int64(cpu, SCHED_TRACE_BUF, gator_get_time());
+ gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, pid);
+ gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, tgid);
+ gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, cookie);
+ gator_buffer_write_packed_int(cpu, SCHED_TRACE_BUF, state);
+ }
+ local_irq_restore(flags);
+
+ // Check and commit; commit is set to occur once buffer is 3/4 full
+ buffer_check(cpu, SCHED_TRACE_BUF);
+}
+
+#if GATOR_CPU_FREQ_SUPPORT
+static void marshal_wfi(int core, int state) {
+ unsigned long flags, cpu;
+
+ local_irq_save(flags);
+ cpu = smp_processor_id();
+ if (buffer_check_space(cpu, WFI_BUF, MAXSIZE_PACK64 + MAXSIZE_PACK32 * 2)) {
+ gator_buffer_write_packed_int64(cpu, WFI_BUF, gator_get_time());
+ gator_buffer_write_packed_int(cpu, WFI_BUF, core);
+ gator_buffer_write_packed_int(cpu, WFI_BUF, state);
+ }
+ local_irq_restore(flags);
+
+ // Check and commit; commit is set to occur once buffer is 3/4 full
+ buffer_check(cpu, WFI_BUF);
+}
+#endif
+
+static void marshal_frame(int cpu, int buftype, int frame) {
+ // add response type
+ if (gator_response_type > 0) {
+ gator_buffer_write_packed_int(cpu, buftype, gator_response_type);
+ }
+
+ // leave space for 4-byte unpacked length
+ per_cpu(gator_buffer_write, cpu)[buftype] = (per_cpu(gator_buffer_write, cpu)[buftype] + 4) & gator_buffer_mask[buftype];
+
+ // add frame type and core number
+ gator_buffer_write_packed_int(cpu, buftype, frame);
+ gator_buffer_write_packed_int(cpu, buftype, cpu);
+}
diff --git a/driver/gator_pack.c b/driver/gator_pack.c
index 985e960..925469a 100644
--- a/driver/gator_pack.c
+++ b/driver/gator_pack.c
@@ -162,101 +162,3 @@ static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long
per_cpu(gator_buffer_write, cpu)[buftype] = write9;
}
}
-
-static int gator_write_packed_int(char *buffer, unsigned int x)
-{
- if ((x & 0xffffff80) == 0) {
- buffer[0] = x & 0x7f;
- return 1;
- } else if ((x & 0xffffc000) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) & 0x7f;
- return 2;
- } else if ((x & 0xffe00000) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) & 0x7f;
- return 3;
- } else if ((x & 0xf0000000) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) | 0x80;
- buffer[3] = (x>>21) & 0x7f;
- return 4;
- } else {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) | 0x80;
- buffer[3] = (x>>21) | 0x80;
- buffer[4] = (x>>28) & 0x0f;
- return 5;
- }
-}
-
-static int gator_write_packed_int64(char *buffer, unsigned long long x)
-{
- if ((x & 0xffffffffffffff80LL) == 0) {
- buffer[0] = x & 0x7f;
- return 1;
- } else if ((x & 0xffffffffffffc000LL) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) & 0x7f;
- return 2;
- } else if ((x & 0xffffffffffe00000LL) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) & 0x7f;
- return 3;
- } else if ((x & 0xfffffffff0000000LL) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) | 0x80;
- buffer[3] = (x>>21) & 0x7f;
- return 4;
- } else if ((x & 0xfffffff800000000LL) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) | 0x80;
- buffer[3] = (x>>21) | 0x80;
- buffer[4] = (x>>28) & 0x7f;
- return 5;
- } else if ((x & 0xfffffc0000000000LL) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) | 0x80;
- buffer[3] = (x>>21) | 0x80;
- buffer[4] = (x>>28) | 0x80;
- buffer[5] = (x>>35) & 0x7f;
- return 6;
- } else if ((x & 0xfffe000000000000LL) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) | 0x80;
- buffer[3] = (x>>21) | 0x80;
- buffer[4] = (x>>28) | 0x80;
- buffer[5] = (x>>35) | 0x80;
- buffer[6] = (x>>42) & 0x7f;
- return 7;
- } else if ((x & 0xff00000000000000LL) == 0) {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) | 0x80;
- buffer[3] = (x>>21) | 0x80;
- buffer[4] = (x>>28) | 0x80;
- buffer[5] = (x>>35) | 0x80;
- buffer[6] = (x>>42) | 0x80;
- buffer[7] = (x>>49) & 0x7f;
- return 8;
- } else {
- buffer[0] = x | 0x80;
- buffer[1] = (x>>7) | 0x80;
- buffer[2] = (x>>14) | 0x80;
- buffer[3] = (x>>21) | 0x80;
- buffer[4] = (x>>28) | 0x80;
- buffer[5] = (x>>35) | 0x80;
- buffer[6] = (x>>42) | 0x80;
- buffer[7] = (x>>49) | 0x80;
- buffer[8] = (x>>56) & 0xff;
- return 9;
- }
-}
diff --git a/driver/gator_trace_gpu.c b/driver/gator_trace_gpu.c
index bc63995..921932c 100644
--- a/driver/gator_trace_gpu.c
+++ b/driver/gator_trace_gpu.c
@@ -25,68 +25,12 @@
static int mali_trace_registered;
static int gpu_trace_registered;
-#define GPU_OVERFLOW -1
#define GPU_START 1
#define GPU_STOP 2
#define GPU_UNIT_VP 1
#define GPU_UNIT_FP 2
-
-#define TRACESIZE (8*1024)
-
-static DEFINE_PER_CPU(uint64_t *[2], theGpuTraceBuf);
-static DEFINE_PER_CPU(int, theGpuTraceSel);
-static DEFINE_PER_CPU(int, theGpuTracePos);
-static DEFINE_PER_CPU(int, theGpuTraceErr);
-
-int gator_trace_gpu_read(long long **buffer);
-
-static void probe_gpu_write(int type, int unit, int core, struct task_struct* task)
-{
- int tracePos;
- unsigned long flags;
- uint64_t *traceBuf, time;
- int pid, tgid;
- int cpu = smp_processor_id();
-
- if (!per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)])
- return;
-
- if (task) {
- tgid = (int)task->tgid;
- pid = (int)task->pid;
- } else {
- tgid = pid = 0;
- }
-
- // disable interrupts to synchronize with gator_trace_gpu_read(); spinlocks not needed since percpu buffers are used
- local_irq_save(flags);
-
- time = gator_get_time();
- tracePos = per_cpu(theGpuTracePos, cpu);
- traceBuf = per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)];
-
- if (tracePos < (TRACESIZE - 100)) {
- // capture
- traceBuf[tracePos++] = type;
- traceBuf[tracePos++] = time;
- traceBuf[tracePos++] = unit;
- traceBuf[tracePos++] = core;
- traceBuf[tracePos++] = tgid;
- traceBuf[tracePos++] = pid;
- } else if (!per_cpu(theGpuTraceErr, cpu)) {
- per_cpu(theGpuTraceErr, cpu) = 1;
- traceBuf[tracePos++] = GPU_OVERFLOW;
- traceBuf[tracePos++] = time;
- traceBuf[tracePos++] = 0;
- traceBuf[tracePos++] = 0;
- traceBuf[tracePos++] = 0;
- traceBuf[tracePos++] = 0;
- pr_debug("gator: gpu trace overflow\n");
- }
- per_cpu(theGpuTracePos, cpu) = tracePos;
- local_irq_restore(flags);
-}
+#define GPU_UNIT_CL 3
#ifdef MALI_SUPPORT
@@ -105,19 +49,23 @@ enum components {
GATOR_DEFINE_PROBE(mali_timeline_event, TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, unsigned int d2, unsigned int d3, unsigned int d4))
{
unsigned int component, state;
+ int tgid = 0, pid = 0;
// do as much work as possible before disabling interrupts
- component = (event_id >> 16) & 0xF;
- state = (event_id >> 24) & 0xF;
+ component = (event_id >> 16) & 0xFF; // component is an 8-bit field
+ state = (event_id >> 24) & 0xF; // state is a 4-bit field
if ((component == COMPONENT_VP0) || (component >= COMPONENT_FP0 && component <= COMPONENT_FP7)) {
if (state == ACTIVITY_START || state == ACTIVITY_STOP) {
unsigned int type = (state == ACTIVITY_START) ? GPU_START : GPU_STOP;
unsigned int unit = (component < COMPONENT_FP0) ? GPU_UNIT_VP : GPU_UNIT_FP;
unsigned int core = (component < COMPONENT_FP0) ? component - COMPONENT_VP0 : component - COMPONENT_FP0;
- struct task_struct* task = (state == ACTIVITY_START) ? (struct task_struct*)d2 : NULL;
+ if (state == ACTIVITY_START) {
+ tgid = d0;
+ pid = d1;
+ }
- probe_gpu_write(type, unit, core, task);
+ marshal_sched_gpu(type, unit, core, tgid, pid);
}
}
}
@@ -125,20 +73,18 @@ GATOR_DEFINE_PROBE(mali_timeline_event, TP_PROTO(unsigned int event_id, unsigned
GATOR_DEFINE_PROBE(gpu_activity_start, TP_PROTO(int gpu_unit, int gpu_core, struct task_struct *p))
{
- probe_gpu_write(GPU_START, gpu_unit, gpu_core, p);
+ marshal_sched_gpu(GPU_START, gpu_unit, gpu_core, (int)p->tgid, (int)p->pid);
}
GATOR_DEFINE_PROBE(gpu_activity_stop, TP_PROTO(int gpu_unit, int gpu_core))
{
- probe_gpu_write(GPU_STOP, gpu_unit, gpu_core, NULL);
+ marshal_sched_gpu(GPU_STOP, gpu_unit, gpu_core, 0, 0);
}
int gator_trace_gpu_start(void)
{
- int cpu;
-
/*
- * Returns 0 for installation failed
+ * Returns nonzero for installation failed
* Absence of gpu trace points is not an error
*/
@@ -161,90 +107,20 @@ int gator_trace_gpu_start(void)
gpu_trace_registered = 1;
}
- if (!gpu_trace_registered && !mali_trace_registered) {
- return 0;
- }
-
- for_each_present_cpu(cpu) {
- per_cpu(theGpuTraceSel, cpu) = 0;
- per_cpu(theGpuTracePos, cpu) = 0;
- per_cpu(theGpuTraceErr, cpu) = 0;
- per_cpu(theGpuTraceBuf, cpu)[0] = kmalloc(TRACESIZE * sizeof(uint64_t), GFP_KERNEL);
- per_cpu(theGpuTraceBuf, cpu)[1] = kmalloc(TRACESIZE * sizeof(uint64_t), GFP_KERNEL);
- if (!per_cpu(theGpuTraceBuf, cpu)[0] || !per_cpu(theGpuTraceBuf, cpu)[1]) {
-#ifdef MALI_SUPPORT
- if (mali_trace_registered) {
- GATOR_UNREGISTER_TRACE(mali_timeline_event);
- }
-#endif
- if (gpu_trace_registered) {
- GATOR_UNREGISTER_TRACE(gpu_activity_stop);
- GATOR_UNREGISTER_TRACE(gpu_activity_start);
- }
-
- gpu_trace_registered = mali_trace_registered = 0;
-
- return -1;
- }
- }
-
return 0;
}
-int gator_trace_gpu_offline(long long **buffer)
-{
- return gator_trace_gpu_read(buffer);
-}
-
void gator_trace_gpu_stop(void)
{
- int cpu;
-
- if (gpu_trace_registered || mali_trace_registered) {
- for_each_present_cpu(cpu) {
- kfree(per_cpu(theGpuTraceBuf, cpu)[0]);
- kfree(per_cpu(theGpuTraceBuf, cpu)[1]);
- per_cpu(theGpuTraceBuf, cpu)[0] = NULL;
- per_cpu(theGpuTraceBuf, cpu)[1] = NULL;
- }
-
#ifdef MALI_SUPPORT
- if (mali_trace_registered) {
- GATOR_UNREGISTER_TRACE(mali_timeline_event);
- }
+ if (mali_trace_registered) {
+ GATOR_UNREGISTER_TRACE(mali_timeline_event);
+ }
#endif
- if (gpu_trace_registered) {
- GATOR_UNREGISTER_TRACE(gpu_activity_stop);
- GATOR_UNREGISTER_TRACE(gpu_activity_start);
- }
-
- gpu_trace_registered = mali_trace_registered = 0;
- }
-}
-
-int gator_trace_gpu_read(long long **buffer)
-{
- int cpu = smp_processor_id();
- unsigned long flags;
- uint64_t *traceBuf;
- int tracePos;
-
- if (!per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)])
- return 0;
-
- local_irq_save(flags);
-
- traceBuf = per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)];
- tracePos = per_cpu(theGpuTracePos, cpu);
-
- per_cpu(theGpuTraceSel, cpu) = !per_cpu(theGpuTraceSel, cpu);
- per_cpu(theGpuTracePos, cpu) = 0;
- per_cpu(theGpuTraceErr, cpu) = 0;
-
- local_irq_restore(flags);
-
- if (buffer)
- *buffer = traceBuf;
+ if (gpu_trace_registered) {
+ GATOR_UNREGISTER_TRACE(gpu_activity_stop);
+ GATOR_UNREGISTER_TRACE(gpu_activity_start);
+ }
- return tracePos;
+ gpu_trace_registered = mali_trace_registered = 0;
}
diff --git a/driver/gator_trace_power.c b/driver/gator_trace_power.c
new file mode 100755
index 0000000..ca89b19
--- /dev/null
+++ b/driver/gator_trace_power.c
@@ -0,0 +1,160 @@
+/**
+ * Copyright (C) ARM Limited 2011-2012. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/cpufreq.h>
+#include <trace/events/power.h>
+
+// cpu_frequency and cpu_idle trace points were introduced in Linux kernel v2.6.38
+// the now deprecated power_frequency trace point was available prior to 2.6.38, but only for x86
+#if GATOR_CPU_FREQ_SUPPORT
+enum {
+ POWER_CPU_FREQ,
+ POWER_CPU_IDLE,
+ POWER_TOTAL
+};
+
+static DEFINE_PER_CPU(ulong, idle_prev_state);
+static ulong power_cpu_enabled[POWER_TOTAL];
+static ulong power_cpu_key[POWER_TOTAL];
+
+static int gator_trace_power_create_files(struct super_block *sb, struct dentry *root)
+{
+ struct dentry *dir;
+
+ // cpu_frequency
+ dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_freq");
+ if (!dir) {
+ return -1;
+ }
+ gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_FREQ]);
+ gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_FREQ]);
+
+ // cpu_idle
+ dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_idle");
+ if (!dir) {
+ return -1;
+ }
+ gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_IDLE]);
+ gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_IDLE]);
+
+ return 0;
+}
+
+// 'cpu' may not equal smp_processor_id(), i.e. may not be running on the core that is having the freq/idle state change
+GATOR_DEFINE_PROBE(cpu_frequency, TP_PROTO(unsigned int frequency, unsigned int cpu))
+{
+ marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], frequency * 1000);
+}
+
+#define WFI_ACTIVE_THRESHOLD 2 // may vary on platform/OS
+#define WFI_EXIT 0
+#define WFI_ENTER 1
+GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu))
+{
+ // the streamline engine treats all counter values as unsigned
+ if (state & 0x80000000) {
+ state = 0;
+ }
+
+ if (state == per_cpu(idle_prev_state, cpu)) {
+ return;
+ }
+
+ if (state < WFI_ACTIVE_THRESHOLD && per_cpu(idle_prev_state, cpu) >= WFI_ACTIVE_THRESHOLD) {
+ // transition from wfi to non-wfi
+ marshal_wfi(cpu, WFI_EXIT);
+ } else if (state >= WFI_ACTIVE_THRESHOLD && per_cpu(idle_prev_state, cpu) < WFI_ACTIVE_THRESHOLD) {
+ // transition from non-wfi to wfi
+ marshal_wfi(cpu, WFI_ENTER);
+ }
+
+ per_cpu(idle_prev_state, cpu) = state;
+
+ if (power_cpu_enabled[POWER_CPU_IDLE]) {
+ marshal_event_single(cpu, power_cpu_key[POWER_CPU_IDLE], state);
+ }
+}
+
+static void gator_trace_power_online(void)
+{
+ int cpu = smp_processor_id();
+ if (power_cpu_enabled[POWER_CPU_FREQ]) {
+ marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], cpufreq_quick_get(cpu) * 1000);
+ }
+}
+
+static void gator_trace_power_offline(void)
+{
+ // Set frequency to zero on an offline
+ int cpu = smp_processor_id();
+ if (power_cpu_enabled[POWER_CPU_FREQ]) {
+ marshal_event_single(cpu, power_cpu_key[POWER_CPU_FREQ], 0);
+ }
+}
+
+static int gator_trace_power_start(void)
+{
+ int cpu;
+
+ // register tracepoints
+ if (power_cpu_enabled[POWER_CPU_FREQ])
+ if (GATOR_REGISTER_TRACE(cpu_frequency))
+ goto fail_cpu_frequency_exit;
+
+ // Always register for cpu:idle for detecting WFI, independent of power_cpu_enabled[POWER_CPU_IDLE]
+ if (GATOR_REGISTER_TRACE(cpu_idle))
+ goto fail_cpu_idle_exit;
+ pr_debug("gator: registered power event tracepoints\n");
+
+ for_each_present_cpu(cpu) {
+ per_cpu(idle_prev_state, cpu) = 0;
+ }
+
+ return 0;
+
+ // unregister tracepoints on error
+fail_cpu_idle_exit:
+ if (power_cpu_enabled[POWER_CPU_FREQ])
+ GATOR_UNREGISTER_TRACE(cpu_frequency);
+fail_cpu_frequency_exit:
+ pr_err("gator: power event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");
+
+ return -1;
+}
+
+static void gator_trace_power_stop(void)
+{
+ int i;
+
+ if (power_cpu_enabled[POWER_CPU_FREQ])
+ GATOR_UNREGISTER_TRACE(cpu_frequency);
+ GATOR_UNREGISTER_TRACE(cpu_idle);
+ pr_debug("gator: unregistered power event tracepoints\n");
+
+ for (i = 0; i < POWER_TOTAL; i++) {
+ power_cpu_enabled[i] = 0;
+ }
+}
+
+void gator_trace_power_init(void)
+{
+ int i;
+ for (i = 0; i < POWER_TOTAL; i++) {
+ power_cpu_enabled[i] = 0;
+ power_cpu_key[i] = gator_events_get_key();
+ }
+}
+#else
+static int gator_trace_power_create_files(struct super_block *sb, struct dentry *root) {return 0;}
+static void gator_trace_power_online(void) {}
+static void gator_trace_power_offline(void) {}
+static int gator_trace_power_start(void) {return 0;}
+static void gator_trace_power_stop(void) {}
+void gator_trace_power_init(void) {}
+#endif
diff --git a/driver/gator_trace_sched.c b/driver/gator_trace_sched.c
index dafacb7..08b0270 100644
--- a/driver/gator_trace_sched.c
+++ b/driver/gator_trace_sched.c
@@ -10,34 +10,26 @@
#include <trace/events/sched.h>
#include "gator.h"
-#define SCHED_OVERFLOW -1
#define SCHED_SWITCH 1
-#define SCHED_PROCESS_FREE 2
+#define SCHED_PROCESS_EXIT 2
-#define SCHEDSIZE (8*1024)
#define TASK_MAP_ENTRIES 1024 /* must be power of 2 */
#define TASK_MAX_COLLISIONS 2
-static DEFINE_PER_CPU(uint64_t *[2], theSchedBuf);
-static DEFINE_PER_CPU(int, theSchedSel);
-static DEFINE_PER_CPU(int, theSchedPos);
-static DEFINE_PER_CPU(int, theSchedErr);
static DEFINE_PER_CPU(uint64_t *, taskname_keys);
enum {
- STATE_CONTENTION = 0,
+ STATE_WAIT_ON_OTHER = 0,
+ STATE_CONTENTION,
STATE_WAIT_ON_IO,
- STATE_WAIT_ON_OTHER
+ STATE_WAIT_ON_MUTEX,
};
-int gator_trace_sched_read(long long **buffer);
-
void emit_pid_name(struct task_struct* task)
{
bool found = false;
- unsigned long flags;
char taskcomm[TASK_COMM_LEN + 3];
- int x, cpu = smp_processor_id();
+ unsigned long x, cpu = smp_processor_id();
uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]);
uint64_t value;
@@ -52,7 +44,7 @@ void emit_pid_name(struct task_struct* task)
}
}
- if (!found && buffer_check_space(cpu, TIMER_BUF, TASK_COMM_LEN + 2 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) {
+ if (!found) {
// shift values, new value always in front
uint64_t oldv, newv = value;
for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
@@ -62,100 +54,70 @@ void emit_pid_name(struct task_struct* task)
}
// emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions
- if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1)
+ if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) {
// append ellipses if task->comm has length of TASK_COMM_LEN - 1
strcat(taskcomm, "...");
+ }
- // disable interrupts to synchronize with hrtimer populating timer buf
- local_irq_save(flags);
- gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_PID_NAME);
- gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time());
- gator_buffer_write_packed_int(cpu, TIMER_BUF, task->pid);
- gator_buffer_write_string(cpu, TIMER_BUF, taskcomm);
- local_irq_restore(flags);
+ marshal_pid_name(task->pid, taskcomm);
+ }
+}
+
+
+static void collect_counters(void)
+{
+ int *buffer, len;
+ long long *buffer64;
+ struct gator_interface *gi;
+
+ if (marshal_event_header()) {
+ list_for_each_entry(gi, &gator_events, list) {
+ if (gi->read) {
+ len = gi->read(&buffer);
+ marshal_event(len, buffer);
+ } else if (gi->read64) {
+ len = gi->read64(&buffer64);
+ marshal_event64(len, buffer64);
+ }
+ }
}
}
static void probe_sched_write(int type, struct task_struct* task, struct task_struct* old_task)
{
- int schedPos, cookie = 0, state = 0;
- unsigned long flags;
- uint64_t *schedBuf, time;
+ int cookie = 0, state = 0;
int cpu = smp_processor_id();
int pid = task->pid;
int tgid = task->tgid;
- if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)])
- return;
-
if (type == SCHED_SWITCH) {
// do as much work as possible before disabling interrupts
- cookie = get_exec_cookie(cpu, TIMER_BUF, task);
+ cookie = get_exec_cookie(cpu, BACKTRACE_BUF, task);
emit_pid_name(task);
- if (old_task->state == 0)
+ if (old_task->state == TASK_RUNNING) {
state = STATE_CONTENTION;
- else if (old_task->in_iowait)
+ } else if (old_task->in_iowait) {
state = STATE_WAIT_ON_IO;
- else
+#ifdef CONFIG_DEBUG_MUTEXES
+ } else if (old_task->blocked_on) {
+ state = STATE_WAIT_ON_MUTEX;
+#endif
+ } else {
state = STATE_WAIT_ON_OTHER;
- }
-
- // disable interrupts to synchronize with gator_trace_sched_read(); spinlocks not needed since percpu buffers are used
- local_irq_save(flags);
-
- time = gator_get_time();
- schedPos = per_cpu(theSchedPos, cpu);
- schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)];
+ }
- if (schedPos < (SCHEDSIZE - 100)) {
- // capture
- schedBuf[schedPos++] = type;
- schedBuf[schedPos++] = time;
- schedBuf[schedPos++] = pid;
- schedBuf[schedPos++] = tgid;
- schedBuf[schedPos++] = cookie;
- schedBuf[schedPos++] = state;
- } else if (!per_cpu(theSchedErr, cpu)) {
- per_cpu(theSchedErr, cpu) = 1;
- schedBuf[schedPos++] = SCHED_OVERFLOW;
- schedBuf[schedPos++] = time;
- schedBuf[schedPos++] = 0;
- schedBuf[schedPos++] = 0;
- schedBuf[schedPos++] = 0;
- schedBuf[schedPos++] = 0;
- pr_debug("gator: tracepoint overflow\n");
+ collect_counters();
}
- per_cpu(theSchedPos, cpu) = schedPos;
- local_irq_restore(flags);
+
+ // marshal_sched_trace() disables interrupts as the free may trigger while switch is writing to the buffer; disabling preemption is not sufficient
+ // is disable interrupts necessary now that exit is used instead of free?
+ marshal_sched_trace(type, pid, tgid, cookie, state);
}
// special case used during a suspend of the system
static void trace_sched_insert_idle(void)
{
- unsigned long flags;
- uint64_t *schedBuf;
- int schedPos, cpu = smp_processor_id();
-
- if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)])
- return;
-
- local_irq_save(flags);
-
- schedPos = per_cpu(theSchedPos, cpu);
- schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)];
-
- if (schedPos < (SCHEDSIZE - (6 * 8))) {
- // capture
- schedBuf[schedPos++] = SCHED_SWITCH;
- schedBuf[schedPos++] = gator_get_time();
- schedBuf[schedPos++] = 0; // idle pid is zero
- schedBuf[schedPos++] = 0; // idle tid is zero
- schedBuf[schedPos++] = 0; // idle cookie is zero
- schedBuf[schedPos++] = STATE_WAIT_ON_OTHER;
- }
-
- per_cpu(theSchedPos, cpu) = schedPos;
- local_irq_restore(flags);
+ marshal_sched_trace(SCHED_SWITCH, 0, 0, 0, 0);
}
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
@@ -167,23 +129,23 @@ GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_
probe_sched_write(SCHED_SWITCH, next, prev);
}
-GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p))
+GATOR_DEFINE_PROBE(sched_process_exit, TP_PROTO(struct task_struct *p))
{
- probe_sched_write(SCHED_PROCESS_FREE, p, 0);
+ probe_sched_write(SCHED_PROCESS_EXIT, p, 0);
}
static int register_scheduler_tracepoints(void) {
// register tracepoints
if (GATOR_REGISTER_TRACE(sched_switch))
goto fail_sched_switch;
- if (GATOR_REGISTER_TRACE(sched_process_free))
- goto fail_sched_process_free;
+ if (GATOR_REGISTER_TRACE(sched_process_exit))
+ goto fail_sched_process_exit;
pr_debug("gator: registered tracepoints\n");
return 0;
// unregister tracepoints on error
-fail_sched_process_free:
+fail_sched_process_exit:
GATOR_UNREGISTER_TRACE(sched_switch);
fail_sched_switch:
pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");
@@ -196,14 +158,6 @@ int gator_trace_sched_start(void)
int cpu, size;
for_each_present_cpu(cpu) {
- per_cpu(theSchedSel, cpu) = 0;
- per_cpu(theSchedPos, cpu) = 0;
- per_cpu(theSchedErr, cpu) = 0;
- per_cpu(theSchedBuf, cpu)[0] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL);
- per_cpu(theSchedBuf, cpu)[1] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL);
- if (!per_cpu(theSchedBuf, cpu))
- return -1;
-
size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t);
per_cpu(taskname_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL);
if (!per_cpu(taskname_keys, cpu))
@@ -214,16 +168,15 @@ int gator_trace_sched_start(void)
return register_scheduler_tracepoints();
}
-int gator_trace_sched_offline(long long **buffer)
+void gator_trace_sched_offline(void)
{
trace_sched_insert_idle();
- return gator_trace_sched_read(buffer);
}
static void unregister_scheduler_tracepoints(void)
{
GATOR_UNREGISTER_TRACE(sched_switch);
- GATOR_UNREGISTER_TRACE(sched_process_free);
+ GATOR_UNREGISTER_TRACE(sched_process_exit);
pr_debug("gator: unregistered tracepoints\n");
}
@@ -233,37 +186,6 @@ void gator_trace_sched_stop(void)
unregister_scheduler_tracepoints();
for_each_present_cpu(cpu) {
- kfree(per_cpu(theSchedBuf, cpu)[0]);
- kfree(per_cpu(theSchedBuf, cpu)[1]);
- per_cpu(theSchedBuf, cpu)[0] = NULL;
- per_cpu(theSchedBuf, cpu)[1] = NULL;
kfree(per_cpu(taskname_keys, cpu));
}
}
-
-int gator_trace_sched_read(long long **buffer)
-{
- int cpu = smp_processor_id();
- unsigned long flags;
- uint64_t *schedBuf;
- int schedPos;
-
- if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)])
- return 0;
-
- local_irq_save(flags);
-
- schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)];
- schedPos = per_cpu(theSchedPos, cpu);
-
- per_cpu(theSchedSel, cpu) = !per_cpu(theSchedSel, cpu);
- per_cpu(theSchedPos, cpu) = 0;
- per_cpu(theSchedErr, cpu) = 0;
-
- local_irq_restore(flags);
-
- if (buffer)
- *buffer = schedBuf;
-
- return schedPos;
-}