gator: ARM DS-5.5 Streamline gator driverDS-5.5

Signed-off-by: Pawel Moll <pawel.moll@arm.com>

5 files changed, 236 insertions, 49 deletions

diff --git a/gator_annotate.c b/gator_annotate.c
index 56656f0..f261f73 100644
--- a/gator_annotate.c
+++ b/gator_annotate.c
@@ -23,17 +23,6 @@ static char *annotateBuf1;
 static int annotatePos;
 static int annotateSel;
 
-static int gatorfs_copy_from_user(char* localbuf, char const __user *buf, size_t count)
-{
-	if (count == 0)
-		return 0;
-
-	if (copy_from_user(localbuf, buf, count))
-		return -EFAULT;
-
-	return 0;
-}
-
 static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
 {
 	char tempBuffer[32];
@@ -51,7 +40,7 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t
 		return 0;
 
 	// copy from user space
-	retval = gatorfs_copy_from_user(tempBuffer, buf, size);
+	retval = copy_from_user(tempBuffer, buf, size);
 	if (retval == 0) {
 		// synchronize shared variables annotateBuf and annotatePos
 		spin_lock_irqsave(&annotate_lock, flags);
@@ -68,6 +57,8 @@ static ssize_t annotate_write(struct file *file, char const __user *buf, size_t
 
 		// return the number of bytes written
 		retval = size;
+	} else {
+		retval = -EINVAL;
 	}
 
 	return retval;
@@ -101,13 +92,14 @@ int gator_annotate_start(void)
 
 void gator_annotate_stop(void)
 {
-	spin_lock(&annotate_lock);
+	unsigned long flags;
+	spin_lock_irqsave(&annotate_lock, flags);
 
 	kfree(annotateBuf0);
 	kfree(annotateBuf1);
 	annotateBuf = annotateBuf0 = annotateBuf1 = NULL;
 
-	spin_unlock(&annotate_lock);
+	spin_unlock_irqrestore(&annotate_lock, flags);
 }
 
 int gator_annotate_read(int **buffer)
diff --git a/gator_cookies.c b/gator_cookies.c
index 4e39667..70fece0 100644
--- a/gator_cookies.c
+++ b/gator_cookies.c
@@ -8,15 +8,26 @@
  */
 
 #define COOKIEMAP_ENTRIES	1024		/* must be power of 2 */
+#define TRANSLATE_SIZE		256
 #define MAX_COLLISIONS		2
 
+static uint32_t *gator_crc32_table;
+static uint32_t translate_buffer_mask;
+
+static DEFINE_PER_CPU(char *, translate_text);
 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
 static DEFINE_PER_CPU(uint32_t *, cookie_values);
+static DEFINE_PER_CPU(int, translate_buffer_read);
+static DEFINE_PER_CPU(int, translate_buffer_write);
+static DEFINE_PER_CPU(unsigned int *, translate_buffer);
 
-static uint32_t *gator_crc32_table;
+static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod);
+static void wq_cookie_handler(struct work_struct *unused);
+DECLARE_WORK(cookie_work, wq_cookie_handler);
 
-static uint32_t cookiemap_code(uint32_t value) {
+static uint32_t cookiemap_code(uint64_t value64) {
+	uint32_t value = (uint32_t)((value64 >> 32) + value64);
 	uint32_t cookiecode = (value >> 24) & 0xff;
 	cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
 	cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
@@ -45,12 +56,14 @@ static uint32_t gator_chksum_crc32(char *data)
  *  Post: [v][0][1][3]..[n-1]
  */
 static uint32_t cookiemap_exists(uint64_t key) {
+	unsigned long x, flags, retval = 0;
 	int cpu = raw_smp_processor_id();
 	uint32_t cookiecode = cookiemap_code(key);
 	uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
 	uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
-	int x;
 
+	// Can be called from interrupt handler or from work queue
+	local_irq_save(flags);
 	for (x = 0; x < MAX_COLLISIONS; x++) {
 		if (keys[x] == key) {
 			uint32_t value = values[x];
@@ -60,11 +73,13 @@ static uint32_t cookiemap_exists(uint64_t key) {
 			}
 			keys[0] = key;
 			values[0] = value;
-			return value;
+			retval = value;
+			break;
 		}
 	}
+	local_irq_restore(flags);
 
-	return 0;
+	return retval;
 }
 
 /*
@@ -87,30 +102,146 @@ static void cookiemap_add(uint64_t key, uint32_t value) {
 	values[0] = value;
 }
 
-static inline uint32_t get_cookie(int cpu, int tgid, struct vm_area_struct *vma)
+static void translate_buffer_write_int(int cpu, unsigned int x)
+{
+	per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_write, cpu)++] = x;
+	per_cpu(translate_buffer_write, cpu) &= translate_buffer_mask;
+}
+
+static unsigned int translate_buffer_read_int(int cpu)
+{
+	unsigned int value = per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_read, cpu)++];
+	per_cpu(translate_buffer_read, cpu) &= translate_buffer_mask;
+	return value;
+}
+
+static void wq_cookie_handler(struct work_struct *unused)
+{
+	struct task_struct *task;
+	struct vm_area_struct *vma;
+	int cpu = smp_processor_id();
+	unsigned int cookie, commit;
+
+	commit = per_cpu(translate_buffer_write, cpu);
+	while (per_cpu(translate_buffer_read, cpu) != commit) {
+		task = (struct task_struct *)translate_buffer_read_int(cpu);
+		vma = (struct vm_area_struct *)translate_buffer_read_int(cpu);
+		cookie = get_cookie(cpu, task, vma, NULL);
+	}
+}
+
+// Retrieve full name from proc/pid/cmdline for java processes on Android
+static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma)
 {
+	void *maddr;
+	unsigned int len;
+	unsigned long addr;
+	struct mm_struct *mm;
+	struct page *page = NULL;
+	struct vm_area_struct *page_vma;
+	int bytes, offset, retval = 0, ptr;
+	char * buf = per_cpu(translate_text, cpu);
+
+	// Push work into a work queue if in atomic context as the kernel functions below might sleep
+	if (in_irq()) {
+		// Check if already in buffer
+		ptr = per_cpu(translate_buffer_read, cpu);
+		while (ptr != per_cpu(translate_buffer_write, cpu)) {
+			if (per_cpu(translate_buffer, cpu)[ptr] == (int)task)
+				goto out;
+			ptr = (ptr + 2) & translate_buffer_mask;
+		}
+
+		translate_buffer_write_int(cpu, (unsigned int)task);
+		translate_buffer_write_int(cpu, (unsigned int)vma);
+		schedule_work(&cookie_work);
+		goto out;
+	}
+
+	mm = get_task_mm(task);
+	if (!mm)
+		goto out;
+	if (!mm->arg_end)
+		goto outmm;
+	addr = mm->arg_start;
+	len = mm->arg_end - mm->arg_start;
+
+	if (len > TRANSLATE_SIZE)
+		len = TRANSLATE_SIZE;
+
+	down_read(&mm->mmap_sem);
+	while (len) {
+		if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
+			goto outsem;
+
+		maddr = kmap(page);
+		offset = addr & (PAGE_SIZE-1);
+		bytes = len;
+		if (bytes > PAGE_SIZE - offset)
+			bytes = PAGE_SIZE - offset;
+
+		copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
+
+		kunmap(page);	// release page allocated by get_user_pages()
+		page_cache_release(page);
+
+		len -= bytes;
+		buf += bytes;
+		addr += bytes;
+
+		*text = per_cpu(translate_text, cpu);
+		retval = 1;
+	}
+
+	// On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period
+	if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
+		retval = 0;
+
+outsem:
+	up_read(&mm->mmap_sem);
+outmm:
+	mmput(mm);
+out:
+	return retval;
+}
+
+static inline uint32_t get_cookie(int cpu, struct task_struct *task, struct vm_area_struct *vma, struct module *mod)
+{
+	unsigned long flags, cookie;
 	struct path *path;
 	uint64_t key;
-	int cookie;
 	char *text;
 
-	if (!vma || !vma->vm_file) {
-		return INVALID_COOKIE;
-	}
-	path = &vma->vm_file->f_path;
-	if (!path || !path->dentry) {
-		return INVALID_COOKIE;
+	if (mod) {
+		text = mod->name;
+	} else {
+		if (!vma || !vma->vm_file) {
+			return INVALID_COOKIE;
+		}
+		path = &vma->vm_file->f_path;
+		if (!path || !path->dentry) {
+			return INVALID_COOKIE;
+		}
+
+		text = (char*)path->dentry->d_name.name;
 	}
 
-	text = (char*)path->dentry->d_name.name;
 	key = gator_chksum_crc32(text);
-	key = (key << 32) | (uint32_t)text;
+	key = (key << 32) | (uint32_t)task->tgid;
 
 	cookie = cookiemap_exists(key);
 	if (cookie) {
-		goto output;
+		return cookie;
 	}
 
+	if (strcmp(text, "app_process") == 0 && !mod) {
+		if (!translate_app_process(&text, cpu, task, vma))
+			return INVALID_COOKIE;
+	}
+
+	// Can be called from interrupt handler or from work queue
+	local_irq_save(flags);
+
 	cookie = per_cpu(cookie_next_key, cpu)+=nr_cpu_ids;
 	cookiemap_add(key, cookie);
 
@@ -118,7 +249,8 @@ static inline uint32_t get_cookie(int cpu, int tgid, struct vm_area_struct *vma)
 	gator_buffer_write_packed_int(cpu, cookie);
 	gator_buffer_write_string(cpu, text);
 
-output:
+	local_irq_restore(flags);
+
 	return cookie;
 }
 
@@ -136,7 +268,7 @@ static int get_exec_cookie(int cpu, struct task_struct *task)
 			continue;
 		if (!(vma->vm_flags & VM_EXECUTABLE))
 			continue;
-		cookie = get_cookie(cpu, task->tgid, vma);
+		cookie = get_cookie(cpu, task, vma, NULL);
 		break;
 	}
 
@@ -157,7 +289,7 @@ static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsig
 			continue;
 
 		if (vma->vm_file) {
-			cookie = get_cookie(cpu, task->tgid, vma);
+			cookie = get_cookie(cpu, task, vma, NULL);
 			*offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
 		} else {
 			/* must be an anonymous map */
@@ -173,11 +305,13 @@ static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsig
 	return cookie;
 }
 
-static void cookies_initialize(void)
+static int cookies_initialize(void)
 {
 	uint32_t crc, poly;
-	int cpu, size;
-	int i, j;
+	int i, j, cpu, size, err = 0;
+
+	int translate_buffer_size = 512; // must be a power of 2
+	translate_buffer_mask = translate_buffer_size / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
 
 	for_each_present_cpu(cpu) {
 		per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
@@ -189,6 +323,21 @@ static void cookies_initialize(void)
 		size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
 		per_cpu(cookie_values, cpu) = (uint32_t*)kmalloc(size, GFP_KERNEL);
 		memset(per_cpu(cookie_values, cpu), 0, size);
+
+		per_cpu(translate_buffer, cpu) = (unsigned int *)kmalloc(translate_buffer_size, GFP_KERNEL);
+		if (!per_cpu(translate_buffer, cpu)) {
+			err = -ENOMEM;
+			goto cookie_setup_error;
+		}
+
+		per_cpu(translate_buffer_write, cpu) = 0;
+		per_cpu(translate_buffer_read, cpu) = 0;
+
+		per_cpu(translate_text, cpu) = (char *)kmalloc(TRANSLATE_SIZE, GFP_KERNEL);
+		if (!per_cpu(translate_text, cpu)) {
+			err = -ENOMEM;
+			goto cookie_setup_error;
+		}
 	}
 
 	// build CRC32 table
@@ -205,6 +354,9 @@ static void cookies_initialize(void)
 		}
 		gator_crc32_table[i] = crc;
 	}
+
+cookie_setup_error:
+	return err;
 }
 
 static void cookies_release(void)
@@ -217,6 +369,14 @@ static void cookies_release(void)
 
 		kfree(per_cpu(cookie_values, cpu));
 		per_cpu(cookie_values, cpu) = NULL;
+
+		kfree(per_cpu(translate_buffer, cpu));
+		per_cpu(translate_buffer, cpu) = NULL;
+		per_cpu(translate_buffer_read, cpu) = 0;
+		per_cpu(translate_buffer_write, cpu) = 0;
+
+		kfree(per_cpu(translate_text, cpu));
+		per_cpu(translate_text, cpu) = NULL;
 	}
 
 	kfree(gator_crc32_table);
diff --git a/gator_events_net.c b/gator_events_net.c
index 3ca5911..15a395e 100644
--- a/gator_events_net.c
+++ b/gator_events_net.c
@@ -136,6 +136,7 @@ static void gator_events_net_stop(void)
 static int gator_events_net_read(int **buffer)
 {
 	int len, drv_delta, rx_delta, tx_delta;
+	static int last_drv_delta = 0, last_rx_delta = 0, last_tx_delta = 0;
 
 	if (raw_smp_processor_id() != 0)
 		return 0;
@@ -144,17 +145,20 @@ static int gator_events_net_read(int **buffer)
 	calculate_delta(&drv_delta, &rx_delta, &tx_delta);
 
 	len = 0;
-	if (netdrv_enabled) {
+	if (netdrv_enabled && last_drv_delta != drv_delta) {
+		last_drv_delta = drv_delta;
 		netGet[len++] = netdrv_key;
 		netGet[len++] = drv_delta;
 	}
 
-	if (netrx_enabled) {
+	if (netrx_enabled && last_rx_delta != rx_delta) {
+		last_rx_delta = rx_delta;
 		netGet[len++] = netrx_key;
 		netGet[len++] = rx_delta;
 	}
 
-	if (nettx_enabled) {
+	if (nettx_enabled && last_tx_delta != tx_delta) {
+		last_tx_delta = tx_delta;
 		netGet[len++] = nettx_key;
 		netGet[len++] = tx_delta;
 	}
diff --git a/gator_fs.c b/gator_fs.c
index 5f8983f..f7cbff2 100644
--- a/gator_fs.c
+++ b/gator_fs.c
@@ -27,6 +27,9 @@ static struct inode *gatorfs_get_inode(struct super_block *sb, int mode)
 	struct inode *inode = new_inode(sb);
 
 	if (inode) {
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)
+		inode->i_ino = get_next_ino();
+#endif
 		inode->i_mode = mode;
 		inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
 	}
@@ -246,16 +249,29 @@ static int gatorfs_fill_super(struct super_block *sb, void *data, int silent)
 	return 0;
 }
 
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 39)
 static int gatorfs_get_sb(struct file_system_type *fs_type,
 	int flags, const char *dev_name, void *data, struct vfsmount *mnt)
 {
 	return get_sb_single(fs_type, flags, data, gatorfs_fill_super, mnt);
 }
+#else
+static struct dentry *gatorfs_mount(struct file_system_type *fs_type,
+	int flags, const char *dev_name, void *data)
+{
+	return mount_nodev(fs_type, flags, data, gatorfs_fill_super);
+}
+#endif
 
 static struct file_system_type gatorfs_type = {
 	.owner		= THIS_MODULE,
 	.name		= "gatorfs",
-	.get_sb		= gatorfs_get_sb,
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 39)
+	 	.get_sb		= gatorfs_get_sb,
+#else
+	 	.mount		= gatorfs_mount,
+#endif
+
 	.kill_sb	= kill_litter_super,
 };
 
diff --git a/gator_main.c b/gator_main.c
index 60b09cb..a1ab776 100644
--- a/gator_main.c
+++ b/gator_main.c
@@ -7,7 +7,7 @@
  *
  */
 
-static unsigned long gator_protocol_version = 3;
+static unsigned long gator_protocol_version = 4;
 
 #include "gator.h"
 #include <linux/slab.h>
@@ -15,6 +15,9 @@ static unsigned long gator_protocol_version = 3;
 #include <linux/sched.h>
 #include <linux/irq.h>
 #include <linux/vmalloc.h>
+#include <linux/hardirq.h>
+#include <linux/highmem.h>
+#include <linux/pagemap.h>
 #include <asm/uaccess.h>
 
 #ifndef CONFIG_GENERIC_TRACER
@@ -31,6 +34,12 @@ static unsigned long gator_protocol_version = 3;
 #warning gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined
 #endif
 
+#ifdef CONFIG_SMP
+#ifndef CONFIG_LOCAL_TIMERS
+#warning gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
+#endif
+#endif
+
 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
 #error kernels prior to 2.6.32 are not supported
 #endif
@@ -38,7 +47,7 @@ static unsigned long gator_protocol_version = 3;
 /******************************************************************************
  * DEFINES
  ******************************************************************************/
-#define BUFFER_SIZE_DEFAULT		262144
+#define BUFFER_SIZE_DEFAULT		(256*1024)
 #define SYNC_FREQ_DEFAULT		1000
 
 #define NO_COOKIE				0UL
@@ -280,13 +289,15 @@ static void gator_add_trace(int cpu, unsigned int address)
 
 static void gator_add_sample(int cpu, struct pt_regs * const regs)
 {
+	struct module *mod;
+	unsigned int addr, cookie = 0;
 	int inKernel = regs ? !user_mode(regs) : 1;
-	unsigned long cookie = !inKernel ? get_exec_cookie(cpu, current) : NO_COOKIE;
+	unsigned long exec_cookie = !inKernel ? get_exec_cookie(cpu, current) : NO_COOKIE;
 
 	gator_buffer_write_packed_int(cpu, PROTOCOL_START_BACKTRACE);
 
 	// TGID::PID::inKernel
-	gator_buffer_write_packed_int(cpu, cookie);
+	gator_buffer_write_packed_int(cpu, exec_cookie);
 	gator_buffer_write_packed_int(cpu, (unsigned int)current->tgid);
 	gator_buffer_write_packed_int(cpu, (unsigned int)current->pid);
 	gator_buffer_write_packed_int(cpu, inKernel);
@@ -294,8 +305,14 @@ static void gator_add_sample(int cpu, struct pt_regs * const regs)
 	// get_irq_regs() will return NULL outside of IRQ context (e.g. nested IRQ)
 	if (regs) {
 		if (inKernel) {
-			gator_buffer_write_packed_int(cpu, PC_REG & ~1);
-			gator_buffer_write_packed_int(cpu, 0); // cookie
+			addr = PC_REG;
+			mod = __module_address(addr);
+			if (mod) {
+				cookie = get_cookie(cpu, current, NULL, mod);
+				addr = addr - (unsigned long)mod->module_core;
+			}
+			gator_buffer_write_packed_int(cpu, addr & ~1);
+			gator_buffer_write_packed_int(cpu, cookie);
 		} else {
 			// Cookie+PC
 			gator_add_trace(cpu, PC_REG);
@@ -739,13 +756,11 @@ static int gator_op_start(void)
 
 	mutex_lock(&start_mutex);
 
-	if (gator_started || gator_start())
+	if (gator_started || gator_start() || cookies_initialize())
 		err = -EINVAL;
 	else
 		gator_started = 1;
 
-	cookies_initialize();
-
 	mutex_unlock(&start_mutex);
 
 	return err;