From 18f88ad877a9d3c8bc21ceb0b565e6a32a7a07cc Mon Sep 17 00:00:00 2001 From: Jon Medhurst Date: Fri, 16 Mar 2012 17:36:42 +0000 Subject: gator/driver: Version 5.9 Signed-off-by: Jon Medhurst --- drivers/gator/driver/Makefile | 56 ++ drivers/gator/driver/gator.h | 99 +++ drivers/gator/driver/gator_annotate.c | 181 +++++ drivers/gator/driver/gator_annotate_kernel.c | 90 +++ drivers/gator/driver/gator_backtrace.c | 116 +++ drivers/gator/driver/gator_cookies.c | 403 ++++++++++ drivers/gator/driver/gator_ebs.c | 192 +++++ drivers/gator/driver/gator_events.sh | 19 + drivers/gator/driver/gator_events_armv6.c | 244 ++++++ drivers/gator/driver/gator_events_armv7.c | 320 ++++++++ drivers/gator/driver/gator_events_block.c | 171 ++++ drivers/gator/driver/gator_events_irq.c | 195 +++++ drivers/gator/driver/gator_events_l2c-310.c | 179 +++++ drivers/gator/driver/gator_events_mali.c | 611 ++++++++++++++ drivers/gator/driver/gator_events_meminfo.c | 196 +++++ drivers/gator/driver/gator_events_mmaped.c | 198 +++++ drivers/gator/driver/gator_events_net.c | 144 ++++ drivers/gator/driver/gator_events_perf_pmu.c | 296 +++++++ drivers/gator/driver/gator_events_power.c | 178 +++++ drivers/gator/driver/gator_events_sched.c | 114 +++ drivers/gator/driver/gator_events_scorpion.c | 678 ++++++++++++++++ drivers/gator/driver/gator_fs.c | 284 +++++++ drivers/gator/driver/gator_hrtimer_gator.c | 76 ++ drivers/gator/driver/gator_hrtimer_perf.c | 113 +++ drivers/gator/driver/gator_main.c | 1109 ++++++++++++++++++++++++++ drivers/gator/driver/gator_pack.c | 262 ++++++ drivers/gator/driver/gator_trace_gpu.c | 250 ++++++ drivers/gator/driver/gator_trace_gpu.h | 79 ++ drivers/gator/driver/gator_trace_sched.c | 269 +++++++ 29 files changed, 7122 insertions(+) create mode 100644 drivers/gator/driver/Makefile create mode 100644 drivers/gator/driver/gator.h create mode 100644 drivers/gator/driver/gator_annotate.c create mode 100644 drivers/gator/driver/gator_annotate_kernel.c create mode 100644 drivers/gator/driver/gator_backtrace.c create mode 100644 drivers/gator/driver/gator_cookies.c create mode 100644 drivers/gator/driver/gator_ebs.c create mode 100644 drivers/gator/driver/gator_events.sh create mode 100644 drivers/gator/driver/gator_events_armv6.c create mode 100644 drivers/gator/driver/gator_events_armv7.c create mode 100644 drivers/gator/driver/gator_events_block.c create mode 100644 drivers/gator/driver/gator_events_irq.c create mode 100644 drivers/gator/driver/gator_events_l2c-310.c create mode 100644 drivers/gator/driver/gator_events_mali.c create mode 100644 drivers/gator/driver/gator_events_meminfo.c create mode 100644 drivers/gator/driver/gator_events_mmaped.c create mode 100644 drivers/gator/driver/gator_events_net.c create mode 100644 drivers/gator/driver/gator_events_perf_pmu.c create mode 100644 drivers/gator/driver/gator_events_power.c create mode 100644 drivers/gator/driver/gator_events_sched.c create mode 100644 drivers/gator/driver/gator_events_scorpion.c create mode 100644 drivers/gator/driver/gator_fs.c create mode 100644 drivers/gator/driver/gator_hrtimer_gator.c create mode 100644 drivers/gator/driver/gator_hrtimer_perf.c create mode 100644 drivers/gator/driver/gator_main.c create mode 100644 drivers/gator/driver/gator_pack.c create mode 100644 drivers/gator/driver/gator_trace_gpu.c create mode 100644 drivers/gator/driver/gator_trace_gpu.h create mode 100644 drivers/gator/driver/gator_trace_sched.c diff --git a/drivers/gator/driver/Makefile b/drivers/gator/driver/Makefile new file mode 100644 index 00000000000..e521b9915dc --- /dev/null +++ b/drivers/gator/driver/Makefile @@ -0,0 +1,56 @@ +ifneq ($(KERNELRELEASE),) + +# Uncomment the following line to enable kernel stack unwinding within gator, or update gator_backtrace.c +# EXTRA_CFLAGS += -DGATOR_KERNEL_STACK_UNWINDING + +obj-m := gator.o + +gator-y := gator_main.o \ + gator_events_irq.o \ + gator_events_sched.o \ + gator_events_net.o \ + gator_events_block.o \ + gator_events_meminfo.o \ + gator_events_power.o \ + gator_events_perf_pmu.o + +gator-y += gator_events_mmaped.o + +ifneq ($(GATOR_WITH_MALI_SUPPORT),) +ifeq ($(GATOR_WITH_MALI_SUPPORT),MALI_T6xx) +gator-y += gator_events_mali_t6xx.o +else +gator-y += gator_events_mali.o +endif +EXTRA_CFLAGS += -DMALI_SUPPORT=$(GATOR_WITH_MALI_SUPPORT) +endif + +gator-$(CONFIG_ARM) += gator_events_armv6.o \ + gator_events_armv7.o \ + gator_events_l2c-310.o \ + gator_events_scorpion.o + +$(obj)/gator_main.o: gator_events.h + +clean-files := gator_events.h + + chk_events.h = : + quiet_chk_events.h = echo ' CHK $@' +silent_chk_events.h = : +gator_events.h: FORCE + @$($(quiet)chk_events.h) + $(Q)cd $(obj) ; $(CONFIG_SHELL) $(obj)/gator_events.sh $@ + +else + +all: + @echo + @echo "usage:" + @echo " make -C M=\`pwd\` ARCH=arm CROSS_COMPILE=<...> modules" + @echo + $(error) + +clean: + rm -f *.o .*.cmd gator_events.h modules.order Module.symvers gator.ko gator.mod.c + +endif diff --git a/drivers/gator/driver/gator.h b/drivers/gator/driver/gator.h new file mode 100644 index 00000000000..a7a323c3fae --- /dev/null +++ b/drivers/gator/driver/gator.h @@ -0,0 +1,99 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef GATOR_H_ +#define GATOR_H_ + +#include +#include +#include +#include + +#define GATOR_PERF_SUPPORT LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0) +#define GATOR_PERF_PMU_SUPPORT GATOR_PERF_SUPPORT && defined(CONFIG_PERF_EVENTS) && defined(CONFIG_HW_PERF_EVENTS) +#define GATOR_NO_PERF_SUPPORT (!(GATOR_PERF_SUPPORT)) +#define GATOR_CPU_FREQ_SUPPORT (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)) && defined(CONFIG_CPU_FREQ) + +// cpu ids +#define ARM1136 0xb36 +#define ARM1156 0xb56 +#define ARM1176 0xb76 +#define ARM11MPCORE 0xb02 +#define CORTEX_A5 0xc05 +#define CORTEX_A7 0xc07 +#define CORTEX_A8 0xc08 +#define CORTEX_A9 0xc09 +#define CORTEX_A15 0xc0f +#define SCORPION 0x00f +#define SCORPIONMP 0x02d +#define KRAITSIM 0x049 +#define KRAIT 0x04d + +/****************************************************************************** + * Filesystem + ******************************************************************************/ +int gatorfs_create_file_perm(struct super_block *sb, struct dentry *root, + char const *name, const struct file_operations *fops, int perm); + +struct dentry *gatorfs_mkdir(struct super_block *sb, + struct dentry *root, char const *name); + +int gatorfs_create_ulong(struct super_block *sb, struct dentry *root, + char const *name, unsigned long *val); + +int gatorfs_create_ro_ulong(struct super_block *sb, struct dentry *root, + char const *name, unsigned long *val); + +void gator_op_create_files(struct super_block *sb, struct dentry *root); + +/****************************************************************************** + * Tracepoints + ******************************************************************************/ +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) +# error Kernels prior to 2.6.32 not supported +#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) +# define GATOR_DEFINE_PROBE(probe_name, proto) \ + static void probe_##probe_name(PARAMS(proto)) +# define GATOR_REGISTER_TRACE(probe_name) \ + register_trace_##probe_name(probe_##probe_name) +# define GATOR_UNREGISTER_TRACE(probe_name) \ + unregister_trace_##probe_name(probe_##probe_name) +#else +# define GATOR_DEFINE_PROBE(probe_name, proto) \ + static void probe_##probe_name(void *data, PARAMS(proto)) +# define GATOR_REGISTER_TRACE(probe_name) \ + register_trace_##probe_name(probe_##probe_name, NULL) +# define GATOR_UNREGISTER_TRACE(probe_name) \ + unregister_trace_##probe_name(probe_##probe_name, NULL) +#endif + +/****************************************************************************** + * Events + ******************************************************************************/ +struct gator_interface { + int (*create_files)(struct super_block *sb, struct dentry *root); + int (*start)(void); + void (*stop)(void); + int (*online)(int** buffer); + int (*offline)(int** buffer); + void (*online_dispatch)(int cpu); // called in process context but may not be running on core 'cpu' + void (*offline_dispatch)(int cpu); // called in process context but may not be running on core 'cpu' + int (*read)(int **buffer); + int (*read64)(long long **buffer); + struct list_head list; +}; + +#define gator_events_init(initfn) \ + static inline int __gator_events_init_test(void) \ + { return initfn(); } + +int gator_events_install(struct gator_interface *interface); +int gator_events_get_key(void); +extern u32 gator_cpuid(void); + +#endif // GATOR_H_ diff --git a/drivers/gator/driver/gator_annotate.c b/drivers/gator/driver/gator_annotate.c new file mode 100644 index 00000000000..36a921cdd25 --- /dev/null +++ b/drivers/gator/driver/gator_annotate.c @@ -0,0 +1,181 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#define ANNOTATE_SIZE (16*1024) +static DEFINE_SPINLOCK(annotate_lock); +static char *annotateBuf; +static char *annotateBuf0; +static char *annotateBuf1; +static int annotatePos; +static int annotateSel; +static bool collect_annotations = false; + +static ssize_t annotate_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) +{ + char tempBuffer[512]; + int remaining, size; + uint32_t tid; + + if (*offset) + return -EINVAL; + + // determine size to capture + size = count < sizeof(tempBuffer) ? count : sizeof(tempBuffer); + + // note: copy may be for naught if remaining is zero, but better to do the copy outside of the spinlock + if (file == NULL) { + // copy from kernel + memcpy(tempBuffer, buf, size); + + // set the thread id to the kernel thread, not the current thread + tid = -1; + } else { + // copy from user space + if (copy_from_user(tempBuffer, buf, size) != 0) + return -EINVAL; + tid = current->pid; + } + + // synchronize shared variables annotateBuf and annotatePos + spin_lock(&annotate_lock); + if (collect_annotations && annotateBuf) { + remaining = ANNOTATE_SIZE - annotatePos - 256; // pad for headers and release + size = size < remaining ? size : remaining; + if (size > 0) { + uint64_t time = gator_get_time(); + uint32_t cpuid = smp_processor_id(); + int pos = annotatePos; + pos += gator_write_packed_int(&annotateBuf[pos], tid); + pos += gator_write_packed_int64(&annotateBuf[pos], time); + pos += gator_write_packed_int(&annotateBuf[pos], cpuid); + pos += gator_write_packed_int(&annotateBuf[pos], size); + memcpy(&annotateBuf[pos], tempBuffer, size); + annotatePos = pos + size; + } + } + spin_unlock(&annotate_lock); + + if (size <= 0) { + wake_up(&gator_buffer_wait); + return 0; + } + + // return the number of bytes written + return size; +} + +#include "gator_annotate_kernel.c" + +static int annotate_release(struct inode *inode, struct file *file) +{ + int remaining = ANNOTATE_SIZE - annotatePos; + if (remaining < 16) { + return -EFAULT; + } + + spin_lock(&annotate_lock); + if (annotateBuf) { + uint32_t tid = current->pid; + int pos = annotatePos; + pos += gator_write_packed_int(&annotateBuf[pos], tid); + pos += gator_write_packed_int64(&annotateBuf[pos], 0); // time + pos += gator_write_packed_int(&annotateBuf[pos], 0); // cpuid + pos += gator_write_packed_int(&annotateBuf[pos], 0); // size + annotatePos = pos; + } + spin_unlock(&annotate_lock); + + return 0; +} + +static const struct file_operations annotate_fops = { + .write = annotate_write, + .release = annotate_release +}; + +static int gator_annotate_create_files(struct super_block *sb, struct dentry *root) +{ + annotateBuf = NULL; + return gatorfs_create_file_perm(sb, root, "annotate", &annotate_fops, 0666); +} + +static int gator_annotate_init(void) +{ + annotateBuf0 = kmalloc(ANNOTATE_SIZE, GFP_KERNEL); + annotateBuf1 = kmalloc(ANNOTATE_SIZE, GFP_KERNEL); + if (!annotateBuf0 || !annotateBuf1) + return -1; + return 0; +} + +static int gator_annotate_start(void) +{ + annotateSel = 0; + annotatePos = 1; + annotateBuf = annotateBuf0; + annotateBuf[0] = FRAME_ANNOTATE; + collect_annotations = true; + return 0; +} + +static void gator_annotate_stop(void) +{ + collect_annotations = false; +} + +static void gator_annotate_shutdown(void) +{ + spin_lock(&annotate_lock); + annotateBuf = NULL; + spin_unlock(&annotate_lock); +} + +static void gator_annotate_exit(void) +{ + spin_lock(&annotate_lock); + kfree(annotateBuf0); + kfree(annotateBuf1); + annotateBuf = annotateBuf0 = annotateBuf1 = NULL; + spin_unlock(&annotate_lock); +} + +static int gator_annotate_ready(void) +{ + return annotatePos > 1 && annotateBuf; +} + +static int gator_annotate_read(char **buffer) +{ + int len; + + if (!gator_annotate_ready()) + return 0; + + annotateSel = !annotateSel; + + if (buffer) + *buffer = annotateBuf; + + spin_lock(&annotate_lock); + len = annotatePos; + annotateBuf = annotateSel ? annotateBuf1 : annotateBuf0; + annotateBuf[0] = FRAME_ANNOTATE; + annotatePos = 1; + spin_unlock(&annotate_lock); + + return len; +} diff --git a/drivers/gator/driver/gator_annotate_kernel.c b/drivers/gator/driver/gator_annotate_kernel.c new file mode 100644 index 00000000000..ffab08795b7 --- /dev/null +++ b/drivers/gator/driver/gator_annotate_kernel.c @@ -0,0 +1,90 @@ +/** + * Copyright (C) ARM Limited 2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +static void kannotate_write(char* ptr, unsigned int size) +{ + int retval; + int pos = 0; + loff_t offset = 0; + while (pos < size) { + retval = annotate_write(NULL, &ptr[pos], size - pos, &offset); + if (retval < 0) { + printk(KERN_WARNING "gator: kannotate_write failed with return value %d\n", retval); + return; + } + pos += retval; + } +} + +// String annotation +void gator_annotate(char* string) +{ + kannotate_write(string, strlen(string) + 1); +} +EXPORT_SYMBOL(gator_annotate); + +// String annotation with color +void gator_annotate_color(int color, char* string) +{ + kannotate_write((char*)&color, sizeof(color)); + kannotate_write(string, strlen(string) + 1); +} +EXPORT_SYMBOL(gator_annotate_color); + +// Terminate an annotation +void gator_annotate_end(void) +{ + char nul = 0; + kannotate_write(&nul, sizeof(nul)); +} +EXPORT_SYMBOL(gator_annotate_end); + +// Image annotation with optional string +void gator_annotate_visual(char* data, unsigned int length, char* string) +{ + long long visual_annotation = 0x011c | (strlen(string) << 16) | ((long long)length << 32); + kannotate_write((char*)&visual_annotation, 8); + kannotate_write(string, strlen(string)); + kannotate_write(data, length); +} +EXPORT_SYMBOL(gator_annotate_visual); + +// Marker annotation +void gator_annotate_marker(void) +{ + int marker_annotation = 0x00021c; + kannotate_write((char*)&marker_annotation, 3); +} +EXPORT_SYMBOL(gator_annotate_marker); + +// Marker annotation with a string +void gator_annotate_marker_str(char* string) +{ + int marker_annotation = 0x021c; + kannotate_write((char*)&marker_annotation, 2); + kannotate_write(string, strlen(string) + 1); +} +EXPORT_SYMBOL(gator_annotate_marker_str); + +// Marker annotation with a color +void gator_annotate_marker_color(int color) +{ + long long marker_annotation = (0x021c | ((long long)color << 16)) & 0x0000ffffffffffffLL; + kannotate_write((char*)&marker_annotation, 7); +} +EXPORT_SYMBOL(gator_annotate_marker_color); + +// Marker annotationw ith a string and color +void gator_annotate_marker_color_str(int color, char* string) +{ + long long marker_annotation = 0x021c | ((long long)color << 16); + kannotate_write((char*)&marker_annotation, 6); + kannotate_write(string, strlen(string) + 1); +} +EXPORT_SYMBOL(gator_annotate_marker_color_str); diff --git a/drivers/gator/driver/gator_backtrace.c b/drivers/gator/driver/gator_backtrace.c new file mode 100644 index 00000000000..26503ef0a55 --- /dev/null +++ b/drivers/gator/driver/gator_backtrace.c @@ -0,0 +1,116 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +/* + * EABI backtrace stores {fp,lr} on the stack. + */ +struct frame_tail_eabi { + unsigned long fp; // points to prev_lr + unsigned long lr; +}; + +static void arm_backtrace_eabi(int cpu, int buftype, struct pt_regs * const regs, unsigned int depth) +{ +#if defined(__arm__) + struct frame_tail_eabi *tail; + struct frame_tail_eabi *next; + struct frame_tail_eabi *ptrtail; + struct frame_tail_eabi buftail; + unsigned long fp = regs->ARM_fp; + unsigned long sp = regs->ARM_sp; + unsigned long lr = regs->ARM_lr; + int is_user_mode = user_mode(regs); + + if (!is_user_mode) { + return; + } + + /* entry preamble may not have executed */ + gator_add_trace(cpu, buftype, lr); + + /* check tail is valid */ + if (fp == 0 || fp < sp) { + return; + } + + tail = (struct frame_tail_eabi *)(fp - 4); + + while (depth-- && tail && !((unsigned long) tail & 3)) { + /* Also check accessibility of one struct frame_tail beyond */ + if (!access_ok(VERIFY_READ, tail, sizeof(struct frame_tail_eabi))) + return; + if (__copy_from_user_inatomic(&buftail, tail, sizeof(struct frame_tail_eabi))) + return; + ptrtail = &buftail; + + lr = ptrtail[0].lr; + gator_add_trace(cpu, buftype, lr); + + /* frame pointers should progress back up the stack, towards higher addresses */ + next = (struct frame_tail_eabi *)(lr - 4); + if (tail >= next || lr == 0) { + fp = ptrtail[0].fp; + next = (struct frame_tail_eabi *)(fp - 4); + /* check tail is valid */ + if (tail >= next || fp == 0) { + return; + } + } + + tail = next; + } +#endif +} + +#if defined(__arm__) +static DEFINE_PER_CPU(int, backtrace_buffer); +static int report_trace(struct stackframe *frame, void *d) +{ + struct module *mod; + unsigned int *depth = d, addr = frame->pc, cookie = NO_COOKIE, cpu = smp_processor_id(); + + if (*depth) { + mod = __module_address(addr); + if (mod) { + cookie = get_cookie(cpu, per_cpu(backtrace_buffer, cpu), current, NULL, mod, true); + addr = addr - (unsigned long)mod->module_core; + } + gator_buffer_write_packed_int(cpu, per_cpu(backtrace_buffer, cpu), addr & ~1); + gator_buffer_write_packed_int(cpu, per_cpu(backtrace_buffer, cpu), cookie); + (*depth)--; + } + + return *depth == 0; +} +#endif + +// Uncomment the following line to enable kernel stack unwinding within gator, note it can also be defined from the Makefile +// #define GATOR_KERNEL_STACK_UNWINDING +static void kernel_backtrace(int cpu, int buftype, struct pt_regs * const regs) +{ +#if defined(__arm__) +#ifdef GATOR_KERNEL_STACK_UNWINDING + int depth = gator_backtrace_depth; +#else + int depth = 1; +#endif + struct stackframe frame; + if (depth == 0) + depth = 1; + frame.fp = regs->ARM_fp; + frame.sp = regs->ARM_sp; + frame.lr = regs->ARM_lr; + frame.pc = regs->ARM_pc; + per_cpu(backtrace_buffer, cpu) = buftype; + walk_stackframe(&frame, report_trace, &depth); +#else + gator_buffer_write_packed_int(cpu, buftype, PC_REG & ~1); + gator_buffer_write_packed_int(cpu, buftype, NO_COOKIE); +#endif +} diff --git a/drivers/gator/driver/gator_cookies.c b/drivers/gator/driver/gator_cookies.c new file mode 100644 index 00000000000..1beb34f5170 --- /dev/null +++ b/drivers/gator/driver/gator_cookies.c @@ -0,0 +1,403 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#define COOKIEMAP_ENTRIES 1024 /* must be power of 2 */ +#define TRANSLATE_SIZE 256 +#define MAX_COLLISIONS 2 + +static uint32_t *gator_crc32_table; +static uint32_t translate_buffer_mask; + +static DEFINE_PER_CPU(char *, translate_text); +static DEFINE_PER_CPU(uint32_t, cookie_next_key); +static DEFINE_PER_CPU(uint64_t *, cookie_keys); +static DEFINE_PER_CPU(uint32_t *, cookie_values); +static DEFINE_PER_CPU(int, translate_buffer_read); +static DEFINE_PER_CPU(int, translate_buffer_write); +static DEFINE_PER_CPU(unsigned int *, translate_buffer); + +static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt); +static void wq_cookie_handler(struct work_struct *unused); +DECLARE_WORK(cookie_work, wq_cookie_handler); + +static uint32_t cookiemap_code(uint64_t value64) { + uint32_t value = (uint32_t)((value64 >> 32) + value64); + uint32_t cookiecode = (value >> 24) & 0xff; + cookiecode = cookiecode * 31 + ((value >> 16) & 0xff); + cookiecode = cookiecode * 31 + ((value >> 8) & 0xff); + cookiecode = cookiecode * 31 + ((value >> 0) & 0xff); + cookiecode &= (COOKIEMAP_ENTRIES-1); + return cookiecode * MAX_COLLISIONS; +} + +static uint32_t gator_chksum_crc32(char *data) +{ + register unsigned long crc; + unsigned char *block = data; + int i, length = strlen(data); + + crc = 0xFFFFFFFF; + for (i = 0; i < length; i++) { + crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF]; + } + + return (crc ^ 0xFFFFFFFF); +} + +/* + * Exists + * Pre: [0][1][v][3]..[n-1] + * Post: [v][0][1][3]..[n-1] + */ +static uint32_t cookiemap_exists(uint64_t key) { + unsigned long x, flags, retval = 0; + int cpu = smp_processor_id(); + uint32_t cookiecode = cookiemap_code(key); + uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]); + uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]); + + // Can be called from interrupt handler or from work queue + local_irq_save(flags); + for (x = 0; x < MAX_COLLISIONS; x++) { + if (keys[x] == key) { + uint32_t value = values[x]; + for (; x > 0; x--) { + keys[x] = keys[x-1]; + values[x] = values[x-1]; + } + keys[0] = key; + values[0] = value; + retval = value; + break; + } + } + local_irq_restore(flags); + + return retval; +} + +/* + * Add + * Pre: [0][1][2][3]..[n-1] + * Post: [v][0][1][2]..[n-2] + */ +static void cookiemap_add(uint64_t key, uint32_t value) { + int cpu = smp_processor_id(); + int cookiecode = cookiemap_code(key); + uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]); + uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]); + int x; + + for (x = MAX_COLLISIONS-1; x > 0; x--) { + keys[x] = keys[x-1]; + values[x] = values[x-1]; + } + keys[0] = key; + values[0] = value; +} + +static void translate_buffer_write_int(int cpu, unsigned int x) +{ + per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_write, cpu)++] = x; + per_cpu(translate_buffer_write, cpu) &= translate_buffer_mask; +} + +static unsigned int translate_buffer_read_int(int cpu) +{ + unsigned int value = per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_read, cpu)++]; + per_cpu(translate_buffer_read, cpu) &= translate_buffer_mask; + return value; +} + +static void wq_cookie_handler(struct work_struct *unused) +{ + struct task_struct *task; + struct vm_area_struct *vma; + int cpu = smp_processor_id(); + unsigned int cookie, commit; + + mutex_lock(&start_mutex); + + if (gator_started != 0) { + commit = per_cpu(translate_buffer_write, cpu); + while (per_cpu(translate_buffer_read, cpu) != commit) { + task = (struct task_struct *)translate_buffer_read_int(cpu); + vma = (struct vm_area_struct *)translate_buffer_read_int(cpu); + cookie = get_cookie(cpu, TIMER_BUF, task, vma, NULL, false); + } + } + + mutex_unlock(&start_mutex); +} + +// Retrieve full name from proc/pid/cmdline for java processes on Android +static int translate_app_process(char** text, int cpu, struct task_struct * task, struct vm_area_struct *vma, bool in_interrupt) +{ + void *maddr; + unsigned int len; + unsigned long addr; + struct mm_struct *mm; + struct page *page = NULL; + struct vm_area_struct *page_vma; + int bytes, offset, retval = 0, ptr; + char * buf = per_cpu(translate_text, cpu); + + // Push work into a work queue if in atomic context as the kernel functions below might sleep + // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems + // inconsistent during a context switch between android/linux versions + if (in_interrupt) { + // Check if already in buffer + ptr = per_cpu(translate_buffer_read, cpu); + while (ptr != per_cpu(translate_buffer_write, cpu)) { + if (per_cpu(translate_buffer, cpu)[ptr] == (int)task) + goto out; + ptr = (ptr + 2) & translate_buffer_mask; + } + + translate_buffer_write_int(cpu, (unsigned int)task); + translate_buffer_write_int(cpu, (unsigned int)vma); + schedule_work(&cookie_work); + goto out; + } + + mm = get_task_mm(task); + if (!mm) + goto out; + if (!mm->arg_end) + goto outmm; + addr = mm->arg_start; + len = mm->arg_end - mm->arg_start; + + if (len > TRANSLATE_SIZE) + len = TRANSLATE_SIZE; + + down_read(&mm->mmap_sem); + while (len) { + if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0) + goto outsem; + + maddr = kmap(page); + offset = addr & (PAGE_SIZE-1); + bytes = len; + if (bytes > PAGE_SIZE - offset) + bytes = PAGE_SIZE - offset; + + copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes); + + kunmap(page); // release page allocated by get_user_pages() + page_cache_release(page); + + len -= bytes; + buf += bytes; + addr += bytes; + + *text = per_cpu(translate_text, cpu); + retval = 1; + } + + // On app_process startup, /proc/pid/cmdline is initially "zygote" then "" but changes after an initial startup period + if (strcmp(*text, "zygote") == 0 || strcmp(*text, "") == 0) + retval = 0; + +outsem: + up_read(&mm->mmap_sem); +outmm: + mmput(mm); +out: + return retval; +} + +static inline uint32_t get_cookie(int cpu, int buftype, struct task_struct *task, struct vm_area_struct *vma, struct module *mod, bool in_interrupt) +{ + unsigned long flags, cookie; + struct path *path; + uint64_t key; + char *text; + + if (mod) { + text = mod->name; + } else { + if (!vma || !vma->vm_file) { + return INVALID_COOKIE; + } + path = &vma->vm_file->f_path; + if (!path || !path->dentry) { + return INVALID_COOKIE; + } + + text = (char*)path->dentry->d_name.name; + } + + key = gator_chksum_crc32(text); + key = (key << 32) | (uint32_t)task->tgid; + + cookie = cookiemap_exists(key); + if (cookie) { + return cookie; + } + + if (strcmp(text, "app_process") == 0 && !mod) { + if (!translate_app_process(&text, cpu, task, vma, in_interrupt)) + return INVALID_COOKIE; + } + + // Can be called from interrupt handler or from work queue or from scheduler trace + local_irq_save(flags); + + cookie = INVALID_COOKIE; + if (buffer_check_space(cpu, buftype, strlen(text) + 2 * MAXSIZE_PACK32)) { + cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids; + cookiemap_add(key, cookie); + + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COOKIE); + gator_buffer_write_packed_int(cpu, buftype, cookie); + gator_buffer_write_string(cpu, buftype, text); + } + + local_irq_restore(flags); + + return cookie; +} + +static int get_exec_cookie(int cpu, int buftype, struct task_struct *task) +{ + unsigned long cookie = NO_COOKIE; + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; + + if (!mm) + return cookie; + + for (vma = mm->mmap; vma; vma = vma->vm_next) { + if (!vma->vm_file) + continue; + if (!(vma->vm_flags & VM_EXECUTABLE)) + continue; + cookie = get_cookie(cpu, buftype, task, vma, NULL, true); + break; + } + + return cookie; +} + +static unsigned long get_address_cookie(int cpu, int buftype, struct task_struct *task, unsigned long addr, off_t *offset) +{ + unsigned long cookie = NO_COOKIE; + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; + + if (!mm) + return cookie; + + for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { + if (addr < vma->vm_start || addr >= vma->vm_end) + continue; + + if (vma->vm_file) { + cookie = get_cookie(cpu, buftype, task, vma, NULL, true); + *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start; + } else { + /* must be an anonymous map */ + *offset = addr; + } + + break; + } + + if (!vma) + cookie = INVALID_COOKIE; + + return cookie; +} + +static int cookies_initialize(void) +{ + uint32_t crc, poly; + int i, j, cpu, size, err = 0; + + int translate_buffer_size = 512; // must be a power of 2 + translate_buffer_mask = translate_buffer_size / sizeof(per_cpu(translate_buffer, 0)[0]) - 1; + + for_each_present_cpu(cpu) { + per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu; + + size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t); + per_cpu(cookie_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL); + if (!per_cpu(cookie_keys, cpu)) { + err = -ENOMEM; + goto cookie_setup_error; + } + memset(per_cpu(cookie_keys, cpu), 0, size); + + size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t); + per_cpu(cookie_values, cpu) = (uint32_t*)kmalloc(size, GFP_KERNEL); + if (!per_cpu(cookie_values, cpu)) { + err = -ENOMEM; + goto cookie_setup_error; + } + memset(per_cpu(cookie_values, cpu), 0, size); + + per_cpu(translate_buffer, cpu) = (unsigned int *)kmalloc(translate_buffer_size, GFP_KERNEL); + if (!per_cpu(translate_buffer, cpu)) { + err = -ENOMEM; + goto cookie_setup_error; + } + + per_cpu(translate_buffer_write, cpu) = 0; + per_cpu(translate_buffer_read, cpu) = 0; + + per_cpu(translate_text, cpu) = (char *)kmalloc(TRANSLATE_SIZE, GFP_KERNEL); + if (!per_cpu(translate_text, cpu)) { + err = -ENOMEM; + goto cookie_setup_error; + } + } + + // build CRC32 table + poly = 0x04c11db7; + gator_crc32_table = (uint32_t*)kmalloc(256 * sizeof(uint32_t), GFP_KERNEL); + for (i = 0; i < 256; i++) { + crc = i; + for (j = 8; j > 0; j--) { + if (crc & 1) { + crc = (crc >> 1) ^ poly; + } else { + crc >>= 1; + } + } + gator_crc32_table[i] = crc; + } + +cookie_setup_error: + return err; +} + +static void cookies_release(void) +{ + int cpu; + + for_each_present_cpu(cpu) { + kfree(per_cpu(cookie_keys, cpu)); + per_cpu(cookie_keys, cpu) = NULL; + + kfree(per_cpu(cookie_values, cpu)); + per_cpu(cookie_values, cpu) = NULL; + + kfree(per_cpu(translate_buffer, cpu)); + per_cpu(translate_buffer, cpu) = NULL; + per_cpu(translate_buffer_read, cpu) = 0; + per_cpu(translate_buffer_write, cpu) = 0; + + kfree(per_cpu(translate_text, cpu)); + per_cpu(translate_text, cpu) = NULL; + } + + kfree(gator_crc32_table); + gator_crc32_table = NULL; +} diff --git a/drivers/gator/driver/gator_ebs.c b/drivers/gator/driver/gator_ebs.c new file mode 100644 index 00000000000..8c2997c57da --- /dev/null +++ b/drivers/gator/driver/gator_ebs.c @@ -0,0 +1,192 @@ +/** + * Copyright (C) ARM Limited 2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#if defined(__arm__) && (GATOR_PERF_PMU_SUPPORT) +#include +#include +#include + +#include + +extern int pmnc_counters; +extern int ccnt; +extern unsigned long pmnc_enabled[]; +extern unsigned long pmnc_event[]; +extern unsigned long pmnc_count[]; +extern unsigned long pmnc_key[]; + +static DEFINE_PER_CPU(struct perf_event *, pevent); +static DEFINE_PER_CPU(struct perf_event_attr *, pevent_attr); +static DEFINE_PER_CPU(int, key); +static DEFINE_PER_CPU(unsigned int, prev_value); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) +static void ebs_overflow_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs) +#else +static void ebs_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) +#endif +{ + unsigned int value, delta, cpu = smp_processor_id(), buftype = EVENT_BUF; + + if (event != per_cpu(pevent, cpu)) + return; + + if (buffer_check_space(cpu, buftype, 5 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) { + value = local64_read(&event->count); + delta = value - per_cpu(prev_value, cpu); + per_cpu(prev_value, cpu) = value; + + // Counters header + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type + gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time + + // Output counter + gator_buffer_write_packed_int(cpu, buftype, 2); // length + gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key + gator_buffer_write_packed_int(cpu, buftype, delta); // delta + + // End Counters, length of zero + gator_buffer_write_packed_int(cpu, buftype, 0); + } + + // Output backtrace + if (buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32)) + gator_add_sample(cpu, buftype, regs); + + // Check and commit; commit is set to occur once buffer is 3/4 full + buffer_check(cpu, buftype); +} + +static void gator_event_sampling_online(void) +{ + int cpu = smp_processor_id(), buftype = EVENT_BUF; + + // read the counter and toss the invalid data, return zero instead + struct perf_event * ev = per_cpu(pevent, cpu); + if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { + ev->pmu->read(ev); + per_cpu(prev_value, cpu) = local64_read(&ev->count); + + // Counters header + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); // type + gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); // time + + // Output counter + gator_buffer_write_packed_int(cpu, buftype, 2); // length + gator_buffer_write_packed_int(cpu, buftype, per_cpu(key, cpu)); // key + gator_buffer_write_packed_int(cpu, buftype, 0); // delta - zero for initialization + + // End Counters, length of zero + gator_buffer_write_packed_int(cpu, buftype, 0); + } +} + +static void gator_event_sampling_online_dispatch(int cpu) +{ + struct perf_event * ev; + + if (!event_based_sampling) + return; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) + ev = per_cpu(pevent, cpu) = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu), cpu, 0, ebs_overflow_handler); +#else + ev = per_cpu(pevent, cpu) = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu), cpu, 0, ebs_overflow_handler, 0); +#endif + + if (IS_ERR(ev)) { + pr_err("gator: unable to start event-based-sampling"); + return; + } + + if (ev->state != PERF_EVENT_STATE_ACTIVE) { + pr_err("gator: unable to start event-based-sampling"); + perf_event_release_kernel(ev); + return; + } + + ev->pmu->read(ev); + per_cpu(prev_value, cpu) = local64_read(&ev->count); +} + +static void gator_event_sampling_offline_dispatch(int cpu) +{ + if (per_cpu(pevent, cpu)) { + perf_event_release_kernel(per_cpu(pevent, cpu)); + per_cpu(pevent, cpu) = NULL; + } +} + +static int gator_event_sampling_start(void) +{ + int cnt, event = 0, count = 0, ebs_key = 0, cpu; + + for_each_present_cpu(cpu) { + per_cpu(pevent, cpu) = NULL; + per_cpu(pevent_attr, cpu) = NULL; + } + + event_based_sampling = false; + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (pmnc_count[cnt] > 0) { + event_based_sampling = true; + event = pmnc_event[cnt]; + count = pmnc_count[cnt]; + ebs_key = pmnc_key[cnt]; + break; + } + } + + if (!event_based_sampling) + return 0; + + for_each_present_cpu(cpu) { + u32 size = sizeof(struct perf_event_attr); + per_cpu(pevent_attr, cpu) = kmalloc(size, GFP_KERNEL); + if (!per_cpu(pevent_attr, cpu)) + return -1; + + memset(per_cpu(pevent_attr, cpu), 0, size); + per_cpu(pevent_attr, cpu)->type = PERF_TYPE_RAW; + per_cpu(pevent_attr, cpu)->size = size; + per_cpu(pevent_attr, cpu)->config = event; + per_cpu(pevent_attr, cpu)->sample_period = count; + per_cpu(pevent_attr, cpu)->pinned = 1; + + // handle special case for ccnt + if (cnt == ccnt) { + per_cpu(pevent_attr, cpu)->type = PERF_TYPE_HARDWARE; + per_cpu(pevent_attr, cpu)->config = PERF_COUNT_HW_CPU_CYCLES; + } + + per_cpu(key, cpu) = ebs_key; + } + + return 0; +} + +static void gator_event_sampling_stop(void) +{ + int cpu; + + for_each_present_cpu(cpu) { + if (per_cpu(pevent_attr, cpu)) { + kfree(per_cpu(pevent_attr, cpu)); + per_cpu(pevent_attr, cpu) = NULL; + } + } +} + +#else +static void gator_event_sampling_online(void) {} +static void gator_event_sampling_online_dispatch(int cpu) {} +static void gator_event_sampling_offline_dispatch(int cpu) {} +static int gator_event_sampling_start(void) {return 0;} +static void gator_event_sampling_stop(void) {} +#endif diff --git a/drivers/gator/driver/gator_events.sh b/drivers/gator/driver/gator_events.sh new file mode 100644 index 00000000000..5467dd6d17d --- /dev/null +++ b/drivers/gator/driver/gator_events.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +EVENTS=`grep gator_events_init *.c | sed 's/.\+gator_events_init(\(.\+\)).\+/\1/'` + +( + echo /\* This file is auto generated \*/ + echo + for EVENT in $EVENTS; do + echo __weak int $EVENT\(void\)\; + done + echo + echo static int \(*gator_events_list[]\)\(void\) = { + for EVENT in $EVENTS; do + echo \ $EVENT, + done + echo }\; +) > $1.tmp + +cmp -s $1 $1.tmp && rm $1.tmp || mv $1.tmp $1 diff --git a/drivers/gator/driver/gator_events_armv6.c b/drivers/gator/driver/gator_events_armv6.c new file mode 100644 index 00000000000..ef518987906 --- /dev/null +++ b/drivers/gator/driver/gator_events_armv6.c @@ -0,0 +1,244 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "gator.h" + +// gator_events_perf_pmu.c is used if perf is supported +#if GATOR_NO_PERF_SUPPORT + +static const char *pmnc_name; + +/* + * Per-CPU PMCR + */ +#define PMCR_E (1 << 0) /* Enable */ +#define PMCR_P (1 << 1) /* Count reset */ +#define PMCR_C (1 << 2) /* Cycle counter reset */ +#define PMCR_OFL_PMN0 (1 << 8) /* Count reg 0 overflow */ +#define PMCR_OFL_PMN1 (1 << 9) /* Count reg 1 overflow */ +#define PMCR_OFL_CCNT (1 << 10) /* Cycle counter overflow */ + +#define PMN0 0 +#define PMN1 1 +#define CCNT 2 +#define CNTMAX (CCNT+1) + +static int pmnc_counters = 0; +static unsigned long pmnc_enabled[CNTMAX]; +static unsigned long pmnc_event[CNTMAX]; +static unsigned long pmnc_key[CNTMAX]; + +static DEFINE_PER_CPU(int[CNTMAX], perfPrev); +static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); + +static inline void armv6_pmnc_write(u32 val) +{ + /* upper 4bits and 7, 11 are write-as-0 */ + val &= 0x0ffff77f; + asm volatile("mcr p15, 0, %0, c15, c12, 0" : : "r" (val)); +} + +static inline u32 armv6_pmnc_read(void) +{ + u32 val; + asm volatile("mrc p15, 0, %0, c15, c12, 0" : "=r" (val)); + return val; +} + +static void armv6_pmnc_reset_counter(unsigned int cnt) +{ + u32 val = 0; + switch (cnt) { + case CCNT: + asm volatile("mcr p15, 0, %0, c15, c12, 1" : : "r" (val)); + break; + case PMN0: + asm volatile("mcr p15, 0, %0, c15, c12, 2" : : "r" (val)); + break; + case PMN1: + asm volatile("mcr p15, 0, %0, c15, c12, 3" : : "r" (val)); + break; + } +} + +int gator_events_armv6_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int i; + + pmnc_counters = 3; + + for (i = PMN0; i <= CCNT; i++) { + char buf[40]; + if (i == CCNT) { + snprintf(buf, sizeof buf, "ARM_%s_ccnt", pmnc_name); + } else { + snprintf(buf, sizeof buf, "ARM_%s_cnt%d", pmnc_name, i); + } + dir = gatorfs_mkdir(sb, root, buf); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); + if (i != CCNT) { + gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); + } + } + + return 0; +} + +static int gator_events_armv6_online(int** buffer) +{ + unsigned int cnt, len = 0, cpu = smp_processor_id(); + u32 pmnc; + + if (armv6_pmnc_read() & PMCR_E) { + armv6_pmnc_write(armv6_pmnc_read() & ~PMCR_E); + } + + /* initialize PMNC, reset overflow, D bit, C bit and P bit. */ + armv6_pmnc_write(PMCR_OFL_PMN0 | PMCR_OFL_PMN1 | PMCR_OFL_CCNT | + PMCR_C | PMCR_P); + + /* configure control register */ + for (pmnc = 0, cnt = PMN0; cnt <= CCNT; cnt++) { + unsigned long event; + + per_cpu(perfPrev, cpu)[cnt] = 0; + + if (!pmnc_enabled[cnt]) + continue; + + event = pmnc_event[cnt] & 255; + + // Set event (if destined for PMNx counters) + if (cnt == PMN0) { + pmnc |= event << 20; + } else if (cnt == PMN1) { + pmnc |= event << 12; + } + + // Reset counter + armv6_pmnc_reset_counter(cnt); + } + armv6_pmnc_write(pmnc | PMCR_E); + + // return zero values, no need to read as the counters were just reset + for (cnt = PMN0; cnt <= CCNT; cnt++) { + if (pmnc_enabled[cnt]) { + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = 0; + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; +} + +static int gator_events_armv6_offline(int** buffer) +{ + unsigned int cnt; + + armv6_pmnc_write(armv6_pmnc_read() & ~PMCR_E); + for (cnt = PMN0; cnt <= CCNT; cnt++) { + armv6_pmnc_reset_counter(cnt); + } + + return 0; +} + +static void gator_events_armv6_stop(void) +{ + unsigned int cnt; + + for (cnt = PMN0; cnt <= CCNT; cnt++) { + pmnc_enabled[cnt] = 0; + pmnc_event[cnt] = 0; + } +} + +static int gator_events_armv6_read(int **buffer) +{ + int cnt, len = 0; + int cpu = smp_processor_id(); + + for (cnt = PMN0; cnt <= CCNT; cnt++) { + if (pmnc_enabled[cnt]) { + u32 value = 0; + switch (cnt) { + case CCNT: + asm volatile("mrc p15, 0, %0, c15, c12, 1" : "=r" (value)); + break; + case PMN0: + asm volatile("mrc p15, 0, %0, c15, c12, 2" : "=r" (value)); + break; + case PMN1: + asm volatile("mrc p15, 0, %0, c15, c12, 3" : "=r" (value)); + break; + } + armv6_pmnc_reset_counter(cnt); + if (value != per_cpu(perfPrev, cpu)[cnt]) { + per_cpu(perfPrev, cpu)[cnt] = value; + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = value; + } + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; +} + +static struct gator_interface gator_events_armv6_interface = { + .create_files = gator_events_armv6_create_files, + .stop = gator_events_armv6_stop, + .online = gator_events_armv6_online, + .offline = gator_events_armv6_offline, + .read = gator_events_armv6_read, +}; + +int gator_events_armv6_init(void) +{ + unsigned int cnt; + + switch (gator_cpuid()) { + case ARM1136: + case ARM1156: + case ARM1176: + pmnc_name = "ARM11"; + break; + case ARM11MPCORE: + pmnc_name = "ARM11MPCore"; + break; + default: + return -1; + } + + for (cnt = PMN0; cnt <= CCNT; cnt++) { + pmnc_enabled[cnt] = 0; + pmnc_event[cnt] = 0; + pmnc_key[cnt] = gator_events_get_key(); + } + + return gator_events_install(&gator_events_armv6_interface); +} + +gator_events_init(gator_events_armv6_init); + +#else +int gator_events_armv6_init(void) +{ + return -1; +} +#endif diff --git a/drivers/gator/driver/gator_events_armv7.c b/drivers/gator/driver/gator_events_armv7.c new file mode 100644 index 00000000000..cdf450f1975 --- /dev/null +++ b/drivers/gator/driver/gator_events_armv7.c @@ -0,0 +1,320 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* Disabling interrupts + * Many of the functions below disable interrupts via local_irq_save(). This disabling of interrupts is done to prevent any race conditions + * between multiple entities (e.g. hrtimer interrupts and event based interrupts) calling the same functions. As accessing the pmu involves + * several steps (disable, select, read, enable), these steps must be performed atomically. Normal synchronization routines cannot be used + * as these functions are being called from interrupt context. + */ + +#include "gator.h" + +// gator_events_perf_pmu.c is used if perf is supported +#if GATOR_NO_PERF_SUPPORT + +// Per-CPU PMNC: config reg +#define PMNC_E (1 << 0) /* Enable all counters */ +#define PMNC_P (1 << 1) /* Reset all counters */ +#define PMNC_C (1 << 2) /* Cycle counter reset */ +#define PMNC_MASK 0x3f /* Mask for writable bits */ + +// ccnt reg +#define CCNT_REG (1 << 31) + +#define CCNT 0 +#define CNT0 1 +#define CNTMAX (6+1) + +static const char *pmnc_name; +static int pmnc_counters; + +static unsigned long pmnc_enabled[CNTMAX]; +static unsigned long pmnc_event[CNTMAX]; +static unsigned long pmnc_key[CNTMAX]; + +static DEFINE_PER_CPU(int[CNTMAX], perfPrev); +static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); + +inline void armv7_pmnc_write(u32 val) +{ + val &= PMNC_MASK; + asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r" (val)); +} + +inline u32 armv7_pmnc_read(void) +{ + u32 val; + asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); + return val; +} + +inline u32 armv7_ccnt_read(u32 reset_value) +{ + unsigned long flags; + u32 newval = -reset_value; + u32 den = CCNT_REG; + u32 val; + + local_irq_save(flags); + asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable + asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); // read + asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (newval));// new value + asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable + local_irq_restore(flags); + + return val; +} + +inline u32 armv7_cntn_read(unsigned int cnt, u32 reset_value) +{ + unsigned long flags; + u32 newval = -reset_value; + u32 sel = (cnt - CNT0); + u32 den = 1 << sel; + u32 oldval; + + local_irq_save(flags); + asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (den)); // disable + asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (sel)); // select + asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (oldval)); // read + asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (newval));// new value + asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (den)); // enable + local_irq_restore(flags); + + return oldval; +} + +static inline void armv7_pmnc_disable_interrupt(unsigned int cnt) +{ + u32 val = cnt ? (1 << (cnt - CNT0)) : (1 << 31); + asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val)); +} + +inline u32 armv7_pmnc_reset_interrupt(void) +{ + // Get and reset overflow status flags + u32 flags; + asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (flags)); + flags &= 0x8000003f; + asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (flags)); + return flags; +} + +static inline u32 armv7_pmnc_enable_counter(unsigned int cnt) +{ + u32 val = cnt ? (1 << (cnt - CNT0)) : CCNT_REG; + asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val)); + return cnt; +} + +static inline u32 armv7_pmnc_disable_counter(unsigned int cnt) +{ + u32 val = cnt ? (1 << (cnt - CNT0)) : CCNT_REG; + asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val)); + return cnt; +} + +static inline int armv7_pmnc_select_counter(unsigned int cnt) +{ + u32 val = (cnt - CNT0); + asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val)); + return cnt; +} + +static inline void armv7_pmnc_write_evtsel(unsigned int cnt, u32 val) +{ + if (armv7_pmnc_select_counter(cnt) == cnt) { + asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val)); + } +} + +static int gator_events_armv7_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int i; + + for (i = 0; i < pmnc_counters; i++) { + char buf[40]; + if (i == 0) { + snprintf(buf, sizeof buf, "ARM_%s_ccnt", pmnc_name); + } else { + snprintf(buf, sizeof buf, "ARM_%s_cnt%d", pmnc_name, i-1); + } + dir = gatorfs_mkdir(sb, root, buf); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); + if (i > 0) { + gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); + } + } + + return 0; +} + +static int gator_events_armv7_online(int** buffer) +{ + unsigned int cnt, len = 0, cpu = smp_processor_id(); + + if (armv7_pmnc_read() & PMNC_E) { + armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); + } + + // Initialize & Reset PMNC: C bit and P bit + armv7_pmnc_write(PMNC_P | PMNC_C); + + // Reset overflow flags + armv7_pmnc_reset_interrupt(); + + for (cnt = CCNT; cnt < CNTMAX; cnt++) { + unsigned long event; + + per_cpu(perfPrev, cpu)[cnt] = 0; + + if (!pmnc_enabled[cnt]) + continue; + + // Disable counter + armv7_pmnc_disable_counter(cnt); + + event = pmnc_event[cnt] & 255; + + // Set event (if destined for PMNx counters), we don't need to set the event if it's a cycle count + if (cnt != CCNT) + armv7_pmnc_write_evtsel(cnt, event); + + armv7_pmnc_disable_interrupt(cnt); + + // Reset counter + cnt ? armv7_cntn_read(cnt, 0) : armv7_ccnt_read(0); + + // Enable counter + armv7_pmnc_enable_counter(cnt); + } + + // enable + armv7_pmnc_write(armv7_pmnc_read() | PMNC_E); + + // return zero values, no need to read as the counters were just reset + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (pmnc_enabled[cnt]) { + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = 0; + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; +} + +static int gator_events_armv7_offline(int** buffer) +{ + // disbale all counters, including PMCCNTR; overflow IRQs will not be signaled + armv7_pmnc_write(armv7_pmnc_read() & ~PMNC_E); + + return 0; +} + +static void gator_events_armv7_stop(void) +{ + unsigned int cnt; + + for (cnt = CCNT; cnt < CNTMAX; cnt++) { + pmnc_enabled[cnt] = 0; + pmnc_event[cnt] = 0; + } +} + +static int gator_events_armv7_read(int **buffer) +{ + int cnt, len = 0; + int cpu = smp_processor_id(); + + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (pmnc_enabled[cnt]) { + int value; + if (cnt == CCNT) { + value = armv7_ccnt_read(0); + } else { + value = armv7_cntn_read(cnt, 0); + } + if (value != per_cpu(perfPrev, cpu)[cnt]) { + per_cpu(perfPrev, cpu)[cnt] = value; + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = value; + } + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; +} + +static struct gator_interface gator_events_armv7_interface = { + .create_files = gator_events_armv7_create_files, + .stop = gator_events_armv7_stop, + .online = gator_events_armv7_online, + .offline = gator_events_armv7_offline, + .read = gator_events_armv7_read, +}; + +int gator_events_armv7_init(void) +{ + unsigned int cnt; + + switch (gator_cpuid()) { + case CORTEX_A5: + pmnc_name = "Cortex-A5"; + pmnc_counters = 2; + break; + case CORTEX_A7: + pmnc_name = "Cortex-A7"; + pmnc_counters = 4; + break; + case CORTEX_A8: + pmnc_name = "Cortex-A8"; + pmnc_counters = 4; + break; + case CORTEX_A9: + pmnc_name = "Cortex-A9"; + pmnc_counters = 6; + break; + case CORTEX_A15: + pmnc_name = "Cortex-A15"; + pmnc_counters = 6; + break; + default: + return -1; + } + + pmnc_counters++; // CNT[n] + CCNT + + for (cnt = CCNT; cnt < CNTMAX; cnt++) { + pmnc_enabled[cnt] = 0; + pmnc_event[cnt] = 0; + pmnc_key[cnt] = gator_events_get_key(); + } + + return gator_events_install(&gator_events_armv7_interface); +} + +gator_events_init(gator_events_armv7_init); + +#else +int gator_events_armv7_init(void) +{ + return -1; +} +#endif diff --git a/drivers/gator/driver/gator_events_block.c b/drivers/gator/driver/gator_events_block.c new file mode 100644 index 00000000000..f1bbbc83bfb --- /dev/null +++ b/drivers/gator/driver/gator_events_block.c @@ -0,0 +1,171 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" +#include + +#define BLOCK_RQ_WR 0 +#define BLOCK_RQ_RD 1 + +#define BLOCK_TOTAL (BLOCK_RQ_RD+1) + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 36) +#define EVENTWRITE REQ_RW +#else +#define EVENTWRITE REQ_WRITE +#endif + +static ulong block_rq_wr_enabled; +static ulong block_rq_rd_enabled; +static ulong block_rq_wr_key; +static ulong block_rq_rd_key; +static DEFINE_PER_CPU(int[BLOCK_TOTAL], blockCnt); +static DEFINE_PER_CPU(int[BLOCK_TOTAL * 2], blockGet); +static DEFINE_PER_CPU(bool, new_data_avail); + +GATOR_DEFINE_PROBE(block_rq_complete, TP_PROTO(struct request_queue *q, struct request *rq)) +{ + unsigned long flags; + int write, size; + int cpu = smp_processor_id(); + + if (!rq) + return; + + write = rq->cmd_flags & EVENTWRITE; + size = rq->resid_len; + + if (!size) + return; + + // disable interrupts to synchronize with gator_events_block_read() + // spinlocks not needed since percpu buffers are used + local_irq_save(flags); + if (write) + per_cpu(blockCnt, cpu)[BLOCK_RQ_WR] += size; + else + per_cpu(blockCnt, cpu)[BLOCK_RQ_RD] += size; + local_irq_restore(flags); + + per_cpu(new_data_avail, cpu) = true; +} + +static int gator_events_block_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + + /* block_complete_wr */ + dir = gatorfs_mkdir(sb, root, "Linux_block_rq_wr"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &block_rq_wr_enabled); + gatorfs_create_ro_ulong(sb, dir, "key", &block_rq_wr_key); + + /* block_complete_rd */ + dir = gatorfs_mkdir(sb, root, "Linux_block_rq_rd"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &block_rq_rd_enabled); + gatorfs_create_ro_ulong(sb, dir, "key", &block_rq_rd_key); + + return 0; +} + +static int gator_events_block_start(void) +{ + int cpu; + + for_each_present_cpu(cpu) + per_cpu(new_data_avail, cpu) = true; + + // register tracepoints + if (block_rq_wr_enabled || block_rq_rd_enabled) + if (GATOR_REGISTER_TRACE(block_rq_complete)) + goto fail_block_rq_exit; + pr_debug("gator: registered block event tracepoints\n"); + + return 0; + + // unregister tracepoints on error +fail_block_rq_exit: + pr_err("gator: block event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +static void gator_events_block_stop(void) +{ + if (block_rq_wr_enabled || block_rq_rd_enabled) + GATOR_UNREGISTER_TRACE(block_rq_complete); + pr_debug("gator: unregistered block event tracepoints\n"); + + block_rq_wr_enabled = 0; + block_rq_rd_enabled = 0; +} + +static int gator_events_block_read(int **buffer) +{ + unsigned long flags; + int len, value, cpu, data = 0; + cpu = smp_processor_id(); + + if (per_cpu(new_data_avail, cpu) == false) + return 0; + + per_cpu(new_data_avail, cpu) = false; + + len = 0; + if (block_rq_wr_enabled) { + local_irq_save(flags); + value = per_cpu(blockCnt, cpu)[BLOCK_RQ_WR]; + per_cpu(blockCnt, cpu)[BLOCK_RQ_WR] = 0; + local_irq_restore(flags); + per_cpu(blockGet, cpu)[len++] = block_rq_wr_key; + per_cpu(blockGet, cpu)[len++] = value; + data += value; + } + if (block_rq_rd_enabled) { + local_irq_save(flags); + value = per_cpu(blockCnt, cpu)[BLOCK_RQ_RD]; + per_cpu(blockCnt, cpu)[BLOCK_RQ_RD] = 0; + local_irq_restore(flags); + per_cpu(blockGet, cpu)[len++] = block_rq_rd_key; + per_cpu(blockGet, cpu)[len++] = value; + data += value; + } + + if (data != 0) + per_cpu(new_data_avail, cpu) = true; + + if (buffer) + *buffer = per_cpu(blockGet, cpu); + + return len; +} + +static struct gator_interface gator_events_block_interface = { + .create_files = gator_events_block_create_files, + .start = gator_events_block_start, + .stop = gator_events_block_stop, + .read = gator_events_block_read, +}; + +int gator_events_block_init(void) +{ + block_rq_wr_enabled = 0; + block_rq_rd_enabled = 0; + + block_rq_wr_key = gator_events_get_key(); + block_rq_rd_key = gator_events_get_key(); + + return gator_events_install(&gator_events_block_interface); +} +gator_events_init(gator_events_block_init); diff --git a/drivers/gator/driver/gator_events_irq.c b/drivers/gator/driver/gator_events_irq.c new file mode 100644 index 00000000000..59461b9799e --- /dev/null +++ b/drivers/gator/driver/gator_events_irq.c @@ -0,0 +1,195 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" +#include + +#define HARDIRQ 0 +#define SOFTIRQ 1 +#define TOTALIRQ (SOFTIRQ+1) + +static ulong hardirq_enabled; +static ulong softirq_enabled; +static ulong hardirq_key; +static ulong softirq_key; +static DEFINE_PER_CPU(int[TOTALIRQ], irqCnt); +static DEFINE_PER_CPU(int[TOTALIRQ], irqPrev); +static DEFINE_PER_CPU(int[TOTALIRQ * 2], irqGet); + +GATOR_DEFINE_PROBE(irq_handler_exit, TP_PROTO(int irq, + struct irqaction *action, int ret)) +{ + unsigned long flags; + + // disable interrupts to synchronize with gator_events_irq_read() + // spinlocks not needed since percpu buffers are used + local_irq_save(flags); + per_cpu(irqCnt, smp_processor_id())[HARDIRQ]++; + local_irq_restore(flags); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37) +GATOR_DEFINE_PROBE(softirq_exit, TP_PROTO(struct softirq_action *h, struct softirq_action *vec)) +#else +GATOR_DEFINE_PROBE(softirq_exit, TP_PROTO(unsigned int vec_nr)) +#endif +{ + unsigned long flags; + + // disable interrupts to synchronize with gator_events_irq_read() + // spinlocks not needed since percpu buffers are used + local_irq_save(flags); + per_cpu(irqCnt, smp_processor_id())[SOFTIRQ]++; + local_irq_restore(flags); +} + +static int gator_events_irq_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + + /* irq */ + dir = gatorfs_mkdir(sb, root, "Linux_irq_irq"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &hardirq_enabled); + gatorfs_create_ro_ulong(sb, dir, "key", &hardirq_key); + + /* soft irq */ + dir = gatorfs_mkdir(sb, root, "Linux_irq_softirq"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &softirq_enabled); + gatorfs_create_ro_ulong(sb, dir, "key", &softirq_key); + + return 0; +} + +static int gator_events_irq_online(int** buffer) +{ + int len = 0, cpu = smp_processor_id(); + unsigned long flags; // not necessary as we are in interrupt context anyway, but doesn't hurt + + // synchronization with the irq_exit functions is not necessary as the values are being reset + if (hardirq_enabled) { + local_irq_save(flags); + per_cpu(irqCnt, cpu)[HARDIRQ] = 0; + local_irq_restore(flags); + per_cpu(irqPrev, cpu)[HARDIRQ] = 0; + per_cpu(irqGet, cpu)[len++] = hardirq_key; + per_cpu(irqGet, cpu)[len++] = 0; + } + + if (softirq_enabled) { + local_irq_save(flags); + per_cpu(irqCnt, cpu)[SOFTIRQ] = 0; + local_irq_restore(flags); + per_cpu(irqPrev, cpu)[SOFTIRQ] = 0; + per_cpu(irqGet, cpu)[len++] = softirq_key; + per_cpu(irqGet, cpu)[len++] = 0; + } + + if (buffer) + *buffer = per_cpu(irqGet, cpu); + + return len; +} + +static int gator_events_irq_start(void) +{ + // register tracepoints + if (hardirq_enabled) + if (GATOR_REGISTER_TRACE(irq_handler_exit)) + goto fail_hardirq_exit; + if (softirq_enabled) + if (GATOR_REGISTER_TRACE(softirq_exit)) + goto fail_softirq_exit; + pr_debug("gator: registered irq tracepoints\n"); + + return 0; + + // unregister tracepoints on error +fail_softirq_exit: + if (hardirq_enabled) + GATOR_UNREGISTER_TRACE(irq_handler_exit); +fail_hardirq_exit: + pr_err("gator: irq tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +static void gator_events_irq_stop(void) +{ + if (hardirq_enabled) + GATOR_UNREGISTER_TRACE(irq_handler_exit); + if (softirq_enabled) + GATOR_UNREGISTER_TRACE(softirq_exit); + pr_debug("gator: unregistered irq tracepoints\n"); + + hardirq_enabled = 0; + softirq_enabled = 0; +} + +static int gator_events_irq_read(int **buffer) +{ + unsigned long flags; // not necessary as we are in interrupt context anyway, but doesn't hurt + int len, value; + int cpu = smp_processor_id(); + + len = 0; + if (hardirq_enabled) { + local_irq_save(flags); + value = per_cpu(irqCnt, cpu)[HARDIRQ]; + per_cpu(irqCnt, cpu)[HARDIRQ] = 0; + local_irq_restore(flags); + if (value != per_cpu(irqPrev, cpu)[HARDIRQ]) { + per_cpu(irqPrev, cpu)[HARDIRQ] = value; + per_cpu(irqGet, cpu)[len++] = hardirq_key; + per_cpu(irqGet, cpu)[len++] = value; + } + } + + if (softirq_enabled) { + local_irq_save(flags); + value = per_cpu(irqCnt, cpu)[SOFTIRQ]; + per_cpu(irqCnt, cpu)[SOFTIRQ] = 0; + local_irq_restore(flags); + if (value != per_cpu(irqPrev, cpu)[SOFTIRQ]) { + per_cpu(irqPrev, cpu)[SOFTIRQ] = value; + per_cpu(irqGet, cpu)[len++] = softirq_key; + per_cpu(irqGet, cpu)[len++] = value; + } + } + + if (buffer) + *buffer = per_cpu(irqGet, cpu); + + return len; +} + +static struct gator_interface gator_events_irq_interface = { + .create_files = gator_events_irq_create_files, + .online = gator_events_irq_online, + .start = gator_events_irq_start, + .stop = gator_events_irq_stop, + .read = gator_events_irq_read, +}; + +int gator_events_irq_init(void) +{ + hardirq_key = gator_events_get_key(); + softirq_key = gator_events_get_key(); + + hardirq_enabled = 0; + softirq_enabled = 0; + + return gator_events_install(&gator_events_irq_interface); +} +gator_events_init(gator_events_irq_init); diff --git a/drivers/gator/driver/gator_events_l2c-310.c b/drivers/gator/driver/gator_events_l2c-310.c new file mode 100644 index 00000000000..bd1c48a64e2 --- /dev/null +++ b/drivers/gator/driver/gator_events_l2c-310.c @@ -0,0 +1,179 @@ +/** + * l2c310 (L2 Cache Controller) event counters for gator + * + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include + +#include "gator.h" + +#define L2C310_COUNTERS_NUM 2 + +static struct { + unsigned long enabled; + unsigned long event; + unsigned long key; +} l2c310_counters[L2C310_COUNTERS_NUM]; + +static int l2c310_buffer[L2C310_COUNTERS_NUM * 2]; + +static void __iomem *l2c310_base; + + + +static void gator_events_l2c310_reset_counters(void) +{ + u32 val = readl(l2c310_base + L2X0_EVENT_CNT_CTRL); + + val |= ((1 << L2C310_COUNTERS_NUM) - 1) << 1; + + writel(val, l2c310_base + L2X0_EVENT_CNT_CTRL); +} + + +static int gator_events_l2c310_create_files(struct super_block *sb, + struct dentry *root) +{ + int i; + + for (i = 0; i < L2C310_COUNTERS_NUM; i++) { + char buf[16]; + struct dentry *dir; + + snprintf(buf, sizeof(buf), "L2C-310_cnt%d", i); + dir = gatorfs_mkdir(sb, root, buf); + if (WARN_ON(!dir)) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", + &l2c310_counters[i].enabled); + gatorfs_create_ulong(sb, dir, "event", + &l2c310_counters[i].event); + gatorfs_create_ro_ulong(sb, dir, "key", + &l2c310_counters[i].key); + } + + return 0; +} + +static int gator_events_l2c310_start(void) +{ + static const unsigned long l2x0_event_cntx_cfg[L2C310_COUNTERS_NUM] = { + L2X0_EVENT_CNT0_CFG, + L2X0_EVENT_CNT1_CFG, + }; + int i; + + /* Counter event sources */ + for (i = 0; i < L2C310_COUNTERS_NUM; i++) + writel((l2c310_counters[i].event & 0xf) << 2, + l2c310_base + l2x0_event_cntx_cfg[i]); + + gator_events_l2c310_reset_counters(); + + /* Event counter enable */ + writel(1, l2c310_base + L2X0_EVENT_CNT_CTRL); + + return 0; +} + +static void gator_events_l2c310_stop(void) +{ + /* Event counter disable */ + writel(0, l2c310_base + L2X0_EVENT_CNT_CTRL); +} + +static int gator_events_l2c310_read(int **buffer) +{ + static const unsigned long l2x0_event_cntx_val[L2C310_COUNTERS_NUM] = { + L2X0_EVENT_CNT0_VAL, + L2X0_EVENT_CNT1_VAL, + }; + int i; + int len = 0; + + if (smp_processor_id()) + return 0; + + for (i = 0; i < L2C310_COUNTERS_NUM; i++) { + if (l2c310_counters[i].enabled) { + l2c310_buffer[len++] = l2c310_counters[i].key; + l2c310_buffer[len++] = readl(l2c310_base + + l2x0_event_cntx_val[i]); + } + } + + /* l2c310 counters are saturating, not wrapping in case of overflow */ + gator_events_l2c310_reset_counters(); + + if (buffer) + *buffer = l2c310_buffer; + + return len; +} + +static struct gator_interface gator_events_l2c310_interface = { + .create_files = gator_events_l2c310_create_files, + .start = gator_events_l2c310_start, + .stop = gator_events_l2c310_stop, + .read = gator_events_l2c310_read, +}; + +static void __maybe_unused gator_events_l2c310_probe(unsigned long phys) +{ + if (l2c310_base) + return; + + l2c310_base = ioremap(phys, SZ_4K); + if (l2c310_base) { + u32 cache_id = readl(l2c310_base + L2X0_CACHE_ID); + + if ((cache_id & 0xff0003c0) != 0x410000c0) { + iounmap(l2c310_base); + l2c310_base = NULL; + } + } +} + +int gator_events_l2c310_init(void) +{ + int i; + + if (gator_cpuid() != CORTEX_A5 && gator_cpuid() != CORTEX_A9) + return -1; + +#if defined(CONFIG_ARCH_EXYNOS4) || defined(CONFIG_ARCH_S5PV310) + gator_events_l2c310_probe(0x10502000); +#endif +#if defined(CONFIG_ARCH_OMAP4) + gator_events_l2c310_probe(0x48242000); +#endif +#if defined(CONFIG_ARCH_TEGRA) + gator_events_l2c310_probe(0x50043000); +#endif +#if defined(CONFIG_ARCH_U8500) + gator_events_l2c310_probe(0xa0412000); +#endif +#if defined(CONFIG_ARCH_VEXPRESS) + // A9x4 core tile (HBI-0191) + gator_events_l2c310_probe(0x1e00a000); + // New memory map tiles + gator_events_l2c310_probe(0x2c0f0000); +#endif + if (!l2c310_base) + return -1; + + for (i = 0; i < L2C310_COUNTERS_NUM; i++) { + l2c310_counters[i].enabled = 0; + l2c310_counters[i].key = gator_events_get_key(); + } + + return gator_events_install(&gator_events_l2c310_interface); +} +gator_events_init(gator_events_l2c310_init); diff --git a/drivers/gator/driver/gator_events_mali.c b/drivers/gator/driver/gator_events_mali.c new file mode 100644 index 00000000000..31e8f0d9030 --- /dev/null +++ b/drivers/gator/driver/gator_events_mali.c @@ -0,0 +1,611 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "gator.h" + +#include +#include +#include + +#include "linux/mali_linux_trace.h" + +#define ACTIVITY_START 1 +#define ACTIVITY_STOP 2 + +#ifndef MALI_SUPPORT +#error MALI_SUPPORT not defined! +#endif + +#define MALI_200 0x0a07 +#define MALI_300 0x0b06 //This is not actually true; Mali-300 is also 0x0b07 +#define MALI_400 0x0b07 +#define MALI_T6xx 0x0056 + +static const char *mali_name; + +enum counters { + /* Timeline activity */ + ACTIVITY_VP = 0, + ACTIVITY_FP0, + ACTIVITY_FP1, + ACTIVITY_FP2, + ACTIVITY_FP3, + + /* L2 cache counters */ + COUNTER_L2_C0, + COUNTER_L2_C1, + + /* Vertex processor counters */ + COUNTER_VP_C0, + COUNTER_VP_C1, + + /* Fragment processor counters */ + COUNTER_FP0_C0, + COUNTER_FP0_C1, + COUNTER_FP1_C0, + COUNTER_FP1_C1, + COUNTER_FP2_C0, + COUNTER_FP2_C1, + COUNTER_FP3_C0, + COUNTER_FP3_C1, + + /* EGL Software Counters */ + COUNTER_EGL_BLIT_TIME, + + /* GLES Software Counters */ + COUNTER_GLES_DRAW_ELEMENTS_CALLS, + COUNTER_GLES_DRAW_ELEMENTS_NUM_INDICES, + COUNTER_GLES_DRAW_ELEMENTS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_ARRAYS_CALLS, + COUNTER_GLES_DRAW_ARRAYS_NUM_TRANSFORMED, + COUNTER_GLES_DRAW_POINTS, + COUNTER_GLES_DRAW_LINES, + COUNTER_GLES_DRAW_LINE_LOOP, + COUNTER_GLES_DRAW_LINE_STRIP, + COUNTER_GLES_DRAW_TRIANGLES, + COUNTER_GLES_DRAW_TRIANGLE_STRIP, + COUNTER_GLES_DRAW_TRIANGLE_FAN, + COUNTER_GLES_NON_VBO_DATA_COPY_TIME, + COUNTER_GLES_UNIFORM_BYTES_COPIED_TO_MALI, + COUNTER_GLES_UPLOAD_TEXTURE_TIME, + COUNTER_GLES_UPLOAD_VBO_TIME, + COUNTER_GLES_NUM_FLUSHES, + COUNTER_GLES_NUM_VSHADERS_GENERATED, + COUNTER_GLES_NUM_FSHADERS_GENERATED, + COUNTER_GLES_VSHADER_GEN_TIME, + COUNTER_GLES_FSHADER_GEN_TIME, + COUNTER_GLES_INPUT_TRIANGLES, + COUNTER_GLES_VXCACHE_HIT, + COUNTER_GLES_VXCACHE_MISS, + COUNTER_GLES_VXCACHE_COLLISION, + COUNTER_GLES_CULLED_TRIANGLES, + COUNTER_GLES_CULLED_LINES, + COUNTER_GLES_BACKFACE_TRIANGLES, + COUNTER_GLES_GBCLIP_TRIANGLES, + COUNTER_GLES_GBCLIP_LINES, + COUNTER_GLES_TRIANGLES_DRAWN, + COUNTER_GLES_DRAWCALL_TIME, + COUNTER_GLES_TRIANGLES_COUNT, + COUNTER_GLES_INDEPENDENT_TRIANGLES_COUNT, + COUNTER_GLES_STRIP_TRIANGLES_COUNT, + COUNTER_GLES_FAN_TRIANGLES_COUNT, + COUNTER_GLES_LINES_COUNT, + COUNTER_GLES_INDEPENDENT_LINES_COUNT, + COUNTER_GLES_STRIP_LINES_COUNT, + COUNTER_GLES_LOOP_LINES_COUNT, + + COUNTER_FILMSTRIP, + + NUMBER_OF_EVENTS +}; + +#define FIRST_ACTIVITY_EVENT ACTIVITY_VP +#define LAST_ACTIVITY_EVENT ACTIVITY_FP3 + +#define FIRST_HW_COUNTER COUNTER_L2_C0 +#define LAST_HW_COUNTER COUNTER_FP3_C1 + +#define FIRST_SW_COUNTER COUNTER_EGL_BLIT_TIME +#define LAST_SW_COUNTER COUNTER_GLES_LOOP_LINES_COUNT + +#define FIRST_SPECIAL_COUNTER COUNTER_FILMSTRIP +#define LAST_SPECIAL_COUNTER COUNTER_FILMSTRIP + +/* gatorfs variables for counter enable state, + * the event the counter should count and the + * 'key' (a unique id set by gatord and returned + * by gator.ko) + */ +static unsigned long counter_enabled[NUMBER_OF_EVENTS]; +static unsigned long counter_event[NUMBER_OF_EVENTS]; +static unsigned long counter_key[NUMBER_OF_EVENTS]; + +/* The data we have recorded */ +static u32 counter_data[NUMBER_OF_EVENTS]; +/* The address to sample (or 0 if samples are sent to us) */ +static u32* counter_address[NUMBER_OF_EVENTS]; + +/* An array used to return the data we recorded + * as key,value pairs hence the *2 + */ +static unsigned long counter_dump[NUMBER_OF_EVENTS * 2]; +static unsigned long counter_prev[NUMBER_OF_EVENTS]; + +/* Note whether tracepoints have been registered */ +static int trace_registered; + +/** + * Calculate the difference and handle the overflow. + */ +static u32 get_difference(u32 start, u32 end) +{ + if (start - end >= 0) + { + return start - end; + } + + // Mali counters are unsigned 32 bit values that wrap. + return (4294967295u - end) + start; +} + +/** + * Returns non-zero if the given counter ID is an activity counter. + */ +static inline int is_activity_counter(unsigned int event_id) +{ + return (event_id >= FIRST_ACTIVITY_EVENT && + event_id <= LAST_ACTIVITY_EVENT); +} + +/** + * Returns non-zero if the given counter ID is a hardware counter. + */ +static inline int is_hw_counter(unsigned int event_id) +{ + return (event_id >= FIRST_HW_COUNTER && event_id <= LAST_HW_COUNTER); +} + +/** + * Returns non-zero if the given counter ID is a software counter. + */ +static inline int is_sw_counter(unsigned int event_id) +{ + return (event_id >= FIRST_SW_COUNTER && event_id <= LAST_SW_COUNTER); +} + +/* + * The Mali DDK uses s64 types to contain software counter values, but gator + * can only use a maximum of 32 bits. This function scales a software counter + * to an appopriate range. + */ +static u32 scale_sw_counter_value(unsigned int event_id, signed long long value) +{ + u32 scaled_value; + + switch (event_id) { + case COUNTER_GLES_UPLOAD_TEXTURE_TIME: + case COUNTER_GLES_UPLOAD_VBO_TIME: + scaled_value = (u32)div_s64(value, 1000000); + break; + default: + scaled_value = (u32)value; + break; + } + + return scaled_value; +} + +/* Probe for continuously sampled counter */ +#if 0 //WE_DONT_CURRENTLY_USE_THIS_SO_SUPPRESS_WARNING +GATOR_DEFINE_PROBE(mali_sample_address, TP_PROTO(unsigned int event_id, u32* addr)) +{ + /* Turning on too many pr_debug statements in frequently called functions + * can cause stability and/or performance problems + */ + //pr_debug("gator: mali_sample_address %d %d\n", event_id, addr); + if (event_id >= ACTIVITY_VP && event_id <= COUNTER_FP3_C1) { + counter_address[event_id] = addr; + } +} +#endif + +/* Probe for hardware counter events */ +GATOR_DEFINE_PROBE(mali_hw_counter, TP_PROTO(unsigned int event_id, unsigned int value)) +{ + /* Turning on too many pr_debug statements in frequently called functions + * can cause stability and/or performance problems + */ + //pr_debug("gator: mali_hw_counter %d %d\n", event_id, value); + if (is_hw_counter(event_id)) { + counter_data[event_id] = value; + } +} + +GATOR_DEFINE_PROBE(mali_sw_counter, TP_PROTO(unsigned int event_id, signed long long value)) +{ + if (is_sw_counter(event_id)) { + counter_data[event_id] = scale_sw_counter_value(event_id, value); + } +} + +//TODO need to work out how many fp units we have +u32 gator_mali_get_n_fp(void) { + return 4; +} + +//TODO need to work out what kind of Mali we are looking at +u32 gator_mali_get_id(void) { + return MALI_SUPPORT; +} + +int gator_events_mali_create_files(struct super_block *sb, struct dentry *root) { + struct dentry *dir; + int event; + int n_fp = gator_mali_get_n_fp(); + + /* + * Create the filesystem entries for vertex processor, fragement processor + * and L2 cache timeline and hardware counters. Software counters get + * special handling after this block. + */ + for (event = FIRST_ACTIVITY_EVENT; event <= LAST_HW_COUNTER; event++) + { + char buf[40]; + + /* + * We can skip this event if it's for a non-existent fragment + * processor. + */ + if (((event - ACTIVITY_FP0 >= n_fp) && (event < COUNTER_L2_C0)) || + (((event - COUNTER_FP0_C0)/2 >= n_fp))) + { + continue; + } + + /* Otherwise, set up the filesystem entry for this event. */ + switch (event) { + case ACTIVITY_VP: + snprintf(buf, sizeof buf, "ARM_%s_VP_active", mali_name); + break; + case ACTIVITY_FP0: + case ACTIVITY_FP1: + case ACTIVITY_FP2: + case ACTIVITY_FP3: + snprintf(buf, sizeof buf, "ARM_%s_FP%d_active", + mali_name, event - ACTIVITY_FP0); + break; + case COUNTER_L2_C0: + case COUNTER_L2_C1: + snprintf(buf, sizeof buf, "ARM_%s_L2_cnt%d", + mali_name, event - COUNTER_L2_C0); + break; + case COUNTER_VP_C0: + case COUNTER_VP_C1: + snprintf(buf, sizeof buf, "ARM_%s_VP_cnt%d", + mali_name, event - COUNTER_VP_C0); + break; + case COUNTER_FP0_C0: + case COUNTER_FP0_C1: + case COUNTER_FP1_C0: + case COUNTER_FP1_C1: + case COUNTER_FP2_C0: + case COUNTER_FP2_C1: + case COUNTER_FP3_C0: + case COUNTER_FP3_C1: + snprintf(buf, sizeof buf, "ARM_%s_FP%d_cnt%d", mali_name, + (event - COUNTER_FP0_C0) / 2, (event - COUNTER_FP0_C0) % 2); + break; + default: + printk("gator: trying to create file for non-existent counter (%d)\n", event); + continue; + } + + dir = gatorfs_mkdir(sb, root, buf); + + if (!dir) { + return -1; + } + + gatorfs_create_ulong(sb, dir, "enabled", &counter_enabled[event]); + + /* Only create an event node for counters that can change what they count */ + if (event >= COUNTER_L2_C0) { + gatorfs_create_ulong(sb, dir, "event", &counter_event[event]); + } + + gatorfs_create_ro_ulong(sb, dir, "key", &counter_key[event]); + } + + /* Now set up the software counter entries */ + for (event = FIRST_SW_COUNTER; event <= LAST_SW_COUNTER; event++) + { + char buf[40]; + + snprintf(buf, sizeof(buf), "ARM_%s_SW_%d", mali_name, event); + + dir = gatorfs_mkdir(sb, root, buf); + + if (!dir) { + return -1; + } + + gatorfs_create_ulong(sb, dir, "enabled", &counter_enabled[event]); + gatorfs_create_ro_ulong(sb, dir, "key", &counter_key[event]); + } + + /* Now set up the special counter entries */ + for (event = FIRST_SPECIAL_COUNTER; event <= LAST_SPECIAL_COUNTER; event++) + { + char buf[40]; + + snprintf(buf, sizeof(buf), "ARM_%s_Filmstrip", mali_name); + + dir = gatorfs_mkdir(sb, root, buf); + + if (!dir) { + return -1; + } + + gatorfs_create_ulong(sb, dir, "event", &counter_event[event]); + gatorfs_create_ulong(sb, dir, "enabled", &counter_enabled[event]); + gatorfs_create_ro_ulong(sb, dir, "key", &counter_key[event]); + } + + + return 0; +} + +//TODO +void _mali_profiling_set_event(unsigned int, unsigned int); +void _mali_osk_fb_control_set(unsigned int, unsigned int); + +void _mali_profiling_get_counters(unsigned int*, unsigned int*, unsigned int*, unsigned int*); +void (*_mali_profiling_get_counters_function_pointer)(unsigned int*, unsigned int*, unsigned int*, unsigned int*); + +static void mali_counter_initialize(void) +{ + /* If a Mali driver is present and exporting the appropriate symbol + * then we can request the HW counters (of which there are only 2) + * be configured to count the desired events + */ + void (*set_hw_event)(unsigned int, unsigned int); + void (*set_fb_event)(unsigned int, unsigned int); + + set_hw_event = symbol_get(_mali_profiling_set_event); + + if (set_hw_event) { + int i; + + pr_debug("gator: mali online _mali_profiling_set_event symbol @ %p\n",set_hw_event); + + for (i = FIRST_HW_COUNTER; i <= LAST_HW_COUNTER; i++) { + if (counter_enabled[i]) { + set_hw_event(i, counter_event[i]); + } else { + set_hw_event(i, 0xFFFFFFFF); + } + } + + symbol_put(_mali_profiling_set_event); + } else { + printk("gator: mali online _mali_profiling_set_event symbol not found\n"); + } + + set_fb_event = symbol_get(_mali_osk_fb_control_set); + + if (set_fb_event) { + pr_debug("gator: mali online _mali_osk_fb_control_set symbol @ %p\n", set_fb_event); + + set_fb_event(0,(counter_enabled[COUNTER_FILMSTRIP]?1:0)); + + symbol_put(_mali_osk_fb_control_set); + } else { + printk("gator: mali online _mali_osk_fb_control_set symbol not found\n"); + } + + _mali_profiling_get_counters_function_pointer = symbol_get(_mali_profiling_get_counters); + if (_mali_profiling_get_counters_function_pointer){ + pr_debug("gator: mali online _mali_profiling_get_counters symbol @ %p\n", _mali_profiling_get_counters_function_pointer); + counter_prev[COUNTER_L2_C0] = 0; + counter_prev[COUNTER_L2_C1] = 0; + } + else{ + pr_debug("gator WARNING: mali _mali_profiling_get_counters symbol not defined"); + } + +} + +static void mali_counter_deinitialize(void) +{ + void (*set_hw_event)(unsigned int, unsigned int); + void (*set_fb_event)(unsigned int, unsigned int); + + set_hw_event = symbol_get(_mali_profiling_set_event); + + if (set_hw_event) { + int i; + + pr_debug("gator: mali offline _mali_profiling_set_event symbol @ %p\n",set_hw_event); + for (i = FIRST_HW_COUNTER; i <= LAST_HW_COUNTER; i++) { + set_hw_event(i, 0xFFFFFFFF); + } + + symbol_put(_mali_profiling_set_event); + } else { + printk("gator: mali offline _mali_profiling_set_event symbol not found\n"); + } + + set_fb_event = symbol_get(_mali_osk_fb_control_set); + + if (set_fb_event) { + pr_debug("gator: mali offline _mali_osk_fb_control_set symbol @ %p\n", set_fb_event); + + set_fb_event(0,0); + + symbol_put(_mali_osk_fb_control_set); + } else { + printk("gator: mali offline _mali_osk_fb_control_set symbol not found\n"); + } + + if (_mali_profiling_get_counters_function_pointer){ + symbol_put(_mali_profiling_get_counters); + } + +} + +static int gator_events_mali_start(void) { + // register tracepoints + if (GATOR_REGISTER_TRACE(mali_hw_counter)) { + printk("gator: mali_hw_counter tracepoint failed to activate\n"); + return -1; + } + + if (GATOR_REGISTER_TRACE(mali_sw_counter)) { + printk("gator: mali_sw_counter tracepoint failed to activate\n"); + return -1; + } + + trace_registered = 1; + + mali_counter_initialize(); + return 0; +} + +static void gator_events_mali_stop(void) { + unsigned int cnt; + + pr_debug("gator: mali stop\n"); + + if (trace_registered) { + GATOR_UNREGISTER_TRACE(mali_hw_counter); + GATOR_UNREGISTER_TRACE(mali_sw_counter); + + pr_debug("gator: mali timeline tracepoint deactivated\n"); + + trace_registered = 0; + } + + for (cnt = FIRST_ACTIVITY_EVENT; cnt < NUMBER_OF_EVENTS; cnt++) { + counter_enabled[cnt] = 0; + counter_event[cnt] = 0; + counter_address[cnt] = NULL; + } + + mali_counter_deinitialize(); +} + +static int gator_events_mali_read(int **buffer) { + int cnt, len = 0; + + if (smp_processor_id()) return 0; + + // Read the L2 C0 and C1 here. + if (counter_enabled[COUNTER_L2_C0] || counter_enabled[COUNTER_L2_C1] ) { + u32 src0 = 0; + u32 val0 = 0; + u32 src1 = 0; + u32 val1 = 0; + + // Poke the driver to get the counter values + if (_mali_profiling_get_counters_function_pointer){ + _mali_profiling_get_counters_function_pointer(&src0, &val0, &src1, &val1); + } + + if (counter_enabled[COUNTER_L2_C0]) + { + // Calculate and save src0's counter val0 + counter_dump[len++] = counter_key[COUNTER_L2_C0]; + counter_dump[len++] = get_difference(val0, counter_prev[COUNTER_L2_C0]); + } + + if (counter_enabled[COUNTER_L2_C1]) + { + // Calculate and save src1's counter val1 + counter_dump[len++] = counter_key[COUNTER_L2_C1]; + counter_dump[len++] = get_difference(val1, counter_prev[COUNTER_L2_C1]); + } + + // Save the previous values for the counters. + counter_prev[COUNTER_L2_C0] = val0; + counter_prev[COUNTER_L2_C1] = val1; + } + + // Process other (non-timeline) counters. + for (cnt = COUNTER_VP_C0; cnt <= LAST_SW_COUNTER; cnt++) { + if (counter_enabled[cnt]) { + u32 value = 0; + + // Determine the current value of the counter. + if( counter_address[cnt] != NULL && 0 ) { // Never true! + value = *counter_address[cnt]; + } else if (counter_data[cnt]!=0) { + value = counter_data[cnt]; + counter_data[cnt] = 0; + } + + // Send the counter value only if it differs from last time. + if (value != counter_prev[cnt]) { + counter_prev[cnt] = value; + counter_dump[len++] = counter_key[cnt]; + counter_dump[len++] = value; + } + } + } + + if (buffer) { + *buffer = (int*) counter_dump; + } + + return len; +} + +static struct gator_interface gator_events_mali_interface = { + .create_files = gator_events_mali_create_files, + .start = gator_events_mali_start, + .stop = gator_events_mali_stop, + .read = gator_events_mali_read, +}; + +int gator_events_mali_init(void) +{ + unsigned int cnt; + u32 id = gator_mali_get_id(); + + switch (id) { + case MALI_T6xx: + mali_name = "Mali-T6xx"; + break; + case MALI_400: + mali_name = "Mali-400"; + break; + case MALI_300: + mali_name = "Mali-300"; + break; + case MALI_200: + mali_name = "Mali-200"; + break; + default: + printk("Unknown Mali ID (%d)\n", id); + return -1; + } + + pr_debug("gator: mali init\n"); + + for (cnt = FIRST_ACTIVITY_EVENT; cnt < NUMBER_OF_EVENTS; cnt++) { + counter_enabled[cnt] = 0; + counter_event[cnt] = 0; + counter_key[cnt] = gator_events_get_key(); + counter_address[cnt] = NULL; + counter_data[cnt] = 0; + } + + trace_registered = 0; + + return gator_events_install(&gator_events_mali_interface); +} +gator_events_init(gator_events_mali_init); diff --git a/drivers/gator/driver/gator_events_meminfo.c b/drivers/gator/driver/gator_events_meminfo.c new file mode 100644 index 00000000000..8af9cfcb6d5 --- /dev/null +++ b/drivers/gator/driver/gator_events_meminfo.c @@ -0,0 +1,196 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" +#include +#include + +#define MEMINFO_MEMFREE 0 +#define MEMINFO_MEMUSED 1 +#define MEMINFO_BUFFERRAM 2 +#define MEMINFO_TOTAL 3 + +static ulong meminfo_global_enabled; +static ulong meminfo_enabled[MEMINFO_TOTAL]; +static ulong meminfo_key[MEMINFO_TOTAL]; +static unsigned long long meminfo_buffer[MEMINFO_TOTAL * 2]; +static int meminfo_length = 0; +static unsigned int mem_event = 0; +static bool new_data_avail; + +static void wq_sched_handler(struct work_struct *wsptr); + +DECLARE_WORK(work, wq_sched_handler); + +GATOR_DEFINE_PROBE(mm_page_free_direct, TP_PROTO(struct page *page, unsigned int order)) { + mem_event++; +} + +GATOR_DEFINE_PROBE(mm_pagevec_free, TP_PROTO(struct page *page, int cold)) { + mem_event++; +} + +GATOR_DEFINE_PROBE(mm_page_alloc, TP_PROTO(struct page *page, unsigned int order, gfp_t gfp_flags, int migratetype)) { + mem_event++; +} + +static int gator_events_meminfo_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int i; + + for (i = 0; i < MEMINFO_TOTAL; i++) { + switch (i) { + case MEMINFO_MEMFREE: + dir = gatorfs_mkdir(sb, root, "Linux_meminfo_memfree"); + break; + case MEMINFO_MEMUSED: + dir = gatorfs_mkdir(sb, root, "Linux_meminfo_memused"); + break; + case MEMINFO_BUFFERRAM: + dir = gatorfs_mkdir(sb, root, "Linux_meminfo_bufferram"); + break; + default: + return -1; + } + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &meminfo_enabled[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &meminfo_key[i]); + } + + return 0; +} + +static int gator_events_meminfo_start(void) +{ + int i; + + new_data_avail = true; + for (i = 0; i < MEMINFO_TOTAL; i++) { + if (meminfo_enabled[i]) { + meminfo_global_enabled = 1; + } + } + + if (meminfo_global_enabled == 0) + return 0; + + if (GATOR_REGISTER_TRACE(mm_page_free_direct)) + goto mm_page_free_direct_exit; + if (GATOR_REGISTER_TRACE(mm_pagevec_free)) + goto mm_pagevec_free_exit; + if (GATOR_REGISTER_TRACE(mm_page_alloc)) + goto mm_page_alloc_exit; + + return 0; + +mm_page_alloc_exit: + GATOR_UNREGISTER_TRACE(mm_pagevec_free); +mm_pagevec_free_exit: + GATOR_UNREGISTER_TRACE(mm_page_free_direct); +mm_page_free_direct_exit: + return -1; +} + +static void gator_events_meminfo_stop(void) +{ + int i; + + if (meminfo_global_enabled) { + GATOR_UNREGISTER_TRACE(mm_page_free_direct); + GATOR_UNREGISTER_TRACE(mm_pagevec_free); + GATOR_UNREGISTER_TRACE(mm_page_alloc); + } + + meminfo_global_enabled = 0; + for (i = 0; i < MEMINFO_TOTAL; i++) { + meminfo_enabled[i] = 0; + } +} + +// Must be run in a work queue as the kernel function si_meminfo() can sleep +static void wq_sched_handler(struct work_struct *wsptr) +{ + struct sysinfo info; + int i, len; + unsigned long long value; + + meminfo_length = len = 0; + + si_meminfo(&info); + for (i = 0; i < MEMINFO_TOTAL; i++) { + if (meminfo_enabled[i]) { + switch (i) { + case MEMINFO_MEMFREE: + value = info.freeram * PAGE_SIZE; + break; + case MEMINFO_MEMUSED: + value = (info.totalram - info.freeram) * PAGE_SIZE; + break; + case MEMINFO_BUFFERRAM: + value = info.bufferram * PAGE_SIZE; + break; + default: + value = 0; + break; + } + meminfo_buffer[len++] = (unsigned long long)meminfo_key[i]; + meminfo_buffer[len++] = value; + } + } + + meminfo_length = len; + new_data_avail = true; +} + +static int gator_events_meminfo_read(long long **buffer) +{ + static unsigned int last_mem_event = 0; + + if (smp_processor_id() || !meminfo_global_enabled) + return 0; + + if (last_mem_event != mem_event) { + last_mem_event = mem_event; + schedule_work(&work); + } + + if (!new_data_avail) + return 0; + + new_data_avail = false; + + if (buffer) + *buffer = meminfo_buffer; + + return meminfo_length; +} + +static struct gator_interface gator_events_meminfo_interface = { + .create_files = gator_events_meminfo_create_files, + .start = gator_events_meminfo_start, + .stop = gator_events_meminfo_stop, + .read64 = gator_events_meminfo_read, +}; + +int gator_events_meminfo_init(void) +{ + int i; + + meminfo_global_enabled = 0; + for (i = 0; i < MEMINFO_TOTAL; i++) { + meminfo_enabled[i] = 0; + meminfo_key[i] = gator_events_get_key(); + } + + return gator_events_install(&gator_events_meminfo_interface); +} +gator_events_init(gator_events_meminfo_init); diff --git a/drivers/gator/driver/gator_events_mmaped.c b/drivers/gator/driver/gator_events_mmaped.c new file mode 100644 index 00000000000..acd5f85ebb7 --- /dev/null +++ b/drivers/gator/driver/gator_events_mmaped.c @@ -0,0 +1,198 @@ +/* + * Example events provider + * + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Similar entries to those below must be present in the events.xml file. + * To add them to the events.xml, create an events-mmap.xml with the + * following contents and rebuild gatord: + * + * + * + * + * + * + * + * + * + * + */ + +#include +#include +#include + +#include "gator.h" + +#define MMAPED_COUNTERS_NUM 3 + +static struct { + unsigned long enabled; + unsigned long event; + unsigned long key; +} mmaped_counters[MMAPED_COUNTERS_NUM]; + +static int mmaped_buffer[MMAPED_COUNTERS_NUM * 2]; + +#ifdef TODO +static void __iomem *mmaped_base; +#endif + +/* Adds mmaped_cntX directories and enabled, event, and key files to /dev/gator/driver/events */ +static int gator_events_mmaped_create_files(struct super_block *sb, + struct dentry *root) +{ + int i; + + for (i = 0; i < MMAPED_COUNTERS_NUM; i++) { + char buf[16]; + struct dentry *dir; + + snprintf(buf, sizeof(buf), "mmaped_cnt%d", i); + dir = gatorfs_mkdir(sb, root, buf); + if (WARN_ON(!dir)) + return -1; + gatorfs_create_ulong(sb, dir, "enabled", + &mmaped_counters[i].enabled); + gatorfs_create_ulong(sb, dir, "event", + &mmaped_counters[i].event); + gatorfs_create_ro_ulong(sb, dir, "key", + &mmaped_counters[i].key); + } + + return 0; +} + +static int gator_events_mmaped_start(void) +{ +#ifdef TODO + for (i = 0; i < MMAPED_COUNTERS_NUM; i++) + writel(mmaped_counters[i].event, + mmaped_base + COUNTERS_CONFIG_OFFSET[i]); + + writel(ENABLED, COUNTERS_CONTROL_OFFSET); +#endif + + return 0; +} + +static void gator_events_mmaped_stop(void) +{ +#ifdef TODO + writel(DISABLED, COUNTERS_CONTROL_OFFSET); +#endif +} + +#ifndef TODO +/* This function "simulates" counters, generating values of fancy + * functions like sine or triangle... */ +static int mmaped_simulate(int counter) +{ + int result = 0; + + switch (counter) { + case 0: /* sort-of-sine */ + { + static int t; + int x; + + if (t % 1024 < 512) + x = 512 - (t % 512); + else + x = t % 512; + + result = 32 * x / 512; + result = result * result; + + if (t < 1024) + result = 1922 - result; + + t = (t + 1) % 2048; + } + break; + case 1: /* triangle */ + { + static int v, d = 1; + + v += d; + if (v % 2000 == 0) + d = -d; + + result = v; + } + break; + case 2: /* PWM signal */ + { + static int t, dc; + + t = (t + 1) % 2000; + if (t % 100) + dc = (dc + 200) % 2000; + + result = t < dc ? 0 : 2000; + } + break; + } + + return result; +} +#endif + +static int gator_events_mmaped_read(int **buffer) +{ + int i; + int len = 0; + + /* System wide counters - read from one core only */ + if (smp_processor_id()) + return 0; + + for (i = 0; i < MMAPED_COUNTERS_NUM; i++) { + if (mmaped_counters[i].enabled) { + mmaped_buffer[len++] = mmaped_counters[i].key; +#ifdef TODO + mmaped_buffer[len++] = readl(mmaped_base + + COUNTERS_VALUE_OFFSET[i]); +#else + mmaped_buffer[len++] = mmaped_simulate( + mmaped_counters[i].event); +#endif + } + } + + if (buffer) + *buffer = mmaped_buffer; + + return len; +} + +static struct gator_interface gator_events_mmaped_interface = { + .create_files = gator_events_mmaped_create_files, + .start = gator_events_mmaped_start, + .stop = gator_events_mmaped_stop, + .read = gator_events_mmaped_read, +}; + +/* Must not be static! */ +int __init gator_events_mmaped_init(void) +{ + int i; + +#ifdef TODO + mmaped_base = ioremap(COUNTERS_PHYS_ADDR, SZ_4K); + if (!mmaped_base) + return -ENOMEM; +#endif + + for (i = 0; i < MMAPED_COUNTERS_NUM; i++) { + mmaped_counters[i].enabled = 0; + mmaped_counters[i].key = gator_events_get_key(); + } + + return gator_events_install(&gator_events_mmaped_interface); +} +gator_events_init(gator_events_mmaped_init); diff --git a/drivers/gator/driver/gator_events_net.c b/drivers/gator/driver/gator_events_net.c new file mode 100644 index 00000000000..ef1623b9a25 --- /dev/null +++ b/drivers/gator/driver/gator_events_net.c @@ -0,0 +1,144 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" +#include + +#define NETRX 0 +#define NETTX 1 +#define TOTALNET 2 + +static ulong netrx_enabled; +static ulong nettx_enabled; +static ulong netrx_key; +static ulong nettx_key; +static int rx_total, tx_total; +static ulong netPrev[TOTALNET]; +static int netGet[TOTALNET * 2]; + +static void get_network_stats(struct work_struct *wsptr) { + int rx = 0, tx = 0; + struct net_device *dev; + + for_each_netdev(&init_net, dev) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 36) + const struct net_device_stats *stats = dev_get_stats(dev); +#else + struct rtnl_link_stats64 temp; + const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp); +#endif + rx += stats->rx_bytes; + tx += stats->tx_bytes; + } + rx_total = rx; + tx_total = tx; +} +DECLARE_WORK(wq_get_stats, get_network_stats); + +static void calculate_delta(int *rx, int *tx) +{ + int rx_calc, tx_calc; + + rx_calc = (int)(rx_total - netPrev[NETRX]); + if (rx_calc < 0) + rx_calc = 0; + netPrev[NETRX] += rx_calc; + + tx_calc = (int)(tx_total - netPrev[NETTX]); + if (tx_calc < 0) + tx_calc = 0; + netPrev[NETTX] += tx_calc; + + *rx = rx_calc; + *tx = tx_calc; +} + +static int gator_events_net_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + + dir = gatorfs_mkdir(sb, root, "Linux_net_rx"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &netrx_enabled); + gatorfs_create_ro_ulong(sb, dir, "key", &netrx_key); + + dir = gatorfs_mkdir(sb, root, "Linux_net_tx"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &nettx_enabled); + gatorfs_create_ro_ulong(sb, dir, "key", &nettx_key); + + return 0; +} + +static int gator_events_net_start(void) +{ + get_network_stats(NULL); + netPrev[NETRX] = rx_total; + netPrev[NETTX] = tx_total; + return 0; +} + +static void gator_events_net_stop(void) +{ + netrx_enabled = 0; + nettx_enabled = 0; +} + +static int gator_events_net_read(int **buffer) +{ + int len, rx_delta, tx_delta; + static int last_rx_delta = 0, last_tx_delta = 0; + + if (smp_processor_id() != 0) + return 0; + + schedule_work(&wq_get_stats); + calculate_delta(&rx_delta, &tx_delta); + + len = 0; + if (netrx_enabled && last_rx_delta != rx_delta) { + last_rx_delta = rx_delta; + netGet[len++] = netrx_key; + netGet[len++] = rx_delta; + } + + if (nettx_enabled && last_tx_delta != tx_delta) { + last_tx_delta = tx_delta; + netGet[len++] = nettx_key; + netGet[len++] = tx_delta; + } + + if (buffer) + *buffer = netGet; + + return len; +} + +static struct gator_interface gator_events_net_interface = { + .create_files = gator_events_net_create_files, + .start = gator_events_net_start, + .stop = gator_events_net_stop, + .read = gator_events_net_read, +}; + +int gator_events_net_init(void) +{ + netrx_key = gator_events_get_key(); + nettx_key = gator_events_get_key(); + + netrx_enabled = 0; + nettx_enabled = 0; + + return gator_events_install(&gator_events_net_interface); +} +gator_events_init(gator_events_net_init); diff --git a/drivers/gator/driver/gator_events_perf_pmu.c b/drivers/gator/driver/gator_events_perf_pmu.c new file mode 100644 index 00000000000..322ebc430ec --- /dev/null +++ b/drivers/gator/driver/gator_events_perf_pmu.c @@ -0,0 +1,296 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include "gator.h" + +// gator_events_armvX.c is used for Linux 2.6.x +#if GATOR_PERF_PMU_SUPPORT + +static const char *pmnc_name; +int pmnc_counters; +int ccnt = 0; + +#define CNTMAX (6+1) + +unsigned long pmnc_enabled[CNTMAX]; +unsigned long pmnc_event[CNTMAX]; +unsigned long pmnc_count[CNTMAX]; +unsigned long pmnc_key[CNTMAX]; + +static DEFINE_PER_CPU(int[CNTMAX], perfCurr); +static DEFINE_PER_CPU(int[CNTMAX], perfPrev); +static DEFINE_PER_CPU(int[CNTMAX], perfPrevDelta); +static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); +static DEFINE_PER_CPU(struct perf_event *[CNTMAX], pevent); +static DEFINE_PER_CPU(struct perf_event_attr *[CNTMAX], pevent_attr); + +static void gator_events_perf_pmu_stop(void); + +static int gator_events_perf_pmu_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int i; + + for (i = 0; i < pmnc_counters; i++) { + char buf[40]; + if (i == 0) { + snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name); + } else { + snprintf(buf, sizeof buf, "%s_cnt%d", pmnc_name, i-1); + } + dir = gatorfs_mkdir(sb, root, buf); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); + gatorfs_create_ulong(sb, dir, "count", &pmnc_count[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); + if (i > 0) { + gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); + } + } + + return 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) +static void dummy_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs) +#else +static void dummy_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) +#endif +{ +// Required as perf_event_create_kernel_counter() requires an overflow handler, even though all we do is poll +} + +static int gator_events_perf_pmu_online(int** buffer) +{ + int cnt, len = 0, cpu = smp_processor_id(); + + // read the counters and toss the invalid data, return zero instead + for (cnt = 0; cnt < pmnc_counters; cnt++) { + struct perf_event * ev = per_cpu(pevent, cpu)[cnt]; + if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { + ev->pmu->read(ev); + per_cpu(perfPrev, cpu)[cnt] = per_cpu(perfCurr, cpu)[cnt] = local64_read(&ev->count); + per_cpu(perfPrevDelta, cpu)[cnt] = 0; + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = 0; + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; +} + +static void gator_events_perf_pmu_online_dispatch(int cpu) +{ + int cnt; + + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (per_cpu(pevent, cpu)[cnt] != NULL || per_cpu(pevent_attr, cpu)[cnt] == 0) + continue; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) + per_cpu(pevent, cpu)[cnt] = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu)[cnt], cpu, 0, dummy_handler); +#else + per_cpu(pevent, cpu)[cnt] = perf_event_create_kernel_counter(per_cpu(pevent_attr, cpu)[cnt], cpu, 0, dummy_handler, 0); +#endif + if (IS_ERR(per_cpu(pevent, cpu)[cnt])) { + pr_debug("gator: unable to online a counter on cpu %d\n", cpu); + per_cpu(pevent, cpu)[cnt] = NULL; + continue; + } + + if (per_cpu(pevent, cpu)[cnt]->state != PERF_EVENT_STATE_ACTIVE) { + pr_debug("gator: inactive counter on cpu %d\n", cpu); + perf_event_release_kernel(per_cpu(pevent, cpu)[cnt]); + per_cpu(pevent, cpu)[cnt] = NULL; + continue; + } + } +} + +static void gator_events_perf_pmu_offline_dispatch(int cpu) +{ + int cnt; + + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (per_cpu(pevent, cpu)[cnt] != NULL) { + perf_event_release_kernel(per_cpu(pevent, cpu)[cnt]); + per_cpu(pevent, cpu)[cnt] = NULL; + } + } +} + +static int gator_events_perf_pmu_start(void) +{ + int cnt, cpu; + u32 size = sizeof(struct perf_event_attr); + + for_each_present_cpu(cpu) { + for (cnt = 0; cnt < pmnc_counters; cnt++) { + per_cpu(pevent, cpu)[cnt] = NULL; + if (!pmnc_enabled[cnt] || pmnc_count[cnt] > 0) // Skip disabled counters and EBS counters + continue; + + per_cpu(perfPrev, cpu)[cnt] = 0; + per_cpu(perfCurr, cpu)[cnt] = 0; + per_cpu(perfPrevDelta, cpu)[cnt] = 0; + per_cpu(pevent_attr, cpu)[cnt] = kmalloc(size, GFP_KERNEL); + if (!per_cpu(pevent_attr, cpu)[cnt]) { + gator_events_perf_pmu_stop(); + return -1; + } + + memset(per_cpu(pevent_attr, cpu)[cnt], 0, size); + per_cpu(pevent_attr, cpu)[cnt]->type = PERF_TYPE_RAW; + per_cpu(pevent_attr, cpu)[cnt]->size = size; + per_cpu(pevent_attr, cpu)[cnt]->config = pmnc_event[cnt]; + per_cpu(pevent_attr, cpu)[cnt]->sample_period = 0; + per_cpu(pevent_attr, cpu)[cnt]->pinned = 1; + + // handle special case for ccnt + if (cnt == ccnt) { + per_cpu(pevent_attr, cpu)[cnt]->type = PERF_TYPE_HARDWARE; + per_cpu(pevent_attr, cpu)[cnt]->config = PERF_COUNT_HW_CPU_CYCLES; + } + } + } + + return 0; +} + +static void gator_events_perf_pmu_stop(void) +{ + unsigned int cnt, cpu; + + for_each_present_cpu(cpu) { + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (per_cpu(pevent_attr, cpu)[cnt]) { + kfree(per_cpu(pevent_attr, cpu)[cnt]); + per_cpu(pevent_attr, cpu)[cnt] = NULL; + } + } + } + + for (cnt = 0; cnt < pmnc_counters; cnt++) { + pmnc_enabled[cnt] = 0; + pmnc_event[cnt] = 0; + pmnc_count[cnt] = 0; + } +} + +static int gator_events_perf_pmu_read(int **buffer) +{ + int cnt, delta, len = 0; + int cpu = smp_processor_id(); + + for (cnt = 0; cnt < pmnc_counters; cnt++) { + struct perf_event * ev = per_cpu(pevent, cpu)[cnt]; + if (ev != NULL && ev->state == PERF_EVENT_STATE_ACTIVE) { + ev->pmu->read(ev); + per_cpu(perfCurr, cpu)[cnt] = local64_read(&ev->count); + delta = per_cpu(perfCurr, cpu)[cnt] - per_cpu(perfPrev, cpu)[cnt]; + if (delta != per_cpu(perfPrevDelta, cpu)[cnt]) { + per_cpu(perfPrevDelta, cpu)[cnt] = delta; + per_cpu(perfPrev, cpu)[cnt] = per_cpu(perfCurr, cpu)[cnt]; + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + if (delta < 0) + delta *= -1; + per_cpu(perfCnt, cpu)[len++] = delta; + } + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; +} + +static struct gator_interface gator_events_perf_pmu_interface = { + .create_files = gator_events_perf_pmu_create_files, + .start = gator_events_perf_pmu_start, + .stop = gator_events_perf_pmu_stop, + .online = gator_events_perf_pmu_online, + .online_dispatch = gator_events_perf_pmu_online_dispatch, + .offline_dispatch = gator_events_perf_pmu_offline_dispatch, + .read = gator_events_perf_pmu_read, +}; + +int gator_events_perf_pmu_init(void) +{ + unsigned int cnt; + + switch (gator_cpuid()) { + case ARM1136: + case ARM1156: + case ARM1176: + pmnc_name = "ARM_ARM11"; + pmnc_counters = 3; + ccnt = 2; + break; + case ARM11MPCORE: + pmnc_name = "ARM_ARM11MPCore"; + pmnc_counters = 3; + break; + case CORTEX_A5: + pmnc_name = "ARM_Cortex-A5"; + pmnc_counters = 2; + break; + case CORTEX_A7: + pmnc_name = "ARM_Cortex-A7"; + pmnc_counters = 4; + break; + case CORTEX_A8: + pmnc_name = "ARM_Cortex-A8"; + pmnc_counters = 4; + break; + case CORTEX_A9: + pmnc_name = "ARM_Cortex-A9"; + pmnc_counters = 6; + break; + case CORTEX_A15: + pmnc_name = "ARM_Cortex-A15"; + pmnc_counters = 6; + break; + case SCORPION: + pmnc_name = "Scorpion"; + pmnc_counters = 4; + break; + case SCORPIONMP: + pmnc_name = "ScorpionMP"; + pmnc_counters = 4; + break; + case KRAITSIM: + case KRAIT: + pmnc_name = "Krait"; + pmnc_counters = 4; + break; + default: + return -1; + } + + pmnc_counters++; // CNT[n] + CCNT + + for (cnt = 0; cnt < CNTMAX; cnt++) { + pmnc_enabled[cnt] = 0; + pmnc_event[cnt] = 0; + pmnc_count[cnt] = 0; + pmnc_key[cnt] = gator_events_get_key(); + } + + return gator_events_install(&gator_events_perf_pmu_interface); +} + +gator_events_init(gator_events_perf_pmu_init); +#endif diff --git a/drivers/gator/driver/gator_events_power.c b/drivers/gator/driver/gator_events_power.c new file mode 100644 index 00000000000..a0ae684827d --- /dev/null +++ b/drivers/gator/driver/gator_events_power.c @@ -0,0 +1,178 @@ +/** + * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" +#include +#include +#include + +// cpu_frequency and cpu_idle trace points were introduced in Linux kernel v2.6.38 +// the now deprecated power_frequency trace point was available prior to 2.6.38, but only for x86 +// CPU Idle is currently disabled in the .xml +#if GATOR_CPU_FREQ_SUPPORT +enum { + POWER_CPU_FREQ, + POWER_CPU_IDLE, + POWER_TOTAL +}; + +static ulong power_cpu_enabled[POWER_TOTAL]; +static ulong power_cpu_key[POWER_TOTAL]; +static DEFINE_PER_CPU(ulong[POWER_TOTAL], power); +static DEFINE_PER_CPU(ulong[POWER_TOTAL], prev); +static DEFINE_PER_CPU(int *, powerGet); + +GATOR_DEFINE_PROBE(cpu_frequency, TP_PROTO(unsigned int frequency, unsigned int cpu)) +{ + per_cpu(power, cpu)[POWER_CPU_FREQ] = frequency * 1000; +} + +GATOR_DEFINE_PROBE(cpu_idle, TP_PROTO(unsigned int state, unsigned int cpu)) +{ + per_cpu(power, cpu)[POWER_CPU_IDLE] = state; +} + +static int gator_events_power_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + + // cpu_frequency + dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_freq"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_FREQ]); + gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_FREQ]); + + // cpu_idle + dir = gatorfs_mkdir(sb, root, "Linux_power_cpu_idle"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &power_cpu_enabled[POWER_CPU_IDLE]); + gatorfs_create_ro_ulong(sb, dir, "key", &power_cpu_key[POWER_CPU_IDLE]); + + return 0; +} + +static int gator_events_power_populate(int cpu, int** buffer) +{ + int i, len = 0; + + for (i = 0; i < POWER_TOTAL; i++) { + if (power_cpu_enabled[i]) { + if (per_cpu(power, cpu)[i] != per_cpu(prev, cpu)[i]) { + per_cpu(prev, cpu)[i] = per_cpu(power, cpu)[i]; + per_cpu(powerGet, cpu)[len++] = power_cpu_key[i]; + per_cpu(powerGet, cpu)[len++] = per_cpu(power, cpu)[i]; + } + } + } + + if (buffer) + *buffer = per_cpu(powerGet, cpu); + + return len; +} + +static int gator_events_power_online(int** buffer) +{ + int i, cpu = smp_processor_id(); + for (i = 0; i < POWER_TOTAL; i++) + per_cpu(prev, cpu)[i] = -1; + per_cpu(power, cpu)[POWER_CPU_FREQ] = cpufreq_quick_get(cpu) * 1000; + return gator_events_power_populate(cpu, buffer); +} + +static int gator_events_power_offline(int** buffer) +{ + int cpu = smp_processor_id(); + // Set frequency to zero on an offline + per_cpu(power, cpu)[POWER_CPU_FREQ] = 0; + return gator_events_power_populate(cpu, buffer); +} + +static int gator_events_power_start(void) +{ + int cpu; + + for_each_present_cpu(cpu) { + per_cpu(powerGet, cpu) = kmalloc(POWER_TOTAL * 2, GFP_KERNEL); + if (!per_cpu(powerGet, cpu)) + return -1; + } + + // register tracepoints + if (power_cpu_enabled[POWER_CPU_FREQ]) + if (GATOR_REGISTER_TRACE(cpu_frequency)) + goto fail_cpu_frequency_exit; + if (power_cpu_enabled[POWER_CPU_IDLE]) + if (GATOR_REGISTER_TRACE(cpu_idle)) + goto fail_cpu_idle_exit; + pr_debug("gator: registered power event tracepoints\n"); + + return 0; + + // unregister tracepoints on error +fail_cpu_idle_exit: + GATOR_UNREGISTER_TRACE(cpu_frequency); +fail_cpu_frequency_exit: + pr_err("gator: power event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +static void gator_events_power_stop(void) +{ + int i, cpu; + if (power_cpu_enabled[POWER_CPU_FREQ]) + GATOR_UNREGISTER_TRACE(cpu_frequency); + if (power_cpu_enabled[POWER_CPU_IDLE]) + GATOR_UNREGISTER_TRACE(cpu_idle); + pr_debug("gator: unregistered power event tracepoints\n"); + + for (i = 0; i < POWER_TOTAL; i++) { + power_cpu_enabled[i] = 0; + } + + for_each_present_cpu(cpu) { + kfree(per_cpu(powerGet, cpu)); + } +} + +static int gator_events_power_read(int **buffer) +{ + return gator_events_power_populate(smp_processor_id(), buffer); +} + +static struct gator_interface gator_events_power_interface = { + .create_files = gator_events_power_create_files, + .online = gator_events_power_online, + .offline = gator_events_power_offline, + .start = gator_events_power_start, + .stop = gator_events_power_stop, + .read = gator_events_power_read, +}; +#endif + +int gator_events_power_init(void) +{ +#if (GATOR_CPU_FREQ_SUPPORT) + int i; + for (i = 0; i < POWER_TOTAL; i++) { + power_cpu_enabled[i] = 0; + power_cpu_key[i] = gator_events_get_key(); + } + + return gator_events_install(&gator_events_power_interface); +#else + return -1; +#endif +} +gator_events_init(gator_events_power_init); diff --git a/drivers/gator/driver/gator_events_sched.c b/drivers/gator/driver/gator_events_sched.c new file mode 100644 index 00000000000..9bed3641dbb --- /dev/null +++ b/drivers/gator/driver/gator_events_sched.c @@ -0,0 +1,114 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include "gator.h" +#include + +#define SCHED_SWITCH 0 +#define SCHED_TOTAL (SCHED_SWITCH+1) + +static ulong sched_switch_enabled; +static ulong sched_switch_key; +static DEFINE_PER_CPU(int[SCHED_TOTAL], schedCnt); +static DEFINE_PER_CPU(int[SCHED_TOTAL * 2], schedGet); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) +GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *prev, struct task_struct *next)) +#else +GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next)) +#endif +{ + unsigned long flags; + + // disable interrupts to synchronize with gator_events_sched_read() + // spinlocks not needed since percpu buffers are used + local_irq_save(flags); + per_cpu(schedCnt, smp_processor_id())[SCHED_SWITCH]++; + local_irq_restore(flags); +} + +static int gator_events_sched_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + + /* switch */ + dir = gatorfs_mkdir(sb, root, "Linux_sched_switch"); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &sched_switch_enabled); + gatorfs_create_ro_ulong(sb, dir, "key", &sched_switch_key); + + return 0; +} + +static int gator_events_sched_start(void) +{ + // register tracepoints + if (sched_switch_enabled) + if (GATOR_REGISTER_TRACE(sched_switch)) + goto sched_switch_exit; + pr_debug("gator: registered scheduler event tracepoints\n"); + + return 0; + + // unregister tracepoints on error +sched_switch_exit: + pr_err("gator: scheduler event tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +static void gator_events_sched_stop(void) +{ + if (sched_switch_enabled) + GATOR_UNREGISTER_TRACE(sched_switch); + pr_debug("gator: unregistered scheduler event tracepoints\n"); + + sched_switch_enabled = 0; +} + +static int gator_events_sched_read(int **buffer) +{ + unsigned long flags; + int len, value; + int cpu = smp_processor_id(); + + len = 0; + if (sched_switch_enabled) { + local_irq_save(flags); + value = per_cpu(schedCnt, cpu)[SCHED_SWITCH]; + per_cpu(schedCnt, cpu)[SCHED_SWITCH] = 0; + local_irq_restore(flags); + per_cpu(schedGet, cpu)[len++] = sched_switch_key; + per_cpu(schedGet, cpu)[len++] = value; + } + + if (buffer) + *buffer = per_cpu(schedGet, cpu); + + return len; +} + +static struct gator_interface gator_events_sched_interface = { + .create_files = gator_events_sched_create_files, + .start = gator_events_sched_start, + .stop = gator_events_sched_stop, + .read = gator_events_sched_read, +}; + +int gator_events_sched_init(void) +{ + sched_switch_enabled = 0; + + sched_switch_key = gator_events_get_key(); + + return gator_events_install(&gator_events_sched_interface); +} +gator_events_init(gator_events_sched_init); diff --git a/drivers/gator/driver/gator_events_scorpion.c b/drivers/gator/driver/gator_events_scorpion.c new file mode 100644 index 00000000000..477e7c98948 --- /dev/null +++ b/drivers/gator/driver/gator_events_scorpion.c @@ -0,0 +1,678 @@ +/** + * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "gator.h" + +// gator_events_perf_pmu.c is used if perf is supported +#if GATOR_NO_PERF_SUPPORT + +static const char *pmnc_name; +static int pmnc_counters; + +// Per-CPU PMNC: config reg +#define PMNC_E (1 << 0) /* Enable all counters */ +#define PMNC_P (1 << 1) /* Reset all counters */ +#define PMNC_C (1 << 2) /* Cycle counter reset */ +#define PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */ +#define PMNC_X (1 << 4) /* Export to ETM */ +#define PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/ +#define PMNC_MASK 0x3f /* Mask for writable bits */ + +// ccnt reg +#define CCNT_REG (1 << 31) + +#define CCNT 0 +#define CNT0 1 +#define CNTMAX (4+1) + +static unsigned long pmnc_enabled[CNTMAX]; +static unsigned long pmnc_event[CNTMAX]; +static unsigned long pmnc_key[CNTMAX]; + +static DEFINE_PER_CPU(int[CNTMAX], perfPrev); +static DEFINE_PER_CPU(int[CNTMAX * 2], perfCnt); + +enum scorpion_perf_types { + SCORPION_ICACHE_EXPL_INV = 0x4c, + SCORPION_ICACHE_MISS = 0x4d, + SCORPION_ICACHE_ACCESS = 0x4e, + SCORPION_ICACHE_CACHEREQ_L2 = 0x4f, + SCORPION_ICACHE_NOCACHE_L2 = 0x50, + SCORPION_HIQUP_NOPED = 0x51, + SCORPION_DATA_ABORT = 0x52, + SCORPION_IRQ = 0x53, + SCORPION_FIQ = 0x54, + SCORPION_ALL_EXCPT = 0x55, + SCORPION_UNDEF = 0x56, + SCORPION_SVC = 0x57, + SCORPION_SMC = 0x58, + SCORPION_PREFETCH_ABORT = 0x59, + SCORPION_INDEX_CHECK = 0x5a, + SCORPION_NULL_CHECK = 0x5b, + SCORPION_EXPL_ICIALLU = 0x5c, + SCORPION_IMPL_ICIALLU = 0x5d, + SCORPION_NONICIALLU_BTAC_INV = 0x5e, + SCORPION_ICIMVAU_IMPL_ICIALLU = 0x5f, + SCORPION_SPIPE_ONLY_CYCLES = 0x60, + SCORPION_XPIPE_ONLY_CYCLES = 0x61, + SCORPION_DUAL_CYCLES = 0x62, + SCORPION_DISPATCH_ANY_CYCLES = 0x63, + SCORPION_FIFO_FULLBLK_CMT = 0x64, + SCORPION_FAIL_COND_INST = 0x65, + SCORPION_PASS_COND_INST = 0x66, + SCORPION_ALLOW_VU_CLK = 0x67, + SCORPION_VU_IDLE = 0x68, + SCORPION_ALLOW_L2_CLK = 0x69, + SCORPION_L2_IDLE = 0x6a, + SCORPION_DTLB_IMPL_INV_SCTLR_DACR = 0x6b, + SCORPION_DTLB_EXPL_INV = 0x6c, + SCORPION_DTLB_MISS = 0x6d, + SCORPION_DTLB_ACCESS = 0x6e, + SCORPION_ITLB_MISS = 0x6f, + SCORPION_ITLB_IMPL_INV = 0x70, + SCORPION_ITLB_EXPL_INV = 0x71, + SCORPION_UTLB_D_MISS = 0x72, + SCORPION_UTLB_D_ACCESS = 0x73, + SCORPION_UTLB_I_MISS = 0x74, + SCORPION_UTLB_I_ACCESS = 0x75, + SCORPION_UTLB_INV_ASID = 0x76, + SCORPION_UTLB_INV_MVA = 0x77, + SCORPION_UTLB_INV_ALL = 0x78, + SCORPION_S2_HOLD_RDQ_UNAVAIL = 0x79, + SCORPION_S2_HOLD = 0x7a, + SCORPION_S2_HOLD_DEV_OP = 0x7b, + SCORPION_S2_HOLD_ORDER = 0x7c, + SCORPION_S2_HOLD_BARRIER = 0x7d, + SCORPION_VIU_DUAL_CYCLE = 0x7e, + SCORPION_VIU_SINGLE_CYCLE = 0x7f, + SCORPION_VX_PIPE_WAR_STALL_CYCLES = 0x80, + SCORPION_VX_PIPE_WAW_STALL_CYCLES = 0x81, + SCORPION_VX_PIPE_RAW_STALL_CYCLES = 0x82, + SCORPION_VX_PIPE_LOAD_USE_STALL = 0x83, + SCORPION_VS_PIPE_WAR_STALL_CYCLES = 0x84, + SCORPION_VS_PIPE_WAW_STALL_CYCLES = 0x85, + SCORPION_VS_PIPE_RAW_STALL_CYCLES = 0x86, + SCORPION_EXCEPTIONS_INV_OPERATION = 0x87, + SCORPION_EXCEPTIONS_DIV_BY_ZERO = 0x88, + SCORPION_COND_INST_FAIL_VX_PIPE = 0x89, + SCORPION_COND_INST_FAIL_VS_PIPE = 0x8a, + SCORPION_EXCEPTIONS_OVERFLOW = 0x8b, + SCORPION_EXCEPTIONS_UNDERFLOW = 0x8c, + SCORPION_EXCEPTIONS_DENORM = 0x8d, +#ifdef CONFIG_ARCH_MSM_SCORPIONMP + SCORPIONMP_NUM_BARRIERS = 0x8e, + SCORPIONMP_BARRIER_CYCLES = 0x8f, +#else + SCORPION_BANK_AB_HIT = 0x8e, + SCORPION_BANK_AB_ACCESS = 0x8f, + SCORPION_BANK_CD_HIT = 0x90, + SCORPION_BANK_CD_ACCESS = 0x91, + SCORPION_BANK_AB_DSIDE_HIT = 0x92, + SCORPION_BANK_AB_DSIDE_ACCESS = 0x93, + SCORPION_BANK_CD_DSIDE_HIT = 0x94, + SCORPION_BANK_CD_DSIDE_ACCESS = 0x95, + SCORPION_BANK_AB_ISIDE_HIT = 0x96, + SCORPION_BANK_AB_ISIDE_ACCESS = 0x97, + SCORPION_BANK_CD_ISIDE_HIT = 0x98, + SCORPION_BANK_CD_ISIDE_ACCESS = 0x99, + SCORPION_ISIDE_RD_WAIT = 0x9a, + SCORPION_DSIDE_RD_WAIT = 0x9b, + SCORPION_BANK_BYPASS_WRITE = 0x9c, + SCORPION_BANK_AB_NON_CASTOUT = 0x9d, + SCORPION_BANK_AB_L2_CASTOUT = 0x9e, + SCORPION_BANK_CD_NON_CASTOUT = 0x9f, + SCORPION_BANK_CD_L2_CASTOUT = 0xa0, +#endif + MSM_MAX_EVT +}; + +struct scorp_evt { + u32 evt_type; + u32 val; + u8 grp; + u32 evt_type_act; +}; + +static const struct scorp_evt sc_evt[] = { + {SCORPION_ICACHE_EXPL_INV, 0x80000500, 0, 0x4d}, + {SCORPION_ICACHE_MISS, 0x80050000, 0, 0x4e}, + {SCORPION_ICACHE_ACCESS, 0x85000000, 0, 0x4f}, + {SCORPION_ICACHE_CACHEREQ_L2, 0x86000000, 0, 0x4f}, + {SCORPION_ICACHE_NOCACHE_L2, 0x87000000, 0, 0x4f}, + {SCORPION_HIQUP_NOPED, 0x80080000, 0, 0x4e}, + {SCORPION_DATA_ABORT, 0x8000000a, 0, 0x4c}, + {SCORPION_IRQ, 0x80000a00, 0, 0x4d}, + {SCORPION_FIQ, 0x800a0000, 0, 0x4e}, + {SCORPION_ALL_EXCPT, 0x8a000000, 0, 0x4f}, + {SCORPION_UNDEF, 0x8000000b, 0, 0x4c}, + {SCORPION_SVC, 0x80000b00, 0, 0x4d}, + {SCORPION_SMC, 0x800b0000, 0, 0x4e}, + {SCORPION_PREFETCH_ABORT, 0x8b000000, 0, 0x4f}, + {SCORPION_INDEX_CHECK, 0x8000000c, 0, 0x4c}, + {SCORPION_NULL_CHECK, 0x80000c00, 0, 0x4d}, + {SCORPION_EXPL_ICIALLU, 0x8000000d, 0, 0x4c}, + {SCORPION_IMPL_ICIALLU, 0x80000d00, 0, 0x4d}, + {SCORPION_NONICIALLU_BTAC_INV, 0x800d0000, 0, 0x4e}, + {SCORPION_ICIMVAU_IMPL_ICIALLU, 0x8d000000, 0, 0x4f}, + + {SCORPION_SPIPE_ONLY_CYCLES, 0x80000600, 1, 0x51}, + {SCORPION_XPIPE_ONLY_CYCLES, 0x80060000, 1, 0x52}, + {SCORPION_DUAL_CYCLES, 0x86000000, 1, 0x53}, + {SCORPION_DISPATCH_ANY_CYCLES, 0x89000000, 1, 0x53}, + {SCORPION_FIFO_FULLBLK_CMT, 0x8000000d, 1, 0x50}, + {SCORPION_FAIL_COND_INST, 0x800d0000, 1, 0x52}, + {SCORPION_PASS_COND_INST, 0x8d000000, 1, 0x53}, + {SCORPION_ALLOW_VU_CLK, 0x8000000e, 1, 0x50}, + {SCORPION_VU_IDLE, 0x80000e00, 1, 0x51}, + {SCORPION_ALLOW_L2_CLK, 0x800e0000, 1, 0x52}, + {SCORPION_L2_IDLE, 0x8e000000, 1, 0x53}, + + {SCORPION_DTLB_IMPL_INV_SCTLR_DACR, 0x80000001, 2, 0x54}, + {SCORPION_DTLB_EXPL_INV, 0x80000100, 2, 0x55}, + {SCORPION_DTLB_MISS, 0x80010000, 2, 0x56}, + {SCORPION_DTLB_ACCESS, 0x81000000, 2, 0x57}, + {SCORPION_ITLB_MISS, 0x80000200, 2, 0x55}, + {SCORPION_ITLB_IMPL_INV, 0x80020000, 2, 0x56}, + {SCORPION_ITLB_EXPL_INV, 0x82000000, 2, 0x57}, + {SCORPION_UTLB_D_MISS, 0x80000003, 2, 0x54}, + {SCORPION_UTLB_D_ACCESS, 0x80000300, 2, 0x55}, + {SCORPION_UTLB_I_MISS, 0x80030000, 2, 0x56}, + {SCORPION_UTLB_I_ACCESS, 0x83000000, 2, 0x57}, + {SCORPION_UTLB_INV_ASID, 0x80000400, 2, 0x55}, + {SCORPION_UTLB_INV_MVA, 0x80040000, 2, 0x56}, + {SCORPION_UTLB_INV_ALL, 0x84000000, 2, 0x57}, + {SCORPION_S2_HOLD_RDQ_UNAVAIL, 0x80000800, 2, 0x55}, + {SCORPION_S2_HOLD, 0x88000000, 2, 0x57}, + {SCORPION_S2_HOLD_DEV_OP, 0x80000900, 2, 0x55}, + {SCORPION_S2_HOLD_ORDER, 0x80090000, 2, 0x56}, + {SCORPION_S2_HOLD_BARRIER, 0x89000000, 2, 0x57}, + + {SCORPION_VIU_DUAL_CYCLE, 0x80000001, 4, 0x5c}, + {SCORPION_VIU_SINGLE_CYCLE, 0x80000100, 4, 0x5d}, + {SCORPION_VX_PIPE_WAR_STALL_CYCLES, 0x80000005, 4, 0x5c}, + {SCORPION_VX_PIPE_WAW_STALL_CYCLES, 0x80000500, 4, 0x5d}, + {SCORPION_VX_PIPE_RAW_STALL_CYCLES, 0x80050000, 4, 0x5e}, + {SCORPION_VX_PIPE_LOAD_USE_STALL, 0x80000007, 4, 0x5c}, + {SCORPION_VS_PIPE_WAR_STALL_CYCLES, 0x80000008, 4, 0x5c}, + {SCORPION_VS_PIPE_WAW_STALL_CYCLES, 0x80000800, 4, 0x5d}, + {SCORPION_VS_PIPE_RAW_STALL_CYCLES, 0x80080000, 4, 0x5e}, + {SCORPION_EXCEPTIONS_INV_OPERATION, 0x8000000b, 4, 0x5c}, + {SCORPION_EXCEPTIONS_DIV_BY_ZERO, 0x80000b00, 4, 0x5d}, + {SCORPION_COND_INST_FAIL_VX_PIPE, 0x800b0000, 4, 0x5e}, + {SCORPION_COND_INST_FAIL_VS_PIPE, 0x8b000000, 4, 0x5f}, + {SCORPION_EXCEPTIONS_OVERFLOW, 0x8000000c, 4, 0x5c}, + {SCORPION_EXCEPTIONS_UNDERFLOW, 0x80000c00, 4, 0x5d}, + {SCORPION_EXCEPTIONS_DENORM, 0x8c000000, 4, 0x5f}, + +#ifdef CONFIG_ARCH_MSM_SCORPIONMP + {SCORPIONMP_NUM_BARRIERS, 0x80000e00, 3, 0x59}, + {SCORPIONMP_BARRIER_CYCLES, 0x800e0000, 3, 0x5a}, +#else + {SCORPION_BANK_AB_HIT, 0x80000001, 3, 0x58}, + {SCORPION_BANK_AB_ACCESS, 0x80000100, 3, 0x59}, + {SCORPION_BANK_CD_HIT, 0x80010000, 3, 0x5a}, + {SCORPION_BANK_CD_ACCESS, 0x81000000, 3, 0x5b}, + {SCORPION_BANK_AB_DSIDE_HIT, 0x80000002, 3, 0x58}, + {SCORPION_BANK_AB_DSIDE_ACCESS, 0x80000200, 3, 0x59}, + {SCORPION_BANK_CD_DSIDE_HIT, 0x80020000, 3, 0x5a}, + {SCORPION_BANK_CD_DSIDE_ACCESS, 0x82000000, 3, 0x5b}, + {SCORPION_BANK_AB_ISIDE_HIT, 0x80000003, 3, 0x58}, + {SCORPION_BANK_AB_ISIDE_ACCESS, 0x80000300, 3, 0x59}, + {SCORPION_BANK_CD_ISIDE_HIT, 0x80030000, 3, 0x5a}, + {SCORPION_BANK_CD_ISIDE_ACCESS, 0x83000000, 3, 0x5b}, + {SCORPION_ISIDE_RD_WAIT, 0x80000009, 3, 0x58}, + {SCORPION_DSIDE_RD_WAIT, 0x80090000, 3, 0x5a}, + {SCORPION_BANK_BYPASS_WRITE, 0x8000000a, 3, 0x58}, + {SCORPION_BANK_AB_NON_CASTOUT, 0x8000000c, 3, 0x58}, + {SCORPION_BANK_AB_L2_CASTOUT, 0x80000c00, 3, 0x59}, + {SCORPION_BANK_CD_NON_CASTOUT, 0x800c0000, 3, 0x5a}, + {SCORPION_BANK_CD_L2_CASTOUT, 0x8c000000, 3, 0x5b}, +#endif +}; + +static inline void scorpion_pmnc_write(u32 val) +{ + val &= PMNC_MASK; + asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r" (val)); +} + +static inline u32 scorpion_pmnc_read(void) +{ + u32 val; + asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); + return val; +} + +static inline u32 scorpion_ccnt_read(void) +{ + u32 val; + asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); + return val; +} + +static inline u32 scorpion_cntn_read(void) +{ + u32 val; + asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); + return val; +} + +static inline u32 scorpion_pmnc_enable_counter(unsigned int cnt) +{ + u32 val; + + if (cnt >= CNTMAX) { + pr_err("gator: CPU%u enabling wrong PMNC counter %d\n", smp_processor_id(), cnt); + return -1; + } + + if (cnt == CCNT) + val = CCNT_REG; + else + val = (1 << (cnt - CNT0)); + + asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val)); + + return cnt; +} + +static inline u32 scorpion_pmnc_disable_counter(unsigned int cnt) +{ + u32 val; + + if (cnt >= CNTMAX) { + pr_err("gator: CPU%u disabling wrong PMNC counter %d\n", smp_processor_id(), cnt); + return -1; + } + + if (cnt == CCNT) + val = CCNT_REG; + else + val = (1 << (cnt - CNT0)); + + asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val)); + + return cnt; +} + +static inline int scorpion_pmnc_select_counter(unsigned int cnt) +{ + u32 val; + + if ((cnt == CCNT) || (cnt >= CNTMAX)) { + pr_err("gator: CPU%u selecting wrong PMNC counter %d\n", smp_processor_id(), cnt); + return -1; + } + + val = (cnt - CNT0); + asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val)); + + return cnt; +} + +static u32 scorpion_read_lpm0(void) +{ + u32 val; + asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val)); + return val; +} + +static void scorpion_write_lpm0(u32 val) +{ + asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val)); +} + +static u32 scorpion_read_lpm1(void) +{ + u32 val; + asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val)); + return val; +} + +static void scorpion_write_lpm1(u32 val) +{ + asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val)); +} + +static u32 scorpion_read_lpm2(void) +{ + u32 val; + asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val)); + return val; +} + +static void scorpion_write_lpm2(u32 val) +{ + asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val)); +} + +static u32 scorpion_read_l2lpm(void) +{ + u32 val; + asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val)); + return val; +} + +static void scorpion_write_l2lpm(u32 val) +{ + asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val)); +} + +static u32 scorpion_read_vlpm(void) +{ + u32 val; + asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val)); + return val; +} + +static void scorpion_write_vlpm(u32 val) +{ + asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val)); +} + +struct scorpion_access_funcs { + u32 (*read) (void); + void (*write) (u32); +}; + +struct scorpion_access_funcs scor_func[] = { + {scorpion_read_lpm0, scorpion_write_lpm0}, + {scorpion_read_lpm1, scorpion_write_lpm1}, + {scorpion_read_lpm2, scorpion_write_lpm2}, + {scorpion_read_l2lpm, scorpion_write_l2lpm}, + {scorpion_read_vlpm, scorpion_write_vlpm}, +}; + +u32 venum_orig_val; +u32 fp_orig_val; + +static void scorpion_pre_vlpm(void) +{ + u32 venum_new_val; + u32 fp_new_val; + + /* CPACR Enable CP10 access*/ + asm volatile("mrc p15, 0, %0, c1, c0, 2" : "=r" (venum_orig_val)); + venum_new_val = venum_orig_val | 0x00300000; + asm volatile("mcr p15, 0, %0, c1, c0, 2" : : "r" (venum_new_val)); + /* Enable FPEXC */ + asm volatile("mrc p10, 7, %0, c8, c0, 0" : "=r" (fp_orig_val)); + fp_new_val = fp_orig_val | 0x40000000; + asm volatile("mcr p10, 7, %0, c8, c0, 0" : : "r" (fp_new_val)); +} + +static void scorpion_post_vlpm(void) +{ + /* Restore FPEXC*/ + asm volatile("mcr p10, 7, %0, c8, c0, 0" : : "r" (fp_orig_val)); + /* Restore CPACR*/ + asm volatile("mcr p15, 0, %0, c1, c0, 2" : : "r" (venum_orig_val)); +} + +#define COLMN0MASK 0x000000ff +#define COLMN1MASK 0x0000ff00 +#define COLMN2MASK 0x00ff0000 +static u32 scorpion_get_columnmask(u32 setval) +{ + if (setval & COLMN0MASK) + return 0xffffff00; + else if (setval & COLMN1MASK) + return 0xffff00ff; + else if (setval & COLMN2MASK) + return 0xff00ffff; + else + return 0x80ffffff; +} + +static void scorpion_evt_setup(u32 gr, u32 setval) +{ + u32 val; + if (gr == 4) + scorpion_pre_vlpm(); + val = scorpion_get_columnmask(setval) & scor_func[gr].read(); + val = val | setval; + scor_func[gr].write(val); + if (gr == 4) + scorpion_post_vlpm(); +} + +static int get_scorpion_evtinfo(unsigned int evt_type, struct scorp_evt *evtinfo) +{ + u32 idx; + if ((evt_type < 0x4c) || (evt_type >= MSM_MAX_EVT)) + return 0; + idx = evt_type - 0x4c; + if (sc_evt[idx].evt_type == evt_type) { + evtinfo->val = sc_evt[idx].val; + evtinfo->grp = sc_evt[idx].grp; + evtinfo->evt_type_act = sc_evt[idx].evt_type_act; + return 1; + } + return 0; +} + +static inline void scorpion_pmnc_write_evtsel(unsigned int cnt, u32 val) +{ + if (scorpion_pmnc_select_counter(cnt) == cnt) { + if (val < 0x40) { + asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val)); + } else { + u32 zero = 0; + struct scorp_evt evtinfo; + // extract evtinfo.grp and evtinfo.tevt_type_act from val + if (get_scorpion_evtinfo(val, &evtinfo) == 0) return; + asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (evtinfo.evt_type_act)); + asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (zero)); + scorpion_evt_setup(evtinfo.grp, val); + } + } +} + +static void scorpion_pmnc_reset_counter(unsigned int cnt) +{ + u32 val = 0; + + if (cnt == CCNT) { + scorpion_pmnc_disable_counter(cnt); + + asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (val)); + + if (pmnc_enabled[cnt] != 0) + scorpion_pmnc_enable_counter(cnt); + + } else if (cnt >= CNTMAX) { + pr_err("gator: CPU%u resetting wrong PMNC counter %d\n", smp_processor_id(), cnt); + } else { + scorpion_pmnc_disable_counter(cnt); + + if (scorpion_pmnc_select_counter(cnt) == cnt) + asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (val)); + + if (pmnc_enabled[cnt] != 0) + scorpion_pmnc_enable_counter(cnt); + } +} + +static int gator_events_scorpion_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + int i; + + for (i = 0; i < pmnc_counters; i++) { + char buf[40]; + if (i == 0) { + snprintf(buf, sizeof buf, "%s_ccnt", pmnc_name); + } else { + snprintf(buf, sizeof buf, "%s_cnt%d", pmnc_name, i-1); + } + dir = gatorfs_mkdir(sb, root, buf); + if (!dir) { + return -1; + } + gatorfs_create_ulong(sb, dir, "enabled", &pmnc_enabled[i]); + gatorfs_create_ro_ulong(sb, dir, "key", &pmnc_key[i]); + if (i > 0) { + gatorfs_create_ulong(sb, dir, "event", &pmnc_event[i]); + } + } + + return 0; +} + +static int gator_events_scorpion_online(int** buffer) +{ + unsigned int cnt, len = 0, cpu = smp_processor_id(); + + if (scorpion_pmnc_read() & PMNC_E) { + scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E); + } + + /* Initialize & Reset PMNC: C bit and P bit */ + scorpion_pmnc_write(PMNC_P | PMNC_C); + + for (cnt = CCNT; cnt < CNTMAX; cnt++) { + unsigned long event; + + per_cpu(perfPrev, smp_processor_id())[cnt] = 0; + + if (!pmnc_enabled[cnt]) + continue; + + // disable counter + scorpion_pmnc_disable_counter(cnt); + + event = pmnc_event[cnt] & 255; + + // Set event (if destined for PMNx counters), We don't need to set the event if it's a cycle count + if (cnt != CCNT) + scorpion_pmnc_write_evtsel(cnt, event); + + // reset counter + scorpion_pmnc_reset_counter(cnt); + + // Enable counter, do not enable interrupt for this counter + scorpion_pmnc_enable_counter(cnt); + } + + // enable + scorpion_pmnc_write(scorpion_pmnc_read() | PMNC_E); + + // read the counters and toss the invalid data, return zero instead + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (pmnc_enabled[cnt]) { + int value; + if (cnt == CCNT) { + value = scorpion_ccnt_read(); + } else if (scorpion_pmnc_select_counter(cnt) == cnt) { + value = scorpion_cntn_read(); + } else { + value = 0; + } + scorpion_pmnc_reset_counter(cnt); + per_cpu(perfPrev, cpu)[cnt] = 0; + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = 0; + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; +} + +static int gator_events_scorpion_offline(int** buffer) +{ + scorpion_pmnc_write(scorpion_pmnc_read() & ~PMNC_E); + return 0; +} + +static void gator_events_scorpion_stop(void) +{ + unsigned int cnt; + + for (cnt = CCNT; cnt < CNTMAX; cnt++) { + pmnc_enabled[cnt] = 0; + pmnc_event[cnt] = 0; + } +} + +static int gator_events_scorpion_read(int **buffer) +{ + int cnt, len = 0; + int cpu = smp_processor_id(); + + for (cnt = 0; cnt < pmnc_counters; cnt++) { + if (pmnc_enabled[cnt]) { + int value; + if (cnt == CCNT) { + value = scorpion_ccnt_read(); + } else if (scorpion_pmnc_select_counter(cnt) == cnt) { + value = scorpion_cntn_read(); + } else { + value = 0; + } + scorpion_pmnc_reset_counter(cnt); + if (value != per_cpu(perfPrev, cpu)[cnt]) { + per_cpu(perfPrev, cpu)[cnt] = value; + per_cpu(perfCnt, cpu)[len++] = pmnc_key[cnt]; + per_cpu(perfCnt, cpu)[len++] = value; + } + } + } + + if (buffer) + *buffer = per_cpu(perfCnt, cpu); + + return len; +} + +static struct gator_interface gator_events_scorpion_interface = { + .create_files = gator_events_scorpion_create_files, + .stop = gator_events_scorpion_stop, + .online = gator_events_scorpion_online, + .offline = gator_events_scorpion_offline, + .read = gator_events_scorpion_read, +}; + +int gator_events_scorpion_init(void) +{ + unsigned int cnt; + + switch (gator_cpuid()) { + case SCORPION: + pmnc_name = "Scorpion"; + pmnc_counters = 4; + break; + case SCORPIONMP: + pmnc_name = "ScorpionMP"; + pmnc_counters = 4; + break; + default: + return -1; + } + + pmnc_counters++; // CNT[n] + CCNT + + for (cnt = CCNT; cnt < CNTMAX; cnt++) { + pmnc_enabled[cnt] = 0; + pmnc_event[cnt] = 0; + pmnc_key[cnt] = gator_events_get_key(); + } + + return gator_events_install(&gator_events_scorpion_interface); +} + +gator_events_init(gator_events_scorpion_init); + +#else +int gator_events_scorpion_init(void) +{ + return -1; +} +#endif diff --git a/drivers/gator/driver/gator_fs.c b/drivers/gator/driver/gator_fs.c new file mode 100644 index 00000000000..8277c3aac3e --- /dev/null +++ b/drivers/gator/driver/gator_fs.c @@ -0,0 +1,284 @@ +/** + * @file gatorfs.c + * + * @remark Copyright 2002 OProfile authors + * @remark Read the file COPYING + * + * @author John Levon + * + * A simple filesystem for configuration and + * access of oprofile. + */ + +#include +#include +#include +#include +#include + +#define gatorfs_MAGIC 0x24051020 +#define TMPBUFSIZE 50 +DEFINE_SPINLOCK(gatorfs_lock); + +static struct inode *gatorfs_get_inode(struct super_block *sb, int mode) +{ + struct inode *inode = new_inode(sb); + + if (inode) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37) + inode->i_ino = get_next_ino(); +#endif + inode->i_mode = mode; + inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; + } + return inode; +} + +static const struct super_operations s_ops = { + .statfs = simple_statfs, + .drop_inode = generic_delete_inode, +}; + +ssize_t gatorfs_str_to_user(char const *str, char __user *buf, size_t count, loff_t *offset) +{ + return simple_read_from_buffer(buf, count, offset, str, strlen(str)); +} + +ssize_t gatorfs_ulong_to_user(unsigned long val, char __user *buf, size_t count, loff_t *offset) +{ + char tmpbuf[TMPBUFSIZE]; + size_t maxlen = snprintf(tmpbuf, TMPBUFSIZE, "%lu\n", val); + if (maxlen > TMPBUFSIZE) + maxlen = TMPBUFSIZE; + return simple_read_from_buffer(buf, count, offset, tmpbuf, maxlen); +} + +int gatorfs_ulong_from_user(unsigned long *val, char const __user *buf, size_t count) +{ + char tmpbuf[TMPBUFSIZE]; + unsigned long flags; + + if (!count) + return 0; + + if (count > TMPBUFSIZE - 1) + return -EINVAL; + + memset(tmpbuf, 0x0, TMPBUFSIZE); + + if (copy_from_user(tmpbuf, buf, count)) + return -EFAULT; + + spin_lock_irqsave(&gatorfs_lock, flags); + *val = simple_strtoul(tmpbuf, NULL, 0); + spin_unlock_irqrestore(&gatorfs_lock, flags); + return 0; +} + +static ssize_t ulong_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + unsigned long *val = file->private_data; + return gatorfs_ulong_to_user(*val, buf, count, offset); +} + +static ssize_t ulong_write_file(struct file *file, char const __user *buf, size_t count, loff_t *offset) +{ + unsigned long *value = file->private_data; + int retval; + + if (*offset) + return -EINVAL; + + retval = gatorfs_ulong_from_user(value, buf, count); + + if (retval) + return retval; + return count; +} + +static int default_open(struct inode *inode, struct file *filp) +{ + if (inode->i_private) + filp->private_data = inode->i_private; + return 0; +} + +static const struct file_operations ulong_fops = { + .read = ulong_read_file, + .write = ulong_write_file, + .open = default_open, +}; + +static const struct file_operations ulong_ro_fops = { + .read = ulong_read_file, + .open = default_open, +}; + +static struct dentry *__gatorfs_create_file(struct super_block *sb, + struct dentry *root, char const *name, const struct file_operations *fops, + int perm) +{ + struct dentry *dentry; + struct inode *inode; + + dentry = d_alloc_name(root, name); + if (!dentry) + return NULL; + inode = gatorfs_get_inode(sb, S_IFREG | perm); + if (!inode) { + dput(dentry); + return NULL; + } + inode->i_fop = fops; + d_add(dentry, inode); + return dentry; +} + +int gatorfs_create_ulong(struct super_block *sb, struct dentry *root, + char const *name, unsigned long *val) +{ + struct dentry *d = __gatorfs_create_file(sb, root, name, + &ulong_fops, 0644); + if (!d) + return -EFAULT; + + d->d_inode->i_private = val; + return 0; +} + +int gatorfs_create_ro_ulong(struct super_block *sb, struct dentry *root, + char const *name, unsigned long *val) +{ + struct dentry *d = __gatorfs_create_file(sb, root, name, + &ulong_ro_fops, 0444); + if (!d) + return -EFAULT; + + d->d_inode->i_private = val; + return 0; +} + +static ssize_t atomic_read_file(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + atomic_t *val = file->private_data; + return gatorfs_ulong_to_user(atomic_read(val), buf, count, offset); +} + +static const struct file_operations atomic_ro_fops = { + .read = atomic_read_file, + .open = default_open, +}; + +int gatorfs_create_ro_atomic(struct super_block *sb, struct dentry *root, + char const *name, atomic_t *val) +{ + struct dentry *d = __gatorfs_create_file(sb, root, name, + &atomic_ro_fops, 0444); + if (!d) + return -EFAULT; + + d->d_inode->i_private = val; + return 0; +} + +int gatorfs_create_file(struct super_block *sb, struct dentry *root, + char const *name, const struct file_operations *fops) +{ + if (!__gatorfs_create_file(sb, root, name, fops, 0644)) + return -EFAULT; + return 0; +} + +int gatorfs_create_file_perm(struct super_block *sb, struct dentry *root, + char const *name, const struct file_operations *fops, int perm) +{ + if (!__gatorfs_create_file(sb, root, name, fops, perm)) + return -EFAULT; + return 0; +} + +struct dentry *gatorfs_mkdir(struct super_block *sb, + struct dentry *root, char const *name) +{ + struct dentry *dentry; + struct inode *inode; + + dentry = d_alloc_name(root, name); + if (!dentry) + return NULL; + inode = gatorfs_get_inode(sb, S_IFDIR | 0755); + if (!inode) { + dput(dentry); + return NULL; + } + inode->i_op = &simple_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + d_add(dentry, inode); + return dentry; +} + +static int gatorfs_fill_super(struct super_block *sb, void *data, int silent) +{ + struct inode *root_inode; + struct dentry *root_dentry; + + sb->s_blocksize = PAGE_CACHE_SIZE; + sb->s_blocksize_bits = PAGE_CACHE_SHIFT; + sb->s_magic = gatorfs_MAGIC; + sb->s_op = &s_ops; + sb->s_time_gran = 1; + + root_inode = gatorfs_get_inode(sb, S_IFDIR | 0755); + if (!root_inode) + return -ENOMEM; + root_inode->i_op = &simple_dir_inode_operations; + root_inode->i_fop = &simple_dir_operations; + root_dentry = d_alloc_root(root_inode); + if (!root_dentry) { + iput(root_inode); + return -ENOMEM; + } + + sb->s_root = root_dentry; + + gator_op_create_files(sb, root_dentry); + + // FIXME: verify kill_litter_super removes our dentries + return 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 39) +static int gatorfs_get_sb(struct file_system_type *fs_type, + int flags, const char *dev_name, void *data, struct vfsmount *mnt) +{ + return get_sb_single(fs_type, flags, data, gatorfs_fill_super, mnt); +} +#else +static struct dentry *gatorfs_mount(struct file_system_type *fs_type, + int flags, const char *dev_name, void *data) +{ + return mount_nodev(fs_type, flags, data, gatorfs_fill_super); +} +#endif + +static struct file_system_type gatorfs_type = { + .owner = THIS_MODULE, + .name = "gatorfs", +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 39) + .get_sb = gatorfs_get_sb, +#else + .mount = gatorfs_mount, +#endif + + .kill_sb = kill_litter_super, +}; + +int __init gatorfs_register(void) +{ + return register_filesystem(&gatorfs_type); +} + +void gatorfs_unregister(void) +{ + unregister_filesystem(&gatorfs_type); +} diff --git a/drivers/gator/driver/gator_hrtimer_gator.c b/drivers/gator/driver/gator_hrtimer_gator.c new file mode 100644 index 00000000000..5896b3c39ae --- /dev/null +++ b/drivers/gator/driver/gator_hrtimer_gator.c @@ -0,0 +1,76 @@ +/** + * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +// gator_hrtimer_perf.c is used if perf is supported +// update, gator_hrtimer_gator.c always used until issues resolved with perf hrtimers +#if 1 + +void (*callback)(void); +DEFINE_PER_CPU(struct hrtimer, percpu_hrtimer); +static ktime_t profiling_interval; +static void gator_hrtimer_online(int cpu); +static void gator_hrtimer_offline(int cpu); + +static enum hrtimer_restart gator_hrtimer_notify(struct hrtimer *hrtimer) +{ + hrtimer_forward_now(hrtimer, profiling_interval); + (*callback)(); + return HRTIMER_RESTART; +} + +static void gator_hrtimer_switch_cpus_online(void *unused) +{ + gator_hrtimer_online(smp_processor_id()); +} + +static void gator_hrtimer_online(int cpu) +{ + struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); + if (cpu != smp_processor_id()) { + smp_call_function_single(cpu, gator_hrtimer_switch_cpus_online, NULL, 1); + return; + } + + hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrtimer->function = gator_hrtimer_notify; + hrtimer_start(hrtimer, profiling_interval, HRTIMER_MODE_REL_PINNED); +} + +static void gator_hrtimer_switch_cpus_offline(void *unused) +{ + gator_hrtimer_offline(smp_processor_id()); +} + +static void gator_hrtimer_offline(int cpu) +{ + struct hrtimer *hrtimer = &per_cpu(percpu_hrtimer, cpu); + if (cpu != smp_processor_id()) { + smp_call_function_single(cpu, gator_hrtimer_switch_cpus_offline, NULL, 1); + return; + } + + hrtimer_cancel(hrtimer); +} + +static int gator_hrtimer_init(int interval, void (*func)(void)) +{ + (callback) = (func); + + // calculate profiling interval + profiling_interval = ns_to_ktime(1000000000UL / interval); + + return 0; +} + +static void gator_hrtimer_shutdown(void) +{ + /* empty */ +} + +#endif diff --git a/drivers/gator/driver/gator_hrtimer_perf.c b/drivers/gator/driver/gator_hrtimer_perf.c new file mode 100644 index 00000000000..7c0333f6445 --- /dev/null +++ b/drivers/gator/driver/gator_hrtimer_perf.c @@ -0,0 +1,113 @@ +/** + * Copyright (C) ARM Limited 2011-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +// gator_hrtimer_gator.c is used if perf is not supported +// update, gator_hrtimer_gator.c always used until issues resolved with perf hrtimers +#if 0 + +// Note: perf Cortex support added in 2.6.35 and PERF_COUNT_SW_CPU_CLOCK/hrtimer broken on 2.6.35 and 2.6.36 +// not relevant as this code is not active until 3.0.0, but wanted to document the issue + +void (*callback)(void); +static int profiling_interval; +static DEFINE_PER_CPU(struct perf_event *, perf_hrtimer); +static DEFINE_PER_CPU(struct perf_event_attr *, perf_hrtimer_attr); + +static void gator_hrtimer_shutdown(void); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) +static void hrtimer_overflow_handler(struct perf_event *event, int unused, struct perf_sample_data *data, struct pt_regs *regs) +#else +static void hrtimer_overflow_handler(struct perf_event *event, struct perf_sample_data *data, struct pt_regs *regs) +#endif +{ + (*callback)(); +} + +static int gator_online_single_hrtimer(int cpu) +{ + if (per_cpu(perf_hrtimer, cpu) != 0 || per_cpu(perf_hrtimer_attr, cpu) == 0) + return 0; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 1, 0) + per_cpu(perf_hrtimer, cpu) = perf_event_create_kernel_counter(per_cpu(perf_hrtimer_attr, cpu), cpu, 0, hrtimer_overflow_handler); +#else + per_cpu(perf_hrtimer, cpu) = perf_event_create_kernel_counter(per_cpu(perf_hrtimer_attr, cpu), cpu, 0, hrtimer_overflow_handler, 0); +#endif + if (IS_ERR(per_cpu(perf_hrtimer, cpu))) { + per_cpu(perf_hrtimer, cpu) = NULL; + return -1; + } + + if (per_cpu(perf_hrtimer, cpu)->state != PERF_EVENT_STATE_ACTIVE) { + perf_event_release_kernel(per_cpu(perf_hrtimer, cpu)); + per_cpu(perf_hrtimer, cpu) = NULL; + return -1; + } + + return 0; +} + +static void gator_hrtimer_online(int cpu) +{ + if (gator_online_single_hrtimer(cpu) < 0) { + pr_debug("gator: unable to online the hrtimer on cpu%d\n", cpu); + } +} + +static void gator_hrtimer_offline(int cpu) +{ + if (per_cpu(perf_hrtimer, cpu)) { + perf_event_release_kernel(per_cpu(perf_hrtimer, cpu)); + per_cpu(perf_hrtimer, cpu) = NULL; + } +} + +static int gator_hrtimer_init(int interval, void (*func)(void)) +{ + u32 size = sizeof(struct perf_event_attr); + int cpu; + + callback = func; + + // calculate profiling interval + profiling_interval = 1000000000 / interval; + + for_each_present_cpu(cpu) { + per_cpu(perf_hrtimer, cpu) = 0; + per_cpu(perf_hrtimer_attr, cpu) = kmalloc(size, GFP_KERNEL); + if (per_cpu(perf_hrtimer_attr, cpu) == 0) { + gator_hrtimer_shutdown(); + return -1; + } + + memset(per_cpu(perf_hrtimer_attr, cpu), 0, size); + per_cpu(perf_hrtimer_attr, cpu)->type = PERF_TYPE_SOFTWARE; + per_cpu(perf_hrtimer_attr, cpu)->size = size; + per_cpu(perf_hrtimer_attr, cpu)->config = PERF_COUNT_SW_CPU_CLOCK; + per_cpu(perf_hrtimer_attr, cpu)->sample_period = profiling_interval; + per_cpu(perf_hrtimer_attr, cpu)->pinned = 1; + } + + return 0; +} + +static void gator_hrtimer_shutdown(void) +{ + int cpu; + + for_each_present_cpu(cpu) { + if (per_cpu(perf_hrtimer_attr, cpu)) { + kfree(per_cpu(perf_hrtimer_attr, cpu)); + per_cpu(perf_hrtimer_attr, cpu) = NULL; + } + } +} + +#endif diff --git a/drivers/gator/driver/gator_main.c b/drivers/gator/driver/gator_main.c new file mode 100644 index 00000000000..6341ef6f051 --- /dev/null +++ b/drivers/gator/driver/gator_main.c @@ -0,0 +1,1109 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +static unsigned long gator_protocol_version = 8; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gator.h" +#include "gator_events.h" + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32) +#error kernels prior to 2.6.32 are not supported +#endif + +#if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING) +#error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined +#endif + +#ifndef CONFIG_PROFILING +#error gator requires the kernel to have CONFIG_PROFILING defined +#endif + +#ifndef CONFIG_HIGH_RES_TIMERS +#error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined +#endif + +#if defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS) +#error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems +#endif + +#if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT)) +#ifndef CONFIG_PERF_EVENTS +#warning gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters +#elif !defined CONFIG_HW_PERF_EVENTS +#warning gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters +#endif +#endif + +#if (!(GATOR_CPU_FREQ_SUPPORT)) +#warning gator requires kernel version 2.6.38 or greater and CONFIG_CPU_FREQ defined in order to enable the CPU Freq timeline chart +#endif + +/****************************************************************************** + * DEFINES + ******************************************************************************/ +#define TIMER_BUFFER_SIZE_DEFAULT (512*1024) +#define EVENT_BUFFER_SIZE_DEFAULT (128*1024) + +#define NO_COOKIE 0UL +#define INVALID_COOKIE ~0UL + +#define FRAME_HRTIMER 1 +#define FRAME_EVENT 2 +#define FRAME_ANNOTATE 3 + +#define MESSAGE_COOKIE 1 +#define MESSAGE_COUNTERS 3 +#define MESSAGE_START_BACKTRACE 5 +#define MESSAGE_END_BACKTRACE 7 +#define MESSAGE_SCHEDULER_TRACE 9 +#define MESSAGE_PID_NAME 11 +#define MESSAGE_GPU_TRACE 13 +#define MESSAGE_OVERFLOW 127 + +#define MAXSIZE_PACK32 5 +#define MAXSIZE_PACK64 9 + +#if defined(__arm__) +#define PC_REG regs->ARM_pc +#else +#define PC_REG regs->ip +#endif + +enum {TIMER_BUF, EVENT_BUF, NUM_GATOR_BUFS}; + +/****************************************************************************** + * Globals + ******************************************************************************/ +static unsigned long gator_cpu_cores; +static unsigned long userspace_buffer_size; +static unsigned long gator_backtrace_depth; + +static unsigned long gator_started; +static unsigned long gator_buffer_opened; +static unsigned long gator_timer_count; +static unsigned long gator_streaming; +static DEFINE_MUTEX(start_mutex); +static DEFINE_MUTEX(gator_buffer_mutex); + +bool event_based_sampling; + +static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait); + +static void buffer_check(int cpu, int buftype); + +/****************************************************************************** + * Prototypes + ******************************************************************************/ +static bool buffer_check_space(int cpu, int buftype, int bytes); +static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x); +static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x); +static void gator_buffer_write_string(int cpu, int buftype, char *x); +static int gator_write_packed_int(char *buffer, unsigned int x); +static int gator_write_packed_int64(char *buffer, unsigned long long x); +static void gator_add_trace(int cpu, int buftype, unsigned int address); +static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs); +static uint64_t gator_get_time(void); + +static uint32_t gator_buffer_size[NUM_GATOR_BUFS]; +static uint32_t gator_buffer_mask[NUM_GATOR_BUFS]; +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read); +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write); +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit); +static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available); +static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer); +static DEFINE_PER_CPU(uint64_t, emit_overflow); + +/****************************************************************************** + * Application Includes + ******************************************************************************/ +#include "gator_hrtimer_perf.c" +#include "gator_hrtimer_gator.c" +#include "gator_cookies.c" +#include "gator_trace_sched.c" +#include "gator_trace_gpu.c" +#include "gator_backtrace.c" +#include "gator_annotate.c" +#include "gator_fs.c" +#include "gator_ebs.c" +#include "gator_pack.c" + +/****************************************************************************** + * Misc + ******************************************************************************/ +#if defined(__arm__) +u32 gator_cpuid(void) +{ + u32 val; + asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val)); + return (val >> 4) & 0xfff; +} +#endif + +/****************************************************************************** + * Commit interface + ******************************************************************************/ +static bool buffer_commit_ready(int* cpu, int* buftype) +{ + int cpu_x, x; + for_each_present_cpu(cpu_x) { + for (x = 0; x < NUM_GATOR_BUFS; x++) + if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) { + *cpu = cpu_x; + *buftype = x; + return true; + } + } + return false; +} + +/****************************************************************************** + * Buffer management + ******************************************************************************/ +static bool buffer_check_space(int cpu, int buftype, int bytes) +{ + int remaining, filled; + + filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_read, cpu)[buftype]; + if (filled < 0) { + filled += gator_buffer_size[buftype]; + } + + remaining = gator_buffer_size[buftype] - filled; + + if (per_cpu(buffer_space_available, cpu)[buftype]) { + // Give some extra room; also allows space to insert the overflow error packet + remaining -= 200; + } else { + // Hysteresis, prevents multiple overflow messages + remaining -= 2000; + } + + if (remaining < bytes) { + if (per_cpu(buffer_space_available, cpu)[buftype] == true) { + // overflow packet to be emitted at a later time, as we may be in the middle of writing a message, e.g. counters + per_cpu(emit_overflow, cpu) = gator_get_time(); + pr_err("overflow: remaining = %d\n", gator_buffer_size[buftype] - filled); + } + per_cpu(buffer_space_available, cpu)[buftype] = false; + } else { + per_cpu(buffer_space_available, cpu)[buftype] = true; + } + + return per_cpu(buffer_space_available, cpu)[buftype]; +} + +static void gator_buffer_write_bytes(int cpu, int buftype, char *x, int len) +{ + int i; + u32 write = per_cpu(gator_buffer_write, cpu)[buftype]; + u32 mask = gator_buffer_mask[buftype]; + char* buffer = per_cpu(gator_buffer, cpu)[buftype]; + + for (i = 0; i < len; i++) { + buffer[write] = x[i]; + write = (write + 1) & mask; + } + + per_cpu(gator_buffer_write, cpu)[buftype] = write; +} + +static void gator_buffer_write_string(int cpu, int buftype, char *x) +{ + int len = strlen(x); + gator_buffer_write_packed_int(cpu, buftype, len); + gator_buffer_write_bytes(cpu, buftype, x, len); +} + +static void gator_buffer_header(int cpu, int buftype) +{ + int frame; + + if (buftype == TIMER_BUF) + frame = FRAME_HRTIMER; + else if (buftype == EVENT_BUF) + frame = FRAME_EVENT; + else + frame = -1; + + gator_buffer_write_packed_int(cpu, buftype, frame); + gator_buffer_write_packed_int(cpu, buftype, cpu); +} + +static void gator_commit_buffer(int cpu, int buftype) +{ + per_cpu(gator_buffer_commit, cpu)[buftype] = per_cpu(gator_buffer_write, cpu)[buftype]; + gator_buffer_header(cpu, buftype); + wake_up(&gator_buffer_wait); +} + +static void buffer_check(int cpu, int buftype) +{ + int filled = per_cpu(gator_buffer_write, cpu)[buftype] - per_cpu(gator_buffer_commit, cpu)[buftype]; + if (filled < 0) { + filled += gator_buffer_size[buftype]; + } + if (filled >= ((gator_buffer_size[buftype] * 3) / 4)) { + gator_commit_buffer(cpu, buftype); + } +} + +static void gator_add_trace(int cpu, int buftype, unsigned int address) +{ + off_t offset = 0; + unsigned long cookie = get_address_cookie(cpu, buftype, current, address & ~1, &offset); + + if (cookie == NO_COOKIE || cookie == INVALID_COOKIE) { + offset = address; + } + + gator_buffer_write_packed_int(cpu, buftype, offset & ~1); + gator_buffer_write_packed_int(cpu, buftype, cookie); +} + +static void gator_add_sample(int cpu, int buftype, struct pt_regs * const regs) +{ + int inKernel = regs ? !user_mode(regs) : 1; + unsigned long exec_cookie = inKernel ? NO_COOKIE : get_exec_cookie(cpu, buftype, current); + + if (!regs) + return; + + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_START_BACKTRACE); + gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); + gator_buffer_write_packed_int(cpu, buftype, exec_cookie); + gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->tgid); + gator_buffer_write_packed_int(cpu, buftype, (unsigned int)current->pid); + gator_buffer_write_packed_int(cpu, buftype, inKernel); + + if (inKernel) { + kernel_backtrace(cpu, buftype, regs); + } else { + // Cookie+PC + gator_add_trace(cpu, buftype, PC_REG); + + // Backtrace + if (gator_backtrace_depth) + arm_backtrace_eabi(cpu, buftype, regs, gator_backtrace_depth); + } + + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_END_BACKTRACE); +} + +/****************************************************************************** + * hrtimer interrupt processing + ******************************************************************************/ +static LIST_HEAD(gator_events); + +static void gator_timer_interrupt(void) +{ + struct pt_regs * const regs = get_irq_regs(); + int cpu = smp_processor_id(); + int *buffer, len, i, buftype = TIMER_BUF; + long long *buffer64; + struct gator_interface *gi; + + // Output scheduler trace + len = gator_trace_sched_read(&buffer64); + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_SCHEDULER_TRACE); + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); + } + } + + // Output GPU trace + len = gator_trace_gpu_read(&buffer64); + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_GPU_TRACE); + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); + } + } + + // Output counters + if (buffer_check_space(cpu, buftype, MAXSIZE_PACK32 * 2 + MAXSIZE_PACK64)) { + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_COUNTERS); + gator_buffer_write_packed_int64(cpu, buftype, gator_get_time()); + list_for_each_entry(gi, &gator_events, list) { + if (gi->read) { + len = gi->read(&buffer); + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int(cpu, buftype, buffer[i]); + } + } + } else if (gi->read64) { + len = gi->read64(&buffer64); + if (len > 0 && buffer_check_space(cpu, buftype, len * MAXSIZE_PACK64 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, buftype, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, buftype, buffer64[i]); + } + } + } + } + gator_buffer_write_packed_int(cpu, buftype, 0); + } + + // Output backtrace + if (!event_based_sampling && buffer_check_space(cpu, buftype, gator_backtrace_depth * 2 * MAXSIZE_PACK32)) + gator_add_sample(cpu, buftype, regs); + + // Overflow message + if (per_cpu(emit_overflow, cpu)) { + gator_buffer_write_packed_int(cpu, buftype, MESSAGE_OVERFLOW); + gator_buffer_write_packed_int64(cpu, buftype, per_cpu(emit_overflow, cpu)); + per_cpu(emit_overflow, cpu) = 0; + } + + // Check and commit; generally, commit is set to occur once per second + buffer_check(cpu, buftype); +} + +DEFINE_PER_CPU(int, hrtimer_is_active); +static int hrtimer_running; + +// This function runs in interrupt context and on the appropriate core +static void gator_timer_offline(void* unused) +{ + int i, len, cpu = smp_processor_id(); + int* buffer; + long long* buffer64; + + if (per_cpu(hrtimer_is_active, cpu)) { + struct gator_interface *gi; + gator_hrtimer_offline(cpu); + per_cpu(hrtimer_is_active, cpu) = 0; + + // Output scheduler trace + len = gator_trace_sched_offline(&buffer64); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_SCHEDULER_TRACE); + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]); + } + } + + // Output GPU trace + len = gator_trace_gpu_offline(&buffer64); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK64 + 2 * MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_GPU_TRACE); + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) { + gator_buffer_write_packed_int64(cpu, TIMER_BUF, buffer64[i]); + } + } + + // offline any events and output counters + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS); + gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time()); + list_for_each_entry(gi, &gator_events, list) { + if (gi->offline) { + len = gi->offline(&buffer); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) + gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]); + } + } + } + gator_buffer_write_packed_int(cpu, TIMER_BUF, 0); + + gator_commit_buffer(cpu, TIMER_BUF); + } + + if (event_based_sampling) { + gator_commit_buffer(cpu, EVENT_BUF); + } +} + +// This function runs in interrupt context and may be running on a core other than core 'cpu' +static void gator_timer_offline_dispatch(int cpu) +{ + struct gator_interface *gi; + + list_for_each_entry(gi, &gator_events, list) + if (gi->offline_dispatch) + gi->offline_dispatch(cpu); + + gator_event_sampling_offline_dispatch(cpu); +} + +static void gator_timer_stop(void) +{ + int cpu; + + if (hrtimer_running) { + on_each_cpu(gator_timer_offline, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_offline_dispatch(cpu); + } + + hrtimer_running = 0; + gator_hrtimer_shutdown(); + } +} + +// This function runs in interrupt context and on the appropriate core +static void gator_timer_online(void* unused) +{ + int i, len, cpu = smp_processor_id(); + int* buffer; + + if (!per_cpu(hrtimer_is_active, cpu)) { + struct gator_interface *gi; + + // online any events and output counters + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_COUNTERS); + gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time()); + list_for_each_entry(gi, &gator_events, list) { + if (gi->online) { + len = gi->online(&buffer); + if (len > 0 && buffer_check_space(cpu, TIMER_BUF, len * MAXSIZE_PACK32 + MAXSIZE_PACK32)) { + gator_buffer_write_packed_int(cpu, TIMER_BUF, len); + for (i = 0; i < len; i++) + gator_buffer_write_packed_int(cpu, TIMER_BUF, buffer[i]); + } + } + } + gator_buffer_write_packed_int(cpu, TIMER_BUF, 0); + + gator_event_sampling_online(); + + gator_hrtimer_online(cpu); + per_cpu(hrtimer_is_active, cpu) = 1; + } +} + +// This function runs in interrupt context and may be running on a core other than core 'cpu' +static void gator_timer_online_dispatch(int cpu) +{ + struct gator_interface *gi; + + list_for_each_entry(gi, &gator_events, list) + if (gi->online_dispatch) + gi->online_dispatch(cpu); + + gator_event_sampling_online_dispatch(cpu); +} + +int gator_timer_start(unsigned long setup) +{ + int cpu; + + if (!setup) { + pr_err("gator: cannot start due to a system tick value of zero\n"); + return -1; + } else if (hrtimer_running) { + pr_notice("gator: high res timer already running\n"); + return 0; + } + + hrtimer_running = 1; + + if (gator_hrtimer_init(setup, gator_timer_interrupt) == -1) + return -1; + + for_each_online_cpu(cpu) { + gator_timer_online_dispatch(cpu); + } + on_each_cpu(gator_timer_online, NULL, 1); + + return 0; +} + +static uint64_t gator_get_time(void) +{ + struct timespec ts; + uint64_t timestamp; + + getnstimeofday(&ts); + timestamp = timespec_to_ns(&ts); + + return timestamp; +} + +/****************************************************************************** + * cpu hotplug and pm notifiers + ******************************************************************************/ +static int __cpuinit gator_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) +{ + long cpu = (long)hcpu; + + switch (action) { + case CPU_DOWN_PREPARE: + case CPU_DOWN_PREPARE_FROZEN: + smp_call_function_single(cpu, gator_timer_offline, NULL, 1); + gator_timer_offline_dispatch(cpu); + break; + case CPU_ONLINE: + case CPU_ONLINE_FROZEN: + gator_timer_online_dispatch(cpu); + smp_call_function_single(cpu, gator_timer_online, NULL, 1); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block __refdata gator_cpu_notifier = { + .notifier_call = gator_cpu_notify, +}; + +// n.b. calling "on_each_cpu" only runs on those that are online +// Registered linux events are not disabled, so their counters will continue to collect +static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy) +{ + int cpu; + + switch (event) { + case PM_HIBERNATION_PREPARE: + case PM_SUSPEND_PREPARE: + unregister_hotcpu_notifier(&gator_cpu_notifier); + unregister_scheduler_tracepoints(); + on_each_cpu(gator_timer_offline, NULL, 1); + for_each_online_cpu(cpu) { + gator_timer_offline_dispatch(cpu); + } + break; + case PM_POST_HIBERNATION: + case PM_POST_SUSPEND: + for_each_online_cpu(cpu) { + gator_timer_online_dispatch(cpu); + } + on_each_cpu(gator_timer_online, NULL, 1); + register_scheduler_tracepoints(); + register_hotcpu_notifier(&gator_cpu_notifier); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block gator_pm_notifier = { + .notifier_call = gator_pm_notify, +}; + +static int gator_notifier_start(void) +{ + int retval; + retval = register_hotcpu_notifier(&gator_cpu_notifier); + if (retval == 0) + retval = register_pm_notifier(&gator_pm_notifier); + return retval; +} + +static void gator_notifier_stop(void) +{ + unregister_pm_notifier(&gator_pm_notifier); + unregister_hotcpu_notifier(&gator_cpu_notifier); +} + +/****************************************************************************** + * Main + ******************************************************************************/ +int gator_events_install(struct gator_interface *interface) +{ + list_add_tail(&interface->list, &gator_events); + + return 0; +} + +int gator_events_get_key(void) +{ + static int key; + + return key++; +} + +static int gator_init(void) +{ + int i; + + if (gator_annotate_init()) + return -1; + + // events sources (gator_events.h, generated by gator_events.sh) + for (i = 0; i < ARRAY_SIZE(gator_events_list); i++) + if (gator_events_list[i]) + gator_events_list[i](); + + return 0; +} + +static int gator_start(void) +{ + struct gator_interface *gi; + + // start all events + list_for_each_entry(gi, &gator_events, list) { + if (gi->start && gi->start() != 0) { + struct list_head *ptr = gi->list.prev; + + while (ptr != &gator_events) { + gi = list_entry(ptr, struct gator_interface, list); + + if (gi->stop) + gi->stop(); + + ptr = ptr->prev; + } + goto events_failure; + } + } + + // cookies shall be initialized before trace_sched_start() and gator_timer_start() + if (cookies_initialize()) + goto cookies_failure; + if (gator_annotate_start()) + goto annotate_failure; + if (gator_trace_sched_start()) + goto sched_failure; + if (gator_trace_gpu_start()) + goto gpu_failure; + if (gator_event_sampling_start()) + goto event_sampling_failure; + if (gator_timer_start(gator_timer_count)) + goto timer_failure; + if (gator_notifier_start()) + goto notifier_failure; + + return 0; + +notifier_failure: + gator_timer_stop(); +timer_failure: + gator_event_sampling_stop(); +event_sampling_failure: + gator_trace_gpu_stop(); +gpu_failure: + gator_trace_sched_stop(); +sched_failure: + gator_annotate_stop(); +annotate_failure: + cookies_release(); +cookies_failure: + // stop all events + list_for_each_entry(gi, &gator_events, list) + if (gi->stop) + gi->stop(); +events_failure: + + return -1; +} + +static void gator_stop(void) +{ + struct gator_interface *gi; + + // stop all events + list_for_each_entry(gi, &gator_events, list) + if (gi->stop) + gi->stop(); + + gator_annotate_stop(); + gator_trace_sched_stop(); + gator_trace_gpu_stop(); + gator_event_sampling_stop(); + + // stop all interrupt callback reads before tearing down other interfaces + gator_notifier_stop(); // should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined + gator_timer_stop(); +} + +static void gator_exit(void) +{ + gator_annotate_exit(); +} + +/****************************************************************************** + * Filesystem + ******************************************************************************/ +/* fopen("buffer") */ +static int gator_op_setup(void) +{ + int err = 0; + int cpu, i; + + mutex_lock(&start_mutex); + + gator_buffer_size[TIMER_BUF] = userspace_buffer_size; + gator_buffer_mask[TIMER_BUF] = userspace_buffer_size - 1; + + // must be a power of 2 + if (gator_buffer_size[TIMER_BUF] & (gator_buffer_size[TIMER_BUF] - 1)) { + err = -ENOEXEC; + goto setup_error; + } + + gator_buffer_size[EVENT_BUF] = EVENT_BUFFER_SIZE_DEFAULT; + gator_buffer_mask[EVENT_BUF] = gator_buffer_size[EVENT_BUF] - 1; + + // Initialize percpu per buffer variables + for (i = 0; i < NUM_GATOR_BUFS; i++) { + for_each_present_cpu(cpu) { + per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]); + if (!per_cpu(gator_buffer, cpu)[i]) { + err = -ENOMEM; + goto setup_error; + } + + per_cpu(gator_buffer_read, cpu)[i] = 0; + per_cpu(gator_buffer_write, cpu)[i] = 0; + per_cpu(gator_buffer_commit, cpu)[i] = 0; + per_cpu(buffer_space_available, cpu)[i] = true; + per_cpu(emit_overflow, cpu) = 0; + gator_buffer_header(cpu, i); + } + } + +setup_error: + mutex_unlock(&start_mutex); + return err; +} + +/* Actually start profiling (echo 1>/dev/gator/driver/enable) */ +static int gator_op_start(void) +{ + int err = 0; + + mutex_lock(&start_mutex); + + if (gator_started || gator_start()) + err = -EINVAL; + else + gator_started = 1; + + mutex_unlock(&start_mutex); + + return err; +} + +/* echo 0>/dev/gator/driver/enable */ +static void gator_op_stop(void) +{ + mutex_lock(&start_mutex); + + if (gator_started) { + gator_stop(); + + mutex_lock(&gator_buffer_mutex); + + gator_started = 0; + cookies_release(); + wake_up(&gator_buffer_wait); + + mutex_unlock(&gator_buffer_mutex); + } + + mutex_unlock(&start_mutex); +} + +static void gator_shutdown(void) +{ + int cpu, i; + + mutex_lock(&start_mutex); + + gator_annotate_shutdown(); + + for_each_present_cpu(cpu) { + mutex_lock(&gator_buffer_mutex); + for (i = 0; i < NUM_GATOR_BUFS; i++) { + vfree(per_cpu(gator_buffer, cpu)[i]); + per_cpu(gator_buffer, cpu)[i] = NULL; + per_cpu(gator_buffer_read, cpu)[i] = 0; + per_cpu(gator_buffer_write, cpu)[i] = 0; + per_cpu(gator_buffer_commit, cpu)[i] = 0; + per_cpu(buffer_space_available, cpu)[i] = true; + per_cpu(emit_overflow, cpu) = 0; + } + mutex_unlock(&gator_buffer_mutex); + } + + mutex_unlock(&start_mutex); +} + +static int gator_set_backtrace(unsigned long val) +{ + int err = 0; + + mutex_lock(&start_mutex); + + if (gator_started) + err = -EBUSY; + else + gator_backtrace_depth = val; + + mutex_unlock(&start_mutex); + + return err; +} + +static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + return gatorfs_ulong_to_user(gator_started, buf, count, offset); +} + +static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) +{ + unsigned long val; + int retval; + + if (*offset) + return -EINVAL; + + retval = gatorfs_ulong_from_user(&val, buf, count); + if (retval) + return retval; + + if (val) + retval = gator_op_start(); + else + gator_op_stop(); + + if (retval) + return retval; + return count; +} + +static const struct file_operations enable_fops = { + .read = enable_read, + .write = enable_write, +}; + +static int userspace_buffer_open(struct inode *inode, struct file *file) +{ + int err = -EPERM; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (test_and_set_bit_lock(0, &gator_buffer_opened)) + return -EBUSY; + + if ((err = gator_op_setup())) + goto fail; + + /* NB: the actual start happens from userspace + * echo 1 >/dev/gator/driver/enable + */ + + return 0; + +fail: + __clear_bit_unlock(0, &gator_buffer_opened); + return err; +} + +static int userspace_buffer_release(struct inode *inode, struct file *file) +{ + gator_op_stop(); + gator_shutdown(); + __clear_bit_unlock(0, &gator_buffer_opened); + return 0; +} + +static ssize_t userspace_buffer_read(struct file *file, char __user *buf, + size_t count, loff_t *offset) +{ + int retval = -EINVAL; + int commit = 0, length1, length2, read; + char *buffer1; + char *buffer2 = NULL; + int cpu, buftype; + + /* do not handle partial reads */ + if (count != userspace_buffer_size || *offset) + return -EINVAL; + + // sleep until the condition is true or a signal is received + // the condition is checked each time gator_buffer_wait is woken up + buftype = cpu = -1; + wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || gator_annotate_ready() || !gator_started); + + if (signal_pending(current)) + return -EINTR; + + length2 = 0; + retval = -EFAULT; + + mutex_lock(&gator_buffer_mutex); + + if (buftype != -1 && cpu != -1) { + read = per_cpu(gator_buffer_read, cpu)[buftype]; + commit = per_cpu(gator_buffer_commit, cpu)[buftype]; + + /* May happen if the buffer is freed during pending reads. */ + if (!per_cpu(gator_buffer, cpu)[buftype]) { + retval = -EFAULT; + goto out; + } + + /* determine the size of two halves */ + length1 = commit - read; + buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]); + buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]); + if (length1 < 0) { + length1 = gator_buffer_size[buftype] - read; + length2 = commit; + } + } else if (gator_annotate_ready()) { + length1 = gator_annotate_read(&buffer1); + if (!length1) + goto out; + } else { + retval = 0; + goto out; + } + + /* start, middle or end */ + if (length1 > 0) { + if (copy_to_user(&buf[0], buffer1, length1)) { + goto out; + } + } + + /* possible wrap around */ + if (length2 > 0) { + if (copy_to_user(&buf[length1], buffer2, length2)) { + goto out; + } + } + + if (buftype != -1 && cpu != -1) + per_cpu(gator_buffer_read, cpu)[buftype] = commit; + + retval = length1 + length2; + + /* kick just in case we've lost an SMP event */ + wake_up(&gator_buffer_wait); + +out: + mutex_unlock(&gator_buffer_mutex); + return retval; +} + +const struct file_operations gator_event_buffer_fops = { + .open = userspace_buffer_open, + .release = userspace_buffer_release, + .read = userspace_buffer_read, +}; + +static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset) +{ + return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, + offset); +} + +static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset) +{ + unsigned long val; + int retval; + + if (*offset) + return -EINVAL; + + retval = gatorfs_ulong_from_user(&val, buf, count); + if (retval) + return retval; + + retval = gator_set_backtrace(val); + + if (retval) + return retval; + return count; +} + +static const struct file_operations depth_fops = { + .read = depth_read, + .write = depth_write +}; + +void gator_op_create_files(struct super_block *sb, struct dentry *root) +{ + struct dentry *dir; + struct gator_interface *gi; + int cpu; + + /* reinitialize default values */ + gator_cpu_cores = 0; + for_each_present_cpu(cpu) { + gator_cpu_cores++; + } + userspace_buffer_size = TIMER_BUFFER_SIZE_DEFAULT; + gator_streaming = 1; + + gatorfs_create_file(sb, root, "enable", &enable_fops); + gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops); + gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops); + gatorfs_create_ulong(sb, root, "cpu_cores", &gator_cpu_cores); + gatorfs_create_ulong(sb, root, "buffer_size", &userspace_buffer_size); + gatorfs_create_ulong(sb, root, "tick", &gator_timer_count); + gatorfs_create_ulong(sb, root, "streaming", &gator_streaming); + gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version); + + // Annotate interface + gator_annotate_create_files(sb, root); + + // Linux Events + dir = gatorfs_mkdir(sb, root, "events"); + list_for_each_entry(gi, &gator_events, list) + if (gi->create_files) + gi->create_files(sb, dir); +} + +/****************************************************************************** + * Module + ******************************************************************************/ +static int __init gator_module_init(void) +{ + if (gatorfs_register()) { + return -1; + } + + if (gator_init()) { + gatorfs_unregister(); + return -1; + } + + return 0; +} + +static void __exit gator_module_exit(void) +{ + tracepoint_synchronize_unregister(); + gatorfs_unregister(); + gator_exit(); +} + +module_init(gator_module_init); +module_exit(gator_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("ARM Ltd"); +MODULE_DESCRIPTION("Gator system profiler"); diff --git a/drivers/gator/driver/gator_pack.c b/drivers/gator/driver/gator_pack.c new file mode 100644 index 00000000000..985e960ad59 --- /dev/null +++ b/drivers/gator/driver/gator_pack.c @@ -0,0 +1,262 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +static void gator_buffer_write_packed_int(int cpu, int buftype, unsigned int x) +{ + uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype]; + uint32_t mask = gator_buffer_mask[buftype]; + char *buffer = per_cpu(gator_buffer, cpu)[buftype]; + int write0 = (write + 0) & mask; + int write1 = (write + 1) & mask; + + if ((x & 0xffffff80) == 0) { + buffer[write0] = x & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write1; + } else if ((x & 0xffffc000) == 0) { + int write2 = (write + 2) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write2; + } else if ((x & 0xffe00000) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write3; + } else if ((x & 0xf0000000) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write4; + } else { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) & 0x0f; + per_cpu(gator_buffer_write, cpu)[buftype] = write5; + } +} + +static void gator_buffer_write_packed_int64(int cpu, int buftype, unsigned long long x) +{ + uint32_t write = per_cpu(gator_buffer_write, cpu)[buftype]; + uint32_t mask = gator_buffer_mask[buftype]; + char *buffer = per_cpu(gator_buffer, cpu)[buftype]; + int write0 = (write + 0) & mask; + int write1 = (write + 1) & mask; + + if ((x & 0xffffffffffffff80LL) == 0) { + buffer[write0] = x & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write1; + } else if ((x & 0xffffffffffffc000LL) == 0) { + int write2 = (write + 2) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write2; + } else if ((x & 0xffffffffffe00000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write3; + } else if ((x & 0xfffffffff0000000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write4; + } else if ((x & 0xfffffff800000000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write5; + } else if ((x & 0xfffffc0000000000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + int write6 = (write + 6) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) | 0x80; + buffer[write5] = (x>>35) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write6; + } else if ((x & 0xfffe000000000000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + int write6 = (write + 6) & mask; + int write7 = (write + 7) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) | 0x80; + buffer[write5] = (x>>35) | 0x80; + buffer[write6] = (x>>42) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write7; + } else if ((x & 0xff00000000000000LL) == 0) { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + int write6 = (write + 6) & mask; + int write7 = (write + 7) & mask; + int write8 = (write + 8) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) | 0x80; + buffer[write5] = (x>>35) | 0x80; + buffer[write6] = (x>>42) | 0x80; + buffer[write7] = (x>>49) & 0x7f; + per_cpu(gator_buffer_write, cpu)[buftype] = write8; + } else { + int write2 = (write + 2) & mask; + int write3 = (write + 3) & mask; + int write4 = (write + 4) & mask; + int write5 = (write + 5) & mask; + int write6 = (write + 6) & mask; + int write7 = (write + 7) & mask; + int write8 = (write + 8) & mask; + int write9 = (write + 9) & mask; + buffer[write0] = x | 0x80; + buffer[write1] = (x>>7) | 0x80; + buffer[write2] = (x>>14) | 0x80; + buffer[write3] = (x>>21) | 0x80; + buffer[write4] = (x>>28) | 0x80; + buffer[write5] = (x>>35) | 0x80; + buffer[write6] = (x>>42) | 0x80; + buffer[write7] = (x>>49) | 0x80; + buffer[write8] = (x>>56) & 0xff; + per_cpu(gator_buffer_write, cpu)[buftype] = write9; + } +} + +static int gator_write_packed_int(char *buffer, unsigned int x) +{ + if ((x & 0xffffff80) == 0) { + buffer[0] = x & 0x7f; + return 1; + } else if ((x & 0xffffc000) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) & 0x7f; + return 2; + } else if ((x & 0xffe00000) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) & 0x7f; + return 3; + } else if ((x & 0xf0000000) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) & 0x7f; + return 4; + } else { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) & 0x0f; + return 5; + } +} + +static int gator_write_packed_int64(char *buffer, unsigned long long x) +{ + if ((x & 0xffffffffffffff80LL) == 0) { + buffer[0] = x & 0x7f; + return 1; + } else if ((x & 0xffffffffffffc000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) & 0x7f; + return 2; + } else if ((x & 0xffffffffffe00000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) & 0x7f; + return 3; + } else if ((x & 0xfffffffff0000000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) & 0x7f; + return 4; + } else if ((x & 0xfffffff800000000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) & 0x7f; + return 5; + } else if ((x & 0xfffffc0000000000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) | 0x80; + buffer[5] = (x>>35) & 0x7f; + return 6; + } else if ((x & 0xfffe000000000000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) | 0x80; + buffer[5] = (x>>35) | 0x80; + buffer[6] = (x>>42) & 0x7f; + return 7; + } else if ((x & 0xff00000000000000LL) == 0) { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) | 0x80; + buffer[5] = (x>>35) | 0x80; + buffer[6] = (x>>42) | 0x80; + buffer[7] = (x>>49) & 0x7f; + return 8; + } else { + buffer[0] = x | 0x80; + buffer[1] = (x>>7) | 0x80; + buffer[2] = (x>>14) | 0x80; + buffer[3] = (x>>21) | 0x80; + buffer[4] = (x>>28) | 0x80; + buffer[5] = (x>>35) | 0x80; + buffer[6] = (x>>42) | 0x80; + buffer[7] = (x>>49) | 0x80; + buffer[8] = (x>>56) & 0xff; + return 9; + } +} diff --git a/drivers/gator/driver/gator_trace_gpu.c b/drivers/gator/driver/gator_trace_gpu.c new file mode 100644 index 00000000000..bc63995c847 --- /dev/null +++ b/drivers/gator/driver/gator_trace_gpu.c @@ -0,0 +1,250 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "gator.h" + +#include +#include +#include +#include + +#ifdef MALI_SUPPORT +#include "linux/mali_linux_trace.h" +#endif +#include "gator_trace_gpu.h" + +#define ACTIVITY_START 1 +#define ACTIVITY_STOP 2 + +/* Note whether tracepoints have been registered */ +static int mali_trace_registered; +static int gpu_trace_registered; + +#define GPU_OVERFLOW -1 +#define GPU_START 1 +#define GPU_STOP 2 + +#define GPU_UNIT_VP 1 +#define GPU_UNIT_FP 2 + +#define TRACESIZE (8*1024) + +static DEFINE_PER_CPU(uint64_t *[2], theGpuTraceBuf); +static DEFINE_PER_CPU(int, theGpuTraceSel); +static DEFINE_PER_CPU(int, theGpuTracePos); +static DEFINE_PER_CPU(int, theGpuTraceErr); + +int gator_trace_gpu_read(long long **buffer); + +static void probe_gpu_write(int type, int unit, int core, struct task_struct* task) +{ + int tracePos; + unsigned long flags; + uint64_t *traceBuf, time; + int pid, tgid; + int cpu = smp_processor_id(); + + if (!per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]) + return; + + if (task) { + tgid = (int)task->tgid; + pid = (int)task->pid; + } else { + tgid = pid = 0; + } + + // disable interrupts to synchronize with gator_trace_gpu_read(); spinlocks not needed since percpu buffers are used + local_irq_save(flags); + + time = gator_get_time(); + tracePos = per_cpu(theGpuTracePos, cpu); + traceBuf = per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]; + + if (tracePos < (TRACESIZE - 100)) { + // capture + traceBuf[tracePos++] = type; + traceBuf[tracePos++] = time; + traceBuf[tracePos++] = unit; + traceBuf[tracePos++] = core; + traceBuf[tracePos++] = tgid; + traceBuf[tracePos++] = pid; + } else if (!per_cpu(theGpuTraceErr, cpu)) { + per_cpu(theGpuTraceErr, cpu) = 1; + traceBuf[tracePos++] = GPU_OVERFLOW; + traceBuf[tracePos++] = time; + traceBuf[tracePos++] = 0; + traceBuf[tracePos++] = 0; + traceBuf[tracePos++] = 0; + traceBuf[tracePos++] = 0; + pr_debug("gator: gpu trace overflow\n"); + } + per_cpu(theGpuTracePos, cpu) = tracePos; + local_irq_restore(flags); +} + +#ifdef MALI_SUPPORT + +enum components { + COMPONENT_VP0 = 1, + COMPONENT_FP0 = 5, + COMPONENT_FP1, + COMPONENT_FP2, + COMPONENT_FP3, + COMPONENT_FP4, + COMPONENT_FP5, + COMPONENT_FP6, + COMPONENT_FP7, +}; + +GATOR_DEFINE_PROBE(mali_timeline_event, TP_PROTO(unsigned int event_id, unsigned int d0, unsigned int d1, unsigned int d2, unsigned int d3, unsigned int d4)) +{ + unsigned int component, state; + + // do as much work as possible before disabling interrupts + component = (event_id >> 16) & 0xF; + state = (event_id >> 24) & 0xF; + + if ((component == COMPONENT_VP0) || (component >= COMPONENT_FP0 && component <= COMPONENT_FP7)) { + if (state == ACTIVITY_START || state == ACTIVITY_STOP) { + unsigned int type = (state == ACTIVITY_START) ? GPU_START : GPU_STOP; + unsigned int unit = (component < COMPONENT_FP0) ? GPU_UNIT_VP : GPU_UNIT_FP; + unsigned int core = (component < COMPONENT_FP0) ? component - COMPONENT_VP0 : component - COMPONENT_FP0; + struct task_struct* task = (state == ACTIVITY_START) ? (struct task_struct*)d2 : NULL; + + probe_gpu_write(type, unit, core, task); + } + } +} +#endif + +GATOR_DEFINE_PROBE(gpu_activity_start, TP_PROTO(int gpu_unit, int gpu_core, struct task_struct *p)) +{ + probe_gpu_write(GPU_START, gpu_unit, gpu_core, p); +} + +GATOR_DEFINE_PROBE(gpu_activity_stop, TP_PROTO(int gpu_unit, int gpu_core)) +{ + probe_gpu_write(GPU_STOP, gpu_unit, gpu_core, NULL); +} + +int gator_trace_gpu_start(void) +{ + int cpu; + + /* + * Returns 0 for installation failed + * Absence of gpu trace points is not an error + */ + + gpu_trace_registered = mali_trace_registered = 0; + +#ifdef MALI_SUPPORT + if (!GATOR_REGISTER_TRACE(mali_timeline_event)) { + mali_trace_registered = 1; + } +#endif + + if (!mali_trace_registered) { + if (GATOR_REGISTER_TRACE(gpu_activity_start)) { + return 0; + } + if (GATOR_REGISTER_TRACE(gpu_activity_stop)) { + GATOR_UNREGISTER_TRACE(gpu_activity_start); + return 0; + } + gpu_trace_registered = 1; + } + + if (!gpu_trace_registered && !mali_trace_registered) { + return 0; + } + + for_each_present_cpu(cpu) { + per_cpu(theGpuTraceSel, cpu) = 0; + per_cpu(theGpuTracePos, cpu) = 0; + per_cpu(theGpuTraceErr, cpu) = 0; + per_cpu(theGpuTraceBuf, cpu)[0] = kmalloc(TRACESIZE * sizeof(uint64_t), GFP_KERNEL); + per_cpu(theGpuTraceBuf, cpu)[1] = kmalloc(TRACESIZE * sizeof(uint64_t), GFP_KERNEL); + if (!per_cpu(theGpuTraceBuf, cpu)[0] || !per_cpu(theGpuTraceBuf, cpu)[1]) { +#ifdef MALI_SUPPORT + if (mali_trace_registered) { + GATOR_UNREGISTER_TRACE(mali_timeline_event); + } +#endif + if (gpu_trace_registered) { + GATOR_UNREGISTER_TRACE(gpu_activity_stop); + GATOR_UNREGISTER_TRACE(gpu_activity_start); + } + + gpu_trace_registered = mali_trace_registered = 0; + + return -1; + } + } + + return 0; +} + +int gator_trace_gpu_offline(long long **buffer) +{ + return gator_trace_gpu_read(buffer); +} + +void gator_trace_gpu_stop(void) +{ + int cpu; + + if (gpu_trace_registered || mali_trace_registered) { + for_each_present_cpu(cpu) { + kfree(per_cpu(theGpuTraceBuf, cpu)[0]); + kfree(per_cpu(theGpuTraceBuf, cpu)[1]); + per_cpu(theGpuTraceBuf, cpu)[0] = NULL; + per_cpu(theGpuTraceBuf, cpu)[1] = NULL; + } + +#ifdef MALI_SUPPORT + if (mali_trace_registered) { + GATOR_UNREGISTER_TRACE(mali_timeline_event); + } +#endif + if (gpu_trace_registered) { + GATOR_UNREGISTER_TRACE(gpu_activity_stop); + GATOR_UNREGISTER_TRACE(gpu_activity_start); + } + + gpu_trace_registered = mali_trace_registered = 0; + } +} + +int gator_trace_gpu_read(long long **buffer) +{ + int cpu = smp_processor_id(); + unsigned long flags; + uint64_t *traceBuf; + int tracePos; + + if (!per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]) + return 0; + + local_irq_save(flags); + + traceBuf = per_cpu(theGpuTraceBuf, cpu)[per_cpu(theGpuTraceSel, cpu)]; + tracePos = per_cpu(theGpuTracePos, cpu); + + per_cpu(theGpuTraceSel, cpu) = !per_cpu(theGpuTraceSel, cpu); + per_cpu(theGpuTracePos, cpu) = 0; + per_cpu(theGpuTraceErr, cpu) = 0; + + local_irq_restore(flags); + + if (buffer) + *buffer = traceBuf; + + return tracePos; +} diff --git a/drivers/gator/driver/gator_trace_gpu.h b/drivers/gator/driver/gator_trace_gpu.h new file mode 100644 index 00000000000..894289b4316 --- /dev/null +++ b/drivers/gator/driver/gator_trace_gpu.h @@ -0,0 +1,79 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#undef TRACE_GPU +#define TRACE_GPU gpu + +#if !defined(_TRACE_GPU_H) +#define _TRACE_GPU_H + +#include + +/* + * UNIT - the GPU processor type + * 1 = Vertex Processor + * 2 = Fragment Processor + * + * CORE - the GPU processor core number + * this is not the CPU core number + */ + +/* + * Tracepoint for calling GPU unit start activity on core + */ +TRACE_EVENT(gpu_activity_start, + + TP_PROTO(int gpu_unit, int gpu_core, struct task_struct *p), + + TP_ARGS(gpu_unit, gpu_core, p), + + TP_STRUCT__entry( + __field( int, gpu_unit ) + __field( int, gpu_core ) + __array( char, comm, TASK_COMM_LEN ) + __field( pid_t, pid ) + ), + + TP_fast_assign( + __entry->gpu_unit = gpu_unit; + __entry->gpu_core = gpu_core; + memcpy(__entry->comm, p->comm, TASK_COMM_LEN); + __entry->pid = p->pid; + ), + + TP_printk("unit=%d core=%d comm=%s pid=%d", + __entry->gpu_unit, __entry->gpu_core, __entry->comm, __entry->pid) +); + +/* + * Tracepoint for calling GPU unit stop activity on core + */ +TRACE_EVENT(gpu_activity_stop, + + TP_PROTO(int gpu_unit, int gpu_core), + + TP_ARGS(gpu_unit, gpu_core), + + TP_STRUCT__entry( + __field( int, gpu_unit ) + __field( int, gpu_core ) + ), + + TP_fast_assign( + __entry->gpu_unit = gpu_unit; + __entry->gpu_core = gpu_core; + ), + + TP_printk("unit=%d core=%d", + __entry->gpu_unit, __entry->gpu_core) +); + +#endif /* _TRACE_GPU_H */ + +/* This part must be outside protection */ +#include diff --git a/drivers/gator/driver/gator_trace_sched.c b/drivers/gator/driver/gator_trace_sched.c new file mode 100644 index 00000000000..dafacb75416 --- /dev/null +++ b/drivers/gator/driver/gator_trace_sched.c @@ -0,0 +1,269 @@ +/** + * Copyright (C) ARM Limited 2010-2012. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include +#include "gator.h" + +#define SCHED_OVERFLOW -1 +#define SCHED_SWITCH 1 +#define SCHED_PROCESS_FREE 2 + +#define SCHEDSIZE (8*1024) +#define TASK_MAP_ENTRIES 1024 /* must be power of 2 */ +#define TASK_MAX_COLLISIONS 2 + +static DEFINE_PER_CPU(uint64_t *[2], theSchedBuf); +static DEFINE_PER_CPU(int, theSchedSel); +static DEFINE_PER_CPU(int, theSchedPos); +static DEFINE_PER_CPU(int, theSchedErr); +static DEFINE_PER_CPU(uint64_t *, taskname_keys); + +enum { + STATE_CONTENTION = 0, + STATE_WAIT_ON_IO, + STATE_WAIT_ON_OTHER +}; + +int gator_trace_sched_read(long long **buffer); + +void emit_pid_name(struct task_struct* task) +{ + bool found = false; + unsigned long flags; + char taskcomm[TASK_COMM_LEN + 3]; + int x, cpu = smp_processor_id(); + uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]); + uint64_t value; + + value = gator_chksum_crc32(task->comm); + value = (value << 32) | (uint32_t)task->pid; + + // determine if the thread name was emitted already + for (x = 0; x < TASK_MAX_COLLISIONS; x++) { + if (keys[x] == value) { + found = true; + break; + } + } + + if (!found && buffer_check_space(cpu, TIMER_BUF, TASK_COMM_LEN + 2 * MAXSIZE_PACK32 + MAXSIZE_PACK64)) { + // shift values, new value always in front + uint64_t oldv, newv = value; + for (x = 0; x < TASK_MAX_COLLISIONS; x++) { + oldv = keys[x]; + keys[x] = newv; + newv = oldv; + } + + // emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions + if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) + // append ellipses if task->comm has length of TASK_COMM_LEN - 1 + strcat(taskcomm, "..."); + + // disable interrupts to synchronize with hrtimer populating timer buf + local_irq_save(flags); + gator_buffer_write_packed_int(cpu, TIMER_BUF, MESSAGE_PID_NAME); + gator_buffer_write_packed_int64(cpu, TIMER_BUF, gator_get_time()); + gator_buffer_write_packed_int(cpu, TIMER_BUF, task->pid); + gator_buffer_write_string(cpu, TIMER_BUF, taskcomm); + local_irq_restore(flags); + } +} + +static void probe_sched_write(int type, struct task_struct* task, struct task_struct* old_task) +{ + int schedPos, cookie = 0, state = 0; + unsigned long flags; + uint64_t *schedBuf, time; + int cpu = smp_processor_id(); + int pid = task->pid; + int tgid = task->tgid; + + if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) + return; + + if (type == SCHED_SWITCH) { + // do as much work as possible before disabling interrupts + cookie = get_exec_cookie(cpu, TIMER_BUF, task); + emit_pid_name(task); + if (old_task->state == 0) + state = STATE_CONTENTION; + else if (old_task->in_iowait) + state = STATE_WAIT_ON_IO; + else + state = STATE_WAIT_ON_OTHER; + } + + // disable interrupts to synchronize with gator_trace_sched_read(); spinlocks not needed since percpu buffers are used + local_irq_save(flags); + + time = gator_get_time(); + schedPos = per_cpu(theSchedPos, cpu); + schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]; + + if (schedPos < (SCHEDSIZE - 100)) { + // capture + schedBuf[schedPos++] = type; + schedBuf[schedPos++] = time; + schedBuf[schedPos++] = pid; + schedBuf[schedPos++] = tgid; + schedBuf[schedPos++] = cookie; + schedBuf[schedPos++] = state; + } else if (!per_cpu(theSchedErr, cpu)) { + per_cpu(theSchedErr, cpu) = 1; + schedBuf[schedPos++] = SCHED_OVERFLOW; + schedBuf[schedPos++] = time; + schedBuf[schedPos++] = 0; + schedBuf[schedPos++] = 0; + schedBuf[schedPos++] = 0; + schedBuf[schedPos++] = 0; + pr_debug("gator: tracepoint overflow\n"); + } + per_cpu(theSchedPos, cpu) = schedPos; + local_irq_restore(flags); +} + +// special case used during a suspend of the system +static void trace_sched_insert_idle(void) +{ + unsigned long flags; + uint64_t *schedBuf; + int schedPos, cpu = smp_processor_id(); + + if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) + return; + + local_irq_save(flags); + + schedPos = per_cpu(theSchedPos, cpu); + schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]; + + if (schedPos < (SCHEDSIZE - (6 * 8))) { + // capture + schedBuf[schedPos++] = SCHED_SWITCH; + schedBuf[schedPos++] = gator_get_time(); + schedBuf[schedPos++] = 0; // idle pid is zero + schedBuf[schedPos++] = 0; // idle tid is zero + schedBuf[schedPos++] = 0; // idle cookie is zero + schedBuf[schedPos++] = STATE_WAIT_ON_OTHER; + } + + per_cpu(theSchedPos, cpu) = schedPos; + local_irq_restore(flags); +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35) +GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *prev, struct task_struct *next)) +#else +GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next)) +#endif +{ + probe_sched_write(SCHED_SWITCH, next, prev); +} + +GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p)) +{ + probe_sched_write(SCHED_PROCESS_FREE, p, 0); +} + +static int register_scheduler_tracepoints(void) { + // register tracepoints + if (GATOR_REGISTER_TRACE(sched_switch)) + goto fail_sched_switch; + if (GATOR_REGISTER_TRACE(sched_process_free)) + goto fail_sched_process_free; + pr_debug("gator: registered tracepoints\n"); + + return 0; + + // unregister tracepoints on error +fail_sched_process_free: + GATOR_UNREGISTER_TRACE(sched_switch); +fail_sched_switch: + pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n"); + + return -1; +} + +int gator_trace_sched_start(void) +{ + int cpu, size; + + for_each_present_cpu(cpu) { + per_cpu(theSchedSel, cpu) = 0; + per_cpu(theSchedPos, cpu) = 0; + per_cpu(theSchedErr, cpu) = 0; + per_cpu(theSchedBuf, cpu)[0] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL); + per_cpu(theSchedBuf, cpu)[1] = kmalloc(SCHEDSIZE * sizeof(uint64_t), GFP_KERNEL); + if (!per_cpu(theSchedBuf, cpu)) + return -1; + + size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t); + per_cpu(taskname_keys, cpu) = (uint64_t*)kmalloc(size, GFP_KERNEL); + if (!per_cpu(taskname_keys, cpu)) + return -1; + memset(per_cpu(taskname_keys, cpu), 0, size); + } + + return register_scheduler_tracepoints(); +} + +int gator_trace_sched_offline(long long **buffer) +{ + trace_sched_insert_idle(); + return gator_trace_sched_read(buffer); +} + +static void unregister_scheduler_tracepoints(void) +{ + GATOR_UNREGISTER_TRACE(sched_switch); + GATOR_UNREGISTER_TRACE(sched_process_free); + pr_debug("gator: unregistered tracepoints\n"); +} + +void gator_trace_sched_stop(void) +{ + int cpu; + unregister_scheduler_tracepoints(); + + for_each_present_cpu(cpu) { + kfree(per_cpu(theSchedBuf, cpu)[0]); + kfree(per_cpu(theSchedBuf, cpu)[1]); + per_cpu(theSchedBuf, cpu)[0] = NULL; + per_cpu(theSchedBuf, cpu)[1] = NULL; + kfree(per_cpu(taskname_keys, cpu)); + } +} + +int gator_trace_sched_read(long long **buffer) +{ + int cpu = smp_processor_id(); + unsigned long flags; + uint64_t *schedBuf; + int schedPos; + + if (!per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]) + return 0; + + local_irq_save(flags); + + schedBuf = per_cpu(theSchedBuf, cpu)[per_cpu(theSchedSel, cpu)]; + schedPos = per_cpu(theSchedPos, cpu); + + per_cpu(theSchedSel, cpu) = !per_cpu(theSchedSel, cpu); + per_cpu(theSchedPos, cpu) = 0; + per_cpu(theSchedErr, cpu) = 0; + + local_irq_restore(flags); + + if (buffer) + *buffer = schedBuf; + + return schedPos; +} -- cgit v1.2.3