From e725c731e3bb1e892e7b564c945b121cb41d1087 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Fri, 3 Mar 2017 13:37:33 -0500 Subject: tracing: Split tracing initialization into two for early initialization Create an early_trace_init() function that will initialize the buffers and allow for ealier use of trace_printk(). This will also allow for future work to have function tracing start earlier at boot up. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/trace.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index f35109514a01..6757561d9617 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -7999,7 +7999,7 @@ out: return ret; } -void __init trace_init(void) +void __init early_trace_init(void) { if (tracepoint_printk) { tracepoint_print_iter = @@ -8010,6 +8010,10 @@ void __init trace_init(void) static_key_enable(&tracepoint_printk_key.key); } tracer_alloc_buffers(); +} + +void __init trace_init(void) +{ trace_event_init(); } -- cgit v1.2.3 From 9afecfbb95198ec3ea6d52cca4711ea314f29ec6 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Fri, 24 Mar 2017 17:59:10 -0400 Subject: tracing: Postpone tracer start-up tests till the system is more robust As tracing can now be enabled very early in boot up, even before some critical system services (like scheduling), do not run the tracer selftests until after early_initcall() is performed. If a tracer is registered before such time, it is saved off in a list and the test is run when the system is able to handle more diverse functions. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/trace.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 6757561d9617..68a6f78f6862 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1424,6 +1424,28 @@ static int wait_on_pipe(struct trace_iterator *iter, bool full) } #ifdef CONFIG_FTRACE_STARTUP_TEST +static bool selftests_can_run; + +struct trace_selftests { + struct list_head list; + struct tracer *type; +}; + +static LIST_HEAD(postponed_selftests); + +static int save_selftest(struct tracer *type) +{ + struct trace_selftests *selftest; + + selftest = kmalloc(sizeof(*selftest), GFP_KERNEL); + if (!selftest) + return -ENOMEM; + + selftest->type = type; + list_add(&selftest->list, &postponed_selftests); + return 0; +} + static int run_tracer_selftest(struct tracer *type) { struct trace_array *tr = &global_trace; @@ -1433,6 +1455,14 @@ static int run_tracer_selftest(struct tracer *type) if (!type->selftest || tracing_selftest_disabled) return 0; + /* + * If a tracer registers early in boot up (before scheduling is + * initialized and such), then do not run its selftests yet. + * Instead, run it a little later in the boot process. + */ + if (!selftests_can_run) + return save_selftest(type); + /* * Run a selftest on this tracer. * Here we reset the trace buffer, and set the current @@ -1482,6 +1512,47 @@ static int run_tracer_selftest(struct tracer *type) printk(KERN_CONT "PASSED\n"); return 0; } + +static __init int init_trace_selftests(void) +{ + struct trace_selftests *p, *n; + struct tracer *t, **last; + int ret; + + selftests_can_run = true; + + mutex_lock(&trace_types_lock); + + if (list_empty(&postponed_selftests)) + goto out; + + pr_info("Running postponed tracer tests:\n"); + + list_for_each_entry_safe(p, n, &postponed_selftests, list) { + ret = run_tracer_selftest(p->type); + /* If the test fails, then warn and remove from available_tracers */ + if (ret < 0) { + WARN(1, "tracer: %s failed selftest, disabling\n", + p->type->name); + last = &trace_types; + for (t = trace_types; t; t = t->next) { + if (t == p->type) { + *last = t->next; + break; + } + last = &t->next; + } + } + list_del(&p->list); + kfree(p); + } + + out: + mutex_unlock(&trace_types_lock); + + return 0; +} +early_initcall(init_trace_selftests); #else static inline int run_tracer_selftest(struct tracer *type) { -- cgit v1.2.3 From dbeafd0d6131d0f6ae8cd7551f5f4bf8c54aa49a Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Fri, 3 Mar 2017 13:48:42 -0500 Subject: ftrace: Have function tracing start in early boot up Register the function tracer right after the tracing buffers are initialized in early boot up. This will allow function tracing to begin early if it is enabled via the kernel command line. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/trace.c | 3 +++ kernel/trace/trace.h | 2 ++ kernel/trace/trace_functions.c | 3 +-- 3 files changed, 6 insertions(+), 2 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 68a6f78f6862..4fa8e8f3c765 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -8036,6 +8036,9 @@ __init static int tracer_alloc_buffers(void) register_tracer(&nop_trace); + /* Function tracing may start here (via kernel command line) */ + init_function_trace(); + /* All seems OK, enable tracing */ tracing_disabled = 0; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index ae1cce91fead..571acee52a32 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -896,6 +896,7 @@ int using_ftrace_ops_list_func(void); void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer); void ftrace_init_tracefs_toplevel(struct trace_array *tr, struct dentry *d_tracer); +int init_function_trace(void); #else static inline int ftrace_trace_task(struct trace_array *tr) { @@ -914,6 +915,7 @@ ftrace_init_global_array_ops(struct trace_array *tr) { } static inline void ftrace_reset_array_ops(struct trace_array *tr) { } static inline void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d) { } static inline void ftrace_init_tracefs_toplevel(struct trace_array *tr, struct dentry *d) { } +static inline int init_function_trace(void) { return 0; } /* ftace_func_t type is not defined, use macro instead of static inline */ #define ftrace_init_array_ops(tr, func) do { } while (0) #endif /* CONFIG_FUNCTION_TRACER */ diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 0efa00d80623..4199ca61b0e5 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -687,9 +687,8 @@ static inline int init_func_cmd_traceon(void) } #endif /* CONFIG_DYNAMIC_FTRACE */ -static __init int init_function_trace(void) +__init int init_function_trace(void) { init_func_cmd_traceon(); return register_tracer(&function_trace); } -core_initcall(init_function_trace); -- cgit v1.2.3 From af0009fc16a45d091f896794e97a6457f9a7eddf Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Thu, 16 Mar 2017 11:01:06 -0400 Subject: tracing: Move trace_handle_return() out of line Currently trace_handle_return() looks like this: static inline enum print_line_t trace_handle_return(struct trace_seq *s) { return trace_seq_has_overflowed(s) ? TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED; } Where trace_seq_overflowed(s) is: static inline bool trace_seq_has_overflowed(struct trace_seq *s) { return s->full || seq_buf_has_overflowed(&s->seq); } And seq_buf_has_overflowed(&s->seq) is: static inline bool seq_buf_has_overflowed(struct seq_buf *s) { return s->len > s->size; } Making trace_handle_return() into: return (s->full || (s->seq->len > s->seq->size)) ? TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED; One would think this is not an issue to keep as an inline. But because this is used in the TRACE_EVENT() macro, it is extended for every tracepoint in the system. Taking a look at a single tracepoint x86_irq_vector (was the first one I randomly chosen). As trace_handle_return is used in the TRACE_EVENT() macro of trace_raw_output_##call() we disassemble trace_raw_output_x86_irq_vector and do a diff: - is the original + is the out-of-line code I removed identical lines that were different just due to different addresses. --- /tmp/irq-vec-orig 2017-03-16 09:12:48.569384851 -0400 +++ /tmp/irq-vec-ool 2017-03-16 09:13:39.378153385 -0400 @@ -6,27 +6,23 @@ 53 push %rbx 48 89 fb mov %rdi,%rbx 4c 8b a7 c0 20 00 00 mov 0x20c0(%rdi),%r12 e8 f7 72 13 00 callq ffffffff81155c80 83 f8 01 cmp $0x1,%eax 74 05 je ffffffff8101e993 5b pop %rbx 41 5c pop %r12 5d pop %rbp c3 retq 41 8b 54 24 08 mov 0x8(%r12),%edx - 48 8d bb 98 10 00 00 lea 0x1098(%rbx),%rdi + 48 81 c3 98 10 00 00 add $0x1098,%rbx - 48 c7 c6 7b 8a a0 81 mov $0xffffffff81a08a7b,%rsi + 48 c7 c6 ab 8a a0 81 mov $0xffffffff81a08aab,%rsi - e8 c5 85 13 00 callq ffffffff81156f70 === here's the start of the main difference === + 48 89 df mov %rbx,%rdi + e8 62 7e 13 00 callq ffffffff81156810 - 8b 93 b8 20 00 00 mov 0x20b8(%rbx),%edx - 31 c0 xor %eax,%eax - 85 d2 test %edx,%edx - 75 11 jne ffffffff8101e9c8 - 48 8b 83 a8 20 00 00 mov 0x20a8(%rbx),%rax - 48 39 83 a0 20 00 00 cmp %rax,0x20a0(%rbx) - 0f 93 c0 setae %al + 48 89 df mov %rbx,%rdi + e8 4a c5 12 00 callq ffffffff8114af00 5b pop %rbx - 0f b6 c0 movzbl %al,%eax === end === 41 5c pop %r12 5d pop %rbp c3 retq If you notice, the original has 22 bytes of text more than the out of line version. As this is for every TRACE_EVENT() defined in the system, this can become quite large. text data bss dec hex filename 8690305 5450490 1298432 15439227 eb957b vmlinux-orig 8681725 5450490 1298432 15430647 eb73f7 vmlinux-handle This change has a total of 8580 bytes in savings. $ objdump -dr /tmp/vmlinux-orig | grep '^[0-9a-f]* Signed-off-by: Steven Rostedt (VMware) --- include/linux/trace_events.h | 11 +---------- kernel/trace/trace.c | 12 ++++++++++++ 2 files changed, 13 insertions(+), 10 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 0af63c4381b9..a556805eff8a 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -138,16 +138,7 @@ enum print_line_t { TRACE_TYPE_NO_CONSUME = 3 /* Handled but ask to not consume */ }; -/* - * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq - * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function - * simplifies those functions and keeps them in sync. - */ -static inline enum print_line_t trace_handle_return(struct trace_seq *s) -{ - return trace_seq_has_overflowed(s) ? - TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED; -} +enum print_line_t trace_handle_return(struct trace_seq *s); void tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags, diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 4fa8e8f3c765..b5d4b80f2d45 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1998,6 +1998,18 @@ void tracing_record_cmdline(struct task_struct *tsk) __this_cpu_write(trace_cmdline_save, false); } +/* + * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq + * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function + * simplifies those functions and keeps them in sync. + */ +enum print_line_t trace_handle_return(struct trace_seq *s) +{ + return trace_seq_has_overflowed(s) ? + TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED; +} +EXPORT_SYMBOL_GPL(trace_handle_return); + void tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags, int pc) -- cgit v1.2.3 From 1e10486ffee0a5b060c58b9c8c712422f7b88b3b Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Mon, 17 Apr 2017 11:44:28 +0900 Subject: ftrace: Add 'function-fork' trace option The function-fork option is same as event-fork that it tracks task fork/exit and set the pid filter properly. This can be useful if user wants to trace selected tasks including their children only. Link: http://lkml.kernel.org/r/20170417024430.21194-3-namhyung@kernel.org Signed-off-by: Namhyung Kim Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/ftrace.c | 37 +++++++++++++++++++++++++++++++++++++ kernel/trace/trace.c | 5 ++++- kernel/trace/trace.h | 6 +++++- 3 files changed, 46 insertions(+), 2 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index b21a3e61ac74..b5ce7ea67e02 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -5600,6 +5600,43 @@ ftrace_filter_pid_sched_switch_probe(void *data, bool preempt, trace_ignore_this_task(pid_list, next)); } +static void +ftrace_pid_follow_sched_process_fork(void *data, + struct task_struct *self, + struct task_struct *task) +{ + struct trace_pid_list *pid_list; + struct trace_array *tr = data; + + pid_list = rcu_dereference_sched(tr->function_pids); + trace_filter_add_remove_task(pid_list, self, task); +} + +static void +ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task) +{ + struct trace_pid_list *pid_list; + struct trace_array *tr = data; + + pid_list = rcu_dereference_sched(tr->function_pids); + trace_filter_add_remove_task(pid_list, NULL, task); +} + +void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) +{ + if (enable) { + register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork, + tr); + register_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit, + tr); + } else { + unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork, + tr); + unregister_trace_sched_process_exit(ftrace_pid_follow_sched_process_exit, + tr); + } +} + static void clear_ftrace_pids(struct trace_array *tr) { struct trace_pid_list *pid_list; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index b5d4b80f2d45..8a5064a03ddf 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -257,7 +257,7 @@ unsigned long long ns2usecs(u64 nsec) /* trace_flags that are default zero for instances */ #define ZEROED_TRACE_FLAGS \ - TRACE_ITER_EVENT_FORK + (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK) /* * The global_trace is the descriptor that holds the top-level tracing @@ -4205,6 +4205,9 @@ int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled) if (mask == TRACE_ITER_EVENT_FORK) trace_event_follow_fork(tr, enabled); + if (mask == TRACE_ITER_FUNC_FORK) + ftrace_pid_follow_fork(tr, enabled); + if (mask == TRACE_ITER_OVERWRITE) { ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled); #ifdef CONFIG_TRACER_MAX_TRACE diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 571acee52a32..31a4997b67c6 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -897,6 +897,7 @@ void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer); void ftrace_init_tracefs_toplevel(struct trace_array *tr, struct dentry *d_tracer); int init_function_trace(void); +void ftrace_pid_follow_fork(struct trace_array *tr, bool enable); #else static inline int ftrace_trace_task(struct trace_array *tr) { @@ -916,6 +917,7 @@ static inline void ftrace_reset_array_ops(struct trace_array *tr) { } static inline void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d) { } static inline void ftrace_init_tracefs_toplevel(struct trace_array *tr, struct dentry *d) { } static inline int init_function_trace(void) { return 0; } +static inline void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) { } /* ftace_func_t type is not defined, use macro instead of static inline */ #define ftrace_init_array_ops(tr, func) do { } while (0) #endif /* CONFIG_FUNCTION_TRACER */ @@ -989,11 +991,13 @@ extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf, #ifdef CONFIG_FUNCTION_TRACER # define FUNCTION_FLAGS \ - C(FUNCTION, "function-trace"), + C(FUNCTION, "function-trace"), \ + C(FUNC_FORK, "function-fork"), # define FUNCTION_DEFAULT_FLAGS TRACE_ITER_FUNCTION #else # define FUNCTION_FLAGS # define FUNCTION_DEFAULT_FLAGS 0UL +# define TRACE_ITER_FUNC_FORK 0UL #endif #ifdef CONFIG_STACKTRACE -- cgit v1.2.3 From bca6c8d0480a8aa5c86f8f416db96c71f6b79e29 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Apr 2017 18:18:47 -0400 Subject: ftrace: Pass probe ops to probe function In preparation to cleaning up the probe function registration code, the "data" parameter will eventually be removed from the probe->func() call. Instead it will receive its own "ops" function, in which it can set up its own data that it needs to map. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/ftrace.c | 2 +- kernel/trace/trace.c | 6 ++++-- kernel/trace/trace.h | 1 + kernel/trace/trace_events.c | 8 +++++--- kernel/trace/trace_functions.c | 24 ++++++++++++++++-------- 5 files changed, 27 insertions(+), 14 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index b6dc29583c86..d8873079bed4 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -3739,7 +3739,7 @@ static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, preempt_disable_notrace(); hlist_for_each_entry_rcu_notrace(entry, hhd, node) { if (entry->ip == ip) - entry->ops->func(ip, parent_ip, &entry->data); + entry->ops->func(ip, parent_ip, entry->ops, &entry->data); } preempt_enable_notrace(); } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 8a5064a03ddf..41e9a20f91f0 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -6735,13 +6735,15 @@ static const struct file_operations tracing_dyn_info_fops = { #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) static void -ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data) +ftrace_snapshot(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **data) { tracing_snapshot(); } static void -ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data) +ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **data) { unsigned long *count = (long *)data; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index a63411c53c5e..0f915c264c19 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -934,6 +934,7 @@ static inline void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) { struct ftrace_probe_ops { void (*func)(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **data); int (*init)(struct ftrace_probe_ops *ops, unsigned long ip, void **data); diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 93116549a284..9dbac1881b03 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2461,7 +2461,8 @@ struct event_probe_data { }; static void -event_enable_probe(unsigned long ip, unsigned long parent_ip, void **_data) +event_enable_probe(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **_data) { struct event_probe_data **pdata = (struct event_probe_data **)_data; struct event_probe_data *data = *pdata; @@ -2476,7 +2477,8 @@ event_enable_probe(unsigned long ip, unsigned long parent_ip, void **_data) } static void -event_enable_count_probe(unsigned long ip, unsigned long parent_ip, void **_data) +event_enable_count_probe(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **_data) { struct event_probe_data **pdata = (struct event_probe_data **)_data; struct event_probe_data *data = *pdata; @@ -2494,7 +2496,7 @@ event_enable_count_probe(unsigned long ip, unsigned long parent_ip, void **_data if (data->count != -1) (data->count)--; - event_enable_probe(ip, parent_ip, _data); + event_enable_probe(ip, parent_ip, ops, _data); } static int diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 4199ca61b0e5..b99f6231281e 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -326,19 +326,22 @@ static void update_traceon_count(void **data, bool on) } static void -ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, void **data) +ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **data) { update_traceon_count(data, 1); } static void -ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, void **data) +ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **data) { update_traceon_count(data, 0); } static void -ftrace_traceon(unsigned long ip, unsigned long parent_ip, void **data) +ftrace_traceon(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **data) { if (tracing_is_on()) return; @@ -347,7 +350,8 @@ ftrace_traceon(unsigned long ip, unsigned long parent_ip, void **data) } static void -ftrace_traceoff(unsigned long ip, unsigned long parent_ip, void **data) +ftrace_traceoff(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **data) { if (!tracing_is_on()) return; @@ -365,13 +369,15 @@ ftrace_traceoff(unsigned long ip, unsigned long parent_ip, void **data) #define STACK_SKIP 4 static void -ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, void **data) +ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **data) { trace_dump_stack(STACK_SKIP); } static void -ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, void **data) +ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **data) { long *count = (long *)data; long old_count; @@ -419,7 +425,8 @@ static int update_count(void **data) } static void -ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, void **data) +ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **data) { if (update_count(data)) ftrace_dump(DUMP_ALL); @@ -427,7 +434,8 @@ ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, void **data) /* Only dump the current CPU buffer. */ static void -ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, void **data) +ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, + struct ftrace_probe_ops *ops, void **data) { if (update_count(data)) ftrace_dump(DUMP_ORIG); -- cgit v1.2.3 From 1a93f8bd19bd24f9b41136e70188d8f4de90c6a2 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 3 Apr 2017 22:09:43 -0400 Subject: tracing: Have the snapshot trigger use the mapping helper functions As the data pointer for individual ips will soon be removed and no longer passed to the callback function probe handlers, convert the snapshot trigger counter over to the new ftrace_func_mapper helper functions. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/trace.c | 52 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 8 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 41e9a20f91f0..7febeb823c62 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -6745,13 +6745,19 @@ static void ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, struct ftrace_probe_ops *ops, void **data) { - unsigned long *count = (long *)data; + struct ftrace_func_mapper *mapper = ops->private_data; + long *count = NULL; - if (!*count) - return; + if (mapper) + count = (long *)ftrace_func_mapper_find_ip(mapper, ip); + + if (count) { + + if (*count <= 0) + return; - if (*count != -1) (*count)--; + } tracing_snapshot(); } @@ -6760,20 +6766,42 @@ static int ftrace_snapshot_print(struct seq_file *m, unsigned long ip, struct ftrace_probe_ops *ops, void *data) { - long count = (long)data; + struct ftrace_func_mapper *mapper = ops->private_data; + long *count = NULL; seq_printf(m, "%ps:", (void *)ip); seq_puts(m, "snapshot"); - if (count == -1) - seq_puts(m, ":unlimited\n"); + if (mapper) + count = (long *)ftrace_func_mapper_find_ip(mapper, ip); + + if (count) + seq_printf(m, ":count=%ld\n", *count); else - seq_printf(m, ":count=%ld\n", count); + seq_puts(m, ":unlimited\n"); return 0; } +static int +ftrace_snapshot_init(struct ftrace_probe_ops *ops, unsigned long ip, + void **data) +{ + struct ftrace_func_mapper *mapper = ops->private_data; + + return ftrace_func_mapper_add_ip(mapper, ip, *data); +} + +static void +ftrace_snapshot_free(struct ftrace_probe_ops *ops, unsigned long ip, + void **_data) +{ + struct ftrace_func_mapper *mapper = ops->private_data; + + ftrace_func_mapper_remove_ip(mapper, ip); +} + static struct ftrace_probe_ops snapshot_probe_ops = { .func = ftrace_snapshot, .print = ftrace_snapshot_print, @@ -6782,6 +6810,8 @@ static struct ftrace_probe_ops snapshot_probe_ops = { static struct ftrace_probe_ops snapshot_count_probe_ops = { .func = ftrace_count_snapshot, .print = ftrace_snapshot_print, + .init = ftrace_snapshot_init, + .free = ftrace_snapshot_free, }; static int @@ -6812,6 +6842,12 @@ ftrace_trace_snapshot_callback(struct ftrace_hash *hash, if (!strlen(number)) goto out_reg; + if (!ops->private_data) { + ops->private_data = allocate_ftrace_func_mapper(); + if (!ops->private_data) + return -ENOMEM; + } + /* * We use the callback data field (which is a pointer) * as our counter. -- cgit v1.2.3 From 1a48df0041c2756194e700affb0e2ff084092e28 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Tue, 4 Apr 2017 10:27:51 -0400 Subject: ftrace: Remove data field from ftrace_func_probe structure No users of the function probes uses the data field anymore. Remove it, and change the init function to take a void *data parameter instead of a void **data, because the init will just get the data that the registering function was received, and there's no state after it is called. The other functions for ftrace_probe_ops still take the data parameter, but it will currently only be passed NULL. It will stay as a parameter for future data to be passed to these functions. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/ftrace.c | 11 ++++------- kernel/trace/trace.c | 4 ++-- kernel/trace/trace.h | 2 +- kernel/trace/trace_events.c | 5 ++--- kernel/trace/trace_functions.c | 4 ++-- 5 files changed, 11 insertions(+), 15 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 15f910a03822..f7fcab8f3aa1 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1102,7 +1102,6 @@ struct ftrace_func_probe { struct hlist_node node; struct ftrace_probe_ops *ops; unsigned long ip; - void *data; struct list_head free_list; }; @@ -3152,7 +3151,7 @@ t_hash_show(struct seq_file *m, struct ftrace_iterator *iter) return -EIO; if (rec->ops->print) - return rec->ops->print(m, rec->ip, rec->ops, rec->data); + return rec->ops->print(m, rec->ip, rec->ops, NULL); seq_printf(m, "%ps:%ps\n", (void *)rec->ip, (void *)rec->ops->func); @@ -3735,7 +3734,7 @@ static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, preempt_disable_notrace(); hlist_for_each_entry_rcu_notrace(entry, hhd, node) { if (entry->ip == ip) - entry->ops->func(ip, parent_ip, entry->ops, &entry->data); + entry->ops->func(ip, parent_ip, entry->ops, NULL); } preempt_enable_notrace(); } @@ -3800,7 +3799,7 @@ static bool __disable_ftrace_function_probe(void) static void ftrace_free_entry(struct ftrace_func_probe *entry) { if (entry->ops->free) - entry->ops->free(entry->ops, entry->ip, &entry->data); + entry->ops->free(entry->ops, entry->ip, NULL); kfree(entry); } @@ -4007,15 +4006,13 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, count++; - entry->data = data; - /* * The caller might want to do something special * for each function we find. We call the callback * to give the caller an opportunity to do so. */ if (ops->init) { - if (ops->init(ops, rec->ip, &entry->data) < 0) { + if (ops->init(ops, rec->ip, data) < 0) { /* caller does not like this func */ kfree(entry); continue; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 7febeb823c62..7a4d578d8887 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -6786,11 +6786,11 @@ ftrace_snapshot_print(struct seq_file *m, unsigned long ip, static int ftrace_snapshot_init(struct ftrace_probe_ops *ops, unsigned long ip, - void **data) + void *data) { struct ftrace_func_mapper *mapper = ops->private_data; - return ftrace_func_mapper_add_ip(mapper, ip, *data); + return ftrace_func_mapper_add_ip(mapper, ip, data); } static void diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 376d5a798489..86aa5a2222ba 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -937,7 +937,7 @@ struct ftrace_probe_ops { struct ftrace_probe_ops *ops, void **data); int (*init)(struct ftrace_probe_ops *ops, - unsigned long ip, void **data); + unsigned long ip, void *data); void (*free)(struct ftrace_probe_ops *ops, unsigned long ip, void **data); int (*print)(struct seq_file *m, diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index ee308312fe87..37902107c44f 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2543,11 +2543,10 @@ event_enable_print(struct seq_file *m, unsigned long ip, static int event_enable_init(struct ftrace_probe_ops *ops, unsigned long ip, - void **_data) + void *_data) { struct ftrace_func_mapper *mapper = ops->private_data; - struct event_probe_data **pdata = (struct event_probe_data **)_data; - struct event_probe_data *data = *pdata; + struct event_probe_data *data = _data; int ret; ret = ftrace_func_mapper_add_ip(mapper, ip, data); diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index d9cbde8575a8..56d0fe1e4ea1 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -509,11 +509,11 @@ ftrace_cpudump_print(struct seq_file *m, unsigned long ip, static int ftrace_count_init(struct ftrace_probe_ops *ops, unsigned long ip, - void **data) + void *data) { struct ftrace_func_mapper *mapper = ops->private_data; - return ftrace_func_mapper_add_ip(mapper, ip, *data); + return ftrace_func_mapper_add_ip(mapper, ip, data); } static void -- cgit v1.2.3 From d3d532d798c5720055ab02a10bf7829a33c3645a Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Tue, 4 Apr 2017 16:44:43 -0400 Subject: ftrace: Have unregister_ftrace_function_probe_func() return a value Currently unregister_ftrace_function_probe_func() is a void function. It does not give any feedback if an error occurred or no item was found to remove and nothing was done. Change it to return status and success if it removed something. Also update the callers to return that feedback to the user. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/ftrace.c | 14 +++++++++++--- kernel/trace/trace.c | 6 ++---- kernel/trace/trace.h | 2 +- kernel/trace/trace_events.c | 3 +-- kernel/trace/trace_functions.c | 6 ++---- 5 files changed, 17 insertions(+), 14 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 5c8d8eea9e7c..cbae7fb1be15 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -4102,7 +4102,7 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, return count; } -void +int unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) { struct ftrace_ops_hash old_hash_ops; @@ -4131,7 +4131,7 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) /* we do not support '!' for function probes */ if (WARN_ON(not)) - return; + return -EINVAL; } mutex_lock(&trace_probe_ops.func_hash->regex_lock); @@ -4140,9 +4140,9 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) /* Probes only have filters */ old_hash_ops.notrace_hash = NULL; + ret = -ENOMEM; hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash); if (!hash) - /* Hmm, should report this somehow */ goto out_unlock; INIT_LIST_HEAD(&free_list); @@ -4173,6 +4173,13 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) list_add(&entry->free_list, &free_list); } } + + /* Nothing found? */ + if (list_empty(&free_list)) { + ret = -EINVAL; + goto out_unlock; + } + mutex_lock(&ftrace_lock); disabled = __disable_ftrace_function_probe(); /* @@ -4198,6 +4205,7 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) out_unlock: mutex_unlock(&trace_probe_ops.func_hash->regex_lock); free_ftrace_hash(hash); + return ret; } static LIST_HEAD(ftrace_commands); diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 7a4d578d8887..64a4418a5106 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -6829,10 +6829,8 @@ ftrace_trace_snapshot_callback(struct ftrace_hash *hash, ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops; - if (glob[0] == '!') { - unregister_ftrace_function_probe_func(glob+1, ops); - return 0; - } + if (glob[0] == '!') + return unregister_ftrace_function_probe_func(glob+1, ops); if (!param) goto out_reg; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 86aa5a2222ba..31d80bff9ee6 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -963,7 +963,7 @@ void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper, extern int register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, void *data); -extern void +extern int unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops); int register_ftrace_command(struct ftrace_func_command *cmd); diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 37902107c44f..9e07a5b3869b 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2652,8 +2652,7 @@ event_enable_func(struct ftrace_hash *hash, ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops; if (glob[0] == '!') { - unregister_ftrace_function_probe_func(glob+1, ops); - ret = 0; + ret = unregister_ftrace_function_probe_func(glob+1, ops); goto out; } diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 56d0fe1e4ea1..dcb4d37ed4bd 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -586,10 +586,8 @@ ftrace_trace_probe_callback(struct ftrace_probe_ops *ops, if (!enable) return -EINVAL; - if (glob[0] == '!') { - unregister_ftrace_function_probe_func(glob+1, ops); - return 0; - } + if (glob[0] == '!') + return unregister_ftrace_function_probe_func(glob+1, ops); if (!param) goto out_reg; -- cgit v1.2.3 From 04ec7bb642b77374b53731b795b5654b5aff1c00 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Wed, 5 Apr 2017 13:12:55 -0400 Subject: tracing: Have the trace_array hold the list of registered func probes Add a link list to the trace_array to hold func probes that are registered. Currently, all function probes are the same for all instances as it was before, that is, only the top level trace_array holds the function probes. But this lays the ground work to have function probes be attached to individual instances, and having the event trigger only affect events in the given instance. But that work is still to be done. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/ftrace.c | 41 +++++++++++++++++++++++++++++------------ kernel/trace/trace.c | 6 ++++-- kernel/trace/trace.h | 13 ++++++++++--- kernel/trace/trace_events.c | 5 ++--- kernel/trace/trace_functions.c | 21 +++++++++++---------- 5 files changed, 56 insertions(+), 30 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 8394055e6793..ea208e93f000 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1096,8 +1096,6 @@ static bool update_all_ops; # error Dynamic ftrace depends on MCOUNT_RECORD #endif -static LIST_HEAD(ftrace_func_probes); - struct ftrace_func_entry { struct hlist_node hlist; unsigned long ip; @@ -3070,6 +3068,8 @@ static void * t_probe_next(struct seq_file *m, loff_t *pos) { struct ftrace_iterator *iter = m->private; + struct trace_array *tr = global_ops.private; + struct list_head *func_probes; struct ftrace_hash *hash; struct list_head *next; struct hlist_node *hnd = NULL; @@ -3079,11 +3079,15 @@ t_probe_next(struct seq_file *m, loff_t *pos) (*pos)++; iter->pos = *pos; - if (list_empty(&ftrace_func_probes)) + if (!tr) + return NULL; + + func_probes = &tr->func_probes; + if (list_empty(func_probes)) return NULL; if (!iter->probe) { - next = ftrace_func_probes.next; + next = func_probes->next; iter->probe = list_entry(next, struct ftrace_probe_ops, list); } @@ -3095,7 +3099,7 @@ t_probe_next(struct seq_file *m, loff_t *pos) retry: if (iter->pidx >= size) { - if (iter->probe->list.next == &ftrace_func_probes) + if (iter->probe->list.next == func_probes) return NULL; next = iter->probe->list.next; iter->probe = list_entry(next, struct ftrace_probe_ops, list); @@ -3752,7 +3756,7 @@ static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, */ static int -ftrace_mod_callback(struct ftrace_hash *hash, +ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash, char *func, char *cmd, char *module, int enable) { int ret; @@ -3942,8 +3946,8 @@ void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper, } int -register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, - void *data) +register_ftrace_function_probe(char *glob, struct trace_array *tr, + struct ftrace_probe_ops *ops, void *data) { struct ftrace_func_entry *entry; struct ftrace_hash **orig_hash; @@ -3954,6 +3958,9 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, int ret; int i; + if (WARN_ON(!tr)) + return -EINVAL; + /* We do not support '!' for function probes */ if (WARN_ON(glob[0] == '!')) return -EINVAL; @@ -4006,7 +4013,7 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, goto err_unlock; if (list_empty(&ops->list)) - list_add(&ops->list, &ftrace_func_probes); + list_add(&ops->list, &tr->func_probes); if (!(ops->ops.flags & FTRACE_OPS_FL_ENABLED)) ret = ftrace_startup(&ops->ops, 0); @@ -4192,9 +4199,11 @@ __init int unregister_ftrace_command(struct ftrace_func_command *cmd) return ret; } -static int ftrace_process_regex(struct ftrace_hash *hash, +static int ftrace_process_regex(struct ftrace_iterator *iter, char *buff, int len, int enable) { + struct ftrace_hash *hash = iter->hash; + struct trace_array *tr = global_ops.private; char *func, *command, *next = buff; struct ftrace_func_command *p; int ret = -EINVAL; @@ -4214,10 +4223,13 @@ static int ftrace_process_regex(struct ftrace_hash *hash, command = strsep(&next, ":"); + if (WARN_ON_ONCE(!tr)) + return -EINVAL; + mutex_lock(&ftrace_cmd_mutex); list_for_each_entry(p, &ftrace_commands, list) { if (strcmp(p->name, command) == 0) { - ret = p->func(hash, func, command, next, enable); + ret = p->func(tr, hash, func, command, next, enable); goto out_unlock; } } @@ -4254,7 +4266,7 @@ ftrace_regex_write(struct file *file, const char __user *ubuf, if (read >= 0 && trace_parser_loaded(parser) && !trace_parser_cont(parser)) { - ret = ftrace_process_regex(iter->hash, parser->buffer, + ret = ftrace_process_regex(iter, parser->buffer, parser->idx, enable); trace_parser_clear(parser); if (ret < 0) @@ -5441,6 +5453,10 @@ static void ftrace_update_trampoline(struct ftrace_ops *ops) arch_ftrace_update_trampoline(ops); } +void ftrace_init_trace_array(struct trace_array *tr) +{ + INIT_LIST_HEAD(&tr->func_probes); +} #else static struct ftrace_ops global_ops = { @@ -5495,6 +5511,7 @@ __init void ftrace_init_global_array_ops(struct trace_array *tr) { tr->ops = &global_ops; tr->ops->private = tr; + ftrace_init_trace_array(tr); } void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 64a4418a5106..86598293787a 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -6815,7 +6815,7 @@ static struct ftrace_probe_ops snapshot_count_probe_ops = { }; static int -ftrace_trace_snapshot_callback(struct ftrace_hash *hash, +ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, char *glob, char *cmd, char *param, int enable) { struct ftrace_probe_ops *ops; @@ -6855,7 +6855,7 @@ ftrace_trace_snapshot_callback(struct ftrace_hash *hash, return ret; out_reg: - ret = register_ftrace_function_probe(glob, ops, count); + ret = register_ftrace_function_probe(glob, tr, ops, count); if (ret >= 0) alloc_snapshot(&global_trace); @@ -7468,6 +7468,8 @@ static int instance_mkdir(const char *name) goto out_free_tr; } + ftrace_init_trace_array(tr); + init_tracer_tracefs(tr, tr->dir); init_trace_flags_index(tr); __update_tracer_options(tr); diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index d457addcc224..68ff25e4cb19 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -262,6 +262,9 @@ struct trace_array { #ifdef CONFIG_FUNCTION_TRACER struct ftrace_ops *ops; struct trace_pid_list __rcu *function_pids; +#ifdef CONFIG_DYNAMIC_FTRACE + struct list_head func_probes; +#endif /* function tracing enabled */ int function_enabled; #endif @@ -696,6 +699,9 @@ extern void trace_event_follow_fork(struct trace_array *tr, bool enable); #ifdef CONFIG_DYNAMIC_FTRACE extern unsigned long ftrace_update_tot_cnt; +void ftrace_init_trace_array(struct trace_array *tr); +#else +static inline void ftrace_init_trace_array(struct trace_array *tr) { } #endif #define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func extern int DYN_FTRACE_TEST_NAME(void); @@ -883,7 +889,8 @@ extern struct list_head ftrace_pids; struct ftrace_func_command { struct list_head list; char *name; - int (*func)(struct ftrace_hash *hash, + int (*func)(struct trace_array *tr, + struct ftrace_hash *hash, char *func, char *cmd, char *params, int enable); }; @@ -963,8 +970,8 @@ void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper, ftrace_mapper_func free_func); extern int -register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, - void *data); +register_ftrace_function_probe(char *glob, struct trace_array *tr, + struct ftrace_probe_ops *ops, void *data); extern int unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops); diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 9e07a5b3869b..f0d6e5aef53e 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2611,10 +2611,9 @@ static struct ftrace_probe_ops event_disable_count_probe_ops = { }; static int -event_enable_func(struct ftrace_hash *hash, +event_enable_func(struct trace_array *tr, struct ftrace_hash *hash, char *glob, char *cmd, char *param, int enabled) { - struct trace_array *tr = top_trace_array(); struct trace_event_file *file; struct ftrace_probe_ops *ops; struct event_probe_data *data; @@ -2701,7 +2700,7 @@ event_enable_func(struct ftrace_hash *hash, if (ret < 0) goto out_put; - ret = register_ftrace_function_probe(glob, ops, data); + ret = register_ftrace_function_probe(glob, tr, ops, data); /* * The above returns on success the # of functions enabled, * but if it didn't find any functions it returns zero. diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index dcb4d37ed4bd..2c8961b35401 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -574,7 +574,8 @@ static struct ftrace_probe_ops stacktrace_probe_ops = { }; static int -ftrace_trace_probe_callback(struct ftrace_probe_ops *ops, +ftrace_trace_probe_callback(struct trace_array *tr, + struct ftrace_probe_ops *ops, struct ftrace_hash *hash, char *glob, char *cmd, char *param, int enable) { @@ -612,13 +613,13 @@ ftrace_trace_probe_callback(struct ftrace_probe_ops *ops, return ret; out_reg: - ret = register_ftrace_function_probe(glob, ops, count); + ret = register_ftrace_function_probe(glob, tr, ops, count); return ret < 0 ? ret : 0; } static int -ftrace_trace_onoff_callback(struct ftrace_hash *hash, +ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash, char *glob, char *cmd, char *param, int enable) { struct ftrace_probe_ops *ops; @@ -629,24 +630,24 @@ ftrace_trace_onoff_callback(struct ftrace_hash *hash, else ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops; - return ftrace_trace_probe_callback(ops, hash, glob, cmd, + return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, param, enable); } static int -ftrace_stacktrace_callback(struct ftrace_hash *hash, +ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash, char *glob, char *cmd, char *param, int enable) { struct ftrace_probe_ops *ops; ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops; - return ftrace_trace_probe_callback(ops, hash, glob, cmd, + return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, param, enable); } static int -ftrace_dump_callback(struct ftrace_hash *hash, +ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash, char *glob, char *cmd, char *param, int enable) { struct ftrace_probe_ops *ops; @@ -654,12 +655,12 @@ ftrace_dump_callback(struct ftrace_hash *hash, ops = &dump_probe_ops; /* Only dump once. */ - return ftrace_trace_probe_callback(ops, hash, glob, cmd, + return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, "1", enable); } static int -ftrace_cpudump_callback(struct ftrace_hash *hash, +ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash, char *glob, char *cmd, char *param, int enable) { struct ftrace_probe_ops *ops; @@ -667,7 +668,7 @@ ftrace_cpudump_callback(struct ftrace_hash *hash, ops = &cpudump_probe_ops; /* Only dump once. */ - return ftrace_trace_probe_callback(ops, hash, glob, cmd, + return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, "1", enable); } -- cgit v1.2.3 From b5f081b563a6cdcb85a543df8c851951a8978275 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 10 Apr 2017 22:30:05 -0400 Subject: tracing: Pass the trace_array into ftrace_probe_ops functions Pass the trace_array associated to a ftrace_probe_ops into the probe_ops func(), init() and free() functions. The trace_array is the descriptor that describes a tracing instance. This will help create the infrastructure that will allow having function probes unique to tracing instances. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/ftrace.c | 13 +++++++++---- kernel/trace/trace.c | 14 ++++++++------ kernel/trace/trace.h | 3 +++ kernel/trace/trace_events.c | 16 +++++++++------- kernel/trace/trace_functions.c | 35 ++++++++++++++++++++++------------- 5 files changed, 51 insertions(+), 30 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index ea208e93f000..e51cd6b51253 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -3791,6 +3791,7 @@ static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct pt_regs *pt_regs) { struct ftrace_probe_ops *probe_ops; + struct trace_array *tr = op->private; probe_ops = container_of(op, struct ftrace_probe_ops, ops); @@ -3800,7 +3801,7 @@ static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, * on the hash. rcu_read_lock is too dangerous here. */ preempt_disable_notrace(); - probe_ops->func(ip, parent_ip, probe_ops, NULL); + probe_ops->func(ip, parent_ip, tr, probe_ops, NULL); preempt_enable_notrace(); } @@ -3969,6 +3970,7 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr, ops->ops.func = function_trace_probe_call; ftrace_ops_init(&ops->ops); INIT_LIST_HEAD(&ops->list); + ops->ops.private = tr; } mutex_lock(&ops->ops.func_hash->regex_lock); @@ -3997,7 +3999,7 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr, * to give the caller an opportunity to do so. */ if (ops->init) { - ret = ops->init(ops, entry->ip, data); + ret = ops->init(ops, tr, entry->ip, data); if (ret < 0) goto out; } @@ -4038,7 +4040,7 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr, hlist_for_each_entry(entry, &hash->buckets[i], hlist) { if (ftrace_lookup_ip(old_hash, entry->ip)) continue; - ops->free(ops, entry->ip, NULL); + ops->free(ops, tr, entry->ip, NULL); } } goto out_unlock; @@ -4055,6 +4057,7 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) struct ftrace_hash *hash = NULL; struct hlist_node *tmp; struct hlist_head hhd; + struct trace_array *tr; char str[KSYM_SYMBOL_LEN]; int i, ret; int size; @@ -4062,6 +4065,8 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) if (!(ops->ops.flags & FTRACE_OPS_FL_INITIALIZED)) return -EINVAL; + tr = ops->ops.private; + if (glob && (strcmp(glob, "*") == 0 || !strlen(glob))) func_g.search = NULL; else if (glob) { @@ -4139,7 +4144,7 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) { hlist_del(&entry->hlist); if (ops->free) - ops->free(ops, entry->ip, NULL); + ops->free(ops, tr, entry->ip, NULL); kfree(entry); } mutex_unlock(&ftrace_lock); diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 86598293787a..368310e78d45 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -6736,14 +6736,16 @@ static const struct file_operations tracing_dyn_info_fops = { #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) static void ftrace_snapshot(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **data) { tracing_snapshot(); } static void ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **data) { struct ftrace_func_mapper *mapper = ops->private_data; long *count = NULL; @@ -6785,8 +6787,8 @@ ftrace_snapshot_print(struct seq_file *m, unsigned long ip, } static int -ftrace_snapshot_init(struct ftrace_probe_ops *ops, unsigned long ip, - void *data) +ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr, + unsigned long ip, void *data) { struct ftrace_func_mapper *mapper = ops->private_data; @@ -6794,8 +6796,8 @@ ftrace_snapshot_init(struct ftrace_probe_ops *ops, unsigned long ip, } static void -ftrace_snapshot_free(struct ftrace_probe_ops *ops, unsigned long ip, - void **_data) +ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr, + unsigned long ip, void **data) { struct ftrace_func_mapper *mapper = ops->private_data; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 68ff25e4cb19..390761804886 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -943,11 +943,14 @@ struct ftrace_probe_ops { struct list_head list; void (*func)(unsigned long ip, unsigned long parent_ip, + struct trace_array *tr, struct ftrace_probe_ops *ops, void **data); int (*init)(struct ftrace_probe_ops *ops, + struct trace_array *tr, unsigned long ip, void *data); void (*free)(struct ftrace_probe_ops *ops, + struct trace_array *tr, unsigned long ip, void **data); int (*print)(struct seq_file *m, unsigned long ip, diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index f0d6e5aef53e..713bec614312 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2470,7 +2470,8 @@ static void update_event_probe(struct event_probe_data *data) static void event_enable_probe(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **_data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **_data) { struct ftrace_func_mapper *mapper = ops->private_data; struct event_probe_data *data; @@ -2486,7 +2487,8 @@ event_enable_probe(unsigned long ip, unsigned long parent_ip, static void event_enable_count_probe(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **_data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **_data) { struct ftrace_func_mapper *mapper = ops->private_data; struct event_probe_data *data; @@ -2513,7 +2515,7 @@ event_enable_count_probe(unsigned long ip, unsigned long parent_ip, static int event_enable_print(struct seq_file *m, unsigned long ip, - struct ftrace_probe_ops *ops, void *_data) + struct ftrace_probe_ops *ops, void *_data) { struct ftrace_func_mapper *mapper = ops->private_data; struct event_probe_data *data; @@ -2542,8 +2544,8 @@ event_enable_print(struct seq_file *m, unsigned long ip, } static int -event_enable_init(struct ftrace_probe_ops *ops, unsigned long ip, - void *_data) +event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr, + unsigned long ip, void *_data) { struct ftrace_func_mapper *mapper = ops->private_data; struct event_probe_data *data = _data; @@ -2559,8 +2561,8 @@ event_enable_init(struct ftrace_probe_ops *ops, unsigned long ip, } static void -event_enable_free(struct ftrace_probe_ops *ops, unsigned long ip, - void **_data) +event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr, + unsigned long ip, void **_data) { struct ftrace_func_mapper *mapper = ops->private_data; struct event_probe_data *data; diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 2c8961b35401..797f087183c5 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -328,21 +328,24 @@ static void update_traceon_count(struct ftrace_probe_ops *ops, static void ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **data) { update_traceon_count(ops, ip, 1); } static void ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **data) { update_traceon_count(ops, ip, 0); } static void ftrace_traceon(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **data) { if (tracing_is_on()) return; @@ -352,7 +355,8 @@ ftrace_traceon(unsigned long ip, unsigned long parent_ip, static void ftrace_traceoff(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **data) { if (!tracing_is_on()) return; @@ -371,14 +375,16 @@ ftrace_traceoff(unsigned long ip, unsigned long parent_ip, static void ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **data) { trace_dump_stack(STACK_SKIP); } static void ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **data) { struct ftrace_func_mapper *mapper = ops->private_data; long *count; @@ -436,7 +442,8 @@ static int update_count(struct ftrace_probe_ops *ops, unsigned long ip) static void ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **data) { if (update_count(ops, ip)) ftrace_dump(DUMP_ALL); @@ -445,7 +452,8 @@ ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, /* Only dump the current CPU buffer. */ static void ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, - struct ftrace_probe_ops *ops, void **data) + struct trace_array *tr, struct ftrace_probe_ops *ops, + void **data) { if (update_count(ops, ip)) ftrace_dump(DUMP_ORIG); @@ -473,7 +481,8 @@ ftrace_probe_print(const char *name, struct seq_file *m, static int ftrace_traceon_print(struct seq_file *m, unsigned long ip, - struct ftrace_probe_ops *ops, void *data) + struct ftrace_probe_ops *ops, + void *data) { return ftrace_probe_print("traceon", m, ip, ops); } @@ -508,8 +517,8 @@ ftrace_cpudump_print(struct seq_file *m, unsigned long ip, static int -ftrace_count_init(struct ftrace_probe_ops *ops, unsigned long ip, - void *data) +ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr, + unsigned long ip, void *data) { struct ftrace_func_mapper *mapper = ops->private_data; @@ -517,8 +526,8 @@ ftrace_count_init(struct ftrace_probe_ops *ops, unsigned long ip, } static void -ftrace_count_free(struct ftrace_probe_ops *ops, unsigned long ip, - void **_data) +ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr, + unsigned long ip, void **_data) { struct ftrace_func_mapper *mapper = ops->private_data; -- cgit v1.2.3 From 7b60f3d8761561d95d7e962522d6338143fc2329 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Tue, 18 Apr 2017 14:50:39 -0400 Subject: ftrace: Dynamically create the probe ftrace_ops for the trace_array In order to eventually have each trace_array instance have its own unique set of function probes (triggers), the trace array needs to hold the ops and the filters for the probes. This is the first step to accomplish this. Instead of having the private data of the probe ops point to the trace_array, create a separate list that the trace_array holds. There's only one private_data for a probe, we need one per trace_array. The probe ftrace_ops will be dynamically created for each instance, instead of being static. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/ftrace.c | 192 ++++++++++++++++++++++++++++++----------- kernel/trace/trace.c | 2 +- kernel/trace/trace.h | 5 +- kernel/trace/trace_events.c | 2 +- kernel/trace/trace_functions.c | 2 +- 5 files changed, 146 insertions(+), 57 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index e51cd6b51253..8fdc18500c61 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1101,6 +1101,14 @@ struct ftrace_func_entry { unsigned long ip; }; +struct ftrace_func_probe { + struct ftrace_probe_ops *probe_ops; + struct ftrace_ops ops; + struct trace_array *tr; + struct list_head list; + int ref; +}; + /* * We make these constant because no one should touch them, * but they are used as the default "empty hash", to avoid allocating @@ -3054,7 +3062,7 @@ struct ftrace_iterator { loff_t func_pos; struct ftrace_page *pg; struct dyn_ftrace *func; - struct ftrace_probe_ops *probe; + struct ftrace_func_probe *probe; struct ftrace_func_entry *probe_entry; struct trace_parser parser; struct ftrace_hash *hash; @@ -3088,7 +3096,7 @@ t_probe_next(struct seq_file *m, loff_t *pos) if (!iter->probe) { next = func_probes->next; - iter->probe = list_entry(next, struct ftrace_probe_ops, list); + iter->probe = list_entry(next, struct ftrace_func_probe, list); } if (iter->probe_entry) @@ -3102,7 +3110,7 @@ t_probe_next(struct seq_file *m, loff_t *pos) if (iter->probe->list.next == func_probes) return NULL; next = iter->probe->list.next; - iter->probe = list_entry(next, struct ftrace_probe_ops, list); + iter->probe = list_entry(next, struct ftrace_func_probe, list); hash = iter->probe->ops.func_hash->filter_hash; size = 1 << hash->size_bits; iter->pidx = 0; @@ -3166,8 +3174,9 @@ static void *t_probe_start(struct seq_file *m, loff_t *pos) static int t_probe_show(struct seq_file *m, struct ftrace_iterator *iter) { - struct ftrace_probe_ops *probe; struct ftrace_func_entry *probe_entry; + struct ftrace_probe_ops *probe_ops; + struct ftrace_func_probe *probe; probe = iter->probe; probe_entry = iter->probe_entry; @@ -3175,10 +3184,13 @@ t_probe_show(struct seq_file *m, struct ftrace_iterator *iter) if (WARN_ON_ONCE(!probe || !probe_entry)) return -EIO; - if (probe->print) - return probe->print(m, probe_entry->ip, probe, NULL); + probe_ops = probe->probe_ops; + + if (probe_ops->print) + return probe_ops->print(m, probe_entry->ip, probe_ops, NULL); - seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip, (void *)probe->func); + seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip, + (void *)probe_ops->func); return 0; } @@ -3791,9 +3803,10 @@ static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct pt_regs *pt_regs) { struct ftrace_probe_ops *probe_ops; - struct trace_array *tr = op->private; + struct ftrace_func_probe *probe; - probe_ops = container_of(op, struct ftrace_probe_ops, ops); + probe = container_of(op, struct ftrace_func_probe, ops); + probe_ops = probe->probe_ops; /* * Disable preemption for these calls to prevent a RCU grace @@ -3801,7 +3814,7 @@ static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, * on the hash. rcu_read_lock is too dangerous here. */ preempt_disable_notrace(); - probe_ops->func(ip, parent_ip, tr, probe_ops, NULL); + probe_ops->func(ip, parent_ip, probe->tr, probe_ops, NULL); preempt_enable_notrace(); } @@ -3946,11 +3959,41 @@ void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper, free_ftrace_hash(&mapper->hash); } +static void release_probe(struct ftrace_func_probe *probe) +{ + struct ftrace_probe_ops *probe_ops; + + mutex_lock(&ftrace_lock); + + WARN_ON(probe->ref <= 0); + + /* Subtract the ref that was used to protect this instance */ + probe->ref--; + + if (!probe->ref) { + probe_ops = probe->probe_ops; + list_del(&probe->list); + kfree(probe); + } + mutex_unlock(&ftrace_lock); +} + +static void acquire_probe_locked(struct ftrace_func_probe *probe) +{ + /* + * Add one ref to keep it from being freed when releasing the + * ftrace_lock mutex. + */ + probe->ref++; +} + int register_ftrace_function_probe(char *glob, struct trace_array *tr, - struct ftrace_probe_ops *ops, void *data) + struct ftrace_probe_ops *probe_ops, + void *data) { struct ftrace_func_entry *entry; + struct ftrace_func_probe *probe; struct ftrace_hash **orig_hash; struct ftrace_hash *old_hash; struct ftrace_hash *hash; @@ -3966,16 +4009,33 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr, if (WARN_ON(glob[0] == '!')) return -EINVAL; - if (!(ops->ops.flags & FTRACE_OPS_FL_INITIALIZED)) { - ops->ops.func = function_trace_probe_call; - ftrace_ops_init(&ops->ops); - INIT_LIST_HEAD(&ops->list); - ops->ops.private = tr; + + mutex_lock(&ftrace_lock); + /* Check if the probe_ops is already registered */ + list_for_each_entry(probe, &tr->func_probes, list) { + if (probe->probe_ops == probe_ops) + break; } + if (&probe->list == &tr->func_probes) { + probe = kzalloc(sizeof(*probe), GFP_KERNEL); + if (!probe) { + mutex_unlock(&ftrace_lock); + return -ENOMEM; + } + probe->probe_ops = probe_ops; + probe->ops.func = function_trace_probe_call; + probe->tr = tr; + ftrace_ops_init(&probe->ops); + list_add(&probe->list, &tr->func_probes); + } + + acquire_probe_locked(probe); - mutex_lock(&ops->ops.func_hash->regex_lock); + mutex_unlock(&ftrace_lock); + + mutex_lock(&probe->ops.func_hash->regex_lock); - orig_hash = &ops->ops.func_hash->filter_hash; + orig_hash = &probe->ops.func_hash->filter_hash; old_hash = *orig_hash; hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash); @@ -3998,8 +4058,9 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr, * for each function we find. We call the callback * to give the caller an opportunity to do so. */ - if (ops->init) { - ret = ops->init(ops, tr, entry->ip, data); + if (probe_ops->init) { + ret = probe_ops->init(probe_ops, tr, + entry->ip, data); if (ret < 0) goto out; } @@ -4009,16 +4070,22 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr, mutex_lock(&ftrace_lock); - ret = ftrace_hash_move_and_update_ops(&ops->ops, orig_hash, - hash, 1); + if (!count) { + /* Nothing was added? */ + ret = -EINVAL; + goto out_unlock; + } + + ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, + hash, 1); if (ret < 0) goto err_unlock; - if (list_empty(&ops->list)) - list_add(&ops->list, &tr->func_probes); + /* One ref for each new function traced */ + probe->ref += count; - if (!(ops->ops.flags & FTRACE_OPS_FL_ENABLED)) - ret = ftrace_startup(&ops->ops, 0); + if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED)) + ret = ftrace_startup(&probe->ops, 0); out_unlock: mutex_unlock(&ftrace_lock); @@ -4026,13 +4093,15 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr, if (!ret) ret = count; out: - mutex_unlock(&ops->ops.func_hash->regex_lock); + mutex_unlock(&probe->ops.func_hash->regex_lock); free_ftrace_hash(hash); + release_probe(probe); + return ret; err_unlock: - if (!ops->free) + if (!probe_ops->free || !count) goto out_unlock; /* Failed to do the move, need to call the free functions */ @@ -4040,33 +4109,30 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr, hlist_for_each_entry(entry, &hash->buckets[i], hlist) { if (ftrace_lookup_ip(old_hash, entry->ip)) continue; - ops->free(ops, tr, entry->ip, NULL); + probe_ops->free(probe_ops, tr, entry->ip, NULL); } } goto out_unlock; } int -unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) +unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr, + struct ftrace_probe_ops *probe_ops) { struct ftrace_ops_hash old_hash_ops; struct ftrace_func_entry *entry; + struct ftrace_func_probe *probe; struct ftrace_glob func_g; struct ftrace_hash **orig_hash; struct ftrace_hash *old_hash; struct ftrace_hash *hash = NULL; struct hlist_node *tmp; struct hlist_head hhd; - struct trace_array *tr; char str[KSYM_SYMBOL_LEN]; - int i, ret; + int count = 0; + int i, ret = -ENODEV; int size; - if (!(ops->ops.flags & FTRACE_OPS_FL_INITIALIZED)) - return -EINVAL; - - tr = ops->ops.private; - if (glob && (strcmp(glob, "*") == 0 || !strlen(glob))) func_g.search = NULL; else if (glob) { @@ -4082,12 +4148,28 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) return -EINVAL; } - mutex_lock(&ops->ops.func_hash->regex_lock); + mutex_lock(&ftrace_lock); + /* Check if the probe_ops is already registered */ + list_for_each_entry(probe, &tr->func_probes, list) { + if (probe->probe_ops == probe_ops) + break; + } + if (&probe->list == &tr->func_probes) + goto err_unlock_ftrace; + + ret = -EINVAL; + if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED)) + goto err_unlock_ftrace; + + acquire_probe_locked(probe); - orig_hash = &ops->ops.func_hash->filter_hash; + mutex_unlock(&ftrace_lock); + + mutex_lock(&probe->ops.func_hash->regex_lock); + + orig_hash = &probe->ops.func_hash->filter_hash; old_hash = *orig_hash; - ret = -EINVAL; if (ftrace_hash_empty(old_hash)) goto out_unlock; @@ -4112,46 +4194,54 @@ unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) if (!ftrace_match(str, &func_g)) continue; } - + count++; remove_hash_entry(hash, entry); hlist_add_head(&entry->hlist, &hhd); } } /* Nothing found? */ - if (hlist_empty(&hhd)) { + if (!count) { ret = -EINVAL; goto out_unlock; } mutex_lock(&ftrace_lock); - if (ftrace_hash_empty(hash)) { - ftrace_shutdown(&ops->ops, 0); - list_del_init(&ops->list); - } + WARN_ON(probe->ref < count); + probe->ref -= count; - ret = ftrace_hash_move_and_update_ops(&ops->ops, orig_hash, + if (ftrace_hash_empty(hash)) + ftrace_shutdown(&probe->ops, 0); + + ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, hash, 1); /* still need to update the function call sites */ if (ftrace_enabled && !ftrace_hash_empty(hash)) - ftrace_run_modify_code(&ops->ops, FTRACE_UPDATE_CALLS, + ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS, &old_hash_ops); synchronize_sched(); hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) { hlist_del(&entry->hlist); - if (ops->free) - ops->free(ops, tr, entry->ip, NULL); + if (probe_ops->free) + probe_ops->free(probe_ops, tr, entry->ip, NULL); kfree(entry); } mutex_unlock(&ftrace_lock); out_unlock: - mutex_unlock(&ops->ops.func_hash->regex_lock); + mutex_unlock(&probe->ops.func_hash->regex_lock); free_ftrace_hash(hash); + + release_probe(probe); + + return ret; + + err_unlock_ftrace: + mutex_unlock(&ftrace_lock); return ret; } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 368310e78d45..e61610e5e6e3 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -6832,7 +6832,7 @@ ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops; if (glob[0] == '!') - return unregister_ftrace_function_probe_func(glob+1, ops); + return unregister_ftrace_function_probe_func(glob+1, tr, ops); if (!param) goto out_reg; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 390761804886..e978ecd257b8 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -939,8 +939,6 @@ static inline void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) { #if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_DYNAMIC_FTRACE) struct ftrace_probe_ops { - struct ftrace_ops ops; - struct list_head list; void (*func)(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, @@ -976,7 +974,8 @@ extern int register_ftrace_function_probe(char *glob, struct trace_array *tr, struct ftrace_probe_ops *ops, void *data); extern int -unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops); +unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr, + struct ftrace_probe_ops *ops); int register_ftrace_command(struct ftrace_func_command *cmd); int unregister_ftrace_command(struct ftrace_func_command *cmd); diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 713bec614312..48c7f70cbac7 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2653,7 +2653,7 @@ event_enable_func(struct trace_array *tr, struct ftrace_hash *hash, ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops; if (glob[0] == '!') { - ret = unregister_ftrace_function_probe_func(glob+1, ops); + ret = unregister_ftrace_function_probe_func(glob+1, tr, ops); goto out; } diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 797f087183c5..b95f56ba9744 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -597,7 +597,7 @@ ftrace_trace_probe_callback(struct trace_array *tr, return -EINVAL; if (glob[0] == '!') - return unregister_ftrace_function_probe_func(glob+1, ops); + return unregister_ftrace_function_probe_func(glob+1, tr, ops); if (!param) goto out_reg; -- cgit v1.2.3 From 6e4443199e5354255e8a4c1e8e5cfc8ef064c3ce Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Wed, 19 Apr 2017 22:39:44 -0400 Subject: tracing/ftrace: Add a better way to pass data via the probe functions With the redesign of the registration and execution of the function probes (triggers), data can now be passed from the setup of the probe to the probe callers that are specific to the trace_array it is on. Although, all probes still only affect the toplevel trace array, this change will allow for instances to have their own probes separated from other instances and the top array. That is, something like the stacktrace probe can be set to trace only in an instance and not the toplevel trace array. This isn't implement yet, but this change sets the ground work for the change. When a probe callback is triggered (someone writes the probe format into set_ftrace_filter), it calls register_ftrace_function_probe() passing in init_data that will be used to initialize the probe. Then for every matching function, register_ftrace_function_probe() will call the probe_ops->init() function with the init data that was passed to it, as well as an address to a place holder that is associated with the probe and the instance. The first occurrence will have a NULL in the pointer. The init() function will then initialize it. If other probes are added, or more functions are part of the probe, the place holder will be passed to the init() function with the place holder data that it was initialized to the last time. Then this place_holder is passed to each of the other probe_ops functions, where it can be used in the function callback. When the probe_ops free() function is called, it can be called either with the rip of the function that is being removed from the probe, or zero, indicating that there are no more functions attached to the probe, and the place holder is about to be freed. This gives the probe_ops a way to free the data it assigned to the place holder if it was allocade during the first init call. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/ftrace.c | 25 +++++++--- kernel/trace/trace.c | 38 ++++++++------ kernel/trace/trace.h | 8 +-- kernel/trace/trace_events.c | 110 +++++++++++++++++++++++------------------ kernel/trace/trace_functions.c | 79 ++++++++++++++++------------- 5 files changed, 153 insertions(+), 107 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 8fdc18500c61..774e9108e5dc 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1106,6 +1106,7 @@ struct ftrace_func_probe { struct ftrace_ops ops; struct trace_array *tr; struct list_head list; + void *data; int ref; }; @@ -3187,7 +3188,7 @@ t_probe_show(struct seq_file *m, struct ftrace_iterator *iter) probe_ops = probe->probe_ops; if (probe_ops->print) - return probe_ops->print(m, probe_entry->ip, probe_ops, NULL); + return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data); seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip, (void *)probe_ops->func); @@ -3814,7 +3815,7 @@ static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, * on the hash. rcu_read_lock is too dangerous here. */ preempt_disable_notrace(); - probe_ops->func(ip, parent_ip, probe->tr, probe_ops, NULL); + probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data); preempt_enable_notrace(); } @@ -3972,6 +3973,12 @@ static void release_probe(struct ftrace_func_probe *probe) if (!probe->ref) { probe_ops = probe->probe_ops; + /* + * Sending zero as ip tells probe_ops to free + * the probe->data itself + */ + if (probe_ops->free) + probe_ops->free(probe_ops, probe->tr, 0, probe->data); list_del(&probe->list); kfree(probe); } @@ -4060,9 +4067,15 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr, */ if (probe_ops->init) { ret = probe_ops->init(probe_ops, tr, - entry->ip, data); - if (ret < 0) + entry->ip, data, + &probe->data); + if (ret < 0) { + if (probe_ops->free && count) + probe_ops->free(probe_ops, tr, + 0, probe->data); + probe->data = NULL; goto out; + } } count++; } @@ -4109,7 +4122,7 @@ register_ftrace_function_probe(char *glob, struct trace_array *tr, hlist_for_each_entry(entry, &hash->buckets[i], hlist) { if (ftrace_lookup_ip(old_hash, entry->ip)) continue; - probe_ops->free(probe_ops, tr, entry->ip, NULL); + probe_ops->free(probe_ops, tr, entry->ip, probe->data); } } goto out_unlock; @@ -4227,7 +4240,7 @@ unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr, hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) { hlist_del(&entry->hlist); if (probe_ops->free) - probe_ops->free(probe_ops, tr, entry->ip, NULL); + probe_ops->free(probe_ops, tr, entry->ip, probe->data); kfree(entry); } mutex_unlock(&ftrace_lock); diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index e61610e5e6e3..18256cd7ad2c 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -6737,7 +6737,7 @@ static const struct file_operations tracing_dyn_info_fops = { static void ftrace_snapshot(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **data) + void *data) { tracing_snapshot(); } @@ -6745,9 +6745,9 @@ ftrace_snapshot(unsigned long ip, unsigned long parent_ip, static void ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **data) + void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; + struct ftrace_func_mapper *mapper = data; long *count = NULL; if (mapper) @@ -6768,7 +6768,7 @@ static int ftrace_snapshot_print(struct seq_file *m, unsigned long ip, struct ftrace_probe_ops *ops, void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; + struct ftrace_func_mapper *mapper = data; long *count = NULL; seq_printf(m, "%ps:", (void *)ip); @@ -6788,18 +6788,32 @@ ftrace_snapshot_print(struct seq_file *m, unsigned long ip, static int ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr, - unsigned long ip, void *data) + unsigned long ip, void *init_data, void **data) { - struct ftrace_func_mapper *mapper = ops->private_data; + struct ftrace_func_mapper *mapper = *data; + + if (!mapper) { + mapper = allocate_ftrace_func_mapper(); + if (!mapper) + return -ENOMEM; + *data = mapper; + } - return ftrace_func_mapper_add_ip(mapper, ip, data); + return ftrace_func_mapper_add_ip(mapper, ip, init_data); } static void ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr, - unsigned long ip, void **data) + unsigned long ip, void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; + struct ftrace_func_mapper *mapper = data; + + if (!ip) { + if (!mapper) + return; + free_ftrace_func_mapper(mapper, NULL); + return; + } ftrace_func_mapper_remove_ip(mapper, ip); } @@ -6842,12 +6856,6 @@ ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, if (!strlen(number)) goto out_reg; - if (!ops->private_data) { - ops->private_data = allocate_ftrace_func_mapper(); - if (!ops->private_data) - return -ENOMEM; - } - /* * We use the callback data field (which is a pointer) * as our counter. diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index e978ecd257b8..8f6754fba778 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -943,18 +943,18 @@ struct ftrace_probe_ops { unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **data); + void *data); int (*init)(struct ftrace_probe_ops *ops, struct trace_array *tr, - unsigned long ip, void *data); + unsigned long ip, void *init_data, + void **data); void (*free)(struct ftrace_probe_ops *ops, struct trace_array *tr, - unsigned long ip, void **data); + unsigned long ip, void *data); int (*print)(struct seq_file *m, unsigned long ip, struct ftrace_probe_ops *ops, void *data); - void *private_data; }; struct ftrace_func_mapper; diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 48c7f70cbac7..e7973e10398c 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2471,54 +2471,54 @@ static void update_event_probe(struct event_probe_data *data) static void event_enable_probe(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **_data) + void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; - struct event_probe_data *data; + struct ftrace_func_mapper *mapper = data; + struct event_probe_data *edata; void **pdata; pdata = ftrace_func_mapper_find_ip(mapper, ip); if (!pdata || !*pdata) return; - data = *pdata; - update_event_probe(data); + edata = *pdata; + update_event_probe(edata); } static void event_enable_count_probe(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **_data) + void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; - struct event_probe_data *data; + struct ftrace_func_mapper *mapper = data; + struct event_probe_data *edata; void **pdata; pdata = ftrace_func_mapper_find_ip(mapper, ip); if (!pdata || !*pdata) return; - data = *pdata; + edata = *pdata; - if (!data->count) + if (!edata->count) return; /* Skip if the event is in a state we want to switch to */ - if (data->enable == !(data->file->flags & EVENT_FILE_FL_SOFT_DISABLED)) + if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED)) return; - if (data->count != -1) - (data->count)--; + if (edata->count != -1) + (edata->count)--; - update_event_probe(data); + update_event_probe(edata); } static int event_enable_print(struct seq_file *m, unsigned long ip, - struct ftrace_probe_ops *ops, void *_data) + struct ftrace_probe_ops *ops, void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; - struct event_probe_data *data; + struct ftrace_func_mapper *mapper = data; + struct event_probe_data *edata; void **pdata; pdata = ftrace_func_mapper_find_ip(mapper, ip); @@ -2526,62 +2526,84 @@ event_enable_print(struct seq_file *m, unsigned long ip, if (WARN_ON_ONCE(!pdata || !*pdata)) return 0; - data = *pdata; + edata = *pdata; seq_printf(m, "%ps:", (void *)ip); seq_printf(m, "%s:%s:%s", - data->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR, - data->file->event_call->class->system, - trace_event_name(data->file->event_call)); + edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR, + edata->file->event_call->class->system, + trace_event_name(edata->file->event_call)); - if (data->count == -1) + if (edata->count == -1) seq_puts(m, ":unlimited\n"); else - seq_printf(m, ":count=%ld\n", data->count); + seq_printf(m, ":count=%ld\n", edata->count); return 0; } static int event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr, - unsigned long ip, void *_data) + unsigned long ip, void *init_data, void **data) { - struct ftrace_func_mapper *mapper = ops->private_data; - struct event_probe_data *data = _data; + struct ftrace_func_mapper *mapper = *data; + struct event_probe_data *edata = init_data; int ret; - ret = ftrace_func_mapper_add_ip(mapper, ip, data); + if (!mapper) { + mapper = allocate_ftrace_func_mapper(); + if (!mapper) + return -ENODEV; + *data = mapper; + } + + ret = ftrace_func_mapper_add_ip(mapper, ip, edata); if (ret < 0) return ret; - data->ref++; + edata->ref++; + + return 0; +} + +static int free_probe_data(void *data) +{ + struct event_probe_data *edata = data; + edata->ref--; + if (!edata->ref) { + /* Remove the SOFT_MODE flag */ + __ftrace_event_enable_disable(edata->file, 0, 1); + module_put(edata->file->event_call->mod); + kfree(edata); + } return 0; } static void event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr, - unsigned long ip, void **_data) + unsigned long ip, void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; - struct event_probe_data *data; + struct ftrace_func_mapper *mapper = data; + struct event_probe_data *edata; + + if (!ip) { + if (!mapper) + return; + free_ftrace_func_mapper(mapper, free_probe_data); + return; + } - data = ftrace_func_mapper_remove_ip(mapper, ip); + edata = ftrace_func_mapper_remove_ip(mapper, ip); - if (WARN_ON_ONCE(!data)) + if (WARN_ON_ONCE(!edata)) return; - if (WARN_ON_ONCE(data->ref <= 0)) + if (WARN_ON_ONCE(edata->ref <= 0)) return; - data->ref--; - if (!data->ref) { - /* Remove the SOFT_MODE flag */ - __ftrace_event_enable_disable(data->file, 0, 1); - module_put(data->file->event_call->mod); - kfree(data); - } + free_probe_data(edata); } static struct ftrace_probe_ops event_enable_probe_ops = { @@ -2659,12 +2681,6 @@ event_enable_func(struct trace_array *tr, struct ftrace_hash *hash, ret = -ENOMEM; - if (!ops->private_data) { - ops->private_data = allocate_ftrace_func_mapper(); - if (!ops->private_data) - goto out; - } - data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) goto out; diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index b95f56ba9744..7775e1ca5bad 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -268,9 +268,10 @@ static struct tracer function_trace __tracer_data = #ifdef CONFIG_DYNAMIC_FTRACE static void update_traceon_count(struct ftrace_probe_ops *ops, - unsigned long ip, bool on) + unsigned long ip, bool on, + void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; + struct ftrace_func_mapper *mapper = data; long *count; long old_count; @@ -329,23 +330,23 @@ static void update_traceon_count(struct ftrace_probe_ops *ops, static void ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **data) + void *data) { - update_traceon_count(ops, ip, 1); + update_traceon_count(ops, ip, 1, data); } static void ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **data) + void *data) { - update_traceon_count(ops, ip, 0); + update_traceon_count(ops, ip, 0, data); } static void ftrace_traceon(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **data) + void *data) { if (tracing_is_on()) return; @@ -356,7 +357,7 @@ ftrace_traceon(unsigned long ip, unsigned long parent_ip, static void ftrace_traceoff(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **data) + void *data) { if (!tracing_is_on()) return; @@ -376,7 +377,7 @@ ftrace_traceoff(unsigned long ip, unsigned long parent_ip, static void ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **data) + void *data) { trace_dump_stack(STACK_SKIP); } @@ -384,9 +385,9 @@ ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, static void ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **data) + void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; + struct ftrace_func_mapper *mapper = data; long *count; long old_count; long new_count; @@ -423,9 +424,10 @@ ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, } while (new_count != old_count); } -static int update_count(struct ftrace_probe_ops *ops, unsigned long ip) +static int update_count(struct ftrace_probe_ops *ops, unsigned long ip, + void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; + struct ftrace_func_mapper *mapper = data; long *count = NULL; if (mapper) @@ -443,9 +445,9 @@ static int update_count(struct ftrace_probe_ops *ops, unsigned long ip) static void ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **data) + void *data) { - if (update_count(ops, ip)) + if (update_count(ops, ip, data)) ftrace_dump(DUMP_ALL); } @@ -453,17 +455,18 @@ ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, static void ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, - void **data) + void *data) { - if (update_count(ops, ip)) + if (update_count(ops, ip, data)) ftrace_dump(DUMP_ORIG); } static int ftrace_probe_print(const char *name, struct seq_file *m, - unsigned long ip, struct ftrace_probe_ops *ops) + unsigned long ip, struct ftrace_probe_ops *ops, + void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; + struct ftrace_func_mapper *mapper = data; long *count = NULL; seq_printf(m, "%ps:%s", (void *)ip, name); @@ -484,52 +487,64 @@ ftrace_traceon_print(struct seq_file *m, unsigned long ip, struct ftrace_probe_ops *ops, void *data) { - return ftrace_probe_print("traceon", m, ip, ops); + return ftrace_probe_print("traceon", m, ip, ops, data); } static int ftrace_traceoff_print(struct seq_file *m, unsigned long ip, struct ftrace_probe_ops *ops, void *data) { - return ftrace_probe_print("traceoff", m, ip, ops); + return ftrace_probe_print("traceoff", m, ip, ops, data); } static int ftrace_stacktrace_print(struct seq_file *m, unsigned long ip, struct ftrace_probe_ops *ops, void *data) { - return ftrace_probe_print("stacktrace", m, ip, ops); + return ftrace_probe_print("stacktrace", m, ip, ops, data); } static int ftrace_dump_print(struct seq_file *m, unsigned long ip, struct ftrace_probe_ops *ops, void *data) { - return ftrace_probe_print("dump", m, ip, ops); + return ftrace_probe_print("dump", m, ip, ops, data); } static int ftrace_cpudump_print(struct seq_file *m, unsigned long ip, struct ftrace_probe_ops *ops, void *data) { - return ftrace_probe_print("cpudump", m, ip, ops); + return ftrace_probe_print("cpudump", m, ip, ops, data); } static int ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr, - unsigned long ip, void *data) + unsigned long ip, void *init_data, void **data) { - struct ftrace_func_mapper *mapper = ops->private_data; + struct ftrace_func_mapper *mapper = *data; + + if (!mapper) { + mapper = allocate_ftrace_func_mapper(); + if (!mapper) + return -ENOMEM; + *data = mapper; + } - return ftrace_func_mapper_add_ip(mapper, ip, data); + return ftrace_func_mapper_add_ip(mapper, ip, init_data); } static void ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr, - unsigned long ip, void **_data) + unsigned long ip, void *data) { - struct ftrace_func_mapper *mapper = ops->private_data; + struct ftrace_func_mapper *mapper = data; + + if (!ip) { + free_ftrace_func_mapper(mapper, NULL); + return; + } ftrace_func_mapper_remove_ip(mapper, ip); } @@ -607,12 +622,6 @@ ftrace_trace_probe_callback(struct trace_array *tr, if (!strlen(number)) goto out_reg; - if (!ops->private_data) { - ops->private_data = allocate_ftrace_func_mapper(); - if (!ops->private_data) - return -ENOMEM; - } - /* * We use the callback data field (which is a pointer) * as our counter. -- cgit v1.2.3 From cab5037950821caa1301df0223de657c6ee202a8 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Thu, 20 Apr 2017 11:34:06 -0400 Subject: tracing/ftrace: Enable snapshot function trigger to work with instances Modify the snapshot probe trigger to work with instances. This way the snapshot function trigger will only affect the instance that it is added to in the set_ftrace_filter file. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/trace.c | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 18256cd7ad2c..57e9c546bebb 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -894,23 +894,8 @@ int __trace_bputs(unsigned long ip, const char *str) EXPORT_SYMBOL_GPL(__trace_bputs); #ifdef CONFIG_TRACER_SNAPSHOT -/** - * trace_snapshot - take a snapshot of the current buffer. - * - * This causes a swap between the snapshot buffer and the current live - * tracing buffer. You can use this to take snapshots of the live - * trace when some condition is triggered, but continue to trace. - * - * Note, make sure to allocate the snapshot with either - * a tracing_snapshot_alloc(), or by doing it manually - * with: echo 1 > /sys/kernel/debug/tracing/snapshot - * - * If the snapshot buffer is not allocated, it will stop tracing. - * Basically making a permanent snapshot. - */ -void tracing_snapshot(void) +static void tracing_snapshot_instance(struct trace_array *tr) { - struct trace_array *tr = &global_trace; struct tracer *tracer = tr->current_trace; unsigned long flags; @@ -938,6 +923,27 @@ void tracing_snapshot(void) update_max_tr(tr, current, smp_processor_id()); local_irq_restore(flags); } + +/** + * trace_snapshot - take a snapshot of the current buffer. + * + * This causes a swap between the snapshot buffer and the current live + * tracing buffer. You can use this to take snapshots of the live + * trace when some condition is triggered, but continue to trace. + * + * Note, make sure to allocate the snapshot with either + * a tracing_snapshot_alloc(), or by doing it manually + * with: echo 1 > /sys/kernel/debug/tracing/snapshot + * + * If the snapshot buffer is not allocated, it will stop tracing. + * Basically making a permanent snapshot. + */ +void tracing_snapshot(void) +{ + struct trace_array *tr = &global_trace; + + tracing_snapshot_instance(tr); +} EXPORT_SYMBOL_GPL(tracing_snapshot); static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf, @@ -6739,7 +6745,7 @@ ftrace_snapshot(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, void *data) { - tracing_snapshot(); + tracing_snapshot_instance(tr); } static void @@ -6761,7 +6767,7 @@ ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, (*count)--; } - tracing_snapshot(); + tracing_snapshot_instance(tr); } static int @@ -6868,7 +6874,7 @@ ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, ret = register_ftrace_function_probe(glob, tr, ops, count); if (ret >= 0) - alloc_snapshot(&global_trace); + alloc_snapshot(tr); return ret < 0 ? ret : 0; } -- cgit v1.2.3 From 2290f2c589285d0031e3b7445afff8949f3fdbb6 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Thu, 20 Apr 2017 11:46:03 -0400 Subject: tracing/ftrace: Allow for the traceonoff probe be unique to instances Have the traceon/off function probe triggers affect only the instance they are set in. This required making the trace_on/off accessible for other files in the tracing directory. Signed-off-by: Steven Rostedt (VMware) --- kernel/trace/trace.c | 4 ++-- kernel/trace/trace.h | 2 ++ kernel/trace/trace_functions.c | 21 +++++++++++---------- 3 files changed, 15 insertions(+), 12 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 57e9c546bebb..60c904fa5480 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -757,7 +757,7 @@ __trace_buffer_lock_reserve(struct ring_buffer *buffer, return event; } -static void tracer_tracing_on(struct trace_array *tr) +void tracer_tracing_on(struct trace_array *tr) { if (tr->trace_buffer.buffer) ring_buffer_record_on(tr->trace_buffer.buffer); @@ -1045,7 +1045,7 @@ void tracing_snapshot_alloc(void) EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); #endif /* CONFIG_TRACER_SNAPSHOT */ -static void tracer_tracing_off(struct trace_array *tr) +void tracer_tracing_off(struct trace_array *tr) { if (tr->trace_buffer.buffer) ring_buffer_record_off(tr->trace_buffer.buffer); diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 8f6754fba778..bc011c1f3d71 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -582,6 +582,8 @@ void tracing_reset_all_online_cpus(void); int tracing_open_generic(struct inode *inode, struct file *filp); bool tracing_is_disabled(void); int tracer_tracing_is_on(struct trace_array *tr); +void tracer_tracing_on(struct trace_array *tr); +void tracer_tracing_off(struct trace_array *tr); struct dentry *trace_create_file(const char *name, umode_t mode, struct dentry *parent, diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 7775e1ca5bad..8c30ca733a5c 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -268,7 +268,8 @@ static struct tracer function_trace __tracer_data = #ifdef CONFIG_DYNAMIC_FTRACE static void update_traceon_count(struct ftrace_probe_ops *ops, - unsigned long ip, bool on, + unsigned long ip, + struct trace_array *tr, bool on, void *data) { struct ftrace_func_mapper *mapper = data; @@ -313,13 +314,13 @@ static void update_traceon_count(struct ftrace_probe_ops *ops, /* Make sure we see count before checking tracing state */ smp_rmb(); - if (on == !!tracing_is_on()) + if (on == !!tracer_tracing_is_on(tr)) return; if (on) - tracing_on(); + tracer_tracing_on(tr); else - tracing_off(); + tracer_tracing_off(tr); /* Make sure tracing state is visible before updating count */ smp_wmb(); @@ -332,7 +333,7 @@ ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, void *data) { - update_traceon_count(ops, ip, 1, data); + update_traceon_count(ops, ip, tr, 1, data); } static void @@ -340,7 +341,7 @@ ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, void *data) { - update_traceon_count(ops, ip, 0, data); + update_traceon_count(ops, ip, tr, 0, data); } static void @@ -348,10 +349,10 @@ ftrace_traceon(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, void *data) { - if (tracing_is_on()) + if (tracer_tracing_is_on(tr)) return; - tracing_on(); + tracer_tracing_on(tr); } static void @@ -359,10 +360,10 @@ ftrace_traceoff(unsigned long ip, unsigned long parent_ip, struct trace_array *tr, struct ftrace_probe_ops *ops, void *data) { - if (!tracing_is_on()) + if (!tracer_tracing_is_on(tr)) return; - tracing_off(); + tracer_tracing_off(tr); } /* -- cgit v1.2.3 From 73a757e63114dfd765f1c5d1ff7e994f123d0234 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Mon, 1 May 2017 09:35:09 -0400 Subject: ring-buffer: Return reader page back into existing ring buffer When reading the ring buffer for consuming, it is optimized for splice, where a page is taken out of the ring buffer (zero copy) and sent to the reading consumer. When the read is finished with the page, it calls ring_buffer_free_read_page(), which simply frees the page. The next time the reader needs to get a page from the ring buffer, it must call ring_buffer_alloc_read_page() which allocates and initializes a reader page for the ring buffer to be swapped into the ring buffer for a new filled page for the reader. The problem is that there's no reason to actually free the page when it is passed back to the ring buffer. It can hold it off and reuse it for the next iteration. This completely removes the interaction with the page_alloc mechanism. Using the trace-cmd utility to record all events (causing trace-cmd to require reading lots of pages from the ring buffer, and calling ring_buffer_alloc/free_read_page() several times), and also assigning a stack trace trigger to the mm_page_alloc event, we can see how many times the ring_buffer_alloc_read_page() needed to allocate a page for the ring buffer. Before this change: # trace-cmd record -e all -e mem_page_alloc -R stacktrace sleep 1 # trace-cmd report |grep ring_buffer_alloc_read_page | wc -l 9968 After this change: # trace-cmd record -e all -e mem_page_alloc -R stacktrace sleep 1 # trace-cmd report |grep ring_buffer_alloc_read_page | wc -l 4 Signed-off-by: Steven Rostedt (VMware) --- include/linux/ring_buffer.h | 2 +- kernel/trace/ring_buffer.c | 40 +++++++++++++++++++++++++++++++++--- kernel/trace/ring_buffer_benchmark.c | 2 +- kernel/trace/trace.c | 17 ++++++++++----- 4 files changed, 51 insertions(+), 10 deletions(-) (limited to 'kernel/trace/trace.c') diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index b6d4568795a7..ee9b461af095 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -185,7 +185,7 @@ size_t ring_buffer_page_len(void *page); void *ring_buffer_alloc_read_page(struct ring_buffer *buffer, int cpu); -void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data); +void ring_buffer_free_read_page(struct ring_buffer *buffer, int cpu, void *data); int ring_buffer_read_page(struct ring_buffer *buffer, void **data_page, size_t len, int cpu, int full); diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 96fc3c043ad6..01b4ee5326cf 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -438,6 +438,7 @@ struct ring_buffer_per_cpu { raw_spinlock_t reader_lock; /* serialize readers */ arch_spinlock_t lock; struct lock_class_key lock_key; + struct buffer_data_page *free_page; unsigned long nr_pages; unsigned int current_context; struct list_head *pages; @@ -4377,9 +4378,25 @@ EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu); */ void *ring_buffer_alloc_read_page(struct ring_buffer *buffer, int cpu) { - struct buffer_data_page *bpage; + struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; + struct buffer_data_page *bpage = NULL; + unsigned long flags; struct page *page; + local_irq_save(flags); + arch_spin_lock(&cpu_buffer->lock); + + if (cpu_buffer->free_page) { + bpage = cpu_buffer->free_page; + cpu_buffer->free_page = NULL; + } + + arch_spin_unlock(&cpu_buffer->lock); + local_irq_restore(flags); + + if (bpage) + goto out; + page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL | __GFP_NORETRY, 0); if (!page) @@ -4387,6 +4404,7 @@ void *ring_buffer_alloc_read_page(struct ring_buffer *buffer, int cpu) bpage = page_address(page); + out: rb_init_page(bpage); return bpage; @@ -4396,13 +4414,29 @@ EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); /** * ring_buffer_free_read_page - free an allocated read page * @buffer: the buffer the page was allocate for + * @cpu: the cpu buffer the page came from * @data: the page to free * * Free a page allocated from ring_buffer_alloc_read_page. */ -void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data) +void ring_buffer_free_read_page(struct ring_buffer *buffer, int cpu, void *data) { - free_page((unsigned long)data); + struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; + struct buffer_data_page *bpage = data; + unsigned long flags; + + local_irq_save(flags); + arch_spin_lock(&cpu_buffer->lock); + + if (!cpu_buffer->free_page) { + cpu_buffer->free_page = bpage; + bpage = NULL; + } + + arch_spin_unlock(&cpu_buffer->lock); + local_irq_restore(flags); + + free_page((unsigned long)bpage); } EXPORT_SYMBOL_GPL(ring_buffer_free_read_page); diff --git a/kernel/trace/ring_buffer_benchmark.c b/kernel/trace/ring_buffer_benchmark.c index c190a4d5013c..9fbcaf567886 100644 --- a/kernel/trace/ring_buffer_benchmark.c +++ b/kernel/trace/ring_buffer_benchmark.c @@ -171,7 +171,7 @@ static enum event_status read_page(int cpu) } } } - ring_buffer_free_read_page(buffer, bpage); + ring_buffer_free_read_page(buffer, cpu, bpage); if (ret < 0) return EVENT_DROPPED; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 60c904fa5480..5b645b0fbbb8 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -6054,6 +6054,7 @@ static int tracing_clock_open(struct inode *inode, struct file *file) struct ftrace_buffer_info { struct trace_iterator iter; void *spare; + unsigned int spare_cpu; unsigned int read; }; @@ -6383,9 +6384,11 @@ tracing_buffers_read(struct file *filp, char __user *ubuf, return -EBUSY; #endif - if (!info->spare) + if (!info->spare) { info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer, iter->cpu_file); + info->spare_cpu = iter->cpu_file; + } if (!info->spare) return -ENOMEM; @@ -6445,7 +6448,8 @@ static int tracing_buffers_release(struct inode *inode, struct file *file) __trace_array_put(iter->tr); if (info->spare) - ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare); + ring_buffer_free_read_page(iter->trace_buffer->buffer, + info->spare_cpu, info->spare); kfree(info); mutex_unlock(&trace_types_lock); @@ -6456,6 +6460,7 @@ static int tracing_buffers_release(struct inode *inode, struct file *file) struct buffer_ref { struct ring_buffer *buffer; void *page; + int cpu; int ref; }; @@ -6467,7 +6472,7 @@ static void buffer_pipe_buf_release(struct pipe_inode_info *pipe, if (--ref->ref) return; - ring_buffer_free_read_page(ref->buffer, ref->page); + ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); kfree(ref); buf->private = 0; } @@ -6501,7 +6506,7 @@ static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i) if (--ref->ref) return; - ring_buffer_free_read_page(ref->buffer, ref->page); + ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); kfree(ref); spd->partial[i].private = 0; } @@ -6566,11 +6571,13 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, kfree(ref); break; } + ref->cpu = iter->cpu_file; r = ring_buffer_read_page(ref->buffer, &ref->page, len, iter->cpu_file, 1); if (r < 0) { - ring_buffer_free_read_page(ref->buffer, ref->page); + ring_buffer_free_read_page(ref->buffer, ref->cpu, + ref->page); kfree(ref); break; } -- cgit v1.2.3