diff options
Diffstat (limited to 'include/trace/ftrace.h')
| -rw-r--r-- | include/trace/ftrace.h | 507 |
1 files changed, 164 insertions, 343 deletions
diff --git a/include/trace/ftrace.h b/include/trace/ftrace.h index c6fe03e902c..5a64905d727 100644 --- a/include/trace/ftrace.h +++ b/include/trace/ftrace.h @@ -62,10 +62,14 @@ struct trace_entry ent; \ tstruct \ char __data[0]; \ - }; + }; \ + \ + static struct ftrace_event_class event_class_##name; + #undef DEFINE_EVENT #define DEFINE_EVENT(template, name, proto, args) \ - static struct ftrace_event_call event_##name + static struct ftrace_event_call __used \ + __attribute__((__aligned__(4))) event_##name #undef DEFINE_EVENT_PRINT #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \ @@ -131,130 +135,6 @@ #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) /* - * Setup the showing format of trace point. - * - * int - * ftrace_format_##call(struct trace_seq *s) - * { - * struct ftrace_raw_##call field; - * int ret; - * - * ret = trace_seq_printf(s, #type " " #item ";" - * " offset:%u; size:%u;\n", - * offsetof(struct ftrace_raw_##call, item), - * sizeof(field.type)); - * - * } - */ - -#undef TP_STRUCT__entry -#define TP_STRUCT__entry(args...) args - -#undef __field -#define __field(type, item) \ - ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \ - "offset:%u;\tsize:%u;\tsigned:%u;\n", \ - (unsigned int)offsetof(typeof(field), item), \ - (unsigned int)sizeof(field.item), \ - (unsigned int)is_signed_type(type)); \ - if (!ret) \ - return 0; - -#undef __field_ext -#define __field_ext(type, item, filter_type) __field(type, item) - -#undef __array -#define __array(type, item, len) \ - ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \ - "offset:%u;\tsize:%u;\tsigned:%u;\n", \ - (unsigned int)offsetof(typeof(field), item), \ - (unsigned int)sizeof(field.item), \ - (unsigned int)is_signed_type(type)); \ - if (!ret) \ - return 0; - -#undef __dynamic_array -#define __dynamic_array(type, item, len) \ - ret = trace_seq_printf(s, "\tfield:__data_loc " #type "[] " #item ";\t"\ - "offset:%u;\tsize:%u;\tsigned:%u;\n", \ - (unsigned int)offsetof(typeof(field), \ - __data_loc_##item), \ - (unsigned int)sizeof(field.__data_loc_##item), \ - (unsigned int)is_signed_type(type)); \ - if (!ret) \ - return 0; - -#undef __string -#define __string(item, src) __dynamic_array(char, item, -1) - -#undef __entry -#define __entry REC - -#undef __print_symbolic -#undef __get_dynamic_array -#undef __get_str - -#undef TP_printk -#define TP_printk(fmt, args...) "\"%s\", %s\n", fmt, __stringify(args) - -#undef TP_fast_assign -#define TP_fast_assign(args...) args - -#undef TP_perf_assign -#define TP_perf_assign(args...) - -#undef DECLARE_EVENT_CLASS -#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \ -static int \ -ftrace_format_setup_##call(struct ftrace_event_call *unused, \ - struct trace_seq *s) \ -{ \ - struct ftrace_raw_##call field __attribute__((unused)); \ - int ret = 0; \ - \ - tstruct; \ - \ - return ret; \ -} \ - \ -static int \ -ftrace_format_##call(struct ftrace_event_call *unused, \ - struct trace_seq *s) \ -{ \ - int ret = 0; \ - \ - ret = ftrace_format_setup_##call(unused, s); \ - if (!ret) \ - return ret; \ - \ - ret = trace_seq_printf(s, "\nprint fmt: " print); \ - \ - return ret; \ -} - -#undef DEFINE_EVENT -#define DEFINE_EVENT(template, name, proto, args) - -#undef DEFINE_EVENT_PRINT -#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \ -static int \ -ftrace_format_##name(struct ftrace_event_call *unused, \ - struct trace_seq *s) \ -{ \ - int ret = 0; \ - \ - ret = ftrace_format_setup_##template(unused, s); \ - if (!ret) \ - return ret; \ - \ - trace_seq_printf(s, "\nprint fmt: " print); \ - \ - return ret; \ -} - -#include TRACE_INCLUDE(TRACE_INCLUDE_FILE) - -/* * Stage 3 of the trace events. * * Override the macros in <trace/trace_events.h> to include the following: @@ -270,16 +150,18 @@ ftrace_format_##name(struct ftrace_event_call *unused, \ * * entry = iter->ent; * - * if (entry->type != event_<call>.id) { + * if (entry->type != event_<call>->event.type) { * WARN_ON_ONCE(1); * return TRACE_TYPE_UNHANDLED; * } * * field = (typeof(field))entry; * - * p = get_cpu_var(ftrace_event_seq); + * p = &get_cpu_var(ftrace_event_seq); * trace_seq_init(p); - * ret = trace_seq_printf(s, <TP_printk> "\n"); + * ret = trace_seq_printf(s, "%s: ", <call>); + * if (ret) + * ret = trace_seq_printf(s, <TP_printk> "\n"); * put_cpu(); * if (!ret) * return TRACE_TYPE_PARTIAL_LINE; @@ -321,21 +203,28 @@ ftrace_format_##name(struct ftrace_event_call *unused, \ ftrace_print_symbols_seq(p, value, symbols); \ }) +#undef __print_hex +#define __print_hex(buf, buf_len) ftrace_print_hex_seq(p, buf, buf_len) + #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ -static enum print_line_t \ -ftrace_raw_output_id_##call(int event_id, const char *name, \ - struct trace_iterator *iter, int flags) \ +static notrace enum print_line_t \ +ftrace_raw_output_##call(struct trace_iterator *iter, int flags, \ + struct trace_event *trace_event) \ { \ + struct ftrace_event_call *event; \ struct trace_seq *s = &iter->seq; \ struct ftrace_raw_##call *field; \ struct trace_entry *entry; \ struct trace_seq *p; \ int ret; \ \ + event = container_of(trace_event, struct ftrace_event_call, \ + event); \ + \ entry = iter->ent; \ \ - if (entry->type != event_id) { \ + if (entry->type != event->event.type) { \ WARN_ON_ONCE(1); \ return TRACE_TYPE_UNHANDLED; \ } \ @@ -344,7 +233,7 @@ ftrace_raw_output_id_##call(int event_id, const char *name, \ \ p = &get_cpu_var(ftrace_event_seq); \ trace_seq_init(p); \ - ret = trace_seq_printf(s, "%s: ", name); \ + ret = trace_seq_printf(s, "%s: ", event->name); \ if (ret) \ ret = trace_seq_printf(s, print); \ put_cpu(); \ @@ -352,21 +241,16 @@ ftrace_raw_output_id_##call(int event_id, const char *name, \ return TRACE_TYPE_PARTIAL_LINE; \ \ return TRACE_TYPE_HANDLED; \ -} - -#undef DEFINE_EVENT -#define DEFINE_EVENT(template, name, proto, args) \ -static enum print_line_t \ -ftrace_raw_output_##name(struct trace_iterator *iter, int flags) \ -{ \ - return ftrace_raw_output_id_##template(event_##name.id, \ - #name, iter, flags); \ -} +} \ +static struct trace_event_functions ftrace_event_type_funcs_##call = { \ + .trace = ftrace_raw_output_##call, \ +}; #undef DEFINE_EVENT_PRINT #define DEFINE_EVENT_PRINT(template, call, proto, args, print) \ -static enum print_line_t \ -ftrace_raw_output_##call(struct trace_iterator *iter, int flags) \ +static notrace enum print_line_t \ +ftrace_raw_output_##call(struct trace_iterator *iter, int flags, \ + struct trace_event *event) \ { \ struct trace_seq *s = &iter->seq; \ struct ftrace_raw_##template *field; \ @@ -376,7 +260,7 @@ ftrace_raw_output_##call(struct trace_iterator *iter, int flags) \ \ entry = iter->ent; \ \ - if (entry->type != event_##call.id) { \ + if (entry->type != event_##call.event.type) { \ WARN_ON_ONCE(1); \ return TRACE_TYPE_UNHANDLED; \ } \ @@ -393,7 +277,10 @@ ftrace_raw_output_##call(struct trace_iterator *iter, int flags) \ return TRACE_TYPE_PARTIAL_LINE; \ \ return TRACE_TYPE_HANDLED; \ -} +} \ +static struct trace_event_functions ftrace_event_type_funcs_##call = { \ + .trace = ftrace_raw_output_##call, \ +}; #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) @@ -431,7 +318,7 @@ ftrace_raw_output_##call(struct trace_iterator *iter, int flags) \ #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \ -static int \ +static int notrace \ ftrace_define_fields_##call(struct ftrace_event_call *event_call) \ { \ struct ftrace_raw_##call field; \ @@ -479,7 +366,7 @@ ftrace_define_fields_##call(struct ftrace_event_call *event_call) \ #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ -static inline int ftrace_get_offsets_##call( \ +static inline notrace int ftrace_get_offsets_##call( \ struct ftrace_data_offsets_##call *__data_offsets, proto) \ { \ int __data_size = 0; \ @@ -499,143 +386,85 @@ static inline int ftrace_get_offsets_##call( \ #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) -#ifdef CONFIG_EVENT_PROFILE - -/* - * Generate the functions needed for tracepoint perf_event support. - * - * NOTE: The insertion profile callback (ftrace_profile_<call>) is defined later - * - * static int ftrace_profile_enable_<call>(void) - * { - * return register_trace_<call>(ftrace_profile_<call>); - * } - * - * static void ftrace_profile_disable_<call>(void) - * { - * unregister_trace_<call>(ftrace_profile_<call>); - * } - * - */ - -#undef DECLARE_EVENT_CLASS -#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) - -#undef DEFINE_EVENT -#define DEFINE_EVENT(template, name, proto, args) \ - \ -static void ftrace_profile_##name(proto); \ - \ -static int ftrace_profile_enable_##name(struct ftrace_event_call *unused)\ -{ \ - return register_trace_##name(ftrace_profile_##name); \ -} \ - \ -static void ftrace_profile_disable_##name(struct ftrace_event_call *unused)\ -{ \ - unregister_trace_##name(ftrace_profile_##name); \ -} - -#undef DEFINE_EVENT_PRINT -#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \ - DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args)) - -#include TRACE_INCLUDE(TRACE_INCLUDE_FILE) - -#endif - /* * Stage 4 of the trace events. * * Override the macros in <trace/trace_events.h> to include the following: * - * static void ftrace_event_<call>(proto) - * { - * event_trace_printk(_RET_IP_, "<call>: " <fmt>); - * } - * - * static int ftrace_reg_event_<call>(struct ftrace_event_call *unused) - * { - * return register_trace_<call>(ftrace_event_<call>); - * } - * - * static void ftrace_unreg_event_<call>(struct ftrace_event_call *unused) - * { - * unregister_trace_<call>(ftrace_event_<call>); - * } - * - * * For those macros defined with TRACE_EVENT: * * static struct ftrace_event_call event_<call>; * - * static void ftrace_raw_event_<call>(proto) + * static void ftrace_raw_event_<call>(void *__data, proto) * { + * struct ftrace_event_call *event_call = __data; + * struct ftrace_data_offsets_<call> __maybe_unused __data_offsets; * struct ring_buffer_event *event; * struct ftrace_raw_<call> *entry; <-- defined in stage 1 * struct ring_buffer *buffer; * unsigned long irq_flags; + * int __data_size; * int pc; * * local_save_flags(irq_flags); * pc = preempt_count(); * + * __data_size = ftrace_get_offsets_<call>(&__data_offsets, args); + * * event = trace_current_buffer_lock_reserve(&buffer, - * event_<call>.id, - * sizeof(struct ftrace_raw_<call>), + * event_<call>->event.type, + * sizeof(*entry) + __data_size, * irq_flags, pc); * if (!event) * return; * entry = ring_buffer_event_data(event); * - * <assign>; <-- Here we assign the entries by the __field and - * __array macros. - * - * trace_current_buffer_unlock_commit(buffer, event, irq_flags, pc); - * } - * - * static int ftrace_raw_reg_event_<call>(struct ftrace_event_call *unused) - * { - * int ret; - * - * ret = register_trace_<call>(ftrace_raw_event_<call>); - * if (!ret) - * pr_info("event trace: Could not activate trace point " - * "probe to <call>"); - * return ret; - * } + * { <assign>; } <-- Here we assign the entries by the __field and + * __array macros. * - * static void ftrace_unreg_event_<call>(struct ftrace_event_call *unused) - * { - * unregister_trace_<call>(ftrace_raw_event_<call>); + * if (!filter_current_check_discard(buffer, event_call, entry, event)) + * trace_current_buffer_unlock_commit(buffer, + * event, irq_flags, pc); * } * * static struct trace_event ftrace_event_type_<call> = { * .trace = ftrace_raw_output_<call>, <-- stage 2 * }; * + * static const char print_fmt_<call>[] = <TP_printk>; + * + * static struct ftrace_event_class __used event_class_<template> = { + * .system = "<system>", + * .define_fields = ftrace_define_fields_<call>, + * .fields = LIST_HEAD_INIT(event_class_##call.fields), + * .raw_init = trace_event_raw_init, + * .probe = ftrace_raw_event_##call, + * }; + * * static struct ftrace_event_call __used * __attribute__((__aligned__(4))) * __attribute__((section("_ftrace_events"))) event_<call> = { * .name = "<call>", - * .system = "<system>", - * .raw_init = trace_event_raw_init, - * .regfunc = ftrace_reg_event_<call>, - * .unregfunc = ftrace_unreg_event_<call>, - * .show_format = ftrace_format_<call>, - * } + * .class = event_class_<template>, + * .event = &ftrace_event_type_<call>, + * .print_fmt = print_fmt_<call>, + * }; * */ -#ifdef CONFIG_EVENT_PROFILE +#ifdef CONFIG_PERF_EVENTS + +#define _TRACE_PERF_PROTO(call, proto) \ + static notrace void \ + perf_trace_##call(void *__data, proto); -#define _TRACE_PROFILE_INIT(call) \ - .profile_enable = ftrace_profile_enable_##call, \ - .profile_disable = ftrace_profile_disable_##call, +#define _TRACE_PERF_INIT(call) \ + .perf_probe = perf_trace_##call, #else -#define _TRACE_PROFILE_INIT(call) -#endif +#define _TRACE_PERF_PROTO(call, proto) +#define _TRACE_PERF_INIT(call) +#endif /* CONFIG_PERF_EVENTS */ #undef __entry #define __entry entry @@ -657,12 +486,19 @@ static void ftrace_profile_disable_##name(struct ftrace_event_call *unused)\ #define __assign_str(dst, src) \ strcpy(__get_str(dst), src); +#undef TP_fast_assign +#define TP_fast_assign(args...) args + +#undef TP_perf_assign +#define TP_perf_assign(args...) + #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ \ -static void ftrace_raw_event_id_##call(struct ftrace_event_call *event_call, \ - proto) \ +static notrace void \ +ftrace_raw_event_##call(void *__data, proto) \ { \ + struct ftrace_event_call *event_call = __data; \ struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\ struct ring_buffer_event *event; \ struct ftrace_raw_##call *entry; \ @@ -677,14 +513,13 @@ static void ftrace_raw_event_id_##call(struct ftrace_event_call *event_call, \ __data_size = ftrace_get_offsets_##call(&__data_offsets, args); \ \ event = trace_current_buffer_lock_reserve(&buffer, \ - event_call->id, \ + event_call->event.type, \ sizeof(*entry) + __data_size, \ irq_flags, pc); \ if (!event) \ return; \ entry = ring_buffer_event_data(event); \ \ - \ tstruct \ \ { assign; } \ @@ -693,37 +528,47 @@ static void ftrace_raw_event_id_##call(struct ftrace_event_call *event_call, \ trace_nowake_buffer_unlock_commit(buffer, \ event, irq_flags, pc); \ } +/* + * The ftrace_test_probe is compiled out, it is only here as a build time check + * to make sure that if the tracepoint handling changes, the ftrace probe will + * fail to compile unless it too is updated. + */ #undef DEFINE_EVENT #define DEFINE_EVENT(template, call, proto, args) \ - \ -static void ftrace_raw_event_##call(proto) \ +static inline void ftrace_test_probe_##call(void) \ { \ - ftrace_raw_event_id_##template(&event_##call, args); \ -} \ - \ -static int ftrace_raw_reg_event_##call(struct ftrace_event_call *unused)\ -{ \ - return register_trace_##call(ftrace_raw_event_##call); \ -} \ - \ -static void ftrace_raw_unreg_event_##call(struct ftrace_event_call *unused)\ -{ \ - unregister_trace_##call(ftrace_raw_event_##call); \ -} \ - \ -static struct trace_event ftrace_event_type_##call = { \ - .trace = ftrace_raw_output_##call, \ -}; + check_trace_callback_type_##call(ftrace_raw_event_##template); \ +} #undef DEFINE_EVENT_PRINT -#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \ - DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args)) +#define DEFINE_EVENT_PRINT(template, name, proto, args, print) #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) +#undef __entry +#define __entry REC + +#undef __print_flags +#undef __print_symbolic +#undef __get_dynamic_array +#undef __get_str + +#undef TP_printk +#define TP_printk(fmt, args...) "\"" fmt "\", " __stringify(args) + #undef DECLARE_EVENT_CLASS -#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) +#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ +_TRACE_PERF_PROTO(call, PARAMS(proto)); \ +static const char print_fmt_##call[] = print; \ +static struct ftrace_event_class __used event_class_##call = { \ + .system = __stringify(TRACE_SYSTEM), \ + .define_fields = ftrace_define_fields_##call, \ + .fields = LIST_HEAD_INIT(event_class_##call.fields),\ + .raw_init = trace_event_raw_init, \ + .probe = ftrace_raw_event_##call, \ + _TRACE_PERF_INIT(call) \ +}; #undef DEFINE_EVENT #define DEFINE_EVENT(template, call, proto, args) \ @@ -732,42 +577,34 @@ static struct ftrace_event_call __used \ __attribute__((__aligned__(4))) \ __attribute__((section("_ftrace_events"))) event_##call = { \ .name = #call, \ - .system = __stringify(TRACE_SYSTEM), \ - .event = &ftrace_event_type_##call, \ - .raw_init = trace_event_raw_init, \ - .regfunc = ftrace_raw_reg_event_##call, \ - .unregfunc = ftrace_raw_unreg_event_##call, \ - .show_format = ftrace_format_##template, \ - .define_fields = ftrace_define_fields_##template, \ - _TRACE_PROFILE_INIT(call) \ -} + .class = &event_class_##template, \ + .event.funcs = &ftrace_event_type_funcs_##template, \ + .print_fmt = print_fmt_##template, \ +}; #undef DEFINE_EVENT_PRINT #define DEFINE_EVENT_PRINT(template, call, proto, args, print) \ \ +static const char print_fmt_##call[] = print; \ + \ static struct ftrace_event_call __used \ __attribute__((__aligned__(4))) \ __attribute__((section("_ftrace_events"))) event_##call = { \ .name = #call, \ - .system = __stringify(TRACE_SYSTEM), \ - .event = &ftrace_event_type_##call, \ - .raw_init = trace_event_raw_init, \ - .regfunc = ftrace_raw_reg_event_##call, \ - .unregfunc = ftrace_raw_unreg_event_##call, \ - .show_format = ftrace_format_##call, \ - .define_fields = ftrace_define_fields_##template, \ - _TRACE_PROFILE_INIT(call) \ + .class = &event_class_##template, \ + .event.funcs = &ftrace_event_type_funcs_##call, \ + .print_fmt = print_fmt_##call, \ } #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) /* - * Define the insertion callback to profile events + * Define the insertion callback to perf events * * The job is very similar to ftrace_raw_event_<call> except that we don't * insert in the ring buffer but in a perf counter. * - * static void ftrace_profile_<call>(proto) + * static void ftrace_perf_<call>(proto) * { * struct ftrace_data_offsets_<call> __maybe_unused __data_offsets; * struct ftrace_event_call *event_call = &event_<call>; @@ -798,9 +635,9 @@ __attribute__((section("_ftrace_events"))) event_##call = { \ * __cpu = smp_processor_id(); * * if (in_nmi()) - * trace_buf = rcu_dereference(perf_trace_buf_nmi); + * trace_buf = rcu_dereference_sched(perf_trace_buf_nmi); * else - * trace_buf = rcu_dereference(perf_trace_buf); + * trace_buf = rcu_dereference_sched(perf_trace_buf); * * if (!trace_buf) * goto end; @@ -835,7 +672,17 @@ __attribute__((section("_ftrace_events"))) event_##call = { \ * } */ -#ifdef CONFIG_EVENT_PROFILE +#ifdef CONFIG_PERF_EVENTS + +#undef __entry +#define __entry entry + +#undef __get_dynamic_array +#define __get_dynamic_array(field) \ + ((void *)__entry + (__entry->__data_loc_##field & 0xffff)) + +#undef __get_str +#define __get_str(field) (char *)__get_dynamic_array(field) #undef __perf_addr #define __perf_addr(a) __addr = (a) @@ -845,89 +692,63 @@ __attribute__((section("_ftrace_events"))) event_##call = { \ #undef DECLARE_EVENT_CLASS #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ -static void \ -ftrace_profile_templ_##call(struct ftrace_event_call *event_call, \ - proto) \ +static notrace void \ +perf_trace_##call(void *__data, proto) \ { \ + struct ftrace_event_call *event_call = __data; \ struct ftrace_data_offsets_##call __maybe_unused __data_offsets;\ - extern int perf_swevent_get_recursion_context(void); \ - extern void perf_swevent_put_recursion_context(int rctx); \ - extern void perf_tp_event(int, u64, u64, void *, int); \ struct ftrace_raw_##call *entry; \ + struct pt_regs __regs; \ u64 __addr = 0, __count = 1; \ - unsigned long irq_flags; \ - struct trace_entry *ent; \ + struct hlist_head *head; \ int __entry_size; \ int __data_size; \ - char *trace_buf; \ - char *raw_data; \ - int __cpu; \ int rctx; \ - int pc; \ \ - pc = preempt_count(); \ + perf_fetch_caller_regs(&__regs, 1); \ \ __data_size = ftrace_get_offsets_##call(&__data_offsets, args); \ __entry_size = ALIGN(__data_size + sizeof(*entry) + sizeof(u32),\ sizeof(u64)); \ __entry_size -= sizeof(u32); \ \ - if (WARN_ONCE(__entry_size > FTRACE_MAX_PROFILE_SIZE, \ + if (WARN_ONCE(__entry_size > PERF_MAX_TRACE_SIZE, \ "profile buffer not large enough")) \ return; \ \ - local_irq_save(irq_flags); \ - \ - rctx = perf_swevent_get_recursion_context(); \ - if (rctx < 0) \ - goto end_recursion; \ - \ - __cpu = smp_processor_id(); \ - \ - if (in_nmi()) \ - trace_buf = rcu_dereference(perf_trace_buf_nmi); \ - else \ - trace_buf = rcu_dereference(perf_trace_buf); \ - \ - if (!trace_buf) \ - goto end; \ - \ - raw_data = per_cpu_ptr(trace_buf, __cpu); \ - \ - *(u64 *)(&raw_data[__entry_size - sizeof(u64)]) = 0ULL; \ - entry = (struct ftrace_raw_##call *)raw_data; \ - ent = &entry->ent; \ - tracing_generic_entry_update(ent, irq_flags, pc); \ - ent->type = event_call->id; \ + entry = (struct ftrace_raw_##call *)perf_trace_buf_prepare( \ + __entry_size, event_call->event.type, &__regs, &rctx); \ + if (!entry) \ + return; \ \ tstruct \ \ { assign; } \ \ - perf_tp_event(event_call->id, __addr, __count, entry, \ - __entry_size); \ - \ -end: \ - perf_swevent_put_recursion_context(rctx); \ -end_recursion: \ - local_irq_restore(irq_flags); \ + head = this_cpu_ptr(event_call->perf_events); \ + perf_trace_buf_submit(entry, __entry_size, rctx, __addr, \ + __count, &__regs, head); \ } +/* + * This part is compiled out, it is only here as a build time check + * to make sure that if the tracepoint handling changes, the + * perf probe will fail to compile unless it too is updated. + */ #undef DEFINE_EVENT -#define DEFINE_EVENT(template, call, proto, args) \ -static void ftrace_profile_##call(proto) \ -{ \ - struct ftrace_event_call *event_call = &event_##call; \ - \ - ftrace_profile_templ_##template(event_call, args); \ +#define DEFINE_EVENT(template, call, proto, args) \ +static inline void perf_test_probe_##call(void) \ +{ \ + check_trace_callback_type_##call(perf_trace_##template); \ } + #undef DEFINE_EVENT_PRINT #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \ DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args)) #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) -#endif /* CONFIG_EVENT_PROFILE */ +#endif /* CONFIG_PERF_EVENTS */ #undef _TRACE_PROFILE_INIT |