/* * Copyright 2006 Andi Kleen, SUSE Labs. * Subject to the GNU Public License, v.2 * * Fast user context implementation of clock_gettime and gettimeofday. * * The code should have no internal unresolved relocations. * Check with readelf after changing. * Also alternative() doesn't work. */ /* Disable profiling for userspace code: */ #define DISABLE_BRANCH_PROFILING #include #include #include #include #include #include #include #include #include #include #include #include #include "vextern.h" #define gtod vdso_vsyscall_gtod_data notrace static long vdso_fallback_gettime(long clock, struct timespec *ts) { long ret; asm("syscall" : "=a" (ret) : "0" (__NR_clock_gettime),"D" (clock), "S" (ts) : "memory"); return ret; } notrace static inline long vgetns(void) { long v; cycles_t (*vread)(void); vread = gtod->clock.vread; v = (vread() - gtod->clock.cycle_last) & gtod->clock.mask; return (v * gtod->clock.mult) >> gtod->clock.shift; } notrace static noinline int do_realtime(struct timespec *ts) { unsigned long seq, ns; do { seq = read_seqbegin(>od->lock); ts->tv_sec = gtod->wall_time_sec; ts->tv_nsec = gtod->wall_time_nsec; ns = vgetns(); } while (unlikely(read_seqretry(>od->lock, seq))); timespec_add_ns(ts, ns); return 0; } /* Copy of the version in kernel/time.c which we cannot directly access */ notrace static void vset_normalized_timespec(struct timespec *ts, long sec, long nsec) { while (nsec >= NSEC_PER_SEC) { nsec -= NSEC_PER_SEC; ++sec; } while (nsec < 0) { nsec += NSEC_PER_SEC; --sec; } ts->tv_sec = sec; ts->tv_nsec = nsec; } notrace static noinline int do_monotonic(struct timespec *ts) { unsigned long seq, ns, secs; do { seq = read_seqbegin(>od->lock); secs = gtod->wall_time_sec; ns = gtod->wall_time_nsec + vgetns(); secs += gtod->wall_to_monotonic.tv_sec; ns += gtod->wall_to_monotonic.tv_nsec; } while (unlikely(read_seqretry(>od->lock, seq))); vset_normalized_timespec(ts, secs, ns); return 0; } notrace static noinline int do_realtime_coarse(struct timespec *ts) { unsigned long seq; do { seq = read_seqbegin(>od->lock); ts->tv_sec = gtod->wall_time_coarse.tv_sec; ts->tv_nsec = gtod->wall_time_coarse.tv_nsec; } while (unlikely(read_seqretry(>od->lock, seq))); return 0; } notrace static noinline int do_monotonic_coarse(struct timespec *ts) { unsigned long seq, ns, secs; do { seq = read_seqbegin(>od->lock); secs = gtod->wall_time_coarse.tv_sec; ns = gtod->wall_time_coarse.tv_nsec; secs += gtod->wall_to_monotonic.tv_sec; ns += gtod->wall_to_monotonic.tv_nsec; } while (unlikely(read_seqretry(>od->lock, seq))); vset_normalized_timespec(ts, secs, ns); return 0; } /* * If the TSC is synchronized across all CPUs, read the current TSC * and export its value in the nsec field of the timespec */ notrace static noinline int do_trace_clock(struct timespec *ts) { unsigned long seq; union lttng_timespec *lts = (union lttng_timespec *) ts; do { seq = read_seqbegin(>od->lock); if (unlikely(!gtod->trace_clock_is_sync)) return vdso_fallback_gettime(CLOCK_TRACE, ts); /* * We don't protect the rdtsc with the rdtsc_barrier because * we can't obtain with tracing that level of precision. * The operation of recording an event is not atomic therefore * the small chance of imprecision doesn't justify the overhead * of a barrier. */ /* * TODO: check that vget_cycles(), using paravirt ops, will * match the TSC read by get_cycles() at the kernel level. */ lts->lttng_ts = vget_cycles(); } while (unlikely(read_seqretry(>od->lock, seq))); return 0; } /* * Returns the cpu_khz, it needs to be a syscall because we can't access * this value from userspace and it will only be called at the beginning * of the tracing session */ notrace static noinline int do_trace_clock_freq(struct timespec *ts) { return vdso_fallback_gettime(CLOCK_TRACE_FREQ, ts); } notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) { if (likely(gtod->sysctl_enabled)) switch (clock) { case CLOCK_REALTIME: if (likely(gtod->clock.vread)) return do_realtime(ts); break; case CLOCK_MONOTONIC: if (likely(gtod->clock.vread)) return do_monotonic(ts); break; case CLOCK_REALTIME_COARSE: return do_realtime_coarse(ts); case CLOCK_MONOTONIC_COARSE: return do_monotonic_coarse(ts); case CLOCK_TRACE: return do_trace_clock(ts); case CLOCK_TRACE_FREQ: return do_trace_clock_freq(ts); default: return -EINVAL; } return vdso_fallback_gettime(clock, ts); } int clock_gettime(clockid_t, struct timespec *) __attribute__((weak, alias("__vdso_clock_gettime"))); notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) { long ret; if (likely(gtod->sysctl_enabled && gtod->clock.vread)) { if (likely(tv != NULL)) { BUILD_BUG_ON(offsetof(struct timeval, tv_usec) != offsetof(struct timespec, tv_nsec) || sizeof(*tv) != sizeof(struct timespec)); do_realtime((struct timespec *)tv); tv->tv_usec /= 1000; } if (unlikely(tz != NULL)) { /* Avoid memcpy. Some old compilers fail to inline it */ tz->tz_minuteswest = gtod->sys_tz.tz_minuteswest; tz->tz_dsttime = gtod->sys_tz.tz_dsttime; } return 0; } asm("syscall" : "=a" (ret) : "0" (__NR_gettimeofday), "D" (tv), "S" (tz) : "memory"); return ret; } int gettimeofday(struct timeval *, struct timezone *) __attribute__((weak, alias("__vdso_gettimeofday")));