From e506ea451254ab17e0bf918ca36232fec2a9b10c Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 15 Oct 2019 16:29:32 -0700 Subject: compiler.h: Split {READ,WRITE}_ONCE definitions out into rwonce.h In preparation for allowing architectures to define their own implementation of the READ_ONCE() macro, move the generic {READ,WRITE}_ONCE() definitions out of the unwieldy 'linux/compiler.h' file and into a new 'rwonce.h' header under 'asm-generic'. Acked-by: Peter Zijlstra (Intel) Acked-by: Paul E. McKenney Signed-off-by: Will Deacon --- include/asm-generic/Kbuild | 1 + include/asm-generic/barrier.h | 2 +- include/asm-generic/rwonce.h | 101 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 include/asm-generic/rwonce.h (limited to 'include/asm-generic') diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 44ec80e70518..74b0612601dd 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -45,6 +45,7 @@ mandatory-y += pci.h mandatory-y += percpu.h mandatory-y += pgalloc.h mandatory-y += preempt.h +mandatory-y += rwonce.h mandatory-y += sections.h mandatory-y += serial.h mandatory-y += shmparam.h diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h index 2eacaf7d62f6..8116744bb82c 100644 --- a/include/asm-generic/barrier.h +++ b/include/asm-generic/barrier.h @@ -13,7 +13,7 @@ #ifndef __ASSEMBLY__ -#include +#include #ifndef nop #define nop() asm volatile ("nop") diff --git a/include/asm-generic/rwonce.h b/include/asm-generic/rwonce.h new file mode 100644 index 000000000000..87584379da43 --- /dev/null +++ b/include/asm-generic/rwonce.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Prevent the compiler from merging or refetching reads or writes. The + * compiler is also forbidden from reordering successive instances of + * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some + * particular ordering. One way to make the compiler aware of ordering is to + * put the two invocations of READ_ONCE or WRITE_ONCE in different C + * statements. + * + * These two macros will also work on aggregate data types like structs or + * unions. + * + * Their two major use cases are: (1) Mediating communication between + * process-level code and irq/NMI handlers, all running on the same CPU, + * and (2) Ensuring that the compiler does not fold, spindle, or otherwise + * mutilate accesses that either do not require ordering or that interact + * with an explicit memory barrier or atomic instruction that provides the + * required ordering. + */ +#ifndef __ASM_GENERIC_RWONCE_H +#define __ASM_GENERIC_RWONCE_H + +#ifndef __ASSEMBLY__ + +#include +#include +#include + +#include + +/* + * Yes, this permits 64-bit accesses on 32-bit architectures. These will + * actually be atomic in some cases (namely Armv7 + LPAE), but for others we + * rely on the access being split into 2x32-bit accesses for a 32-bit quantity + * (e.g. a virtual address) and a strong prevailing wind. + */ +#define compiletime_assert_rwonce_type(t) \ + compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long), \ + "Unsupported access size for {READ,WRITE}_ONCE().") + +/* + * Use __READ_ONCE() instead of READ_ONCE() if you do not require any + * atomicity or dependency ordering guarantees. Note that this may result + * in tears! + */ +#define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x)) + +#define __READ_ONCE_SCALAR(x) \ +({ \ + __unqual_scalar_typeof(x) __x = __READ_ONCE(x); \ + smp_read_barrier_depends(); \ + (typeof(x))__x; \ +}) + +#define READ_ONCE(x) \ +({ \ + compiletime_assert_rwonce_type(x); \ + __READ_ONCE_SCALAR(x); \ +}) + +#define __WRITE_ONCE(x, val) \ +do { \ + *(volatile typeof(x) *)&(x) = (val); \ +} while (0) + +#define WRITE_ONCE(x, val) \ +do { \ + compiletime_assert_rwonce_type(x); \ + __WRITE_ONCE(x, val); \ +} while (0) + +static __no_sanitize_or_inline +unsigned long __read_once_word_nocheck(const void *addr) +{ + return __READ_ONCE(*(unsigned long *)addr); +} + +/* + * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need to load a + * word from memory atomically but without telling KASAN/KCSAN. This is + * usually used by unwinding code when walking the stack of a running process. + */ +#define READ_ONCE_NOCHECK(x) \ +({ \ + unsigned long __x; \ + compiletime_assert(sizeof(x) == sizeof(__x), \ + "Unsupported access size for READ_ONCE_NOCHECK()."); \ + __x = __read_once_word_nocheck(&(x)); \ + smp_read_barrier_depends(); \ + (typeof(x))__x; \ +}) + +static __no_kasan_or_inline +unsigned long read_word_at_a_time(const void *addr) +{ + kasan_check_read(addr, 1); + return *(unsigned long *)addr; +} + +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_GENERIC_RWONCE_H */ -- cgit v1.2.3 From b78b331a3f5c0773171dadd6bbfa2a2242b45604 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 15 Oct 2019 17:30:47 -0700 Subject: asm/rwonce: Allow __READ_ONCE to be overridden by the architecture The meat and potatoes of READ_ONCE() is defined by the __READ_ONCE() macro, which uses a volatile casts in an attempt to avoid tearing of byte, halfword, word and double-word accesses. Allow this to be overridden by the architecture code in the case that things like memory barriers are also required. Acked-by: Peter Zijlstra (Intel) Acked-by: Paul E. McKenney Signed-off-by: Will Deacon --- include/asm-generic/rwonce.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/asm-generic') diff --git a/include/asm-generic/rwonce.h b/include/asm-generic/rwonce.h index 87584379da43..04586b55a7c2 100644 --- a/include/asm-generic/rwonce.h +++ b/include/asm-generic/rwonce.h @@ -43,7 +43,9 @@ * atomicity or dependency ordering guarantees. Note that this may result * in tears! */ +#ifndef __READ_ONCE #define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x)) +#endif #define __READ_ONCE_SCALAR(x) \ ({ \ -- cgit v1.2.3 From 3c9184109e78ea2371ca8fa66d7f36986a53af98 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 30 Oct 2019 16:51:07 +0000 Subject: asm/rwonce: Remove smp_read_barrier_depends() invocation Alpha overrides __READ_ONCE() directly, so there's no need to use smp_read_barrier_depends() in the core code. This also means that __READ_ONCE() can be relied upon to provide dependency ordering. Acked-by: Peter Zijlstra (Intel) Acked-by: Paul E. McKenney Signed-off-by: Will Deacon --- include/asm-generic/rwonce.h | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) (limited to 'include/asm-generic') diff --git a/include/asm-generic/rwonce.h b/include/asm-generic/rwonce.h index 04586b55a7c2..3a7f737c77bd 100644 --- a/include/asm-generic/rwonce.h +++ b/include/asm-generic/rwonce.h @@ -40,24 +40,16 @@ /* * Use __READ_ONCE() instead of READ_ONCE() if you do not require any - * atomicity or dependency ordering guarantees. Note that this may result - * in tears! + * atomicity. Note that this may result in tears! */ #ifndef __READ_ONCE #define __READ_ONCE(x) (*(const volatile __unqual_scalar_typeof(x) *)&(x)) #endif -#define __READ_ONCE_SCALAR(x) \ -({ \ - __unqual_scalar_typeof(x) __x = __READ_ONCE(x); \ - smp_read_barrier_depends(); \ - (typeof(x))__x; \ -}) - #define READ_ONCE(x) \ ({ \ compiletime_assert_rwonce_type(x); \ - __READ_ONCE_SCALAR(x); \ + __READ_ONCE(x); \ }) #define __WRITE_ONCE(x, val) \ @@ -84,12 +76,9 @@ unsigned long __read_once_word_nocheck(const void *addr) */ #define READ_ONCE_NOCHECK(x) \ ({ \ - unsigned long __x; \ - compiletime_assert(sizeof(x) == sizeof(__x), \ + compiletime_assert(sizeof(x) == sizeof(unsigned long), \ "Unsupported access size for READ_ONCE_NOCHECK()."); \ - __x = __read_once_word_nocheck(&(x)); \ - smp_read_barrier_depends(); \ - (typeof(x))__x; \ + (typeof(x))__read_once_word_nocheck(&(x)); \ }) static __no_kasan_or_inline -- cgit v1.2.3 From 002dff36acfba3476b685a09f78ffb7c452f5951 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 10 Jul 2020 14:49:40 +0100 Subject: asm/rwonce: Don't pull into 'asm-generic/rwonce.h' Now that 'smp_read_barrier_depends()' has gone the way of the Norwegian Blue, drop the inclusion of in 'asm-generic/rwonce.h'. This requires fixups to some architecture vdso headers which were previously relying on 'asm/barrier.h' coming in via 'linux/compiler.h'. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Will Deacon --- arch/arm/include/asm/vdso/gettimeofday.h | 1 + arch/arm64/include/asm/vdso/compat_gettimeofday.h | 1 + arch/arm64/include/asm/vdso/gettimeofday.h | 1 + arch/riscv/include/asm/vdso/gettimeofday.h | 1 + include/asm-generic/rwonce.h | 2 -- include/linux/nospec.h | 2 ++ 6 files changed, 6 insertions(+), 2 deletions(-) (limited to 'include/asm-generic') diff --git a/arch/arm/include/asm/vdso/gettimeofday.h b/arch/arm/include/asm/vdso/gettimeofday.h index 36dc18553ed8..1b207cf07697 100644 --- a/arch/arm/include/asm/vdso/gettimeofday.h +++ b/arch/arm/include/asm/vdso/gettimeofday.h @@ -7,6 +7,7 @@ #ifndef __ASSEMBLY__ +#include #include #include #include diff --git a/arch/arm64/include/asm/vdso/compat_gettimeofday.h b/arch/arm64/include/asm/vdso/compat_gettimeofday.h index b6907ae78e53..bcf7649999a4 100644 --- a/arch/arm64/include/asm/vdso/compat_gettimeofday.h +++ b/arch/arm64/include/asm/vdso/compat_gettimeofday.h @@ -7,6 +7,7 @@ #ifndef __ASSEMBLY__ +#include #include #include diff --git a/arch/arm64/include/asm/vdso/gettimeofday.h b/arch/arm64/include/asm/vdso/gettimeofday.h index afba6ba332f8..127fa63893e2 100644 --- a/arch/arm64/include/asm/vdso/gettimeofday.h +++ b/arch/arm64/include/asm/vdso/gettimeofday.h @@ -7,6 +7,7 @@ #ifndef __ASSEMBLY__ +#include #include #define VDSO_HAS_CLOCK_GETRES 1 diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h index c8e818688ec1..3099362d9f26 100644 --- a/arch/riscv/include/asm/vdso/gettimeofday.h +++ b/arch/riscv/include/asm/vdso/gettimeofday.h @@ -4,6 +4,7 @@ #ifndef __ASSEMBLY__ +#include #include #include #include diff --git a/include/asm-generic/rwonce.h b/include/asm-generic/rwonce.h index 3a7f737c77bd..8d0a6280e982 100644 --- a/include/asm-generic/rwonce.h +++ b/include/asm-generic/rwonce.h @@ -26,8 +26,6 @@ #include #include -#include - /* * Yes, this permits 64-bit accesses on 32-bit architectures. These will * actually be atomic in some cases (namely Armv7 + LPAE), but for others we diff --git a/include/linux/nospec.h b/include/linux/nospec.h index 0c5ef54fd416..c1e79f72cd89 100644 --- a/include/linux/nospec.h +++ b/include/linux/nospec.h @@ -5,6 +5,8 @@ #ifndef _LINUX_NOSPEC_H #define _LINUX_NOSPEC_H + +#include #include struct task_struct; -- cgit v1.2.3 From 93fab07c22930c9ac4f01212fd92913c9a812f9f Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 30 Oct 2019 17:17:22 +0000 Subject: locking/barriers: Remove definitions for [smp_]read_barrier_depends() There are no remaining users of [smp_]read_barrier_depends(), so remove it from the generic implementation of 'barrier.h'. Acked-by: Peter Zijlstra (Intel) Acked-by: Paul E. McKenney Signed-off-by: Will Deacon --- include/asm-generic/barrier.h | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'include/asm-generic') diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h index 8116744bb82c..fec97dc34de7 100644 --- a/include/asm-generic/barrier.h +++ b/include/asm-generic/barrier.h @@ -46,10 +46,6 @@ #define dma_wmb() wmb() #endif -#ifndef read_barrier_depends -#define read_barrier_depends() do { } while (0) -#endif - #ifndef __smp_mb #define __smp_mb() mb() #endif @@ -62,10 +58,6 @@ #define __smp_wmb() wmb() #endif -#ifndef __smp_read_barrier_depends -#define __smp_read_barrier_depends() read_barrier_depends() -#endif - #ifdef CONFIG_SMP #ifndef smp_mb @@ -80,10 +72,6 @@ #define smp_wmb() __smp_wmb() #endif -#ifndef smp_read_barrier_depends -#define smp_read_barrier_depends() __smp_read_barrier_depends() -#endif - #else /* !CONFIG_SMP */ #ifndef smp_mb @@ -98,10 +86,6 @@ #define smp_wmb() barrier() #endif -#ifndef smp_read_barrier_depends -#define smp_read_barrier_depends() do { } while (0) -#endif - #endif /* CONFIG_SMP */ #ifndef __smp_store_mb @@ -196,7 +180,6 @@ do { \ #define virt_mb() __smp_mb() #define virt_rmb() __smp_rmb() #define virt_wmb() __smp_wmb() -#define virt_read_barrier_depends() __smp_read_barrier_depends() #define virt_store_mb(var, value) __smp_store_mb(var, value) #define virt_mb__before_atomic() __smp_mb__before_atomic() #define virt_mb__after_atomic() __smp_mb__after_atomic() -- cgit v1.2.3