From bb90d4bc7b6a536b2e4db45f4763e467c2008251 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Tue, 9 Feb 2021 22:22:14 -0800 Subject: mm/highmem: Lift memcpy_[to|from]_page to core Working through a conversion to a call kmap_local_page() instead of kmap() revealed many places where the pattern kmap/memcpy/kunmap occurred. Eric Biggers, Matthew Wilcox, Christoph Hellwig, Dan Williams, and Al Viro all suggested putting this code into helper functions. Al Viro further pointed out that these functions already existed in the iov_iter code.[1] Various locations for the lifted functions were considered. Headers like mm.h or string.h seem ok but don't really portray the functionality well. pagemap.h made some sense but is for page cache functionality.[2] Another alternative would be to create a new header for the promoted memcpy functions, but it masks the fact that these are designed to copy to/from pages using the kernel direct mappings and complicates matters with a new header. Placing these functions in 'highmem.h' is suboptimal especially with the changes being proposed in the functionality of kmap. From a caller perspective including/using 'highmem.h' implies that the functions defined in that header are only required when highmem is in use which is increasingly not the case with modern processors. However, highmem.h is where all the current functions like this reside (zero_user(), clear_highpage(), clear_user_highpage(), copy_user_highpage(), and copy_highpage()). So it makes the most sense even though it is distasteful for some.[3] Lift memcpy_to_page() and memcpy_from_page() to pagemap.h. [1] https://lore.kernel.org/lkml/20201013200149.GI3576660@ZenIV.linux.org.uk/ https://lore.kernel.org/lkml/20201013112544.GA5249@infradead.org/ [2] https://lore.kernel.org/lkml/20201208122316.GH7338@casper.infradead.org/ [3] https://lore.kernel.org/lkml/20201013200149.GI3576660@ZenIV.linux.org.uk/#t https://lore.kernel.org/lkml/20201208163814.GN1563847@iweiny-DESK2.sc.intel.com/ Cc: Boris Pismenny Cc: Or Gerlitz Cc: Dave Hansen Suggested-by: Matthew Wilcox Suggested-by: Christoph Hellwig Suggested-by: Dan Williams Suggested-by: Al Viro Suggested-by: Eric Biggers Reviewed-by: Chaitanya Kulkarni Reviewed-by: Christoph Hellwig Signed-off-by: Ira Weiny Signed-off-by: David Sterba --- include/linux/highmem.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'include') diff --git a/include/linux/highmem.h b/include/linux/highmem.h index d2c70d3772a3..736b6a9f144d 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -276,4 +276,22 @@ static inline void copy_highpage(struct page *to, struct page *from) #endif +static inline void memcpy_from_page(char *to, struct page *page, + size_t offset, size_t len) +{ + char *from = kmap_atomic(page); + + memcpy(to, from + offset, len); + kunmap_atomic(from); +} + +static inline void memcpy_to_page(struct page *page, size_t offset, + const char *from, size_t len) +{ + char *to = kmap_atomic(page); + + memcpy(to + offset, from, len); + kunmap_atomic(to); +} + #endif /* _LINUX_HIGHMEM_H */ -- cgit v1.2.3 From 61b205f579911a11f0b576f73275eca2aed0d108 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Tue, 9 Feb 2021 22:22:15 -0800 Subject: mm/highmem: Convert memcpy_[to|from]_page() to kmap_local_page() kmap_local_page() is more efficient and is well suited for these calls. Convert the kmap() to kmap_local_page() Cc: Andrew Morton Cc: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Christoph Hellwig Signed-off-by: Ira Weiny Signed-off-by: David Sterba --- include/linux/highmem.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 736b6a9f144d..c17a175fe5fe 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -279,19 +279,19 @@ static inline void copy_highpage(struct page *to, struct page *from) static inline void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) { - char *from = kmap_atomic(page); + char *from = kmap_local_page(page); memcpy(to, from + offset, len); - kunmap_atomic(from); + kunmap_local(from); } static inline void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len) { - char *to = kmap_atomic(page); + char *to = kmap_local_page(page); memcpy(to + offset, from, len); - kunmap_atomic(to); + kunmap_local(to); } #endif /* _LINUX_HIGHMEM_H */ -- cgit v1.2.3 From 6a0996db6879cf09f989c5f44f9edd38240cb346 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Tue, 9 Feb 2021 22:22:16 -0800 Subject: mm/highmem: Introduce memcpy_page(), memmove_page(), and memset_page() 3 more common kmap patterns are kmap/memcpy/kunmap, kmap/memmove/kunmap. and kmap/memset/kunmap. Add helper functions for those patterns which use kmap_local_page(). Cc: Andrew Morton Cc: Christoph Hellwig Reviewed-by: Chaitanya Kulkarni Reviewed-by: Christoph Hellwig Signed-off-by: Ira Weiny Signed-off-by: David Sterba --- include/linux/highmem.h | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'include') diff --git a/include/linux/highmem.h b/include/linux/highmem.h index c17a175fe5fe..0b5d89621cb9 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -276,6 +276,39 @@ static inline void copy_highpage(struct page *to, struct page *from) #endif +static inline void memcpy_page(struct page *dst_page, size_t dst_off, + struct page *src_page, size_t src_off, + size_t len) +{ + char *dst = kmap_local_page(dst_page); + char *src = kmap_local_page(src_page); + + memcpy(dst + dst_off, src + src_off, len); + kunmap_local(src); + kunmap_local(dst); +} + +static inline void memmove_page(struct page *dst_page, size_t dst_off, + struct page *src_page, size_t src_off, + size_t len) +{ + char *dst = kmap_local_page(dst_page); + char *src = kmap_local_page(src_page); + + memmove(dst + dst_off, src + src_off, len); + kunmap_local(src); + kunmap_local(dst); +} + +static inline void memset_page(struct page *page, size_t offset, int val, + size_t len) +{ + char *addr = kmap_local_page(page); + + memset(addr + offset, val, len); + kunmap_local(addr); +} + static inline void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) { -- cgit v1.2.3 From ca18f6ea012bf30236b76c3480ac2c97131b6f8f Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 10 Feb 2021 09:49:28 -0800 Subject: mm/highmem: Add VM_BUG_ON() to mem*_page() calls Add VM_BUG_ON bounds checks to ensure the newly lifted and created page memory operations do not result in corrupted data in neighbor pages.[1][2] [1] https://lore.kernel.org/lkml/20201210053502.GS1563847@iweiny-DESK2.sc.intel.com/ [2] https://lore.kernel.org/lkml/20210209110931.00f00e47d9a0529fcee2ff01@linux-foundation.org/ Suggested-by: Matthew Wilcox Suggested-by: Andrew Morton Reviewed-by: Christoph Hellwig Signed-off-by: Ira Weiny Signed-off-by: David Sterba --- include/linux/highmem.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include') diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 0b5d89621cb9..44170f312ae7 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -283,6 +283,7 @@ static inline void memcpy_page(struct page *dst_page, size_t dst_off, char *dst = kmap_local_page(dst_page); char *src = kmap_local_page(src_page); + VM_BUG_ON(dst_off + len > PAGE_SIZE || src_off + len > PAGE_SIZE); memcpy(dst + dst_off, src + src_off, len); kunmap_local(src); kunmap_local(dst); @@ -295,6 +296,7 @@ static inline void memmove_page(struct page *dst_page, size_t dst_off, char *dst = kmap_local_page(dst_page); char *src = kmap_local_page(src_page); + VM_BUG_ON(dst_off + len > PAGE_SIZE || src_off + len > PAGE_SIZE); memmove(dst + dst_off, src + src_off, len); kunmap_local(src); kunmap_local(dst); @@ -305,6 +307,7 @@ static inline void memset_page(struct page *page, size_t offset, int val, { char *addr = kmap_local_page(page); + VM_BUG_ON(offset + len > PAGE_SIZE); memset(addr + offset, val, len); kunmap_local(addr); } @@ -314,6 +317,7 @@ static inline void memcpy_from_page(char *to, struct page *page, { char *from = kmap_local_page(page); + VM_BUG_ON(offset + len > PAGE_SIZE); memcpy(to, from + offset, len); kunmap_local(from); } @@ -323,6 +327,7 @@ static inline void memcpy_to_page(struct page *page, size_t offset, { char *to = kmap_local_page(page); + VM_BUG_ON(offset + len > PAGE_SIZE); memcpy(to + offset, from, len); kunmap_local(to); } -- cgit v1.2.3