From 9f49df1ad3ba49d479aa9a1ff997d2236509163b Mon Sep 17 00:00:00 2001 From: Ville Syrjälä Date: Thu, 18 Apr 2019 19:17:31 +0300 Subject: lib/igt_fb: Fix 32bit integer overflow in the shadow buffer size calculation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 16k*16k*16 == 1<<32, so 32bits isn't enough for the result when we start to have big framebuffers. Signed-off-by: Ville Syrjälä Reviewed-by: Chris Wilson --- lib/igt_fb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/igt_fb.c b/lib/igt_fb.c index f3609680..d4929019 100644 --- a/lib/igt_fb.c +++ b/lib/igt_fb.c @@ -1912,7 +1912,7 @@ static void *igt_fb_create_cairo_shadow_buffer(int fd, IGT_COLOR_YCBCR_BT709, IGT_COLOR_YCBCR_LIMITED_RANGE); shadow->strides[0] = ALIGN(width * (shadow->plane_bpp[0] / 8), 16); - shadow->size = ALIGN(shadow->strides[0] * height, + shadow->size = ALIGN((uint64_t)shadow->strides[0] * height, sysconf(_SC_PAGESIZE)); ptr = mmap(NULL, shadow->size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); -- cgit v1.2.3