128 lines
4.7 KiB
Diff
128 lines
4.7 KiB
Diff
From 19d9966572a410804349e1a8ee2017fed49a6dab Mon Sep 17 00:00:00 2001
|
|
From: Wan-Teh Chang <wtc@google.com>
|
|
Date: Wed, 3 Apr 2024 20:08:16 +0000
|
|
Subject: [PATCH 1/2] Fix integer overflows in calc of stride_in_bytes
|
|
|
|
Fix unsigned integer overflows in the calculation of stride_in_bytes in
|
|
img_alloc_helper() when d_w is huge.
|
|
|
|
Change the type of stride_in_bytes from unsigned int to int because it
|
|
will be assigned to img->stride[AOM_PLANE_Y], which is of the int type.
|
|
|
|
Test:
|
|
cmake ../aom -G Ninja -DCMAKE_C_COMPILER=clang \
|
|
-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Debug \
|
|
-DSANITIZE=unsigned-integer-overflow
|
|
ninja
|
|
./test_libaom --gtest_filter=AomImageTest.AomImgAllocHugeWidth
|
|
|
|
Bug: chromium:332382766
|
|
Change-Id: Iaccb83bcd13ddc3ea5e6f01da91bb01215ddb461
|
|
(cherry picked from commit 7aa2edc2b09f98c32820923d813fd73eb23b5861)
|
|
---
|
|
aom/src/aom_image.c | 15 ++++++++-------
|
|
test/aom_image_test.cc | 36 ++++++++++++++++++++++++++++++++++++
|
|
2 files changed, 44 insertions(+), 7 deletions(-)
|
|
|
|
diff --git a/aom/src/aom_image.c b/aom/src/aom_image.c
|
|
index 8e94d5d..acd3694 100644
|
|
--- a/aom/src/aom_image.c
|
|
+++ b/aom/src/aom_image.c
|
|
@@ -36,8 +36,7 @@ static aom_image_t *img_alloc_helper(
|
|
/* NOTE: In this function, bit_depth is either 8 or 16 (if
|
|
* AOM_IMG_FMT_HIGHBITDEPTH is set), never 10 or 12.
|
|
*/
|
|
- unsigned int h, w, s, xcs, ycs, bps, bit_depth;
|
|
- unsigned int stride_in_bytes;
|
|
+ unsigned int h, w, xcs, ycs, bps, bit_depth;
|
|
|
|
if (img != NULL) memset(img, 0, sizeof(aom_image_t));
|
|
|
|
@@ -106,9 +105,11 @@ static aom_image_t *img_alloc_helper(
|
|
w = align_image_dimension(d_w, xcs, size_align);
|
|
h = align_image_dimension(d_h, ycs, size_align);
|
|
|
|
- s = (fmt & AOM_IMG_FMT_PLANAR) ? w : bps * w / bit_depth;
|
|
+ uint64_t s = (fmt & AOM_IMG_FMT_PLANAR) ? w : (uint64_t)bps * w / bit_depth;
|
|
s = (s + 2 * border + stride_align - 1) & ~(stride_align - 1);
|
|
- stride_in_bytes = s * bit_depth / 8;
|
|
+ s = s * bit_depth / 8;
|
|
+ if (s > INT_MAX) goto fail;
|
|
+ const int stride_in_bytes = (int)s;
|
|
|
|
/* Allocate the new image */
|
|
if (!img) {
|
|
@@ -230,7 +231,7 @@ int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y,
|
|
|
|
img->planes[AOM_PLANE_Y] =
|
|
data + x * bytes_per_sample + y * img->stride[AOM_PLANE_Y];
|
|
- data += (img->h + 2 * border) * img->stride[AOM_PLANE_Y];
|
|
+ data += ((size_t)img->h + 2 * border) * img->stride[AOM_PLANE_Y];
|
|
|
|
unsigned int uv_border_h = border >> img->y_chroma_shift;
|
|
unsigned int uv_x = x >> img->x_chroma_shift;
|
|
@@ -242,14 +243,14 @@ int aom_img_set_rect(aom_image_t *img, unsigned int x, unsigned int y,
|
|
} else if (!(img->fmt & AOM_IMG_FMT_UV_FLIP)) {
|
|
img->planes[AOM_PLANE_U] =
|
|
data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_U];
|
|
- data += ((img->h >> img->y_chroma_shift) + 2 * uv_border_h) *
|
|
+ data += ((size_t)(img->h >> img->y_chroma_shift) + 2 * uv_border_h) *
|
|
img->stride[AOM_PLANE_U];
|
|
img->planes[AOM_PLANE_V] =
|
|
data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_V];
|
|
} else {
|
|
img->planes[AOM_PLANE_V] =
|
|
data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_V];
|
|
- data += ((img->h >> img->y_chroma_shift) + 2 * uv_border_h) *
|
|
+ data += ((size_t)(img->h >> img->y_chroma_shift) + 2 * uv_border_h) *
|
|
img->stride[AOM_PLANE_V];
|
|
img->planes[AOM_PLANE_U] =
|
|
data + uv_x * bytes_per_sample + uv_y * img->stride[AOM_PLANE_U];
|
|
diff --git a/test/aom_image_test.cc b/test/aom_image_test.cc
|
|
index ad48e73..69b777b 100644
|
|
--- a/test/aom_image_test.cc
|
|
+++ b/test/aom_image_test.cc
|
|
@@ -60,3 +60,39 @@ TEST(AomImageTest, AomImgAllocNv12) {
|
|
EXPECT_EQ(img.planes[AOM_PLANE_V], nullptr);
|
|
aom_img_free(&img);
|
|
}
|
|
+
|
|
+TEST(AomImageTest, AomImgAllocHugeWidth) {
|
|
+ // The stride (0x80000000 * 2) would overflow unsigned int.
|
|
+ aom_image_t *image =
|
|
+ aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, 0x80000000, 1, 1);
|
|
+ ASSERT_EQ(image, nullptr);
|
|
+
|
|
+ // The stride (0x80000000) would overflow int.
|
|
+ image = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 0x80000000, 1, 1);
|
|
+ ASSERT_EQ(image, nullptr);
|
|
+
|
|
+ image = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 0x7ffffffe, 1, 1);
|
|
+ if (image) {
|
|
+ aom_img_free(image);
|
|
+ }
|
|
+
|
|
+ image = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 285245883, 64, 1);
|
|
+ if (image) {
|
|
+ aom_img_free(image);
|
|
+ }
|
|
+
|
|
+ image = aom_img_alloc(nullptr, AOM_IMG_FMT_NV12, 285245883, 64, 1);
|
|
+ if (image) {
|
|
+ aom_img_free(image);
|
|
+ }
|
|
+
|
|
+ image = aom_img_alloc(nullptr, AOM_IMG_FMT_YV12, 285245883, 64, 1);
|
|
+ if (image) {
|
|
+ aom_img_free(image);
|
|
+ }
|
|
+
|
|
+ image = aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, 285245883, 2, 1);
|
|
+ if (image) {
|
|
+ aom_img_free(image);
|
|
+ }
|
|
+}
|
|
--
|
|
2.41.0
|
|
|