[Backport] Security bug 329674887 (1/2)

Cherry-pick of patch orignally reviewed on
https://chromium-review.googlesource.com/c/webm/libvpx/+/5370376:
Fix to buffer alloc for vp9_bitstream_worker_data

The code was using the bitstream_worker_data when it
wasn't allocated for big enough size. This is because
the existing condition was to only re-alloc the
bitstream_worker_data when current dest_size was larger
than the current frame_size. But under resolution change
where frame_size is increased, beyond the current dest_size,
we need to allow re-alloc to the new size.

The existing condition to re-alloc when dest_size is
larger than frame_size (which is not required) is kept
for now.

Also increase the dest_size to account for image format.

Added tests, for both ROW_MT=0 and 1, that reproduce
the failures in the bugs below.

Note: this issue only affects the REALTIME encoding path.

Bug: b/329088759, b/329674887, b/329179808

Change-Id: Icd65dbc5317120304d803f648d4bd9405710db6f
Reviewed-on: https://codereview.qt-project.org/c/qt/qtwebengine-chromium/+/554667
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>

1 files changed, 11 insertions, 3 deletions

diff --git a/chromium/third_party/libvpx/source/libvpx/vp9/encoder/vp9_bitstream.c b/chromium/third_party/libvpx/source/libvpx/vp9/encoder/vp9_bitstream.c
index 3eff4ce830d..22db3971492 100644
--- a/chromium/third_party/libvpx/source/libvpx/vp9/encoder/vp9_bitstream.c
+++ b/chromium/third_party/libvpx/source/libvpx/vp9/encoder/vp9_bitstream.c
@@ -963,6 +963,14 @@ void vp9_bitstream_encode_tiles_buffer_dealloc(VP9_COMP *const cpi) {
   }
 }
 
+static int encode_tiles_buffer_alloc_size(VP9_COMP *const cpi) {
+  VP9_COMMON *const cm = &cpi->common;
+  const int image_bps =
+      (8 + 2 * (8 >> (cm->subsampling_x + cm->subsampling_y))) *
+      (1 + (cm->bit_depth > 8));
+  return cpi->oxcf.width * cpi->oxcf.height * image_bps / 8;
+}
+
 static int encode_tiles_buffer_alloc(VP9_COMP *const cpi) {
   int i;
   const size_t worker_data_size =
@@ -972,7 +980,7 @@ static int encode_tiles_buffer_alloc(VP9_COMP *const cpi) {
   if (!cpi->vp9_bitstream_worker_data) return 1;
   for (i = 1; i < cpi->num_workers; ++i) {
     cpi->vp9_bitstream_worker_data[i].dest_size =
-        cpi->oxcf.width * cpi->oxcf.height;
+        encode_tiles_buffer_alloc_size(cpi);
     cpi->vp9_bitstream_worker_data[i].dest =
         vpx_malloc(cpi->vp9_bitstream_worker_data[i].dest_size);
     if (!cpi->vp9_bitstream_worker_data[i].dest) return 1;
@@ -989,8 +997,8 @@ static size_t encode_tiles_mt(VP9_COMP *cpi, uint8_t *data_ptr) {
   int tile_col = 0;
 
   if (!cpi->vp9_bitstream_worker_data ||
-      cpi->vp9_bitstream_worker_data[1].dest_size >
-          (cpi->oxcf.width * cpi->oxcf.height)) {
+      cpi->vp9_bitstream_worker_data[1].dest_size !=
+          encode_tiles_buffer_alloc_size(cpi)) {
     vp9_bitstream_encode_tiles_buffer_dealloc(cpi);
     if (encode_tiles_buffer_alloc(cpi)) return 0;
   }