Validate grid cell dimensions are <= AV1 maximum

Author: wantehchang

src/write.c

@@ -1620,8 +1620,14 @@ static avifResult avifValidateGrid(uint32_t gridCols,
     }
     const uint32_t tileWidth = firstCell->width;
     const uint32_t tileHeight = firstCell->height;
-    const uint32_t gridWidth = avifGridWidth(gridCols, firstCell, bottomRightCell);
-    const uint32_t gridHeight = avifGridHeight(gridRows, firstCell, bottomRightCell);
+    if ((tileWidth > 65536) || (tileHeight > 65536)) {
+        avifDiagnosticsPrintf(diag,
+                              "the first %s cell has invalid dimensions for AV1: %ux%u",
+                              validateGainMap ? "gain map" : "image",
+                              tileWidth,
+                              tileHeight);
+        return AVIF_RESULT_INVALID_ARGUMENT;
+    }
     for (uint32_t cellIndex = 0; cellIndex < cellCount; ++cellIndex) {
         const avifImage * cellImage = cellImages[cellIndex];
         if (validateGainMap) {
@@ -1682,6 +1688,8 @@ static avifResult avifValidateGrid(uint32_t gridCols,
                               bottomRightCell->height);
         return AVIF_RESULT_INVALID_IMAGE_GRID;
     }
+    const uint32_t gridWidth = avifGridWidth(gridCols, firstCell, bottomRightCell);
+    const uint32_t gridHeight = avifGridHeight(gridRows, firstCell, bottomRightCell);
     if ((cellCount > 1) && !avifAreGridDimensionsValid(firstCell->yuvFormat, gridWidth, gridHeight, tileWidth, tileHeight, diag)) {
         return AVIF_RESULT_INVALID_IMAGE_GRID;
     }

tests/gtest/avifallocationtest.cc

@@ -171,19 +171,19 @@ TEST(EncodingTest, MinimumInvalidDimensions) {
   TestEncoding(0, 1, 8, AVIF_RESULT_NO_CONTENT);
   TestEncoding(1, 0, 8, AVIF_RESULT_NO_CONTENT);
   TestEncoding(1, 1, 0, AVIF_RESULT_UNSUPPORTED_DEPTH);
-  TestEncoding(65536 + 1, 1, 8, AVIF_RESULT_ENCODE_COLOR_FAILED);
-  TestEncoding(1, 65536 + 1, 8, AVIF_RESULT_ENCODE_COLOR_FAILED);
-  TestEncoding(65536 + 1, 65536 + 1, 8, AVIF_RESULT_ENCODE_COLOR_FAILED);
+  TestEncoding(65536 + 1, 1, 8, AVIF_RESULT_INVALID_ARGUMENT);
+  TestEncoding(1, 65536 + 1, 8, AVIF_RESULT_INVALID_ARGUMENT);
+  TestEncoding(65536 + 1, 65536 + 1, 8, AVIF_RESULT_INVALID_ARGUMENT);
 }
 
 TEST(EncodingTest, MaximumInvalidDimensions) {
   TestEncoding(std::numeric_limits<decltype(avifImage::width)>::max(), 1, 8,
-               AVIF_RESULT_ENCODE_COLOR_FAILED);
+               AVIF_RESULT_INVALID_ARGUMENT);
   TestEncoding(1, std::numeric_limits<decltype(avifImage::height)>::max(), 8,
-               AVIF_RESULT_ENCODE_COLOR_FAILED);
+               AVIF_RESULT_INVALID_ARGUMENT);
   TestEncoding(std::numeric_limits<decltype(avifImage::width)>::max(),
                std::numeric_limits<decltype(avifImage::height)>::max(), 12,
-               AVIF_RESULT_ENCODE_COLOR_FAILED);
+               AVIF_RESULT_INVALID_ARGUMENT);
   TestEncoding(1, 1, std::numeric_limits<decltype(avifImage::depth)>::max(),
                AVIF_RESULT_UNSUPPORTED_DEPTH);
 }