swscale: fix overflows in vertical scaling at top/bottom edges.

This fixes integer multiplication overflows in RGB48 output
(vertical) scaling as detected by IOC. What happens is that for
certain types of filters (lanczos, spline, bicubic), the
intermediate sum of coefficients in the middle of a filter can
be larger than the fixed-point equivalent of 1.0, even if the
final sum is 1.0. This is fine and we support that.

However, at frame edges, initFilter() will merge the coefficients
for the off-screen pixels into the top or bottom pixel, such as
to emulate edge extension. This means that suddenly, a single
coefficient can be larger than the fixed-point equivalent of
1.0, which the vertical scaling routines do not support.

Therefore, remove the merging of coefficients for edges for
the vertical scaling filter, and instead add edge detection
to the scaler itself so that it copies the pointers (not data)
for the edges (i.e. it uses line[0] for line[-1] as well), so
that a single coefficient is never larger than the fixed-point
equivalent of 1.0.

1 files changed, 48 insertions, 3 deletions

diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index c9dfc8df2f..f24561b8bf 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -2487,9 +2487,11 @@ static int swScale(SwsContext *c, const uint8_t* src[],
         const int firstLumSrcY= vLumFilterPos[dstY]; //First line needed as input
         const int firstLumSrcY2= vLumFilterPos[FFMIN(dstY | ((1<<c->chrDstVSubSample) - 1), dstH-1)];
         const int firstChrSrcY= vChrFilterPos[chrDstY]; //First line needed as input
-        int lastLumSrcY= firstLumSrcY + vLumFilterSize -1; // Last line needed as input
-        int lastLumSrcY2=firstLumSrcY2+ vLumFilterSize -1; // Last line needed as input
-        int lastChrSrcY= firstChrSrcY + vChrFilterSize -1; // Last line needed as input
+
+        // Last line needed as input
+        int lastLumSrcY  = FFMIN(c->srcH,    firstLumSrcY  + vLumFilterSize) - 1;
+        int lastLumSrcY2 = FFMIN(c->srcH,    firstLumSrcY2 + vLumFilterSize) - 1;
+        int lastChrSrcY  = FFMIN(c->chrSrcH, firstChrSrcY  + vChrFilterSize) - 1;
         int enough_lines;
 
         //handle holes (FAST_BILINEAR & weird filters)
@@ -2585,6 +2587,49 @@ static int swScale(SwsContext *c, const uint8_t* src[],
             const int16_t **chrUSrcPtr= (const int16_t **) chrUPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
             const int16_t **chrVSrcPtr= (const int16_t **) chrVPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
             const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **) alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL;
+
+            if (firstLumSrcY < 0 || firstLumSrcY + vLumFilterSize > c->srcH) {
+                const int16_t **tmpY = (const int16_t **) lumPixBuf + 2 * vLumBufSize;
+                int neg = -firstLumSrcY, i, end = FFMIN(c->srcH - firstLumSrcY, vLumFilterSize);
+                for (i = 0; i < neg;            i++)
+                    tmpY[i] = lumSrcPtr[neg];
+                for (     ; i < end;            i++)
+                    tmpY[i] = lumSrcPtr[i];
+                for (     ; i < vLumFilterSize; i++)
+                    tmpY[i] = tmpY[i-1];
+                lumSrcPtr = tmpY;
+
+                if (alpSrcPtr) {
+                    const int16_t **tmpA = (const int16_t **) alpPixBuf + 2 * vLumBufSize;
+                    for (i = 0; i < neg;            i++)
+                        tmpA[i] = alpSrcPtr[neg];
+                    for (     ; i < end;            i++)
+                        tmpA[i] = alpSrcPtr[i];
+                    for (     ; i < vLumFilterSize; i++)
+                        tmpA[i] = tmpA[i - 1];
+                    alpSrcPtr = tmpA;
+                }
+            }
+            if (firstChrSrcY < 0 || firstChrSrcY + vChrFilterSize > c->chrSrcH) {
+                const int16_t **tmpU = (const int16_t **) chrUPixBuf + 2 * vChrBufSize,
+                              **tmpV = (const int16_t **) chrVPixBuf + 2 * vChrBufSize;
+                int neg = -firstChrSrcY, i, end = FFMIN(c->chrSrcH - firstChrSrcY, vChrFilterSize);
+                for (i = 0; i < neg;            i++) {
+                    tmpU[i] = chrUSrcPtr[neg];
+                    tmpV[i] = chrVSrcPtr[neg];
+                }
+                for (     ; i < end;            i++) {
+                    tmpU[i] = chrUSrcPtr[i];
+                    tmpV[i] = chrVSrcPtr[i];
+                }
+                for (     ; i < vChrFilterSize; i++) {
+                    tmpU[i] = tmpU[i - 1];
+                    tmpV[i] = tmpV[i - 1];
+                }
+                chrUSrcPtr = tmpU;
+                chrVSrcPtr = tmpV;
+            }
+
             if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) { //YV12 like
                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;