From 8b94df0f2047e9728cb872adc9e64557b7a5152f Mon Sep 17 00:00:00 2001
From: Reinhard Tartler <siretart@tauware.de>
Date: Sun, 4 Dec 2011 10:10:33 +0100
Subject: [PATCH] vp3dec: Check coefficient index in vp3_dequant()

Based on a patch by Michael Niedermayer <michaelni@gmx.at>

Fixes NGS00145, CVE-2011-4352

Found-by: Phillip Langlois
Signed-off-by: Reinhard Tartler <siretart@tauware.de>


Upstream-Status: Backport

http://git.videolan.org/?p=ffmpeg.git;a=commit;h=8b94df0f2047e9728cb872adc9e64557b7a5152f

Signed-off-by: Kai Kang <kai.kang@windriver.com>
---
 libavcodec/vp3.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/libavcodec/vp3.c b/libavcodec/vp3.c
index 51ab048..f44d084 100644
--- a/gst-libs/ext/libav/libavcodec/vp3.c
+++ b/gst-libs/ext/libav/libavcodec/vp3.c
@@ -1363,6 +1363,10 @@ static inline int vp3_dequant(Vp3DecodeContext *s, Vp3Fragment *frag,
         case 1: // zero run
             s->dct_tokens[plane][i]++;
             i += (token >> 2) & 0x7f;
+            if (i > 63) {
+                av_log(s->avctx, AV_LOG_ERROR, "Coefficient index overflow\n");
+                return i;
+            }
             block[perm[i]] = (token >> 9) * dequantizer[perm[i]];
             i++;
             break;
@@ -1566,7 +1570,10 @@ static void render_slice(Vp3DecodeContext *s, int slice)
                     /* invert DCT and place (or add) in final output */
 
                     if (s->all_fragments[i].coding_method == MODE_INTRA) {
-                        vp3_dequant(s, s->all_fragments + i, plane, 0, block);
+                        int index;
+                        index = vp3_dequant(s, s->all_fragments + i, plane, 0, block);
+                        if (index > 63)
+                            continue;
                         if(s->avctx->idct_algo!=FF_IDCT_VP3)
                             block[0] += 128<<3;
                         s->dsp.idct_put(
@@ -1574,7 +1581,10 @@ static void render_slice(Vp3DecodeContext *s, int slice)
                             stride,
                             block);
                     } else {
-                        if (vp3_dequant(s, s->all_fragments + i, plane, 1, block)) {
+                        int index = vp3_dequant(s, s->all_fragments + i, plane, 1, block);
+                        if (index > 63)
+                            continue;
+                        if (index > 0) {
                         s->dsp.idct_add(
                             output_plane + first_pixel,
                             stride,
-- 
2.1.1