golomb: use unsigned arithmetics in svq3_get_ue_golomb()

This prevents undefined behaviour of signed left shift if the coded
value is larger than 2^31. Large values are most likely invalid and
caused errors or by feeding random.

Validate every use of svq3_get_ue_golomb() and changed the place there
the return value was compared with negative numbers. dirac.c was clean,
fixed rv30 and svq3.

1 files changed, 8 insertions, 9 deletions

diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c
index f2c14ea739..013dee8dd3 100644
--- a/libavcodec/svq3.c
+++ b/libavcodec/svq3.c
@@ -216,17 +216,15 @@ static inline int svq3_decode_block(GetBitContext *gb, DCTELEM *block,
     static const uint8_t *const scan_patterns[4] =
     { luma_dc_zigzag_scan, zigzag_scan, svq3_scan, chroma_dc_scan };
 
-    int run, level, sign, vlc, limit;
+    int run, level, limit;
+    unsigned vlc;
     const int intra           = 3 * type >> 2;
     const uint8_t *const scan = scan_patterns[type];
 
     for (limit = (16 >> intra); index < 16; index = limit, limit += 8) {
         for (; (vlc = svq3_get_ue_golomb(gb)) != 0; index++) {
-            if (vlc == INVALID_VLC)
-                return -1;
-
-            sign = (vlc & 0x1) - 1;
-            vlc  = vlc + 1 >> 1;
+            int sign = (vlc & 1) ? 0 : -1;
+            vlc      = vlc + 1 >> 1;
 
             if (type == 3) {
                 if (vlc < 3) {
@@ -786,7 +784,7 @@ static int svq3_decode_slice_header(AVCodecContext *avctx)
         skip_bits_long(&s->gb, 0);
     }
 
-    if ((i = svq3_get_ue_golomb(&s->gb)) == INVALID_VLC || i >= 3) {
+    if ((i = svq3_get_ue_golomb(&s->gb)) >= 3) {
         av_log(h->s.avctx, AV_LOG_ERROR, "illegal slice type %d \n", i);
         return -1;
     }
@@ -1010,7 +1008,7 @@ static int svq3_decode_frame(AVCodecContext *avctx, void *data,
     H264Context *h     = &svq3->h;
     MpegEncContext *s  = &h->s;
     int buf_size       = avpkt->size;
-    int m, mb_type;
+    int m;
 
     /* special case for last picture */
     if (buf_size == 0) {
@@ -1093,6 +1091,7 @@ static int svq3_decode_frame(AVCodecContext *avctx, void *data,
 
     for (s->mb_y = 0; s->mb_y < s->mb_height; s->mb_y++) {
         for (s->mb_x = 0; s->mb_x < s->mb_width; s->mb_x++) {
+            unsigned mb_type;
             h->mb_xy = s->mb_x + s->mb_y * s->mb_stride;
 
             if ((get_bits_count(&s->gb) + 7) >= s->gb.size_in_bits &&
@@ -1113,7 +1112,7 @@ static int svq3_decode_frame(AVCodecContext *avctx, void *data,
                 mb_type += 8;
             else if (s->pict_type == AV_PICTURE_TYPE_B && mb_type >= 4)
                 mb_type += 4;
-            if ((unsigned)mb_type > 33 || svq3_decode_mb(svq3, mb_type)) {
+            if (mb_type > 33 || svq3_decode_mb(svq3, mb_type)) {
                 av_log(h->s.avctx, AV_LOG_ERROR,
                        "error while decoding MB %d %d\n", s->mb_x, s->mb_y);
                 return -1;