Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame^] | 1 | From 13c13109759090b7f7182480d075e13b36ed8edd Mon Sep 17 00:00:00 2001 |
| 2 | From: Paul B Mahol <onemda@gmail.com> |
| 3 | Date: Sat, 12 Nov 2022 15:19:21 +0100 |
| 4 | Subject: [PATCH] avcodec/smcenc: stop accessing out of bounds frame |
| 5 | |
| 6 | Upstream-Status: Backport [https://github.com/FFmpeg/FFmpeg/commit/13c13109759090b7f7182480d075e13b36ed8edd] |
| 7 | |
| 8 | Signed-off-by: <narpat.mali@windriver.com> |
| 9 | |
| 10 | --- |
| 11 | libavcodec/smcenc.c | 18 ++++++++++++++---- |
| 12 | 1 file changed, 14 insertions(+), 4 deletions(-) |
| 13 | |
| 14 | diff --git a/libavcodec/smcenc.c b/libavcodec/smcenc.c |
| 15 | index f3d26a4e8d..33549b8ab4 100644 |
| 16 | --- a/libavcodec/smcenc.c |
| 17 | +++ b/libavcodec/smcenc.c |
| 18 | @@ -61,6 +61,7 @@ typedef struct SMCContext { |
| 19 | { \ |
| 20 | row_ptr += stride * 4; \ |
| 21 | pixel_ptr = row_ptr; \ |
| 22 | + cur_y += 4; \ |
| 23 | } \ |
| 24 | } \ |
| 25 | } |
| 26 | @@ -117,6 +118,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame, |
| 27 | const uint8_t *prev_pixels = (const uint8_t *)s->prev_frame->data[0]; |
| 28 | uint8_t *distinct_values = s->distinct_values; |
| 29 | const uint8_t *pixel_ptr, *row_ptr; |
| 30 | + const int height = frame->height; |
| 31 | const int width = frame->width; |
| 32 | uint8_t block_values[16]; |
| 33 | int block_counter = 0; |
| 34 | @@ -125,13 +127,14 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame, |
| 35 | int color_octet_index = 0; |
| 36 | int color_table_index; /* indexes to color pair, quad, or octet tables */ |
| 37 | int total_blocks; |
| 38 | + int cur_y = 0; |
| 39 | |
| 40 | memset(s->color_pairs, 0, sizeof(s->color_pairs)); |
| 41 | memset(s->color_quads, 0, sizeof(s->color_quads)); |
| 42 | memset(s->color_octets, 0, sizeof(s->color_octets)); |
| 43 | |
| 44 | /* Number of 4x4 blocks in frame. */ |
| 45 | - total_blocks = ((frame->width + 3) / 4) * ((frame->height + 3) / 4); |
| 46 | + total_blocks = ((width + 3) / 4) * ((height + 3) / 4); |
| 47 | |
| 48 | pixel_ptr = row_ptr = src_pixels; |
| 49 | |
| 50 | @@ -145,11 +148,13 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame, |
| 51 | int cache_index; |
| 52 | int distinct = 0; |
| 53 | int blocks = 0; |
| 54 | + int frame_y = cur_y; |
| 55 | |
| 56 | while (prev_pixels && s->key_frame == 0 && block_counter + inter_skip_blocks < total_blocks) { |
| 57 | + const int y_size = FFMIN(4, height - cur_y); |
| 58 | int compare = 0; |
| 59 | |
| 60 | - for (int y = 0; y < 4; y++) { |
| 61 | + for (int y = 0; y < y_size; y++) { |
| 62 | const ptrdiff_t offset = pixel_ptr - src_pixels; |
| 63 | const uint8_t *prev_pixel_ptr = prev_pixels + offset; |
| 64 | |
| 65 | @@ -170,8 +175,10 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame, |
| 66 | |
| 67 | pixel_ptr = xpixel_ptr; |
| 68 | row_ptr = xrow_ptr; |
| 69 | + cur_y = frame_y; |
| 70 | |
| 71 | while (block_counter > 0 && block_counter + intra_skip_blocks < total_blocks) { |
| 72 | + const int y_size = FFMIN(4, height - cur_y); |
| 73 | const ptrdiff_t offset = pixel_ptr - src_pixels; |
| 74 | const int sy = offset / stride; |
| 75 | const int sx = offset % stride; |
| 76 | @@ -180,7 +187,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame, |
| 77 | const uint8_t *old_pixel_ptr = src_pixels + nx + ny * stride; |
| 78 | int compare = 0; |
| 79 | |
| 80 | - for (int y = 0; y < 4; y++) { |
| 81 | + for (int y = 0; y < y_size; y++) { |
| 82 | compare |= memcmp(old_pixel_ptr + y * stride, pixel_ptr + y * stride, 4); |
| 83 | if (compare) |
| 84 | break; |
| 85 | @@ -197,9 +204,11 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame, |
| 86 | |
| 87 | pixel_ptr = xpixel_ptr; |
| 88 | row_ptr = xrow_ptr; |
| 89 | + cur_y = frame_y; |
| 90 | |
| 91 | while (block_counter + coded_blocks < total_blocks && coded_blocks < 256) { |
| 92 | - for (int y = 0; y < 4; y++) |
| 93 | + const int y_size = FFMIN(4, height - cur_y); |
| 94 | + for (int y = 0; y < y_size; y++) |
| 95 | memcpy(block_values + y * 4, pixel_ptr + y * stride, 4); |
| 96 | |
| 97 | qsort(block_values, 16, sizeof(block_values[0]), smc_cmp_values); |
| 98 | @@ -224,6 +233,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame, |
| 99 | |
| 100 | pixel_ptr = xpixel_ptr; |
| 101 | row_ptr = xrow_ptr; |
| 102 | + cur_y = frame_y; |
| 103 | |
| 104 | blocks = coded_blocks; |
| 105 | distinct = coded_distinct; |
| 106 | -- |
| 107 | 2.34.1 |
| 108 | |