| Andrew Geissler | 517393d | 2023-01-13 08:55:19 -0600 | [diff] [blame] | 1 | From 4fcace61801f418786c42487c6b06b693ee87666 Mon Sep 17 00:00:00 2001 |
| 2 | From: Romain Vimont <rom1v@videolabs.io> |
| 3 | Date: Mon, 19 Sep 2022 17:17:01 +0200 |
| 4 | Subject: [PATCH] vnc: fix possible buffer overflow |
| 5 | |
| 6 | Upstream-Status: Inappropriate |
| 7 | |
| 8 | RPI-Distro repo forks original vlc and applies patches |
| 9 | to enable raspiberry pi support. |
| 10 | |
| 11 | Thanks to 0xMitsurugi [1] from Synacktiv [2] for the bug report and fix. |
| 12 | |
| 13 | [1] https://twitter.com/0xMitsurugi |
| 14 | [2] https://www.synacktiv.com/ |
| 15 | |
| 16 | Fixes #27335 |
| 17 | |
| 18 | (cherry picked from commit 5eb783fd44ed6298db3e38f7765f21c42e4405f9) |
| 19 | --- |
| 20 | modules/access/vnc.c | 23 ++++++++++++++++------- |
| 21 | 1 file changed, 16 insertions(+), 7 deletions(-) |
| 22 | |
| 23 | --- a/modules/access/vnc.c |
| 24 | +++ b/modules/access/vnc.c |
| 25 | @@ -33,6 +33,7 @@ |
| 26 | #ifdef HAVE_CONFIG_H |
| 27 | # include "config.h" |
| 28 | #endif |
| 29 | +#include <assert.h> |
| 30 | |
| 31 | #include <vlc_common.h> |
| 32 | #include <vlc_plugin.h> |
| 33 | @@ -115,7 +116,7 @@ |
| 34 | int i_cancel_state; |
| 35 | |
| 36 | rfbClient* p_client; |
| 37 | - int i_framebuffersize; |
| 38 | + size_t i_framebuffersize; |
| 39 | block_t *p_block; |
| 40 | |
| 41 | float f_fps; |
| 42 | @@ -143,11 +144,16 @@ |
| 43 | p_sys->es = NULL; |
| 44 | } |
| 45 | |
| 46 | - int i_width = p_client->width; |
| 47 | - int i_height = p_client->height; |
| 48 | - int i_depth = p_client->format.bitsPerPixel; |
| 49 | + assert(!(p_client->width & ~0xffff)); // fits in 16 bits |
| 50 | + uint16_t i_width = p_client->width; |
| 51 | |
| 52 | - switch( i_depth ) |
| 53 | + assert(!(p_client->height & ~0xffff)); // fits in 16 bits |
| 54 | + uint16_t i_height = p_client->height; |
| 55 | + |
| 56 | + uint8_t i_bits_per_pixel = p_client->format.bitsPerPixel; |
| 57 | + assert((i_bits_per_pixel & 0x7) == 0); // multiple of 8 |
| 58 | + |
| 59 | + switch( i_bits_per_pixel ) |
| 60 | { |
| 61 | case 8: |
| 62 | i_chroma = VLC_CODEC_RGB8; |
| 63 | @@ -180,7 +186,10 @@ |
| 64 | } |
| 65 | |
| 66 | /* Set up framebuffer */ |
| 67 | - p_sys->i_framebuffersize = i_width * i_height * i_depth / 8; |
| 68 | + if (mul_overflow(i_width, i_height * (i_bits_per_pixel / 8), &p_sys->i_framebuffersize)) { |
| 69 | + msg_Err(p_demux, "VNC framebuffersize overflow"); |
| 70 | + return FALSE; |
| 71 | + } |
| 72 | |
| 73 | /* Reuse unsent block */ |
| 74 | if ( p_sys->p_block ) |
| 75 | @@ -211,7 +220,7 @@ |
| 76 | fmt.video.i_frame_rate_base = 1000; |
| 77 | fmt.video.i_frame_rate = 1000 * p_sys->f_fps; |
| 78 | |
| 79 | - fmt.video.i_bits_per_pixel = i_depth; |
| 80 | + fmt.video.i_bits_per_pixel = i_bits_per_pixel; |
| 81 | fmt.video.i_rmask = p_client->format.redMax << p_client->format.redShift; |
| 82 | fmt.video.i_gmask = p_client->format.greenMax << p_client->format.greenShift; |
| 83 | fmt.video.i_bmask = p_client->format.blueMax << p_client->format.blueShift; |