Patrick Williams | b48b7b4 | 2016-08-17 15:04:38 -0500 | [diff] [blame^] | 1 | gst-ffmpeg: h264: set parameters from SPS whenever it changes |
| 2 | |
| 3 | Fixes a crash in the fuzzed sample sample_varPAR.avi_s26638 with |
| 4 | alternating bit depths. |
| 5 | |
| 6 | Upstream-Status: Backport |
| 7 | |
| 8 | Signed-off-by: Yue Tao <yue.tao@windriver.com> |
| 9 | |
| 10 | diff --git a/gst-libs/ext/libav/libavcodec/h264.c.old b/gst-libs/ext/libav/libavcodec/h264.c |
| 11 | index 3621f41..718906a 100644 |
| 12 | --- a/gst-libs/ext/libav/libavcodec/h264.c.old |
| 13 | +++ b/gst-libs/ext/libav/libavcodec/h264.c |
| 14 | @@ -2491,6 +2491,34 @@ int ff_h264_get_profile(SPS *sps) |
| 15 | return profile; |
| 16 | } |
| 17 | |
| 18 | +static int h264_set_parameter_from_sps(H264Context *h) |
| 19 | +{ |
| 20 | + MpegEncContext *s = &h->s; |
| 21 | + AVCodecContext * avctx= s->avctx; |
| 22 | + |
| 23 | + if (s->flags& CODEC_FLAG_LOW_DELAY || |
| 24 | + (h->sps.bitstream_restriction_flag && !h->sps.num_reorder_frames)) |
| 25 | + s->low_delay=1; |
| 26 | + |
| 27 | + if(avctx->has_b_frames < 2) |
| 28 | + avctx->has_b_frames= !s->low_delay; |
| 29 | + |
| 30 | + if (avctx->bits_per_raw_sample != h->sps.bit_depth_luma) { |
| 31 | + if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 10) { |
| 32 | + avctx->bits_per_raw_sample = h->sps.bit_depth_luma; |
| 33 | + h->pixel_shift = h->sps.bit_depth_luma > 8; |
| 34 | + |
| 35 | + ff_h264dsp_init(&h->h264dsp, h->sps.bit_depth_luma); |
| 36 | + ff_h264_pred_init(&h->hpc, s->codec_id, h->sps.bit_depth_luma); |
| 37 | + dsputil_init(&s->dsp, s->avctx); |
| 38 | + } else { |
| 39 | + av_log(avctx, AV_LOG_DEBUG, "Unsupported bit depth: %d\n", h->sps.bit_depth_luma); |
| 40 | + return -1; |
| 41 | + } |
| 42 | + } |
| 43 | + return 0; |
| 44 | +} |
| 45 | + |
| 46 | /** |
| 47 | * decodes a slice header. |
| 48 | * This will also call MPV_common_init() and frame_start() as needed. |
| 49 | @@ -2505,7 +2533,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ |
| 50 | MpegEncContext * const s0 = &h0->s; |
| 51 | unsigned int first_mb_in_slice; |
| 52 | unsigned int pps_id; |
| 53 | - int num_ref_idx_active_override_flag; |
| 54 | + int num_ref_idx_active_override_flag, ret; |
| 55 | unsigned int slice_type, tmp, i, j; |
| 56 | int default_ref_list_done = 0; |
| 57 | int last_pic_structure; |
| 58 | @@ -2569,7 +2597,17 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ |
| 59 | av_log(h->s.avctx, AV_LOG_ERROR, "non-existing SPS %u referenced\n", h->pps.sps_id); |
| 60 | return -1; |
| 61 | } |
| 62 | - h->sps = *h0->sps_buffers[h->pps.sps_id]; |
| 63 | + |
| 64 | + if (h->pps.sps_id != h->current_sps_id || |
| 65 | + h0->sps_buffers[h->pps.sps_id]->new) { |
| 66 | + h0->sps_buffers[h->pps.sps_id]->new = 0; |
| 67 | + |
| 68 | + h->current_sps_id = h->pps.sps_id; |
| 69 | + h->sps = *h0->sps_buffers[h->pps.sps_id]; |
| 70 | + |
| 71 | + if ((ret = h264_set_parameter_from_sps(h)) < 0) |
| 72 | + return ret; |
| 73 | + } |
| 74 | |
| 75 | s->avctx->profile = ff_h264_get_profile(&h->sps); |
| 76 | s->avctx->level = h->sps.level_idc; |
| 77 | @@ -3811,26 +3811,8 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){ |
| 78 | case NAL_SPS: |
| 79 | init_get_bits(&s->gb, ptr, bit_length); |
| 80 | ff_h264_decode_seq_parameter_set(h); |
| 81 | - |
| 82 | - if (s->flags& CODEC_FLAG_LOW_DELAY || |
| 83 | - (h->sps.bitstream_restriction_flag && !h->sps.num_reorder_frames)) |
| 84 | - s->low_delay=1; |
| 85 | - |
| 86 | - if(avctx->has_b_frames < 2) |
| 87 | - avctx->has_b_frames= !s->low_delay; |
| 88 | - |
| 89 | - if (avctx->bits_per_raw_sample != h->sps.bit_depth_luma) { |
| 90 | - if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 10) { |
| 91 | - avctx->bits_per_raw_sample = h->sps.bit_depth_luma; |
| 92 | - h->pixel_shift = h->sps.bit_depth_luma > 8; |
| 93 | - |
| 94 | - ff_h264dsp_init(&h->h264dsp, h->sps.bit_depth_luma); |
| 95 | - ff_h264_pred_init(&h->hpc, s->codec_id, h->sps.bit_depth_luma); |
| 96 | - dsputil_init(&s->dsp, s->avctx); |
| 97 | - } else { |
| 98 | - av_log(avctx, AV_LOG_DEBUG, "Unsupported bit depth: %d\n", h->sps.bit_depth_luma); |
| 99 | - return -1; |
| 100 | - } |
| 101 | + if (h264_set_parameter_from_sps(h) < 0) { |
| 102 | + return -1; |
| 103 | } |
| 104 | break; |
| 105 | case NAL_PPS: |
| 106 | diff --git a/gst-libs/ext/libav/libavcodec/h264.h.old b/gst-libs/ext/libav/libavcodec/h264.h |
| 107 | index e3cc815..b77ad98 100644 |
| 108 | --- a/gst-libs/ext/libav/libavcodec/h264.h.old |
| 109 | +++ b/gst-libs/ext/libav/libavcodec/h264.h |
| 110 | @@ -202,6 +202,7 @@ typedef struct SPS{ |
| 111 | int bit_depth_chroma; ///< bit_depth_chroma_minus8 + 8 |
| 112 | int residual_color_transform_flag; ///< residual_colour_transform_flag |
| 113 | int constraint_set_flags; ///< constraint_set[0-3]_flag |
| 114 | + int new; ///< flag to keep track if the decoder context needs re-init due to changed SPS |
| 115 | }SPS; |
| 116 | |
| 117 | /** |
| 118 | @@ -333,6 +334,7 @@ typedef struct H264Context{ |
| 119 | int emu_edge_width; |
| 120 | int emu_edge_height; |
| 121 | |
| 122 | + unsigned current_sps_id; ///< id of the current SPS |
| 123 | SPS sps; ///< current sps |
| 124 | |
| 125 | /** |
| 126 | diff --git a/gst-libs/ext/libav/libavcodec/h264_ps.c.old b/gst-libs/ext/libav/libavcodec/h264_ps.c |
| 127 | index 7491807..0929098 100644 |
| 128 | --- a/gst-libs/ext/libav/libavcodec/h264_ps.c.old |
| 129 | +++ b/gst-libs/ext/libav/libavcodec/h264_ps.c |
| 130 | @@ -438,10 +438,13 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){ |
| 131 | sps->timing_info_present_flag ? sps->time_scale : 0 |
| 132 | ); |
| 133 | } |
| 134 | + sps->new = 1; |
| 135 | |
| 136 | av_free(h->sps_buffers[sps_id]); |
| 137 | - h->sps_buffers[sps_id]= sps; |
| 138 | - h->sps = *sps; |
| 139 | + h->sps_buffers[sps_id] = sps; |
| 140 | + h->sps = *sps; |
| 141 | + h->current_sps_id = sps_id; |
| 142 | + |
| 143 | return 0; |
| 144 | fail: |
| 145 | av_free(sps); |