From c454f3a329c68ba24ff08ef84fcf4bd7f0eff759 Mon Sep 17 00:00:00 2001 From: Martin Pulec Date: Thu, 27 Aug 2020 16:21:03 +0200 Subject: [PATCH] Decompress multiple tile: fixes - fixed decompress deinit when one init fails - fixed a crash when number of substream changes - destroyed the new number of substreams but old was allocated --- src/rtp/video_decoders.cpp | 33 ++++++++++++++------------------- src/video_decompress.cpp | 9 +++++---- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/rtp/video_decoders.cpp b/src/rtp/video_decoders.cpp index caad47fdd..d28b3401e 100644 --- a/src/rtp/video_decoders.cpp +++ b/src/rtp/video_decoders.cpp @@ -362,7 +362,7 @@ struct state_video_decoder enum decoder_type_t decoder_type = {}; ///< how will the video data be decoded struct line_decoder *line_decoder = NULL; ///< if the video is uncompressed and only pixelformat change ///< is neeeded, use this structure - struct state_decompress **decompress_state = NULL; ///< state of the decompress (for every substream) + vector decompress_state; ///< state of the decompress (for every substream) bool accepts_corrupted_frame = false; ///< whether we should pass corrupted frame to decompress bool buffer_swapped = true; /**< variable indicating that display buffer * has been processed and we can write to a new one */ @@ -600,7 +600,7 @@ static void *decompress_worker(void *data) if (!d->compressed->tiles[d->pos].data) return NULL; - d->ret = decompress_frame(decoder->decompress_state[d->pos], + d->ret = decompress_frame(decoder->decompress_state.at(d->pos), (unsigned char *) d->out, (unsigned char *) d->compressed->tiles[d->pos].data, d->compressed->tiles[d->pos].data_len, @@ -940,13 +940,10 @@ void video_decoder_remove_display(struct state_video_decoder *decoder) static void cleanup(struct state_video_decoder *decoder) { decoder->decoder_type = UNSET; - if(decoder->decompress_state) { - for(unsigned int i = 0; i < decoder->max_substreams; ++i) { - decompress_done(decoder->decompress_state[i]); - } - free(decoder->decompress_state); - decoder->decompress_state = NULL; + for (auto &d : decoder->decompress_state) { + decompress_done(d); } + decoder->decompress_state.clear(); if(decoder->line_decoder) { free(decoder->line_decoder); decoder->line_decoder = NULL; @@ -1109,14 +1106,13 @@ after_linedecoder_lookup: /* we didn't find line decoder. So try now regular (aka DXT) decoder */ if(*decode_line == NULL) { - decoder->decompress_state = (struct state_decompress **) - calloc(decoder->max_substreams, sizeof(struct state_decompress *)); + decoder->decompress_state.resize(decoder->max_substreams); // try to probe video format if (comp_int_fmt == VIDEO_CODEC_NONE && decoder->out_codec != VIDEO_CODEC_END) { bool supports_autodetection = decompress_init_multi(desc.color_spec, - VIDEO_CODEC_NONE, VIDEO_CODEC_NONE, decoder->decompress_state, - decoder->max_substreams); + VIDEO_CODEC_NONE, VIDEO_CODEC_NONE, decoder->decompress_state.data(), + decoder->decompress_state.size()); if (supports_autodetection) { decoder->decoder_type = EXTERNAL_DECODER; return VIDEO_CODEC_END; @@ -1131,14 +1127,13 @@ after_linedecoder_lookup: out_codec = (*it).second; if (decompress_init_multi(desc.color_spec, (*it).first, (*it).second, - decoder->decompress_state, - decoder->max_substreams)) { + decoder->decompress_state.data(), + decoder->decompress_state.size())) { decoder->decoder_type = EXTERNAL_DECODER; goto after_decoder_lookup; } } - free(decoder->decompress_state); - decoder->decompress_state = 0; + decoder->decompress_state.clear(); } after_decoder_lookup: @@ -1370,8 +1365,8 @@ static bool reconfigure_decoder(struct state_video_decoder *decoder, } else if (decoder->decoder_type == EXTERNAL_DECODER) { int buf_size; - for(unsigned int i = 0; i < decoder->max_substreams; ++i) { - buf_size = decompress_reconfigure(decoder->decompress_state[i], desc, + for(unsigned int i = 0; i < decoder->decompress_state.size(); ++i) { + buf_size = decompress_reconfigure(decoder->decompress_state.at(i), desc, display_requested_rgb_shift[0], display_requested_rgb_shift[1], display_requested_rgb_shift[2], @@ -1384,7 +1379,7 @@ static bool reconfigure_decoder(struct state_video_decoder *decoder, decoder->merged_fb = display_mode != DISPLAY_PROPERTY_VIDEO_SEPARATE_TILES; int res = 0, ret; size_t size = sizeof(res); - ret = decompress_get_property(decoder->decompress_state[0], + ret = decompress_get_property(decoder->decompress_state.at(0), DECOMPRESS_PROPERTY_ACCEPTS_CORRUPTED_FRAME, &res, &size); decoder->accepts_corrupted_frame = ret && res; diff --git a/src/video_decompress.cpp b/src/video_decompress.cpp index 4ae9288ab..e79a4a7c9 100644 --- a/src/video_decompress.cpp +++ b/src/video_decompress.cpp @@ -160,10 +160,11 @@ static bool try_initialize_decompress(const video_decompress_info *vdi, decompress_state[i] = decompress_init(vdi); if (!decompress_state[i]) { - for(int j = 0; j < substreams; ++j) { - if (decompress_state[i] != NULL) - decompress_done(decompress_state[i]); - decompress_state[i] = NULL; + for(int j = 0; j < i; ++j) { + if (decompress_state[j] != nullptr) { + decompress_done(decompress_state[j]); + } + decompress_state[j] = nullptr; } return false; }