From 9e1df595e1746babfdfe0546bc507a4dca9bc220 Mon Sep 17 00:00:00 2001 From: Martin Pulec Date: Tue, 31 Jan 2023 11:19:08 +0100 Subject: [PATCH] fixed a crash when R10k % 64 != 0 fixed crashing: uv -t testcard:codec=R10k:size=3600x2160 -d dummy -c \ libavcode:encoder=libx265:disable_intra_refresh --param \ force-lavd-decoder=hevc_qsv,decoder-use-codec=R10k The problem was that 3600 width rounds up to 3648 pixel for which there was not allocated enough space (even with MAX_PADDING, which doesn't protect against such cases). vc_get_size() should be used instead of vc_get_linesize in all this cases. --- src/libavcodec/from_lavc_vid_conv.c | 8 ++++---- src/video_codec.c | 14 ++++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/libavcodec/from_lavc_vid_conv.c b/src/libavcodec/from_lavc_vid_conv.c index ec027aaae..28d9185c8 100644 --- a/src/libavcodec/from_lavc_vid_conv.c +++ b/src/libavcodec/from_lavc_vid_conv.c @@ -2342,14 +2342,14 @@ void av_to_uv_convert(av_to_uv_convert_t *state, char * __restrict dst_buffer, A return; } src_linesize = vc_get_linesize(width, priv->src_pixfmt); - dec_input = tmp = malloc(vc_get_datalen(width, height, priv->src_pixfmt)); + dec_input = tmp = malloc(vc_get_datalen(width, height, priv->src_pixfmt) + MAX_PADDING); int default_rgb_shift[] = { DEFAULT_R_SHIFT, DEFAULT_G_SHIFT, DEFAULT_B_SHIFT }; - priv->convert((char *) dec_input, in_frame, width, height, src_linesize, default_rgb_shift); + priv->convert((char *) dec_input, in_frame, width, height, vc_get_size(width, priv->src_pixfmt), default_rgb_shift); } if (priv->dec) { - int dst_linesize = vc_get_linesize(width, priv->dst_pixfmt); + int dst_size = vc_get_size(width, priv->dst_pixfmt); for (ptrdiff_t i = 0; i < height; ++i) { - priv->dec((unsigned char *) dst_buffer + i * pitch, dec_input + i * src_linesize, dst_linesize, rgb_shift[0], rgb_shift[1], rgb_shift[2]); + priv->dec((unsigned char *) dst_buffer + i * pitch, dec_input + i * src_linesize, dst_size, rgb_shift[0], rgb_shift[1], rgb_shift[2]); } free(tmp); return; diff --git a/src/video_codec.c b/src/video_codec.c index 3eddfb96d..cd1e674f0 100644 --- a/src/video_codec.c +++ b/src/video_codec.c @@ -480,7 +480,11 @@ bool codec_is_hw_accelerated(codec_t codec) { return codec == HW_VDPAU; } -/** @brief Returns aligned linesize according to pixelformat specification (in bytes) */ +/** + * @returns aligned linesize according to pixelformat specification (in bytes) + * + * @sa vc_get_size that should be used eg. as decoder_t linesize parameter instead + * */ int vc_get_linesize(unsigned int width, codec_t codec) { if (codec >= sizeof codec_info / sizeof(struct codec_info_t)) { @@ -498,9 +502,11 @@ int vc_get_linesize(unsigned int width, codec_t codec) } /** - * Returns size of "width" pixels in codec _excluding_ padding. - * This is most likely only distinctive for vc_get_linesize for v210, - * eg. for width=1 that function returns 128, while this function 16. + * @returns size of "width" pixels in codec _excluding_ line padding + * + * This differs from vc_get_linesize for v210, eg. for width=1 that function + * returns 128, while this function 16. Also for R10k, lines are aligned to + * 256 B (64 pixels), while single pixel is only 4 B. */ int vc_get_size(unsigned int width, codec_t codec) {