fixed a crash when R10k % 64 != 0

fixed crashing:

    uv -t testcard:codec=R10k:size=3600x2160 -d dummy -c \
        libavcode:encoder=libx265:disable_intra_refresh --param \
        force-lavd-decoder=hevc_qsv,decoder-use-codec=R10k

The problem was that 3600 width rounds up to 3648 pixel for which there
was not allocated enough space (even with MAX_PADDING, which doesn't
protect against such cases).

vc_get_size() should be used instead of vc_get_linesize in all this
cases.
This commit is contained in:
Martin Pulec
2023-01-31 11:19:08 +01:00
parent 144fa13ff4
commit 9e1df595e1
2 changed files with 14 additions and 8 deletions

View File

@@ -2342,14 +2342,14 @@ void av_to_uv_convert(av_to_uv_convert_t *state, char * __restrict dst_buffer, A
return;
}
src_linesize = vc_get_linesize(width, priv->src_pixfmt);
dec_input = tmp = malloc(vc_get_datalen(width, height, priv->src_pixfmt));
dec_input = tmp = malloc(vc_get_datalen(width, height, priv->src_pixfmt) + MAX_PADDING);
int default_rgb_shift[] = { DEFAULT_R_SHIFT, DEFAULT_G_SHIFT, DEFAULT_B_SHIFT };
priv->convert((char *) dec_input, in_frame, width, height, src_linesize, default_rgb_shift);
priv->convert((char *) dec_input, in_frame, width, height, vc_get_size(width, priv->src_pixfmt), default_rgb_shift);
}
if (priv->dec) {
int dst_linesize = vc_get_linesize(width, priv->dst_pixfmt);
int dst_size = vc_get_size(width, priv->dst_pixfmt);
for (ptrdiff_t i = 0; i < height; ++i) {
priv->dec((unsigned char *) dst_buffer + i * pitch, dec_input + i * src_linesize, dst_linesize, rgb_shift[0], rgb_shift[1], rgb_shift[2]);
priv->dec((unsigned char *) dst_buffer + i * pitch, dec_input + i * src_linesize, dst_size, rgb_shift[0], rgb_shift[1], rgb_shift[2]);
}
free(tmp);
return;

View File

@@ -480,7 +480,11 @@ bool codec_is_hw_accelerated(codec_t codec) {
return codec == HW_VDPAU;
}
/** @brief Returns aligned linesize according to pixelformat specification (in bytes) */
/**
* @returns aligned linesize according to pixelformat specification (in bytes)
*
* @sa vc_get_size that should be used eg. as decoder_t linesize parameter instead
* */
int vc_get_linesize(unsigned int width, codec_t codec)
{
if (codec >= sizeof codec_info / sizeof(struct codec_info_t)) {
@@ -498,9 +502,11 @@ int vc_get_linesize(unsigned int width, codec_t codec)
}
/**
* Returns size of "width" pixels in codec _excluding_ padding.
* This is most likely only distinctive for vc_get_linesize for v210,
* eg. for width=1 that function returns 128, while this function 16.
* @returns size of "width" pixels in codec _excluding_ line padding
*
* This differs from vc_get_linesize for v210, eg. for width=1 that function
* returns 128, while this function 16. Also for R10k, lines are aligned to
* 256 B (64 pixels), while single pixel is only 4 B.
*/
int vc_get_size(unsigned int width, codec_t codec)
{