From d9dfd8c1ff8ae76de28acceb954d35c27f906923 Mon Sep 17 00:00:00 2001 From: Martin Piatka <445597@mail.muni.cz> Date: Wed, 3 May 2017 13:20:13 +0200 Subject: [PATCH] Support for VDPAU & VAAPI hw accelerated decoding --- configure.ac | 30 ++ src/video_decompress/libavcodec.c | 443 +++++++++++++++++++++++++++++- 2 files changed, 469 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 47219325f..4a1d6eda8 100644 --- a/configure.ac +++ b/configure.ac @@ -2688,6 +2688,35 @@ if test $video_mix_req = yes -a $video_mix = no; then AC_MSG_ERROR([Could not found OpenCV needed for video mixer!]); fi +# ------------------------------------------------------------------------------------------------- +# libavcodec hw-accelerated decoding +# ------------------------------------------------------------------------------------------------- +lavc_hwdec=no + +define(lavc_hwdec_dep, libavutil >= 55.22.1 libva) + +AC_ARG_ENABLE(lavc-hw-decoding, +[ --disable-lavc-hw-decoding disable lavc-hw-decoding (default is auto)] +[ Requires: lavc_hwdec_dep], + [lavc_hwdec_req=$enableval], + [lavc_hwdec_req=auto] + ) + +PKG_CHECK_MODULES([LAVC_HWDEC], [lavc_hwdec_dep], [FOUND_HWDEC_DEP=yes], [FOUND_HWDEC_DEP=no]) + +if test $lavc_hwdec_req != no -a $FOUND_HWDEC_DEP = yes +then + CFLAGS="$CFLAGS -DUSE_HWDEC ${LAVC_HWDEC_CFLAGS}" + CXXFLAGS="$CXXFLAGS -DUSE_HWDEC ${LAVC_HWDEC_CFLAGS}" + LIBS="$LIBS ${LAVC_HWDEC_LIBS}" + lavc_hwdec=yes +fi + +if test $lavc_hwdec_req = yes -a $lavc_hwdec = no; then + AC_MSG_ERROR([Could not find hwdec dependencies!]); +fi + + # ------------------------------------------------------------------------------------------------- # We need to add libraries then # ------------------------------------------------------------------------------------------------- @@ -2767,6 +2796,7 @@ RESULT=\ testcard extras ............. $testcard_extras_req GPU accelerated LDGM ........ $ldgm_gpu MCU-like video mixer ........ $video_mix + Libavcodec hw decoding ...... $lavc_hwdec " AC_MSG_RESULT( diff --git a/src/video_decompress/libavcodec.c b/src/video_decompress/libavcodec.c index 5bc9643a0..11e33b1ad 100644 --- a/src/video_decompress/libavcodec.c +++ b/src/video_decompress/libavcodec.c @@ -51,6 +51,15 @@ #include "video.h" #include "video_decompress.h" +#ifdef USE_HWDEC +#include +#include +#include +#include +#include +#define DEFAULT_SURFACES 20 +#endif + #ifdef __cplusplus #include using std::max; @@ -64,6 +73,23 @@ using std::min; #define MOD_NAME "[lavd] " +#ifdef USE_HWDEC +struct hw_accel_state { + enum { + HWACCEL_NONE, + HWACCEL_VDPAU, + HWACCEL_VAAPI + } type; + + bool copy; + AVFrame *tmp_frame; + + void (*uninit)(struct hw_accel_state*); + + void *ctx; //Type depends on hwaccel type +}; +#endif + struct state_libavcodec_decompress { pthread_mutex_t *global_lavcd_lock; AVCodecContext *codec_ctx; @@ -83,6 +109,10 @@ struct state_libavcodec_decompress { struct video_desc saved_desc; unsigned int broken_h264_mt_decoding_workaroud_warning_displayed; bool broken_h264_mt_decoding_workaroud_active; + +#ifdef USE_HWDEC + struct hw_accel_state hwaccel; +#endif }; static int change_pixfmt(AVFrame *frame, unsigned char *dst, int av_codec, @@ -92,6 +122,29 @@ static enum AVPixelFormat get_format_callback(struct AVCodecContext *s, const en static bool broken_h264_mt_decoding = false; +#ifdef USE_HWDEC +static void hwaccel_state_init(struct hw_accel_state *hwaccel){ + hwaccel->type = HWACCEL_NONE; + hwaccel->copy = false; + hwaccel->uninit = NULL; + hwaccel->tmp_frame = NULL; + hwaccel->uninit = NULL; + hwaccel->ctx = NULL; +} + +static void hwaccel_state_reset(struct hw_accel_state *hwaccel){ + if(hwaccel->ctx){ + hwaccel->uninit(hwaccel); + } + + if(hwaccel->tmp_frame){ + av_frame_free(&hwaccel->tmp_frame); + } + + hwaccel_state_init(hwaccel); +} +#endif + static void deconfigure(struct state_libavcodec_decompress *s) { #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 37, 100) @@ -122,6 +175,10 @@ static void deconfigure(struct state_libavcodec_decompress *s) av_free(s->frame); s->frame = NULL; av_packet_unref(&s->pkt); + +#ifdef USE_HWDEC + hwaccel_state_reset(&s->hwaccel); +#endif } static void set_codec_context_params(struct state_libavcodec_decompress *s) @@ -153,6 +210,8 @@ static void set_codec_context_params(struct state_libavcodec_decompress *s) s->codec_ctx->pix_fmt = AV_PIX_FMT_NONE; // callback to negotiate pixel format that is supported by UG s->codec_ctx->get_format = get_format_callback; + + s->codec_ctx->opaque = s; } static void jpeg_callback(void) @@ -189,6 +248,11 @@ static const struct decoder_info decoders[] = { ADD_TO_PARAM(force_lavd_decoder, "force-lavd-decoder", "* force-lavd-decoder=[:...]\n" " Forces specified Libavcodec decoder. If more need to be specified, use colon as a delimiter\n"); + +#ifdef USE_HWDEC +ADD_TO_PARAM(force_hw_accel, "use-hw-accel", "* use-hw-accel\n" + " Tries to use hardware acceleration. \n"); +#endif static bool configure_with(struct state_libavcodec_decompress *s, struct video_desc desc) { @@ -303,7 +367,7 @@ static bool configure_with(struct state_libavcodec_decompress *s, static void * libavcodec_decompress_init(void) { struct state_libavcodec_decompress *s; - + s = (struct state_libavcodec_decompress *) calloc(1, sizeof(struct state_libavcodec_decompress)); @@ -325,6 +389,10 @@ static void * libavcodec_decompress_init(void) av_log_set_callback(error_callback); +#ifdef USE_HWDEC + hwaccel_state_init(&s->hwaccel); +#endif + return s; } @@ -333,7 +401,7 @@ static int libavcodec_decompress_reconfigure(void *state, struct video_desc desc { struct state_libavcodec_decompress *s = (struct state_libavcodec_decompress *) state; - + s->pitch = pitch; assert(out_codec == UYVY || out_codec == RGB || out_codec == v210); @@ -1019,8 +1087,328 @@ static const struct { {AV_PIX_FMT_RGB24, RGB, rgb24_to_rgb}, }; +#ifdef USE_HWDEC +static int create_hw_device_ctx(enum AVHWDeviceType type, AVBufferRef **device_ref){ + int ret; + ret = av_hwdevice_ctx_create(device_ref, type, NULL, NULL, 0); + + if(ret < 0){ + log_msg(LOG_LEVEL_ERROR, "[lavd] Unable to create hwdevice!!\n"); + return ret; + } + + return 0; +} + +static int create_hw_frame_ctx(AVBufferRef *device_ref, + AVCodecContext *s, + enum AVPixelFormat format, + enum AVPixelFormat sw_format, + int decode_surfaces, + AVBufferRef **ctx) +{ + *ctx = av_hwframe_ctx_alloc(device_ref); + if(!*ctx){ + log_msg(LOG_LEVEL_ERROR, "[lavd] Failed to allocate hwframe_ctx!!\n"); + return -1; + } + + AVHWFramesContext *frames_ctx = (AVHWFramesContext *) (*ctx)->data; + frames_ctx->format = format; + frames_ctx->width = s->coded_width; + frames_ctx->height = s->coded_height; + frames_ctx->sw_format = sw_format; + frames_ctx->initial_pool_size = decode_surfaces; + + int ret = av_hwframe_ctx_init(*ctx); + if (ret < 0) { + av_buffer_unref(ctx); + *ctx = NULL; + log_msg(LOG_LEVEL_ERROR, "[lavd] Unable to init hwframe_ctx!!\n\n"); + return ret; + } + + return 0; +} + +static int vdpau_init(struct AVCodecContext *s){ + + struct state_libavcodec_decompress *state = s->opaque; + + AVBufferRef *device_ref = NULL; + int ret = create_hw_device_ctx(AV_HWDEVICE_TYPE_VDPAU, &device_ref); + if(ret < 0) + return ret; + + AVHWDeviceContext *device_ctx = (AVHWDeviceContext*)device_ref->data; + AVVDPAUDeviceContext *device_vdpau_ctx = device_ctx->hwctx; + + AVBufferRef *hw_frames_ctx = NULL; + ret = create_hw_frame_ctx(device_ref, + s, + AV_PIX_FMT_VDPAU, + s->sw_pix_fmt, + DEFAULT_SURFACES, + &hw_frames_ctx); + if(ret < 0) + goto fail; + + s->hw_frames_ctx = hw_frames_ctx; + + state->hwaccel.type = HWACCEL_VDPAU; + state->hwaccel.copy = true; + state->hwaccel.tmp_frame = av_frame_alloc(); + if(!state->hwaccel.tmp_frame){ + ret = -1; + goto fail; + } + + if(av_vdpau_bind_context(s, device_vdpau_ctx->device, device_vdpau_ctx->get_proc_address, + AV_HWACCEL_FLAG_ALLOW_HIGH_DEPTH | + AV_HWACCEL_FLAG_IGNORE_LEVEL)){ + log_msg(LOG_LEVEL_ERROR, "[lavd] Unable to bind vdpau context!\n"); + ret = -1; + goto fail; + } + + av_buffer_unref(&device_ref); + return 0; + +fail: + av_frame_free(&state->hwaccel.tmp_frame); + av_buffer_unref(&hw_frames_ctx); + av_buffer_unref(&device_ref); + return ret; +} + +struct vaapi_ctx{ + AVBufferRef *device_ref; + AVHWDeviceContext *device_ctx; + AVVAAPIDeviceContext *device_vaapi_ctx; + + AVBufferRef *hw_frames_ctx; + AVHWFramesContext *frame_ctx; + +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 74, 100) + VAProfile va_profile; + VAEntrypoint va_entrypoint; + VAConfigID va_config; + VAContextID va_context; + + struct vaapi_context decoder_context; +#endif +}; + +static void vaapi_uninit(struct hw_accel_state *s){ + + free(s->ctx); +} + +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 74, 100) +static const struct { + enum AVCodecID av_codec_id; + int codec_profile; + VAProfile va_profile; +} vaapi_profiles[] = { + {AV_CODEC_ID_MPEG2VIDEO, FF_PROFILE_MPEG2_SIMPLE, VAProfileMPEG2Simple}, + {AV_CODEC_ID_MPEG2VIDEO, FF_PROFILE_MPEG2_MAIN, VAProfileMPEG2Main}, + {AV_CODEC_ID_H264, FF_PROFILE_H264_CONSTRAINED_BASELINE, VAProfileH264ConstrainedBaseline}, + {AV_CODEC_ID_H264, FF_PROFILE_H264_BASELINE, VAProfileH264Baseline}, + {AV_CODEC_ID_H264, FF_PROFILE_H264_MAIN, VAProfileH264Main}, + {AV_CODEC_ID_H264, FF_PROFILE_H264_HIGH, VAProfileH264High}, +#if VA_CHECK_VERSION(0, 37, 0) + {AV_CODEC_ID_HEVC, FF_PROFILE_HEVC_MAIN, VAProfileHEVCMain}, +#endif +}; + +static int vaapi_create_context(struct vaapi_ctx *ctx, + AVCodecContext *codec_ctx) +{ + const AVCodecDescriptor *codec_desc; + + codec_desc = avcodec_descriptor_get(codec_ctx->codec_id); + if(!codec_desc){ + return -1; + } + + int profile_count = vaMaxNumProfiles(ctx->device_vaapi_ctx->display); + log_msg(LOG_LEVEL_VERBOSE, "VAAPI Profile count: %d\n", profile_count); + + VAProfile *list = av_malloc(profile_count * sizeof(VAProfile)); + if(!list){ + return -1; + } + + VAStatus status = vaQueryConfigProfiles(ctx->device_vaapi_ctx->display, + list, &profile_count); + if(status != VA_STATUS_SUCCESS){ + log_msg(LOG_LEVEL_ERROR, "[lavd] Profile query failed: %d (%s)\n", status, vaErrorStr(status)); + av_free(list); + return -1; + } + + VAProfile profile = VAProfileNone; + int match = 0; + + for(unsigned i = 0; i < FF_ARRAY_ELEMS(vaapi_profiles); i++){ + if(vaapi_profiles[i].av_codec_id != codec_ctx->codec_id) + continue; + + if(vaapi_profiles[i].codec_profile == codec_ctx->profile){ + profile = vaapi_profiles[i].va_profile; + break; + } + } + + for(int i = 0; i < profile_count; i++){ + if(profile == list[i]) + match = 1; + } + + av_freep(&list); + + if(!match){ + log_msg(LOG_LEVEL_ERROR, "[lavd] Profile not supported \n"); + return -1; + } + + ctx->va_profile = profile; + ctx->va_entrypoint = VAEntrypointVLD; + + status = vaCreateConfig(ctx->device_vaapi_ctx->display, ctx->va_profile, + ctx->va_entrypoint, 0, 0, &ctx->va_config); + if(status != VA_STATUS_SUCCESS){ + log_msg(LOG_LEVEL_ERROR, "[lavd] Create config failed: %d (%s)\n", status, vaErrorStr(status)); + return -1; + } + + AVVAAPIHWConfig *hwconfig = av_hwdevice_hwconfig_alloc(ctx->device_ref); + if(!hwconfig){ + log_msg(LOG_LEVEL_WARNING, "[lavd] Failed to get constraints. Will try to continue anyways...\n"); + return 0; + } + + hwconfig->config_id = ctx->va_config; + AVHWFramesConstraints *constraints = av_hwdevice_get_hwframe_constraints(ctx->device_ref, hwconfig); + if (!constraints){ + log_msg(LOG_LEVEL_WARNING, "[lavd] Failed to get constraints. Will try to continue anyways...\n"); + av_freep(&hwconfig); + return 0; + } + + if (codec_ctx->coded_width < constraints->min_width || + codec_ctx->coded_width > constraints->max_width || + codec_ctx->coded_height < constraints->min_height || + codec_ctx->coded_height > constraints->max_height) + { + log_msg(LOG_LEVEL_WARNING, "[lavd] VAAPI hw does not support the resolution %dx%d\n", + codec_ctx->coded_width, + codec_ctx->coded_height); + av_hwframe_constraints_free(&constraints); + av_freep(&hwconfig); + return -1; + } + + av_hwframe_constraints_free(&constraints); + av_freep(&hwconfig); + + return 0; +} +#endif + +static int vaapi_init(struct AVCodecContext *s){ + + struct state_libavcodec_decompress *state = s->opaque; + + struct vaapi_ctx *ctx = calloc(1, sizeof(struct vaapi_ctx)); + if(!ctx){ + return -1; + } + + int ret = create_hw_device_ctx(AV_HWDEVICE_TYPE_VAAPI, &ctx->device_ref); + if(ret < 0) + goto fail; + + ctx->device_ctx = (AVHWDeviceContext*)ctx->device_ref->data; + ctx->device_vaapi_ctx = ctx->device_ctx->hwctx; + +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 74, 100) + ret = vaapi_create_context(ctx, s); + if(ret < 0) + goto fail; +#endif + + int decode_surfaces = DEFAULT_SURFACES; + + if (s->active_thread_type & FF_THREAD_FRAME) + decode_surfaces += s->thread_count; + + ret = create_hw_frame_ctx(ctx->device_ref, + s, + AV_PIX_FMT_VAAPI, + s->sw_pix_fmt, + decode_surfaces, + &ctx->hw_frames_ctx); + if(ret < 0) + goto fail; + + ctx->frame_ctx = (AVHWFramesContext *) (ctx->hw_frames_ctx->data); + + s->hw_frames_ctx = ctx->hw_frames_ctx; + + state->hwaccel.tmp_frame = av_frame_alloc(); + if(!state->hwaccel.tmp_frame){ + ret = -1; + goto fail; + } + state->hwaccel.type = HWACCEL_VAAPI; + state->hwaccel.copy = true; + state->hwaccel.ctx = ctx; + state->hwaccel.uninit = vaapi_uninit; + +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 74, 100) + AVVAAPIFramesContext *avfc = ctx->frame_ctx->hwctx; + VAStatus status = vaCreateContext(ctx->device_vaapi_ctx->display, + ctx->va_config, s->coded_width, s->coded_height, + VA_PROGRESSIVE, + avfc->surface_ids, + avfc->nb_surfaces, + &ctx->va_context); + + if(status != VA_STATUS_SUCCESS){ + log_msg(LOG_LEVEL_ERROR, "[lavd] Create config failed: %d (%s)\n", status, vaErrorStr(status)); + ret = -1; + goto fail; + } + + ctx->decoder_context.display = ctx->device_vaapi_ctx->display; + ctx->decoder_context.config_id = ctx->va_config; + ctx->decoder_context.context_id = ctx->va_context; + + s->hwaccel_context = &ctx->decoder_context; +#endif + + av_buffer_unref(&ctx->device_ref); + return 0; + + +fail: + av_frame_free(&state->hwaccel.tmp_frame); + av_buffer_unref(&ctx->hw_frames_ctx); +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 74, 100) + if(ctx->device_vaapi_ctx) + vaDestroyConfig(ctx->device_vaapi_ctx->display, ctx->va_config); +#endif + av_buffer_unref(&ctx->device_ref); + free(ctx); + return ret; +} +#endif + static enum AVPixelFormat get_format_callback(struct AVCodecContext *s __attribute__((unused)), const enum AVPixelFormat *fmt) { + struct state_libavcodec_decompress *state = (struct state_libavcodec_decompress *) s->opaque; + if (log_level >= LOG_LEVEL_DEBUG) { char out[1024] = "[lavd] Available output pixel formats:"; const enum AVPixelFormat *it = fmt; @@ -1031,6 +1419,37 @@ static enum AVPixelFormat get_format_callback(struct AVCodecContext *s __attribu log_msg(LOG_LEVEL_DEBUG, "%s\n", out); } + +#ifdef USE_HWDEC + hwaccel_state_reset(&state->hwaccel); + const char *param = get_commandline_param("use-hw-accel"); + bool hwaccel = param != NULL; + + if(hwaccel){ + for(const enum AVPixelFormat *it = fmt; *it != AV_PIX_FMT_NONE; it++){ + if (*it == AV_PIX_FMT_VDPAU){ + int ret = vdpau_init(s); + if(ret < 0){ + hwaccel_state_reset(&state->hwaccel); + continue; + } + return AV_PIX_FMT_VDPAU; + } + if (*it == AV_PIX_FMT_VAAPI){ + int ret = vaapi_init(s); + if(ret < 0){ + hwaccel_state_reset(&state->hwaccel); + continue; + } + return AV_PIX_FMT_VAAPI; + } + } + + log_msg(LOG_LEVEL_WARNING, "[lavd] Falling back to software decoding!\n"); + } + +#endif + while (*fmt != AV_PIX_FMT_NONE) { for (unsigned int i = 0; i < sizeof convert_funcs / sizeof convert_funcs[0]; ++i) { if (convert_funcs[i].av_codec == *fmt) { @@ -1091,6 +1510,17 @@ static void error_callback(void *ptr, int level, const char *fmt, va_list vl) { av_log_default_callback(ptr, level, fmt, vl); } +#ifdef USE_HWDEC +static void transfer_frame(struct hw_accel_state *s, AVFrame *frame){ + av_hwframe_transfer_data(s->tmp_frame, frame, 0); + + av_frame_copy_props(s->tmp_frame, frame); + + av_frame_unref(frame); + av_frame_move_ref(frame, s->tmp_frame); +} +#endif + static int libavcodec_decompress(void *state, unsigned char *dst, unsigned char *src, unsigned int src_len, int frame_seq) { @@ -1100,7 +1530,7 @@ static int libavcodec_decompress(void *state, unsigned char *dst, unsigned char s->pkt.size = src_len; s->pkt.data = src; - + while (s->pkt.size > 0) { struct timeval t0, t1; gettimeofday(&t0, NULL); @@ -1158,7 +1588,12 @@ static int libavcodec_decompress(void *state, unsigned char *dst, unsigned char s->last_frame_seq : -1, (unsigned) frame_seq); res = FALSE; } else { - res = change_pixfmt(s->frame, dst, s->codec_ctx->pix_fmt, +#ifdef USE_HWDEC + if(s->hwaccel.copy){ + transfer_frame(&s->hwaccel, s->frame); + } +#endif + res = change_pixfmt(s->frame, dst, s->frame->format, s->out_codec, s->width, s->height, s->pitch); if(res == TRUE) { s->last_frame_seq_initialized = true;