diff --git a/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h b/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h index e429bbca..0e95aefd 100644 --- a/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h +++ b/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h @@ -80,6 +80,7 @@ namespace cmvr::device { AVFormatContext* capture_format_context_ = nullptr; AVCodecContext* capture_decoder_context_ = nullptr; + AVBufferRef* capture_hw_device_context_ = nullptr; SwsContext* capture_sws_context_ = nullptr; int capture_video_stream_index_ = -1; std::atomic capture_running_{false}; diff --git a/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp b/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp index bad4499c..a789a79b 100644 --- a/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp +++ b/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp @@ -11,6 +11,7 @@ extern "C" { #include #include +#include } using namespace std; @@ -27,6 +28,17 @@ std::string ffmpeg_error_string(const int error_code) return error_buffer; } +AVPixelFormat select_qsv_pixel_format(AVCodecContext*, const AVPixelFormat* formats) +{ + for (const AVPixelFormat* format = formats; *format != AV_PIX_FMT_NONE; ++format) { + if (*format == AV_PIX_FMT_QSV) { + return *format; + } + } + CMVR_LOG(ERROR) << "[UVCCamera] mjpeg_qsv decoder did not offer QSV frames"; + return AV_PIX_FMT_NONE; +} + AVPixelFormat normalize_deprecated_yuv_format(const AVPixelFormat format) { switch (format) { @@ -394,9 +406,16 @@ bool UVCCamera::open_capture_() } AVStream* video_stream = capture_format_context_->streams[capture_video_stream_index_]; - const AVCodec* decoder = avcodec_find_decoder(video_stream->codecpar->codec_id); + if (video_stream->codecpar->codec_id != AV_CODEC_ID_MJPEG) { + const std::string error = "camera input is not MJPEG; QSV MJPEG decoding is unavailable"; + close_capture_(); + capture_error_ = error; + return false; + } + + const AVCodec* decoder = avcodec_find_decoder_by_name("mjpeg_qsv"); if (!decoder) { - const std::string error = "FFmpeg decoder is unavailable for camera input"; + const std::string error = "FFmpeg was built without the mjpeg_qsv decoder"; close_capture_(); capture_error_ = error; return false; @@ -411,12 +430,37 @@ bool UVCCamera::open_capture_() } result = avcodec_parameters_to_context(capture_decoder_context_, video_stream->codecpar); + if (result >= 0) { + AVBufferRef* vaapi_device_context = nullptr; + result = av_hwdevice_ctx_create( + &vaapi_device_context, + AV_HWDEVICE_TYPE_VAAPI, + "/dev/dri/renderD128", + nullptr, + 0); + if (result >= 0) { + result = av_hwdevice_ctx_create_derived( + &capture_hw_device_context_, + AV_HWDEVICE_TYPE_QSV, + vaapi_device_context, + 0); + } + av_buffer_unref(&vaapi_device_context); + } + if (result >= 0) { + capture_decoder_context_->hw_device_ctx = av_buffer_ref(capture_hw_device_context_); + if (!capture_decoder_context_->hw_device_ctx) { + result = AVERROR(ENOMEM); + } else { + capture_decoder_context_->get_format = select_qsv_pixel_format; + } + } if (result >= 0) { result = avcodec_open2(capture_decoder_context_, decoder, nullptr); } if (result < 0) { const std::string error = - "failed to open camera decoder: " + ffmpeg_error_string(result); + "failed to initialize mjpeg_qsv decoder: " + ffmpeg_error_string(result); close_capture_(); capture_error_ = error; return false; @@ -425,6 +469,8 @@ bool UVCCamera::open_capture_() CMVR_LOG(INFO) << "[UVCCamera] FFmpeg capture opened" << ", device=" << serial_ << ", input_codec=" << avcodec_get_name(video_stream->codecpar->codec_id) + << ", decoder=" << decoder->name + << ", hardware_device=qsv" << ", width=" << capture_decoder_context_->width << ", height=" << capture_decoder_context_->height << ", requested_fps=" << fps_; @@ -450,6 +496,7 @@ void UVCCamera::close_capture_() if (capture_decoder_context_) { avcodec_free_context(&capture_decoder_context_); } + av_buffer_unref(&capture_hw_device_context_); if (capture_format_context_) { avformat_close_input(&capture_format_context_); } @@ -460,8 +507,9 @@ void UVCCamera::capture_worker_() { AVPacket* input_packet = av_packet_alloc(); AVFrame* decoded_frame = av_frame_alloc(); + AVFrame* software_frame = av_frame_alloc(); uint64_t discarded_frame_count = 0; - if (!input_packet || !decoded_frame) { + if (!input_packet || !decoded_frame || !software_frame) { { std::lock_guard frame_lock(capture_frame_mutex_); capture_error_ = "failed to allocate FFmpeg capture frame resources"; @@ -470,6 +518,7 @@ void UVCCamera::capture_worker_() capture_frame_cv_.notify_all(); av_packet_free(&input_packet); av_frame_free(&decoded_frame); + av_frame_free(&software_frame); return; } @@ -545,14 +594,30 @@ void UVCCamera::capture_worker_() break; } - const auto decoded_format = static_cast(decoded_frame->format); + AVFrame* source_frame = decoded_frame; + if (decoded_frame->hw_frames_ctx) { + av_frame_unref(software_frame); + const int transfer_result = av_hwframe_transfer_data( + software_frame, decoded_frame, 0); + if (transfer_result < 0) { + std::lock_guard frame_lock(capture_frame_mutex_); + capture_error_ = "failed to download QSV camera frame: " + + ffmpeg_error_string(transfer_result); + capture_running_.store(false, std::memory_order_relaxed); + break; + } + av_frame_copy_props(software_frame, decoded_frame); + source_frame = software_frame; + } + + const auto decoded_format = static_cast(source_frame->format); const auto source_format = normalize_deprecated_yuv_format(decoded_format); - const bool source_full_range = decoded_frame->color_range == AVCOL_RANGE_JPEG + const bool source_full_range = source_frame->color_range == AVCOL_RANGE_JPEG || source_format != decoded_format; capture_sws_context_ = sws_getCachedContext( capture_sws_context_, - decoded_frame->width, - decoded_frame->height, + source_frame->width, + source_frame->height, source_format, width_, height_, @@ -569,7 +634,7 @@ void UVCCamera::capture_worker_() } const int* color_coefficients = sws_getCoefficients( - swscale_colorspace(decoded_frame->colorspace)); + swscale_colorspace(source_frame->colorspace)); const int color_result = sws_setColorspaceDetails( capture_sws_context_, color_coefficients, @@ -594,10 +659,10 @@ void UVCCamera::capture_worker_() }; const int converted_rows = sws_scale( capture_sws_context_, - decoded_frame->data, - decoded_frame->linesize, + source_frame->data, + source_frame->linesize, 0, - decoded_frame->height, + source_frame->height, destination_data, destination_linesize); if (converted_rows != height_) { @@ -627,6 +692,7 @@ void UVCCamera::capture_worker_() capture_frame_cv_.notify_all(); av_packet_free(&input_packet); av_frame_free(&decoded_frame); + av_frame_free(&software_frame); } bool UVCCamera::wait_for_capture_frame_(cv::Mat& frame,