feat(camera): decode UVC MJPEG with QSV

cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h

@@ -80,6 +80,7 @@ namespace cmvr::device {
 
         AVFormatContext* capture_format_context_ = nullptr;
         AVCodecContext* capture_decoder_context_ = nullptr;
+        AVBufferRef* capture_hw_device_context_ = nullptr;
         SwsContext* capture_sws_context_ = nullptr;
         int capture_video_stream_index_ = -1;
         std::atomic<bool> capture_running_{false};

cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp

@@ -11,6 +11,7 @@
 extern "C" {
 #include <libavdevice/avdevice.h>
 #include <libavutil/error.h>
+#include <libavutil/hwcontext.h>
 }
 
 using namespace std;
@@ -27,6 +28,17 @@ std::string ffmpeg_error_string(const int error_code)
     return error_buffer;
 }
 
+AVPixelFormat select_qsv_pixel_format(AVCodecContext*, const AVPixelFormat* formats)
+{
+    for (const AVPixelFormat* format = formats; *format != AV_PIX_FMT_NONE; ++format) {
+        if (*format == AV_PIX_FMT_QSV) {
+            return *format;
+        }
+    }
+    CMVR_LOG(ERROR) << "[UVCCamera] mjpeg_qsv decoder did not offer QSV frames";
+    return AV_PIX_FMT_NONE;
+}
+
 AVPixelFormat normalize_deprecated_yuv_format(const AVPixelFormat format)
 {
     switch (format) {
@@ -394,9 +406,16 @@ bool UVCCamera::open_capture_()
     }
 
     AVStream* video_stream = capture_format_context_->streams[capture_video_stream_index_];
-    const AVCodec* decoder = avcodec_find_decoder(video_stream->codecpar->codec_id);
+    if (video_stream->codecpar->codec_id != AV_CODEC_ID_MJPEG) {
+        const std::string error = "camera input is not MJPEG; QSV MJPEG decoding is unavailable";
+        close_capture_();
+        capture_error_ = error;
+        return false;
+    }
+
+    const AVCodec* decoder = avcodec_find_decoder_by_name("mjpeg_qsv");
     if (!decoder) {
-        const std::string error = "FFmpeg decoder is unavailable for camera input";
+        const std::string error = "FFmpeg was built without the mjpeg_qsv decoder";
         close_capture_();
         capture_error_ = error;
         return false;
@@ -411,12 +430,37 @@ bool UVCCamera::open_capture_()
     }
 
     result = avcodec_parameters_to_context(capture_decoder_context_, video_stream->codecpar);
+    if (result >= 0) {
+        AVBufferRef* vaapi_device_context = nullptr;
+        result = av_hwdevice_ctx_create(
+            &vaapi_device_context,
+            AV_HWDEVICE_TYPE_VAAPI,
+            "/dev/dri/renderD128",
+            nullptr,
+            0);
+        if (result >= 0) {
+            result = av_hwdevice_ctx_create_derived(
+                &capture_hw_device_context_,
+                AV_HWDEVICE_TYPE_QSV,
+                vaapi_device_context,
+                0);
+        }
+        av_buffer_unref(&vaapi_device_context);
+    }
+    if (result >= 0) {
+        capture_decoder_context_->hw_device_ctx = av_buffer_ref(capture_hw_device_context_);
+        if (!capture_decoder_context_->hw_device_ctx) {
+            result = AVERROR(ENOMEM);
+        } else {
+            capture_decoder_context_->get_format = select_qsv_pixel_format;
+        }
+    }
     if (result >= 0) {
         result = avcodec_open2(capture_decoder_context_, decoder, nullptr);
     }
     if (result < 0) {
         const std::string error =
-            "failed to open camera decoder: " + ffmpeg_error_string(result);
+            "failed to initialize mjpeg_qsv decoder: " + ffmpeg_error_string(result);
         close_capture_();
         capture_error_ = error;
         return false;
@@ -425,6 +469,8 @@ bool UVCCamera::open_capture_()
     CMVR_LOG(INFO) << "[UVCCamera] FFmpeg capture opened"
                    << ", device=" << serial_
                    << ", input_codec=" << avcodec_get_name(video_stream->codecpar->codec_id)
+                   << ", decoder=" << decoder->name
+                   << ", hardware_device=qsv"
                    << ", width=" << capture_decoder_context_->width
                    << ", height=" << capture_decoder_context_->height
                    << ", requested_fps=" << fps_;
@@ -450,6 +496,7 @@ void UVCCamera::close_capture_()
     if (capture_decoder_context_) {
         avcodec_free_context(&capture_decoder_context_);
     }
+    av_buffer_unref(&capture_hw_device_context_);
     if (capture_format_context_) {
         avformat_close_input(&capture_format_context_);
     }
@@ -460,8 +507,9 @@ void UVCCamera::capture_worker_()
 {
     AVPacket* input_packet = av_packet_alloc();
     AVFrame* decoded_frame = av_frame_alloc();
+    AVFrame* software_frame = av_frame_alloc();
     uint64_t discarded_frame_count = 0;
-    if (!input_packet || !decoded_frame) {
+    if (!input_packet || !decoded_frame || !software_frame) {
         {
             std::lock_guard frame_lock(capture_frame_mutex_);
             capture_error_ = "failed to allocate FFmpeg capture frame resources";
@@ -470,6 +518,7 @@ void UVCCamera::capture_worker_()
         capture_frame_cv_.notify_all();
         av_packet_free(&input_packet);
         av_frame_free(&decoded_frame);
+        av_frame_free(&software_frame);
         return;
     }
 
@@ -545,14 +594,30 @@ void UVCCamera::capture_worker_()
                 break;
             }
 
-            const auto decoded_format = static_cast<AVPixelFormat>(decoded_frame->format);
+            AVFrame* source_frame = decoded_frame;
+            if (decoded_frame->hw_frames_ctx) {
+                av_frame_unref(software_frame);
+                const int transfer_result = av_hwframe_transfer_data(
+                    software_frame, decoded_frame, 0);
+                if (transfer_result < 0) {
+                    std::lock_guard frame_lock(capture_frame_mutex_);
+                    capture_error_ = "failed to download QSV camera frame: "
+                        + ffmpeg_error_string(transfer_result);
+                    capture_running_.store(false, std::memory_order_relaxed);
+                    break;
+                }
+                av_frame_copy_props(software_frame, decoded_frame);
+                source_frame = software_frame;
+            }
+
+            const auto decoded_format = static_cast<AVPixelFormat>(source_frame->format);
             const auto source_format = normalize_deprecated_yuv_format(decoded_format);
-            const bool source_full_range = decoded_frame->color_range == AVCOL_RANGE_JPEG
+            const bool source_full_range = source_frame->color_range == AVCOL_RANGE_JPEG
                 || source_format != decoded_format;
             capture_sws_context_ = sws_getCachedContext(
                 capture_sws_context_,
-                decoded_frame->width,
+                source_frame->width,
-                decoded_frame->height,
+                source_frame->height,
                 source_format,
                 width_,
                 height_,
@@ -569,7 +634,7 @@ void UVCCamera::capture_worker_()
             }
 
             const int* color_coefficients = sws_getCoefficients(
-                swscale_colorspace(decoded_frame->colorspace));
+                swscale_colorspace(source_frame->colorspace));
             const int color_result = sws_setColorspaceDetails(
                 capture_sws_context_,
                 color_coefficients,
@@ -594,10 +659,10 @@ void UVCCamera::capture_worker_()
             };
             const int converted_rows = sws_scale(
                 capture_sws_context_,
-                decoded_frame->data,
+                source_frame->data,
-                decoded_frame->linesize,
+                source_frame->linesize,
                 0,
-                decoded_frame->height,
+                source_frame->height,
                 destination_data,
                 destination_linesize);
             if (converted_rows != height_) {
@@ -627,6 +692,7 @@ void UVCCamera::capture_worker_()
     capture_frame_cv_.notify_all();
     av_packet_free(&input_packet);
     av_frame_free(&decoded_frame);
+    av_frame_free(&software_frame);
 }
 
 bool UVCCamera::wait_for_capture_frame_(cv::Mat& frame,