feat(camera): decode UVC MJPEG with QSV
This commit is contained in:
parent
2596a82c2c
commit
41d8482faf
@ -80,6 +80,7 @@ namespace cmvr::device {
|
||||
|
||||
AVFormatContext* capture_format_context_ = nullptr;
|
||||
AVCodecContext* capture_decoder_context_ = nullptr;
|
||||
AVBufferRef* capture_hw_device_context_ = nullptr;
|
||||
SwsContext* capture_sws_context_ = nullptr;
|
||||
int capture_video_stream_index_ = -1;
|
||||
std::atomic<bool> capture_running_{false};
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
extern "C" {
|
||||
#include <libavdevice/avdevice.h>
|
||||
#include <libavutil/error.h>
|
||||
#include <libavutil/hwcontext.h>
|
||||
}
|
||||
|
||||
using namespace std;
|
||||
@ -27,6 +28,17 @@ std::string ffmpeg_error_string(const int error_code)
|
||||
return error_buffer;
|
||||
}
|
||||
|
||||
AVPixelFormat select_qsv_pixel_format(AVCodecContext*, const AVPixelFormat* formats)
|
||||
{
|
||||
for (const AVPixelFormat* format = formats; *format != AV_PIX_FMT_NONE; ++format) {
|
||||
if (*format == AV_PIX_FMT_QSV) {
|
||||
return *format;
|
||||
}
|
||||
}
|
||||
CMVR_LOG(ERROR) << "[UVCCamera] mjpeg_qsv decoder did not offer QSV frames";
|
||||
return AV_PIX_FMT_NONE;
|
||||
}
|
||||
|
||||
AVPixelFormat normalize_deprecated_yuv_format(const AVPixelFormat format)
|
||||
{
|
||||
switch (format) {
|
||||
@ -394,9 +406,16 @@ bool UVCCamera::open_capture_()
|
||||
}
|
||||
|
||||
AVStream* video_stream = capture_format_context_->streams[capture_video_stream_index_];
|
||||
const AVCodec* decoder = avcodec_find_decoder(video_stream->codecpar->codec_id);
|
||||
if (video_stream->codecpar->codec_id != AV_CODEC_ID_MJPEG) {
|
||||
const std::string error = "camera input is not MJPEG; QSV MJPEG decoding is unavailable";
|
||||
close_capture_();
|
||||
capture_error_ = error;
|
||||
return false;
|
||||
}
|
||||
|
||||
const AVCodec* decoder = avcodec_find_decoder_by_name("mjpeg_qsv");
|
||||
if (!decoder) {
|
||||
const std::string error = "FFmpeg decoder is unavailable for camera input";
|
||||
const std::string error = "FFmpeg was built without the mjpeg_qsv decoder";
|
||||
close_capture_();
|
||||
capture_error_ = error;
|
||||
return false;
|
||||
@ -411,12 +430,37 @@ bool UVCCamera::open_capture_()
|
||||
}
|
||||
|
||||
result = avcodec_parameters_to_context(capture_decoder_context_, video_stream->codecpar);
|
||||
if (result >= 0) {
|
||||
AVBufferRef* vaapi_device_context = nullptr;
|
||||
result = av_hwdevice_ctx_create(
|
||||
&vaapi_device_context,
|
||||
AV_HWDEVICE_TYPE_VAAPI,
|
||||
"/dev/dri/renderD128",
|
||||
nullptr,
|
||||
0);
|
||||
if (result >= 0) {
|
||||
result = av_hwdevice_ctx_create_derived(
|
||||
&capture_hw_device_context_,
|
||||
AV_HWDEVICE_TYPE_QSV,
|
||||
vaapi_device_context,
|
||||
0);
|
||||
}
|
||||
av_buffer_unref(&vaapi_device_context);
|
||||
}
|
||||
if (result >= 0) {
|
||||
capture_decoder_context_->hw_device_ctx = av_buffer_ref(capture_hw_device_context_);
|
||||
if (!capture_decoder_context_->hw_device_ctx) {
|
||||
result = AVERROR(ENOMEM);
|
||||
} else {
|
||||
capture_decoder_context_->get_format = select_qsv_pixel_format;
|
||||
}
|
||||
}
|
||||
if (result >= 0) {
|
||||
result = avcodec_open2(capture_decoder_context_, decoder, nullptr);
|
||||
}
|
||||
if (result < 0) {
|
||||
const std::string error =
|
||||
"failed to open camera decoder: " + ffmpeg_error_string(result);
|
||||
"failed to initialize mjpeg_qsv decoder: " + ffmpeg_error_string(result);
|
||||
close_capture_();
|
||||
capture_error_ = error;
|
||||
return false;
|
||||
@ -425,6 +469,8 @@ bool UVCCamera::open_capture_()
|
||||
CMVR_LOG(INFO) << "[UVCCamera] FFmpeg capture opened"
|
||||
<< ", device=" << serial_
|
||||
<< ", input_codec=" << avcodec_get_name(video_stream->codecpar->codec_id)
|
||||
<< ", decoder=" << decoder->name
|
||||
<< ", hardware_device=qsv"
|
||||
<< ", width=" << capture_decoder_context_->width
|
||||
<< ", height=" << capture_decoder_context_->height
|
||||
<< ", requested_fps=" << fps_;
|
||||
@ -450,6 +496,7 @@ void UVCCamera::close_capture_()
|
||||
if (capture_decoder_context_) {
|
||||
avcodec_free_context(&capture_decoder_context_);
|
||||
}
|
||||
av_buffer_unref(&capture_hw_device_context_);
|
||||
if (capture_format_context_) {
|
||||
avformat_close_input(&capture_format_context_);
|
||||
}
|
||||
@ -460,8 +507,9 @@ void UVCCamera::capture_worker_()
|
||||
{
|
||||
AVPacket* input_packet = av_packet_alloc();
|
||||
AVFrame* decoded_frame = av_frame_alloc();
|
||||
AVFrame* software_frame = av_frame_alloc();
|
||||
uint64_t discarded_frame_count = 0;
|
||||
if (!input_packet || !decoded_frame) {
|
||||
if (!input_packet || !decoded_frame || !software_frame) {
|
||||
{
|
||||
std::lock_guard frame_lock(capture_frame_mutex_);
|
||||
capture_error_ = "failed to allocate FFmpeg capture frame resources";
|
||||
@ -470,6 +518,7 @@ void UVCCamera::capture_worker_()
|
||||
capture_frame_cv_.notify_all();
|
||||
av_packet_free(&input_packet);
|
||||
av_frame_free(&decoded_frame);
|
||||
av_frame_free(&software_frame);
|
||||
return;
|
||||
}
|
||||
|
||||
@ -545,14 +594,30 @@ void UVCCamera::capture_worker_()
|
||||
break;
|
||||
}
|
||||
|
||||
const auto decoded_format = static_cast<AVPixelFormat>(decoded_frame->format);
|
||||
AVFrame* source_frame = decoded_frame;
|
||||
if (decoded_frame->hw_frames_ctx) {
|
||||
av_frame_unref(software_frame);
|
||||
const int transfer_result = av_hwframe_transfer_data(
|
||||
software_frame, decoded_frame, 0);
|
||||
if (transfer_result < 0) {
|
||||
std::lock_guard frame_lock(capture_frame_mutex_);
|
||||
capture_error_ = "failed to download QSV camera frame: "
|
||||
+ ffmpeg_error_string(transfer_result);
|
||||
capture_running_.store(false, std::memory_order_relaxed);
|
||||
break;
|
||||
}
|
||||
av_frame_copy_props(software_frame, decoded_frame);
|
||||
source_frame = software_frame;
|
||||
}
|
||||
|
||||
const auto decoded_format = static_cast<AVPixelFormat>(source_frame->format);
|
||||
const auto source_format = normalize_deprecated_yuv_format(decoded_format);
|
||||
const bool source_full_range = decoded_frame->color_range == AVCOL_RANGE_JPEG
|
||||
const bool source_full_range = source_frame->color_range == AVCOL_RANGE_JPEG
|
||||
|| source_format != decoded_format;
|
||||
capture_sws_context_ = sws_getCachedContext(
|
||||
capture_sws_context_,
|
||||
decoded_frame->width,
|
||||
decoded_frame->height,
|
||||
source_frame->width,
|
||||
source_frame->height,
|
||||
source_format,
|
||||
width_,
|
||||
height_,
|
||||
@ -569,7 +634,7 @@ void UVCCamera::capture_worker_()
|
||||
}
|
||||
|
||||
const int* color_coefficients = sws_getCoefficients(
|
||||
swscale_colorspace(decoded_frame->colorspace));
|
||||
swscale_colorspace(source_frame->colorspace));
|
||||
const int color_result = sws_setColorspaceDetails(
|
||||
capture_sws_context_,
|
||||
color_coefficients,
|
||||
@ -594,10 +659,10 @@ void UVCCamera::capture_worker_()
|
||||
};
|
||||
const int converted_rows = sws_scale(
|
||||
capture_sws_context_,
|
||||
decoded_frame->data,
|
||||
decoded_frame->linesize,
|
||||
source_frame->data,
|
||||
source_frame->linesize,
|
||||
0,
|
||||
decoded_frame->height,
|
||||
source_frame->height,
|
||||
destination_data,
|
||||
destination_linesize);
|
||||
if (converted_rows != height_) {
|
||||
@ -627,6 +692,7 @@ void UVCCamera::capture_worker_()
|
||||
capture_frame_cv_.notify_all();
|
||||
av_packet_free(&input_packet);
|
||||
av_frame_free(&decoded_frame);
|
||||
av_frame_free(&software_frame);
|
||||
}
|
||||
|
||||
bool UVCCamera::wait_for_capture_frame_(cv::Mat& frame,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user