SmartBench/cmvr-es
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(camera): avoid unnecessary UVC BGR conversion

Browse Source
This commit is contained in:
linbo 2026-08-12 17:40:43 +08:00
parent 69b96cb60b
commit 400327269a
4 changed files with 307 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmvr-es/devices/camera/common/include/camera_stream_encoder.h
View File

@ -19,6 +19,7 @@ struct FfmpegEncoderInfo {
bool bRunning = false;
AVCodecContext* codec_context = nullptr;
AVFrame* frame = nullptr;
AVFrame* transfer_frame = nullptr;
AVPacket* packet = nullptr;
SwsContext* sws_context = nullptr;
@ -42,6 +43,11 @@ public:
std::vector<uint8_t>& encoded_frame,
bool& is_key,
const CameraStreamEncodeOptions& options = {});
static bool encode(std::shared_ptr<FfmpegEncoderInfo>& encoder,
const AVFrame* frame,
std::vector<uint8_t>& encoded_frame,
bool& is_key);
};
} // namespace cmvr::device

171
cmvr-es/devices/camera/common/src/camera_stream_encoder.cpp
View File

@ -7,6 +7,7 @@
#include <libavutil/opt.h>
#include <libavutil/pixdesc.h>
#include <libavutil/hwcontext.h>
#include <opencv2/imgproc.hpp>
#include "common/base/logging/logger.h"
@ -81,6 +82,55 @@ AVPixelFormat sourcePixelFormat(const cv::Mat& frame)
return AV_PIX_FMT_NONE;
}
bool encodePreparedFrame(FfmpegEncoderInfo& encoder,
std::vector<uint8_t>& encoded_frame,
bool& is_key)
{
encoder.frame->pts = encoder.frame_pts++;
int ret = avcodec_send_frame(encoder.codec_context, encoder.frame);
if (ret < 0) {
char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0};
av_strerror(ret, errbuf, sizeof(errbuf));
CMVR_LOG(ERROR) << "[CameraStreamEncoder] error sending frame to encoder: " << errbuf;
return false;
}
while (true) {
ret = avcodec_receive_packet(encoder.codec_context, encoder.packet);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
if (ret < 0) {
char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0};
av_strerror(ret, errbuf, sizeof(errbuf));
CMVR_LOG(ERROR) << "[CameraStreamEncoder] error receiving packet from encoder: " << errbuf;
return false;
}
if (encoder.packet->flags & AV_PKT_FLAG_KEY) {
is_key = true;
}
encoded_frame.reserve(encoded_frame.size() + encoder.packet->size);
encoded_frame.insert(encoded_frame.end(),
encoder.packet->data,
encoder.packet->data + encoder.packet->size);
av_packet_unref(encoder.packet);
}
if (encoded_frame.size() < 4) {
return false;
}
const bool has_start_code =
(encoded_frame[0] == 0 && encoded_frame[1] == 0 && encoded_frame[2] == 1) ||
(encoded_frame[0] == 0 && encoded_frame[1] == 0 && encoded_frame[2] == 0 && encoded_frame[3] == 1);
if (!has_start_code) {
CMVR_LOG(ERROR) << "[CameraStreamEncoder] invalid frame: no NALU start code";
return false;
}
return true;
}
} // namespace
FfmpegEncoderInfo::~FfmpegEncoderInfo()
@ -89,6 +139,10 @@ FfmpegEncoderInfo::~FfmpegEncoderInfo()
av_frame_free(&frame);
frame = nullptr;
}
if (transfer_frame) {
av_frame_free(&transfer_frame);
transfer_frame = nullptr;
}
if (packet) {
av_packet_free(&packet);
packet = nullptr;
@ -277,49 +331,102 @@ bool CameraStreamEncoder::encode(std::shared_ptr<FfmpegEncoderInfo>& encoder,
return false;
}
encoder->frame->pts = encoder->frame_pts++;
int ret = avcodec_send_frame(encoder->codec_context, encoder->frame);
if (ret < 0) {
return encodePreparedFrame(*encoder, encoded_frame, is_key);
}
bool CameraStreamEncoder::encode(std::shared_ptr<FfmpegEncoderInfo>& encoder,
const AVFrame* frame,
std::vector<uint8_t>& encoded_frame,
bool& is_key)
{
encoded_frame.clear();
is_key = false;
if (!encoder || !encoder->codec_context || !encoder->frame || !encoder->packet || !frame) {
return false;
}
const AVFrame* source_frame = frame;
if (frame->hw_frames_ctx) {
if (!encoder->transfer_frame) {
encoder->transfer_frame = av_frame_alloc();
}
if (!encoder->transfer_frame) {
return false;
}
av_frame_unref(encoder->transfer_frame);
const int transfer_ret = av_hwframe_transfer_data(encoder->transfer_frame, frame, 0);
if (transfer_ret < 0) {
char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0};
av_strerror(ret, errbuf, sizeof(errbuf));
CMVR_LOG(ERROR) << "[CameraStreamEncoder] error sending frame to encoder: " << errbuf;
av_strerror(transfer_ret, errbuf, sizeof(errbuf));
CMVR_LOG(ERROR) << "[CameraStreamEncoder] failed to download hardware frame: " << errbuf;
return false;
}
while (true) {
ret = avcodec_receive_packet(encoder->codec_context, encoder->packet);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
}
if (ret < 0) {
const int props_ret = av_frame_copy_props(encoder->transfer_frame, frame);
if (props_ret < 0) {
char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0};
av_strerror(ret, errbuf, sizeof(errbuf));
CMVR_LOG(ERROR) << "[CameraStreamEncoder] error receiving packet from encoder: " << errbuf;
break;
av_strerror(props_ret, errbuf, sizeof(errbuf));
CMVR_LOG(ERROR) << "[CameraStreamEncoder] failed to copy hardware frame properties: " << errbuf;
return false;
}
source_frame = encoder->transfer_frame;
}
if (encoder->packet->flags & AV_PKT_FLAG_KEY) {
is_key = true;
}
encoded_frame.reserve(encoded_frame.size() + encoder->packet->size);
encoded_frame.insert(encoded_frame.end(),
encoder->packet->data,
encoder->packet->data + encoder->packet->size);
av_packet_unref(encoder->packet);
}
if (encoded_frame.size() < 4) {
const auto source_format = static_cast<AVPixelFormat>(source_frame->format);
if (source_format == AV_PIX_FMT_NONE) {
CMVR_LOG(ERROR) << "[CameraStreamEncoder] source frame has no pixel format";
return false;
}
const bool has_start_code =
(encoded_frame[0] == 0 && encoded_frame[1] == 0 && encoded_frame[2] == 1) ||
(encoded_frame[0] == 0 && encoded_frame[1] == 0 && encoded_frame[2] == 0 && encoded_frame[3] == 1);
if (!has_start_code) {
CMVR_LOG(ERROR) << "[CameraStreamEncoder] invalid frame: no NALU start code";
const int writable_ret = av_frame_make_writable(encoder->frame);
if (writable_ret < 0) {
char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0};
av_strerror(writable_ret, errbuf, sizeof(errbuf));
CMVR_LOG(ERROR) << "[CameraStreamEncoder] frame is not writable: " << errbuf;
return false;
}
return true;
int copy_ret = 0;
if (source_frame->width == encoder->codec_context->width
&& source_frame->height == encoder->codec_context->height
&& source_format == encoder->codec_context->pix_fmt) {
copy_ret = av_frame_copy(encoder->frame, source_frame);
} else {
encoder->sws_context = sws_getCachedContext(
encoder->sws_context,
source_frame->width,
source_frame->height,
source_format,
encoder->codec_context->width,
encoder->codec_context->height,
encoder->codec_context->pix_fmt,
SWS_BILINEAR,
nullptr,
nullptr,
nullptr);
if (!encoder->sws_context) {
CMVR_LOG(ERROR) << "[CameraStreamEncoder] failed to create native frame conversion context";
return false;
}
const int scaled_rows = sws_scale(
encoder->sws_context,
source_frame->data,
source_frame->linesize,
0,
source_frame->height,
encoder->frame->data,
encoder->frame->linesize);
copy_ret = scaled_rows == encoder->codec_context->height
? 0
: scaled_rows < 0 ? scaled_rows : AVERROR_INVALIDDATA;
}
if (copy_ret < 0) {
char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0};
av_strerror(copy_ret, errbuf, sizeof(errbuf));
CMVR_LOG(ERROR) << "[CameraStreamEncoder] failed to copy native frame: " << errbuf;
return false;
}
return encodePreparedFrame(*encoder, encoded_frame, is_key);
}
} // namespace cmvr::device

6
cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h
View File

@ -52,10 +52,11 @@ namespace cmvr::device {
bool open_capture_();
void close_capture_();
void capture_worker_();
bool wait_for_capture_frame_(cv::Mat& frame,
bool wait_for_capture_frame_(AVFrame* frame,
uint64_t& last_sequence,
int64_t& capture_monotonic_ns,
int64_t& capture_utc_ns);
bool convert_capture_frame_to_bgr_(const AVFrame* frame, cv::Mat& bgr_frame);
static int interrupt_capture_(void* opaque);
int fps_;
@ -85,8 +86,9 @@ namespace cmvr::device {
int capture_video_stream_index_ = -1;
std::atomic<bool> capture_running_{false};
std::mutex capture_frame_mutex_;
std::mutex capture_conversion_mutex_;
std::condition_variable capture_frame_cv_;
cv::Mat latest_capture_frame_;
AVFrame* latest_capture_frame_ = nullptr;
uint64_t latest_capture_sequence_ = 0;
int64_t latest_capture_monotonic_ns_ = 0;
int64_t latest_capture_utc_ns_ = 0;

233
cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp
View File

@ -234,11 +234,11 @@ bool UVCCamera::start() {
return false;
}
cv::Mat first_frame;
AVFrame* first_frame = av_frame_alloc();
uint64_t first_sequence = 0;
int64_t first_capture_monotonic_ns = 0;
int64_t first_capture_utc_ns = 0;
if (!wait_for_capture_frame_(first_frame,
if (!first_frame || !wait_for_capture_frame_(first_frame,
first_sequence,
first_capture_monotonic_ns,
first_capture_utc_ns)) {
@ -253,8 +253,10 @@ bool UVCCamera::start() {
? "timed out waiting for the first camera frame"
: capture_error;
CMVR_LOG(ERROR) << "[UVCCamera] (start): " << state_.error_message;
av_frame_free(&first_frame);
return false;
}
av_frame_free(&first_frame);
state_.is_opened = true;
return true;
}
@ -314,10 +316,16 @@ void UVCCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics)
int64_t capture_monotonic_ns = 0;
int64_t capture_utc_ns = 0;
if (!wait_for_capture_frame_(color,
AVFrame* capture_frame = av_frame_alloc();
const bool received_frame = capture_frame
&& wait_for_capture_frame_(capture_frame,
requested_sequence,
capture_monotonic_ns,
capture_utc_ns) || color.empty()) {
capture_utc_ns);
const bool converted_frame = received_frame
&& convert_capture_frame_to_bgr_(capture_frame, color);
av_frame_free(&capture_frame);
if (!converted_frame || color.empty()) {
std::string capture_error;
{
std::lock_guard capture_lock(capture_frame_mutex_);
@ -344,7 +352,7 @@ bool UVCCamera::open_capture_()
{
std::lock_guard frame_lock(capture_frame_mutex_);
latest_capture_frame_.release();
av_frame_free(&latest_capture_frame_);
latest_capture_sequence_ = 0;
latest_capture_monotonic_ns_ = 0;
latest_capture_utc_ns_ = 0;
@ -511,6 +519,10 @@ void UVCCamera::close_capture_()
if (capture_format_context_) {
avformat_close_input(&capture_format_context_);
}
{
std::lock_guard frame_lock(capture_frame_mutex_);
av_frame_free(&latest_capture_frame_);
}
capture_video_stream_index_ = -1;
}
@ -518,9 +530,8 @@ void UVCCamera::capture_worker_()
{
AVPacket* input_packet = av_packet_alloc();
AVFrame* decoded_frame = av_frame_alloc();
AVFrame* software_frame = av_frame_alloc();
uint64_t discarded_frame_count = 0;
if (!input_packet || !decoded_frame || !software_frame) {
if (!input_packet || !decoded_frame) {
{
std::lock_guard frame_lock(capture_frame_mutex_);
capture_error_ = "failed to allocate FFmpeg capture frame resources";
@ -529,7 +540,6 @@ void UVCCamera::capture_worker_()
capture_frame_cv_.notify_all();
av_packet_free(&input_packet);
av_frame_free(&decoded_frame);
av_frame_free(&software_frame);
return;
}
@ -605,19 +615,92 @@ void UVCCamera::capture_worker_()
break;
}
AVFrame* source_frame = decoded_frame;
if (decoded_frame->hw_frames_ctx) {
av_frame_unref(software_frame);
const int transfer_result = av_hwframe_transfer_data(
software_frame, decoded_frame, 0);
if (transfer_result < 0) {
const auto capture_monotonic = std::chrono::steady_clock::now();
const auto capture_utc = std::chrono::system_clock::now();
AVFrame* cached_frame = av_frame_clone(decoded_frame);
if (!cached_frame) {
std::lock_guard frame_lock(capture_frame_mutex_);
capture_error_ = "failed to download QSV camera frame: "
+ ffmpeg_error_string(transfer_result);
capture_error_ = "failed to retain QSV camera frame";
capture_running_.store(false, std::memory_order_relaxed);
break;
}
av_frame_copy_props(software_frame, decoded_frame);
{
std::lock_guard frame_lock(capture_frame_mutex_);
av_frame_free(&latest_capture_frame_);
latest_capture_frame_ = cached_frame;
++latest_capture_sequence_;
latest_capture_monotonic_ns_ = std::chrono::duration_cast<std::chrono::nanoseconds>(
capture_monotonic.time_since_epoch()).count();
latest_capture_utc_ns_ = std::chrono::duration_cast<std::chrono::nanoseconds>(
capture_utc.time_since_epoch()).count();
capture_error_.clear();
}
capture_frame_cv_.notify_all();
av_frame_unref(decoded_frame);
}
}
capture_running_.store(false, std::memory_order_relaxed);
capture_frame_cv_.notify_all();
av_packet_free(&input_packet);
av_frame_free(&decoded_frame);
}
bool UVCCamera::wait_for_capture_frame_(AVFrame* frame,
uint64_t& last_sequence,
int64_t& capture_monotonic_ns,
int64_t& capture_utc_ns)
{
if (!frame) {
return false;
}
std::unique_lock frame_lock(capture_frame_mutex_);
const auto timeout = std::chrono::milliseconds(
std::max(500, fps_ > 0 ? 3000 / fps_ : 500));
const bool ready = capture_frame_cv_.wait_for(frame_lock, timeout, [&] {
return latest_capture_sequence_ > last_sequence
|| !capture_running_.load(std::memory_order_relaxed);
});
if (!ready || latest_capture_sequence_ <= last_sequence || !latest_capture_frame_) {
return false;
}
av_frame_unref(frame);
if (av_frame_ref(frame, latest_capture_frame_) < 0) {
return false;
}
last_sequence = latest_capture_sequence_;
capture_monotonic_ns = latest_capture_monotonic_ns_;
capture_utc_ns = latest_capture_utc_ns_;
return true;
}
bool UVCCamera::convert_capture_frame_to_bgr_(const AVFrame* frame, cv::Mat& bgr_frame)
{
bgr_frame.release();
if (!frame) {
return false;
}
std::lock_guard conversion_lock(capture_conversion_mutex_);
AVFrame* software_frame = av_frame_alloc();
if (!software_frame) {
return false;
}
const AVFrame* source_frame = frame;
int result = 0;
if (frame->hw_frames_ctx) {
result = av_hwframe_transfer_data(software_frame, frame, 0);
if (result >= 0) {
result = av_frame_copy_props(software_frame, frame);
}
if (result < 0) {
CMVR_LOG(ERROR) << "[UVCCamera] failed to prepare BGR source frame: "
<< ffmpeg_error_string(result);
av_frame_free(&software_frame);
return false;
}
source_frame = software_frame;
}
@ -638,15 +721,14 @@ void UVCCamera::capture_worker_()
nullptr,
nullptr);
if (!capture_sws_context_) {
std::lock_guard frame_lock(capture_frame_mutex_);
capture_error_ = "failed to create camera pixel conversion context";
capture_running_.store(false, std::memory_order_relaxed);
break;
CMVR_LOG(ERROR) << "[UVCCamera] failed to create BGR conversion context";
av_frame_free(&software_frame);
return false;
}
const int* color_coefficients = sws_getCoefficients(
swscale_colorspace(source_frame->colorspace));
const int color_result = sws_setColorspaceDetails(
result = sws_setColorspaceDetails(
capture_sws_context_,
color_coefficients,
source_full_range ? 1 : 0,
@ -655,15 +737,14 @@ void UVCCamera::capture_worker_()
0,
1 << 16,
1 << 16);
if (color_result < 0) {
std::lock_guard frame_lock(capture_frame_mutex_);
capture_error_ = "failed to configure camera color conversion: "
+ ffmpeg_error_string(color_result);
capture_running_.store(false, std::memory_order_relaxed);
break;
if (result < 0) {
CMVR_LOG(ERROR) << "[UVCCamera] failed to configure BGR color conversion: "
<< ffmpeg_error_string(result);
av_frame_free(&software_frame);
return false;
}
cv::Mat bgr_frame(height_, width_, CV_8UC3);
bgr_frame.create(height_, width_, CV_8UC3);
uint8_t* destination_data[] = {bgr_frame.data, nullptr, nullptr, nullptr};
int destination_linesize[] = {
static_cast<int>(bgr_frame.step[0]), 0, 0, 0
@ -676,56 +757,13 @@ void UVCCamera::capture_worker_()
source_frame->height,
destination_data,
destination_linesize);
if (converted_rows != height_) {
std::lock_guard frame_lock(capture_frame_mutex_);
capture_error_ = "camera pixel conversion returned an incomplete frame";
continue;
}
const auto capture_monotonic = std::chrono::steady_clock::now();
const auto capture_utc = std::chrono::system_clock::now();
{
std::lock_guard frame_lock(capture_frame_mutex_);
latest_capture_frame_ = std::move(bgr_frame);
++latest_capture_sequence_;
latest_capture_monotonic_ns_ = std::chrono::duration_cast<std::chrono::nanoseconds>(
capture_monotonic.time_since_epoch()).count();
latest_capture_utc_ns_ = std::chrono::duration_cast<std::chrono::nanoseconds>(
capture_utc.time_since_epoch()).count();
capture_error_.clear();
}
capture_frame_cv_.notify_all();
av_frame_unref(decoded_frame);
}
}
capture_running_.store(false, std::memory_order_relaxed);
capture_frame_cv_.notify_all();
av_packet_free(&input_packet);
av_frame_free(&decoded_frame);
av_frame_free(&software_frame);
}
bool UVCCamera::wait_for_capture_frame_(cv::Mat& frame,
uint64_t& last_sequence,
int64_t& capture_monotonic_ns,
int64_t& capture_utc_ns)
{
std::unique_lock frame_lock(capture_frame_mutex_);
const auto timeout = std::chrono::milliseconds(
std::max(500, fps_ > 0 ? 3000 / fps_ : 500));
const bool ready = capture_frame_cv_.wait_for(frame_lock, timeout, [&] {
return latest_capture_sequence_ > last_sequence
|| !capture_running_.load(std::memory_order_relaxed);
});
if (!ready || latest_capture_sequence_ <= last_sequence || latest_capture_frame_.empty()) {
if (converted_rows != height_) {
CMVR_LOG(ERROR) << "[UVCCamera] BGR conversion returned " << converted_rows
<< " rows, expected " << height_;
bgr_frame.release();
return false;
}
frame = latest_capture_frame_;
last_sequence = latest_capture_sequence_;
capture_monotonic_ns = latest_capture_monotonic_ns_;
capture_utc_ns = latest_capture_utc_ns_;
return true;
}
@ -960,14 +998,20 @@ void UVCCamera::resumeRecording() {
}
void UVCCamera::streaming_worker_() {
AVFrame* frame = av_frame_alloc();
if (!frame) {
is_streaming_running = false;
state_.is_error = true;
state_.error_message = "failed to allocate streaming frame";
CMVR_LOG(ERROR) << "[UVCCamera]streaming_worker_: " << state_.error_message;
return;
}
try {
// 计算理论上每帧之间的间隔时间(毫秒)
const int frame_interval = 1000 / fps_;
bool success = false;
is_streaming_running = true;
cv::Mat frame;
uint64_t capture_sequence = 0;
int64_t frame_capture_monotonic_ns = 0;
int64_t frame_capture_utc_ns = 0;
@ -993,7 +1037,7 @@ void UVCCamera::streaming_worker_() {
if (!wait_for_capture_frame_(frame,
capture_sequence,
frame_capture_monotonic_ns,
frame_capture_utc_ns) || frame.empty()) {
frame_capture_utc_ns)) {
std::string capture_error;
{
std::lock_guard capture_lock(capture_frame_mutex_);
@ -1011,31 +1055,33 @@ void UVCCamera::streaming_worker_() {
// 以下为编码部分，用于流模式
StreamFrameData frame_data;
// 保存图像数据
{
frame_data.rgbImage = frame;
}
cv::Mat bgr_to_encode;
const auto copy_end_time = std::chrono::steady_clock::now();
// rgb图像编码
cv::Mat rgb_to_encode = frame_data.rgbImage;
if (encode_width_ > 0 && encode_height_ > 0 &&
(frame_data.rgbImage.cols != encode_width_ || frame_data.rgbImage.rows != encode_height_)) {
cv::resize(frame_data.rgbImage,
rgb_to_encode,
if (enable_stream_timestamp_) {
success = convert_capture_frame_to_bgr_(frame, bgr_to_encode);
if (success && encode_width_ > 0 && encode_height_ > 0 &&
(bgr_to_encode.cols != encode_width_ || bgr_to_encode.rows != encode_height_)) {
cv::resize(bgr_to_encode,
bgr_to_encode,
cv::Size(encode_width_, encode_height_),
0.0,
0.0,
cv::INTER_LINEAR);
}
}
const auto resize_end_time = std::chrono::steady_clock::now();
if (enable_stream_timestamp_) {
CameraStreamEncodeOptions encode_options;
encode_options.draw_timestamp = enable_stream_timestamp_;
success = CameraStreamEncoder::encode(rgbEncoder_,
rgb_to_encode,
frame_data.rgbFrame,
frame_data.bKey,
encode_options);
encode_options.draw_timestamp = true;
success = success && CameraStreamEncoder::encode(
rgbEncoder_, bgr_to_encode, frame_data.rgbFrame, frame_data.bKey, encode_options);
} else {
success = CameraStreamEncoder::encode(
rgbEncoder_, frame, frame_data.rgbFrame, frame_data.bKey);
}
const auto encode_end_time = std::chrono::steady_clock::now();
if (success) {
@ -1123,6 +1169,7 @@ void UVCCamera::streaming_worker_() {
state_.error_message = e.what();
CMVR_LOG(ERROR) << "[UVCCamera]streaming_worker_ error:" << state_.error_message;
}
av_frame_free(&frame);
}
void UVCCamera::recording_worker_() {