update uvc_camera ffmpeg encoder

cmvr-es/common/CMakeLists.txt

@@ -3,6 +3,10 @@
 
 file(GLOB SRC
         ${CMAKE_CURRENT_SOURCE_DIR}/utils/config_helper/src/config_setting.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/utils/ffmpeg/src/CameraCapture.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/utils/ffmpeg/src/VideoFrameEncoder.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/utils/ffmpeg/src/VideoWriter.cpp
+
 )
 
 
@@ -13,6 +17,11 @@ target_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 
 target_link_libraries(common PUBLIC
         cmvr_es::proto
+        avformat
+        avdevice
+        avutil
+        avcodec
+        swscale
 )
 
 add_library(cmvr_es::common ALIAS common)

cmvr-es/common/config/camera_config/camera_config.pb.txt

@@ -20,7 +20,7 @@ uvc_cameras {
   height: 480
   fps: 30
   codec: "H265"
-  camera_mode: CAMERA_MODE_PHOTO
+  camera_mode: CAMERA_MODE_VIDEO
   stream_mode: STREAM_MODE_RGB
   buffer_size: 30
   enable: true

cmvr-es/common/config/microphone_config/microphone_config.pb.txt

@@ -3,5 +3,6 @@ ffmpeg_microphones {
     channels: 2
     sampleRate: 44100
     volume: 100
-    enable: true
+    enable: false
+    input_device: "default"
 }

cmvr-es/common/config/speaker_config/speaker_config.pb.txt

@@ -1,4 +1,4 @@
 ffmpeg_speakers {
     id: "spk1"
-    enable: true
+    enable: false
 }

cmvr-es/common/utils/ffmpeg/include/CameraCapture.h

@@ -0,0 +1,68 @@
+// CameraCapture.h
+#pragma once
+
+#include <string>
+#include <memory>
+#include <functional>
+
+extern "C"{
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+#include <libavdevice/avdevice.h>
+}
+
+namespace ffmpeg {
+
+    struct CameraConfig {
+        std::string device_name = "0";  // Windows: "0" (dshow), Linux: "0" (对应 /dev/video0)
+        int width = 1280;
+        int height = 720;
+        int fps = 30;
+        AVPixelFormat pixel_format = AV_PIX_FMT_YUV420P;
+    };
+
+    class CameraCapture {
+    public:
+        using FrameCallback = std::function<void(AVFrame* frame)>;
+
+        CameraCapture();
+        ~CameraCapture();
+
+        // 初始化摄像头
+        int initialize(const CameraConfig& config);
+
+        // 开始捕获
+        int start_capture(FrameCallback callback);
+
+        // 停止捕获
+        void stop_capture();
+
+        // 获取当前配置
+        CameraConfig get_config() const { return config_; }
+
+        // 获取输入格式上下文
+        AVFormatContext* get_format_context() { return fmt_ctx_; }
+
+        // 获取视频流索引
+        int get_video_stream_index() const { return video_stream_index_; }
+
+    private:
+        CameraConfig config_;
+        AVFormatContext* fmt_ctx_ = nullptr;
+        AVCodecContext* decoder_ctx_ = nullptr;
+        int video_stream_index_ = -1;
+        bool is_capturing_ = false;
+        AVPacket* packet_ = nullptr;
+        AVFrame* frame_ = nullptr;
+
+        // 初始化设备（平台相关）
+        int init_device();
+
+        // 初始化解码器
+        int init_decoder();
+
+        // 释放资源
+        void cleanup();
+    };
+
+} // namespace ffmpeg

cmvr-es/common/utils/ffmpeg/include/VideoFrameEncoder.h

@@ -0,0 +1,101 @@
+// VideoFrameEncoder.h
+#pragma once
+
+#include <string>
+#include <memory>
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+#include <functional>
+
+extern "C" {
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+#include <libswscale/swscale.h>
+#include <libavutil/opt.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/time.h>
+}
+
+namespace ffmpeg {
+
+struct EncoderConfig {
+    int width = 1280;
+    int height = 720;
+    int fps = 30;
+    int bitrate = 5000000;  // 5Mbps
+    int gop_size = 15;      // GOP大小
+    int max_b_frames = 2;   // 最大B帧数
+    AVPixelFormat pixel_format = AV_PIX_FMT_YUV420P;
+    std::string preset = "fast";     // 编码速度预设
+    std::string tune = "zerolatency"; // 调优参数
+    std::string profile = "main";    // 编码profile
+    std::string codec = "libx265";
+};
+
+class VideoFrameEncoder {
+public:
+    using PacketCallback = std::function<void(AVPacket* packet, int64_t pts, int64_t dts)>;
+
+    VideoFrameEncoder();
+    ~VideoFrameEncoder();
+
+    // 初始化编码器
+    int initialize(const EncoderConfig& config);
+
+    // 编码一帧
+    int encode_frame(AVFrame* frame);
+
+    // 设置编码数据包回调
+    void set_packet_callback(PacketCallback callback) {
+        packet_callback_ = callback;
+    }
+
+    // 设置时间戳类型
+    void set_time_base(AVRational time_base) {
+        time_base_ = time_base;
+    }
+
+    // 刷新编码器（处理剩余数据）
+    int flush();
+
+    // 获取编码器上下文
+    AVCodecContext* get_codec_context() { return encoder_ctx_; }
+
+    // 获取时间基
+    AVRational get_time_base() const {
+        return encoder_ctx_ ? encoder_ctx_->time_base : AVRational{0, 1};
+    }
+
+    // 获取帧率
+    AVRational get_framerate() const {
+        return encoder_ctx_ ? encoder_ctx_->framerate : AVRational{0, 1};
+    }
+
+    // 获取当前帧索引
+    int64_t get_frame_index() const { return frame_index_; }
+
+private:
+    EncoderConfig config_;
+    AVCodecContext* encoder_ctx_ = nullptr;
+    SwsContext* sws_ctx_ = nullptr;
+    AVFrame* converted_frame_ = nullptr;
+    PacketCallback packet_callback_;
+    int64_t frame_index_ = 0;
+    AVRational time_base_ = {1, 90000}; // 默认时间基
+    int64_t start_time_ = 0; // 编码开始时间
+
+    // 初始化SWS上下文（格式转换）
+    int init_sws_context(AVFrame* frame);
+
+    // 发送数据包到回调
+    void send_packet(AVPacket* packet);
+
+    // 释放资源
+    void cleanup();
+
+    // 获取当前时间戳
+    int64_t get_current_timestamp();
+};
+
+} // namespace ffmpeg

cmvr-es/common/utils/ffmpeg/include/VideoWriter.h

@@ -0,0 +1,63 @@
+// VideoWriter.h
+#pragma once
+
+#include <string>
+#include <memory>
+
+
+extern "C"{
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+}
+
+namespace ffmpeg {
+
+    struct WriterConfig {
+        std::string output_file = "output.mp4";
+        int width = 1280;
+        int height = 720;
+        int fps = 30;
+        int bitrate = 5000000;
+        AVRational time_base = {1, 90000}; // 默认时间基
+    };
+
+    class VideoWriter {
+    public:
+        VideoWriter();
+        ~VideoWriter();
+
+        // 初始化写入器
+        int initialize(const WriterConfig& config, AVCodecContext* codec_ctx);
+
+        // 写入数据包
+        int write_packet(AVPacket* packet, int64_t pts, int64_t dts);
+
+        // 写入文件尾
+        int finish();
+
+        // 获取输出格式上下文
+        AVFormatContext* get_format_context() { return fmt_ctx_; }
+
+        // 获取视频流
+        AVStream* get_video_stream() { return video_stream_; }
+
+        // 获取输出时间基
+        AVRational get_output_time_base() const {
+            return video_stream_ ? video_stream_->time_base : AVRational{0, 1};
+        }
+
+    private:
+        WriterConfig config_;
+        AVFormatContext* fmt_ctx_ = nullptr;
+        AVStream* video_stream_ = nullptr;
+        AVCodecContext* codec_ctx_ref_ = nullptr;
+        int64_t frame_count_ = 0;
+
+        // 初始化输出格式
+        int init_output_format();
+
+        // 释放资源
+        void cleanup();
+    };
+
+} // namespace ffmpeg

cmvr-es/common/utils/ffmpeg/src/CameraCapture.cpp

@@ -0,0 +1,188 @@
+// CameraCapture.cpp
+#include "../include/CameraCapture.h"
+#include <iostream>
+#include <thread>
+#include <chrono>
+
+namespace ffmpeg {
+
+CameraCapture::CameraCapture() {
+    packet_ = av_packet_alloc();
+    frame_ = av_frame_alloc();
+}
+
+CameraCapture::~CameraCapture() {
+    stop_capture();
+    cleanup();
+
+    if (packet_) av_packet_free(&packet_);
+    if (frame_) av_frame_free(&frame_);
+}
+
+int CameraCapture::initialize(const CameraConfig& config) {
+    config_ = config;
+
+    // 初始化设备
+    int ret = init_device();
+    if (ret < 0) {
+        std::cerr << "初始化设备失败" << std::endl;
+        return ret;
+    }
+
+    // 初始化解码器
+    ret = init_decoder();
+    if (ret < 0) {
+        std::cerr << "初始化解码器失败" << std::endl;
+        return ret;
+    }
+
+    return 0;
+}
+
+int CameraCapture::init_device() {
+    AVInputFormat* input_fmt = nullptr;
+    std::string device_path;
+
+#ifdef _WIN32
+    input_fmt = av_find_input_format("dshow");
+    device_path = "video=" + config_.device_name;
+#else
+    input_fmt = av_find_input_format("v4l2");
+    device_path = config_.device_name;
+#endif
+
+    if (!input_fmt) {
+        std::cerr << "找不到输入格式" << std::endl;
+        return -1;
+    }
+
+    // 打开摄像头
+    int ret = avformat_open_input(&fmt_ctx_, device_path.c_str(), input_fmt, nullptr);
+    if (ret < 0) {
+        char err_buf[1024];
+        av_strerror(ret, err_buf, sizeof(err_buf));
+        std::cerr << "打开摄像头失败: " << err_buf << std::endl;
+        return ret;
+    }
+
+    // 查找流信息
+    ret = avformat_find_stream_info(fmt_ctx_, nullptr);
+    if (ret < 0) {
+        std::cerr << "查找流信息失败" << std::endl;
+        return ret;
+    }
+
+    // 查找视频流
+    video_stream_index_ = av_find_best_stream(fmt_ctx_, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
+    if (video_stream_index_ < 0) {
+        std::cerr << "找不到视频流" << std::endl;
+        return video_stream_index_;
+    }
+
+    return 0;
+}
+
+int CameraCapture::init_decoder() {
+    AVStream* stream = fmt_ctx_->streams[video_stream_index_];
+    const AVCodec* decoder = avcodec_find_decoder(stream->codecpar->codec_id);
+
+    if (!decoder) {
+        std::cerr << "找不到解码器" << std::endl;
+        return -1;
+    }
+
+    decoder_ctx_ = avcodec_alloc_context3(decoder);
+    if (!decoder_ctx_) {
+        std::cerr << "分配解码器上下文失败" << std::endl;
+        return -1;
+    }
+
+    // 复制参数到解码器上下文
+    int ret = avcodec_parameters_to_context(decoder_ctx_, stream->codecpar);
+    if (ret < 0) {
+        std::cerr << "复制解码器参数失败" << std::endl;
+        return ret;
+    }
+
+    // 打开解码器
+    ret = avcodec_open2(decoder_ctx_, decoder, nullptr);
+    if (ret < 0) {
+        std::cerr << "打开解码器失败" << std::endl;
+        return ret;
+    }
+
+    return 0;
+}
+
+int CameraCapture::start_capture(FrameCallback callback) {
+    if (!callback || !fmt_ctx_ || !decoder_ctx_) {
+        std::cerr << "参数无效或未初始化" << std::endl;
+        return -1;
+    }
+
+    is_capturing_ = true;
+
+    while (is_capturing_) {
+        // 读取数据包
+        int ret = av_read_frame(fmt_ctx_, packet_);
+        if (ret < 0) {
+            if (ret == AVERROR(EAGAIN)) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+                continue;
+            }
+            std::cerr << "读取帧失败: " << ret << std::endl;
+            break;
+        }
+
+        // 只处理视频流
+        if (packet_->stream_index == video_stream_index_) {
+            // 发送数据包到解码器
+            ret = avcodec_send_packet(decoder_ctx_, packet_);
+            if (ret < 0 && ret != AVERROR(EAGAIN)) {
+                std::cerr << "发送数据包到解码器失败: " << ret << std::endl;
+                av_packet_unref(packet_);
+                continue;
+            }
+
+            // 接收解码后的帧
+            while (ret >= 0) {
+                ret = avcodec_receive_frame(decoder_ctx_, frame_);
+                if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
+                    break;
+                } else if (ret < 0) {
+                    std::cerr << "接收解码帧失败: " << ret << std::endl;
+                    break;
+                }
+
+                // 回调处理帧
+                callback(frame_);
+            }
+        }
+
+        av_packet_unref(packet_);
+
+        // 控制帧率
+        std::this_thread::sleep_for(std::chrono::milliseconds(1000 / config_.fps));
+    }
+
+    return 0;
+}
+
+void CameraCapture::stop_capture() {
+    is_capturing_ = false;
+}
+
+void CameraCapture::cleanup() {
+    if (decoder_ctx_) {
+        avcodec_close(decoder_ctx_);
+        avcodec_free_context(&decoder_ctx_);
+    }
+
+    if (fmt_ctx_) {
+        avformat_close_input(&fmt_ctx_);
+        avformat_free_context(fmt_ctx_);
+        fmt_ctx_ = nullptr;
+    }
+}
+
+} // namespace ffmpeg

cmvr-es/common/utils/ffmpeg/src/VideoFrameEncoder.cpp

@@ -0,0 +1,240 @@
+// VideoFrameEncoder.cpp
+#include "../include/VideoFrameEncoder.h"
+#include <iostream>
+#include <chrono>
+
+namespace ffmpeg {
+
+VideoFrameEncoder::VideoFrameEncoder() {
+    converted_frame_ = av_frame_alloc();
+    start_time_ = av_gettime(); // 获取初始时间
+}
+
+VideoFrameEncoder::~VideoFrameEncoder() {
+    cleanup();
+    if (converted_frame_) av_frame_free(&converted_frame_);
+}
+
+int64_t VideoFrameEncoder::get_current_timestamp() {
+    int64_t current_time = av_gettime();
+    int64_t elapsed = current_time - start_time_;
+
+    // 将微秒转换为时间基单位
+    return av_rescale_q(elapsed, {1, 1000000}, time_base_);
+}
+
+int VideoFrameEncoder::initialize(const EncoderConfig& config) {
+    config_ = config;
+
+    // 查找编码器
+    //const AVCodec* encoder = avcodec_find_encoder(AV_CODEC_ID_H264);
+    const AVCodec* encoder = avcodec_find_encoder_by_name(config.codec.c_str());
+    if (!encoder) {
+        std::cerr << "找不到H.264编码器" << std::endl;
+        return -1;
+    }
+
+    // 分配编码器上下文
+    encoder_ctx_ = avcodec_alloc_context3(encoder);
+    if (!encoder_ctx_) {
+        std::cerr << "分配编码器上下文失败" << std::endl;
+        return -1;
+    }
+
+    // 配置编码器参数
+    encoder_ctx_->width = config_.width;
+    encoder_ctx_->height = config_.height;
+    encoder_ctx_->pix_fmt = config_.pixel_format;
+
+    // 正确设置时间基
+    encoder_ctx_->time_base = av_make_q(1, config_.fps);
+    encoder_ctx_->framerate = av_make_q(config_.fps, 1);
+
+    // 设置编码器的时间基为帧率倒数
+    time_base_ = encoder_ctx_->time_base;
+
+    encoder_ctx_->bit_rate = config_.bitrate;
+    encoder_ctx_->gop_size = config_.gop_size;
+    encoder_ctx_->max_b_frames = config_.max_b_frames;
+
+    // 设置编码器选项
+    if (encoder->id == AV_CODEC_ID_H264) {
+        av_opt_set(encoder_ctx_->priv_data, "preset", config_.preset.c_str(), 0);
+        av_opt_set(encoder_ctx_->priv_data, "tune", config_.tune.c_str(), 0);
+        av_opt_set(encoder_ctx_->priv_data, "profile", config_.profile.c_str(), 0);
+        av_opt_set(encoder_ctx_->priv_data, "x264-params", "log-level=error", 0);
+    }
+
+    // 打开编码器
+    int ret = avcodec_open2(encoder_ctx_, encoder, nullptr);
+    if (ret < 0) {
+        char err_buf[1024];
+        av_strerror(ret, err_buf, sizeof(err_buf));
+        std::cerr << "打开编码器失败: " << err_buf << std::endl;
+        return ret;
+    }
+
+    // 准备转换后的帧
+    converted_frame_->width = config_.width;
+    converted_frame_->height = config_.height;
+    converted_frame_->format = config_.pixel_format;
+
+    ret = av_frame_get_buffer(converted_frame_, 0);
+    if (ret < 0) {
+        std::cerr << "分配帧缓冲区失败" << std::endl;
+        return ret;
+    }
+
+    return 0;
+}
+
+int VideoFrameEncoder::init_sws_context(AVFrame* frame) {
+    if (!frame) {
+        std::cerr << "输入帧为空" << std::endl;
+        return -1;
+    }
+
+    // 如果格式相同，不需要转换
+    if (frame->format == config_.pixel_format &&
+        frame->width == config_.width &&
+        frame->height == config_.height) {
+        sws_ctx_ = nullptr;
+        return 0;
+    }
+
+    // 创建转换上下文
+    sws_ctx_ = sws_getContext(
+        frame->width, frame->height, static_cast<AVPixelFormat>(frame->format),
+        config_.width, config_.height, config_.pixel_format,
+        SWS_BILINEAR, nullptr, nullptr, nullptr
+    );
+
+    if (!sws_ctx_) {
+        std::cerr << "创建SWS上下文失败" << std::endl;
+        return -1;
+    }
+
+    return 0;
+}
+
+int VideoFrameEncoder::encode_frame(AVFrame* frame) {
+    if (!encoder_ctx_ || !frame) {
+        std::cerr << "编码器未初始化或输入帧为空" << std::endl;
+        return -1;
+    }
+
+    // 初始化格式转换（如果需要）
+    if (!sws_ctx_) {
+        int ret = init_sws_context(frame);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    AVFrame* frame_to_encode = frame;
+
+    // 如果需要格式转换
+    if (sws_ctx_) {
+        sws_scale(
+            sws_ctx_,
+            frame->data, frame->linesize, 0, frame->height,
+            converted_frame_->data, converted_frame_->linesize
+        );
+        frame_to_encode = converted_frame_;
+    }
+
+    // 设置正确的PTS
+    frame_to_encode->pts = frame_index_;
+
+    // 发送帧到编码器
+    int ret = avcodec_send_frame(encoder_ctx_, frame_to_encode);
+    if (ret < 0) {
+        char err_buf[1024];
+        av_strerror(ret, err_buf, sizeof(err_buf));
+        std::cerr << "发送帧到编码器失败: " << err_buf << std::endl;
+        return ret;
+    }
+
+    // 接收编码后的数据包
+    AVPacket* packet = av_packet_alloc();
+    while (ret >= 0) {
+        ret = avcodec_receive_packet(encoder_ctx_, packet);
+        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
+            break;
+        } else if (ret < 0) {
+            std::cerr << "接收编码数据包失败" << std::endl;
+            av_packet_free(&packet);
+            return ret;
+        }
+
+        // 通过回调发送数据包，传递正确的PTS/DTS
+        if (packet_callback_) {
+            packet_callback_(packet, packet->pts, packet->dts);
+        }
+
+        av_packet_unref(packet);
+    }
+
+    av_packet_free(&packet);
+
+    // 递增帧索引
+    frame_index_++;
+
+    return 0;
+}
+
+int VideoFrameEncoder::flush() {
+    if (!encoder_ctx_) {
+        return 0;
+    }
+
+    // 发送空帧刷新编码器
+    int ret = avcodec_send_frame(encoder_ctx_, nullptr);
+    if (ret < 0) {
+        std::cerr << "发送刷新帧失败" << std::endl;
+        return ret;
+    }
+
+    // 接收所有剩余数据包
+    AVPacket* packet = av_packet_alloc();
+    while (true) {
+        ret = avcodec_receive_packet(encoder_ctx_, packet);
+        if (ret == AVERROR_EOF) {
+            break;
+        } else if (ret < 0) {
+            std::cerr << "接收刷新数据包失败" << std::endl;
+            av_packet_free(&packet);
+            return ret;
+        }
+
+        // 通过回调发送数据包
+        if (packet_callback_) {
+            packet_callback_(packet, packet->pts, packet->dts);
+        }
+
+        av_packet_unref(packet);
+    }
+
+    av_packet_free(&packet);
+    return 0;
+}
+
+void VideoFrameEncoder::send_packet(AVPacket* packet) {
+    if (packet_callback_) {
+        packet_callback_(packet, packet->pts, packet->dts);
+    }
+}
+
+void VideoFrameEncoder::cleanup() {
+    if (sws_ctx_) {
+        sws_freeContext(sws_ctx_);
+        sws_ctx_ = nullptr;
+    }
+
+    if (encoder_ctx_) {
+        avcodec_close(encoder_ctx_);
+        avcodec_free_context(&encoder_ctx_);
+    }
+}
+
+} // namespace ffmpeg

cmvr-es/common/utils/ffmpeg/src/VideoWriter.cpp

@@ -0,0 +1,153 @@
+// VideoWriter.cpp
+#include "../include/VideoWriter.h"
+#include <iostream>
+
+namespace ffmpeg {
+
+VideoWriter::VideoWriter() {}
+
+VideoWriter::~VideoWriter() {
+    cleanup();
+}
+
+int VideoWriter::initialize(const WriterConfig& config, AVCodecContext* codec_ctx) {
+    config_ = config;
+    codec_ctx_ref_ = codec_ctx;
+
+    if (!codec_ctx) {
+        std::cerr << "编码器上下文为空" << std::endl;
+        return -1;
+    }
+
+    // 分配输出格式上下文
+    int ret = avformat_alloc_output_context2(&fmt_ctx_, nullptr, nullptr, config_.output_file.c_str());
+    if (ret < 0 || !fmt_ctx_) {
+        std::cerr << "分配输出格式上下文失败" << std::endl;
+        return ret;
+    }
+
+    // 创建输出流
+    video_stream_ = avformat_new_stream(fmt_ctx_, nullptr);
+    if (!video_stream_) {
+        std::cerr << "创建输出流失败" << std::endl;
+        return -1;
+    }
+
+    // 复制编码器参数到输出流
+    ret = avcodec_parameters_from_context(video_stream_->codecpar, codec_ctx);
+    if (ret < 0) {
+        std::cerr << "复制编码器参数失败" << std::endl;
+        return ret;
+    }
+
+    // 设置流时间基（使用配置的时间基）
+    video_stream_->time_base = config_.time_base;
+
+    // 设置流的平均帧率
+    video_stream_->avg_frame_rate = av_make_q(config_.fps, 1);
+
+    // 打开输出文件
+    if (!(fmt_ctx_->oformat->flags & AVFMT_NOFILE)) {
+        ret = avio_open(&fmt_ctx_->pb, config_.output_file.c_str(), AVIO_FLAG_WRITE);
+        if (ret < 0) {
+            char err_buf[1024];
+            av_strerror(ret, err_buf, sizeof(err_buf));
+            std::cerr << "打开输出文件失败: " << err_buf << std::endl;
+            return ret;
+        }
+    }
+
+    // 写入文件头
+    ret = avformat_write_header(fmt_ctx_, nullptr);
+    if (ret < 0) {
+        std::cerr << "写入文件头失败" << std::endl;
+        return ret;
+    }
+
+    std::cout << "视频写入器初始化完成，输出文件: " << config_.output_file
+              << " 时间基: " << video_stream_->time_base.num << "/" << video_stream_->time_base.den
+              << std::endl;
+    return 0;
+}
+
+int VideoWriter::write_packet(AVPacket* packet, int64_t pts, int64_t dts) {
+    if (!fmt_ctx_ || !video_stream_) {
+        std::cerr << "写入器未初始化" << std::endl;
+        return -1;
+    }
+
+    // 克隆数据包，避免修改原始数据
+    AVPacket* cloned_packet = av_packet_clone(packet);
+    if (!cloned_packet) {
+        std::cerr << "克隆数据包失败" << std::endl;
+        return -1;
+    }
+
+    // 设置数据包属性
+    cloned_packet->stream_index = video_stream_->index;
+
+    // 如果传入的PTS/DTS有效，使用它们
+    // if (pts != AV_NOPTS_VALUE) {
+    //     cloned_packet->pts = pts;
+    // } else {
+    //     // 基于帧计数计算PTS
+    //     cloned_packet->pts = av_rescale_q(frame_count_, codec_ctx_ref_->time_base, video_stream_->time_base);
+    // }
+    // 基于帧计数计算PTS
+    cloned_packet->pts = av_rescale_q(frame_count_, codec_ctx_ref_->time_base, video_stream_->time_base);
+    // if (dts != AV_NOPTS_VALUE) {
+    //     cloned_packet->dts = dts;
+    // } else {
+    //     // 如果没有DTS，使用PTS
+    //     cloned_packet->dts = cloned_packet->pts;
+    // }
+    cloned_packet->dts = cloned_packet->pts;
+
+
+    cloned_packet->duration = av_rescale_q(1, codec_ctx_ref_->time_base, video_stream_->time_base);
+
+    // 写入数据包
+    int ret = av_interleaved_write_frame(fmt_ctx_, cloned_packet);
+
+    if (ret < 0) {
+        char err_buf[1024];
+        av_strerror(ret, err_buf, sizeof(err_buf));
+        std::cerr << "写入数据包失败: " << err_buf << std::endl;
+    } else {
+        frame_count_++;
+    }
+
+    av_packet_free(&cloned_packet);
+    return ret;
+}
+
+int VideoWriter::finish() {
+    if (!fmt_ctx_) {
+        return 0;
+    }
+
+    // 写入文件尾
+    int ret = av_write_trailer(fmt_ctx_);
+    if (ret < 0) {
+        std::cerr << "写入文件尾失败" << std::endl;
+    } else {
+        std::cout << "视频写入完成，总帧数: " << frame_count_ << std::endl;
+    }
+
+    cleanup();
+    return ret;
+}
+
+void VideoWriter::cleanup() {
+    if (fmt_ctx_) {
+        if (!(fmt_ctx_->oformat->flags & AVFMT_NOFILE) && fmt_ctx_->pb) {
+            avio_close(fmt_ctx_->pb);
+        }
+        avformat_free_context(fmt_ctx_);
+        fmt_ctx_ = nullptr;
+    }
+    video_stream_ = nullptr;
+    codec_ctx_ref_ = nullptr;
+}
+
+} // namespace ffmpeg

cmvr-es/devices/camera/realsense_camera/CMakeLists.txt

@@ -1,3 +1,5 @@
+
+find_package(OpenCV REQUIRED)
 add_library(realsense_camera SHARED src/realsense_camera.cpp)
 
 target_include_directories(realsense_camera PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
@@ -5,7 +7,7 @@ target_include_directories(realsense_camera PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 add_library(cmvr_es::device::realsense_camera ALIAS realsense_camera) # ✅ 命名空间别名 OK
 
 # 链接 librealsense2
-target_link_libraries(realsense_camera PRIVATE realsense2)
+target_link_libraries(realsense_camera PRIVATE realsense2 ${OpenCV_LIBS} )
 
 install(TARGETS realsense_camera LIBRARY DESTINATION lib)
 # --------------------------------------------------------

cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h

@@ -9,9 +9,47 @@
 #include <librealsense2/rs.hpp>
 #include <librealsense2/hpp/rs_internal.hpp>
 
+
 namespace cmvr::device{
     enum StreamMode {RGBD_MODE, COLOR_MODE,DEPTH_MODE};
+    struct FfmpegEncoderInfo {
+        std::string codec_name;                 // 编码器名称（如"h264"、"hevc"）
+        int width = 0;                          // 图像宽度
+        int height = 0;                         // 图像高度
+        int fps = 0;                            // 帧率
+        int64_t frame_pts = 0;
+        bool bRunning = false; // 是否进行编码
+        AVCodecContext* codec_context = nullptr; // 编码器上下文
+        AVFrame* frame = nullptr; // 输入帧
+        AVPacket* packet = nullptr; // 输出包
+        SwsContext* sws_context = nullptr; // 图像转换上下文（如果需要格式转换）
 
+        // 析构函数：释放FFmpeg资源（核心！避免内存泄漏）
+        ~FfmpegEncoderInfo() {
+            // 释放编码帧
+            if (frame) {
+                av_frame_free(&frame);
+                frame = nullptr;
+            }
+            // 释放编码包
+            if (packet) {
+                av_packet_free(&packet);
+                packet = nullptr;
+            }
+            // 释放编码器上下文
+            if (codec_context) {
+                avcodec_close(codec_context);    // 关闭编码器
+                avcodec_free_context(&codec_context);  // 释放上下文
+                codec_context = nullptr;
+            }
+            // 释放格式转换上下文
+            if (sws_context) {
+                sws_freeContext(sws_context);
+                sws_context = nullptr;
+            }
+            //std::cout << "FfmpegEncoderInfo resources released." << std::endl;
+        }
+    };
 
     class RealsenseCamera final : public AbstractCamera {
     public:

cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h

@@ -1,72 +1,25 @@
-//
-// Created by xtkuang on 2025/5/30.
-//
-
+// uvc_camera.h
 #ifndef CMVR_ES_UVC_CAMERA_H
 #define CMVR_ES_UVC_CAMERA_H
 
 #include "utils/base/include/ring_buffer.h"
 #include "camera/abstract_camera.h"
 
-//使用ffmpeg来编码保存视频文件
-#define USE_FFMPEG_ENCODER 1
-
-#if USE_FFMPEG_ENCODER
-#include "speaker/ffmpeg_speaker/include/ffmpeg_ptr.h"
-#endif
+// 使用新的FFmpeg封装类
+#include "common/utils/ffmpeg/include/CameraCapture.h"
+#include "common/utils/ffmpeg/include/VideoFrameEncoder.h"
+#include "common/utils/ffmpeg/include/VideoWriter.h"
 
 namespace cmvr::device {
     enum CameraMode {PHOTO_MODE, VIDEO_MODE};
 
-
     struct ImageData
     {
-        //原始图像数据
         cv::Mat rgbImage;
-        //深度图原始数据
         cv::Mat depthImage;
     };
 
-    struct FfmpegEncoderInfo {
-        std::string codec_name;                 // 编码器名称（如"h264"、"hevc"）
-        int width = 0;                          // 图像宽度
-        int height = 0;                         // 图像高度
-        int fps = 0;                            // 帧率
-        int64_t frame_pts = 0;
-        bool bRunning = false; // 是否进行编码
-        AVCodecContext* codec_context = nullptr; // 编码器上下文
-        AVFrame* frame = nullptr; // 输入帧
-        AVPacket* packet = nullptr; // 输出包
-        SwsContext* sws_context = nullptr; // 图像转换上下文（如果需要格式转换）
-
-        // 析构函数：释放FFmpeg资源（核心！避免内存泄漏）
-        ~FfmpegEncoderInfo() {
-            // 释放编码帧
-            if (frame) {
-                av_frame_free(&frame);
-                frame = nullptr;
-            }
-            // 释放编码包
-            if (packet) {
-                av_packet_free(&packet);
-                packet = nullptr;
-            }
-            // 释放编码器上下文
-            if (codec_context) {
-                avcodec_close(codec_context);    // 关闭编码器
-                avcodec_free_context(&codec_context);  // 释放上下文
-                codec_context = nullptr;
-            }
-            // 释放格式转换上下文
-            if (sws_context) {
-                sws_freeContext(sws_context);
-                sws_context = nullptr;
-            }
-            //std::cout << "FfmpegEncoderInfo resources released." << std::endl;
-        }
-    };
-
-    //USB摄像机
+    // USB摄像机
     class UVCCamera final : public AbstractCamera {
     public:
         explicit UVCCamera(const XmlNode& config);
@@ -82,56 +35,41 @@ namespace cmvr::device {
         void updateParams(const std::pair<std::string, std::string>& param) override;
         void startRecording(const std::string &video_path) override;
         void stopRecording() override;
-        void pauseRecording() override;
-        void resumeRecording() override;
-        // 初始化单个编码器的通用函数（复用逻辑，避免重复代码）
-        static bool initSingleEncoder(std::shared_ptr<FfmpegEncoderInfo>& encoder,
-                              const std::string& codec_name,
-                              int width, int height, int fps);
-        static bool encodeFrameWithEncoder(std::shared_ptr<FfmpegEncoderInfo>& encoder,const cv::Mat& frame,std::vector<uint8_t>& encoded_frame,bool& is_key);
         void getEncodedFrame(StreamFrameData& frame_data, size_t& index) override;
 
         bool startStreaming() override;
         void stopStreaming() override;
+
     private:
         void streaming_worker_();
-        void recording_worker_();
+
+        // 辅助函数
+        static bool convertAVFrameToMat(AVFrame* frame, cv::Mat& mat);
 
         int fps_;
         int width_;
         int height_;
         std::string serial_;
-        cv::VideoCapture cap_;
         size_t buffer_size_;
         std::string codec_;
         CameraMode mode_;
 
-        // std::shared_ptr<cv::Mat> current_image_;
         std::shared_ptr<SPMCRingBuffer<StreamFrameData>> stream_frame_buffer_;
         std::string current_video_path_;
 
         std::mutex ctrl_mtx_{};
-        std::unique_ptr<cv::VideoWriter> video_writer_;
         std::shared_ptr<std::thread> stream_thread_;
-        std::shared_ptr<std::thread> recording_thread_;
 
+        // 新的FFmpeg封装类实例
+        std::unique_ptr<ffmpeg::CameraCapture> camera_capture_;
+        std::unique_ptr<ffmpeg::VideoFrameEncoder> video_encoder_;
+        std::unique_ptr<ffmpeg::VideoWriter> video_writer_;
 
-        cv::Mat latest_frame_;  // 存储最新帧
-        std::mutex frame_mutex_;  // 保护最新帧的访问
+        // 用于临时存储最新帧
+        std::mutex latest_frame_mutex_;
+        cv::Mat latest_rgb_frame_;
+        bool latest_frame_valid_ = false;
 
-        std::string output_path_;
-        bool is_recording_ = false;
-        // ffmpeg录像
-        AVFormatContext* format_context_ = nullptr;
-        AVCodecContext* codec_context_ = nullptr;
-        AVStream* stream_ = nullptr;
-        AVFrame* frame_ = nullptr;
-        AVPacket* packet_ = nullptr;
-        SwsContext* sws_context_ = nullptr;
-        int frame_count_ = 0;
-
-        std::shared_ptr<FfmpegEncoderInfo> rgbEncoder_;//rgb图像编码
-        std::shared_ptr<FfmpegEncoderInfo> depthEncoder_;//深度图编码
         size_t streamIndex_ = 0;
         size_t recordingIndex_ = 0;
         size_t getImageIndex_ = 0;
@@ -145,6 +83,4 @@ namespace cmvr::device {
     };
 }
 
-
-
-#endif //CMVR_ES_UVC_CAMERA_H
+#endif // CMVR_ES_UVC_CAMERA_H

cmvr-es/devices/microphone/ffmpeg_microphone/src/ffmpeg_microphone.cpp

@@ -32,7 +32,7 @@ ffmpegMicroPhone::ffmpegMicroPhone(const config::FFMpegMicroPhoneConfig& cfg):in
     sample_rate_(44100), channels_(2), config_(cfg)
 {
     id_ = config_.id();
-    input_device_ = cfg_.getAttrString("alsa");
+    input_device_ = config_.input_device();
     channels_ = config_.channels();
     sample_rate_ = config_.samplerate();
     state_.volume = config_.volume();
@@ -174,9 +174,9 @@ bool ffmpegMicroPhone::initFFmpeg() {
     int err;
     avdevice_register_all();
     // 初始化输入设备
-    AVInputFormat* input_fmt = av_find_input_format("alsa");
+    AVInputFormat* input_fmt = av_find_input_format("pulse");
     if (!input_fmt) {
-        LOG(ERROR) << "Could not find ALSA input format";
+        LOG(ERROR) << "Could not find pulse input format";
         return false;
     }
 

protos/cmvr/config/microphone_config/microphone_config.proto

@@ -7,6 +7,7 @@ message FFMpegMicroPhoneConfig{
     int32 sampleRate = 3;
     int32 volume = 4;
     bool enable = 5;
+    string input_device = 6;
 }
 
 message MicroPhoneConfig{