Merge branch 'dev' of http://192.168.0.100:18088/smart_bench/cmvr-es into dev

CMakeLists.txt

@@ -82,7 +82,6 @@ install(TARGETS proto LIBRARY DESTINATION lib)
 # include
 ############################################################
 include_directories(
-        /usr/include/opencv4
         /usr/local/include/
         ${PROTO_BINARY_DIR}
         ${PROJECT_SOURCE_DIR}/include

cmvr-es/common/CMakeLists.txt

@@ -3,6 +3,10 @@
 
 file(GLOB SRC
         ${CMAKE_CURRENT_SOURCE_DIR}/utils/config_helper/src/config_setting.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/utils/ffmpeg/src/CameraCapture.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/utils/ffmpeg/src/VideoFrameEncoder.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/utils/ffmpeg/src/VideoWriter.cpp
+
         ${CMAKE_CURRENT_SOURCE_DIR}/curve/src/s_curve.cpp
 )
 
@@ -14,6 +18,11 @@ target_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 
 target_link_libraries(common PUBLIC
         cmvr_es::proto
+        avformat
+        avdevice
+        avutil
+        avcodec
+        swscale
 )
 
 add_library(cmvr_es::common ALIAS common)

cmvr-es/common/config/camera_config/camera_config.pb.txt

@@ -1,5 +1,20 @@
 realsense_cameras {
-  id: "cam2"
+  id: "cam1"
+  serialNumber: "243122074587"
+  width: 640
+  height: 480
+  fps: 30
+  codec: "H265"
+  camera_mode: CAMERA_MODE_VIDEO
+  stream_mode: STREAM_MODE_RGBD
+  align_mode: ALIGN_MODE_COLOR
+  buffer_size: 30
+  sync: false
+  enable: false
+}
+
+realsense_cameras {
+  id: "right_hand_cam"
   serialNumber: "243122072252"
   width: 1280
   height: 720
@@ -9,19 +24,49 @@ realsense_cameras {
   stream_mode: STREAM_MODE_RGBD
   align_mode: ALIGN_MODE_COLOR
   buffer_size: 30
-  sync: true
+  sync: false
   enable: true
 }
 
+realsense_cameras {
+  id: "cam3"
+  serialNumber: "243122075614"
+  width: 640
+  height: 480
+  fps: 30
+  codec: "H265"
+  camera_mode: CAMERA_MODE_VIDEO
+  stream_mode: STREAM_MODE_RGBD
+  align_mode: ALIGN_MODE_COLOR
+  buffer_size: 30
+  sync: false
+  enable: false
+}
+
 uvc_cameras {
-  id: "cam1"
+  id: "left_eye_cam"
   usb: "/dev/uvc_left_camera"
   width: 640
   height: 480
   fps: 30
   codec: "H265"
-  camera_mode: CAMERA_MODE_PHOTO
+  camera_mode: CAMERA_MODE_VIDEO
   stream_mode: STREAM_MODE_RGB
   buffer_size: 30
   enable: false
 }
+
+realsense_cameras {
+  id: "cam5"
+  serialNumber: "243122075389"
+  width: 640
+  height: 480
+  fps: 30
+  codec: "H265"
+  camera_mode: CAMERA_MODE_VIDEO
+  stream_mode: STREAM_MODE_RGBD
+  align_mode: ALIGN_MODE_COLOR
+  buffer_size: 30
+  sync: false
+  enable: false
+}

cmvr-es/common/config/microphone_config/microphone_config.pb.txt

@@ -3,5 +3,6 @@ ffmpeg_microphones {
     channels: 2
     sampleRate: 44100
     volume: 100
-    enable: true
+    enable: false
+    input_device: "default"
 }

cmvr-es/common/config/speaker_config/speaker_config.pb.txt

@@ -1,4 +1,4 @@
 ffmpeg_speakers {
     id: "spk1"
-    enable: true
+    enable: false
 }

cmvr-es/common/utils/ffmpeg/include/CameraCapture.h

@@ -0,0 +1,68 @@
+// CameraCapture.h
+#pragma once
+
+#include <string>
+#include <memory>
+#include <functional>
+
+extern "C"{
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+#include <libavdevice/avdevice.h>
+}
+
+namespace ffmpeg {
+
+    struct CameraConfig {
+        std::string device_name = "0";  // Windows: "0" (dshow), Linux: "0" (对应 /dev/video0)
+        int width = 1280;
+        int height = 720;
+        int fps = 30;
+        AVPixelFormat pixel_format = AV_PIX_FMT_YUV420P;
+    };
+
+    class CameraCapture {
+    public:
+        using FrameCallback = std::function<void(AVFrame* frame)>;
+
+        CameraCapture();
+        ~CameraCapture();
+
+        // 初始化摄像头
+        int initialize(const CameraConfig& config);
+
+        // 开始捕获
+        int start_capture(FrameCallback callback);
+
+        // 停止捕获
+        void stop_capture();
+
+        // 获取当前配置
+        CameraConfig get_config() const { return config_; }
+
+        // 获取输入格式上下文
+        AVFormatContext* get_format_context() { return fmt_ctx_; }
+
+        // 获取视频流索引
+        int get_video_stream_index() const { return video_stream_index_; }
+
+    private:
+        CameraConfig config_;
+        AVFormatContext* fmt_ctx_ = nullptr;
+        AVCodecContext* decoder_ctx_ = nullptr;
+        int video_stream_index_ = -1;
+        bool is_capturing_ = false;
+        AVPacket* packet_ = nullptr;
+        AVFrame* frame_ = nullptr;
+
+        // 初始化设备（平台相关）
+        int init_device();
+
+        // 初始化解码器
+        int init_decoder();
+
+        // 释放资源
+        void cleanup();
+    };
+
+} // namespace ffmpeg

cmvr-es/common/utils/ffmpeg/include/VideoFrameEncoder.h

@@ -0,0 +1,101 @@
+// VideoFrameEncoder.h
+#pragma once
+
+#include <string>
+#include <memory>
+#include <queue>
+#include <mutex>
+#include <condition_variable>
+#include <functional>
+
+extern "C" {
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+#include <libswscale/swscale.h>
+#include <libavutil/opt.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/time.h>
+}
+
+namespace ffmpeg {
+
+struct EncoderConfig {
+    int width = 1280;
+    int height = 720;
+    int fps = 30;
+    int bitrate = 5000000;  // 5Mbps
+    int gop_size = 15;      // GOP大小
+    int max_b_frames = 2;   // 最大B帧数
+    AVPixelFormat pixel_format = AV_PIX_FMT_YUV420P;
+    std::string preset = "fast";     // 编码速度预设
+    std::string tune = "zerolatency"; // 调优参数
+    std::string profile = "main";    // 编码profile
+    std::string codec = "libx265";
+};
+
+class VideoFrameEncoder {
+public:
+    using PacketCallback = std::function<void(AVPacket* packet, int64_t pts, int64_t dts)>;
+
+    VideoFrameEncoder();
+    ~VideoFrameEncoder();
+
+    // 初始化编码器
+    int initialize(const EncoderConfig& config);
+
+    // 编码一帧
+    int encode_frame(AVFrame* frame);
+
+    // 设置编码数据包回调
+    void set_packet_callback(PacketCallback callback) {
+        packet_callback_ = callback;
+    }
+
+    // 设置时间戳类型
+    void set_time_base(AVRational time_base) {
+        time_base_ = time_base;
+    }
+
+    // 刷新编码器（处理剩余数据）
+    int flush();
+
+    // 获取编码器上下文
+    AVCodecContext* get_codec_context() { return encoder_ctx_; }
+
+    // 获取时间基
+    AVRational get_time_base() const {
+        return encoder_ctx_ ? encoder_ctx_->time_base : AVRational{0, 1};
+    }
+
+    // 获取帧率
+    AVRational get_framerate() const {
+        return encoder_ctx_ ? encoder_ctx_->framerate : AVRational{0, 1};
+    }
+
+    // 获取当前帧索引
+    int64_t get_frame_index() const { return frame_index_; }
+
+private:
+    EncoderConfig config_;
+    AVCodecContext* encoder_ctx_ = nullptr;
+    SwsContext* sws_ctx_ = nullptr;
+    AVFrame* converted_frame_ = nullptr;
+    PacketCallback packet_callback_;
+    int64_t frame_index_ = 0;
+    AVRational time_base_ = {1, 90000}; // 默认时间基
+    int64_t start_time_ = 0; // 编码开始时间
+
+    // 初始化SWS上下文（格式转换）
+    int init_sws_context(AVFrame* frame);
+
+    // 发送数据包到回调
+    void send_packet(AVPacket* packet);
+
+    // 释放资源
+    void cleanup();
+
+    // 获取当前时间戳
+    int64_t get_current_timestamp();
+};
+
+} // namespace ffmpeg

cmvr-es/common/utils/ffmpeg/include/VideoWriter.h

@@ -0,0 +1,63 @@
+// VideoWriter.h
+#pragma once
+
+#include <string>
+#include <memory>
+
+
+extern "C"{
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+}
+
+namespace ffmpeg {
+
+    struct WriterConfig {
+        std::string output_file = "output.mp4";
+        int width = 1280;
+        int height = 720;
+        int fps = 30;
+        int bitrate = 5000000;
+        AVRational time_base = {1, 90000}; // 默认时间基
+    };
+
+    class VideoWriter {
+    public:
+        VideoWriter();
+        ~VideoWriter();
+
+        // 初始化写入器
+        int initialize(const WriterConfig& config, AVCodecContext* codec_ctx);
+
+        // 写入数据包
+        int write_packet(AVPacket* packet, int64_t pts, int64_t dts);
+
+        // 写入文件尾
+        int finish();
+
+        // 获取输出格式上下文
+        AVFormatContext* get_format_context() { return fmt_ctx_; }
+
+        // 获取视频流
+        AVStream* get_video_stream() { return video_stream_; }
+
+        // 获取输出时间基
+        AVRational get_output_time_base() const {
+            return video_stream_ ? video_stream_->time_base : AVRational{0, 1};
+        }
+
+    private:
+        WriterConfig config_;
+        AVFormatContext* fmt_ctx_ = nullptr;
+        AVStream* video_stream_ = nullptr;
+        AVCodecContext* codec_ctx_ref_ = nullptr;
+        int64_t frame_count_ = 0;
+
+        // 初始化输出格式
+        int init_output_format();
+
+        // 释放资源
+        void cleanup();
+    };
+
+} // namespace ffmpeg

cmvr-es/common/utils/ffmpeg/src/CameraCapture.cpp

@@ -0,0 +1,188 @@
+// CameraCapture.cpp
+#include "../include/CameraCapture.h"
+#include <iostream>
+#include <thread>
+#include <chrono>
+
+namespace ffmpeg {
+
+CameraCapture::CameraCapture() {
+    packet_ = av_packet_alloc();
+    frame_ = av_frame_alloc();
+}
+
+CameraCapture::~CameraCapture() {
+    stop_capture();
+    cleanup();
+
+    if (packet_) av_packet_free(&packet_);
+    if (frame_) av_frame_free(&frame_);
+}
+
+int CameraCapture::initialize(const CameraConfig& config) {
+    config_ = config;
+
+    // 初始化设备
+    int ret = init_device();
+    if (ret < 0) {
+        std::cerr << "初始化设备失败" << std::endl;
+        return ret;
+    }
+
+    // 初始化解码器
+    ret = init_decoder();
+    if (ret < 0) {
+        std::cerr << "初始化解码器失败" << std::endl;
+        return ret;
+    }
+
+    return 0;
+}
+
+int CameraCapture::init_device() {
+    AVInputFormat* input_fmt = nullptr;
+    std::string device_path;
+
+#ifdef _WIN32
+    input_fmt = av_find_input_format("dshow");
+    device_path = "video=" + config_.device_name;
+#else
+    input_fmt = av_find_input_format("v4l2");
+    device_path = config_.device_name;
+#endif
+
+    if (!input_fmt) {
+        std::cerr << "找不到输入格式" << std::endl;
+        return -1;
+    }
+
+    // 打开摄像头
+    int ret = avformat_open_input(&fmt_ctx_, device_path.c_str(), input_fmt, nullptr);
+    if (ret < 0) {
+        char err_buf[1024];
+        av_strerror(ret, err_buf, sizeof(err_buf));
+        std::cerr << "打开摄像头失败: " << err_buf << std::endl;
+        return ret;
+    }
+
+    // 查找流信息
+    ret = avformat_find_stream_info(fmt_ctx_, nullptr);
+    if (ret < 0) {
+        std::cerr << "查找流信息失败" << std::endl;
+        return ret;
+    }
+
+    // 查找视频流
+    video_stream_index_ = av_find_best_stream(fmt_ctx_, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
+    if (video_stream_index_ < 0) {
+        std::cerr << "找不到视频流" << std::endl;
+        return video_stream_index_;
+    }
+
+    return 0;
+}
+
+int CameraCapture::init_decoder() {
+    AVStream* stream = fmt_ctx_->streams[video_stream_index_];
+    const AVCodec* decoder = avcodec_find_decoder(stream->codecpar->codec_id);
+
+    if (!decoder) {
+        std::cerr << "找不到解码器" << std::endl;
+        return -1;
+    }
+
+    decoder_ctx_ = avcodec_alloc_context3(decoder);
+    if (!decoder_ctx_) {
+        std::cerr << "分配解码器上下文失败" << std::endl;
+        return -1;
+    }
+
+    // 复制参数到解码器上下文
+    int ret = avcodec_parameters_to_context(decoder_ctx_, stream->codecpar);
+    if (ret < 0) {
+        std::cerr << "复制解码器参数失败" << std::endl;
+        return ret;
+    }
+
+    // 打开解码器
+    ret = avcodec_open2(decoder_ctx_, decoder, nullptr);
+    if (ret < 0) {
+        std::cerr << "打开解码器失败" << std::endl;
+        return ret;
+    }
+
+    return 0;
+}
+
+int CameraCapture::start_capture(FrameCallback callback) {
+    if (!callback || !fmt_ctx_ || !decoder_ctx_) {
+        std::cerr << "参数无效或未初始化" << std::endl;
+        return -1;
+    }
+
+    is_capturing_ = true;
+
+    while (is_capturing_) {
+        // 读取数据包
+        int ret = av_read_frame(fmt_ctx_, packet_);
+        if (ret < 0) {
+            if (ret == AVERROR(EAGAIN)) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(1));
+                continue;
+            }
+            std::cerr << "读取帧失败: " << ret << std::endl;
+            break;
+        }
+
+        // 只处理视频流
+        if (packet_->stream_index == video_stream_index_) {
+            // 发送数据包到解码器
+            ret = avcodec_send_packet(decoder_ctx_, packet_);
+            if (ret < 0 && ret != AVERROR(EAGAIN)) {
+                std::cerr << "发送数据包到解码器失败: " << ret << std::endl;
+                av_packet_unref(packet_);
+                continue;
+            }
+
+            // 接收解码后的帧
+            while (ret >= 0) {
+                ret = avcodec_receive_frame(decoder_ctx_, frame_);
+                if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
+                    break;
+                } else if (ret < 0) {
+                    std::cerr << "接收解码帧失败: " << ret << std::endl;
+                    break;
+                }
+
+                // 回调处理帧
+                callback(frame_);
+            }
+        }
+
+        av_packet_unref(packet_);
+
+        // 控制帧率
+        std::this_thread::sleep_for(std::chrono::milliseconds(1000 / config_.fps));
+    }
+
+    return 0;
+}
+
+void CameraCapture::stop_capture() {
+    is_capturing_ = false;
+}
+
+void CameraCapture::cleanup() {
+    if (decoder_ctx_) {
+        avcodec_close(decoder_ctx_);
+        avcodec_free_context(&decoder_ctx_);
+    }
+
+    if (fmt_ctx_) {
+        avformat_close_input(&fmt_ctx_);
+        avformat_free_context(fmt_ctx_);
+        fmt_ctx_ = nullptr;
+    }
+}
+
+} // namespace ffmpeg

cmvr-es/common/utils/ffmpeg/src/VideoFrameEncoder.cpp

@@ -0,0 +1,240 @@
+// VideoFrameEncoder.cpp
+#include "../include/VideoFrameEncoder.h"
+#include <iostream>
+#include <chrono>
+
+namespace ffmpeg {
+
+VideoFrameEncoder::VideoFrameEncoder() {
+    converted_frame_ = av_frame_alloc();
+    start_time_ = av_gettime(); // 获取初始时间
+}
+
+VideoFrameEncoder::~VideoFrameEncoder() {
+    cleanup();
+    if (converted_frame_) av_frame_free(&converted_frame_);
+}
+
+int64_t VideoFrameEncoder::get_current_timestamp() {
+    int64_t current_time = av_gettime();
+    int64_t elapsed = current_time - start_time_;
+
+    // 将微秒转换为时间基单位
+    return av_rescale_q(elapsed, {1, 1000000}, time_base_);
+}
+
+int VideoFrameEncoder::initialize(const EncoderConfig& config) {
+    config_ = config;
+
+    // 查找编码器
+    //const AVCodec* encoder = avcodec_find_encoder(AV_CODEC_ID_H264);
+    const AVCodec* encoder = avcodec_find_encoder_by_name(config.codec.c_str());
+    if (!encoder) {
+        std::cerr << "找不到H.264编码器" << std::endl;
+        return -1;
+    }
+
+    // 分配编码器上下文
+    encoder_ctx_ = avcodec_alloc_context3(encoder);
+    if (!encoder_ctx_) {
+        std::cerr << "分配编码器上下文失败" << std::endl;
+        return -1;
+    }
+
+    // 配置编码器参数
+    encoder_ctx_->width = config_.width;
+    encoder_ctx_->height = config_.height;
+    encoder_ctx_->pix_fmt = config_.pixel_format;
+
+    // 正确设置时间基
+    encoder_ctx_->time_base = av_make_q(1, config_.fps);
+    encoder_ctx_->framerate = av_make_q(config_.fps, 1);
+
+    // 设置编码器的时间基为帧率倒数
+    time_base_ = encoder_ctx_->time_base;
+
+    encoder_ctx_->bit_rate = config_.bitrate;
+    encoder_ctx_->gop_size = config_.gop_size;
+    encoder_ctx_->max_b_frames = config_.max_b_frames;
+
+    // 设置编码器选项
+    if (encoder->id == AV_CODEC_ID_H264) {
+        av_opt_set(encoder_ctx_->priv_data, "preset", config_.preset.c_str(), 0);
+        av_opt_set(encoder_ctx_->priv_data, "tune", config_.tune.c_str(), 0);
+        av_opt_set(encoder_ctx_->priv_data, "profile", config_.profile.c_str(), 0);
+        av_opt_set(encoder_ctx_->priv_data, "x264-params", "log-level=error", 0);
+    }
+
+    // 打开编码器
+    int ret = avcodec_open2(encoder_ctx_, encoder, nullptr);
+    if (ret < 0) {
+        char err_buf[1024];
+        av_strerror(ret, err_buf, sizeof(err_buf));
+        std::cerr << "打开编码器失败: " << err_buf << std::endl;
+        return ret;
+    }
+
+    // 准备转换后的帧
+    converted_frame_->width = config_.width;
+    converted_frame_->height = config_.height;
+    converted_frame_->format = config_.pixel_format;
+
+    ret = av_frame_get_buffer(converted_frame_, 0);
+    if (ret < 0) {
+        std::cerr << "分配帧缓冲区失败" << std::endl;
+        return ret;
+    }
+
+    return 0;
+}
+
+int VideoFrameEncoder::init_sws_context(AVFrame* frame) {
+    if (!frame) {
+        std::cerr << "输入帧为空" << std::endl;
+        return -1;
+    }
+
+    // 如果格式相同，不需要转换
+    if (frame->format == config_.pixel_format &&
+        frame->width == config_.width &&
+        frame->height == config_.height) {
+        sws_ctx_ = nullptr;
+        return 0;
+    }
+
+    // 创建转换上下文
+    sws_ctx_ = sws_getContext(
+        frame->width, frame->height, static_cast<AVPixelFormat>(frame->format),
+        config_.width, config_.height, config_.pixel_format,
+        SWS_BILINEAR, nullptr, nullptr, nullptr
+    );
+
+    if (!sws_ctx_) {
+        std::cerr << "创建SWS上下文失败" << std::endl;
+        return -1;
+    }
+
+    return 0;
+}
+
+int VideoFrameEncoder::encode_frame(AVFrame* frame) {
+    if (!encoder_ctx_ || !frame) {
+        std::cerr << "编码器未初始化或输入帧为空" << std::endl;
+        return -1;
+    }
+
+    // 初始化格式转换（如果需要）
+    if (!sws_ctx_) {
+        int ret = init_sws_context(frame);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    AVFrame* frame_to_encode = frame;
+
+    // 如果需要格式转换
+    if (sws_ctx_) {
+        sws_scale(
+            sws_ctx_,
+            frame->data, frame->linesize, 0, frame->height,
+            converted_frame_->data, converted_frame_->linesize
+        );
+        frame_to_encode = converted_frame_;
+    }
+
+    // 设置正确的PTS
+    frame_to_encode->pts = frame_index_;
+
+    // 发送帧到编码器
+    int ret = avcodec_send_frame(encoder_ctx_, frame_to_encode);
+    if (ret < 0) {
+        char err_buf[1024];
+        av_strerror(ret, err_buf, sizeof(err_buf));
+        std::cerr << "发送帧到编码器失败: " << err_buf << std::endl;
+        return ret;
+    }
+
+    // 接收编码后的数据包
+    AVPacket* packet = av_packet_alloc();
+    while (ret >= 0) {
+        ret = avcodec_receive_packet(encoder_ctx_, packet);
+        if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
+            break;
+        } else if (ret < 0) {
+            std::cerr << "接收编码数据包失败" << std::endl;
+            av_packet_free(&packet);
+            return ret;
+        }
+
+        // 通过回调发送数据包，传递正确的PTS/DTS
+        if (packet_callback_) {
+            packet_callback_(packet, packet->pts, packet->dts);
+        }
+
+        av_packet_unref(packet);
+    }
+
+    av_packet_free(&packet);
+
+    // 递增帧索引
+    frame_index_++;
+
+    return 0;
+}
+
+int VideoFrameEncoder::flush() {
+    if (!encoder_ctx_) {
+        return 0;
+    }
+
+    // 发送空帧刷新编码器
+    int ret = avcodec_send_frame(encoder_ctx_, nullptr);
+    if (ret < 0) {
+        std::cerr << "发送刷新帧失败" << std::endl;
+        return ret;
+    }
+
+    // 接收所有剩余数据包
+    AVPacket* packet = av_packet_alloc();
+    while (true) {
+        ret = avcodec_receive_packet(encoder_ctx_, packet);
+        if (ret == AVERROR_EOF) {
+            break;
+        } else if (ret < 0) {
+            std::cerr << "接收刷新数据包失败" << std::endl;
+            av_packet_free(&packet);
+            return ret;
+        }
+
+        // 通过回调发送数据包
+        if (packet_callback_) {
+            packet_callback_(packet, packet->pts, packet->dts);
+        }
+
+        av_packet_unref(packet);
+    }
+
+    av_packet_free(&packet);
+    return 0;
+}
+
+void VideoFrameEncoder::send_packet(AVPacket* packet) {
+    if (packet_callback_) {
+        packet_callback_(packet, packet->pts, packet->dts);
+    }
+}
+
+void VideoFrameEncoder::cleanup() {
+    if (sws_ctx_) {
+        sws_freeContext(sws_ctx_);
+        sws_ctx_ = nullptr;
+    }
+
+    if (encoder_ctx_) {
+        avcodec_close(encoder_ctx_);
+        avcodec_free_context(&encoder_ctx_);
+    }
+}
+
+} // namespace ffmpeg

cmvr-es/common/utils/ffmpeg/src/VideoWriter.cpp

@@ -0,0 +1,153 @@
+// VideoWriter.cpp
+#include "../include/VideoWriter.h"
+#include <iostream>
+
+namespace ffmpeg {
+
+VideoWriter::VideoWriter() {}
+
+VideoWriter::~VideoWriter() {
+    cleanup();
+}
+
+int VideoWriter::initialize(const WriterConfig& config, AVCodecContext* codec_ctx) {
+    config_ = config;
+    codec_ctx_ref_ = codec_ctx;
+
+    if (!codec_ctx) {
+        std::cerr << "编码器上下文为空" << std::endl;
+        return -1;
+    }
+
+    // 分配输出格式上下文
+    int ret = avformat_alloc_output_context2(&fmt_ctx_, nullptr, nullptr, config_.output_file.c_str());
+    if (ret < 0 || !fmt_ctx_) {
+        std::cerr << "分配输出格式上下文失败" << std::endl;
+        return ret;
+    }
+
+    // 创建输出流
+    video_stream_ = avformat_new_stream(fmt_ctx_, nullptr);
+    if (!video_stream_) {
+        std::cerr << "创建输出流失败" << std::endl;
+        return -1;
+    }
+
+    // 复制编码器参数到输出流
+    ret = avcodec_parameters_from_context(video_stream_->codecpar, codec_ctx);
+    if (ret < 0) {
+        std::cerr << "复制编码器参数失败" << std::endl;
+        return ret;
+    }
+
+    // 设置流时间基（使用配置的时间基）
+    video_stream_->time_base = config_.time_base;
+
+    // 设置流的平均帧率
+    video_stream_->avg_frame_rate = av_make_q(config_.fps, 1);
+
+    // 打开输出文件
+    if (!(fmt_ctx_->oformat->flags & AVFMT_NOFILE)) {
+        ret = avio_open(&fmt_ctx_->pb, config_.output_file.c_str(), AVIO_FLAG_WRITE);
+        if (ret < 0) {
+            char err_buf[1024];
+            av_strerror(ret, err_buf, sizeof(err_buf));
+            std::cerr << "打开输出文件失败: " << err_buf << std::endl;
+            return ret;
+        }
+    }
+
+    // 写入文件头
+    ret = avformat_write_header(fmt_ctx_, nullptr);
+    if (ret < 0) {
+        std::cerr << "写入文件头失败" << std::endl;
+        return ret;
+    }
+
+    std::cout << "视频写入器初始化完成，输出文件: " << config_.output_file
+              << " 时间基: " << video_stream_->time_base.num << "/" << video_stream_->time_base.den
+              << std::endl;
+    return 0;
+}
+
+int VideoWriter::write_packet(AVPacket* packet, int64_t pts, int64_t dts) {
+    if (!fmt_ctx_ || !video_stream_) {
+        std::cerr << "写入器未初始化" << std::endl;
+        return -1;
+    }
+
+    // 克隆数据包，避免修改原始数据
+    AVPacket* cloned_packet = av_packet_clone(packet);
+    if (!cloned_packet) {
+        std::cerr << "克隆数据包失败" << std::endl;
+        return -1;
+    }
+
+    // 设置数据包属性
+    cloned_packet->stream_index = video_stream_->index;
+
+    // 如果传入的PTS/DTS有效，使用它们
+    // if (pts != AV_NOPTS_VALUE) {
+    //     cloned_packet->pts = pts;
+    // } else {
+    //     // 基于帧计数计算PTS
+    //     cloned_packet->pts = av_rescale_q(frame_count_, codec_ctx_ref_->time_base, video_stream_->time_base);
+    // }
+    // 基于帧计数计算PTS
+    cloned_packet->pts = av_rescale_q(frame_count_, codec_ctx_ref_->time_base, video_stream_->time_base);
+    // if (dts != AV_NOPTS_VALUE) {
+    //     cloned_packet->dts = dts;
+    // } else {
+    //     // 如果没有DTS，使用PTS
+    //     cloned_packet->dts = cloned_packet->pts;
+    // }
+    cloned_packet->dts = cloned_packet->pts;
+
+
+    cloned_packet->duration = av_rescale_q(1, codec_ctx_ref_->time_base, video_stream_->time_base);
+
+    // 写入数据包
+    int ret = av_interleaved_write_frame(fmt_ctx_, cloned_packet);
+
+    if (ret < 0) {
+        char err_buf[1024];
+        av_strerror(ret, err_buf, sizeof(err_buf));
+        std::cerr << "写入数据包失败: " << err_buf << std::endl;
+    } else {
+        frame_count_++;
+    }
+
+    av_packet_free(&cloned_packet);
+    return ret;
+}
+
+int VideoWriter::finish() {
+    if (!fmt_ctx_) {
+        return 0;
+    }
+
+    // 写入文件尾
+    int ret = av_write_trailer(fmt_ctx_);
+    if (ret < 0) {
+        std::cerr << "写入文件尾失败" << std::endl;
+    } else {
+        std::cout << "视频写入完成，总帧数: " << frame_count_ << std::endl;
+    }
+
+    cleanup();
+    return ret;
+}
+
+void VideoWriter::cleanup() {
+    if (fmt_ctx_) {
+        if (!(fmt_ctx_->oformat->flags & AVFMT_NOFILE) && fmt_ctx_->pb) {
+            avio_close(fmt_ctx_->pb);
+        }
+        avformat_free_context(fmt_ctx_);
+        fmt_ctx_ = nullptr;
+    }
+    video_stream_ = nullptr;
+    codec_ctx_ref_ = nullptr;
+}
+
+} // namespace ffmpeg

cmvr-es/devices/camera/realsense_camera/CMakeLists.txt

@@ -5,6 +5,7 @@ target_include_directories(realsense_camera PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 add_library(cmvr_es::device::realsense_camera ALIAS realsense_camera) # ✅ 命名空间别名 OK
 
 # 链接 librealsense2
+target_link_libraries(realsense_camera PRIVATE realsense2 opencv_core opencv_imgproc opencv_videoio)
 target_link_libraries(realsense_camera PRIVATE
         realsense2
         avcodec

cmvr-es/devices/camera/uvc_camera/CMakeLists.txt

@@ -1,10 +1,8 @@
-find_package(OpenCV REQUIRED)
-
 add_library(uvc_camera SHARED src/uvc_camera.cpp)
 
 target_include_directories(uvc_camera PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 
-target_link_libraries(uvc_camera PUBLIC ${OpenCV_LIBS} glog)
+target_link_libraries(uvc_camera PUBLIC glog opencv_core opencv_imgproc)
 
 add_library(cmvr_es::device::uvc_camera ALIAS uvc_camera)
 install(TARGETS uvc_camera LIBRARY DESTINATION lib)

cmvr-es/devices/microphone/ffmpeg_microphone/src/ffmpeg_microphone.cpp

@@ -32,7 +32,7 @@ ffmpegMicroPhone::ffmpegMicroPhone(const config::FFMpegMicroPhoneConfig& cfg):in
     sample_rate_(44100), channels_(2), config_(cfg)
 {
     id_ = config_.id();
-    input_device_ = cfg_.getAttrString("alsa");
+    input_device_ = config_.input_device();
     channels_ = config_.channels();
     sample_rate_ = config_.samplerate();
     state_.volume = config_.volume();
@@ -174,9 +174,9 @@ bool ffmpegMicroPhone::initFFmpeg() {
     int err;
     avdevice_register_all();
     // 初始化输入设备
-    AVInputFormat* input_fmt = av_find_input_format("alsa");
+    AVInputFormat* input_fmt = av_find_input_format("pulse");
     if (!input_fmt) {
-        LOG(ERROR) << "Could not find ALSA input format";
+        LOG(ERROR) << "Could not find pulse input format";
         return false;
     }
 

cmvr-es/service/grpc/include/grpc_humanoid_robot_service.h

@@ -27,7 +27,10 @@ namespace cmvr {
             grpc::Status speedL(grpc::ServerContext* context, const cmvr::api::SpeedL_Request* request, cmvr::api::SpeedL_Response* response) override;
 
             grpc::Status getJointState(grpc::ServerContext *context, const cmvr::api::JointRequest *request, cmvr::api::JointResponse *response) override;
+
             grpc::Status getPose(grpc::ServerContext *context, const cmvr::api::GetPose_Request *request, cmvr::api::GetPose_Response *response) override;
+
+            grpc::Status calibrateZeroQ(grpc::ServerContext* context, const cmvr::api::CalibrateZeroQ_Request* request, cmvr::api::CalibrateZeroQ_Response* response) override;
         private:
             device::DeviceManager& dmgr_;
         };

cmvr-es/service/grpc/src/grpc_camera_service.cpp

@@ -372,7 +372,7 @@ grpc::Status gRPCCameraServiceImpl::GetRGBDImagesStream(grpc::ServerContext* con
         {
             if (context->IsCancelled())
             {
-                LOG(INFO) << "[gRPCCameraServiceImpl](GetRGBImageStream) context is cancelled,id=" << dev_id;
+                LOG(INFO) << "[gRPCCameraServiceImpl](GetRGBDImagesStream) context is cancelled,id=" << dev_id;
                 break;
             }
 
@@ -411,7 +411,7 @@ grpc::Status gRPCCameraServiceImpl::GetRGBDImagesStream(grpc::ServerContext* con
                 }
             }
         }
-        LOG(INFO) << "[gRPCCameraServiceImpl] (GetRGBImageStream): end,id=" << dev_id;
+        LOG(INFO) << "[gRPCCameraServiceImpl] (GetRGBDImagesStream): end,id=" << dev_id;
         dev->stopStreaming();
         return grpc::Status::OK;
     }

cmvr-es/service/grpc/src/grpc_humanoid_robot_service.cpp

@@ -219,3 +219,28 @@ grpc::Status gRPCHumanoidRobotServiceImpl::getJointState(grpc::ServerContext *co
     *response->mutable_header()->mutable_timestamp() = TimeUtil::GetCurrentTime();
     return ret;
 }
+
+
+grpc::Status gRPCHumanoidRobotServiceImpl::calibrateZeroQ(grpc::ServerContext* context
+    , const cmvr::api::CalibrateZeroQ_Request* request
+    , cmvr::api::CalibrateZeroQ_Response* response) {
+
+    grpc::Status ret = grpc::Status::OK;
+
+    try {
+        auto robot = dmgr_.getDevice<AbstractRobot>(request->header().device_id());
+
+        std::string joint_name = request->joint_name();
+
+        robot->calibrateZeroQ(joint_name);
+
+        response->mutable_header()->set_success(true);
+        response->mutable_header()->set_error_message("");
+    }catch (const std::exception& e) {
+        response->mutable_header()->set_success(false);
+        response->mutable_header()->set_error_message(e.what());
+        ret = grpc::Status(grpc::StatusCode::INTERNAL, e.what());
+    }
+    *response->mutable_header()->mutable_timestamp() = TimeUtil::GetCurrentTime();
+    return ret;
+}

protos/cmvr/api/humanoid_robot_command.proto

@@ -133,3 +133,14 @@ message JointRequest {
   CommandHeader.Request header = 1;
 }
 
+
+message CalibrateZeroQ {
+    message Request{
+        CommandHeader.Request header = 1;
+        string joint_name = 2;
+    }
+
+    message Response{
+        CommandHeader.Feedback header= 1;
+    }
+}

protos/cmvr/api/humanoid_robot_service.proto

@@ -14,4 +14,5 @@ service HumanoidRobotService{
   rpc speedL(SpeedL.Request) returns (SpeedL.Response);
   rpc getJointState(JointRequest) returns (JointResponse);
   rpc getPose(GetPose.Request) returns (GetPose.Response);
+  rpc calibrateZeroQ(CalibrateZeroQ.Request) returns (CalibrateZeroQ.Response);
 }

protos/cmvr/config/microphone_config/microphone_config.proto

@@ -7,6 +7,7 @@ message FFMpegMicroPhoneConfig{
     int32 sampleRate = 3;
     int32 volume = 4;
     bool enable = 5;
+    string input_device = 6;
 }
 
 message MicroPhoneConfig{