diff --git a/cmvr-es/common/CMakeLists.txt b/cmvr-es/common/CMakeLists.txt index 86256ff0..db2c2c06 100644 --- a/cmvr-es/common/CMakeLists.txt +++ b/cmvr-es/common/CMakeLists.txt @@ -3,6 +3,10 @@ file(GLOB SRC ${CMAKE_CURRENT_SOURCE_DIR}/utils/config_helper/src/config_setting.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/utils/ffmpeg/src/CameraCapture.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/utils/ffmpeg/src/VideoFrameEncoder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/utils/ffmpeg/src/VideoWriter.cpp + ) @@ -13,6 +17,11 @@ target_include_directories(common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(common PUBLIC cmvr_es::proto + avformat + avdevice + avutil + avcodec + swscale ) add_library(cmvr_es::common ALIAS common) diff --git a/cmvr-es/common/config/camera_config/camera_config.pb.txt b/cmvr-es/common/config/camera_config/camera_config.pb.txt index db18cd7e..5e9bf488 100644 --- a/cmvr-es/common/config/camera_config/camera_config.pb.txt +++ b/cmvr-es/common/config/camera_config/camera_config.pb.txt @@ -20,7 +20,7 @@ uvc_cameras { height: 480 fps: 30 codec: "H265" - camera_mode: CAMERA_MODE_PHOTO + camera_mode: CAMERA_MODE_VIDEO stream_mode: STREAM_MODE_RGB buffer_size: 30 enable: true diff --git a/cmvr-es/common/config/microphone_conifg/microphone_config.pb.txt b/cmvr-es/common/config/microphone_config/microphone_config.pb.txt similarity index 66% rename from cmvr-es/common/config/microphone_conifg/microphone_config.pb.txt rename to cmvr-es/common/config/microphone_config/microphone_config.pb.txt index 541151ea..46fe5bf4 100644 --- a/cmvr-es/common/config/microphone_conifg/microphone_config.pb.txt +++ b/cmvr-es/common/config/microphone_config/microphone_config.pb.txt @@ -3,5 +3,6 @@ ffmpeg_microphones { channels: 2 sampleRate: 44100 volume: 100 - enable: true + enable: false + input_device: "default" } \ No newline at end of file diff --git a/cmvr-es/common/config/speaker_config/speaker_config.pb.txt b/cmvr-es/common/config/speaker_config/speaker_config.pb.txt index 318258de..cd1287ee 100644 --- a/cmvr-es/common/config/speaker_config/speaker_config.pb.txt +++ b/cmvr-es/common/config/speaker_config/speaker_config.pb.txt @@ -1,4 +1,4 @@ ffmpeg_speakers { id: "spk1" - enable: true + enable: false } \ No newline at end of file diff --git a/cmvr-es/common/utils/ffmpeg/include/CameraCapture.h b/cmvr-es/common/utils/ffmpeg/include/CameraCapture.h new file mode 100644 index 00000000..b3561c27 --- /dev/null +++ b/cmvr-es/common/utils/ffmpeg/include/CameraCapture.h @@ -0,0 +1,68 @@ +// CameraCapture.h +#pragma once + +#include +#include +#include + +extern "C"{ +#include +#include +#include +} + +namespace ffmpeg { + + struct CameraConfig { + std::string device_name = "0"; // Windows: "0" (dshow), Linux: "0" (对应 /dev/video0) + int width = 1280; + int height = 720; + int fps = 30; + AVPixelFormat pixel_format = AV_PIX_FMT_YUV420P; + }; + + class CameraCapture { + public: + using FrameCallback = std::function; + + CameraCapture(); + ~CameraCapture(); + + // 初始化摄像头 + int initialize(const CameraConfig& config); + + // 开始捕获 + int start_capture(FrameCallback callback); + + // 停止捕获 + void stop_capture(); + + // 获取当前配置 + CameraConfig get_config() const { return config_; } + + // 获取输入格式上下文 + AVFormatContext* get_format_context() { return fmt_ctx_; } + + // 获取视频流索引 + int get_video_stream_index() const { return video_stream_index_; } + + private: + CameraConfig config_; + AVFormatContext* fmt_ctx_ = nullptr; + AVCodecContext* decoder_ctx_ = nullptr; + int video_stream_index_ = -1; + bool is_capturing_ = false; + AVPacket* packet_ = nullptr; + AVFrame* frame_ = nullptr; + + // 初始化设备（平台相关） + int init_device(); + + // 初始化解码器 + int init_decoder(); + + // 释放资源 + void cleanup(); + }; + +} // namespace ffmpeg \ No newline at end of file diff --git a/cmvr-es/common/utils/ffmpeg/include/VideoFrameEncoder.h b/cmvr-es/common/utils/ffmpeg/include/VideoFrameEncoder.h new file mode 100644 index 00000000..68f19152 --- /dev/null +++ b/cmvr-es/common/utils/ffmpeg/include/VideoFrameEncoder.h @@ -0,0 +1,101 @@ +// VideoFrameEncoder.h +#pragma once + +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +} + +namespace ffmpeg { + +struct EncoderConfig { + int width = 1280; + int height = 720; + int fps = 30; + int bitrate = 5000000; // 5Mbps + int gop_size = 15; // GOP大小 + int max_b_frames = 2; // 最大B帧数 + AVPixelFormat pixel_format = AV_PIX_FMT_YUV420P; + std::string preset = "fast"; // 编码速度预设 + std::string tune = "zerolatency"; // 调优参数 + std::string profile = "main"; // 编码profile + std::string codec = "libx265"; +}; + +class VideoFrameEncoder { +public: + using PacketCallback = std::function; + + VideoFrameEncoder(); + ~VideoFrameEncoder(); + + // 初始化编码器 + int initialize(const EncoderConfig& config); + + // 编码一帧 + int encode_frame(AVFrame* frame); + + // 设置编码数据包回调 + void set_packet_callback(PacketCallback callback) { + packet_callback_ = callback; + } + + // 设置时间戳类型 + void set_time_base(AVRational time_base) { + time_base_ = time_base; + } + + // 刷新编码器（处理剩余数据） + int flush(); + + // 获取编码器上下文 + AVCodecContext* get_codec_context() { return encoder_ctx_; } + + // 获取时间基 + AVRational get_time_base() const { + return encoder_ctx_ ? encoder_ctx_->time_base : AVRational{0, 1}; + } + + // 获取帧率 + AVRational get_framerate() const { + return encoder_ctx_ ? encoder_ctx_->framerate : AVRational{0, 1}; + } + + // 获取当前帧索引 + int64_t get_frame_index() const { return frame_index_; } + +private: + EncoderConfig config_; + AVCodecContext* encoder_ctx_ = nullptr; + SwsContext* sws_ctx_ = nullptr; + AVFrame* converted_frame_ = nullptr; + PacketCallback packet_callback_; + int64_t frame_index_ = 0; + AVRational time_base_ = {1, 90000}; // 默认时间基 + int64_t start_time_ = 0; // 编码开始时间 + + // 初始化SWS上下文（格式转换） + int init_sws_context(AVFrame* frame); + + // 发送数据包到回调 + void send_packet(AVPacket* packet); + + // 释放资源 + void cleanup(); + + // 获取当前时间戳 + int64_t get_current_timestamp(); +}; + +} // namespace ffmpeg \ No newline at end of file diff --git a/cmvr-es/common/utils/ffmpeg/include/VideoWriter.h b/cmvr-es/common/utils/ffmpeg/include/VideoWriter.h new file mode 100644 index 00000000..0ed8d197 --- /dev/null +++ b/cmvr-es/common/utils/ffmpeg/include/VideoWriter.h @@ -0,0 +1,63 @@ +// VideoWriter.h +#pragma once + +#include +#include + + +extern "C"{ +#include +#include +} + +namespace ffmpeg { + + struct WriterConfig { + std::string output_file = "output.mp4"; + int width = 1280; + int height = 720; + int fps = 30; + int bitrate = 5000000; + AVRational time_base = {1, 90000}; // 默认时间基 + }; + + class VideoWriter { + public: + VideoWriter(); + ~VideoWriter(); + + // 初始化写入器 + int initialize(const WriterConfig& config, AVCodecContext* codec_ctx); + + // 写入数据包 + int write_packet(AVPacket* packet, int64_t pts, int64_t dts); + + // 写入文件尾 + int finish(); + + // 获取输出格式上下文 + AVFormatContext* get_format_context() { return fmt_ctx_; } + + // 获取视频流 + AVStream* get_video_stream() { return video_stream_; } + + // 获取输出时间基 + AVRational get_output_time_base() const { + return video_stream_ ? video_stream_->time_base : AVRational{0, 1}; + } + + private: + WriterConfig config_; + AVFormatContext* fmt_ctx_ = nullptr; + AVStream* video_stream_ = nullptr; + AVCodecContext* codec_ctx_ref_ = nullptr; + int64_t frame_count_ = 0; + + // 初始化输出格式 + int init_output_format(); + + // 释放资源 + void cleanup(); + }; + +} // namespace ffmpeg \ No newline at end of file diff --git a/cmvr-es/common/utils/ffmpeg/src/CameraCapture.cpp b/cmvr-es/common/utils/ffmpeg/src/CameraCapture.cpp new file mode 100644 index 00000000..75ac0e70 --- /dev/null +++ b/cmvr-es/common/utils/ffmpeg/src/CameraCapture.cpp @@ -0,0 +1,188 @@ +// CameraCapture.cpp +#include "../include/CameraCapture.h" +#include +#include +#include + +namespace ffmpeg { + +CameraCapture::CameraCapture() { + packet_ = av_packet_alloc(); + frame_ = av_frame_alloc(); +} + +CameraCapture::~CameraCapture() { + stop_capture(); + cleanup(); + + if (packet_) av_packet_free(&packet_); + if (frame_) av_frame_free(&frame_); +} + +int CameraCapture::initialize(const CameraConfig& config) { + config_ = config; + + // 初始化设备 + int ret = init_device(); + if (ret < 0) { + std::cerr << "初始化设备失败" << std::endl; + return ret; + } + + // 初始化解码器 + ret = init_decoder(); + if (ret < 0) { + std::cerr << "初始化解码器失败" << std::endl; + return ret; + } + + return 0; +} + +int CameraCapture::init_device() { + AVInputFormat* input_fmt = nullptr; + std::string device_path; + +#ifdef _WIN32 + input_fmt = av_find_input_format("dshow"); + device_path = "video=" + config_.device_name; +#else + input_fmt = av_find_input_format("v4l2"); + device_path = config_.device_name; +#endif + + if (!input_fmt) { + std::cerr << "找不到输入格式" << std::endl; + return -1; + } + + // 打开摄像头 + int ret = avformat_open_input(&fmt_ctx_, device_path.c_str(), input_fmt, nullptr); + if (ret < 0) { + char err_buf[1024]; + av_strerror(ret, err_buf, sizeof(err_buf)); + std::cerr << "打开摄像头失败: " << err_buf << std::endl; + return ret; + } + + // 查找流信息 + ret = avformat_find_stream_info(fmt_ctx_, nullptr); + if (ret < 0) { + std::cerr << "查找流信息失败" << std::endl; + return ret; + } + + // 查找视频流 + video_stream_index_ = av_find_best_stream(fmt_ctx_, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); + if (video_stream_index_ < 0) { + std::cerr << "找不到视频流" << std::endl; + return video_stream_index_; + } + + return 0; +} + +int CameraCapture::init_decoder() { + AVStream* stream = fmt_ctx_->streams[video_stream_index_]; + const AVCodec* decoder = avcodec_find_decoder(stream->codecpar->codec_id); + + if (!decoder) { + std::cerr << "找不到解码器" << std::endl; + return -1; + } + + decoder_ctx_ = avcodec_alloc_context3(decoder); + if (!decoder_ctx_) { + std::cerr << "分配解码器上下文失败" << std::endl; + return -1; + } + + // 复制参数到解码器上下文 + int ret = avcodec_parameters_to_context(decoder_ctx_, stream->codecpar); + if (ret < 0) { + std::cerr << "复制解码器参数失败" << std::endl; + return ret; + } + + // 打开解码器 + ret = avcodec_open2(decoder_ctx_, decoder, nullptr); + if (ret < 0) { + std::cerr << "打开解码器失败" << std::endl; + return ret; + } + + return 0; +} + +int CameraCapture::start_capture(FrameCallback callback) { + if (!callback || !fmt_ctx_ || !decoder_ctx_) { + std::cerr << "参数无效或未初始化" << std::endl; + return -1; + } + + is_capturing_ = true; + + while (is_capturing_) { + // 读取数据包 + int ret = av_read_frame(fmt_ctx_, packet_); + if (ret < 0) { + if (ret == AVERROR(EAGAIN)) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + std::cerr << "读取帧失败: " << ret << std::endl; + break; + } + + // 只处理视频流 + if (packet_->stream_index == video_stream_index_) { + // 发送数据包到解码器 + ret = avcodec_send_packet(decoder_ctx_, packet_); + if (ret < 0 && ret != AVERROR(EAGAIN)) { + std::cerr << "发送数据包到解码器失败: " << ret << std::endl; + av_packet_unref(packet_); + continue; + } + + // 接收解码后的帧 + while (ret >= 0) { + ret = avcodec_receive_frame(decoder_ctx_, frame_); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + break; + } else if (ret < 0) { + std::cerr << "接收解码帧失败: " << ret << std::endl; + break; + } + + // 回调处理帧 + callback(frame_); + } + } + + av_packet_unref(packet_); + + // 控制帧率 + std::this_thread::sleep_for(std::chrono::milliseconds(1000 / config_.fps)); + } + + return 0; +} + +void CameraCapture::stop_capture() { + is_capturing_ = false; +} + +void CameraCapture::cleanup() { + if (decoder_ctx_) { + avcodec_close(decoder_ctx_); + avcodec_free_context(&decoder_ctx_); + } + + if (fmt_ctx_) { + avformat_close_input(&fmt_ctx_); + avformat_free_context(fmt_ctx_); + fmt_ctx_ = nullptr; + } +} + +} // namespace ffmpeg \ No newline at end of file diff --git a/cmvr-es/common/utils/ffmpeg/src/VideoFrameEncoder.cpp b/cmvr-es/common/utils/ffmpeg/src/VideoFrameEncoder.cpp new file mode 100644 index 00000000..38a9850b --- /dev/null +++ b/cmvr-es/common/utils/ffmpeg/src/VideoFrameEncoder.cpp @@ -0,0 +1,240 @@ +// VideoFrameEncoder.cpp +#include "../include/VideoFrameEncoder.h" +#include +#include + +namespace ffmpeg { + +VideoFrameEncoder::VideoFrameEncoder() { + converted_frame_ = av_frame_alloc(); + start_time_ = av_gettime(); // 获取初始时间 +} + +VideoFrameEncoder::~VideoFrameEncoder() { + cleanup(); + if (converted_frame_) av_frame_free(&converted_frame_); +} + +int64_t VideoFrameEncoder::get_current_timestamp() { + int64_t current_time = av_gettime(); + int64_t elapsed = current_time - start_time_; + + // 将微秒转换为时间基单位 + return av_rescale_q(elapsed, {1, 1000000}, time_base_); +} + +int VideoFrameEncoder::initialize(const EncoderConfig& config) { + config_ = config; + + // 查找编码器 + //const AVCodec* encoder = avcodec_find_encoder(AV_CODEC_ID_H264); + const AVCodec* encoder = avcodec_find_encoder_by_name(config.codec.c_str()); + if (!encoder) { + std::cerr << "找不到H.264编码器" << std::endl; + return -1; + } + + // 分配编码器上下文 + encoder_ctx_ = avcodec_alloc_context3(encoder); + if (!encoder_ctx_) { + std::cerr << "分配编码器上下文失败" << std::endl; + return -1; + } + + // 配置编码器参数 + encoder_ctx_->width = config_.width; + encoder_ctx_->height = config_.height; + encoder_ctx_->pix_fmt = config_.pixel_format; + + // 正确设置时间基 + encoder_ctx_->time_base = av_make_q(1, config_.fps); + encoder_ctx_->framerate = av_make_q(config_.fps, 1); + + // 设置编码器的时间基为帧率倒数 + time_base_ = encoder_ctx_->time_base; + + encoder_ctx_->bit_rate = config_.bitrate; + encoder_ctx_->gop_size = config_.gop_size; + encoder_ctx_->max_b_frames = config_.max_b_frames; + + // 设置编码器选项 + if (encoder->id == AV_CODEC_ID_H264) { + av_opt_set(encoder_ctx_->priv_data, "preset", config_.preset.c_str(), 0); + av_opt_set(encoder_ctx_->priv_data, "tune", config_.tune.c_str(), 0); + av_opt_set(encoder_ctx_->priv_data, "profile", config_.profile.c_str(), 0); + av_opt_set(encoder_ctx_->priv_data, "x264-params", "log-level=error", 0); + } + + // 打开编码器 + int ret = avcodec_open2(encoder_ctx_, encoder, nullptr); + if (ret < 0) { + char err_buf[1024]; + av_strerror(ret, err_buf, sizeof(err_buf)); + std::cerr << "打开编码器失败: " << err_buf << std::endl; + return ret; + } + + // 准备转换后的帧 + converted_frame_->width = config_.width; + converted_frame_->height = config_.height; + converted_frame_->format = config_.pixel_format; + + ret = av_frame_get_buffer(converted_frame_, 0); + if (ret < 0) { + std::cerr << "分配帧缓冲区失败" << std::endl; + return ret; + } + + return 0; +} + +int VideoFrameEncoder::init_sws_context(AVFrame* frame) { + if (!frame) { + std::cerr << "输入帧为空" << std::endl; + return -1; + } + + // 如果格式相同，不需要转换 + if (frame->format == config_.pixel_format && + frame->width == config_.width && + frame->height == config_.height) { + sws_ctx_ = nullptr; + return 0; + } + + // 创建转换上下文 + sws_ctx_ = sws_getContext( + frame->width, frame->height, static_cast(frame->format), + config_.width, config_.height, config_.pixel_format, + SWS_BILINEAR, nullptr, nullptr, nullptr + ); + + if (!sws_ctx_) { + std::cerr << "创建SWS上下文失败" << std::endl; + return -1; + } + + return 0; +} + +int VideoFrameEncoder::encode_frame(AVFrame* frame) { + if (!encoder_ctx_ || !frame) { + std::cerr << "编码器未初始化或输入帧为空" << std::endl; + return -1; + } + + // 初始化格式转换（如果需要） + if (!sws_ctx_) { + int ret = init_sws_context(frame); + if (ret < 0) { + return ret; + } + } + + AVFrame* frame_to_encode = frame; + + // 如果需要格式转换 + if (sws_ctx_) { + sws_scale( + sws_ctx_, + frame->data, frame->linesize, 0, frame->height, + converted_frame_->data, converted_frame_->linesize + ); + frame_to_encode = converted_frame_; + } + + // 设置正确的PTS + frame_to_encode->pts = frame_index_; + + // 发送帧到编码器 + int ret = avcodec_send_frame(encoder_ctx_, frame_to_encode); + if (ret < 0) { + char err_buf[1024]; + av_strerror(ret, err_buf, sizeof(err_buf)); + std::cerr << "发送帧到编码器失败: " << err_buf << std::endl; + return ret; + } + + // 接收编码后的数据包 + AVPacket* packet = av_packet_alloc(); + while (ret >= 0) { + ret = avcodec_receive_packet(encoder_ctx_, packet); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + break; + } else if (ret < 0) { + std::cerr << "接收编码数据包失败" << std::endl; + av_packet_free(&packet); + return ret; + } + + // 通过回调发送数据包，传递正确的PTS/DTS + if (packet_callback_) { + packet_callback_(packet, packet->pts, packet->dts); + } + + av_packet_unref(packet); + } + + av_packet_free(&packet); + + // 递增帧索引 + frame_index_++; + + return 0; +} + +int VideoFrameEncoder::flush() { + if (!encoder_ctx_) { + return 0; + } + + // 发送空帧刷新编码器 + int ret = avcodec_send_frame(encoder_ctx_, nullptr); + if (ret < 0) { + std::cerr << "发送刷新帧失败" << std::endl; + return ret; + } + + // 接收所有剩余数据包 + AVPacket* packet = av_packet_alloc(); + while (true) { + ret = avcodec_receive_packet(encoder_ctx_, packet); + if (ret == AVERROR_EOF) { + break; + } else if (ret < 0) { + std::cerr << "接收刷新数据包失败" << std::endl; + av_packet_free(&packet); + return ret; + } + + // 通过回调发送数据包 + if (packet_callback_) { + packet_callback_(packet, packet->pts, packet->dts); + } + + av_packet_unref(packet); + } + + av_packet_free(&packet); + return 0; +} + +void VideoFrameEncoder::send_packet(AVPacket* packet) { + if (packet_callback_) { + packet_callback_(packet, packet->pts, packet->dts); + } +} + +void VideoFrameEncoder::cleanup() { + if (sws_ctx_) { + sws_freeContext(sws_ctx_); + sws_ctx_ = nullptr; + } + + if (encoder_ctx_) { + avcodec_close(encoder_ctx_); + avcodec_free_context(&encoder_ctx_); + } +} + +} // namespace ffmpeg \ No newline at end of file diff --git a/cmvr-es/common/utils/ffmpeg/src/VideoWriter.cpp b/cmvr-es/common/utils/ffmpeg/src/VideoWriter.cpp new file mode 100644 index 00000000..da9c0553 --- /dev/null +++ b/cmvr-es/common/utils/ffmpeg/src/VideoWriter.cpp @@ -0,0 +1,153 @@ +// VideoWriter.cpp +#include "../include/VideoWriter.h" +#include + +namespace ffmpeg { + +VideoWriter::VideoWriter() {} + +VideoWriter::~VideoWriter() { + cleanup(); +} + +int VideoWriter::initialize(const WriterConfig& config, AVCodecContext* codec_ctx) { + config_ = config; + codec_ctx_ref_ = codec_ctx; + + if (!codec_ctx) { + std::cerr << "编码器上下文为空" << std::endl; + return -1; + } + + // 分配输出格式上下文 + int ret = avformat_alloc_output_context2(&fmt_ctx_, nullptr, nullptr, config_.output_file.c_str()); + if (ret < 0 || !fmt_ctx_) { + std::cerr << "分配输出格式上下文失败" << std::endl; + return ret; + } + + // 创建输出流 + video_stream_ = avformat_new_stream(fmt_ctx_, nullptr); + if (!video_stream_) { + std::cerr << "创建输出流失败" << std::endl; + return -1; + } + + // 复制编码器参数到输出流 + ret = avcodec_parameters_from_context(video_stream_->codecpar, codec_ctx); + if (ret < 0) { + std::cerr << "复制编码器参数失败" << std::endl; + return ret; + } + + // 设置流时间基（使用配置的时间基） + video_stream_->time_base = config_.time_base; + + // 设置流的平均帧率 + video_stream_->avg_frame_rate = av_make_q(config_.fps, 1); + + // 打开输出文件 + if (!(fmt_ctx_->oformat->flags & AVFMT_NOFILE)) { + ret = avio_open(&fmt_ctx_->pb, config_.output_file.c_str(), AVIO_FLAG_WRITE); + if (ret < 0) { + char err_buf[1024]; + av_strerror(ret, err_buf, sizeof(err_buf)); + std::cerr << "打开输出文件失败: " << err_buf << std::endl; + return ret; + } + } + + // 写入文件头 + ret = avformat_write_header(fmt_ctx_, nullptr); + if (ret < 0) { + std::cerr << "写入文件头失败" << std::endl; + return ret; + } + + std::cout << "视频写入器初始化完成，输出文件: " << config_.output_file + << " 时间基: " << video_stream_->time_base.num << "/" << video_stream_->time_base.den + << std::endl; + return 0; +} + +int VideoWriter::write_packet(AVPacket* packet, int64_t pts, int64_t dts) { + if (!fmt_ctx_ || !video_stream_) { + std::cerr << "写入器未初始化" << std::endl; + return -1; + } + + // 克隆数据包，避免修改原始数据 + AVPacket* cloned_packet = av_packet_clone(packet); + if (!cloned_packet) { + std::cerr << "克隆数据包失败" << std::endl; + return -1; + } + + // 设置数据包属性 + cloned_packet->stream_index = video_stream_->index; + + // 如果传入的PTS/DTS有效，使用它们 + // if (pts != AV_NOPTS_VALUE) { + // cloned_packet->pts = pts; + // } else { + // // 基于帧计数计算PTS + // cloned_packet->pts = av_rescale_q(frame_count_, codec_ctx_ref_->time_base, video_stream_->time_base); + // } + // 基于帧计数计算PTS + cloned_packet->pts = av_rescale_q(frame_count_, codec_ctx_ref_->time_base, video_stream_->time_base); + // if (dts != AV_NOPTS_VALUE) { + // cloned_packet->dts = dts; + // } else { + // // 如果没有DTS，使用PTS + // cloned_packet->dts = cloned_packet->pts; + // } + cloned_packet->dts = cloned_packet->pts; + + + cloned_packet->duration = av_rescale_q(1, codec_ctx_ref_->time_base, video_stream_->time_base); + + // 写入数据包 + int ret = av_interleaved_write_frame(fmt_ctx_, cloned_packet); + + if (ret < 0) { + char err_buf[1024]; + av_strerror(ret, err_buf, sizeof(err_buf)); + std::cerr << "写入数据包失败: " << err_buf << std::endl; + } else { + frame_count_++; + } + + av_packet_free(&cloned_packet); + return ret; +} + +int VideoWriter::finish() { + if (!fmt_ctx_) { + return 0; + } + + // 写入文件尾 + int ret = av_write_trailer(fmt_ctx_); + if (ret < 0) { + std::cerr << "写入文件尾失败" << std::endl; + } else { + std::cout << "视频写入完成，总帧数: " << frame_count_ << std::endl; + } + + cleanup(); + return ret; +} + +void VideoWriter::cleanup() { + if (fmt_ctx_) { + if (!(fmt_ctx_->oformat->flags & AVFMT_NOFILE) && fmt_ctx_->pb) { + avio_close(fmt_ctx_->pb); + } + avformat_free_context(fmt_ctx_); + fmt_ctx_ = nullptr; + } + video_stream_ = nullptr; + codec_ctx_ref_ = nullptr; +} + +} // namespace ffmpeg \ No newline at end of file diff --git a/cmvr-es/devices/camera/realsense_camera/CMakeLists.txt b/cmvr-es/devices/camera/realsense_camera/CMakeLists.txt index f5fd8ba9..f82d2a3d 100644 --- a/cmvr-es/devices/camera/realsense_camera/CMakeLists.txt +++ b/cmvr-es/devices/camera/realsense_camera/CMakeLists.txt @@ -1,3 +1,5 @@ + +find_package(OpenCV REQUIRED) add_library(realsense_camera SHARED src/realsense_camera.cpp) target_include_directories(realsense_camera PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) @@ -5,7 +7,7 @@ target_include_directories(realsense_camera PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) add_library(cmvr_es::device::realsense_camera ALIAS realsense_camera) # ✅ 命名空间别名 OK # 链接 librealsense2 -target_link_libraries(realsense_camera PRIVATE realsense2) +target_link_libraries(realsense_camera PRIVATE realsense2 ${OpenCV_LIBS} ) install(TARGETS realsense_camera LIBRARY DESTINATION lib) # -------------------------------------------------------- diff --git a/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h b/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h index 7fce04b7..8a361956 100644 --- a/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h +++ b/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h @@ -9,9 +9,47 @@ #include #include + namespace cmvr::device{ enum StreamMode {RGBD_MODE, COLOR_MODE,DEPTH_MODE}; + struct FfmpegEncoderInfo { + std::string codec_name; // 编码器名称（如"h264"、"hevc"） + int width = 0; // 图像宽度 + int height = 0; // 图像高度 + int fps = 0; // 帧率 + int64_t frame_pts = 0; + bool bRunning = false; // 是否进行编码 + AVCodecContext* codec_context = nullptr; // 编码器上下文 + AVFrame* frame = nullptr; // 输入帧 + AVPacket* packet = nullptr; // 输出包 + SwsContext* sws_context = nullptr; // 图像转换上下文（如果需要格式转换） + // 析构函数：释放FFmpeg资源（核心！避免内存泄漏） + ~FfmpegEncoderInfo() { + // 释放编码帧 + if (frame) { + av_frame_free(&frame); + frame = nullptr; + } + // 释放编码包 + if (packet) { + av_packet_free(&packet); + packet = nullptr; + } + // 释放编码器上下文 + if (codec_context) { + avcodec_close(codec_context); // 关闭编码器 + avcodec_free_context(&codec_context); // 释放上下文 + codec_context = nullptr; + } + // 释放格式转换上下文 + if (sws_context) { + sws_freeContext(sws_context); + sws_context = nullptr; + } + //std::cout << "FfmpegEncoderInfo resources released." << std::endl; + } + }; class RealsenseCamera final : public AbstractCamera { public: diff --git a/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h b/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h index 9c5cd4a0..87a6d8b8 100644 --- a/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h +++ b/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h @@ -1,72 +1,25 @@ -// -// Created by xtkuang on 2025/5/30. -// - +// uvc_camera.h #ifndef CMVR_ES_UVC_CAMERA_H #define CMVR_ES_UVC_CAMERA_H #include "utils/base/include/ring_buffer.h" #include "camera/abstract_camera.h" -//使用ffmpeg来编码保存视频文件 -#define USE_FFMPEG_ENCODER 1 - -#if USE_FFMPEG_ENCODER -#include "speaker/ffmpeg_speaker/include/ffmpeg_ptr.h" -#endif +// 使用新的FFmpeg封装类 +#include "common/utils/ffmpeg/include/CameraCapture.h" +#include "common/utils/ffmpeg/include/VideoFrameEncoder.h" +#include "common/utils/ffmpeg/include/VideoWriter.h" namespace cmvr::device { enum CameraMode {PHOTO_MODE, VIDEO_MODE}; - struct ImageData { - //原始图像数据 cv::Mat rgbImage; - //深度图原始数据 cv::Mat depthImage; }; - struct FfmpegEncoderInfo { - std::string codec_name; // 编码器名称（如"h264"、"hevc"） - int width = 0; // 图像宽度 - int height = 0; // 图像高度 - int fps = 0; // 帧率 - int64_t frame_pts = 0; - bool bRunning = false; // 是否进行编码 - AVCodecContext* codec_context = nullptr; // 编码器上下文 - AVFrame* frame = nullptr; // 输入帧 - AVPacket* packet = nullptr; // 输出包 - SwsContext* sws_context = nullptr; // 图像转换上下文（如果需要格式转换） - - // 析构函数：释放FFmpeg资源（核心！避免内存泄漏） - ~FfmpegEncoderInfo() { - // 释放编码帧 - if (frame) { - av_frame_free(&frame); - frame = nullptr; - } - // 释放编码包 - if (packet) { - av_packet_free(&packet); - packet = nullptr; - } - // 释放编码器上下文 - if (codec_context) { - avcodec_close(codec_context); // 关闭编码器 - avcodec_free_context(&codec_context); // 释放上下文 - codec_context = nullptr; - } - // 释放格式转换上下文 - if (sws_context) { - sws_freeContext(sws_context); - sws_context = nullptr; - } - //std::cout << "FfmpegEncoderInfo resources released." << std::endl; - } - }; - - //USB摄像机 + // USB摄像机 class UVCCamera final : public AbstractCamera { public: explicit UVCCamera(const XmlNode& config); @@ -82,56 +35,41 @@ namespace cmvr::device { void updateParams(const std::pair& param) override; void startRecording(const std::string &video_path) override; void stopRecording() override; - void pauseRecording() override; - void resumeRecording() override; - // 初始化单个编码器的通用函数（复用逻辑，避免重复代码） - static bool initSingleEncoder(std::shared_ptr& encoder, - const std::string& codec_name, - int width, int height, int fps); - static bool encodeFrameWithEncoder(std::shared_ptr& encoder,const cv::Mat& frame,std::vector& encoded_frame,bool& is_key); void getEncodedFrame(StreamFrameData& frame_data, size_t& index) override; bool startStreaming() override; void stopStreaming() override; + private: void streaming_worker_(); - void recording_worker_(); + + // 辅助函数 + static bool convertAVFrameToMat(AVFrame* frame, cv::Mat& mat); int fps_; int width_; int height_; std::string serial_; - cv::VideoCapture cap_; size_t buffer_size_; std::string codec_; CameraMode mode_; - // std::shared_ptr current_image_; std::shared_ptr> stream_frame_buffer_; std::string current_video_path_; std::mutex ctrl_mtx_{}; - std::unique_ptr video_writer_; std::shared_ptr stream_thread_; - std::shared_ptr recording_thread_; + // 新的FFmpeg封装类实例 + std::unique_ptr camera_capture_; + std::unique_ptr video_encoder_; + std::unique_ptr video_writer_; - cv::Mat latest_frame_; // 存储最新帧 - std::mutex frame_mutex_; // 保护最新帧的访问 + // 用于临时存储最新帧 + std::mutex latest_frame_mutex_; + cv::Mat latest_rgb_frame_; + bool latest_frame_valid_ = false; - std::string output_path_; - bool is_recording_ = false; - // ffmpeg录像 - AVFormatContext* format_context_ = nullptr; - AVCodecContext* codec_context_ = nullptr; - AVStream* stream_ = nullptr; - AVFrame* frame_ = nullptr; - AVPacket* packet_ = nullptr; - SwsContext* sws_context_ = nullptr; - int frame_count_ = 0; - - std::shared_ptr rgbEncoder_;//rgb图像编码 - std::shared_ptr depthEncoder_;//深度图编码 size_t streamIndex_ = 0; size_t recordingIndex_ = 0; size_t getImageIndex_ = 0; @@ -145,6 +83,4 @@ namespace cmvr::device { }; } - - -#endif //CMVR_ES_UVC_CAMERA_H \ No newline at end of file +#endif // CMVR_ES_UVC_CAMERA_H \ No newline at end of file diff --git a/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp b/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp index fe7f2c97..beae798b 100644 --- a/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp +++ b/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp @@ -1,14 +1,12 @@ -// -// Created by xtkuang on 2025/5/30. -// - +// uvc_camera.cpp #include "../include/uvc_camera.h" +#include +#include +#include using namespace std; using namespace cmvr::device; -#define USE_LIST_IMAGE 1 - UVCCamera::UVCCamera(const XmlNode& config) : AbstractCamera(config) { try { serial_ = cfg_.getAttrString("serial"); @@ -36,14 +34,13 @@ UVCCamera::UVCCamera(const XmlNode& config) : AbstractCamera(config) { throw runtime_error("invalid argument mode=" + mode_str); } - codec_ = cfg_.getAttrDefault("codec", "H265"); - - //initFFmpeg(); + codec_ = cfg_.getAttrDefault("codec", "H264"); } catch (exception &e) { throw runtime_error(e.what()); } } + UVCCamera::UVCCamera(const config::UVCCameraConfig& camera):camera_(camera) { serial_ = camera_.usb(); @@ -73,12 +70,18 @@ UVCCamera::UVCCamera(const config::UVCCameraConfig& camera):camera_(camera) codec_ = camera_.codec(); } + UVCCamera::~UVCCamera() { stop(); + + // 释放FFmpeg资源 + video_writer_.reset(); + video_encoder_.reset(); + camera_capture_.reset(); + + // 等待线程结束 if (stream_thread_ && stream_thread_->joinable()) stream_thread_->join(); - if (recording_thread_ && recording_thread_->joinable()) - recording_thread_->join(); } void UVCCamera::init() { @@ -86,128 +89,89 @@ void UVCCamera::init() { clear_error_(); state_.is_initialized = false; + // 初始化环形缓冲区 stream_frame_buffer_ = std::make_shared>(buffer_size_); stream_frame_buffer_->clear(); - if (cap_.isOpened()) { - cap_.release(); - } - cap_.open(serial_, cv::CAP_V4L2); - this_thread::sleep_for(chrono::milliseconds(100)); + // 初始化FFmpeg库 + avdevice_register_all(); + av_log_set_level(AV_LOG_ERROR); - if (!cap_.isOpened()) { + // 初始化CameraCapture + try { + ffmpeg::CameraConfig camera_config; + camera_config.device_name = serial_; + camera_config.width = width_; + camera_config.height = height_; + camera_config.fps = fps_; + + camera_capture_ = std::make_unique(); + int ret = camera_capture_->initialize(camera_config); + if (ret < 0) { + throw runtime_error("Failed to initialize camera capture"); + } + + LOG(INFO) << "[UVCCamera] (init): CameraCapture initialized successfully"; + } catch (const std::exception& e) { state_.is_error = true; - state_.error_message = "Failed to open USB camera at index " + serial_; - LOG(ERROR) << "[UVCCamera] (init)" << state_.error_message; + state_.error_message = "Failed to init camera capture: " + std::string(e.what()); + LOG(ERROR) << "[UVCCamera] (init) " << state_.error_message; throw runtime_error(state_.error_message); } - // 设置格式为MJPG - cap_.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); - - if (!cap_.set(cv::CAP_PROP_FRAME_WIDTH, width_)) { - state_.is_error = true; - state_.error_message = "set width failed"; - LOG(ERROR) << "[UVCCamera] (init): set width failed"; - throw runtime_error("[UVCCamera] (init): set width failed"); - } - state_.width = width_; - if (!cap_.set(cv::CAP_PROP_FRAME_HEIGHT, height_)) { - state_.is_error = true; - state_.error_message = "set height failed"; - LOG(ERROR) << "[UVCCamera] (init): set height failed"; - throw runtime_error("[UVCCamera] (init): set height failed"); - } - state_.height = height_; - if (!cap_.set(cv::CAP_PROP_FPS, fps_)) { - state_.is_error = true; - state_.error_message = "set fps failed"; - LOG(ERROR) << "[UVCCamera] (init): set fps failed"; - throw runtime_error("[UVCCamera] (init): set fps failed"); - } - //初始化编码器 - // 初始化RGB编码器（示例参数：640x480，30fps，H.264） - if (!initSingleEncoder(rgbEncoder_, codec_, width_, height_, fps_)) { - LOG(ERROR) << "[UVCCamera] (start): Failed to init RGB encoder!"; - throw runtime_error("Failed to init RGB encoder!"); - } - state_.fps = fps_; state_.is_initialized = true; state_.is_error = false; - LOG(INFO) << "[UVCCamera] (init): UVC camera initialized at index " << serial_; // 记录初始化成功日志 + LOG(INFO) << "[UVCCamera] (init): UVC camera initialized at index " << serial_; } void UVCCamera::start() { std::lock_guard lock(ctrl_mtx_); clear_error_(); + if (!state_.is_initialized) { - LOG(ERROR) << "[UVCCamera] (init): Camera not initialized"; + LOG(ERROR) << "[UVCCamera] (start): Camera not initialized"; throw runtime_error("Camera not initialized"); } + if (state_.is_opened) { - LOG(WARNING) << "[RealsenseCamera] (start): camera already started"; + LOG(WARNING) << "[UVCCamera] (start): camera already started"; return; } - cap_.open(serial_, cv::CAP_V4L2); - this_thread::sleep_for(chrono::milliseconds(100)); - if (!cap_.isOpened()) { - state_.is_error = true; - state_.error_message = "Failed to open USB camera at index " + serial_; - LOG(ERROR) << "[UVCCamera] (init)" << state_.error_message; - throw runtime_error(state_.error_message); - } - - // 设置格式为MJPG - cap_.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); - - if (!cap_.set(cv::CAP_PROP_FRAME_WIDTH, width_)) { - state_.is_error = true; - state_.error_message = "set width failed"; - LOG(ERROR) << "[UVCCamera] (init): set width failed"; - throw runtime_error("[UVCCamera] (init): set width failed"); - } - state_.width = width_; - if (!cap_.set(cv::CAP_PROP_FRAME_HEIGHT, height_)) { - state_.is_error = true; - state_.error_message = "set height failed"; - LOG(ERROR) << "[UVCCamera] (init): set height failed"; - throw runtime_error("[UVCCamera] (init): set height failed"); - } - state_.height = height_; - if (!cap_.set(cv::CAP_PROP_FPS, fps_)) { - state_.is_error = true; - state_.error_message = "set fps failed"; - LOG(ERROR) << "[UVCCamera] (init): set fps failed"; - throw runtime_error("[UVCCamera] (init): set fps failed"); - } state_.is_opened = true; + LOG(INFO) << "[UVCCamera] (start): Camera started"; } void UVCCamera::stop() { - //先停止录制再关闭摄像头 + // 先停止录制 if (state_.is_recording) { stopRecording(); } + std::lock_guard lock(ctrl_mtx_); clear_error_(); + try { if (!state_.is_opened || !state_.is_initialized) { throw runtime_error("Camera already closed"); } - if (mode_ == VIDEO_MODE){ + + if (mode_ == VIDEO_MODE) { state_.is_streaming = false; - if (stream_thread_->joinable()) { + if (stream_thread_ && stream_thread_->joinable()) { stream_thread_->join(); stream_thread_.reset(); stream_thread_ = nullptr; } } - if (cap_.isOpened()) { - cap_.release(); + // 停止摄像头采集 + if (camera_capture_) { + camera_capture_->stop_capture(); } + state_.is_opened = false; + LOG(INFO) << "[UVCCamera] (stop): Camera stopped"; } catch (exception &e) { LOG(ERROR) << "[UVCCamera] (stop): " << e.what(); @@ -215,22 +179,26 @@ void UVCCamera::stop() { } } -void UVCCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) -{ +void UVCCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) { std::lock_guard lock(ctrl_mtx_); clear_error_(); - if (mode_ == PHOTO_MODE) { - if (!state_.is_opened) { - throw runtime_error("camera not opened"); - } - if (!cap_.read(color)) { - throw runtime_error("read color image failed"); - } + + if (!state_.is_opened) { + throw runtime_error("camera not opened"); } - else if (mode_ == VIDEO_MODE) - { - if (!state_.is_opened) { - throw runtime_error("camera not opened"); + + if (mode_ == PHOTO_MODE) { + // 单拍模式 - 这里需要从摄像头直接读取一帧 + // 由于我们使用CameraCapture类，这需要不同的处理方式 + throw runtime_error("[UVCCamera] (getRGBImage): PHOTO_MODE not supported with new FFmpeg architecture"); + } + else if (mode_ == VIDEO_MODE) { + // 从最新帧缓存中获取 + std::lock_guard frame_lock(latest_frame_mutex_); + if (latest_frame_valid_) { + latest_rgb_frame_.copyTo(color); + } else { + throw runtime_error("No frame available"); } } } @@ -240,7 +208,7 @@ void UVCCamera::getDepthImage(cv::Mat& depth, Rs2Intrinsics& intrinsics) { } void UVCCamera::getRGBDImages(cv::Mat &color, cv::Mat &depth, Rs2Intrinsics& intrinsics) { - throw runtime_error("[UVCCamera] (getDepthImage): unsupported usage"); + throw runtime_error("[UVCCamera] (getRGBDImages): unsupported usage"); } void UVCCamera::updateParams(const std::pair& param) { @@ -267,118 +235,103 @@ void UVCCamera::updateParams(const std::pair& param) { } } + +// UVCCamera.cpp 中的相关部分修改 + void UVCCamera::startRecording(const std::string &video_path) { std::lock_guard lock(ctrl_mtx_); clear_error_(); + if (mode_ != VIDEO_MODE) { throw runtime_error("[UVCCamera] (startRecording): 仅支持VIDEO_MODE"); } + if (!state_.is_opened) { throw runtime_error("[UVCCamera] (startRecording): 没有打开设备"); } + if (state_.is_recording) { throw runtime_error("[UVCCamera] (startRecording): 已在录像中"); } try { current_video_path_ = video_path; - std::string temp_path = current_video_path_ + ".temp"; // 临时文件 - // 1. 确定编码格式对应的AVCodecID（H.264/H.265） - AVCodecID codec_id; + // 根据codec设置编码器参数 + ffmpeg::EncoderConfig encoder_config; + encoder_config.width = width_; + encoder_config.height = height_; + encoder_config.fps = fps_; + + // 设置编码器类型 if (codec_ == "H264" || codec_ == "h264") { - codec_id = AV_CODEC_ID_H264; - } else if (codec_ == "H265" || codec_ == "hevc") { - codec_id = AV_CODEC_ID_HEVC; + encoder_config.bitrate = 8000000; // 8Mbps for H264 + encoder_config.codec = "libx264"; + } else if (codec_ == "H265" || codec_ == "hevc" || codec_ == "H265" || codec_ == "HEVC") { + encoder_config.bitrate = 5000000; // 5Mbps for H265 + encoder_config.codec = "libx265"; } else { - throw runtime_error("不支持的编码格式: " + codec_); + encoder_config.bitrate = 8000000; // 默认 + encoder_config.codec = "libx265"; } - const char* format_name = (codec_ == "H265" || codec_ == "h265" || codec_ == "HEVC" || codec_ == "hevc") ? "mp4" : "avi"; - auto ret = avformat_alloc_output_context2(&format_context_, nullptr, format_name, output_path_.c_str()); - // 2. 创建输出格式上下文（封装器核心） + + // 初始化视频编码器 + video_encoder_ = std::make_unique(); + int ret = video_encoder_->initialize(encoder_config); if (ret < 0) { - throw runtime_error("创建输出格式上下文失败"); + throw runtime_error("Failed to initialize video encoder"); } - // 3. 添加视频流并配置参数 - AVStream* stream = avformat_new_stream(format_context_, nullptr); - if (!stream) { - throw runtime_error("创建视频流失败"); - } - stream_ = stream; - AVCodecParameters* codecpar = stream->codecpar; - codecpar->codec_type = AVMEDIA_TYPE_VIDEO; - codecpar->codec_id = codec_id; - codecpar->width = width_; - codecpar->height = height_; - stream->time_base = {1, fps_}; // 时间基：1/fps（每帧间隔1个时间单位） + // 初始化视频写入器 + ffmpeg::WriterConfig writer_config; + writer_config.output_file = current_video_path_; + writer_config.width = width_; + writer_config.height = height_; + writer_config.fps = fps_; + writer_config.bitrate = encoder_config.bitrate; + writer_config.time_base = video_encoder_->get_time_base(); - // 4. 复制编码器的extradata（如H.264的SPS/PPS，确保播放器能解析） - if (rgbEncoder_ && rgbEncoder_->codec_context && rgbEncoder_->codec_context->extradata) { - codecpar->extradata = static_cast(av_mallocz(rgbEncoder_->codec_context->extradata_size + AV_INPUT_BUFFER_PADDING_SIZE)); - if (!codecpar->extradata) { - throw runtime_error("分配extradata内存失败"); + video_writer_ = std::make_unique(); + ret = video_writer_->initialize(writer_config, video_encoder_->get_codec_context()); + if (ret < 0) { + throw runtime_error("Failed to initialize video writer"); + } + + // 设置编码器回调，将编码后的数据包写入文件 + video_encoder_->set_packet_callback([this](AVPacket* packet, int64_t pts, int64_t dts) { + if (video_writer_) { + video_writer_->write_packet(packet, pts, dts); } - memcpy(codecpar->extradata, rgbEncoder_->codec_context->extradata, rgbEncoder_->codec_context->extradata_size); - codecpar->extradata_size = rgbEncoder_->codec_context->extradata_size; - } + }); - // 5. 打开输出文件 - if (!(format_context_->oformat->flags & AVFMT_NOFILE)) { - if (avio_open(&format_context_->pb, temp_path.c_str(), AVIO_FLAG_WRITE) < 0) { - throw runtime_error("打开输出文件失败: " + temp_path); + // 停止旧的流线程（如果存在） + if (stream_thread_) { + if (stream_thread_->joinable()) { + stream_thread_->join(); + is_streaming_running = false; } + stream_thread_.reset(); } - // 6. 写入文件头 - if (avformat_write_header(format_context_, nullptr) < 0) { - throw runtime_error("写入文件头失败"); - } - - // 7. 初始化数据包（用于封装编码帧） - packet_ = av_packet_alloc(); - if (!packet_) { - throw runtime_error("分配AVPacket失败"); - } - //不在录像也不在流传输，但是采集线程没有退出时。 - if (!state_.is_streaming && !state_.is_recording) { - if (stream_thread_) { - if (stream_thread_->joinable()) { - stream_thread_->join(); - is_streaming_running = false; - } - stream_thread_.reset(); - } - } - // 开启录像 + // 开启录像状态 state_.is_recording = true; - //开启流采集线程 + + // 开启流采集线程 if (!stream_thread_) { stream_thread_ = make_shared(&UVCCamera::streaming_worker_, this); - //延时100ms，等待流线程获取图像 std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - // 启动录像线程 - if (recording_thread_) { - if (recording_thread_->joinable()) { - recording_thread_->join(); - is_recording_running = false; - } - recording_thread_.reset(); - } - frame_count_ = 0; - recording_thread_ = make_shared(&UVCCamera::recording_worker_, this); + + LOG(INFO) << "[UVCCamera] (startRecording): Recording started to " << current_video_path_ + << " with timebase " << writer_config.time_base.num << "/" << writer_config.time_base.den; } catch (const std::exception& e) { // 异常清理 - if (packet_) av_packet_free(&packet_); - if (stream_ && stream_->codecpar->extradata) av_free(stream_->codecpar->extradata); - if (format_context_) { - if (!(format_context_->oformat->flags & AVFMT_NOFILE) && format_context_->pb) avio_closep(&format_context_->pb); - avformat_free_context(format_context_); + if (video_writer_) { + video_writer_->finish(); + video_writer_.reset(); } - stream_ = nullptr; - format_context_ = nullptr; + video_encoder_.reset(); state_.is_recording = false; throw runtime_error("[UVCCamera] (startRecording): " + std::string(e.what())); } @@ -387,412 +340,34 @@ void UVCCamera::startRecording(const std::string &video_path) { void UVCCamera::stopRecording() { std::lock_guard lock(ctrl_mtx_); clear_error_(); + if (mode_ != VIDEO_MODE) { throw runtime_error("[UVCCamera] (stopRecording): 仅支持VIDEO_MODE"); } + if (!state_.is_recording) { throw runtime_error("[UVCCamera] (stopRecording): 未在录像中"); } // 1. 停止录像线程 state_.is_recording = false; - if (recording_thread_ && recording_thread_->joinable()) { - recording_thread_->join(); - recording_thread_.reset(); + + // 2. 刷新编码器并完成写入 + if (video_encoder_) { + video_encoder_->flush(); } - // 2. 清理FFmpeg资源 - if (packet_) { - av_packet_free(&packet_); - packet_ = nullptr; + if (video_writer_) { + video_writer_->finish(); + video_writer_.reset(); } - if (stream_ && stream_->codecpar->extradata) { - av_free(stream_->codecpar->extradata); - stream_->codecpar->extradata = nullptr; - stream_->codecpar->extradata_size = 0; - } - if (format_context_) { - if (!(format_context_->oformat->flags & AVFMT_NOFILE) && format_context_->pb) { - avio_closep(&format_context_->pb); // 关闭文件 - } - avformat_free_context(format_context_); // 释放格式上下文 - format_context_ = nullptr; - } - stream_ = nullptr; - // 3. 重命名临时文件为目标文件 - std::string temp_path = current_video_path_ + ".temp"; - if (rename(temp_path.c_str(), current_video_path_.c_str()) != 0) { - throw runtime_error("重命名临时文件失败: " + temp_path + " -> " + current_video_path_); - } + video_encoder_.reset(); + + LOG(INFO) << "[UVCCamera] (stopRecording): Recording saved to " << current_video_path_; current_video_path_.clear(); } -void UVCCamera::pauseRecording() { - -} - -void UVCCamera::resumeRecording() { - -} - -void UVCCamera::streaming_worker_() { - - try { - // 计算理论上每帧之间的间隔时间(毫秒) - const int frame_interval = 1000 / fps_; - - bool success = false; - is_streaming_running = true; - cv::Mat frame; - int64_t frame_count = 0; - // 处于流传输或者录像状态时就不退出线程 - while (state_.is_streaming || state_.is_recording) { - // 记录当前帧处理开始时间 - auto frame_start_time = std::chrono::high_resolution_clock::now(); - if (!cap_.read(frame) || frame.empty()) { - throw runtime_error("[UVCCamera]: Failed to read frame."); - } - - // 以下为编码部分，用于流模式 - StreamFrameData frame_data; - // 保存图像数据 - { - frame.copyTo(frame_data.rgbImage); // 执行深拷贝 - } - // rgb图像编码 - success = encodeFrameWithEncoder(rgbEncoder_, frame_data.rgbImage, frame_data.rgbFrame, frame_data.bKey); - if (success) { - frame_data.fps = fps_; - frame_data.width = width_; - frame_data.height = height_; - frame_data.codec = codec_; - stream_frame_buffer_->push(frame_data); - } - else - { - std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); - continue; - } - // 计算从帧开始到现在的总耗时 - auto total_duration = std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - frame_start_time - ).count(); - - // 计算需要休眠的时间(确保总耗时达到frame_interval) - int sleep_time = frame_interval - total_duration; - // 只有当需要休眠的时间为正数时才休眠 - if (sleep_time > 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time)); - } - } - - is_streaming_running = false; - // 线程结束时清空队列 - stream_frame_buffer_->clear(); - recordingIndex_ = 0; - getImageIndex_ = 0; - streamIndex_ = 0; - } - catch (const std::exception &e) { - // 线程结束时清空队列 - stream_frame_buffer_->clear(); - // 确保线程状态正确更新 - is_streaming_running = false; - state_.is_error = true; - state_.error_message = e.what(); - LOG(ERROR) << "[UVCCamera]streaming_worker_ error:" << state_.error_message; - } -} - -void UVCCamera::recording_worker_() { - is_recording_running = true; - const int frame_interval = 1000 / fps_; - bool is_first_key = false; - try { - //保证当前采集线程正常运行 - while (state_.is_recording && is_streaming_running) { - // 等待缓冲区有数据 - if (stream_frame_buffer_->empty()) { - std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); - continue; - } - - // 1. 从缓冲区取编码好的帧 - auto frame_data_opt = stream_frame_buffer_->pop(recordingIndex_); - - if (!frame_data_opt.has_value()) { - std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); - continue; - } - StreamFrameData frame_data = frame_data_opt.value(); - - // 2. 验证编码数据有效性 - if (frame_data.rgbFrame.empty()) { - std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); - continue; - } - - // 先等待至第一个I帧 - if (!is_first_key) - { - is_first_key = frame_data.bKey; - if (!is_first_key) - { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - continue; - } - } - // 3. 初始化AVPacket（复用编码数据） - av_packet_unref(packet_); - - // 设置数据包数据 - packet_->data = const_cast(frame_data.rgbFrame.data()); // 注意：const_cast安全，因为仅读取 - packet_->size = static_cast(frame_data.rgbFrame.size()); - packet_->stream_index = stream_->index; // 指定流索引 - - // 设置时间戳（递增） - packet_->pts = frame_count_++; - packet_->dts = packet_->pts; - packet_->duration = 1; // 每帧持续1个时间单位（根据time_base） - - // 标记关键帧 - if (frame_data.bKey) { - packet_->flags |= AV_PKT_FLAG_KEY; - } - - // 转换时间基 - av_packet_rescale_ts(packet_, {1, fps_}, stream_->time_base); - - // 写入帧 - if (av_interleaved_write_frame(format_context_, packet_) < 0) { - std::cerr << "写入第" << frame_count_ << "帧失败" << std::endl; - } - - std::this_thread::sleep_for(std::chrono::milliseconds(33)); - } - - // 7. 写入文件尾（完成封装） - av_write_trailer(format_context_); - - } catch (const std::exception& e) { - std::cerr << "录像线程错误: " << e.what() << std::endl; - } - - is_recording_running = false; - state_.is_recording = false; -} - -// 初始化单个编码器的通用函数 -bool UVCCamera::initSingleEncoder(std::shared_ptr& encoder, - const std::string& codec_name, - int width, int height, int fps) { - // 1. 创建编码器实例（不变） - encoder = std::make_shared(); - encoder->codec_name = codec_name; - encoder->width = width; - encoder->height = height; - encoder->fps = fps; - - // 2. 查找编码器（不变） - const AVCodec* codec = nullptr; - if (codec_name == "h264" || codec_name == "H264") { - codec = avcodec_find_encoder_by_name("libx264"); - if (!codec) codec = avcodec_find_encoder(AV_CODEC_ID_H264); - } else if (codec_name == "h265" || codec_name == "HEVC" || codec_name == "H265") { - codec = avcodec_find_encoder_by_name("libx265"); - if (!codec) codec = avcodec_find_encoder(AV_CODEC_ID_HEVC); - } - if (!codec) { - std::cerr << "Failed to find " << codec_name << " encoder!" << std::endl; - return false; - } - - // 3. 初始化编码器上下文（不变） - encoder->codec_context = avcodec_alloc_context3(codec); - if (!encoder->codec_context) { - std::cerr << "Failed to allocate codec context!" << std::endl; - return false; - } - - // 4. 设置编码器参数（不变） - AVCodecContext* ctx = encoder->codec_context; - ctx->codec_type = AVMEDIA_TYPE_VIDEO; - ctx->width = (width + 1) & ~1; // 确保宽度为偶数 - ctx->height = (height + 1) & ~1;// 确保高度为偶数 - ctx->time_base = {1, fps}; // 时间基（1/fps） - ctx->framerate = {fps, 1}; // 帧率 - ctx->max_b_frames = 0; // 禁用B帧（降低延迟） - ctx->gop_size = 10;//I帧间隔1，每一帧都是I帧 - // 5. 设置编码器私有参数 - if (codec->id == AV_CODEC_ID_H264) { - av_opt_set(ctx->priv_data, "preset", "ultrafast", 0); - av_opt_set(ctx->priv_data, "tune", "zerolatency", 0); - av_opt_set(ctx->priv_data, "profile", "baseline", 0); - av_opt_set(ctx->priv_data, "repeat-headers", "1", 0); - av_opt_set(ctx->priv_data, "annexb", "1", 0); - } else if (codec->id == AV_CODEC_ID_HEVC) { - // 直接使用默认参数，不自定义 - // av_opt_set(ctx->priv_data, "preset", "ultrafast", 0); - // av_opt_set(ctx->priv_data, "tune", "zerolatency", 0); - } - - // 6. 设置像素格式（不变） - const enum AVPixelFormat* pix_fmts = codec->pix_fmts; - if (!pix_fmts) { - std::cout << "Using default pixel format: YUV420P" << std::endl; - ctx->pix_fmt = AV_PIX_FMT_YUV420P; - } else { - ctx->pix_fmt = pix_fmts[0]; - std::cout << "Selected pixel format: " << av_get_pix_fmt_name(ctx->pix_fmt) << std::endl; - } - - // 7. 打开编码器（不变） - int ret = avcodec_open2(ctx, codec, nullptr); - if (ret < 0) { - char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0}; - av_strerror(ret, errbuf, sizeof(errbuf)); - std::cerr << "Failed to open " << codec_name << " encoder: " << errbuf << std::endl; - return false; - } - - // 8. 分配AVFrame（不变） - encoder->frame = av_frame_alloc(); - if (!encoder->frame) { - std::cerr << "Failed to allocate AVFrame!" << std::endl; - return false; - } - encoder->frame->format = ctx->pix_fmt; - encoder->frame->width = ctx->width; - encoder->frame->height = ctx->height; - if (av_frame_get_buffer(encoder->frame, 0) < 0) { - std::cerr << "Failed to allocate AVFrame buffer!" << std::endl; - av_frame_free(&encoder->frame); - return false; - } - - // 9. 分配AVPacket（不变） - encoder->packet = av_packet_alloc(); - if (!encoder->packet) { - std::cerr << "Failed to allocate AVPacket!" << std::endl; - return false; - } - - std::cout << "Successfully initialized " << codec_name << " encoder ( " - << width << "x" << height << "@" << fps << "fps )" << std::endl; - return true; -} - -bool UVCCamera::encodeFrameWithEncoder(std::shared_ptr& encoder, const cv::Mat& frame,std::vector& encoded_frame,bool& is_key) { - - if (!encoder || !encoder->codec_context || !encoder->frame || !encoder->packet) { - return false; - } - - // 确保输入帧尺寸匹配（不变） - if (frame.cols != encoder->width || frame.rows != encoder->height) { - std::cerr << "Frame size does not match encoder dimensions" << std::endl; - return false; - } - - // 【修复3：设置递增的PTS，确保编码器正确处理I帧请求】 - encoder->frame->pts = encoder->frame_pts++; // 分配唯一PTS - - // 根据cv::Mat的类型设置源格式（不变） - AVPixelFormat src_pix_fmt; - if (frame.channels() == 3) { - src_pix_fmt = AV_PIX_FMT_BGR24; // OpenCV默认BGR - } else if (frame.channels() == 4) { - src_pix_fmt = AV_PIX_FMT_BGRA; // 4通道为BGRA - } else if (frame.channels() == 1) { - src_pix_fmt = AV_PIX_FMT_GRAY8; // 单通道灰度图 - } else { - std::cerr << "Unsupported number of channels: " << frame.channels() << std::endl; - return false; - } - // 【修复4：动态创建sws_context（匹配当前输入格式）】 - if (encoder->sws_context) { - sws_freeContext(encoder->sws_context); // 释放旧上下文 - } - encoder->sws_context = sws_getContext( - frame.cols, frame.rows, src_pix_fmt, // 输入格式由当前frame决定 - encoder->codec_context->width, encoder->codec_context->height, encoder->codec_context->pix_fmt, - SWS_BILINEAR, nullptr, nullptr, nullptr - ); - if (!encoder->sws_context) { - std::cerr << "Failed to create SwsContext" << std::endl; - return false; - } - - // 转换输入帧格式为编码器所需格式（不变） - const uint8_t* src_data[AV_NUM_DATA_POINTERS] = {frame.data}; - int src_linesize[AV_NUM_DATA_POINTERS] = {static_cast(frame.step)}; - - int ret = sws_scale(encoder->sws_context, src_data, src_linesize, 0, frame.rows, - encoder->frame->data, encoder->frame->linesize); - if (ret < 0) { - std::cerr << "Error scaling frame" << std::endl; - return false; - } - // 发送帧到编码器（不变） - ret = avcodec_send_frame(encoder->codec_context, encoder->frame); - if (ret < 0) { - char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0}; - av_strerror(ret, errbuf, sizeof(errbuf)); - std::cerr << "Error sending frame to encoder: " << errbuf << std::endl; - return false; - } - while (true) - { - // 接收编码后的数据（不变） - ret = avcodec_receive_packet(encoder->codec_context, encoder->packet); - if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { - break; - } else if (ret < 0) { - char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0}; - av_strerror(ret, errbuf, sizeof(errbuf)); - std::cerr << "Error receiving packet from encoder: " << errbuf << std::endl; - break; - } - - // 调试：打印帧类型（I帧/P帧） - if (encoder->packet->flags & AV_PKT_FLAG_KEY) { - is_key = true; - //std::cout << "Encoded I frame (size: " << encoder->packet->size << " bytes)" << std::endl; - } else { - is_key = false; - //std::cout << "Encoded P frame (size: " << encoder->packet->size << " bytes)" << std::endl; - } - - // 预留足够空间，避免多次内存分配 - encoded_frame.reserve(encoded_frame.size() + encoder->packet->size); - // 复制数据包内容到输出向量 - encoded_frame.insert(encoded_frame.end(), - encoder->packet->data, - encoder->packet->data + encoder->packet->size); - av_packet_unref(encoder->packet); - } - - - // 验证帧有效性（NALU起始码、元数据） - if (!encoded_frame.empty()) { - // 检查NALU起始码 - bool has_start_code = false; - if ((encoded_frame[0] == 0 && encoded_frame[1] == 0 && encoded_frame[2] == 1) || - (encoded_frame[0] == 0 && encoded_frame[1] == 0 && encoded_frame[2] == 0 && encoded_frame[3] == 1)) { - has_start_code = true; - } - if (!has_start_code) { - std::cerr << "Invalid frame: no NALU start code!" << std::endl; - return false; - } - } else { - //std::cerr << "Encoded frame is empty!" << std::endl; - return false; - } - return true; -} - void UVCCamera::getEncodedFrame(StreamFrameData& frame_data, size_t& index) { // 环形队列取数据的index由接口传入 auto frame = stream_frame_buffer_->pop(index); @@ -801,10 +376,20 @@ void UVCCamera::getEncodedFrame(StreamFrameData& frame_data, size_t& index) { } } -bool UVCCamera::startStreaming() -{ +bool UVCCamera::startStreaming() { std::lock_guard lock(ctrl_mtx_); - //不在录像也不在流传输，但是采集线程没有退出时。 + + if (mode_ != VIDEO_MODE) { + LOG(ERROR) << "[UVCCamera] (startStreaming): 仅支持VIDEO_MODE"; + return false; + } + + if (!state_.is_opened) { + LOG(ERROR) << "[UVCCamera] (startStreaming): 没有打开设备"; + return false; + } + + // 如果已经停止，重启流线程 if (!state_.is_streaming && !state_.is_recording) { if (stream_thread_) { if (stream_thread_->joinable()) { @@ -814,25 +399,135 @@ bool UVCCamera::startStreaming() stream_thread_.reset(); } } - //开启流采集线程 + + // 开启流采集线程 if (!stream_thread_) { state_.is_streaming = true; stream_thread_ = make_shared(&UVCCamera::streaming_worker_, this); - - //延时100ms，等待流线程获取图像 std::this_thread::sleep_for(std::chrono::milliseconds(100)); } + stream_count_++; + LOG(INFO) << "[UVCCamera] (startStreaming): Streaming started, count: " << stream_count_; return true; } -void UVCCamera::stopStreaming() -{ +void UVCCamera::stopStreaming() { std::lock_guard lock(ctrl_mtx_); - stream_count_--; - if (stream_count_ == 0) - { - // 当前已经没有正在使用的流了，编码采集线程状态修改 + + if (stream_count_ > 0) { + stream_count_--; + } + + if (stream_count_ == 0) { + // 当前已经没有正在使用的流了，停止流线程 state_.is_streaming = false; + LOG(INFO) << "[UVCCamera] (stopStreaming): Streaming stopped"; + } else { + LOG(INFO) << "[UVCCamera] (stopStreaming): Streaming count reduced to " << stream_count_; } } + +// AVFrame转换为cv::Mat的辅助函数 +bool UVCCamera::convertAVFrameToMat(AVFrame* frame, cv::Mat& mat) { + if (!frame || !frame->data[0]) { + return false; + } + + // 根据不同的像素格式进行转换 + switch (frame->format) { + case AV_PIX_FMT_YUV420P: + { + cv::Mat yuv(frame->height + frame->height/2, frame->width, CV_8UC1, frame->data[0]); + cv::cvtColor(yuv, mat, cv::COLOR_YUV2RGB_I420); + return true; + } + case AV_PIX_FMT_YUYV422: + { + cv::Mat yuyv(frame->height, frame->width, CV_8UC2, frame->data[0]); + cv::cvtColor(yuyv, mat, cv::COLOR_YUV2RGB_YUYV); + return true; + } + case AV_PIX_FMT_BGR24: + { + mat = cv::Mat(frame->height, frame->width, CV_8UC3, frame->data[0]); + return true; + } + case AV_PIX_FMT_RGB24: + { + mat = cv::Mat(frame->height, frame->width, CV_8UC3, frame->data[0]); + cv::cvtColor(mat, mat, cv::COLOR_RGB2BGR); + return true; + } + default: + LOG(WARNING) << "[UVCCamera] Unsupported pixel format: " << frame->format; + return false; + } +} + +void UVCCamera::streaming_worker_() { + try { + const int frame_interval = 1000 / fps_; + is_streaming_running = true; + + LOG(INFO) << "[UVCCamera] (streaming_worker_): Streaming worker started"; + + // 开始摄像头采集 + camera_capture_->start_capture([this](AVFrame* frame) { + if (!frame) { + return; + } + + StreamFrameData frame_data; + + // 更新最新帧缓存（用于getRGBImage） + { + std::lock_guard frame_lock(latest_frame_mutex_); + if (convertAVFrameToMat(frame, latest_rgb_frame_)) { + latest_frame_valid_ = true; + + // 保存到frame_data + latest_rgb_frame_.copyTo(frame_data.rgbImage); + } + } + + // 编码帧（如果是录制状态） + if (state_.is_recording && video_encoder_) { + int ret = video_encoder_->encode_frame(frame); + if (ret < 0) { + LOG(WARNING) << "[UVCCamera] Failed to encode frame"; + } + } + + // 设置帧数据信息 + frame_data.fps = fps_; + frame_data.width = width_; + frame_data.height = height_; + frame_data.codec = codec_; + frame_data.bKey = false; // 由编码器回调设置 + + // 推送到缓冲区 + stream_frame_buffer_->push(frame_data); + }); + + // 等待直到停止 + while (state_.is_streaming || state_.is_recording) { + std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); + } + + is_streaming_running = false; + stream_frame_buffer_->clear(); + recordingIndex_ = 0; + getImageIndex_ = 0; + streamIndex_ = 0; + + LOG(INFO) << "[UVCCamera] (streaming_worker_): Streaming worker stopped"; + + } catch (const std::exception &e) { + stream_frame_buffer_->clear(); + is_streaming_running = false; + state_.is_error = true; + state_.error_message = e.what(); + LOG(ERROR) << "[UVCCamera] streaming_worker_ error:" << state_.error_message; + } +} \ No newline at end of file diff --git a/cmvr-es/devices/microphone/ffmpeg_microphone/src/ffmpeg_microphone.cpp b/cmvr-es/devices/microphone/ffmpeg_microphone/src/ffmpeg_microphone.cpp index a78beff7..d6aa1370 100644 --- a/cmvr-es/devices/microphone/ffmpeg_microphone/src/ffmpeg_microphone.cpp +++ b/cmvr-es/devices/microphone/ffmpeg_microphone/src/ffmpeg_microphone.cpp @@ -32,7 +32,7 @@ ffmpegMicroPhone::ffmpegMicroPhone(const config::FFMpegMicroPhoneConfig& cfg):in sample_rate_(44100), channels_(2), config_(cfg) { id_ = config_.id(); - input_device_ = cfg_.getAttrString("alsa"); + input_device_ = config_.input_device(); channels_ = config_.channels(); sample_rate_ = config_.samplerate(); state_.volume = config_.volume(); @@ -174,9 +174,9 @@ bool ffmpegMicroPhone::initFFmpeg() { int err; avdevice_register_all(); // 初始化输入设备 - AVInputFormat* input_fmt = av_find_input_format("alsa"); + AVInputFormat* input_fmt = av_find_input_format("pulse"); if (!input_fmt) { - LOG(ERROR) << "Could not find ALSA input format"; + LOG(ERROR) << "Could not find pulse input format"; return false; } diff --git a/protos/cmvr/config/microphone_config/microphone_config.proto b/protos/cmvr/config/microphone_config/microphone_config.proto index e98874f2..be4451f0 100644 --- a/protos/cmvr/config/microphone_config/microphone_config.proto +++ b/protos/cmvr/config/microphone_config/microphone_config.proto @@ -7,6 +7,7 @@ message FFMpegMicroPhoneConfig{ int32 sampleRate = 3; int32 volume = 4; bool enable = 5; + string input_device = 6; } message MicroPhoneConfig{