diff --git a/CMakeLists.txt b/CMakeLists.txt index ee32b263..0b0da92d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,7 +82,6 @@ install(TARGETS proto LIBRARY DESTINATION lib) # include ############################################################ include_directories( - /usr/include/opencv4 /usr/local/include/ ${PROTO_BINARY_DIR} ${PROJECT_SOURCE_DIR}/include diff --git a/cmvr-es/common/config/camera_config/camera_config.pb.txt b/cmvr-es/common/config/camera_config/camera_config.pb.txt index 5e9bf488..b99b75d2 100644 --- a/cmvr-es/common/config/camera_config/camera_config.pb.txt +++ b/cmvr-es/common/config/camera_config/camera_config.pb.txt @@ -1,16 +1,16 @@ realsense_cameras { id: "cam2" - serialNumber: "123456789012" - width: 1280 - height: 720 + serialNumber: "243122074587" + width: 640 + height: 480 fps: 30 codec: "H265" camera_mode: CAMERA_MODE_VIDEO stream_mode: STREAM_MODE_RGBD align_mode: ALIGN_MODE_COLOR buffer_size: 30 - sync: true - enable: false + sync: false + enable: true } uvc_cameras { @@ -23,5 +23,5 @@ uvc_cameras { camera_mode: CAMERA_MODE_VIDEO stream_mode: STREAM_MODE_RGB buffer_size: 30 - enable: true + enable: false } diff --git a/cmvr-es/devices/camera/realsense_camera/CMakeLists.txt b/cmvr-es/devices/camera/realsense_camera/CMakeLists.txt index f82d2a3d..e00c7a9e 100644 --- a/cmvr-es/devices/camera/realsense_camera/CMakeLists.txt +++ b/cmvr-es/devices/camera/realsense_camera/CMakeLists.txt @@ -1,5 +1,3 @@ - -find_package(OpenCV REQUIRED) add_library(realsense_camera SHARED src/realsense_camera.cpp) target_include_directories(realsense_camera PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) @@ -7,7 +5,7 @@ target_include_directories(realsense_camera PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) add_library(cmvr_es::device::realsense_camera ALIAS realsense_camera) # ✅ 命名空间别名 OK # 链接 librealsense2 -target_link_libraries(realsense_camera PRIVATE realsense2 ${OpenCV_LIBS} ) +target_link_libraries(realsense_camera PRIVATE realsense2 opencv_core opencv_imgproc opencv_videoio) install(TARGETS realsense_camera LIBRARY DESTINATION lib) # -------------------------------------------------------- diff --git a/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h b/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h index 8a361956..7fce04b7 100644 --- a/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h +++ b/cmvr-es/devices/camera/realsense_camera/include/realsense_camera.h @@ -9,47 +9,9 @@ #include #include - namespace cmvr::device{ enum StreamMode {RGBD_MODE, COLOR_MODE,DEPTH_MODE}; - struct FfmpegEncoderInfo { - std::string codec_name; // 编码器名称（如"h264"、"hevc"） - int width = 0; // 图像宽度 - int height = 0; // 图像高度 - int fps = 0; // 帧率 - int64_t frame_pts = 0; - bool bRunning = false; // 是否进行编码 - AVCodecContext* codec_context = nullptr; // 编码器上下文 - AVFrame* frame = nullptr; // 输入帧 - AVPacket* packet = nullptr; // 输出包 - SwsContext* sws_context = nullptr; // 图像转换上下文（如果需要格式转换） - // 析构函数：释放FFmpeg资源（核心！避免内存泄漏） - ~FfmpegEncoderInfo() { - // 释放编码帧 - if (frame) { - av_frame_free(&frame); - frame = nullptr; - } - // 释放编码包 - if (packet) { - av_packet_free(&packet); - packet = nullptr; - } - // 释放编码器上下文 - if (codec_context) { - avcodec_close(codec_context); // 关闭编码器 - avcodec_free_context(&codec_context); // 释放上下文 - codec_context = nullptr; - } - // 释放格式转换上下文 - if (sws_context) { - sws_freeContext(sws_context); - sws_context = nullptr; - } - //std::cout << "FfmpegEncoderInfo resources released." << std::endl; - } - }; class RealsenseCamera final : public AbstractCamera { public: diff --git a/cmvr-es/devices/camera/uvc_camera/CMakeLists.txt b/cmvr-es/devices/camera/uvc_camera/CMakeLists.txt index 39ccb218..2079ddb6 100644 --- a/cmvr-es/devices/camera/uvc_camera/CMakeLists.txt +++ b/cmvr-es/devices/camera/uvc_camera/CMakeLists.txt @@ -1,10 +1,8 @@ -find_package(OpenCV REQUIRED) - add_library(uvc_camera SHARED src/uvc_camera.cpp) target_include_directories(uvc_camera PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(uvc_camera PUBLIC ${OpenCV_LIBS} glog) +target_link_libraries(uvc_camera PUBLIC glog opencv_core opencv_imgproc) add_library(cmvr_es::device::uvc_camera ALIAS uvc_camera) install(TARGETS uvc_camera LIBRARY DESTINATION lib) \ No newline at end of file diff --git a/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h b/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h index 87a6d8b8..9c5cd4a0 100644 --- a/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h +++ b/cmvr-es/devices/camera/uvc_camera/include/uvc_camera.h @@ -1,25 +1,72 @@ -// uvc_camera.h +// +// Created by xtkuang on 2025/5/30. +// + #ifndef CMVR_ES_UVC_CAMERA_H #define CMVR_ES_UVC_CAMERA_H #include "utils/base/include/ring_buffer.h" #include "camera/abstract_camera.h" -// 使用新的FFmpeg封装类 -#include "common/utils/ffmpeg/include/CameraCapture.h" -#include "common/utils/ffmpeg/include/VideoFrameEncoder.h" -#include "common/utils/ffmpeg/include/VideoWriter.h" +//使用ffmpeg来编码保存视频文件 +#define USE_FFMPEG_ENCODER 1 + +#if USE_FFMPEG_ENCODER +#include "speaker/ffmpeg_speaker/include/ffmpeg_ptr.h" +#endif namespace cmvr::device { enum CameraMode {PHOTO_MODE, VIDEO_MODE}; + struct ImageData { + //原始图像数据 cv::Mat rgbImage; + //深度图原始数据 cv::Mat depthImage; }; - // USB摄像机 + struct FfmpegEncoderInfo { + std::string codec_name; // 编码器名称（如"h264"、"hevc"） + int width = 0; // 图像宽度 + int height = 0; // 图像高度 + int fps = 0; // 帧率 + int64_t frame_pts = 0; + bool bRunning = false; // 是否进行编码 + AVCodecContext* codec_context = nullptr; // 编码器上下文 + AVFrame* frame = nullptr; // 输入帧 + AVPacket* packet = nullptr; // 输出包 + SwsContext* sws_context = nullptr; // 图像转换上下文（如果需要格式转换） + + // 析构函数：释放FFmpeg资源（核心！避免内存泄漏） + ~FfmpegEncoderInfo() { + // 释放编码帧 + if (frame) { + av_frame_free(&frame); + frame = nullptr; + } + // 释放编码包 + if (packet) { + av_packet_free(&packet); + packet = nullptr; + } + // 释放编码器上下文 + if (codec_context) { + avcodec_close(codec_context); // 关闭编码器 + avcodec_free_context(&codec_context); // 释放上下文 + codec_context = nullptr; + } + // 释放格式转换上下文 + if (sws_context) { + sws_freeContext(sws_context); + sws_context = nullptr; + } + //std::cout << "FfmpegEncoderInfo resources released." << std::endl; + } + }; + + //USB摄像机 class UVCCamera final : public AbstractCamera { public: explicit UVCCamera(const XmlNode& config); @@ -35,41 +82,56 @@ namespace cmvr::device { void updateParams(const std::pair& param) override; void startRecording(const std::string &video_path) override; void stopRecording() override; + void pauseRecording() override; + void resumeRecording() override; + // 初始化单个编码器的通用函数（复用逻辑，避免重复代码） + static bool initSingleEncoder(std::shared_ptr& encoder, + const std::string& codec_name, + int width, int height, int fps); + static bool encodeFrameWithEncoder(std::shared_ptr& encoder,const cv::Mat& frame,std::vector& encoded_frame,bool& is_key); void getEncodedFrame(StreamFrameData& frame_data, size_t& index) override; bool startStreaming() override; void stopStreaming() override; - private: void streaming_worker_(); - - // 辅助函数 - static bool convertAVFrameToMat(AVFrame* frame, cv::Mat& mat); + void recording_worker_(); int fps_; int width_; int height_; std::string serial_; + cv::VideoCapture cap_; size_t buffer_size_; std::string codec_; CameraMode mode_; + // std::shared_ptr current_image_; std::shared_ptr> stream_frame_buffer_; std::string current_video_path_; std::mutex ctrl_mtx_{}; + std::unique_ptr video_writer_; std::shared_ptr stream_thread_; + std::shared_ptr recording_thread_; - // 新的FFmpeg封装类实例 - std::unique_ptr camera_capture_; - std::unique_ptr video_encoder_; - std::unique_ptr video_writer_; - // 用于临时存储最新帧 - std::mutex latest_frame_mutex_; - cv::Mat latest_rgb_frame_; - bool latest_frame_valid_ = false; + cv::Mat latest_frame_; // 存储最新帧 + std::mutex frame_mutex_; // 保护最新帧的访问 + std::string output_path_; + bool is_recording_ = false; + // ffmpeg录像 + AVFormatContext* format_context_ = nullptr; + AVCodecContext* codec_context_ = nullptr; + AVStream* stream_ = nullptr; + AVFrame* frame_ = nullptr; + AVPacket* packet_ = nullptr; + SwsContext* sws_context_ = nullptr; + int frame_count_ = 0; + + std::shared_ptr rgbEncoder_;//rgb图像编码 + std::shared_ptr depthEncoder_;//深度图编码 size_t streamIndex_ = 0; size_t recordingIndex_ = 0; size_t getImageIndex_ = 0; @@ -83,4 +145,6 @@ namespace cmvr::device { }; } -#endif // CMVR_ES_UVC_CAMERA_H \ No newline at end of file + + +#endif //CMVR_ES_UVC_CAMERA_H \ No newline at end of file diff --git a/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp b/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp index beae798b..fe7f2c97 100644 --- a/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp +++ b/cmvr-es/devices/camera/uvc_camera/src/uvc_camera.cpp @@ -1,12 +1,14 @@ -// uvc_camera.cpp +// +// Created by xtkuang on 2025/5/30. +// + #include "../include/uvc_camera.h" -#include -#include -#include using namespace std; using namespace cmvr::device; +#define USE_LIST_IMAGE 1 + UVCCamera::UVCCamera(const XmlNode& config) : AbstractCamera(config) { try { serial_ = cfg_.getAttrString("serial"); @@ -34,13 +36,14 @@ UVCCamera::UVCCamera(const XmlNode& config) : AbstractCamera(config) { throw runtime_error("invalid argument mode=" + mode_str); } - codec_ = cfg_.getAttrDefault("codec", "H264"); + codec_ = cfg_.getAttrDefault("codec", "H265"); + + //initFFmpeg(); } catch (exception &e) { throw runtime_error(e.what()); } } - UVCCamera::UVCCamera(const config::UVCCameraConfig& camera):camera_(camera) { serial_ = camera_.usb(); @@ -70,18 +73,12 @@ UVCCamera::UVCCamera(const config::UVCCameraConfig& camera):camera_(camera) codec_ = camera_.codec(); } - UVCCamera::~UVCCamera() { stop(); - - // 释放FFmpeg资源 - video_writer_.reset(); - video_encoder_.reset(); - camera_capture_.reset(); - - // 等待线程结束 if (stream_thread_ && stream_thread_->joinable()) stream_thread_->join(); + if (recording_thread_ && recording_thread_->joinable()) + recording_thread_->join(); } void UVCCamera::init() { @@ -89,89 +86,128 @@ void UVCCamera::init() { clear_error_(); state_.is_initialized = false; - // 初始化环形缓冲区 stream_frame_buffer_ = std::make_shared>(buffer_size_); stream_frame_buffer_->clear(); + if (cap_.isOpened()) { + cap_.release(); + } - // 初始化FFmpeg库 - avdevice_register_all(); - av_log_set_level(AV_LOG_ERROR); + cap_.open(serial_, cv::CAP_V4L2); + this_thread::sleep_for(chrono::milliseconds(100)); - // 初始化CameraCapture - try { - ffmpeg::CameraConfig camera_config; - camera_config.device_name = serial_; - camera_config.width = width_; - camera_config.height = height_; - camera_config.fps = fps_; - - camera_capture_ = std::make_unique(); - int ret = camera_capture_->initialize(camera_config); - if (ret < 0) { - throw runtime_error("Failed to initialize camera capture"); - } - - LOG(INFO) << "[UVCCamera] (init): CameraCapture initialized successfully"; - } catch (const std::exception& e) { + if (!cap_.isOpened()) { state_.is_error = true; - state_.error_message = "Failed to init camera capture: " + std::string(e.what()); - LOG(ERROR) << "[UVCCamera] (init) " << state_.error_message; + state_.error_message = "Failed to open USB camera at index " + serial_; + LOG(ERROR) << "[UVCCamera] (init)" << state_.error_message; throw runtime_error(state_.error_message); } + // 设置格式为MJPG + cap_.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + + if (!cap_.set(cv::CAP_PROP_FRAME_WIDTH, width_)) { + state_.is_error = true; + state_.error_message = "set width failed"; + LOG(ERROR) << "[UVCCamera] (init): set width failed"; + throw runtime_error("[UVCCamera] (init): set width failed"); + } + state_.width = width_; + if (!cap_.set(cv::CAP_PROP_FRAME_HEIGHT, height_)) { + state_.is_error = true; + state_.error_message = "set height failed"; + LOG(ERROR) << "[UVCCamera] (init): set height failed"; + throw runtime_error("[UVCCamera] (init): set height failed"); + } + state_.height = height_; + if (!cap_.set(cv::CAP_PROP_FPS, fps_)) { + state_.is_error = true; + state_.error_message = "set fps failed"; + LOG(ERROR) << "[UVCCamera] (init): set fps failed"; + throw runtime_error("[UVCCamera] (init): set fps failed"); + } + //初始化编码器 + // 初始化RGB编码器（示例参数：640x480，30fps，H.264） + if (!initSingleEncoder(rgbEncoder_, codec_, width_, height_, fps_)) { + LOG(ERROR) << "[UVCCamera] (start): Failed to init RGB encoder!"; + throw runtime_error("Failed to init RGB encoder!"); + } + state_.fps = fps_; state_.is_initialized = true; state_.is_error = false; - LOG(INFO) << "[UVCCamera] (init): UVC camera initialized at index " << serial_; + LOG(INFO) << "[UVCCamera] (init): UVC camera initialized at index " << serial_; // 记录初始化成功日志 } void UVCCamera::start() { std::lock_guard lock(ctrl_mtx_); clear_error_(); - if (!state_.is_initialized) { - LOG(ERROR) << "[UVCCamera] (start): Camera not initialized"; + LOG(ERROR) << "[UVCCamera] (init): Camera not initialized"; throw runtime_error("Camera not initialized"); } - if (state_.is_opened) { - LOG(WARNING) << "[UVCCamera] (start): camera already started"; + LOG(WARNING) << "[RealsenseCamera] (start): camera already started"; return; } + cap_.open(serial_, cv::CAP_V4L2); + this_thread::sleep_for(chrono::milliseconds(100)); + if (!cap_.isOpened()) { + state_.is_error = true; + state_.error_message = "Failed to open USB camera at index " + serial_; + LOG(ERROR) << "[UVCCamera] (init)" << state_.error_message; + throw runtime_error(state_.error_message); + } + + // 设置格式为MJPG + cap_.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + + if (!cap_.set(cv::CAP_PROP_FRAME_WIDTH, width_)) { + state_.is_error = true; + state_.error_message = "set width failed"; + LOG(ERROR) << "[UVCCamera] (init): set width failed"; + throw runtime_error("[UVCCamera] (init): set width failed"); + } + state_.width = width_; + if (!cap_.set(cv::CAP_PROP_FRAME_HEIGHT, height_)) { + state_.is_error = true; + state_.error_message = "set height failed"; + LOG(ERROR) << "[UVCCamera] (init): set height failed"; + throw runtime_error("[UVCCamera] (init): set height failed"); + } + state_.height = height_; + if (!cap_.set(cv::CAP_PROP_FPS, fps_)) { + state_.is_error = true; + state_.error_message = "set fps failed"; + LOG(ERROR) << "[UVCCamera] (init): set fps failed"; + throw runtime_error("[UVCCamera] (init): set fps failed"); + } state_.is_opened = true; - LOG(INFO) << "[UVCCamera] (start): Camera started"; } void UVCCamera::stop() { - // 先停止录制 + //先停止录制再关闭摄像头 if (state_.is_recording) { stopRecording(); } - std::lock_guard lock(ctrl_mtx_); clear_error_(); - try { if (!state_.is_opened || !state_.is_initialized) { throw runtime_error("Camera already closed"); } - - if (mode_ == VIDEO_MODE) { + if (mode_ == VIDEO_MODE){ state_.is_streaming = false; - if (stream_thread_ && stream_thread_->joinable()) { + if (stream_thread_->joinable()) { stream_thread_->join(); stream_thread_.reset(); stream_thread_ = nullptr; } } - // 停止摄像头采集 - if (camera_capture_) { - camera_capture_->stop_capture(); + if (cap_.isOpened()) { + cap_.release(); } - state_.is_opened = false; - LOG(INFO) << "[UVCCamera] (stop): Camera stopped"; } catch (exception &e) { LOG(ERROR) << "[UVCCamera] (stop): " << e.what(); @@ -179,26 +215,22 @@ void UVCCamera::stop() { } } -void UVCCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) { +void UVCCamera::getRGBImage(cv::Mat& color, Rs2Intrinsics& intrinsics) +{ std::lock_guard lock(ctrl_mtx_); clear_error_(); - - if (!state_.is_opened) { - throw runtime_error("camera not opened"); - } - if (mode_ == PHOTO_MODE) { - // 单拍模式 - 这里需要从摄像头直接读取一帧 - // 由于我们使用CameraCapture类，这需要不同的处理方式 - throw runtime_error("[UVCCamera] (getRGBImage): PHOTO_MODE not supported with new FFmpeg architecture"); + if (!state_.is_opened) { + throw runtime_error("camera not opened"); + } + if (!cap_.read(color)) { + throw runtime_error("read color image failed"); + } } - else if (mode_ == VIDEO_MODE) { - // 从最新帧缓存中获取 - std::lock_guard frame_lock(latest_frame_mutex_); - if (latest_frame_valid_) { - latest_rgb_frame_.copyTo(color); - } else { - throw runtime_error("No frame available"); + else if (mode_ == VIDEO_MODE) + { + if (!state_.is_opened) { + throw runtime_error("camera not opened"); } } } @@ -208,7 +240,7 @@ void UVCCamera::getDepthImage(cv::Mat& depth, Rs2Intrinsics& intrinsics) { } void UVCCamera::getRGBDImages(cv::Mat &color, cv::Mat &depth, Rs2Intrinsics& intrinsics) { - throw runtime_error("[UVCCamera] (getRGBDImages): unsupported usage"); + throw runtime_error("[UVCCamera] (getDepthImage): unsupported usage"); } void UVCCamera::updateParams(const std::pair& param) { @@ -235,103 +267,118 @@ void UVCCamera::updateParams(const std::pair& param) { } } - -// UVCCamera.cpp 中的相关部分修改 - void UVCCamera::startRecording(const std::string &video_path) { std::lock_guard lock(ctrl_mtx_); clear_error_(); - if (mode_ != VIDEO_MODE) { throw runtime_error("[UVCCamera] (startRecording): 仅支持VIDEO_MODE"); } - if (!state_.is_opened) { throw runtime_error("[UVCCamera] (startRecording): 没有打开设备"); } - if (state_.is_recording) { throw runtime_error("[UVCCamera] (startRecording): 已在录像中"); } try { current_video_path_ = video_path; + std::string temp_path = current_video_path_ + ".temp"; // 临时文件 - // 根据codec设置编码器参数 - ffmpeg::EncoderConfig encoder_config; - encoder_config.width = width_; - encoder_config.height = height_; - encoder_config.fps = fps_; - - // 设置编码器类型 + // 1. 确定编码格式对应的AVCodecID（H.264/H.265） + AVCodecID codec_id; if (codec_ == "H264" || codec_ == "h264") { - encoder_config.bitrate = 8000000; // 8Mbps for H264 - encoder_config.codec = "libx264"; - } else if (codec_ == "H265" || codec_ == "hevc" || codec_ == "H265" || codec_ == "HEVC") { - encoder_config.bitrate = 5000000; // 5Mbps for H265 - encoder_config.codec = "libx265"; + codec_id = AV_CODEC_ID_H264; + } else if (codec_ == "H265" || codec_ == "hevc") { + codec_id = AV_CODEC_ID_HEVC; } else { - encoder_config.bitrate = 8000000; // 默认 - encoder_config.codec = "libx265"; + throw runtime_error("不支持的编码格式: " + codec_); } - - // 初始化视频编码器 - video_encoder_ = std::make_unique(); - int ret = video_encoder_->initialize(encoder_config); + const char* format_name = (codec_ == "H265" || codec_ == "h265" || codec_ == "HEVC" || codec_ == "hevc") ? "mp4" : "avi"; + auto ret = avformat_alloc_output_context2(&format_context_, nullptr, format_name, output_path_.c_str()); + // 2. 创建输出格式上下文（封装器核心） if (ret < 0) { - throw runtime_error("Failed to initialize video encoder"); + throw runtime_error("创建输出格式上下文失败"); } - // 初始化视频写入器 - ffmpeg::WriterConfig writer_config; - writer_config.output_file = current_video_path_; - writer_config.width = width_; - writer_config.height = height_; - writer_config.fps = fps_; - writer_config.bitrate = encoder_config.bitrate; - writer_config.time_base = video_encoder_->get_time_base(); - - video_writer_ = std::make_unique(); - ret = video_writer_->initialize(writer_config, video_encoder_->get_codec_context()); - if (ret < 0) { - throw runtime_error("Failed to initialize video writer"); + // 3. 添加视频流并配置参数 + AVStream* stream = avformat_new_stream(format_context_, nullptr); + if (!stream) { + throw runtime_error("创建视频流失败"); } + stream_ = stream; + AVCodecParameters* codecpar = stream->codecpar; + codecpar->codec_type = AVMEDIA_TYPE_VIDEO; + codecpar->codec_id = codec_id; + codecpar->width = width_; + codecpar->height = height_; + stream->time_base = {1, fps_}; // 时间基：1/fps（每帧间隔1个时间单位） - // 设置编码器回调，将编码后的数据包写入文件 - video_encoder_->set_packet_callback([this](AVPacket* packet, int64_t pts, int64_t dts) { - if (video_writer_) { - video_writer_->write_packet(packet, pts, dts); + // 4. 复制编码器的extradata（如H.264的SPS/PPS，确保播放器能解析） + if (rgbEncoder_ && rgbEncoder_->codec_context && rgbEncoder_->codec_context->extradata) { + codecpar->extradata = static_cast(av_mallocz(rgbEncoder_->codec_context->extradata_size + AV_INPUT_BUFFER_PADDING_SIZE)); + if (!codecpar->extradata) { + throw runtime_error("分配extradata内存失败"); } - }); - - // 停止旧的流线程（如果存在） - if (stream_thread_) { - if (stream_thread_->joinable()) { - stream_thread_->join(); - is_streaming_running = false; - } - stream_thread_.reset(); + memcpy(codecpar->extradata, rgbEncoder_->codec_context->extradata, rgbEncoder_->codec_context->extradata_size); + codecpar->extradata_size = rgbEncoder_->codec_context->extradata_size; } - // 开启录像状态 + // 5. 打开输出文件 + if (!(format_context_->oformat->flags & AVFMT_NOFILE)) { + if (avio_open(&format_context_->pb, temp_path.c_str(), AVIO_FLAG_WRITE) < 0) { + throw runtime_error("打开输出文件失败: " + temp_path); + } + } + + // 6. 写入文件头 + if (avformat_write_header(format_context_, nullptr) < 0) { + throw runtime_error("写入文件头失败"); + } + + // 7. 初始化数据包（用于封装编码帧） + packet_ = av_packet_alloc(); + if (!packet_) { + throw runtime_error("分配AVPacket失败"); + } + //不在录像也不在流传输，但是采集线程没有退出时。 + if (!state_.is_streaming && !state_.is_recording) { + if (stream_thread_) { + if (stream_thread_->joinable()) { + stream_thread_->join(); + is_streaming_running = false; + } + stream_thread_.reset(); + } + } + // 开启录像 state_.is_recording = true; - - // 开启流采集线程 + //开启流采集线程 if (!stream_thread_) { stream_thread_ = make_shared(&UVCCamera::streaming_worker_, this); + //延时100ms，等待流线程获取图像 std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - - LOG(INFO) << "[UVCCamera] (startRecording): Recording started to " << current_video_path_ - << " with timebase " << writer_config.time_base.num << "/" << writer_config.time_base.den; + // 启动录像线程 + if (recording_thread_) { + if (recording_thread_->joinable()) { + recording_thread_->join(); + is_recording_running = false; + } + recording_thread_.reset(); + } + frame_count_ = 0; + recording_thread_ = make_shared(&UVCCamera::recording_worker_, this); } catch (const std::exception& e) { // 异常清理 - if (video_writer_) { - video_writer_->finish(); - video_writer_.reset(); + if (packet_) av_packet_free(&packet_); + if (stream_ && stream_->codecpar->extradata) av_free(stream_->codecpar->extradata); + if (format_context_) { + if (!(format_context_->oformat->flags & AVFMT_NOFILE) && format_context_->pb) avio_closep(&format_context_->pb); + avformat_free_context(format_context_); } - video_encoder_.reset(); + stream_ = nullptr; + format_context_ = nullptr; state_.is_recording = false; throw runtime_error("[UVCCamera] (startRecording): " + std::string(e.what())); } @@ -340,34 +387,412 @@ void UVCCamera::startRecording(const std::string &video_path) { void UVCCamera::stopRecording() { std::lock_guard lock(ctrl_mtx_); clear_error_(); - if (mode_ != VIDEO_MODE) { throw runtime_error("[UVCCamera] (stopRecording): 仅支持VIDEO_MODE"); } - if (!state_.is_recording) { throw runtime_error("[UVCCamera] (stopRecording): 未在录像中"); } // 1. 停止录像线程 state_.is_recording = false; - - // 2. 刷新编码器并完成写入 - if (video_encoder_) { - video_encoder_->flush(); + if (recording_thread_ && recording_thread_->joinable()) { + recording_thread_->join(); + recording_thread_.reset(); } - if (video_writer_) { - video_writer_->finish(); - video_writer_.reset(); + // 2. 清理FFmpeg资源 + if (packet_) { + av_packet_free(&packet_); + packet_ = nullptr; } + if (stream_ && stream_->codecpar->extradata) { + av_free(stream_->codecpar->extradata); + stream_->codecpar->extradata = nullptr; + stream_->codecpar->extradata_size = 0; + } + if (format_context_) { + if (!(format_context_->oformat->flags & AVFMT_NOFILE) && format_context_->pb) { + avio_closep(&format_context_->pb); // 关闭文件 + } + avformat_free_context(format_context_); // 释放格式上下文 + format_context_ = nullptr; + } + stream_ = nullptr; - video_encoder_.reset(); - - LOG(INFO) << "[UVCCamera] (stopRecording): Recording saved to " << current_video_path_; + // 3. 重命名临时文件为目标文件 + std::string temp_path = current_video_path_ + ".temp"; + if (rename(temp_path.c_str(), current_video_path_.c_str()) != 0) { + throw runtime_error("重命名临时文件失败: " + temp_path + " -> " + current_video_path_); + } current_video_path_.clear(); } +void UVCCamera::pauseRecording() { + +} + +void UVCCamera::resumeRecording() { + +} + +void UVCCamera::streaming_worker_() { + + try { + // 计算理论上每帧之间的间隔时间(毫秒) + const int frame_interval = 1000 / fps_; + + bool success = false; + is_streaming_running = true; + cv::Mat frame; + int64_t frame_count = 0; + // 处于流传输或者录像状态时就不退出线程 + while (state_.is_streaming || state_.is_recording) { + // 记录当前帧处理开始时间 + auto frame_start_time = std::chrono::high_resolution_clock::now(); + if (!cap_.read(frame) || frame.empty()) { + throw runtime_error("[UVCCamera]: Failed to read frame."); + } + + // 以下为编码部分，用于流模式 + StreamFrameData frame_data; + // 保存图像数据 + { + frame.copyTo(frame_data.rgbImage); // 执行深拷贝 + } + // rgb图像编码 + success = encodeFrameWithEncoder(rgbEncoder_, frame_data.rgbImage, frame_data.rgbFrame, frame_data.bKey); + if (success) { + frame_data.fps = fps_; + frame_data.width = width_; + frame_data.height = height_; + frame_data.codec = codec_; + stream_frame_buffer_->push(frame_data); + } + else + { + std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); + continue; + } + // 计算从帧开始到现在的总耗时 + auto total_duration = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - frame_start_time + ).count(); + + // 计算需要休眠的时间(确保总耗时达到frame_interval) + int sleep_time = frame_interval - total_duration; + // 只有当需要休眠的时间为正数时才休眠 + if (sleep_time > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_time)); + } + } + + is_streaming_running = false; + // 线程结束时清空队列 + stream_frame_buffer_->clear(); + recordingIndex_ = 0; + getImageIndex_ = 0; + streamIndex_ = 0; + } + catch (const std::exception &e) { + // 线程结束时清空队列 + stream_frame_buffer_->clear(); + // 确保线程状态正确更新 + is_streaming_running = false; + state_.is_error = true; + state_.error_message = e.what(); + LOG(ERROR) << "[UVCCamera]streaming_worker_ error:" << state_.error_message; + } +} + +void UVCCamera::recording_worker_() { + is_recording_running = true; + const int frame_interval = 1000 / fps_; + bool is_first_key = false; + try { + //保证当前采集线程正常运行 + while (state_.is_recording && is_streaming_running) { + // 等待缓冲区有数据 + if (stream_frame_buffer_->empty()) { + std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); + continue; + } + + // 1. 从缓冲区取编码好的帧 + auto frame_data_opt = stream_frame_buffer_->pop(recordingIndex_); + + if (!frame_data_opt.has_value()) { + std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); + continue; + } + StreamFrameData frame_data = frame_data_opt.value(); + + // 2. 验证编码数据有效性 + if (frame_data.rgbFrame.empty()) { + std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); + continue; + } + + // 先等待至第一个I帧 + if (!is_first_key) + { + is_first_key = frame_data.bKey; + if (!is_first_key) + { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + } + // 3. 初始化AVPacket（复用编码数据） + av_packet_unref(packet_); + + // 设置数据包数据 + packet_->data = const_cast(frame_data.rgbFrame.data()); // 注意：const_cast安全，因为仅读取 + packet_->size = static_cast(frame_data.rgbFrame.size()); + packet_->stream_index = stream_->index; // 指定流索引 + + // 设置时间戳（递增） + packet_->pts = frame_count_++; + packet_->dts = packet_->pts; + packet_->duration = 1; // 每帧持续1个时间单位（根据time_base） + + // 标记关键帧 + if (frame_data.bKey) { + packet_->flags |= AV_PKT_FLAG_KEY; + } + + // 转换时间基 + av_packet_rescale_ts(packet_, {1, fps_}, stream_->time_base); + + // 写入帧 + if (av_interleaved_write_frame(format_context_, packet_) < 0) { + std::cerr << "写入第" << frame_count_ << "帧失败" << std::endl; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(33)); + } + + // 7. 写入文件尾（完成封装） + av_write_trailer(format_context_); + + } catch (const std::exception& e) { + std::cerr << "录像线程错误: " << e.what() << std::endl; + } + + is_recording_running = false; + state_.is_recording = false; +} + +// 初始化单个编码器的通用函数 +bool UVCCamera::initSingleEncoder(std::shared_ptr& encoder, + const std::string& codec_name, + int width, int height, int fps) { + // 1. 创建编码器实例（不变） + encoder = std::make_shared(); + encoder->codec_name = codec_name; + encoder->width = width; + encoder->height = height; + encoder->fps = fps; + + // 2. 查找编码器（不变） + const AVCodec* codec = nullptr; + if (codec_name == "h264" || codec_name == "H264") { + codec = avcodec_find_encoder_by_name("libx264"); + if (!codec) codec = avcodec_find_encoder(AV_CODEC_ID_H264); + } else if (codec_name == "h265" || codec_name == "HEVC" || codec_name == "H265") { + codec = avcodec_find_encoder_by_name("libx265"); + if (!codec) codec = avcodec_find_encoder(AV_CODEC_ID_HEVC); + } + if (!codec) { + std::cerr << "Failed to find " << codec_name << " encoder!" << std::endl; + return false; + } + + // 3. 初始化编码器上下文（不变） + encoder->codec_context = avcodec_alloc_context3(codec); + if (!encoder->codec_context) { + std::cerr << "Failed to allocate codec context!" << std::endl; + return false; + } + + // 4. 设置编码器参数（不变） + AVCodecContext* ctx = encoder->codec_context; + ctx->codec_type = AVMEDIA_TYPE_VIDEO; + ctx->width = (width + 1) & ~1; // 确保宽度为偶数 + ctx->height = (height + 1) & ~1;// 确保高度为偶数 + ctx->time_base = {1, fps}; // 时间基（1/fps） + ctx->framerate = {fps, 1}; // 帧率 + ctx->max_b_frames = 0; // 禁用B帧（降低延迟） + ctx->gop_size = 10;//I帧间隔1，每一帧都是I帧 + // 5. 设置编码器私有参数 + if (codec->id == AV_CODEC_ID_H264) { + av_opt_set(ctx->priv_data, "preset", "ultrafast", 0); + av_opt_set(ctx->priv_data, "tune", "zerolatency", 0); + av_opt_set(ctx->priv_data, "profile", "baseline", 0); + av_opt_set(ctx->priv_data, "repeat-headers", "1", 0); + av_opt_set(ctx->priv_data, "annexb", "1", 0); + } else if (codec->id == AV_CODEC_ID_HEVC) { + // 直接使用默认参数，不自定义 + // av_opt_set(ctx->priv_data, "preset", "ultrafast", 0); + // av_opt_set(ctx->priv_data, "tune", "zerolatency", 0); + } + + // 6. 设置像素格式（不变） + const enum AVPixelFormat* pix_fmts = codec->pix_fmts; + if (!pix_fmts) { + std::cout << "Using default pixel format: YUV420P" << std::endl; + ctx->pix_fmt = AV_PIX_FMT_YUV420P; + } else { + ctx->pix_fmt = pix_fmts[0]; + std::cout << "Selected pixel format: " << av_get_pix_fmt_name(ctx->pix_fmt) << std::endl; + } + + // 7. 打开编码器（不变） + int ret = avcodec_open2(ctx, codec, nullptr); + if (ret < 0) { + char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0}; + av_strerror(ret, errbuf, sizeof(errbuf)); + std::cerr << "Failed to open " << codec_name << " encoder: " << errbuf << std::endl; + return false; + } + + // 8. 分配AVFrame（不变） + encoder->frame = av_frame_alloc(); + if (!encoder->frame) { + std::cerr << "Failed to allocate AVFrame!" << std::endl; + return false; + } + encoder->frame->format = ctx->pix_fmt; + encoder->frame->width = ctx->width; + encoder->frame->height = ctx->height; + if (av_frame_get_buffer(encoder->frame, 0) < 0) { + std::cerr << "Failed to allocate AVFrame buffer!" << std::endl; + av_frame_free(&encoder->frame); + return false; + } + + // 9. 分配AVPacket（不变） + encoder->packet = av_packet_alloc(); + if (!encoder->packet) { + std::cerr << "Failed to allocate AVPacket!" << std::endl; + return false; + } + + std::cout << "Successfully initialized " << codec_name << " encoder ( " + << width << "x" << height << "@" << fps << "fps )" << std::endl; + return true; +} + +bool UVCCamera::encodeFrameWithEncoder(std::shared_ptr& encoder, const cv::Mat& frame,std::vector& encoded_frame,bool& is_key) { + + if (!encoder || !encoder->codec_context || !encoder->frame || !encoder->packet) { + return false; + } + + // 确保输入帧尺寸匹配（不变） + if (frame.cols != encoder->width || frame.rows != encoder->height) { + std::cerr << "Frame size does not match encoder dimensions" << std::endl; + return false; + } + + // 【修复3：设置递增的PTS，确保编码器正确处理I帧请求】 + encoder->frame->pts = encoder->frame_pts++; // 分配唯一PTS + + // 根据cv::Mat的类型设置源格式（不变） + AVPixelFormat src_pix_fmt; + if (frame.channels() == 3) { + src_pix_fmt = AV_PIX_FMT_BGR24; // OpenCV默认BGR + } else if (frame.channels() == 4) { + src_pix_fmt = AV_PIX_FMT_BGRA; // 4通道为BGRA + } else if (frame.channels() == 1) { + src_pix_fmt = AV_PIX_FMT_GRAY8; // 单通道灰度图 + } else { + std::cerr << "Unsupported number of channels: " << frame.channels() << std::endl; + return false; + } + // 【修复4：动态创建sws_context（匹配当前输入格式）】 + if (encoder->sws_context) { + sws_freeContext(encoder->sws_context); // 释放旧上下文 + } + encoder->sws_context = sws_getContext( + frame.cols, frame.rows, src_pix_fmt, // 输入格式由当前frame决定 + encoder->codec_context->width, encoder->codec_context->height, encoder->codec_context->pix_fmt, + SWS_BILINEAR, nullptr, nullptr, nullptr + ); + if (!encoder->sws_context) { + std::cerr << "Failed to create SwsContext" << std::endl; + return false; + } + + // 转换输入帧格式为编码器所需格式（不变） + const uint8_t* src_data[AV_NUM_DATA_POINTERS] = {frame.data}; + int src_linesize[AV_NUM_DATA_POINTERS] = {static_cast(frame.step)}; + + int ret = sws_scale(encoder->sws_context, src_data, src_linesize, 0, frame.rows, + encoder->frame->data, encoder->frame->linesize); + if (ret < 0) { + std::cerr << "Error scaling frame" << std::endl; + return false; + } + // 发送帧到编码器（不变） + ret = avcodec_send_frame(encoder->codec_context, encoder->frame); + if (ret < 0) { + char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0}; + av_strerror(ret, errbuf, sizeof(errbuf)); + std::cerr << "Error sending frame to encoder: " << errbuf << std::endl; + return false; + } + while (true) + { + // 接收编码后的数据（不变） + ret = avcodec_receive_packet(encoder->codec_context, encoder->packet); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + break; + } else if (ret < 0) { + char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0}; + av_strerror(ret, errbuf, sizeof(errbuf)); + std::cerr << "Error receiving packet from encoder: " << errbuf << std::endl; + break; + } + + // 调试：打印帧类型（I帧/P帧） + if (encoder->packet->flags & AV_PKT_FLAG_KEY) { + is_key = true; + //std::cout << "Encoded I frame (size: " << encoder->packet->size << " bytes)" << std::endl; + } else { + is_key = false; + //std::cout << "Encoded P frame (size: " << encoder->packet->size << " bytes)" << std::endl; + } + + // 预留足够空间，避免多次内存分配 + encoded_frame.reserve(encoded_frame.size() + encoder->packet->size); + // 复制数据包内容到输出向量 + encoded_frame.insert(encoded_frame.end(), + encoder->packet->data, + encoder->packet->data + encoder->packet->size); + av_packet_unref(encoder->packet); + } + + + // 验证帧有效性（NALU起始码、元数据） + if (!encoded_frame.empty()) { + // 检查NALU起始码 + bool has_start_code = false; + if ((encoded_frame[0] == 0 && encoded_frame[1] == 0 && encoded_frame[2] == 1) || + (encoded_frame[0] == 0 && encoded_frame[1] == 0 && encoded_frame[2] == 0 && encoded_frame[3] == 1)) { + has_start_code = true; + } + if (!has_start_code) { + std::cerr << "Invalid frame: no NALU start code!" << std::endl; + return false; + } + } else { + //std::cerr << "Encoded frame is empty!" << std::endl; + return false; + } + return true; +} + void UVCCamera::getEncodedFrame(StreamFrameData& frame_data, size_t& index) { // 环形队列取数据的index由接口传入 auto frame = stream_frame_buffer_->pop(index); @@ -376,20 +801,10 @@ void UVCCamera::getEncodedFrame(StreamFrameData& frame_data, size_t& index) { } } -bool UVCCamera::startStreaming() { +bool UVCCamera::startStreaming() +{ std::lock_guard lock(ctrl_mtx_); - - if (mode_ != VIDEO_MODE) { - LOG(ERROR) << "[UVCCamera] (startStreaming): 仅支持VIDEO_MODE"; - return false; - } - - if (!state_.is_opened) { - LOG(ERROR) << "[UVCCamera] (startStreaming): 没有打开设备"; - return false; - } - - // 如果已经停止，重启流线程 + //不在录像也不在流传输，但是采集线程没有退出时。 if (!state_.is_streaming && !state_.is_recording) { if (stream_thread_) { if (stream_thread_->joinable()) { @@ -399,135 +814,25 @@ bool UVCCamera::startStreaming() { stream_thread_.reset(); } } - - // 开启流采集线程 + //开启流采集线程 if (!stream_thread_) { state_.is_streaming = true; stream_thread_ = make_shared(&UVCCamera::streaming_worker_, this); + + //延时100ms，等待流线程获取图像 std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - stream_count_++; - LOG(INFO) << "[UVCCamera] (startStreaming): Streaming started, count: " << stream_count_; return true; } -void UVCCamera::stopStreaming() { +void UVCCamera::stopStreaming() +{ std::lock_guard lock(ctrl_mtx_); - - if (stream_count_ > 0) { - stream_count_--; - } - - if (stream_count_ == 0) { - // 当前已经没有正在使用的流了，停止流线程 + stream_count_--; + if (stream_count_ == 0) + { + // 当前已经没有正在使用的流了，编码采集线程状态修改 state_.is_streaming = false; - LOG(INFO) << "[UVCCamera] (stopStreaming): Streaming stopped"; - } else { - LOG(INFO) << "[UVCCamera] (stopStreaming): Streaming count reduced to " << stream_count_; } } - -// AVFrame转换为cv::Mat的辅助函数 -bool UVCCamera::convertAVFrameToMat(AVFrame* frame, cv::Mat& mat) { - if (!frame || !frame->data[0]) { - return false; - } - - // 根据不同的像素格式进行转换 - switch (frame->format) { - case AV_PIX_FMT_YUV420P: - { - cv::Mat yuv(frame->height + frame->height/2, frame->width, CV_8UC1, frame->data[0]); - cv::cvtColor(yuv, mat, cv::COLOR_YUV2RGB_I420); - return true; - } - case AV_PIX_FMT_YUYV422: - { - cv::Mat yuyv(frame->height, frame->width, CV_8UC2, frame->data[0]); - cv::cvtColor(yuyv, mat, cv::COLOR_YUV2RGB_YUYV); - return true; - } - case AV_PIX_FMT_BGR24: - { - mat = cv::Mat(frame->height, frame->width, CV_8UC3, frame->data[0]); - return true; - } - case AV_PIX_FMT_RGB24: - { - mat = cv::Mat(frame->height, frame->width, CV_8UC3, frame->data[0]); - cv::cvtColor(mat, mat, cv::COLOR_RGB2BGR); - return true; - } - default: - LOG(WARNING) << "[UVCCamera] Unsupported pixel format: " << frame->format; - return false; - } -} - -void UVCCamera::streaming_worker_() { - try { - const int frame_interval = 1000 / fps_; - is_streaming_running = true; - - LOG(INFO) << "[UVCCamera] (streaming_worker_): Streaming worker started"; - - // 开始摄像头采集 - camera_capture_->start_capture([this](AVFrame* frame) { - if (!frame) { - return; - } - - StreamFrameData frame_data; - - // 更新最新帧缓存（用于getRGBImage） - { - std::lock_guard frame_lock(latest_frame_mutex_); - if (convertAVFrameToMat(frame, latest_rgb_frame_)) { - latest_frame_valid_ = true; - - // 保存到frame_data - latest_rgb_frame_.copyTo(frame_data.rgbImage); - } - } - - // 编码帧（如果是录制状态） - if (state_.is_recording && video_encoder_) { - int ret = video_encoder_->encode_frame(frame); - if (ret < 0) { - LOG(WARNING) << "[UVCCamera] Failed to encode frame"; - } - } - - // 设置帧数据信息 - frame_data.fps = fps_; - frame_data.width = width_; - frame_data.height = height_; - frame_data.codec = codec_; - frame_data.bKey = false; // 由编码器回调设置 - - // 推送到缓冲区 - stream_frame_buffer_->push(frame_data); - }); - - // 等待直到停止 - while (state_.is_streaming || state_.is_recording) { - std::this_thread::sleep_for(std::chrono::milliseconds(frame_interval)); - } - - is_streaming_running = false; - stream_frame_buffer_->clear(); - recordingIndex_ = 0; - getImageIndex_ = 0; - streamIndex_ = 0; - - LOG(INFO) << "[UVCCamera] (streaming_worker_): Streaming worker stopped"; - - } catch (const std::exception &e) { - stream_frame_buffer_->clear(); - is_streaming_running = false; - state_.is_error = true; - state_.error_message = e.what(); - LOG(ERROR) << "[UVCCamera] streaming_worker_ error:" << state_.error_message; - } -} \ No newline at end of file