diff --git a/cmvr-es/config/devices/microphone/microphone.pb.txt b/cmvr-es/config/devices/microphone/microphone.pb.txt index 03c0ea1b..9ceba156 100644 --- a/cmvr-es/config/devices/microphone/microphone.pb.txt +++ b/cmvr-es/config/devices/microphone/microphone.pb.txt @@ -3,7 +3,7 @@ microphone { id: "mic1" ffmpeg { channels: 2 - sampleRate: 44100 + sampleRate: 48000 volume: 100 input_device: "default" } diff --git a/cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_ptr.h b/cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_ptr.h index da147bf7..c9aa251f 100644 --- a/cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_ptr.h +++ b/cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_ptr.h @@ -114,291 +114,6 @@ enum class ImageType { DEPTH, // 深度图像 GRAYSCALE // 灰度图像 }; - -class HEVCEncoder { -public: - HEVCEncoder(int width, int height, int fps, int bitrate, - const std::string& deviceName = "/dev/dri/renderD128", - ImageType imageType = ImageType::COLOR) - : width_(width), height_(height), fps_(fps), bitrate_(bitrate), - deviceName_(deviceName), imageType_(imageType) { - initialized_ = init(); - } - - bool encodeFrame(const cv::Mat& image) { - if (!initialized_ || !codec_ctx_ || !frame_) { - CMVR_LOG(ERROR) << "HEVC encoder is not initialized"; - return false; - } - if (!prepareFrame(image)) { - CMVR_LOG(ERROR) << "Failed to prepare frame"; - return false; - } - - if (avcodec_send_frame(codec_ctx_.get(), frame_.get()) < 0) { - CMVR_LOG(ERROR) << "Error sending frame to encoder"; - return false; - } - - return true; - } - - std::vector receivePacket(bool& is_key) { - std::vector packetData; - if (!initialized_ || !codec_ctx_) { - CMVR_LOG(ERROR) << "HEVC encoder is not initialized"; - return packetData; - } - FFmpegPtr packet(av_packet_alloc()); - - while (true) - { - int ret = avcodec_receive_packet(codec_ctx_.get(), packet.get()); - if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { - break; - } else if (ret < 0) { - CMVR_LOG(ERROR) << "Error during encoding"; - break; - } - is_key = (packet->flags & AV_PKT_FLAG_KEY) != 0; - // 预留足够空间,避免多次内存分配 - packetData.reserve(packetData.size() + packet->size); - // 复制数据包内容到输出向量 - packetData.insert(packetData.end(), - packet->data, - packet->data + packet->size); - av_packet_unref(packet.get()); - } - return packetData; - } - -private: - bool init() { - // 1. 创建硬件设备上下文 - AVBufferRef* hw_device_ctx = nullptr; - int ret = av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, - deviceName_.c_str(), nullptr, 0); - if (ret < 0) { - char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0}; - av_strerror(ret, errbuf, sizeof(errbuf)); - CMVR_LOG(ERROR) << "Failed to create VAAPI device context: " << errbuf; - return false; - } - hw_device_ctx_.reset(hw_device_ctx); - - // 2. 查找编码器 - const AVCodec* codec = avcodec_find_encoder_by_name("hevc_vaapi"); - if (!codec) { - CMVR_LOG(ERROR) << "HEVC VAAPI encoder not found"; - return false; - } - - // 3. 设置编码器上下文 - codec_ctx_.reset(avcodec_alloc_context3(codec)); - codec_ctx_->width = width_; - codec_ctx_->height = height_; - codec_ctx_->time_base = {1, fps_}; - codec_ctx_->pix_fmt = AV_PIX_FMT_VAAPI; - codec_ctx_->bit_rate = bitrate_; - codec_ctx_->gop_size = 1; - codec_ctx_->hw_device_ctx = av_buffer_ref(hw_device_ctx_.get()); - - // 4. 创建硬件帧上下文 - hw_frames_ctx_.reset(av_hwframe_ctx_alloc(hw_device_ctx_.get())); - if (!hw_frames_ctx_) { - CMVR_LOG(ERROR) << "Failed to allocate hardware frame context"; - return false; - } - - AVHWFramesContext* frames_ctx = (AVHWFramesContext*)hw_frames_ctx_->data; - frames_ctx->format = AV_PIX_FMT_VAAPI; - frames_ctx->sw_format = AV_PIX_FMT_NV12; - frames_ctx->width = width_; - frames_ctx->height = height_; - frames_ctx->initial_pool_size = 20; - - if (av_hwframe_ctx_init(hw_frames_ctx_.get()) < 0) { - CMVR_LOG(ERROR) << "Failed to initialize hardware frame context"; - return false; - } - - codec_ctx_->hw_frames_ctx = av_buffer_ref(hw_frames_ctx_.get()); - - // 5. 打开编码器 - if (avcodec_open2(codec_ctx_.get(), codec, nullptr) < 0) { - CMVR_LOG(ERROR) << "Cannot open encoder"; - return false; - } - - // 6. 初始化硬件帧 - frame_.reset(av_frame_alloc()); - frame_->format = AV_PIX_FMT_VAAPI; - frame_->width = width_; - frame_->height = height_; - - if (av_hwframe_get_buffer(hw_frames_ctx_.get(), frame_.get(), 0) < 0) { - CMVR_LOG(ERROR) << "Could not allocate hardware frame data"; - return false; - } - - // 7. 初始化软件帧 - sw_frame_.reset(av_frame_alloc()); - sw_frame_->width = width_; - sw_frame_->height = height_; - sw_frame_->format = (imageType_ == ImageType::DEPTH) ? AV_PIX_FMT_GRAY8 : AV_PIX_FMT_NV12; - - if (av_frame_get_buffer(sw_frame_.get(), 0) < 0) { - CMVR_LOG(ERROR) << "Could not allocate software frame buffer"; - return false; - } - return true; - } - - bool prepareFrame(const cv::Mat& image) { - if (imageType_ == ImageType::DEPTH) { - return prepareDepthFrame(image); - } else { - return prepareColorOrGrayscaleFrame(image); - } - } - - bool prepareDepthFrame(const cv::Mat& depthImage) { - cv::Mat processedDepth; - - if (depthImage.channels() != 1) { - CMVR_LOG(ERROR) << "Depth image must be single-channel"; - return false; - } - - if (depthImage.depth() != CV_8U) { - if (depthImage.depth() == CV_16U) { - depthImage.convertTo(processedDepth, CV_8U, 255.0 / 65535.0); - } else { - CMVR_LOG(ERROR) << "Unsupported depth image format"; - return false; - } - } else { - processedDepth = depthImage.clone(); - } - - // 确保图像尺寸匹配 - if (processedDepth.cols != width_ || processedDepth.rows != height_) { - CMVR_LOG(ERROR) << "Depth image size mismatch"; - return false; - } - - // 设置软件帧属性 - sw_frame_->pts = frame_counter_++; - sw_frame_->format = AV_PIX_FMT_GRAY8; - av_frame_make_writable(sw_frame_.get()); - - // 使用av_image_fill_arrays处理行对齐 - av_image_fill_arrays(sw_frame_->data, sw_frame_->linesize, - processedDepth.data, - AV_PIX_FMT_GRAY8, - width_, height_, 1); - - // 上传到硬件帧 - if (av_hwframe_transfer_data(frame_.get(), sw_frame_.get(), 0) < 0) { - CMVR_LOG(ERROR) << "Error transferring depth data to hardware frame"; - return false; - } - - frame_->pts = sw_frame_->pts; - return true; - } - - bool prepareColorOrGrayscaleFrame(const cv::Mat& image) { - if (image.cols != width_ || image.rows != height_) { - CMVR_LOG(ERROR) << "Image size mismatch: " << image.cols << "x" << image.rows - << " vs " << width_ << "x" << height_; - return false; - } - - // 1. 确定输入图像的像素格式(OpenCV的Mat格式) - AVPixelFormat src_pix_fmt; - if (image.channels() == 3) { - src_pix_fmt = AV_PIX_FMT_BGR24; // OpenCV默认是BGR格式(3通道) - } else if (image.channels() == 1) { - src_pix_fmt = AV_PIX_FMT_GRAY8; // 灰度图(1通道) - } else { - CMVR_LOG(ERROR) << "Unsupported channel count: " << image.channels(); - return false; - } - - // 2. 初始化格式转换上下文(swscale) - SwsContext* sws_ctx = sws_getContext( - width_, height_, src_pix_fmt, // 源宽高和格式 - width_, height_, AV_PIX_FMT_NV12, // 目标宽高和格式(NV12) - SWS_BILINEAR, // 缩放算法(可根据需求调整) - nullptr, nullptr, nullptr - ); - if (!sws_ctx) { - CMVR_LOG(ERROR) << "Failed to create sws context"; - return false; - } - - // 3. 配置源数据(OpenCV的Mat) - uint8_t* src_data[4] = {nullptr}; - int src_linesize[4] = {0}; - if (image.channels() == 3) { - src_data[0] = image.data; // BGR数据起始地址 - src_linesize[0] = image.step; // 每行字节数(含OpenCV的对齐填充) - } else { - src_data[0] = image.data; // 灰度数据起始地址 - src_linesize[0] = image.step; // 灰度图每行字节数 - } - - // 4. 配置目标数据(NV12格式的AVFrame) - av_frame_make_writable(sw_frame_.get()); - sw_frame_->format = AV_PIX_FMT_NV12; - sw_frame_->width = width_; - sw_frame_->height = height_; - - // 5. 执行格式转换(BGR/GRAY -> NV12) - int ret = sws_scale( - sws_ctx, - src_data, // 源图像数据 - src_linesize, // 源图像每行字节数 - 0, // 起始行 - height_, // 转换的行数 - sw_frame_->data, // 目标图像数据(sw_frame_的data指针) - sw_frame_->linesize // 目标图像每行字节数 - ); - if (ret <= 0) { - CMVR_LOG(ERROR) << "sws_scale failed, ret=" << ret; - sws_freeContext(sws_ctx); - return false; - } - - // 6. 释放转换上下文 - sws_freeContext(sws_ctx); - - // 7. 设置帧属性并上传到硬件 - sw_frame_->pts = frame_counter_++; - if (av_hwframe_transfer_data(frame_.get(), sw_frame_.get(), 0) < 0) { - CMVR_LOG(ERROR) << "Error transferring data to hardware frame"; - return false; - } - frame_->pts = sw_frame_->pts; - - return true; - } - int width_; - int height_; - int fps_; - int bitrate_; - int frame_counter_ = 0; - std::string deviceName_; - ImageType imageType_; - bool initialized_{false}; - - FFmpegPtr codec_ctx_; - FFmpegPtr frame_; - FFmpegPtr sw_frame_; - FFmpegPtr hw_device_ctx_; - FFmpegPtr hw_frames_ctx_; -}; } // namespace ffmpeg #endif //FFMPEG_PTR_H diff --git a/cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_speaker.h b/cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_speaker.h index 9d8e56ed..124213b3 100644 --- a/cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_speaker.h +++ b/cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_speaker.h @@ -3,10 +3,14 @@ #ifndef CMVR_ES_FFMPEG_SPEAKER_H #define CMVR_ES_FFMPEG_SPEAKER_H -#include -#include +#include +#include #include +#include +#include #include +#include +#include #include "speaker/abstract_speaker.h" #include #include @@ -48,6 +52,7 @@ namespace cmvr::device { bool decodeStreamFrame_(const AudioStreamFrameData& frame_data); bool initStreamDecoder_(const AudioStreamFrameData& frame_data); void releaseStreamDecoder_(); + void releaseStreamDecoderUnlocked_(); pa_simple* pulse_simple_ = nullptr; // 修改:PulseAudio 简单 API 句柄 pa_sample_spec sample_spec_{}; // 新增:PulseAudio 采样规格 @@ -59,6 +64,7 @@ namespace cmvr::device { std::shared_ptr decode_thread_; std::shared_ptr play_thread_; std::mutex mtx_; + std::mutex stop_mtx_; std::string audio_path_; // Boost 单生产者单消费者无锁队列 @@ -68,6 +74,7 @@ namespace cmvr::device { mutable std::mutex mtx_pause_; std::condition_variable cv_pause_; std::atomic is_paused_{false}; + std::atomic is_stopping_{false}; // 音频时钟同步 std::atomic audio_pts_{0}; diff --git a/cmvr-es/devices/speaker/ffmpeg_speaker/src/ffmpeg_speaker.cpp b/cmvr-es/devices/speaker/ffmpeg_speaker/src/ffmpeg_speaker.cpp index ee19882c..125d9e04 100644 --- a/cmvr-es/devices/speaker/ffmpeg_speaker/src/ffmpeg_speaker.cpp +++ b/cmvr-es/devices/speaker/ffmpeg_speaker/src/ffmpeg_speaker.cpp @@ -31,6 +31,7 @@ ffmpegSpeaker::ffmpegSpeaker(const config::FFMpegSpeakerConfig& cfg):config_(cfg ffmpegSpeaker::~ffmpegSpeaker() { + is_stopping_ = true; { std::lock_guard lock(mtx_); state_.is_running = false; @@ -101,13 +102,17 @@ void ffmpegSpeaker::resetPlayState() } bool ffmpegSpeaker::stop() { + std::lock_guard stop_lock(stop_mtx_); + is_stopping_ = true; { lock_guard lock(mtx_); state_.is_running = false; state_.is_decoding = false; state_.is_paused = false; + is_streaming_input_ = false; } + cv_pause_.notify_all(); // 等待线程结束 if (decode_thread_ && decode_thread_->joinable()) { decode_thread_->join(); @@ -120,6 +125,7 @@ bool ffmpegSpeaker::stop() { } resetPlayState(); + is_stopping_ = false; return true; } @@ -540,6 +546,10 @@ void ffmpegSpeaker::play_audio_() { bool ffmpegSpeaker::pushAudioFrame(const AudioStreamFrameData& frame_data) { + if (is_stopping_) { + return false; + } + if (frame_data.data.empty()) { return true; } @@ -577,9 +587,12 @@ bool ffmpegSpeaker::pushAudioFrame(const AudioStreamFrameData& frame_data) void ffmpegSpeaker::stopStreaming() { + std::lock_guard stop_lock(stop_mtx_); + is_stopping_ = true; { std::lock_guard lock(mtx_); if (!is_streaming_input_) { + is_stopping_ = false; return; } state_.is_decoding = false; @@ -595,6 +608,7 @@ void ffmpegSpeaker::stopStreaming() } resetPlayState(); + is_stopping_ = false; } bool ffmpegSpeaker::startStreamingPlayback_(const AudioStreamFrameData& frame_data) @@ -653,12 +667,17 @@ bool ffmpegSpeaker::pushPcmFrame_(const AudioStreamFrameData& frame_data) sample = static_cast(std::clamp(scaled, -32768.f, 32767.f)); } - while (!audio_queue_.push(buffer)) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - std::lock_guard lock(mtx_); - if (!state_.is_running) { - return false; + while (true) { + { + std::lock_guard lock(mtx_); + if (!state_.is_running || is_stopping_) { + return false; + } } + if (audio_queue_.push(buffer)) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); } return true; } @@ -670,7 +689,7 @@ bool ffmpegSpeaker::decodeStreamFrame_(const AudioStreamFrameData& frame_data) stream_codec_ != frame_data.codec || stream_input_sample_rate_ != frame_data.sample_rate || stream_input_channels_ != frame_data.channels) { - releaseStreamDecoder_(); + releaseStreamDecoderUnlocked_(); if (!initStreamDecoder_(frame_data)) { return false; } @@ -738,12 +757,17 @@ bool ffmpegSpeaker::decodeStreamFrame_(const AudioStreamFrameData& frame_data) sample = static_cast(std::clamp(scaled, -32768.f, 32767.f)); } - while (!audio_queue_.push(buffer)) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - std::lock_guard lock(mtx_); - if (!state_.is_running) { - return false; + while (true) { + { + std::lock_guard lock(mtx_); + if (!state_.is_running || is_stopping_) { + return false; + } } + if (audio_queue_.push(buffer)) { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); } } @@ -782,14 +806,14 @@ bool ffmpegSpeaker::initStreamDecoder_(const AudioStreamFrameData& frame_data) stream_decoder_ctx_->channels == 1 ? AV_CH_LAYOUT_MONO : AV_CH_LAYOUT_STEREO; if (avcodec_open2(stream_decoder_ctx_, decoder, nullptr) < 0) { - releaseStreamDecoder_(); + releaseStreamDecoderUnlocked_(); return false; } stream_decode_frame_ = av_frame_alloc(); stream_decode_packet_ = av_packet_alloc(); if (!stream_decode_frame_ || !stream_decode_packet_) { - releaseStreamDecoder_(); + releaseStreamDecoderUnlocked_(); return false; } @@ -800,6 +824,12 @@ bool ffmpegSpeaker::initStreamDecoder_(const AudioStreamFrameData& frame_data) } void ffmpegSpeaker::releaseStreamDecoder_() +{ + std::lock_guard decode_lock(stream_decode_mtx_); + releaseStreamDecoderUnlocked_(); +} + +void ffmpegSpeaker::releaseStreamDecoderUnlocked_() { if (stream_swr_ctx_) { swr_free(&stream_swr_ctx_);