update speaker

cmvr-es/config/devices/microphone/microphone.pb.txt

@@ -3,7 +3,7 @@ microphone {
     id: "mic1"
     ffmpeg {
       channels: 2
-      sampleRate: 44100
+      sampleRate: 48000
       volume: 100
       input_device: "default"
     }

cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_ptr.h

@@ -114,291 +114,6 @@ enum class ImageType {
     DEPTH,      // 深度图像
     GRAYSCALE   // 灰度图像
 };
-
-class HEVCEncoder {
-public:
-    HEVCEncoder(int width, int height, int fps, int bitrate,
-                const std::string& deviceName = "/dev/dri/renderD128",
-                ImageType imageType = ImageType::COLOR)
-        : width_(width), height_(height), fps_(fps), bitrate_(bitrate),
-          deviceName_(deviceName), imageType_(imageType) {
-        initialized_ = init();
-    }
-
-    bool encodeFrame(const cv::Mat& image) {
-        if (!initialized_ || !codec_ctx_ || !frame_) {
-            CMVR_LOG(ERROR) << "HEVC encoder is not initialized";
-            return false;
-        }
-        if (!prepareFrame(image)) {
-            CMVR_LOG(ERROR) << "Failed to prepare frame";
-            return false;
-        }
-
-        if (avcodec_send_frame(codec_ctx_.get(), frame_.get()) < 0) {
-            CMVR_LOG(ERROR) << "Error sending frame to encoder";
-            return false;
-        }
-
-        return true;
-    }
-
-    std::vector<uint8_t> receivePacket(bool& is_key) {
-        std::vector<uint8_t> packetData;
-        if (!initialized_ || !codec_ctx_) {
-            CMVR_LOG(ERROR) << "HEVC encoder is not initialized";
-            return packetData;
-        }
-        FFmpegPtr<AVPacket> packet(av_packet_alloc());
-
-        while (true)
-        {
-            int ret = avcodec_receive_packet(codec_ctx_.get(), packet.get());
-            if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
-                break;
-            } else if (ret < 0) {
-                CMVR_LOG(ERROR) << "Error during encoding";
-                break;
-            }
-            is_key = (packet->flags & AV_PKT_FLAG_KEY) != 0;
-            // 预留足够空间，避免多次内存分配
-            packetData.reserve(packetData.size() + packet->size);
-            // 复制数据包内容到输出向量
-            packetData.insert(packetData.end(),
-                                 packet->data,
-                                 packet->data + packet->size);
-            av_packet_unref(packet.get());
-        }
-        return packetData;
-    }
-
-private:
-    bool init() {
-        // 1. 创建硬件设备上下文
-        AVBufferRef* hw_device_ctx = nullptr;
-        int ret = av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI,
-                                  deviceName_.c_str(), nullptr, 0);
-        if (ret < 0) {
-            char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0};
-            av_strerror(ret, errbuf, sizeof(errbuf));
-            CMVR_LOG(ERROR) << "Failed to create VAAPI device context: " << errbuf;
-            return false;
-        }
-        hw_device_ctx_.reset(hw_device_ctx);
-
-        // 2. 查找编码器
-        const AVCodec* codec = avcodec_find_encoder_by_name("hevc_vaapi");
-        if (!codec) {
-            CMVR_LOG(ERROR) << "HEVC VAAPI encoder not found";
-            return false;
-        }
-
-        // 3. 设置编码器上下文
-        codec_ctx_.reset(avcodec_alloc_context3(codec));
-        codec_ctx_->width = width_;
-        codec_ctx_->height = height_;
-        codec_ctx_->time_base = {1, fps_};
-        codec_ctx_->pix_fmt = AV_PIX_FMT_VAAPI;
-        codec_ctx_->bit_rate = bitrate_;
-        codec_ctx_->gop_size = 1;
-        codec_ctx_->hw_device_ctx = av_buffer_ref(hw_device_ctx_.get());
-
-        // 4. 创建硬件帧上下文
-        hw_frames_ctx_.reset(av_hwframe_ctx_alloc(hw_device_ctx_.get()));
-        if (!hw_frames_ctx_) {
-            CMVR_LOG(ERROR) << "Failed to allocate hardware frame context";
-            return false;
-        }
-
-        AVHWFramesContext* frames_ctx = (AVHWFramesContext*)hw_frames_ctx_->data;
-        frames_ctx->format = AV_PIX_FMT_VAAPI;
-        frames_ctx->sw_format = AV_PIX_FMT_NV12;
-        frames_ctx->width = width_;
-        frames_ctx->height = height_;
-        frames_ctx->initial_pool_size = 20;
-
-        if (av_hwframe_ctx_init(hw_frames_ctx_.get()) < 0) {
-            CMVR_LOG(ERROR) << "Failed to initialize hardware frame context";
-            return false;
-        }
-
-        codec_ctx_->hw_frames_ctx = av_buffer_ref(hw_frames_ctx_.get());
-
-        // 5. 打开编码器
-        if (avcodec_open2(codec_ctx_.get(), codec, nullptr) < 0) {
-            CMVR_LOG(ERROR) << "Cannot open encoder";
-            return false;
-        }
-
-        // 6. 初始化硬件帧
-        frame_.reset(av_frame_alloc());
-        frame_->format = AV_PIX_FMT_VAAPI;
-        frame_->width = width_;
-        frame_->height = height_;
-
-        if (av_hwframe_get_buffer(hw_frames_ctx_.get(), frame_.get(), 0) < 0) {
-            CMVR_LOG(ERROR) << "Could not allocate hardware frame data";
-            return false;
-        }
-
-        // 7. 初始化软件帧
-        sw_frame_.reset(av_frame_alloc());
-        sw_frame_->width = width_;
-        sw_frame_->height = height_;
-        sw_frame_->format = (imageType_ == ImageType::DEPTH) ? AV_PIX_FMT_GRAY8 : AV_PIX_FMT_NV12;
-
-        if (av_frame_get_buffer(sw_frame_.get(), 0) < 0) {
-            CMVR_LOG(ERROR) << "Could not allocate software frame buffer";
-            return false;
-        }
-        return true;
-    }
-
-    bool prepareFrame(const cv::Mat& image) {
-        if (imageType_ == ImageType::DEPTH) {
-            return prepareDepthFrame(image);
-        } else {
-            return prepareColorOrGrayscaleFrame(image);
-        }
-    }
-
-    bool prepareDepthFrame(const cv::Mat& depthImage) {
-        cv::Mat processedDepth;
-
-        if (depthImage.channels() != 1) {
-            CMVR_LOG(ERROR) << "Depth image must be single-channel";
-            return false;
-        }
-
-        if (depthImage.depth() != CV_8U) {
-            if (depthImage.depth() == CV_16U) {
-                depthImage.convertTo(processedDepth, CV_8U, 255.0 / 65535.0);
-            } else {
-                CMVR_LOG(ERROR) << "Unsupported depth image format";
-                return false;
-            }
-        } else {
-            processedDepth = depthImage.clone();
-        }
-
-        // 确保图像尺寸匹配
-        if (processedDepth.cols != width_ || processedDepth.rows != height_) {
-            CMVR_LOG(ERROR) << "Depth image size mismatch";
-            return false;
-        }
-
-        // 设置软件帧属性
-        sw_frame_->pts = frame_counter_++;
-        sw_frame_->format = AV_PIX_FMT_GRAY8;
-        av_frame_make_writable(sw_frame_.get());
-
-        // 使用av_image_fill_arrays处理行对齐
-        av_image_fill_arrays(sw_frame_->data, sw_frame_->linesize,
-                            processedDepth.data,
-                            AV_PIX_FMT_GRAY8,
-                            width_, height_, 1);
-
-        // 上传到硬件帧
-        if (av_hwframe_transfer_data(frame_.get(), sw_frame_.get(), 0) < 0) {
-            CMVR_LOG(ERROR) << "Error transferring depth data to hardware frame";
-            return false;
-        }
-
-        frame_->pts = sw_frame_->pts;
-        return true;
-    }
-
-    bool prepareColorOrGrayscaleFrame(const cv::Mat& image) {
-        if (image.cols != width_ || image.rows != height_) {
-            CMVR_LOG(ERROR) << "Image size mismatch: " << image.cols << "x" << image.rows
-                      << " vs " << width_ << "x" << height_;
-            return false;
-        }
-
-        // 1. 确定输入图像的像素格式（OpenCV的Mat格式）
-        AVPixelFormat src_pix_fmt;
-        if (image.channels() == 3) {
-            src_pix_fmt = AV_PIX_FMT_BGR24;  // OpenCV默认是BGR格式（3通道）
-        } else if (image.channels() == 1) {
-            src_pix_fmt = AV_PIX_FMT_GRAY8;  // 灰度图（1通道）
-        } else {
-            CMVR_LOG(ERROR) << "Unsupported channel count: " << image.channels();
-            return false;
-        }
-
-        // 2. 初始化格式转换上下文（swscale）
-        SwsContext* sws_ctx = sws_getContext(
-            width_, height_, src_pix_fmt,       // 源宽高和格式
-            width_, height_, AV_PIX_FMT_NV12,   // 目标宽高和格式（NV12）
-            SWS_BILINEAR,                       // 缩放算法（可根据需求调整）
-            nullptr, nullptr, nullptr
-        );
-        if (!sws_ctx) {
-            CMVR_LOG(ERROR) << "Failed to create sws context";
-            return false;
-        }
-
-        // 3. 配置源数据（OpenCV的Mat）
-        uint8_t* src_data[4] = {nullptr};
-        int src_linesize[4] = {0};
-        if (image.channels() == 3) {
-            src_data[0] = image.data;                  // BGR数据起始地址
-            src_linesize[0] = image.step;              // 每行字节数（含OpenCV的对齐填充）
-        } else {
-            src_data[0] = image.data;                  // 灰度数据起始地址
-            src_linesize[0] = image.step;              // 灰度图每行字节数
-        }
-
-        // 4. 配置目标数据（NV12格式的AVFrame）
-        av_frame_make_writable(sw_frame_.get());
-        sw_frame_->format = AV_PIX_FMT_NV12;
-        sw_frame_->width = width_;
-        sw_frame_->height = height_;
-
-        // 5. 执行格式转换（BGR/GRAY -> NV12）
-        int ret = sws_scale(
-            sws_ctx,
-            src_data,       // 源图像数据
-            src_linesize,   // 源图像每行字节数
-            0,              // 起始行
-            height_,        // 转换的行数
-            sw_frame_->data,  // 目标图像数据（sw_frame_的data指针）
-            sw_frame_->linesize  // 目标图像每行字节数
-        );
-        if (ret <= 0) {
-            CMVR_LOG(ERROR) << "sws_scale failed, ret=" << ret;
-            sws_freeContext(sws_ctx);
-            return false;
-        }
-
-        // 6. 释放转换上下文
-        sws_freeContext(sws_ctx);
-
-        // 7. 设置帧属性并上传到硬件
-        sw_frame_->pts = frame_counter_++;
-        if (av_hwframe_transfer_data(frame_.get(), sw_frame_.get(), 0) < 0) {
-            CMVR_LOG(ERROR) << "Error transferring data to hardware frame";
-            return false;
-        }
-        frame_->pts = sw_frame_->pts;
-
-        return true;
-    }
-    int width_;
-    int height_;
-    int fps_;
-    int bitrate_;
-    int frame_counter_ = 0;
-    std::string deviceName_;
-    ImageType imageType_;
-    bool initialized_{false};
-
-    FFmpegPtr<AVCodecContext> codec_ctx_;
-    FFmpegPtr<AVFrame> frame_;
-    FFmpegPtr<AVFrame> sw_frame_;
-    FFmpegPtr<AVBufferRef> hw_device_ctx_;
-    FFmpegPtr<AVBufferRef> hw_frames_ctx_;
-};
 }  // namespace ffmpeg
 
 #endif //FFMPEG_PTR_H

cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_speaker.h

@@ -3,10 +3,14 @@
 #ifndef CMVR_ES_FFMPEG_SPEAKER_H
 #define CMVR_ES_FFMPEG_SPEAKER_H
 
-#include <thread>
+#include <atomic>
-#include <mutex>
+#include <chrono>
 #include <condition_variable>
+#include <memory>
+#include <mutex>
 #include <string>
+#include <thread>
+#include <vector>
 #include "speaker/abstract_speaker.h"
 #include <boost/lockfree/spsc_queue.hpp>
 #include <pulse/simple.h>
@@ -48,6 +52,7 @@ namespace cmvr::device {
         bool decodeStreamFrame_(const AudioStreamFrameData& frame_data);
         bool initStreamDecoder_(const AudioStreamFrameData& frame_data);
         void releaseStreamDecoder_();
+        void releaseStreamDecoderUnlocked_();
 
         pa_simple* pulse_simple_ = nullptr;  // 修改：PulseAudio 简单 API 句柄
         pa_sample_spec sample_spec_{};         // 新增：PulseAudio 采样规格
@@ -59,6 +64,7 @@ namespace cmvr::device {
         std::shared_ptr<std::thread> decode_thread_;
         std::shared_ptr<std::thread> play_thread_;
         std::mutex mtx_;
+        std::mutex stop_mtx_;
         std::string audio_path_;
 
         // Boost 单生产者单消费者无锁队列
@@ -68,6 +74,7 @@ namespace cmvr::device {
         mutable std::mutex mtx_pause_;
         std::condition_variable cv_pause_;
         std::atomic<bool> is_paused_{false};
+        std::atomic<bool> is_stopping_{false};
 
         // 音频时钟同步
         std::atomic<int64_t> audio_pts_{0};

cmvr-es/devices/speaker/ffmpeg_speaker/src/ffmpeg_speaker.cpp

@@ -31,6 +31,7 @@ ffmpegSpeaker::ffmpegSpeaker(const config::FFMpegSpeakerConfig& cfg):config_(cfg
 
 
 ffmpegSpeaker::~ffmpegSpeaker() {
+    is_stopping_ = true;
     {
         std::lock_guard<std::mutex> lock(mtx_);
         state_.is_running = false;
@@ -101,13 +102,17 @@ void ffmpegSpeaker::resetPlayState()
 }
 
 bool ffmpegSpeaker::stop() {
+    std::lock_guard<std::mutex> stop_lock(stop_mtx_);
+    is_stopping_ = true;
 
     {
         lock_guard lock(mtx_);
         state_.is_running = false;
         state_.is_decoding = false;
         state_.is_paused = false;
+        is_streaming_input_ = false;
     }
+    cv_pause_.notify_all();
     // 等待线程结束
     if (decode_thread_ && decode_thread_->joinable()) {
         decode_thread_->join();
@@ -120,6 +125,7 @@ bool ffmpegSpeaker::stop() {
     }
 
     resetPlayState();
+    is_stopping_ = false;
     return true;
 }
 
@@ -540,6 +546,10 @@ void ffmpegSpeaker::play_audio_() {
 
 bool ffmpegSpeaker::pushAudioFrame(const AudioStreamFrameData& frame_data)
 {
+    if (is_stopping_) {
+        return false;
+    }
+
     if (frame_data.data.empty()) {
         return true;
     }
@@ -577,9 +587,12 @@ bool ffmpegSpeaker::pushAudioFrame(const AudioStreamFrameData& frame_data)
 
 void ffmpegSpeaker::stopStreaming()
 {
+    std::lock_guard<std::mutex> stop_lock(stop_mtx_);
+    is_stopping_ = true;
     {
         std::lock_guard<std::mutex> lock(mtx_);
         if (!is_streaming_input_) {
+            is_stopping_ = false;
             return;
         }
         state_.is_decoding = false;
@@ -595,6 +608,7 @@ void ffmpegSpeaker::stopStreaming()
     }
 
     resetPlayState();
+    is_stopping_ = false;
 }
 
 bool ffmpegSpeaker::startStreamingPlayback_(const AudioStreamFrameData& frame_data)
@@ -653,13 +667,18 @@ bool ffmpegSpeaker::pushPcmFrame_(const AudioStreamFrameData& frame_data)
         sample = static_cast<int16_t>(std::clamp(scaled, -32768.f, 32767.f));
     }
 
-    while (!audio_queue_.push(buffer)) {
+    while (true) {
-        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        {
             std::lock_guard<std::mutex> lock(mtx_);
-        if (!state_.is_running) {
+            if (!state_.is_running || is_stopping_) {
                 return false;
             }
         }
+        if (audio_queue_.push(buffer)) {
+            break;
+        }
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
     return true;
 }
 
@@ -670,7 +689,7 @@ bool ffmpegSpeaker::decodeStreamFrame_(const AudioStreamFrameData& frame_data)
         stream_codec_ != frame_data.codec ||
         stream_input_sample_rate_ != frame_data.sample_rate ||
         stream_input_channels_ != frame_data.channels) {
-        releaseStreamDecoder_();
+        releaseStreamDecoderUnlocked_();
         if (!initStreamDecoder_(frame_data)) {
             return false;
         }
@@ -738,13 +757,18 @@ bool ffmpegSpeaker::decodeStreamFrame_(const AudioStreamFrameData& frame_data)
             sample = static_cast<int16_t>(std::clamp(scaled, -32768.f, 32767.f));
         }
 
-        while (!audio_queue_.push(buffer)) {
+        while (true) {
-            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+            {
                 std::lock_guard<std::mutex> lock(mtx_);
-            if (!state_.is_running) {
+                if (!state_.is_running || is_stopping_) {
                     return false;
                 }
             }
+            if (audio_queue_.push(buffer)) {
+                break;
+            }
+            std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
     }
 
     return ret == AVERROR(EAGAIN) || ret == AVERROR_EOF;
@@ -782,14 +806,14 @@ bool ffmpegSpeaker::initStreamDecoder_(const AudioStreamFrameData& frame_data)
         stream_decoder_ctx_->channels == 1 ? AV_CH_LAYOUT_MONO : AV_CH_LAYOUT_STEREO;
 
     if (avcodec_open2(stream_decoder_ctx_, decoder, nullptr) < 0) {
-        releaseStreamDecoder_();
+        releaseStreamDecoderUnlocked_();
         return false;
     }
 
     stream_decode_frame_ = av_frame_alloc();
     stream_decode_packet_ = av_packet_alloc();
     if (!stream_decode_frame_ || !stream_decode_packet_) {
-        releaseStreamDecoder_();
+        releaseStreamDecoderUnlocked_();
         return false;
     }
 
@@ -800,6 +824,12 @@ bool ffmpegSpeaker::initStreamDecoder_(const AudioStreamFrameData& frame_data)
 }
 
 void ffmpegSpeaker::releaseStreamDecoder_()
+{
+    std::lock_guard<std::mutex> decode_lock(stream_decode_mtx_);
+    releaseStreamDecoderUnlocked_();
+}
+
+void ffmpegSpeaker::releaseStreamDecoderUnlocked_()
 {
     if (stream_swr_ctx_) {
         swr_free(&stream_swr_ctx_);