SmartBench/cmvr-es
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update speaker

Browse Source
This commit is contained in:
linbo 2026-07-22 13:48:11 +08:00
parent 0ac04290e4
commit 1fc5179344
4 changed files with 53 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmvr-es/config/devices/microphone/microphone.pb.txt
View File

@ -3,7 +3,7 @@ microphone {
id: "mic1"
ffmpeg {
channels: 2
sampleRate: 44100
sampleRate: 48000
volume: 100
input_device: "default"
}

285
cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_ptr.h
View File

@ -114,291 +114,6 @@ enum class ImageType {
DEPTH, // 深度图像
GRAYSCALE // 灰度图像
};
class HEVCEncoder {
public:
HEVCEncoder(int width, int height, int fps, int bitrate,
const std::string& deviceName = "/dev/dri/renderD128",
ImageType imageType = ImageType::COLOR)
: width_(width), height_(height), fps_(fps), bitrate_(bitrate),
deviceName_(deviceName), imageType_(imageType) {
initialized_ = init();
}
bool encodeFrame(const cv::Mat& image) {
if (!initialized_ || !codec_ctx_ || !frame_) {
CMVR_LOG(ERROR) << "HEVC encoder is not initialized";
return false;
}
if (!prepareFrame(image)) {
CMVR_LOG(ERROR) << "Failed to prepare frame";
return false;
}
if (avcodec_send_frame(codec_ctx_.get(), frame_.get()) < 0) {
CMVR_LOG(ERROR) << "Error sending frame to encoder";
return false;
}
return true;
}
std::vector<uint8_t> receivePacket(bool& is_key) {
std::vector<uint8_t> packetData;
if (!initialized_ || !codec_ctx_) {
CMVR_LOG(ERROR) << "HEVC encoder is not initialized";
return packetData;
}
FFmpegPtr<AVPacket> packet(av_packet_alloc());
while (true)
{
int ret = avcodec_receive_packet(codec_ctx_.get(), packet.get());
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
break;
} else if (ret < 0) {
CMVR_LOG(ERROR) << "Error during encoding";
break;
}
is_key = (packet->flags & AV_PKT_FLAG_KEY) != 0;
// 预留足够空间，避免多次内存分配
packetData.reserve(packetData.size() + packet->size);
// 复制数据包内容到输出向量
packetData.insert(packetData.end(),
packet->data,
packet->data + packet->size);
av_packet_unref(packet.get());
}
return packetData;
}
private:
bool init() {
// 1. 创建硬件设备上下文
AVBufferRef* hw_device_ctx = nullptr;
int ret = av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI,
deviceName_.c_str(), nullptr, 0);
if (ret < 0) {
char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0};
av_strerror(ret, errbuf, sizeof(errbuf));
CMVR_LOG(ERROR) << "Failed to create VAAPI device context: " << errbuf;
return false;
}
hw_device_ctx_.reset(hw_device_ctx);
// 2. 查找编码器
const AVCodec* codec = avcodec_find_encoder_by_name("hevc_vaapi");
if (!codec) {
CMVR_LOG(ERROR) << "HEVC VAAPI encoder not found";
return false;
}
// 3. 设置编码器上下文
codec_ctx_.reset(avcodec_alloc_context3(codec));
codec_ctx_->width = width_;
codec_ctx_->height = height_;
codec_ctx_->time_base = {1, fps_};
codec_ctx_->pix_fmt = AV_PIX_FMT_VAAPI;
codec_ctx_->bit_rate = bitrate_;
codec_ctx_->gop_size = 1;
codec_ctx_->hw_device_ctx = av_buffer_ref(hw_device_ctx_.get());
// 4. 创建硬件帧上下文
hw_frames_ctx_.reset(av_hwframe_ctx_alloc(hw_device_ctx_.get()));
if (!hw_frames_ctx_) {
CMVR_LOG(ERROR) << "Failed to allocate hardware frame context";
return false;
}
AVHWFramesContext* frames_ctx = (AVHWFramesContext*)hw_frames_ctx_->data;
frames_ctx->format = AV_PIX_FMT_VAAPI;
frames_ctx->sw_format = AV_PIX_FMT_NV12;
frames_ctx->width = width_;
frames_ctx->height = height_;
frames_ctx->initial_pool_size = 20;
if (av_hwframe_ctx_init(hw_frames_ctx_.get()) < 0) {
CMVR_LOG(ERROR) << "Failed to initialize hardware frame context";
return false;
}
codec_ctx_->hw_frames_ctx = av_buffer_ref(hw_frames_ctx_.get());
// 5. 打开编码器
if (avcodec_open2(codec_ctx_.get(), codec, nullptr) < 0) {
CMVR_LOG(ERROR) << "Cannot open encoder";
return false;
}
// 6. 初始化硬件帧
frame_.reset(av_frame_alloc());
frame_->format = AV_PIX_FMT_VAAPI;
frame_->width = width_;
frame_->height = height_;
if (av_hwframe_get_buffer(hw_frames_ctx_.get(), frame_.get(), 0) < 0) {
CMVR_LOG(ERROR) << "Could not allocate hardware frame data";
return false;
}
// 7. 初始化软件帧
sw_frame_.reset(av_frame_alloc());
sw_frame_->width = width_;
sw_frame_->height = height_;
sw_frame_->format = (imageType_ == ImageType::DEPTH) ? AV_PIX_FMT_GRAY8 : AV_PIX_FMT_NV12;
if (av_frame_get_buffer(sw_frame_.get(), 0) < 0) {
CMVR_LOG(ERROR) << "Could not allocate software frame buffer";
return false;
}
return true;
}
bool prepareFrame(const cv::Mat& image) {
if (imageType_ == ImageType::DEPTH) {
return prepareDepthFrame(image);
} else {
return prepareColorOrGrayscaleFrame(image);
}
}
bool prepareDepthFrame(const cv::Mat& depthImage) {
cv::Mat processedDepth;
if (depthImage.channels() != 1) {
CMVR_LOG(ERROR) << "Depth image must be single-channel";
return false;
}
if (depthImage.depth() != CV_8U) {
if (depthImage.depth() == CV_16U) {
depthImage.convertTo(processedDepth, CV_8U, 255.0 / 65535.0);
} else {
CMVR_LOG(ERROR) << "Unsupported depth image format";
return false;
}
} else {
processedDepth = depthImage.clone();
}
// 确保图像尺寸匹配
if (processedDepth.cols != width_ || processedDepth.rows != height_) {
CMVR_LOG(ERROR) << "Depth image size mismatch";
return false;
}
// 设置软件帧属性
sw_frame_->pts = frame_counter_++;
sw_frame_->format = AV_PIX_FMT_GRAY8;
av_frame_make_writable(sw_frame_.get());
// 使用av_image_fill_arrays处理行对齐
av_image_fill_arrays(sw_frame_->data, sw_frame_->linesize,
processedDepth.data,
AV_PIX_FMT_GRAY8,
width_, height_, 1);
// 上传到硬件帧
if (av_hwframe_transfer_data(frame_.get(), sw_frame_.get(), 0) < 0) {
CMVR_LOG(ERROR) << "Error transferring depth data to hardware frame";
return false;
}
frame_->pts = sw_frame_->pts;
return true;
}
bool prepareColorOrGrayscaleFrame(const cv::Mat& image) {
if (image.cols != width_ || image.rows != height_) {
CMVR_LOG(ERROR) << "Image size mismatch: " << image.cols << "x" << image.rows
<< " vs " << width_ << "x" << height_;
return false;
}
// 1. 确定输入图像的像素格式（OpenCV的Mat格式）
AVPixelFormat src_pix_fmt;
if (image.channels() == 3) {
src_pix_fmt = AV_PIX_FMT_BGR24; // OpenCV默认是BGR格式（3通道）
} else if (image.channels() == 1) {
src_pix_fmt = AV_PIX_FMT_GRAY8; // 灰度图（1通道）
} else {
CMVR_LOG(ERROR) << "Unsupported channel count: " << image.channels();
return false;
}
// 2. 初始化格式转换上下文（swscale）
SwsContext* sws_ctx = sws_getContext(
width_, height_, src_pix_fmt, // 源宽高和格式
width_, height_, AV_PIX_FMT_NV12, // 目标宽高和格式（NV12）
SWS_BILINEAR, // 缩放算法（可根据需求调整）
nullptr, nullptr, nullptr
);
if (!sws_ctx) {
CMVR_LOG(ERROR) << "Failed to create sws context";
return false;
}
// 3. 配置源数据（OpenCV的Mat）
uint8_t* src_data[4] = {nullptr};
int src_linesize[4] = {0};
if (image.channels() == 3) {
src_data[0] = image.data; // BGR数据起始地址
src_linesize[0] = image.step; // 每行字节数（含OpenCV的对齐填充）
} else {
src_data[0] = image.data; // 灰度数据起始地址
src_linesize[0] = image.step; // 灰度图每行字节数
}
// 4. 配置目标数据（NV12格式的AVFrame）
av_frame_make_writable(sw_frame_.get());
sw_frame_->format = AV_PIX_FMT_NV12;
sw_frame_->width = width_;
sw_frame_->height = height_;
// 5. 执行格式转换（BGR/GRAY -> NV12）
int ret = sws_scale(
sws_ctx,
src_data, // 源图像数据
src_linesize, // 源图像每行字节数
0, // 起始行
height_, // 转换的行数
sw_frame_->data, // 目标图像数据（sw_frame_的data指针）
sw_frame_->linesize // 目标图像每行字节数
);
if (ret <= 0) {
CMVR_LOG(ERROR) << "sws_scale failed, ret=" << ret;
sws_freeContext(sws_ctx);
return false;
}
// 6. 释放转换上下文
sws_freeContext(sws_ctx);
// 7. 设置帧属性并上传到硬件
sw_frame_->pts = frame_counter_++;
if (av_hwframe_transfer_data(frame_.get(), sw_frame_.get(), 0) < 0) {
CMVR_LOG(ERROR) << "Error transferring data to hardware frame";
return false;
}
frame_->pts = sw_frame_->pts;
return true;
}
int width_;
int height_;
int fps_;
int bitrate_;
int frame_counter_ = 0;
std::string deviceName_;
ImageType imageType_;
bool initialized_{false};
FFmpegPtr<AVCodecContext> codec_ctx_;
FFmpegPtr<AVFrame> frame_;
FFmpegPtr<AVFrame> sw_frame_;
FFmpegPtr<AVBufferRef> hw_device_ctx_;
FFmpegPtr<AVBufferRef> hw_frames_ctx_;
};
} // namespace ffmpeg
#endif //FFMPEG_PTR_H

11
cmvr-es/devices/speaker/ffmpeg_speaker/include/ffmpeg_speaker.h
View File

@ -3,10 +3,14 @@
#ifndef CMVR_ES_FFMPEG_SPEAKER_H
#define CMVR_ES_FFMPEG_SPEAKER_H
#include <thread>
#include <mutex>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <memory>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
#include "speaker/abstract_speaker.h"
#include <boost/lockfree/spsc_queue.hpp>
#include <pulse/simple.h>
@ -48,6 +52,7 @@ namespace cmvr::device {
bool decodeStreamFrame_(const AudioStreamFrameData& frame_data);
bool initStreamDecoder_(const AudioStreamFrameData& frame_data);
void releaseStreamDecoder_();
void releaseStreamDecoderUnlocked_();
pa_simple* pulse_simple_ = nullptr; // 修改：PulseAudio 简单 API 句柄
pa_sample_spec sample_spec_{}; // 新增：PulseAudio 采样规格
@ -59,6 +64,7 @@ namespace cmvr::device {
std::shared_ptr<std::thread> decode_thread_;
std::shared_ptr<std::thread> play_thread_;
std::mutex mtx_;
std::mutex stop_mtx_;
std::string audio_path_;
// Boost 单生产者单消费者无锁队列
@ -68,6 +74,7 @@ namespace cmvr::device {
mutable std::mutex mtx_pause_;
std::condition_variable cv_pause_;
std::atomic<bool> is_paused_{false};
std::atomic<bool> is_stopping_{false};
// 音频时钟同步
std::atomic<int64_t> audio_pts_{0};

56
cmvr-es/devices/speaker/ffmpeg_speaker/src/ffmpeg_speaker.cpp
View File

@ -31,6 +31,7 @@ ffmpegSpeaker::ffmpegSpeaker(const config::FFMpegSpeakerConfig& cfg):config_(cfg
ffmpegSpeaker::~ffmpegSpeaker() {
is_stopping_ = true;
{
std::lock_guard<std::mutex> lock(mtx_);
state_.is_running = false;
@ -101,13 +102,17 @@ void ffmpegSpeaker::resetPlayState()
}
bool ffmpegSpeaker::stop() {
std::lock_guard<std::mutex> stop_lock(stop_mtx_);
is_stopping_ = true;
{
lock_guard lock(mtx_);
state_.is_running = false;
state_.is_decoding = false;
state_.is_paused = false;
is_streaming_input_ = false;
}
cv_pause_.notify_all();
// 等待线程结束
if (decode_thread_ && decode_thread_->joinable()) {
decode_thread_->join();
@ -120,6 +125,7 @@ bool ffmpegSpeaker::stop() {
}
resetPlayState();
is_stopping_ = false;
return true;
}
@ -540,6 +546,10 @@ void ffmpegSpeaker::play_audio_() {
bool ffmpegSpeaker::pushAudioFrame(const AudioStreamFrameData& frame_data)
{
if (is_stopping_) {
return false;
}
if (frame_data.data.empty()) {
return true;
}
@ -577,9 +587,12 @@ bool ffmpegSpeaker::pushAudioFrame(const AudioStreamFrameData& frame_data)
void ffmpegSpeaker::stopStreaming()
{
std::lock_guard<std::mutex> stop_lock(stop_mtx_);
is_stopping_ = true;
{
std::lock_guard<std::mutex> lock(mtx_);
if (!is_streaming_input_) {
is_stopping_ = false;
return;
}
state_.is_decoding = false;
@ -595,6 +608,7 @@ void ffmpegSpeaker::stopStreaming()
}
resetPlayState();
is_stopping_ = false;
}
bool ffmpegSpeaker::startStreamingPlayback_(const AudioStreamFrameData& frame_data)
@ -653,12 +667,17 @@ bool ffmpegSpeaker::pushPcmFrame_(const AudioStreamFrameData& frame_data)
sample = static_cast<int16_t>(std::clamp(scaled, -32768.f, 32767.f));
}
while (!audio_queue_.push(buffer)) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
std::lock_guard<std::mutex> lock(mtx_);
if (!state_.is_running) {
return false;
while (true) {
{
std::lock_guard<std::mutex> lock(mtx_);
if (!state_.is_running || is_stopping_) {
return false;
}
}
if (audio_queue_.push(buffer)) {
break;
}
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
return true;
}
@ -670,7 +689,7 @@ bool ffmpegSpeaker::decodeStreamFrame_(const AudioStreamFrameData& frame_data)
stream_codec_ != frame_data.codec ||
stream_input_sample_rate_ != frame_data.sample_rate ||
stream_input_channels_ != frame_data.channels) {
releaseStreamDecoder_();
releaseStreamDecoderUnlocked_();
if (!initStreamDecoder_(frame_data)) {
return false;
}
@ -738,12 +757,17 @@ bool ffmpegSpeaker::decodeStreamFrame_(const AudioStreamFrameData& frame_data)
sample = static_cast<int16_t>(std::clamp(scaled, -32768.f, 32767.f));
}
while (!audio_queue_.push(buffer)) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
std::lock_guard<std::mutex> lock(mtx_);
if (!state_.is_running) {
return false;
while (true) {
{
std::lock_guard<std::mutex> lock(mtx_);
if (!state_.is_running || is_stopping_) {
return false;
}
}
if (audio_queue_.push(buffer)) {
break;
}
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
}
@ -782,14 +806,14 @@ bool ffmpegSpeaker::initStreamDecoder_(const AudioStreamFrameData& frame_data)
stream_decoder_ctx_->channels == 1 ? AV_CH_LAYOUT_MONO : AV_CH_LAYOUT_STEREO;
if (avcodec_open2(stream_decoder_ctx_, decoder, nullptr) < 0) {
releaseStreamDecoder_();
releaseStreamDecoderUnlocked_();
return false;
}
stream_decode_frame_ = av_frame_alloc();
stream_decode_packet_ = av_packet_alloc();
if (!stream_decode_frame_ || !stream_decode_packet_) {
releaseStreamDecoder_();
releaseStreamDecoderUnlocked_();
return false;
}
@ -800,6 +824,12 @@ bool ffmpegSpeaker::initStreamDecoder_(const AudioStreamFrameData& frame_data)
}
void ffmpegSpeaker::releaseStreamDecoder_()
{
std::lock_guard<std::mutex> decode_lock(stream_decode_mtx_);
releaseStreamDecoderUnlocked_();
}
void ffmpegSpeaker::releaseStreamDecoderUnlocked_()
{
if (stream_swr_ctx_) {
swr_free(&stream_swr_ctx_);