update speaker
This commit is contained in:
parent
0ac04290e4
commit
1fc5179344
@ -3,7 +3,7 @@ microphone {
|
||||
id: "mic1"
|
||||
ffmpeg {
|
||||
channels: 2
|
||||
sampleRate: 44100
|
||||
sampleRate: 48000
|
||||
volume: 100
|
||||
input_device: "default"
|
||||
}
|
||||
|
||||
@ -114,291 +114,6 @@ enum class ImageType {
|
||||
DEPTH, // 深度图像
|
||||
GRAYSCALE // 灰度图像
|
||||
};
|
||||
|
||||
class HEVCEncoder {
|
||||
public:
|
||||
HEVCEncoder(int width, int height, int fps, int bitrate,
|
||||
const std::string& deviceName = "/dev/dri/renderD128",
|
||||
ImageType imageType = ImageType::COLOR)
|
||||
: width_(width), height_(height), fps_(fps), bitrate_(bitrate),
|
||||
deviceName_(deviceName), imageType_(imageType) {
|
||||
initialized_ = init();
|
||||
}
|
||||
|
||||
bool encodeFrame(const cv::Mat& image) {
|
||||
if (!initialized_ || !codec_ctx_ || !frame_) {
|
||||
CMVR_LOG(ERROR) << "HEVC encoder is not initialized";
|
||||
return false;
|
||||
}
|
||||
if (!prepareFrame(image)) {
|
||||
CMVR_LOG(ERROR) << "Failed to prepare frame";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (avcodec_send_frame(codec_ctx_.get(), frame_.get()) < 0) {
|
||||
CMVR_LOG(ERROR) << "Error sending frame to encoder";
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> receivePacket(bool& is_key) {
|
||||
std::vector<uint8_t> packetData;
|
||||
if (!initialized_ || !codec_ctx_) {
|
||||
CMVR_LOG(ERROR) << "HEVC encoder is not initialized";
|
||||
return packetData;
|
||||
}
|
||||
FFmpegPtr<AVPacket> packet(av_packet_alloc());
|
||||
|
||||
while (true)
|
||||
{
|
||||
int ret = avcodec_receive_packet(codec_ctx_.get(), packet.get());
|
||||
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
|
||||
break;
|
||||
} else if (ret < 0) {
|
||||
CMVR_LOG(ERROR) << "Error during encoding";
|
||||
break;
|
||||
}
|
||||
is_key = (packet->flags & AV_PKT_FLAG_KEY) != 0;
|
||||
// 预留足够空间，避免多次内存分配
|
||||
packetData.reserve(packetData.size() + packet->size);
|
||||
// 复制数据包内容到输出向量
|
||||
packetData.insert(packetData.end(),
|
||||
packet->data,
|
||||
packet->data + packet->size);
|
||||
av_packet_unref(packet.get());
|
||||
}
|
||||
return packetData;
|
||||
}
|
||||
|
||||
private:
|
||||
bool init() {
|
||||
// 1. 创建硬件设备上下文
|
||||
AVBufferRef* hw_device_ctx = nullptr;
|
||||
int ret = av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI,
|
||||
deviceName_.c_str(), nullptr, 0);
|
||||
if (ret < 0) {
|
||||
char errbuf[AV_ERROR_MAX_STRING_SIZE] = {0};
|
||||
av_strerror(ret, errbuf, sizeof(errbuf));
|
||||
CMVR_LOG(ERROR) << "Failed to create VAAPI device context: " << errbuf;
|
||||
return false;
|
||||
}
|
||||
hw_device_ctx_.reset(hw_device_ctx);
|
||||
|
||||
// 2. 查找编码器
|
||||
const AVCodec* codec = avcodec_find_encoder_by_name("hevc_vaapi");
|
||||
if (!codec) {
|
||||
CMVR_LOG(ERROR) << "HEVC VAAPI encoder not found";
|
||||
return false;
|
||||
}
|
||||
|
||||
// 3. 设置编码器上下文
|
||||
codec_ctx_.reset(avcodec_alloc_context3(codec));
|
||||
codec_ctx_->width = width_;
|
||||
codec_ctx_->height = height_;
|
||||
codec_ctx_->time_base = {1, fps_};
|
||||
codec_ctx_->pix_fmt = AV_PIX_FMT_VAAPI;
|
||||
codec_ctx_->bit_rate = bitrate_;
|
||||
codec_ctx_->gop_size = 1;
|
||||
codec_ctx_->hw_device_ctx = av_buffer_ref(hw_device_ctx_.get());
|
||||
|
||||
// 4. 创建硬件帧上下文
|
||||
hw_frames_ctx_.reset(av_hwframe_ctx_alloc(hw_device_ctx_.get()));
|
||||
if (!hw_frames_ctx_) {
|
||||
CMVR_LOG(ERROR) << "Failed to allocate hardware frame context";
|
||||
return false;
|
||||
}
|
||||
|
||||
AVHWFramesContext* frames_ctx = (AVHWFramesContext*)hw_frames_ctx_->data;
|
||||
frames_ctx->format = AV_PIX_FMT_VAAPI;
|
||||
frames_ctx->sw_format = AV_PIX_FMT_NV12;
|
||||
frames_ctx->width = width_;
|
||||
frames_ctx->height = height_;
|
||||
frames_ctx->initial_pool_size = 20;
|
||||
|
||||
if (av_hwframe_ctx_init(hw_frames_ctx_.get()) < 0) {
|
||||
CMVR_LOG(ERROR) << "Failed to initialize hardware frame context";
|
||||
return false;
|
||||
}
|
||||
|
||||
codec_ctx_->hw_frames_ctx = av_buffer_ref(hw_frames_ctx_.get());
|
||||
|
||||
// 5. 打开编码器
|
||||
if (avcodec_open2(codec_ctx_.get(), codec, nullptr) < 0) {
|
||||
CMVR_LOG(ERROR) << "Cannot open encoder";
|
||||
return false;
|
||||
}
|
||||
|
||||
// 6. 初始化硬件帧
|
||||
frame_.reset(av_frame_alloc());
|
||||
frame_->format = AV_PIX_FMT_VAAPI;
|
||||
frame_->width = width_;
|
||||
frame_->height = height_;
|
||||
|
||||
if (av_hwframe_get_buffer(hw_frames_ctx_.get(), frame_.get(), 0) < 0) {
|
||||
CMVR_LOG(ERROR) << "Could not allocate hardware frame data";
|
||||
return false;
|
||||
}
|
||||
|
||||
// 7. 初始化软件帧
|
||||
sw_frame_.reset(av_frame_alloc());
|
||||
sw_frame_->width = width_;
|
||||
sw_frame_->height = height_;
|
||||
sw_frame_->format = (imageType_ == ImageType::DEPTH) ? AV_PIX_FMT_GRAY8 : AV_PIX_FMT_NV12;
|
||||
|
||||
if (av_frame_get_buffer(sw_frame_.get(), 0) < 0) {
|
||||
CMVR_LOG(ERROR) << "Could not allocate software frame buffer";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool prepareFrame(const cv::Mat& image) {
|
||||
if (imageType_ == ImageType::DEPTH) {
|
||||
return prepareDepthFrame(image);
|
||||
} else {
|
||||
return prepareColorOrGrayscaleFrame(image);
|
||||
}
|
||||
}
|
||||
|
||||
bool prepareDepthFrame(const cv::Mat& depthImage) {
|
||||
cv::Mat processedDepth;
|
||||
|
||||
if (depthImage.channels() != 1) {
|
||||
CMVR_LOG(ERROR) << "Depth image must be single-channel";
|
||||
return false;
|
||||
}
|
||||
|
||||
if (depthImage.depth() != CV_8U) {
|
||||
if (depthImage.depth() == CV_16U) {
|
||||
depthImage.convertTo(processedDepth, CV_8U, 255.0 / 65535.0);
|
||||
} else {
|
||||
CMVR_LOG(ERROR) << "Unsupported depth image format";
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
processedDepth = depthImage.clone();
|
||||
}
|
||||
|
||||
// 确保图像尺寸匹配
|
||||
if (processedDepth.cols != width_ || processedDepth.rows != height_) {
|
||||
CMVR_LOG(ERROR) << "Depth image size mismatch";
|
||||
return false;
|
||||
}
|
||||
|
||||
// 设置软件帧属性
|
||||
sw_frame_->pts = frame_counter_++;
|
||||
sw_frame_->format = AV_PIX_FMT_GRAY8;
|
||||
av_frame_make_writable(sw_frame_.get());
|
||||
|
||||
// 使用av_image_fill_arrays处理行对齐
|
||||
av_image_fill_arrays(sw_frame_->data, sw_frame_->linesize,
|
||||
processedDepth.data,
|
||||
AV_PIX_FMT_GRAY8,
|
||||
width_, height_, 1);
|
||||
|
||||
// 上传到硬件帧
|
||||
if (av_hwframe_transfer_data(frame_.get(), sw_frame_.get(), 0) < 0) {
|
||||
CMVR_LOG(ERROR) << "Error transferring depth data to hardware frame";
|
||||
return false;
|
||||
}
|
||||
|
||||
frame_->pts = sw_frame_->pts;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool prepareColorOrGrayscaleFrame(const cv::Mat& image) {
|
||||
if (image.cols != width_ || image.rows != height_) {
|
||||
CMVR_LOG(ERROR) << "Image size mismatch: " << image.cols << "x" << image.rows
|
||||
<< " vs " << width_ << "x" << height_;
|
||||
return false;
|
||||
}
|
||||
|
||||
// 1. 确定输入图像的像素格式（OpenCV的Mat格式）
|
||||
AVPixelFormat src_pix_fmt;
|
||||
if (image.channels() == 3) {
|
||||
src_pix_fmt = AV_PIX_FMT_BGR24; // OpenCV默认是BGR格式（3通道）
|
||||
} else if (image.channels() == 1) {
|
||||
src_pix_fmt = AV_PIX_FMT_GRAY8; // 灰度图（1通道）
|
||||
} else {
|
||||
CMVR_LOG(ERROR) << "Unsupported channel count: " << image.channels();
|
||||
return false;
|
||||
}
|
||||
|
||||
// 2. 初始化格式转换上下文（swscale）
|
||||
SwsContext* sws_ctx = sws_getContext(
|
||||
width_, height_, src_pix_fmt, // 源宽高和格式
|
||||
width_, height_, AV_PIX_FMT_NV12, // 目标宽高和格式（NV12）
|
||||
SWS_BILINEAR, // 缩放算法（可根据需求调整）
|
||||
nullptr, nullptr, nullptr
|
||||
);
|
||||
if (!sws_ctx) {
|
||||
CMVR_LOG(ERROR) << "Failed to create sws context";
|
||||
return false;
|
||||
}
|
||||
|
||||
// 3. 配置源数据（OpenCV的Mat）
|
||||
uint8_t* src_data[4] = {nullptr};
|
||||
int src_linesize[4] = {0};
|
||||
if (image.channels() == 3) {
|
||||
src_data[0] = image.data; // BGR数据起始地址
|
||||
src_linesize[0] = image.step; // 每行字节数（含OpenCV的对齐填充）
|
||||
} else {
|
||||
src_data[0] = image.data; // 灰度数据起始地址
|
||||
src_linesize[0] = image.step; // 灰度图每行字节数
|
||||
}
|
||||
|
||||
// 4. 配置目标数据（NV12格式的AVFrame）
|
||||
av_frame_make_writable(sw_frame_.get());
|
||||
sw_frame_->format = AV_PIX_FMT_NV12;
|
||||
sw_frame_->width = width_;
|
||||
sw_frame_->height = height_;
|
||||
|
||||
// 5. 执行格式转换（BGR/GRAY -> NV12）
|
||||
int ret = sws_scale(
|
||||
sws_ctx,
|
||||
src_data, // 源图像数据
|
||||
src_linesize, // 源图像每行字节数
|
||||
0, // 起始行
|
||||
height_, // 转换的行数
|
||||
sw_frame_->data, // 目标图像数据（sw_frame_的data指针）
|
||||
sw_frame_->linesize // 目标图像每行字节数
|
||||
);
|
||||
if (ret <= 0) {
|
||||
CMVR_LOG(ERROR) << "sws_scale failed, ret=" << ret;
|
||||
sws_freeContext(sws_ctx);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 6. 释放转换上下文
|
||||
sws_freeContext(sws_ctx);
|
||||
|
||||
// 7. 设置帧属性并上传到硬件
|
||||
sw_frame_->pts = frame_counter_++;
|
||||
if (av_hwframe_transfer_data(frame_.get(), sw_frame_.get(), 0) < 0) {
|
||||
CMVR_LOG(ERROR) << "Error transferring data to hardware frame";
|
||||
return false;
|
||||
}
|
||||
frame_->pts = sw_frame_->pts;
|
||||
|
||||
return true;
|
||||
}
|
||||
int width_;
|
||||
int height_;
|
||||
int fps_;
|
||||
int bitrate_;
|
||||
int frame_counter_ = 0;
|
||||
std::string deviceName_;
|
||||
ImageType imageType_;
|
||||
bool initialized_{false};
|
||||
|
||||
FFmpegPtr<AVCodecContext> codec_ctx_;
|
||||
FFmpegPtr<AVFrame> frame_;
|
||||
FFmpegPtr<AVFrame> sw_frame_;
|
||||
FFmpegPtr<AVBufferRef> hw_device_ctx_;
|
||||
FFmpegPtr<AVBufferRef> hw_frames_ctx_;
|
||||
};
|
||||
} // namespace ffmpeg
|
||||
|
||||
#endif //FFMPEG_PTR_H
|
||||
|
||||
@ -3,10 +3,14 @@
|
||||
#ifndef CMVR_ES_FFMPEG_SPEAKER_H
|
||||
#define CMVR_ES_FFMPEG_SPEAKER_H
|
||||
|
||||
#include <thread>
|
||||
#include <mutex>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include "speaker/abstract_speaker.h"
|
||||
#include <boost/lockfree/spsc_queue.hpp>
|
||||
#include <pulse/simple.h>
|
||||
@ -48,6 +52,7 @@ namespace cmvr::device {
|
||||
bool decodeStreamFrame_(const AudioStreamFrameData& frame_data);
|
||||
bool initStreamDecoder_(const AudioStreamFrameData& frame_data);
|
||||
void releaseStreamDecoder_();
|
||||
void releaseStreamDecoderUnlocked_();
|
||||
|
||||
pa_simple* pulse_simple_ = nullptr; // 修改：PulseAudio 简单 API 句柄
|
||||
pa_sample_spec sample_spec_{}; // 新增：PulseAudio 采样规格
|
||||
@ -59,6 +64,7 @@ namespace cmvr::device {
|
||||
std::shared_ptr<std::thread> decode_thread_;
|
||||
std::shared_ptr<std::thread> play_thread_;
|
||||
std::mutex mtx_;
|
||||
std::mutex stop_mtx_;
|
||||
std::string audio_path_;
|
||||
|
||||
// Boost 单生产者单消费者无锁队列
|
||||
@ -68,6 +74,7 @@ namespace cmvr::device {
|
||||
mutable std::mutex mtx_pause_;
|
||||
std::condition_variable cv_pause_;
|
||||
std::atomic<bool> is_paused_{false};
|
||||
std::atomic<bool> is_stopping_{false};
|
||||
|
||||
// 音频时钟同步
|
||||
std::atomic<int64_t> audio_pts_{0};
|
||||
|
||||
@ -31,6 +31,7 @@ ffmpegSpeaker::ffmpegSpeaker(const config::FFMpegSpeakerConfig& cfg):config_(cfg
|
||||
|
||||
|
||||
ffmpegSpeaker::~ffmpegSpeaker() {
|
||||
is_stopping_ = true;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
state_.is_running = false;
|
||||
@ -101,13 +102,17 @@ void ffmpegSpeaker::resetPlayState()
|
||||
}
|
||||
|
||||
bool ffmpegSpeaker::stop() {
|
||||
std::lock_guard<std::mutex> stop_lock(stop_mtx_);
|
||||
is_stopping_ = true;
|
||||
|
||||
{
|
||||
lock_guard lock(mtx_);
|
||||
state_.is_running = false;
|
||||
state_.is_decoding = false;
|
||||
state_.is_paused = false;
|
||||
is_streaming_input_ = false;
|
||||
}
|
||||
cv_pause_.notify_all();
|
||||
// 等待线程结束
|
||||
if (decode_thread_ && decode_thread_->joinable()) {
|
||||
decode_thread_->join();
|
||||
@ -120,6 +125,7 @@ bool ffmpegSpeaker::stop() {
|
||||
}
|
||||
|
||||
resetPlayState();
|
||||
is_stopping_ = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -540,6 +546,10 @@ void ffmpegSpeaker::play_audio_() {
|
||||
|
||||
bool ffmpegSpeaker::pushAudioFrame(const AudioStreamFrameData& frame_data)
|
||||
{
|
||||
if (is_stopping_) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (frame_data.data.empty()) {
|
||||
return true;
|
||||
}
|
||||
@ -577,9 +587,12 @@ bool ffmpegSpeaker::pushAudioFrame(const AudioStreamFrameData& frame_data)
|
||||
|
||||
void ffmpegSpeaker::stopStreaming()
|
||||
{
|
||||
std::lock_guard<std::mutex> stop_lock(stop_mtx_);
|
||||
is_stopping_ = true;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
if (!is_streaming_input_) {
|
||||
is_stopping_ = false;
|
||||
return;
|
||||
}
|
||||
state_.is_decoding = false;
|
||||
@ -595,6 +608,7 @@ void ffmpegSpeaker::stopStreaming()
|
||||
}
|
||||
|
||||
resetPlayState();
|
||||
is_stopping_ = false;
|
||||
}
|
||||
|
||||
bool ffmpegSpeaker::startStreamingPlayback_(const AudioStreamFrameData& frame_data)
|
||||
@ -653,13 +667,18 @@ bool ffmpegSpeaker::pushPcmFrame_(const AudioStreamFrameData& frame_data)
|
||||
sample = static_cast<int16_t>(std::clamp(scaled, -32768.f, 32767.f));
|
||||
}
|
||||
|
||||
while (!audio_queue_.push(buffer)) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
||||
while (true) {
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
if (!state_.is_running) {
|
||||
if (!state_.is_running || is_stopping_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (audio_queue_.push(buffer)) {
|
||||
break;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -670,7 +689,7 @@ bool ffmpegSpeaker::decodeStreamFrame_(const AudioStreamFrameData& frame_data)
|
||||
stream_codec_ != frame_data.codec ||
|
||||
stream_input_sample_rate_ != frame_data.sample_rate ||
|
||||
stream_input_channels_ != frame_data.channels) {
|
||||
releaseStreamDecoder_();
|
||||
releaseStreamDecoderUnlocked_();
|
||||
if (!initStreamDecoder_(frame_data)) {
|
||||
return false;
|
||||
}
|
||||
@ -738,13 +757,18 @@ bool ffmpegSpeaker::decodeStreamFrame_(const AudioStreamFrameData& frame_data)
|
||||
sample = static_cast<int16_t>(std::clamp(scaled, -32768.f, 32767.f));
|
||||
}
|
||||
|
||||
while (!audio_queue_.push(buffer)) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
||||
while (true) {
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
if (!state_.is_running) {
|
||||
if (!state_.is_running || is_stopping_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (audio_queue_.push(buffer)) {
|
||||
break;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(1));
|
||||
}
|
||||
}
|
||||
|
||||
return ret == AVERROR(EAGAIN) || ret == AVERROR_EOF;
|
||||
@ -782,14 +806,14 @@ bool ffmpegSpeaker::initStreamDecoder_(const AudioStreamFrameData& frame_data)
|
||||
stream_decoder_ctx_->channels == 1 ? AV_CH_LAYOUT_MONO : AV_CH_LAYOUT_STEREO;
|
||||
|
||||
if (avcodec_open2(stream_decoder_ctx_, decoder, nullptr) < 0) {
|
||||
releaseStreamDecoder_();
|
||||
releaseStreamDecoderUnlocked_();
|
||||
return false;
|
||||
}
|
||||
|
||||
stream_decode_frame_ = av_frame_alloc();
|
||||
stream_decode_packet_ = av_packet_alloc();
|
||||
if (!stream_decode_frame_ || !stream_decode_packet_) {
|
||||
releaseStreamDecoder_();
|
||||
releaseStreamDecoderUnlocked_();
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -800,6 +824,12 @@ bool ffmpegSpeaker::initStreamDecoder_(const AudioStreamFrameData& frame_data)
|
||||
}
|
||||
|
||||
void ffmpegSpeaker::releaseStreamDecoder_()
|
||||
{
|
||||
std::lock_guard<std::mutex> decode_lock(stream_decode_mtx_);
|
||||
releaseStreamDecoderUnlocked_();
|
||||
}
|
||||
|
||||
void ffmpegSpeaker::releaseStreamDecoderUnlocked_()
|
||||
{
|
||||
if (stream_swr_ctx_) {
|
||||
swr_free(&stream_swr_ctx_);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user