diff --git a/package.json b/package.json index 3d9d273..5002f4d 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "urdf-loader": "^0.12.6", "vue": "^3.5.38", "vue-router": "^5.1.0", + "ws": "^8.18.3", "xacro-parser": "^0.3.11", "xgplayer": "^3.0.24" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 28f0dc7..cca8c2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,6 +56,9 @@ importers: vue-router: specifier: ^5.1.0 version: 5.1.0(@vue/compiler-sfc@3.5.38)(pinia@3.0.4(vue@3.5.38))(vite@8.1.0(@types/node@26.0.0)(sass@1.101.0)(yaml@2.9.0))(vue@3.5.38) + ws: + specifier: ^8.18.3 + version: 8.21.1 xacro-parser: specifier: ^0.3.11 version: 0.3.11 @@ -2373,6 +2376,18 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + ws@8.21.1: + resolution: {integrity: sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + xacro-parser@0.3.11: resolution: {integrity: sha512-zyRzHmf6/BySLZQP77zp6iEtdryq1g7tWtfwj/28GOPyf9ijtPvouivOAeaA/yAu0/Bwh20DtA4x+BrYiV6qKQ==} @@ -4948,6 +4963,8 @@ snapshots: wrappy@1.0.2: {} + ws@8.21.1: {} + xacro-parser@0.3.11: dependencies: expr-eval-fork: 3.0.3 diff --git a/server/audioWebSocket.js b/server/audioWebSocket.js new file mode 100644 index 0000000..cf4beba --- /dev/null +++ b/server/audioWebSocket.js @@ -0,0 +1,170 @@ +import { WebSocketServer, WebSocket } from 'ws' +import { createMicrophoneGrpcClient, createSpeakerGrpcClient } from './grpcClient.js' + +// 设备实际返回 48kHz；上下行统一使用 48kHz，避免设备按固定采样率解释 44.1kHz 数据。 +const SAMPLE_RATE = 48000 +const CHANNELS = 2 +const MAX_BUFFERED_BYTES = SAMPLE_RATE * CHANNELS * 2 * 2 + +function sendJson(socket, payload) { + if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify(payload)) +} + +function errorText(error) { + return error?.details || error?.message || '机器人音频服务异常' +} + +function waitForGrpcReady(client, timeoutMs = 5000) { + return new Promise((resolve, reject) => { + client.waitForReady(Date.now() + timeoutMs, (error) => { + if (error) reject(error) + else resolve() + }) + }) +} + +export function attachAudioWebSocket(httpServer) { + const wss = new WebSocketServer({ server: httpServer, path: '/audio' }) + + wss.on('connection', (socket) => { + let initialized = false + let speakerId = '' + let microphoneStream + let speakerStream + let microphoneClient + let speakerClient + let remoteSampleRate = SAMPLE_RATE + // 设为 0，确保首帧一定向浏览器发送真实声道数。 + let remoteChannels = 0 + let hasLoggedRemoteFormat = false + let grpcReady = false + let hasFailed = false + let closing = false + + const closeGrpc = () => { + closing = true + microphoneStream?.cancel() + microphoneStream = undefined + speakerStream?.end() + speakerStream = undefined + microphoneClient?.close() + speakerClient?.close() + microphoneClient = undefined + speakerClient = undefined + } + + const fail = (error) => { + if (hasFailed || closing) return + hasFailed = true + console.error('[audio] gRPC error:', error) + sendJson(socket, { + type: grpcReady ? 'error' : 'connection-failed', + message: grpcReady + ? errorText(error) + : `无法连接机器人音频服务：${errorText(error)}` + }) + closeGrpc() + } + + socket.on('message', (data, isBinary) => { + if (isBinary) { + if (!initialized || !speakerStream || speakerStream.destroyed) return + const pcm = Buffer.from(data) + if (!pcm.length || pcm.length % 2 !== 0) return + speakerStream.write({ + header: { device_id: speakerId }, + audio: { + data: pcm, + sample_rate: SAMPLE_RATE, + channels: CHANNELS, + format: 'PCM', + codec: 'pcm_s16le', + // nb_samples 表示每个声道的采样帧数，不是所有声道样本总数。 + nb_samples: pcm.length / (2 * CHANNELS) + } + }) + return + } + + let message + try { + message = JSON.parse(data.toString()) + } catch { + sendJson(socket, { type: 'error', message: '无效的控制消息' }) + return + } + + if (message.type === 'stop') { + closeGrpc() + socket.close(1000, 'call ended') + return + } + if (message.type !== 'init' || initialized) return + + const { microphoneId, robotAddress } = message + speakerId = message.speakerId?.trim() + if (![microphoneId, speakerId, robotAddress].every((value) => typeof value === 'string' && value.trim())) { + sendJson(socket, { type: 'error', message: '机器人地址、麦克风 ID 和扬声器 ID 不能为空' }) + socket.close(1008, 'invalid init') + return + } + + initialized = true + microphoneClient = createMicrophoneGrpcClient(robotAddress.trim()) + speakerClient = createSpeakerGrpcClient(robotAddress.trim()) + + // 只有两条 gRPC 通道都真正可用后，前端才进入“通话中”状态。 + Promise.all([ + waitForGrpcReady(microphoneClient), + waitForGrpcReady(speakerClient) + ]).then(() => { + if (hasFailed || socket.readyState !== WebSocket.OPEN) return + grpcReady = true + sendJson(socket, { type: 'ready', sampleRate: SAMPLE_RATE, channels: CHANNELS }) + }).catch(fail) + + // 现有 proto 用一条服务端流和一条客户端流共同组成全双工音频通道。 + microphoneStream = microphoneClient.streamAudio({ + header: { device_id: microphoneId.trim() } + }) + speakerStream = speakerClient.streamAudio((error, response) => { + if (error) fail(error) + else if (response?.header?.success === false) { + fail(new Error(response.header.error_message || '机器人扬声器拒绝音频流')) + } + }) + + microphoneStream.on('data', (frame) => { + const audio = frame?.audio + if (!audio?.data?.length || socket.readyState !== WebSocket.OPEN) return + if (!hasLoggedRemoteFormat) { + hasLoggedRemoteFormat = true + } + const frameSampleRate = Number(audio.sample_rate) || SAMPLE_RATE + const frameChannels = Number(audio.channels) || 1 + if (frameSampleRate !== remoteSampleRate || frameChannels !== remoteChannels) { + remoteSampleRate = frameSampleRate + remoteChannels = frameChannels + // WebSocket 保证消息顺序，浏览器会先收到格式通知，再收到对应二进制帧。 + sendJson(socket, { + type: 'audio-format', + sampleRate: remoteSampleRate, + channels: remoteChannels + }) + } + // 浏览器播放跟不上时丢弃新帧，防止内存持续增长。 + if (socket.bufferedAmount <= MAX_BUFFERED_BYTES) socket.send(audio.data, { binary: true }) + }) + microphoneStream.on('error', (error) => { + if (error.code !== 1) fail(error) // CANCELLED(1) 是正常清理 + }) + microphoneStream.on('end', () => sendJson(socket, { type: 'remote-ended' })) + speakerStream.on('error', fail) + }) + + socket.on('close', closeGrpc) + socket.on('error', (error) => console.error('[audio] WebSocket error:', error)) + }) + + return wss +} diff --git a/server/grpcClient.js b/server/grpcClient.js index 82180f9..bd7e998 100644 --- a/server/grpcClient.js +++ b/server/grpcClient.js @@ -11,6 +11,17 @@ export const __dirname = path.dirname(__filename) const PROTO_ROOT = path.resolve(__dirname, './proto') // 指向 server/proto/ const AGV_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/agv_service.proto') const ARM_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/arm_service.proto') +const MICROPHONE_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/microphone_service.proto') +const SPEAKER_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/speaker_service.proto') + +const audioLoaderOptions = { + includeDirs: [PROTO_ROOT, path.join(PROTO_ROOT, 'cmvr/api')], + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true +} const agvPackageDefinition = protoLoader.loadSync( AGV_PROTO_ENTRY, @@ -36,8 +47,13 @@ const armPackageDefinition = protoLoader.loadSync( } ) +const microphonePackageDefinition = protoLoader.loadSync(MICROPHONE_PROTO_ENTRY, audioLoaderOptions) +const speakerPackageDefinition = protoLoader.loadSync(SPEAKER_PROTO_ENTRY, audioLoaderOptions) + const agvProto = grpc.loadPackageDefinition(agvPackageDefinition).cmvr.api const armProto = grpc.loadPackageDefinition(armPackageDefinition).cmvr.api +const microphoneProto = grpc.loadPackageDefinition(microphonePackageDefinition).cmvr.api +const speakerProto = grpc.loadPackageDefinition(speakerPackageDefinition).cmvr.api export const createAGVGrpcClient = (address) => { const grpcClient = new agvProto.AgvService( @@ -53,4 +69,14 @@ export const createARMGrpcClient = (address) => { grpc.credentials.createInsecure() ) return grpcClient -} \ No newline at end of file +} + +export const createMicrophoneGrpcClient = (address) => new microphoneProto.MicPhoneService( + address, + grpc.credentials.createInsecure() +) + +export const createSpeakerGrpcClient = (address) => new speakerProto.SpeakerService( + address, + grpc.credentials.createInsecure() +) diff --git a/server/index.js b/server/index.js index 51173e0..f81409b 100644 --- a/server/index.js +++ b/server/index.js @@ -5,6 +5,7 @@ import path from 'path'; import 'dotenv/config'; import { createAGVGrpcClient, createARMGrpcClient, __dirname } from './grpcClient.js'; +import { attachAudioWebSocket } from './audioWebSocket.js'; const app = express() app.use(express.static(path.join(__dirname, '../dist'))); @@ -157,7 +158,6 @@ app.post('/api/agv/stopRobot', async (req, res) => { // 获取机械臂末端位姿 app.post('/api/arm/getPose', async (req, res) => { - console.log('ssssssss') const request = { header: { device_id: req.body.deviceId || 'huayan_arm' @@ -376,6 +376,8 @@ if (isProd) { // ========== 启动 ========= -app.listen(PORT, () => { +const httpServer = app.listen(PORT, () => { console.log(`gRPC 桥接服务已启动: http://localhost:${PORT}`) -}) \ No newline at end of file +}) + +attachAudioWebSocket(httpServer) diff --git a/src/stores/robot.js b/src/stores/robot.js index 77def62..fdc1d62 100644 --- a/src/stores/robot.js +++ b/src/stores/robot.js @@ -3,9 +3,11 @@ import { defineStore } from 'pinia' // 唯一id：robot export const useRobotStore = defineStore('robot', { state: () => ({ - ip: '192.168.1.110:50052', + ip: '192.168.0.28:50052', agvDeviceId:'src1100', - almDeviceId: 'aubo_arm', + almDeviceId: 'huayan_arm', + spkDeviceId: 'spk1', + micDeviceId: 'mic1', position: { x: 0, y: 0, theta: 0 }, battery: { percentage: 0.99, @@ -35,6 +37,12 @@ export const useRobotStore = defineStore('robot', { setAlmDeviceId(str) { this.almDeviceId = str }, + setSpkDeviceId(str) { + this.spkDeviceId = str + }, + setMicDeviceId(str) { + this.micDeviceId = str + }, // 更新机器人位置 setPosition(data = {}) { this.position = data diff --git a/src/views/Address.vue b/src/views/Address.vue index fb62453..d5693b2 100644 --- a/src/views/Address.vue +++ b/src/views/Address.vue @@ -15,6 +15,12 @@ + + + + + + 进入上位机控制页面 @@ -37,8 +43,10 @@ const loading = ref(false) const ruleFormRef = ref() const ruleForm = ref({ ip: '', - agvDeviceId:'', - almDeviceId: '' + agvDeviceId:'src1100', + almDeviceId: 'huayan_arm', + spkDeviceId: 'spk1', + micDeviceId: 'mic1' }) const rules = ref({ @@ -51,6 +59,12 @@ const rules = ref({ almDeviceId: [ { required: true, message: '请输入机械臂设备ID', trigger: 'blur' } ], + spkDeviceId: [ + { required: true, message: '请输入扬声器设备ID', trigger: 'blur' } + ], + micDeviceId: [ + { required: true, message: '请输入麦克风设备ID', trigger: 'blur' } + ], }) const testConnect = async () => { @@ -60,6 +74,8 @@ const testConnect = async () => { robotStore.setIp(ruleForm.value.ip) robotStore.setAgvDeviceId(ruleForm.value.agvDeviceId) robotStore.setAlmDeviceId(ruleForm.value.almDeviceId) + robotStore.setSpkDeviceId(ruleForm.value.spkDeviceId) + robotStore.setMicDeviceId(ruleForm.value.micDeviceId) // const result = await getRuntimeState({ // ip: robotStore.ip, // deviceId: ruleForm.value.agvDeviceId diff --git a/src/views/camera/VoiceConversation.vue b/src/views/camera/VoiceConversation.vue new file mode 100644 index 0000000..87f9d92 --- /dev/null +++ b/src/views/camera/VoiceConversation.vue @@ -0,0 +1,772 @@ + + + + + diff --git a/src/views/camera/index.vue b/src/views/camera/index.vue index 8dffc60..a18abb2 100644 --- a/src/views/camera/index.vue +++ b/src/views/camera/index.vue @@ -1,15 +1,34 @@