From 10a62c51d26ab0b38096aacfb4e23839866b28b6 Mon Sep 17 00:00:00 2001 From: zhanghao <774378400@qq.com> Date: Fri, 31 Jul 2026 10:44:15 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E7=9B=B8=E6=9C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- components.d.ts | 15 +- server/cameraVideoStream.js | 193 ++++++++++++++++++ server/grpcClient.js | 8 + server/index.js | 5 + server/proto/cmvr/api/camera_command.proto | 216 +++++++++++++++++++++ server/proto/cmvr/api/camera_service.proto | 22 +++ server/proto/cmvr/api/common.proto | 13 ++ src/components/IPlayer/index.vue | 19 +- src/stores/robot.js | 6 +- src/views/Address.vue | 10 +- src/views/camera/index.vue | 14 +- 11 files changed, 499 insertions(+), 22 deletions(-) create mode 100644 server/cameraVideoStream.js create mode 100644 server/proto/cmvr/api/camera_command.proto create mode 100644 server/proto/cmvr/api/camera_service.proto diff --git a/components.d.ts b/components.d.ts index b668fe7..1121baa 100644 --- a/components.d.ts +++ b/components.d.ts @@ -11,11 +11,7 @@ export {} /* prettier-ignore */ declare module 'vue' { export interface GlobalComponents { - ElB: typeof import('element-plus/es')['ElB'] ElButton: typeof import('element-plus/es')['ElButton'] - ElButtonGroup: typeof import('element-plus/es')['ElButtonGroup'] - ElCard: typeof import('element-plus/es')['ElCard'] - ElCheckbox: typeof import('element-plus/es')['ElCheckbox'] ElCol: typeof import('element-plus/es')['ElCol'] ElContainer: typeof import('element-plus/es')['ElContainer'] ElForm: typeof import('element-plus/es')['ElForm'] @@ -23,23 +19,14 @@ declare module 'vue' { ElHeader: typeof import('element-plus/es')['ElHeader'] ElIcon: typeof import('element-plus/es')['ElIcon'] ElInput: typeof import('element-plus/es')['ElInput'] - ElInputNumber: typeof import('element-plus/es')['ElInputNumber'] ElMain: typeof import('element-plus/es')['ElMain'] - ElMenu: typeof import('element-plus/es')['ElMenu'] - ElMenuItem: typeof import('element-plus/es')['ElMenuItem'] - ElOption: typeof import('element-plus/es')['ElOption'] - ElRadioButton: typeof import('element-plus/es')['ElRadioButton'] - ElRadioGroup: typeof import('element-plus/es')['ElRadioGroup'] ElRow: typeof import('element-plus/es')['ElRow'] - ElSelect: typeof import('element-plus/es')['ElSelect'] ElSlider: typeof import('element-plus/es')['ElSlider'] - ElSubMenu: typeof import('element-plus/es')['ElSubMenu'] ElSwitch: typeof import('element-plus/es')['ElSwitch'] ElTag: typeof import('element-plus/es')['ElTag'] - HelloWorld: typeof import('./src/components/HelloWorld.vue')['default'] IPlayer: typeof import('./src/components/IPlayer/index.vue')['default'] + LandscapeGuard: typeof import('./src/components/LandscapeGuard.vue')['default'] RouterLink: typeof import('vue-router')['RouterLink'] RouterView: typeof import('vue-router')['RouterView'] - SvgIcon: typeof import('./src/components/SvgIcon/index.vue')['default'] } } diff --git a/server/cameraVideoStream.js b/server/cameraVideoStream.js new file mode 100644 index 0000000..27c279d --- /dev/null +++ b/server/cameraVideoStream.js @@ -0,0 +1,193 @@ +import { spawn } from 'child_process' +import { createCameraGrpcClient } from './grpcClient.js' + +const FIRST_FRAME_TIMEOUT_MS = 10000 +const FRAME_TYPES = ['U8C1', 'U16C1', 'U8C3', 'U16C3', 'F16C1', 'F32C1'] + +function normalizeCodec(codec = '') { + return codec.toLowerCase().replace(/[^a-z0-9]/g, '') +} + +function inputArguments(frame) { + const codec = normalizeCodec(frame.codec) + + if (['jpeg', 'jpg', 'mjpeg', 'imagejpeg'].includes(codec)) return ['-f', 'mjpeg'] + if (['png', 'imagepng'].includes(codec)) return ['-f', 'image2pipe', '-vcodec', 'png'] + if (['h264', 'avc', 'avc1'].includes(codec)) return ['-f', 'h264'] + if (['h265', 'hevc', 'hev1', 'hvc1'].includes(codec)) return ['-f', 'hevc'] + + const width = Number(frame.width) + const height = Number(frame.height) + if (!Number.isInteger(width) || width <= 0 || !Number.isInteger(height) || height <= 0) { + throw new Error(`未压缩视频帧的尺寸无效: ${frame.width}x${frame.height}`) + } + + const declaredPixelFormats = { + gray: 'gray', + gray8: 'gray', + gray16: 'gray16le', + gray16le: 'gray16le', + rgb24: 'rgb24', + bgr24: 'bgr24', + rgb48: 'rgb48le', + rgb48le: 'rgb48le', + bgr48: 'bgr48le', + bgr48le: 'bgr48le', + grayf32: 'grayf32le', + grayf32le: 'grayf32le' + } + const rawPixelFormats = { + U8C1: 'gray', + U16C1: 'gray16le', + U8C3: 'rgb24', + U16C3: 'rgb48le', + F32C1: 'grayf32le' + } + const frameType = typeof frame.type === 'number' ? FRAME_TYPES[frame.type] : frame.type + const pixelFormat = declaredPixelFormats[codec] || rawPixelFormats[frameType] + if (!pixelFormat) { + throw new Error(`暂不支持未压缩帧类型 ${frameType || frame.type}，请让相机返回 JPEG/H.264/H.265`) + } + + return [ + '-f', 'rawvideo', + '-pixel_format', pixelFormat, + '-video_size', `${width}x${height}`, + '-framerate', String(Number(frame.source_fps) || 25) + ] +} + +function createTranscoder(frame) { + const fps = Math.max(1, Number(frame.source_fps) || 25) + const ffmpeg = process.env.FFMPEG_PATH || 'ffmpeg' + const args = [ + '-hide_banner', '-loglevel', 'warning', + ...inputArguments(frame), + '-i', 'pipe:0', + '-an', + '-c:v', 'libx264', + '-preset', 'ultrafast', + '-tune', 'zerolatency', + '-pix_fmt', 'yuv420p', + '-g', String(fps), + '-keyint_min', String(fps), + '-sc_threshold', '0', + '-f', 'mp4', + '-movflags', 'frag_keyframe+empty_moov+default_base_moof', + 'pipe:1' + ] + return spawn(ffmpeg, args, { windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] }) +} + +function message(error) { + return error?.details || error?.message || String(error) +} + +export function streamCameraVideo(req, res) { + const ip = typeof req.query.ip === 'string' ? req.query.ip.trim() : '' + const deviceId = typeof req.query.deviceId === 'string' ? req.query.deviceId.trim() : '' + if (!ip || !deviceId) { + res.status(400).json({ code: 400, message: 'ip 和 deviceId 不能为空' }) + return + } + + const client = createCameraGrpcClient(ip) + const grpcStream = client.getRgbImageStream() + let transcoder + let closed = false + let responseStarted = false + let stderr = '' + + const close = () => { + if (closed) return + closed = true + clearTimeout(firstFrameTimer) + grpcStream.cancel() + client.close() + if (transcoder && !transcoder.killed) { + transcoder.stdin.destroy() + transcoder.kill() + } + } + + const fail = (error) => { + if (closed) return + console.error('[camera] video stream failed:', message(error)) + if (!responseStarted) { + res.status(502).json({ code: 502, message: `相机视频流不可用: ${message(error)}` }) + } else { + res.destroy(error instanceof Error ? error : undefined) + } + close() + } + + const firstFrameTimer = setTimeout( + () => fail(new Error('等待相机首帧超时')), + FIRST_FRAME_TIMEOUT_MS + ) + + grpcStream.on('data', (feedback) => { + if (closed) return + if (feedback?.header?.success === false) { + fail(new Error(feedback.header.error_message || '相机拒绝视频流请求')) + return + } + + const frame = feedback?.color_frame + if (!frame?.data?.length) return + + if (!transcoder) { + clearTimeout(firstFrameTimer) + try { + transcoder = createTranscoder(frame) + } catch (error) { + fail(error) + return + } + + res.status(200) + res.set({ + 'Content-Type': 'video/mp4', + 'Cache-Control': 'no-store, no-cache, must-revalidate', + Connection: 'keep-alive', + 'X-Accel-Buffering': 'no' + }) + responseStarted = true + transcoder.stdout.pipe(res) + transcoder.stderr.on('data', (chunk) => { + stderr = (stderr + chunk.toString()).slice(-4000) + }) + transcoder.stdin.on('error', (error) => { + if (!closed && error.code !== 'EPIPE') fail(error) + }) + transcoder.on('error', fail) + transcoder.on('exit', (code, signal) => { + if (!closed && code !== 0) { + fail(new Error(`FFmpeg 异常退出 (${code ?? signal}): ${stderr.trim()}`)) + } + }) + } + + if (!transcoder.stdin.write(frame.data)) { + grpcStream.pause() + transcoder.stdin.once('drain', () => grpcStream.resume()) + } + }) + + grpcStream.on('error', (error) => { + // CANCELLED 是浏览器断开后主动清理产生的正常状态。 + if (!closed && error.code !== 1) fail(error) + }) + grpcStream.on('end', () => { + if (!closed) { + transcoder?.stdin.end() + if (!responseStarted) fail(new Error('相机未返回任何视频帧')) + } + }) + + res.on('close', close) + grpcStream.write({ + header: { device_id: deviceId }, + eof: false + }) +} diff --git a/server/grpcClient.js b/server/grpcClient.js index bd7e998..b851503 100644 --- a/server/grpcClient.js +++ b/server/grpcClient.js @@ -11,6 +11,7 @@ export const __dirname = path.dirname(__filename) const PROTO_ROOT = path.resolve(__dirname, './proto') // 指向 server/proto/ const AGV_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/agv_service.proto') const ARM_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/arm_service.proto') +const CAMERA_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/camera_service.proto') const MICROPHONE_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/microphone_service.proto') const SPEAKER_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/speaker_service.proto') @@ -47,11 +48,13 @@ const armPackageDefinition = protoLoader.loadSync( } ) +const cameraPackageDefinition = protoLoader.loadSync(CAMERA_PROTO_ENTRY, audioLoaderOptions) const microphonePackageDefinition = protoLoader.loadSync(MICROPHONE_PROTO_ENTRY, audioLoaderOptions) const speakerPackageDefinition = protoLoader.loadSync(SPEAKER_PROTO_ENTRY, audioLoaderOptions) const agvProto = grpc.loadPackageDefinition(agvPackageDefinition).cmvr.api const armProto = grpc.loadPackageDefinition(armPackageDefinition).cmvr.api +const cameraProto = grpc.loadPackageDefinition(cameraPackageDefinition).cmvr.api const microphoneProto = grpc.loadPackageDefinition(microphonePackageDefinition).cmvr.api const speakerProto = grpc.loadPackageDefinition(speakerPackageDefinition).cmvr.api @@ -71,6 +74,11 @@ export const createARMGrpcClient = (address) => { return grpcClient } +export const createCameraGrpcClient = (address) => new cameraProto.CameraService( + address, + grpc.credentials.createInsecure() +) + export const createMicrophoneGrpcClient = (address) => new microphoneProto.MicPhoneService( address, grpc.credentials.createInsecure() diff --git a/server/index.js b/server/index.js index bdbba16..7c9e45d 100644 --- a/server/index.js +++ b/server/index.js @@ -15,12 +15,17 @@ import { __dirname } from './grpcClient.js'; import { attachAudioWebSocket } from './audioWebSocket.js'; +import { streamCameraVideo } from './cameraVideoStream.js'; const app = express() app.use(express.static(path.join(__dirname, '../dist'))); app.use(cors()) app.use(express.json()) +// 将相机逐帧 gRPC 数据转码为浏览器可直接播放的 fragmented MP4。 +// 示例：/api/camera/video?ip=192.168.0.28%3A50052&deviceId=camera1 +app.get('/api/camera/video', streamCameraVideo) + // 获取机器人麦克风音量（0-100）。 app.post('/api/audio/microphone/volume/get', (req, res) => { const { ip, deviceId } = req.body diff --git a/server/proto/cmvr/api/camera_command.proto b/server/proto/cmvr/api/camera_command.proto new file mode 100644 index 0000000..7305955 --- /dev/null +++ b/server/proto/cmvr/api/camera_command.proto @@ -0,0 +1,216 @@ +syntax = "proto3"; + +import "cmvr/api/common.proto"; + +package cmvr.api; + +message FrameData { + enum FrameType { + U8C1 = 0; + U16C1 = 1; + U8C3 = 2; + U16C3 = 3; + F16C1 = 4; + F32C1 = 5; + } + bytes data = 1; + int32 width = 2; + int32 height = 3; + FrameType type = 4; + string codec = 5; + bool is_key_frame = 6; + // Optional capture and source metadata. Fields 1-6 remain wire-compatible + // with existing clients; older clients safely ignore these additions. + int64 capture_utc_ns = 7; + uint64 source_sequence = 8; + int64 pts = 9; + int64 dts = 10; + uint32 source_fps = 11; + uint64 source_timestamp = 12; + uint64 source_frame_number = 13; +} + +message CameraIntrinsics { + float cx = 1; // 主点水平坐标（从左边缘的像素偏移） + float cy = 2; // 主点垂直坐标（从上边缘的像素偏移） + float fx = 3; // x方向焦距（像素宽度的倍数） + float fy = 4; // y方向焦距（像素高度的倍数） + repeated float coeffs = 5 [packed = true]; // 畸变系数（固定5个元素） +} + +message CameraState { + bool is_initialized = 1; + bool is_opened = 2; + bool is_streaming = 3; + bool is_recording = 4; + bool is_error = 5; + string error_message = 6; + int32 fps = 7; + int32 width = 8; + int32 height = 9; +} + +message GetCameraStateCommand { + message Request { + CommandHeader.Request header = 1; + } + message Feedback { + CommandHeader.Feedback header = 1; + CameraState state = 2; + } +} + +message StartCameraCommand { + message Request { + CommandHeader.Request header = 1; + } + + message Feedback { + CommandHeader.Feedback header = 1; + } +} + +message StopCameraCommand { + message Request { + CommandHeader.Request header = 1; + } + + message Feedback { + CommandHeader.Feedback header = 1; + } +} + +message GetRGBImageCommand { + message Request { + CommandHeader.Request header = 1; + } + + message Feedback { + CommandHeader.Feedback header = 1; + FrameData color_frame = 2; + CameraIntrinsics intrinsics = 3; + } +} + +message GetDepthImageCommand { + message Request { + CommandHeader.Request header = 1; + } + + message Feedback { + CommandHeader.Feedback header = 1; + FrameData depth_frame = 2; + CameraIntrinsics intrinsics = 3; + } +} + +message GetRGBDImagesCommand { + message Request { + CommandHeader.Request header = 1; + } + + message Feedback { + CommandHeader.Feedback header = 1; + FrameData color_frame = 2; + FrameData depth_frame = 3; + CameraIntrinsics intrinsics = 4; + } +} + +message StartCameraRecordingCommand { + message Request { + CommandHeader.Request header = 1; + string video_path = 2; + } + message Feedback { + CommandHeader.Feedback header = 1; + } +} + +message StopCameraRecordingCommand { + message Request { + CommandHeader.Request header = 1; + } + + message Feedback { + CommandHeader.Feedback header = 1; + } +} + +message ControlPtzCommand { + enum Command { + COMMAND_UNSPECIFIED = 0; + TILT_UP = 1; + TILT_DOWN = 2; + PAN_LEFT = 3; + PAN_RIGHT = 4; + UP_LEFT = 5; + UP_RIGHT = 6; + DOWN_LEFT = 7; + DOWN_RIGHT = 8; + ZOOM_IN = 9; + ZOOM_OUT = 10; + PAN_AUTO = 11; + } + + enum Action { + START = 0; + STOP = 1; + } + + message Request { + CommandHeader.Request header = 1; + Command command = 2; + Action action = 3; + uint32 speed = 4; + } + + message Feedback { + CommandHeader.Feedback header = 1; + } +} + +message GetRGBImageStreamCommand { + message Request { + CommandHeader.Request header = 1; + bool eof = 2; + } + + message Feedback { + CommandHeader.Feedback header = 1; + FrameData color_frame = 2; + CameraIntrinsics intrinsics = 3; + int32 seq_no = 4; + } +} + +message GetDepthImageStreamCommand { + message Request { + CommandHeader.Request header = 1; + bool eof = 2; + } + + message Feedback { + CommandHeader.Feedback header = 1; + FrameData depth_frame = 2; + CameraIntrinsics intrinsics = 3; + int32 seq_no = 4; + } +} + +message GetRGBDImagesStreamCommand { + message Request { + CommandHeader.Request header = 1; + bool eof = 2; + } + + message Feedback { + CommandHeader.Feedback header = 1; + FrameData color_frame = 2; + FrameData depth_frame = 3; + CameraIntrinsics intrinsics = 4; + int32 seq_no = 5; + } +} + + diff --git a/server/proto/cmvr/api/camera_service.proto b/server/proto/cmvr/api/camera_service.proto new file mode 100644 index 0000000..9ab7d7f --- /dev/null +++ b/server/proto/cmvr/api/camera_service.proto @@ -0,0 +1,22 @@ + +syntax = "proto3"; + +import "cmvr/api/camera_command.proto"; + +package cmvr.api; + +service CameraService { + rpc GetStatus(GetCameraStateCommand.Request) returns (GetCameraStateCommand.Feedback) {} + rpc StartCamera(StartCameraCommand.Request) returns (StartCameraCommand.Feedback) {} + rpc StopCamera(StopCameraCommand.Request) returns (StopCameraCommand.Feedback) {} + rpc GetRGBImage(GetRGBImageCommand.Request) returns (GetRGBImageCommand.Feedback) {} + rpc GetDepthImage(GetDepthImageCommand.Request) returns (GetDepthImageCommand.Feedback) {} + rpc GetRGBDImages(GetRGBDImagesCommand.Request) returns (GetRGBDImagesCommand.Feedback) {} + rpc StartRecording(StartCameraRecordingCommand.Request) returns (StartCameraRecordingCommand.Feedback) {} + rpc StopRecording(StopCameraRecordingCommand.Request) returns (StopCameraRecordingCommand.Feedback) {} + rpc ControlPtz(ControlPtzCommand.Request) returns (ControlPtzCommand.Feedback) {} + + rpc GetRGBImageStream(stream GetRGBImageStreamCommand.Request) returns (stream GetRGBImageStreamCommand.Feedback) {} + rpc GetDepthImageStream(stream GetDepthImageStreamCommand.Request) returns (stream GetDepthImageStreamCommand.Feedback) {} + rpc GetRGBDImagesStream(stream GetRGBDImagesStreamCommand.Request) returns (stream GetRGBDImagesStreamCommand.Feedback) {} +} diff --git a/server/proto/cmvr/api/common.proto b/server/proto/cmvr/api/common.proto index 53ecd55..5686a18 100644 --- a/server/proto/cmvr/api/common.proto +++ b/server/proto/cmvr/api/common.proto @@ -35,6 +35,7 @@ message AudioData { MP3 = 1; AAC = 2; WAV = 3; + OPUS = 4; } bytes data = 1; int32 sample_rate = 2; @@ -57,3 +58,15 @@ message ConfigParam { bytes bytes_value = 6; // 二进制数据类型 } } + +message JsonDeviceCommand { + message Request { + CommandHeader.Request header = 1; + string request_json = 2; + } + + message Feedback { + CommandHeader.Feedback header = 1; + string response_json = 2; + } +} diff --git a/src/components/IPlayer/index.vue b/src/components/IPlayer/index.vue index 6237a2a..561b2db 100644 --- a/src/components/IPlayer/index.vue +++ b/src/components/IPlayer/index.vue @@ -5,11 +5,11 @@