diff --git a/components.d.ts b/components.d.ts
index b668fe7..1121baa 100644
--- a/components.d.ts
+++ b/components.d.ts
@@ -11,11 +11,7 @@ export {}
/* prettier-ignore */
declare module 'vue' {
export interface GlobalComponents {
- ElB: typeof import('element-plus/es')['ElB']
ElButton: typeof import('element-plus/es')['ElButton']
- ElButtonGroup: typeof import('element-plus/es')['ElButtonGroup']
- ElCard: typeof import('element-plus/es')['ElCard']
- ElCheckbox: typeof import('element-plus/es')['ElCheckbox']
ElCol: typeof import('element-plus/es')['ElCol']
ElContainer: typeof import('element-plus/es')['ElContainer']
ElForm: typeof import('element-plus/es')['ElForm']
@@ -23,23 +19,14 @@ declare module 'vue' {
ElHeader: typeof import('element-plus/es')['ElHeader']
ElIcon: typeof import('element-plus/es')['ElIcon']
ElInput: typeof import('element-plus/es')['ElInput']
- ElInputNumber: typeof import('element-plus/es')['ElInputNumber']
ElMain: typeof import('element-plus/es')['ElMain']
- ElMenu: typeof import('element-plus/es')['ElMenu']
- ElMenuItem: typeof import('element-plus/es')['ElMenuItem']
- ElOption: typeof import('element-plus/es')['ElOption']
- ElRadioButton: typeof import('element-plus/es')['ElRadioButton']
- ElRadioGroup: typeof import('element-plus/es')['ElRadioGroup']
ElRow: typeof import('element-plus/es')['ElRow']
- ElSelect: typeof import('element-plus/es')['ElSelect']
ElSlider: typeof import('element-plus/es')['ElSlider']
- ElSubMenu: typeof import('element-plus/es')['ElSubMenu']
ElSwitch: typeof import('element-plus/es')['ElSwitch']
ElTag: typeof import('element-plus/es')['ElTag']
- HelloWorld: typeof import('./src/components/HelloWorld.vue')['default']
IPlayer: typeof import('./src/components/IPlayer/index.vue')['default']
+ LandscapeGuard: typeof import('./src/components/LandscapeGuard.vue')['default']
RouterLink: typeof import('vue-router')['RouterLink']
RouterView: typeof import('vue-router')['RouterView']
- SvgIcon: typeof import('./src/components/SvgIcon/index.vue')['default']
}
}
diff --git a/server/cameraVideoStream.js b/server/cameraVideoStream.js
new file mode 100644
index 0000000..27c279d
--- /dev/null
+++ b/server/cameraVideoStream.js
@@ -0,0 +1,193 @@
+import { spawn } from 'child_process'
+import { createCameraGrpcClient } from './grpcClient.js'
+
+const FIRST_FRAME_TIMEOUT_MS = 10000
+const FRAME_TYPES = ['U8C1', 'U16C1', 'U8C3', 'U16C3', 'F16C1', 'F32C1']
+
+function normalizeCodec(codec = '') {
+ return codec.toLowerCase().replace(/[^a-z0-9]/g, '')
+}
+
+function inputArguments(frame) {
+ const codec = normalizeCodec(frame.codec)
+
+ if (['jpeg', 'jpg', 'mjpeg', 'imagejpeg'].includes(codec)) return ['-f', 'mjpeg']
+ if (['png', 'imagepng'].includes(codec)) return ['-f', 'image2pipe', '-vcodec', 'png']
+ if (['h264', 'avc', 'avc1'].includes(codec)) return ['-f', 'h264']
+ if (['h265', 'hevc', 'hev1', 'hvc1'].includes(codec)) return ['-f', 'hevc']
+
+ const width = Number(frame.width)
+ const height = Number(frame.height)
+ if (!Number.isInteger(width) || width <= 0 || !Number.isInteger(height) || height <= 0) {
+ throw new Error(`未压缩视频帧的尺寸无效: ${frame.width}x${frame.height}`)
+ }
+
+ const declaredPixelFormats = {
+ gray: 'gray',
+ gray8: 'gray',
+ gray16: 'gray16le',
+ gray16le: 'gray16le',
+ rgb24: 'rgb24',
+ bgr24: 'bgr24',
+ rgb48: 'rgb48le',
+ rgb48le: 'rgb48le',
+ bgr48: 'bgr48le',
+ bgr48le: 'bgr48le',
+ grayf32: 'grayf32le',
+ grayf32le: 'grayf32le'
+ }
+ const rawPixelFormats = {
+ U8C1: 'gray',
+ U16C1: 'gray16le',
+ U8C3: 'rgb24',
+ U16C3: 'rgb48le',
+ F32C1: 'grayf32le'
+ }
+ const frameType = typeof frame.type === 'number' ? FRAME_TYPES[frame.type] : frame.type
+ const pixelFormat = declaredPixelFormats[codec] || rawPixelFormats[frameType]
+ if (!pixelFormat) {
+ throw new Error(`暂不支持未压缩帧类型 ${frameType || frame.type},请让相机返回 JPEG/H.264/H.265`)
+ }
+
+ return [
+ '-f', 'rawvideo',
+ '-pixel_format', pixelFormat,
+ '-video_size', `${width}x${height}`,
+ '-framerate', String(Number(frame.source_fps) || 25)
+ ]
+}
+
+function createTranscoder(frame) {
+ const fps = Math.max(1, Number(frame.source_fps) || 25)
+ const ffmpeg = process.env.FFMPEG_PATH || 'ffmpeg'
+ const args = [
+ '-hide_banner', '-loglevel', 'warning',
+ ...inputArguments(frame),
+ '-i', 'pipe:0',
+ '-an',
+ '-c:v', 'libx264',
+ '-preset', 'ultrafast',
+ '-tune', 'zerolatency',
+ '-pix_fmt', 'yuv420p',
+ '-g', String(fps),
+ '-keyint_min', String(fps),
+ '-sc_threshold', '0',
+ '-f', 'mp4',
+ '-movflags', 'frag_keyframe+empty_moov+default_base_moof',
+ 'pipe:1'
+ ]
+ return spawn(ffmpeg, args, { windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] })
+}
+
+function message(error) {
+ return error?.details || error?.message || String(error)
+}
+
+export function streamCameraVideo(req, res) {
+ const ip = typeof req.query.ip === 'string' ? req.query.ip.trim() : ''
+ const deviceId = typeof req.query.deviceId === 'string' ? req.query.deviceId.trim() : ''
+ if (!ip || !deviceId) {
+ res.status(400).json({ code: 400, message: 'ip 和 deviceId 不能为空' })
+ return
+ }
+
+ const client = createCameraGrpcClient(ip)
+ const grpcStream = client.getRgbImageStream()
+ let transcoder
+ let closed = false
+ let responseStarted = false
+ let stderr = ''
+
+ const close = () => {
+ if (closed) return
+ closed = true
+ clearTimeout(firstFrameTimer)
+ grpcStream.cancel()
+ client.close()
+ if (transcoder && !transcoder.killed) {
+ transcoder.stdin.destroy()
+ transcoder.kill()
+ }
+ }
+
+ const fail = (error) => {
+ if (closed) return
+ console.error('[camera] video stream failed:', message(error))
+ if (!responseStarted) {
+ res.status(502).json({ code: 502, message: `相机视频流不可用: ${message(error)}` })
+ } else {
+ res.destroy(error instanceof Error ? error : undefined)
+ }
+ close()
+ }
+
+ const firstFrameTimer = setTimeout(
+ () => fail(new Error('等待相机首帧超时')),
+ FIRST_FRAME_TIMEOUT_MS
+ )
+
+ grpcStream.on('data', (feedback) => {
+ if (closed) return
+ if (feedback?.header?.success === false) {
+ fail(new Error(feedback.header.error_message || '相机拒绝视频流请求'))
+ return
+ }
+
+ const frame = feedback?.color_frame
+ if (!frame?.data?.length) return
+
+ if (!transcoder) {
+ clearTimeout(firstFrameTimer)
+ try {
+ transcoder = createTranscoder(frame)
+ } catch (error) {
+ fail(error)
+ return
+ }
+
+ res.status(200)
+ res.set({
+ 'Content-Type': 'video/mp4',
+ 'Cache-Control': 'no-store, no-cache, must-revalidate',
+ Connection: 'keep-alive',
+ 'X-Accel-Buffering': 'no'
+ })
+ responseStarted = true
+ transcoder.stdout.pipe(res)
+ transcoder.stderr.on('data', (chunk) => {
+ stderr = (stderr + chunk.toString()).slice(-4000)
+ })
+ transcoder.stdin.on('error', (error) => {
+ if (!closed && error.code !== 'EPIPE') fail(error)
+ })
+ transcoder.on('error', fail)
+ transcoder.on('exit', (code, signal) => {
+ if (!closed && code !== 0) {
+ fail(new Error(`FFmpeg 异常退出 (${code ?? signal}): ${stderr.trim()}`))
+ }
+ })
+ }
+
+ if (!transcoder.stdin.write(frame.data)) {
+ grpcStream.pause()
+ transcoder.stdin.once('drain', () => grpcStream.resume())
+ }
+ })
+
+ grpcStream.on('error', (error) => {
+ // CANCELLED 是浏览器断开后主动清理产生的正常状态。
+ if (!closed && error.code !== 1) fail(error)
+ })
+ grpcStream.on('end', () => {
+ if (!closed) {
+ transcoder?.stdin.end()
+ if (!responseStarted) fail(new Error('相机未返回任何视频帧'))
+ }
+ })
+
+ res.on('close', close)
+ grpcStream.write({
+ header: { device_id: deviceId },
+ eof: false
+ })
+}
diff --git a/server/grpcClient.js b/server/grpcClient.js
index bd7e998..b851503 100644
--- a/server/grpcClient.js
+++ b/server/grpcClient.js
@@ -11,6 +11,7 @@ export const __dirname = path.dirname(__filename)
const PROTO_ROOT = path.resolve(__dirname, './proto') // 指向 server/proto/
const AGV_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/agv_service.proto')
const ARM_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/arm_service.proto')
+const CAMERA_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/camera_service.proto')
const MICROPHONE_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/microphone_service.proto')
const SPEAKER_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/speaker_service.proto')
@@ -47,11 +48,13 @@ const armPackageDefinition = protoLoader.loadSync(
}
)
+const cameraPackageDefinition = protoLoader.loadSync(CAMERA_PROTO_ENTRY, audioLoaderOptions)
const microphonePackageDefinition = protoLoader.loadSync(MICROPHONE_PROTO_ENTRY, audioLoaderOptions)
const speakerPackageDefinition = protoLoader.loadSync(SPEAKER_PROTO_ENTRY, audioLoaderOptions)
const agvProto = grpc.loadPackageDefinition(agvPackageDefinition).cmvr.api
const armProto = grpc.loadPackageDefinition(armPackageDefinition).cmvr.api
+const cameraProto = grpc.loadPackageDefinition(cameraPackageDefinition).cmvr.api
const microphoneProto = grpc.loadPackageDefinition(microphonePackageDefinition).cmvr.api
const speakerProto = grpc.loadPackageDefinition(speakerPackageDefinition).cmvr.api
@@ -71,6 +74,11 @@ export const createARMGrpcClient = (address) => {
return grpcClient
}
+export const createCameraGrpcClient = (address) => new cameraProto.CameraService(
+ address,
+ grpc.credentials.createInsecure()
+)
+
export const createMicrophoneGrpcClient = (address) => new microphoneProto.MicPhoneService(
address,
grpc.credentials.createInsecure()
diff --git a/server/index.js b/server/index.js
index bdbba16..7c9e45d 100644
--- a/server/index.js
+++ b/server/index.js
@@ -15,12 +15,17 @@ import {
__dirname
} from './grpcClient.js';
import { attachAudioWebSocket } from './audioWebSocket.js';
+import { streamCameraVideo } from './cameraVideoStream.js';
const app = express()
app.use(express.static(path.join(__dirname, '../dist')));
app.use(cors())
app.use(express.json())
+// 将相机逐帧 gRPC 数据转码为浏览器可直接播放的 fragmented MP4。
+// 示例:/api/camera/video?ip=192.168.0.28%3A50052&deviceId=camera1
+app.get('/api/camera/video', streamCameraVideo)
+
// 获取机器人麦克风音量(0-100)。
app.post('/api/audio/microphone/volume/get', (req, res) => {
const { ip, deviceId } = req.body
diff --git a/server/proto/cmvr/api/camera_command.proto b/server/proto/cmvr/api/camera_command.proto
new file mode 100644
index 0000000..7305955
--- /dev/null
+++ b/server/proto/cmvr/api/camera_command.proto
@@ -0,0 +1,216 @@
+syntax = "proto3";
+
+import "cmvr/api/common.proto";
+
+package cmvr.api;
+
+message FrameData {
+ enum FrameType {
+ U8C1 = 0;
+ U16C1 = 1;
+ U8C3 = 2;
+ U16C3 = 3;
+ F16C1 = 4;
+ F32C1 = 5;
+ }
+ bytes data = 1;
+ int32 width = 2;
+ int32 height = 3;
+ FrameType type = 4;
+ string codec = 5;
+ bool is_key_frame = 6;
+ // Optional capture and source metadata. Fields 1-6 remain wire-compatible
+ // with existing clients; older clients safely ignore these additions.
+ int64 capture_utc_ns = 7;
+ uint64 source_sequence = 8;
+ int64 pts = 9;
+ int64 dts = 10;
+ uint32 source_fps = 11;
+ uint64 source_timestamp = 12;
+ uint64 source_frame_number = 13;
+}
+
+message CameraIntrinsics {
+ float cx = 1; // 主点水平坐标(从左边缘的像素偏移)
+ float cy = 2; // 主点垂直坐标(从上边缘的像素偏移)
+ float fx = 3; // x方向焦距(像素宽度的倍数)
+ float fy = 4; // y方向焦距(像素高度的倍数)
+ repeated float coeffs = 5 [packed = true]; // 畸变系数(固定5个元素)
+}
+
+message CameraState {
+ bool is_initialized = 1;
+ bool is_opened = 2;
+ bool is_streaming = 3;
+ bool is_recording = 4;
+ bool is_error = 5;
+ string error_message = 6;
+ int32 fps = 7;
+ int32 width = 8;
+ int32 height = 9;
+}
+
+message GetCameraStateCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ }
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ CameraState state = 2;
+ }
+}
+
+message StartCameraCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ }
+
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ }
+}
+
+message StopCameraCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ }
+
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ }
+}
+
+message GetRGBImageCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ }
+
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ FrameData color_frame = 2;
+ CameraIntrinsics intrinsics = 3;
+ }
+}
+
+message GetDepthImageCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ }
+
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ FrameData depth_frame = 2;
+ CameraIntrinsics intrinsics = 3;
+ }
+}
+
+message GetRGBDImagesCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ }
+
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ FrameData color_frame = 2;
+ FrameData depth_frame = 3;
+ CameraIntrinsics intrinsics = 4;
+ }
+}
+
+message StartCameraRecordingCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ string video_path = 2;
+ }
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ }
+}
+
+message StopCameraRecordingCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ }
+
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ }
+}
+
+message ControlPtzCommand {
+ enum Command {
+ COMMAND_UNSPECIFIED = 0;
+ TILT_UP = 1;
+ TILT_DOWN = 2;
+ PAN_LEFT = 3;
+ PAN_RIGHT = 4;
+ UP_LEFT = 5;
+ UP_RIGHT = 6;
+ DOWN_LEFT = 7;
+ DOWN_RIGHT = 8;
+ ZOOM_IN = 9;
+ ZOOM_OUT = 10;
+ PAN_AUTO = 11;
+ }
+
+ enum Action {
+ START = 0;
+ STOP = 1;
+ }
+
+ message Request {
+ CommandHeader.Request header = 1;
+ Command command = 2;
+ Action action = 3;
+ uint32 speed = 4;
+ }
+
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ }
+}
+
+message GetRGBImageStreamCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ bool eof = 2;
+ }
+
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ FrameData color_frame = 2;
+ CameraIntrinsics intrinsics = 3;
+ int32 seq_no = 4;
+ }
+}
+
+message GetDepthImageStreamCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ bool eof = 2;
+ }
+
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ FrameData depth_frame = 2;
+ CameraIntrinsics intrinsics = 3;
+ int32 seq_no = 4;
+ }
+}
+
+message GetRGBDImagesStreamCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ bool eof = 2;
+ }
+
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ FrameData color_frame = 2;
+ FrameData depth_frame = 3;
+ CameraIntrinsics intrinsics = 4;
+ int32 seq_no = 5;
+ }
+}
+
+
diff --git a/server/proto/cmvr/api/camera_service.proto b/server/proto/cmvr/api/camera_service.proto
new file mode 100644
index 0000000..9ab7d7f
--- /dev/null
+++ b/server/proto/cmvr/api/camera_service.proto
@@ -0,0 +1,22 @@
+
+syntax = "proto3";
+
+import "cmvr/api/camera_command.proto";
+
+package cmvr.api;
+
+service CameraService {
+ rpc GetStatus(GetCameraStateCommand.Request) returns (GetCameraStateCommand.Feedback) {}
+ rpc StartCamera(StartCameraCommand.Request) returns (StartCameraCommand.Feedback) {}
+ rpc StopCamera(StopCameraCommand.Request) returns (StopCameraCommand.Feedback) {}
+ rpc GetRGBImage(GetRGBImageCommand.Request) returns (GetRGBImageCommand.Feedback) {}
+ rpc GetDepthImage(GetDepthImageCommand.Request) returns (GetDepthImageCommand.Feedback) {}
+ rpc GetRGBDImages(GetRGBDImagesCommand.Request) returns (GetRGBDImagesCommand.Feedback) {}
+ rpc StartRecording(StartCameraRecordingCommand.Request) returns (StartCameraRecordingCommand.Feedback) {}
+ rpc StopRecording(StopCameraRecordingCommand.Request) returns (StopCameraRecordingCommand.Feedback) {}
+ rpc ControlPtz(ControlPtzCommand.Request) returns (ControlPtzCommand.Feedback) {}
+
+ rpc GetRGBImageStream(stream GetRGBImageStreamCommand.Request) returns (stream GetRGBImageStreamCommand.Feedback) {}
+ rpc GetDepthImageStream(stream GetDepthImageStreamCommand.Request) returns (stream GetDepthImageStreamCommand.Feedback) {}
+ rpc GetRGBDImagesStream(stream GetRGBDImagesStreamCommand.Request) returns (stream GetRGBDImagesStreamCommand.Feedback) {}
+}
diff --git a/server/proto/cmvr/api/common.proto b/server/proto/cmvr/api/common.proto
index 53ecd55..5686a18 100644
--- a/server/proto/cmvr/api/common.proto
+++ b/server/proto/cmvr/api/common.proto
@@ -35,6 +35,7 @@ message AudioData {
MP3 = 1;
AAC = 2;
WAV = 3;
+ OPUS = 4;
}
bytes data = 1;
int32 sample_rate = 2;
@@ -57,3 +58,15 @@ message ConfigParam {
bytes bytes_value = 6; // 二进制数据类型
}
}
+
+message JsonDeviceCommand {
+ message Request {
+ CommandHeader.Request header = 1;
+ string request_json = 2;
+ }
+
+ message Feedback {
+ CommandHeader.Feedback header = 1;
+ string response_json = 2;
+ }
+}
diff --git a/src/components/IPlayer/index.vue b/src/components/IPlayer/index.vue
index 6237a2a..561b2db 100644
--- a/src/components/IPlayer/index.vue
+++ b/src/components/IPlayer/index.vue
@@ -5,11 +5,11 @@