SmartBench/inspection-host-computer
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 相机

Browse Source
This commit is contained in:
zhanghao 2026-07-31 10:44:15 +08:00
parent c09f05ac8d
commit 10a62c51d2
11 changed files with 499 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
components.d.ts vendored
View File

@ -11,11 +11,7 @@ export {}
/* prettier-ignore */
declare module 'vue' {
export interface GlobalComponents {
ElB: typeof import('element-plus/es')['ElB']
ElButton: typeof import('element-plus/es')['ElButton']
ElButtonGroup: typeof import('element-plus/es')['ElButtonGroup']
ElCard: typeof import('element-plus/es')['ElCard']
ElCheckbox: typeof import('element-plus/es')['ElCheckbox']
ElCol: typeof import('element-plus/es')['ElCol']
ElContainer: typeof import('element-plus/es')['ElContainer']
ElForm: typeof import('element-plus/es')['ElForm']
@ -23,23 +19,14 @@ declare module 'vue' {
ElHeader: typeof import('element-plus/es')['ElHeader']
ElIcon: typeof import('element-plus/es')['ElIcon']
ElInput: typeof import('element-plus/es')['ElInput']
ElInputNumber: typeof import('element-plus/es')['ElInputNumber']
ElMain: typeof import('element-plus/es')['ElMain']
ElMenu: typeof import('element-plus/es')['ElMenu']
ElMenuItem: typeof import('element-plus/es')['ElMenuItem']
ElOption: typeof import('element-plus/es')['ElOption']
ElRadioButton: typeof import('element-plus/es')['ElRadioButton']
ElRadioGroup: typeof import('element-plus/es')['ElRadioGroup']
ElRow: typeof import('element-plus/es')['ElRow']
ElSelect: typeof import('element-plus/es')['ElSelect']
ElSlider: typeof import('element-plus/es')['ElSlider']
ElSubMenu: typeof import('element-plus/es')['ElSubMenu']
ElSwitch: typeof import('element-plus/es')['ElSwitch']
ElTag: typeof import('element-plus/es')['ElTag']
HelloWorld: typeof import('./src/components/HelloWorld.vue')['default']
IPlayer: typeof import('./src/components/IPlayer/index.vue')['default']
LandscapeGuard: typeof import('./src/components/LandscapeGuard.vue')['default']
RouterLink: typeof import('vue-router')['RouterLink']
RouterView: typeof import('vue-router')['RouterView']
SvgIcon: typeof import('./src/components/SvgIcon/index.vue')['default']
}
}

193
server/cameraVideoStream.js Normal file
View File

@ -0,0 +1,193 @@
import { spawn } from 'child_process'
import { createCameraGrpcClient } from './grpcClient.js'
const FIRST_FRAME_TIMEOUT_MS = 10000
const FRAME_TYPES = ['U8C1', 'U16C1', 'U8C3', 'U16C3', 'F16C1', 'F32C1']
function normalizeCodec(codec = '') {
return codec.toLowerCase().replace(/[^a-z0-9]/g, '')
}
function inputArguments(frame) {
const codec = normalizeCodec(frame.codec)
if (['jpeg', 'jpg', 'mjpeg', 'imagejpeg'].includes(codec)) return ['-f', 'mjpeg']
if (['png', 'imagepng'].includes(codec)) return ['-f', 'image2pipe', '-vcodec', 'png']
if (['h264', 'avc', 'avc1'].includes(codec)) return ['-f', 'h264']
if (['h265', 'hevc', 'hev1', 'hvc1'].includes(codec)) return ['-f', 'hevc']
const width = Number(frame.width)
const height = Number(frame.height)
if (!Number.isInteger(width) || width <= 0 || !Number.isInteger(height) || height <= 0) {
throw new Error(`未压缩视频帧的尺寸无效: ${frame.width}x${frame.height}`)
}
const declaredPixelFormats = {
gray: 'gray',
gray8: 'gray',
gray16: 'gray16le',
gray16le: 'gray16le',
rgb24: 'rgb24',
bgr24: 'bgr24',
rgb48: 'rgb48le',
rgb48le: 'rgb48le',
bgr48: 'bgr48le',
bgr48le: 'bgr48le',
grayf32: 'grayf32le',
grayf32le: 'grayf32le'
}
const rawPixelFormats = {
U8C1: 'gray',
U16C1: 'gray16le',
U8C3: 'rgb24',
U16C3: 'rgb48le',
F32C1: 'grayf32le'
}
const frameType = typeof frame.type === 'number' ? FRAME_TYPES[frame.type] : frame.type
const pixelFormat = declaredPixelFormats[codec] || rawPixelFormats[frameType]
if (!pixelFormat) {
throw new Error(`暂不支持未压缩帧类型 ${frameType || frame.type}，请让相机返回 JPEG/H.264/H.265`)
}
return [
'-f', 'rawvideo',
'-pixel_format', pixelFormat,
'-video_size', `${width}x${height}`,
'-framerate', String(Number(frame.source_fps) || 25)
]
}
function createTranscoder(frame) {
const fps = Math.max(1, Number(frame.source_fps) || 25)
const ffmpeg = process.env.FFMPEG_PATH || 'ffmpeg'
const args = [
'-hide_banner', '-loglevel', 'warning',
...inputArguments(frame),
'-i', 'pipe:0',
'-an',
'-c:v', 'libx264',
'-preset', 'ultrafast',
'-tune', 'zerolatency',
'-pix_fmt', 'yuv420p',
'-g', String(fps),
'-keyint_min', String(fps),
'-sc_threshold', '0',
'-f', 'mp4',
'-movflags', 'frag_keyframe+empty_moov+default_base_moof',
'pipe:1'
]
return spawn(ffmpeg, args, { windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] })
}
function message(error) {
return error?.details || error?.message || String(error)
}
export function streamCameraVideo(req, res) {
const ip = typeof req.query.ip === 'string' ? req.query.ip.trim() : ''
const deviceId = typeof req.query.deviceId === 'string' ? req.query.deviceId.trim() : ''
if (!ip || !deviceId) {
res.status(400).json({ code: 400, message: 'ip 和 deviceId 不能为空' })
return
}
const client = createCameraGrpcClient(ip)
const grpcStream = client.getRgbImageStream()
let transcoder
let closed = false
let responseStarted = false
let stderr = ''
const close = () => {
if (closed) return
closed = true
clearTimeout(firstFrameTimer)
grpcStream.cancel()
client.close()
if (transcoder && !transcoder.killed) {
transcoder.stdin.destroy()
transcoder.kill()
}
}
const fail = (error) => {
if (closed) return
console.error('[camera] video stream failed:', message(error))
if (!responseStarted) {
res.status(502).json({ code: 502, message: `相机视频流不可用: ${message(error)}` })
} else {
res.destroy(error instanceof Error ? error : undefined)
}
close()
}
const firstFrameTimer = setTimeout(
() => fail(new Error('等待相机首帧超时')),
FIRST_FRAME_TIMEOUT_MS
)
grpcStream.on('data', (feedback) => {
if (closed) return
if (feedback?.header?.success === false) {
fail(new Error(feedback.header.error_message || '相机拒绝视频流请求'))
return
}
const frame = feedback?.color_frame
if (!frame?.data?.length) return
if (!transcoder) {
clearTimeout(firstFrameTimer)
try {
transcoder = createTranscoder(frame)
} catch (error) {
fail(error)
return
}
res.status(200)
res.set({
'Content-Type': 'video/mp4',
'Cache-Control': 'no-store, no-cache, must-revalidate',
Connection: 'keep-alive',
'X-Accel-Buffering': 'no'
})
responseStarted = true
transcoder.stdout.pipe(res)
transcoder.stderr.on('data', (chunk) => {
stderr = (stderr + chunk.toString()).slice(-4000)
})
transcoder.stdin.on('error', (error) => {
if (!closed && error.code !== 'EPIPE') fail(error)
})
transcoder.on('error', fail)
transcoder.on('exit', (code, signal) => {
if (!closed && code !== 0) {
fail(new Error(`FFmpeg 异常退出 (${code ?? signal}): ${stderr.trim()}`))
}
})
}
if (!transcoder.stdin.write(frame.data)) {
grpcStream.pause()
transcoder.stdin.once('drain', () => grpcStream.resume())
}
})
grpcStream.on('error', (error) => {
// CANCELLED 是浏览器断开后主动清理产生的正常状态。
if (!closed && error.code !== 1) fail(error)
})
grpcStream.on('end', () => {
if (!closed) {
transcoder?.stdin.end()
if (!responseStarted) fail(new Error('相机未返回任何视频帧'))
}
})
res.on('close', close)
grpcStream.write({
header: { device_id: deviceId },
eof: false
})
}

8
server/grpcClient.js
View File

@ -11,6 +11,7 @@ export const __dirname = path.dirname(__filename)
const PROTO_ROOT = path.resolve(__dirname, './proto') // 指向 server/proto/
const AGV_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/agv_service.proto')
const ARM_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/arm_service.proto')
const CAMERA_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/camera_service.proto')
const MICROPHONE_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/microphone_service.proto')
const SPEAKER_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/speaker_service.proto')
@ -47,11 +48,13 @@ const armPackageDefinition = protoLoader.loadSync(
}
)
const cameraPackageDefinition = protoLoader.loadSync(CAMERA_PROTO_ENTRY, audioLoaderOptions)
const microphonePackageDefinition = protoLoader.loadSync(MICROPHONE_PROTO_ENTRY, audioLoaderOptions)
const speakerPackageDefinition = protoLoader.loadSync(SPEAKER_PROTO_ENTRY, audioLoaderOptions)
const agvProto = grpc.loadPackageDefinition(agvPackageDefinition).cmvr.api
const armProto = grpc.loadPackageDefinition(armPackageDefinition).cmvr.api
const cameraProto = grpc.loadPackageDefinition(cameraPackageDefinition).cmvr.api
const microphoneProto = grpc.loadPackageDefinition(microphonePackageDefinition).cmvr.api
const speakerProto = grpc.loadPackageDefinition(speakerPackageDefinition).cmvr.api
@ -71,6 +74,11 @@ export const createARMGrpcClient = (address) => {
return grpcClient
}
export const createCameraGrpcClient = (address) => new cameraProto.CameraService(
address,
grpc.credentials.createInsecure()
)
export const createMicrophoneGrpcClient = (address) => new microphoneProto.MicPhoneService(
address,
grpc.credentials.createInsecure()

5
server/index.js
View File

@ -15,12 +15,17 @@ import {
__dirname
} from './grpcClient.js';
import { attachAudioWebSocket } from './audioWebSocket.js';
import { streamCameraVideo } from './cameraVideoStream.js';
const app = express()
app.use(express.static(path.join(__dirname, '../dist')));
app.use(cors())
app.use(express.json())
// 将相机逐帧 gRPC 数据转码为浏览器可直接播放的 fragmented MP4。
// 示例：/api/camera/video?ip=192.168.0.28%3A50052&deviceId=camera1
app.get('/api/camera/video', streamCameraVideo)
// 获取机器人麦克风音量（0-100）。
app.post('/api/audio/microphone/volume/get', (req, res) => {
const { ip, deviceId } = req.body

216
server/proto/cmvr/api/camera_command.proto Normal file
View File

@ -0,0 +1,216 @@
syntax = "proto3";
import "cmvr/api/common.proto";
package cmvr.api;
message FrameData {
enum FrameType {
U8C1 = 0;
U16C1 = 1;
U8C3 = 2;
U16C3 = 3;
F16C1 = 4;
F32C1 = 5;
}
bytes data = 1;
int32 width = 2;
int32 height = 3;
FrameType type = 4;
string codec = 5;
bool is_key_frame = 6;
// Optional capture and source metadata. Fields 1-6 remain wire-compatible
// with existing clients; older clients safely ignore these additions.
int64 capture_utc_ns = 7;
uint64 source_sequence = 8;
int64 pts = 9;
int64 dts = 10;
uint32 source_fps = 11;
uint64 source_timestamp = 12;
uint64 source_frame_number = 13;
}
message CameraIntrinsics {
float cx = 1; // 主点水平坐标（从左边缘的像素偏移）
float cy = 2; // 主点垂直坐标（从上边缘的像素偏移）
float fx = 3; // x方向焦距（像素宽度的倍数）
float fy = 4; // y方向焦距（像素高度的倍数）
repeated float coeffs = 5 [packed = true]; // 畸变系数（固定5个元素）
}
message CameraState {
bool is_initialized = 1;
bool is_opened = 2;
bool is_streaming = 3;
bool is_recording = 4;
bool is_error = 5;
string error_message = 6;
int32 fps = 7;
int32 width = 8;
int32 height = 9;
}
message GetCameraStateCommand {
message Request {
CommandHeader.Request header = 1;
}
message Feedback {
CommandHeader.Feedback header = 1;
CameraState state = 2;
}
}
message StartCameraCommand {
message Request {
CommandHeader.Request header = 1;
}
message Feedback {
CommandHeader.Feedback header = 1;
}
}
message StopCameraCommand {
message Request {
CommandHeader.Request header = 1;
}
message Feedback {
CommandHeader.Feedback header = 1;
}
}
message GetRGBImageCommand {
message Request {
CommandHeader.Request header = 1;
}
message Feedback {
CommandHeader.Feedback header = 1;
FrameData color_frame = 2;
CameraIntrinsics intrinsics = 3;
}
}
message GetDepthImageCommand {
message Request {
CommandHeader.Request header = 1;
}
message Feedback {
CommandHeader.Feedback header = 1;
FrameData depth_frame = 2;
CameraIntrinsics intrinsics = 3;
}
}
message GetRGBDImagesCommand {
message Request {
CommandHeader.Request header = 1;
}
message Feedback {
CommandHeader.Feedback header = 1;
FrameData color_frame = 2;
FrameData depth_frame = 3;
CameraIntrinsics intrinsics = 4;
}
}
message StartCameraRecordingCommand {
message Request {
CommandHeader.Request header = 1;
string video_path = 2;
}
message Feedback {
CommandHeader.Feedback header = 1;
}
}
message StopCameraRecordingCommand {
message Request {
CommandHeader.Request header = 1;
}
message Feedback {
CommandHeader.Feedback header = 1;
}
}
message ControlPtzCommand {
enum Command {
COMMAND_UNSPECIFIED = 0;
TILT_UP = 1;
TILT_DOWN = 2;
PAN_LEFT = 3;
PAN_RIGHT = 4;
UP_LEFT = 5;
UP_RIGHT = 6;
DOWN_LEFT = 7;
DOWN_RIGHT = 8;
ZOOM_IN = 9;
ZOOM_OUT = 10;
PAN_AUTO = 11;
}
enum Action {
START = 0;
STOP = 1;
}
message Request {
CommandHeader.Request header = 1;
Command command = 2;
Action action = 3;
uint32 speed = 4;
}
message Feedback {
CommandHeader.Feedback header = 1;
}
}
message GetRGBImageStreamCommand {
message Request {
CommandHeader.Request header = 1;
bool eof = 2;
}
message Feedback {
CommandHeader.Feedback header = 1;
FrameData color_frame = 2;
CameraIntrinsics intrinsics = 3;
int32 seq_no = 4;
}
}
message GetDepthImageStreamCommand {
message Request {
CommandHeader.Request header = 1;
bool eof = 2;
}
message Feedback {
CommandHeader.Feedback header = 1;
FrameData depth_frame = 2;
CameraIntrinsics intrinsics = 3;
int32 seq_no = 4;
}
}
message GetRGBDImagesStreamCommand {
message Request {
CommandHeader.Request header = 1;
bool eof = 2;
}
message Feedback {
CommandHeader.Feedback header = 1;
FrameData color_frame = 2;
FrameData depth_frame = 3;
CameraIntrinsics intrinsics = 4;
int32 seq_no = 5;
}
}

22
server/proto/cmvr/api/camera_service.proto Normal file
View File

@ -0,0 +1,22 @@
syntax = "proto3";
import "cmvr/api/camera_command.proto";
package cmvr.api;
service CameraService {
rpc GetStatus(GetCameraStateCommand.Request) returns (GetCameraStateCommand.Feedback) {}
rpc StartCamera(StartCameraCommand.Request) returns (StartCameraCommand.Feedback) {}
rpc StopCamera(StopCameraCommand.Request) returns (StopCameraCommand.Feedback) {}
rpc GetRGBImage(GetRGBImageCommand.Request) returns (GetRGBImageCommand.Feedback) {}
rpc GetDepthImage(GetDepthImageCommand.Request) returns (GetDepthImageCommand.Feedback) {}
rpc GetRGBDImages(GetRGBDImagesCommand.Request) returns (GetRGBDImagesCommand.Feedback) {}
rpc StartRecording(StartCameraRecordingCommand.Request) returns (StartCameraRecordingCommand.Feedback) {}
rpc StopRecording(StopCameraRecordingCommand.Request) returns (StopCameraRecordingCommand.Feedback) {}
rpc ControlPtz(ControlPtzCommand.Request) returns (ControlPtzCommand.Feedback) {}
rpc GetRGBImageStream(stream GetRGBImageStreamCommand.Request) returns (stream GetRGBImageStreamCommand.Feedback) {}
rpc GetDepthImageStream(stream GetDepthImageStreamCommand.Request) returns (stream GetDepthImageStreamCommand.Feedback) {}
rpc GetRGBDImagesStream(stream GetRGBDImagesStreamCommand.Request) returns (stream GetRGBDImagesStreamCommand.Feedback) {}
}

13
server/proto/cmvr/api/common.proto
View File

@ -35,6 +35,7 @@ message AudioData {
MP3 = 1;
AAC = 2;
WAV = 3;
OPUS = 4;
}
bytes data = 1;
int32 sample_rate = 2;
@ -57,3 +58,15 @@ message ConfigParam {
bytes bytes_value = 6; // 二进制数据类型
}
}
message JsonDeviceCommand {
message Request {
CommandHeader.Request header = 1;
string request_json = 2;
}
message Feedback {
CommandHeader.Feedback header = 1;
string response_json = 2;
}
}

19
src/components/IPlayer/index.vue
View File

@ -5,11 +5,11 @@
</template>
<script setup>
import { ref, onMounted } from 'vue';
import { ref, onMounted, onBeforeUnmount, watch } from 'vue';
import Player, { Events } from 'xgplayer'; // 引入西瓜视频模块
import 'xgplayer/dist/index.min.css'; // 引入西瓜视频样式
const { videoUrl } = defineProps({
const props = defineProps({
videoUrl: {
type: String,
default: ''
@ -20,16 +20,25 @@ const xgPlayerRef = ref()
import { conf } from "./config"; // 配置文件单独拎出来一个js文件
onMounted(() => { init() })
onMounted(() => { init(props.videoUrl) })
onBeforeUnmount(() => {
player?.destroy()
player = null
})
watch(() => props.videoUrl, (url) => {
if (!player || !url) return
player.src = url
})
let player = null // 实例
const init = () => {
const init = (videoUrl) => {
player = new Player({
el: xgPlayerRef.value,
// width: 600,
// height: 400, // 视频宽高尺寸
isLive: false,
isLive: true,
url: videoUrl, // 视频源
// poster: "http://ashuai.work/static/img/avantar.png", // 视频封面
...conf,

6
src/stores/robot.js
View File

@ -8,6 +8,7 @@ export const useRobotStore = defineStore('robot', {
almDeviceId: 'huayan_arm',
spkDeviceId: 'spk1',
micDeviceId: 'mic1',
cameraDeviceId: 'real_cam1',
position: { x: 0, y: 0, theta: 0 },
battery: {
percentage: 0.99,
@ -43,6 +44,9 @@ export const useRobotStore = defineStore('robot', {
setMicDeviceId(str) {
this.micDeviceId = str
},
setCameraDeviceId(str) {
this.cameraDeviceId = str
},
// 更新机器人位置
setPosition(data = {}) {
this.position = data
@ -61,4 +65,4 @@ export const useRobotStore = defineStore('robot', {
},
// 开启本地存储
persist: true
})
})

10
src/views/Address.vue
View File

@ -21,6 +21,9 @@
<el-form-item label="麦克风设备ID" prop="micDeviceId">
<el-input v-model="ruleForm.micDeviceId" placeholder="请输入麦克风设备ID" />
</el-form-item>
<el-form-item label="相机设备ID" prop="cameraDeviceId">
<el-input v-model="ruleForm.cameraDeviceId" placeholder="请输入相机设备ID" />
</el-form-item>
</el-form>
<el-button :loading="loading" @click="testConnect" class="connect">进入上位机控制页面</el-button>
</div>
@ -46,7 +49,8 @@ const ruleForm = ref({
agvDeviceId:'src1100',
almDeviceId: 'huayan_arm',
spkDeviceId: 'spk1',
micDeviceId: 'mic1'
micDeviceId: 'mic1',
cameraDeviceId: 'real_cam1'
})
const rules = ref({
@ -65,6 +69,9 @@ const rules = ref({
micDeviceId: [
{ required: true, message: '请输入麦克风设备ID', trigger: 'blur' }
],
cameraDeviceId: [
{ required: true, message: '请输入相机设备ID', trigger: 'blur' }
],
})
const testConnect = async () => {
@ -76,6 +83,7 @@ const testConnect = async () => {
robotStore.setAlmDeviceId(ruleForm.value.almDeviceId)
robotStore.setSpkDeviceId(ruleForm.value.spkDeviceId)
robotStore.setMicDeviceId(ruleForm.value.micDeviceId)
robotStore.setCameraDeviceId(ruleForm.value.cameraDeviceId)
// const result = await getRuntimeState({
// ip: robotStore.ip,
// deviceId: ruleForm.value.agvDeviceId

14
src/views/camera/index.vue
View File

@ -1,7 +1,7 @@
<template>
<div class="page-container">
<div class="video">
<IPlayer />
<IPlayer :video-url="videoUrl" />
</div>
<div class="audio">
<VoiceConversation />
@ -11,6 +11,18 @@
<script setup>
import IPlayer from '@/components/IPlayer/index.vue'
import VoiceConversation from './VoiceConversation.vue';
import { computed } from 'vue';
import { useRobotStore } from '@/stores/robot';
const robotStore = useRobotStore()
const videoUrl = computed(() => {
if (!robotStore.ip || !robotStore.cameraDeviceId) return ''
const params = new URLSearchParams({
ip: robotStore.ip,
deviceId: robotStore.cameraDeviceId
})
return `/api/camera/video?${params}`
})
</script>
<style lang="scss" scoped>
.page-container {