feat: 相机
This commit is contained in:
parent
c09f05ac8d
commit
10a62c51d2
15
components.d.ts
vendored
15
components.d.ts
vendored
@ -11,11 +11,7 @@ export {}
|
|||||||
/* prettier-ignore */
|
/* prettier-ignore */
|
||||||
declare module 'vue' {
|
declare module 'vue' {
|
||||||
export interface GlobalComponents {
|
export interface GlobalComponents {
|
||||||
ElB: typeof import('element-plus/es')['ElB']
|
|
||||||
ElButton: typeof import('element-plus/es')['ElButton']
|
ElButton: typeof import('element-plus/es')['ElButton']
|
||||||
ElButtonGroup: typeof import('element-plus/es')['ElButtonGroup']
|
|
||||||
ElCard: typeof import('element-plus/es')['ElCard']
|
|
||||||
ElCheckbox: typeof import('element-plus/es')['ElCheckbox']
|
|
||||||
ElCol: typeof import('element-plus/es')['ElCol']
|
ElCol: typeof import('element-plus/es')['ElCol']
|
||||||
ElContainer: typeof import('element-plus/es')['ElContainer']
|
ElContainer: typeof import('element-plus/es')['ElContainer']
|
||||||
ElForm: typeof import('element-plus/es')['ElForm']
|
ElForm: typeof import('element-plus/es')['ElForm']
|
||||||
@ -23,23 +19,14 @@ declare module 'vue' {
|
|||||||
ElHeader: typeof import('element-plus/es')['ElHeader']
|
ElHeader: typeof import('element-plus/es')['ElHeader']
|
||||||
ElIcon: typeof import('element-plus/es')['ElIcon']
|
ElIcon: typeof import('element-plus/es')['ElIcon']
|
||||||
ElInput: typeof import('element-plus/es')['ElInput']
|
ElInput: typeof import('element-plus/es')['ElInput']
|
||||||
ElInputNumber: typeof import('element-plus/es')['ElInputNumber']
|
|
||||||
ElMain: typeof import('element-plus/es')['ElMain']
|
ElMain: typeof import('element-plus/es')['ElMain']
|
||||||
ElMenu: typeof import('element-plus/es')['ElMenu']
|
|
||||||
ElMenuItem: typeof import('element-plus/es')['ElMenuItem']
|
|
||||||
ElOption: typeof import('element-plus/es')['ElOption']
|
|
||||||
ElRadioButton: typeof import('element-plus/es')['ElRadioButton']
|
|
||||||
ElRadioGroup: typeof import('element-plus/es')['ElRadioGroup']
|
|
||||||
ElRow: typeof import('element-plus/es')['ElRow']
|
ElRow: typeof import('element-plus/es')['ElRow']
|
||||||
ElSelect: typeof import('element-plus/es')['ElSelect']
|
|
||||||
ElSlider: typeof import('element-plus/es')['ElSlider']
|
ElSlider: typeof import('element-plus/es')['ElSlider']
|
||||||
ElSubMenu: typeof import('element-plus/es')['ElSubMenu']
|
|
||||||
ElSwitch: typeof import('element-plus/es')['ElSwitch']
|
ElSwitch: typeof import('element-plus/es')['ElSwitch']
|
||||||
ElTag: typeof import('element-plus/es')['ElTag']
|
ElTag: typeof import('element-plus/es')['ElTag']
|
||||||
HelloWorld: typeof import('./src/components/HelloWorld.vue')['default']
|
|
||||||
IPlayer: typeof import('./src/components/IPlayer/index.vue')['default']
|
IPlayer: typeof import('./src/components/IPlayer/index.vue')['default']
|
||||||
|
LandscapeGuard: typeof import('./src/components/LandscapeGuard.vue')['default']
|
||||||
RouterLink: typeof import('vue-router')['RouterLink']
|
RouterLink: typeof import('vue-router')['RouterLink']
|
||||||
RouterView: typeof import('vue-router')['RouterView']
|
RouterView: typeof import('vue-router')['RouterView']
|
||||||
SvgIcon: typeof import('./src/components/SvgIcon/index.vue')['default']
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
193
server/cameraVideoStream.js
Normal file
193
server/cameraVideoStream.js
Normal file
@ -0,0 +1,193 @@
|
|||||||
|
import { spawn } from 'child_process'
|
||||||
|
import { createCameraGrpcClient } from './grpcClient.js'
|
||||||
|
|
||||||
|
const FIRST_FRAME_TIMEOUT_MS = 10000
|
||||||
|
const FRAME_TYPES = ['U8C1', 'U16C1', 'U8C3', 'U16C3', 'F16C1', 'F32C1']
|
||||||
|
|
||||||
|
function normalizeCodec(codec = '') {
|
||||||
|
return codec.toLowerCase().replace(/[^a-z0-9]/g, '')
|
||||||
|
}
|
||||||
|
|
||||||
|
function inputArguments(frame) {
|
||||||
|
const codec = normalizeCodec(frame.codec)
|
||||||
|
|
||||||
|
if (['jpeg', 'jpg', 'mjpeg', 'imagejpeg'].includes(codec)) return ['-f', 'mjpeg']
|
||||||
|
if (['png', 'imagepng'].includes(codec)) return ['-f', 'image2pipe', '-vcodec', 'png']
|
||||||
|
if (['h264', 'avc', 'avc1'].includes(codec)) return ['-f', 'h264']
|
||||||
|
if (['h265', 'hevc', 'hev1', 'hvc1'].includes(codec)) return ['-f', 'hevc']
|
||||||
|
|
||||||
|
const width = Number(frame.width)
|
||||||
|
const height = Number(frame.height)
|
||||||
|
if (!Number.isInteger(width) || width <= 0 || !Number.isInteger(height) || height <= 0) {
|
||||||
|
throw new Error(`未压缩视频帧的尺寸无效: ${frame.width}x${frame.height}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
const declaredPixelFormats = {
|
||||||
|
gray: 'gray',
|
||||||
|
gray8: 'gray',
|
||||||
|
gray16: 'gray16le',
|
||||||
|
gray16le: 'gray16le',
|
||||||
|
rgb24: 'rgb24',
|
||||||
|
bgr24: 'bgr24',
|
||||||
|
rgb48: 'rgb48le',
|
||||||
|
rgb48le: 'rgb48le',
|
||||||
|
bgr48: 'bgr48le',
|
||||||
|
bgr48le: 'bgr48le',
|
||||||
|
grayf32: 'grayf32le',
|
||||||
|
grayf32le: 'grayf32le'
|
||||||
|
}
|
||||||
|
const rawPixelFormats = {
|
||||||
|
U8C1: 'gray',
|
||||||
|
U16C1: 'gray16le',
|
||||||
|
U8C3: 'rgb24',
|
||||||
|
U16C3: 'rgb48le',
|
||||||
|
F32C1: 'grayf32le'
|
||||||
|
}
|
||||||
|
const frameType = typeof frame.type === 'number' ? FRAME_TYPES[frame.type] : frame.type
|
||||||
|
const pixelFormat = declaredPixelFormats[codec] || rawPixelFormats[frameType]
|
||||||
|
if (!pixelFormat) {
|
||||||
|
throw new Error(`暂不支持未压缩帧类型 ${frameType || frame.type},请让相机返回 JPEG/H.264/H.265`)
|
||||||
|
}
|
||||||
|
|
||||||
|
return [
|
||||||
|
'-f', 'rawvideo',
|
||||||
|
'-pixel_format', pixelFormat,
|
||||||
|
'-video_size', `${width}x${height}`,
|
||||||
|
'-framerate', String(Number(frame.source_fps) || 25)
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
function createTranscoder(frame) {
|
||||||
|
const fps = Math.max(1, Number(frame.source_fps) || 25)
|
||||||
|
const ffmpeg = process.env.FFMPEG_PATH || 'ffmpeg'
|
||||||
|
const args = [
|
||||||
|
'-hide_banner', '-loglevel', 'warning',
|
||||||
|
...inputArguments(frame),
|
||||||
|
'-i', 'pipe:0',
|
||||||
|
'-an',
|
||||||
|
'-c:v', 'libx264',
|
||||||
|
'-preset', 'ultrafast',
|
||||||
|
'-tune', 'zerolatency',
|
||||||
|
'-pix_fmt', 'yuv420p',
|
||||||
|
'-g', String(fps),
|
||||||
|
'-keyint_min', String(fps),
|
||||||
|
'-sc_threshold', '0',
|
||||||
|
'-f', 'mp4',
|
||||||
|
'-movflags', 'frag_keyframe+empty_moov+default_base_moof',
|
||||||
|
'pipe:1'
|
||||||
|
]
|
||||||
|
return spawn(ffmpeg, args, { windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] })
|
||||||
|
}
|
||||||
|
|
||||||
|
function message(error) {
|
||||||
|
return error?.details || error?.message || String(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function streamCameraVideo(req, res) {
|
||||||
|
const ip = typeof req.query.ip === 'string' ? req.query.ip.trim() : ''
|
||||||
|
const deviceId = typeof req.query.deviceId === 'string' ? req.query.deviceId.trim() : ''
|
||||||
|
if (!ip || !deviceId) {
|
||||||
|
res.status(400).json({ code: 400, message: 'ip 和 deviceId 不能为空' })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = createCameraGrpcClient(ip)
|
||||||
|
const grpcStream = client.getRgbImageStream()
|
||||||
|
let transcoder
|
||||||
|
let closed = false
|
||||||
|
let responseStarted = false
|
||||||
|
let stderr = ''
|
||||||
|
|
||||||
|
const close = () => {
|
||||||
|
if (closed) return
|
||||||
|
closed = true
|
||||||
|
clearTimeout(firstFrameTimer)
|
||||||
|
grpcStream.cancel()
|
||||||
|
client.close()
|
||||||
|
if (transcoder && !transcoder.killed) {
|
||||||
|
transcoder.stdin.destroy()
|
||||||
|
transcoder.kill()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const fail = (error) => {
|
||||||
|
if (closed) return
|
||||||
|
console.error('[camera] video stream failed:', message(error))
|
||||||
|
if (!responseStarted) {
|
||||||
|
res.status(502).json({ code: 502, message: `相机视频流不可用: ${message(error)}` })
|
||||||
|
} else {
|
||||||
|
res.destroy(error instanceof Error ? error : undefined)
|
||||||
|
}
|
||||||
|
close()
|
||||||
|
}
|
||||||
|
|
||||||
|
const firstFrameTimer = setTimeout(
|
||||||
|
() => fail(new Error('等待相机首帧超时')),
|
||||||
|
FIRST_FRAME_TIMEOUT_MS
|
||||||
|
)
|
||||||
|
|
||||||
|
grpcStream.on('data', (feedback) => {
|
||||||
|
if (closed) return
|
||||||
|
if (feedback?.header?.success === false) {
|
||||||
|
fail(new Error(feedback.header.error_message || '相机拒绝视频流请求'))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const frame = feedback?.color_frame
|
||||||
|
if (!frame?.data?.length) return
|
||||||
|
|
||||||
|
if (!transcoder) {
|
||||||
|
clearTimeout(firstFrameTimer)
|
||||||
|
try {
|
||||||
|
transcoder = createTranscoder(frame)
|
||||||
|
} catch (error) {
|
||||||
|
fail(error)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
res.status(200)
|
||||||
|
res.set({
|
||||||
|
'Content-Type': 'video/mp4',
|
||||||
|
'Cache-Control': 'no-store, no-cache, must-revalidate',
|
||||||
|
Connection: 'keep-alive',
|
||||||
|
'X-Accel-Buffering': 'no'
|
||||||
|
})
|
||||||
|
responseStarted = true
|
||||||
|
transcoder.stdout.pipe(res)
|
||||||
|
transcoder.stderr.on('data', (chunk) => {
|
||||||
|
stderr = (stderr + chunk.toString()).slice(-4000)
|
||||||
|
})
|
||||||
|
transcoder.stdin.on('error', (error) => {
|
||||||
|
if (!closed && error.code !== 'EPIPE') fail(error)
|
||||||
|
})
|
||||||
|
transcoder.on('error', fail)
|
||||||
|
transcoder.on('exit', (code, signal) => {
|
||||||
|
if (!closed && code !== 0) {
|
||||||
|
fail(new Error(`FFmpeg 异常退出 (${code ?? signal}): ${stderr.trim()}`))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!transcoder.stdin.write(frame.data)) {
|
||||||
|
grpcStream.pause()
|
||||||
|
transcoder.stdin.once('drain', () => grpcStream.resume())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
grpcStream.on('error', (error) => {
|
||||||
|
// CANCELLED 是浏览器断开后主动清理产生的正常状态。
|
||||||
|
if (!closed && error.code !== 1) fail(error)
|
||||||
|
})
|
||||||
|
grpcStream.on('end', () => {
|
||||||
|
if (!closed) {
|
||||||
|
transcoder?.stdin.end()
|
||||||
|
if (!responseStarted) fail(new Error('相机未返回任何视频帧'))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
res.on('close', close)
|
||||||
|
grpcStream.write({
|
||||||
|
header: { device_id: deviceId },
|
||||||
|
eof: false
|
||||||
|
})
|
||||||
|
}
|
||||||
@ -11,6 +11,7 @@ export const __dirname = path.dirname(__filename)
|
|||||||
const PROTO_ROOT = path.resolve(__dirname, './proto') // 指向 server/proto/
|
const PROTO_ROOT = path.resolve(__dirname, './proto') // 指向 server/proto/
|
||||||
const AGV_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/agv_service.proto')
|
const AGV_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/agv_service.proto')
|
||||||
const ARM_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/arm_service.proto')
|
const ARM_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/arm_service.proto')
|
||||||
|
const CAMERA_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/camera_service.proto')
|
||||||
const MICROPHONE_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/microphone_service.proto')
|
const MICROPHONE_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/microphone_service.proto')
|
||||||
const SPEAKER_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/speaker_service.proto')
|
const SPEAKER_PROTO_ENTRY = path.join(PROTO_ROOT, 'cmvr/api/speaker_service.proto')
|
||||||
|
|
||||||
@ -47,11 +48,13 @@ const armPackageDefinition = protoLoader.loadSync(
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const cameraPackageDefinition = protoLoader.loadSync(CAMERA_PROTO_ENTRY, audioLoaderOptions)
|
||||||
const microphonePackageDefinition = protoLoader.loadSync(MICROPHONE_PROTO_ENTRY, audioLoaderOptions)
|
const microphonePackageDefinition = protoLoader.loadSync(MICROPHONE_PROTO_ENTRY, audioLoaderOptions)
|
||||||
const speakerPackageDefinition = protoLoader.loadSync(SPEAKER_PROTO_ENTRY, audioLoaderOptions)
|
const speakerPackageDefinition = protoLoader.loadSync(SPEAKER_PROTO_ENTRY, audioLoaderOptions)
|
||||||
|
|
||||||
const agvProto = grpc.loadPackageDefinition(agvPackageDefinition).cmvr.api
|
const agvProto = grpc.loadPackageDefinition(agvPackageDefinition).cmvr.api
|
||||||
const armProto = grpc.loadPackageDefinition(armPackageDefinition).cmvr.api
|
const armProto = grpc.loadPackageDefinition(armPackageDefinition).cmvr.api
|
||||||
|
const cameraProto = grpc.loadPackageDefinition(cameraPackageDefinition).cmvr.api
|
||||||
const microphoneProto = grpc.loadPackageDefinition(microphonePackageDefinition).cmvr.api
|
const microphoneProto = grpc.loadPackageDefinition(microphonePackageDefinition).cmvr.api
|
||||||
const speakerProto = grpc.loadPackageDefinition(speakerPackageDefinition).cmvr.api
|
const speakerProto = grpc.loadPackageDefinition(speakerPackageDefinition).cmvr.api
|
||||||
|
|
||||||
@ -71,6 +74,11 @@ export const createARMGrpcClient = (address) => {
|
|||||||
return grpcClient
|
return grpcClient
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const createCameraGrpcClient = (address) => new cameraProto.CameraService(
|
||||||
|
address,
|
||||||
|
grpc.credentials.createInsecure()
|
||||||
|
)
|
||||||
|
|
||||||
export const createMicrophoneGrpcClient = (address) => new microphoneProto.MicPhoneService(
|
export const createMicrophoneGrpcClient = (address) => new microphoneProto.MicPhoneService(
|
||||||
address,
|
address,
|
||||||
grpc.credentials.createInsecure()
|
grpc.credentials.createInsecure()
|
||||||
|
|||||||
@ -15,12 +15,17 @@ import {
|
|||||||
__dirname
|
__dirname
|
||||||
} from './grpcClient.js';
|
} from './grpcClient.js';
|
||||||
import { attachAudioWebSocket } from './audioWebSocket.js';
|
import { attachAudioWebSocket } from './audioWebSocket.js';
|
||||||
|
import { streamCameraVideo } from './cameraVideoStream.js';
|
||||||
|
|
||||||
const app = express()
|
const app = express()
|
||||||
app.use(express.static(path.join(__dirname, '../dist')));
|
app.use(express.static(path.join(__dirname, '../dist')));
|
||||||
app.use(cors())
|
app.use(cors())
|
||||||
app.use(express.json())
|
app.use(express.json())
|
||||||
|
|
||||||
|
// 将相机逐帧 gRPC 数据转码为浏览器可直接播放的 fragmented MP4。
|
||||||
|
// 示例:/api/camera/video?ip=192.168.0.28%3A50052&deviceId=camera1
|
||||||
|
app.get('/api/camera/video', streamCameraVideo)
|
||||||
|
|
||||||
// 获取机器人麦克风音量(0-100)。
|
// 获取机器人麦克风音量(0-100)。
|
||||||
app.post('/api/audio/microphone/volume/get', (req, res) => {
|
app.post('/api/audio/microphone/volume/get', (req, res) => {
|
||||||
const { ip, deviceId } = req.body
|
const { ip, deviceId } = req.body
|
||||||
|
|||||||
216
server/proto/cmvr/api/camera_command.proto
Normal file
216
server/proto/cmvr/api/camera_command.proto
Normal file
@ -0,0 +1,216 @@
|
|||||||
|
syntax = "proto3";
|
||||||
|
|
||||||
|
import "cmvr/api/common.proto";
|
||||||
|
|
||||||
|
package cmvr.api;
|
||||||
|
|
||||||
|
message FrameData {
|
||||||
|
enum FrameType {
|
||||||
|
U8C1 = 0;
|
||||||
|
U16C1 = 1;
|
||||||
|
U8C3 = 2;
|
||||||
|
U16C3 = 3;
|
||||||
|
F16C1 = 4;
|
||||||
|
F32C1 = 5;
|
||||||
|
}
|
||||||
|
bytes data = 1;
|
||||||
|
int32 width = 2;
|
||||||
|
int32 height = 3;
|
||||||
|
FrameType type = 4;
|
||||||
|
string codec = 5;
|
||||||
|
bool is_key_frame = 6;
|
||||||
|
// Optional capture and source metadata. Fields 1-6 remain wire-compatible
|
||||||
|
// with existing clients; older clients safely ignore these additions.
|
||||||
|
int64 capture_utc_ns = 7;
|
||||||
|
uint64 source_sequence = 8;
|
||||||
|
int64 pts = 9;
|
||||||
|
int64 dts = 10;
|
||||||
|
uint32 source_fps = 11;
|
||||||
|
uint64 source_timestamp = 12;
|
||||||
|
uint64 source_frame_number = 13;
|
||||||
|
}
|
||||||
|
|
||||||
|
message CameraIntrinsics {
|
||||||
|
float cx = 1; // 主点水平坐标(从左边缘的像素偏移)
|
||||||
|
float cy = 2; // 主点垂直坐标(从上边缘的像素偏移)
|
||||||
|
float fx = 3; // x方向焦距(像素宽度的倍数)
|
||||||
|
float fy = 4; // y方向焦距(像素高度的倍数)
|
||||||
|
repeated float coeffs = 5 [packed = true]; // 畸变系数(固定5个元素)
|
||||||
|
}
|
||||||
|
|
||||||
|
message CameraState {
|
||||||
|
bool is_initialized = 1;
|
||||||
|
bool is_opened = 2;
|
||||||
|
bool is_streaming = 3;
|
||||||
|
bool is_recording = 4;
|
||||||
|
bool is_error = 5;
|
||||||
|
string error_message = 6;
|
||||||
|
int32 fps = 7;
|
||||||
|
int32 width = 8;
|
||||||
|
int32 height = 9;
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetCameraStateCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
}
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
CameraState state = 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message StartCameraCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message StopCameraCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetRGBImageCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
FrameData color_frame = 2;
|
||||||
|
CameraIntrinsics intrinsics = 3;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetDepthImageCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
FrameData depth_frame = 2;
|
||||||
|
CameraIntrinsics intrinsics = 3;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetRGBDImagesCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
FrameData color_frame = 2;
|
||||||
|
FrameData depth_frame = 3;
|
||||||
|
CameraIntrinsics intrinsics = 4;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message StartCameraRecordingCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
string video_path = 2;
|
||||||
|
}
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message StopCameraRecordingCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message ControlPtzCommand {
|
||||||
|
enum Command {
|
||||||
|
COMMAND_UNSPECIFIED = 0;
|
||||||
|
TILT_UP = 1;
|
||||||
|
TILT_DOWN = 2;
|
||||||
|
PAN_LEFT = 3;
|
||||||
|
PAN_RIGHT = 4;
|
||||||
|
UP_LEFT = 5;
|
||||||
|
UP_RIGHT = 6;
|
||||||
|
DOWN_LEFT = 7;
|
||||||
|
DOWN_RIGHT = 8;
|
||||||
|
ZOOM_IN = 9;
|
||||||
|
ZOOM_OUT = 10;
|
||||||
|
PAN_AUTO = 11;
|
||||||
|
}
|
||||||
|
|
||||||
|
enum Action {
|
||||||
|
START = 0;
|
||||||
|
STOP = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
Command command = 2;
|
||||||
|
Action action = 3;
|
||||||
|
uint32 speed = 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetRGBImageStreamCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
bool eof = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
FrameData color_frame = 2;
|
||||||
|
CameraIntrinsics intrinsics = 3;
|
||||||
|
int32 seq_no = 4;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetDepthImageStreamCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
bool eof = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
FrameData depth_frame = 2;
|
||||||
|
CameraIntrinsics intrinsics = 3;
|
||||||
|
int32 seq_no = 4;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetRGBDImagesStreamCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
bool eof = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
FrameData color_frame = 2;
|
||||||
|
FrameData depth_frame = 3;
|
||||||
|
CameraIntrinsics intrinsics = 4;
|
||||||
|
int32 seq_no = 5;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
22
server/proto/cmvr/api/camera_service.proto
Normal file
22
server/proto/cmvr/api/camera_service.proto
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
|
||||||
|
syntax = "proto3";
|
||||||
|
|
||||||
|
import "cmvr/api/camera_command.proto";
|
||||||
|
|
||||||
|
package cmvr.api;
|
||||||
|
|
||||||
|
service CameraService {
|
||||||
|
rpc GetStatus(GetCameraStateCommand.Request) returns (GetCameraStateCommand.Feedback) {}
|
||||||
|
rpc StartCamera(StartCameraCommand.Request) returns (StartCameraCommand.Feedback) {}
|
||||||
|
rpc StopCamera(StopCameraCommand.Request) returns (StopCameraCommand.Feedback) {}
|
||||||
|
rpc GetRGBImage(GetRGBImageCommand.Request) returns (GetRGBImageCommand.Feedback) {}
|
||||||
|
rpc GetDepthImage(GetDepthImageCommand.Request) returns (GetDepthImageCommand.Feedback) {}
|
||||||
|
rpc GetRGBDImages(GetRGBDImagesCommand.Request) returns (GetRGBDImagesCommand.Feedback) {}
|
||||||
|
rpc StartRecording(StartCameraRecordingCommand.Request) returns (StartCameraRecordingCommand.Feedback) {}
|
||||||
|
rpc StopRecording(StopCameraRecordingCommand.Request) returns (StopCameraRecordingCommand.Feedback) {}
|
||||||
|
rpc ControlPtz(ControlPtzCommand.Request) returns (ControlPtzCommand.Feedback) {}
|
||||||
|
|
||||||
|
rpc GetRGBImageStream(stream GetRGBImageStreamCommand.Request) returns (stream GetRGBImageStreamCommand.Feedback) {}
|
||||||
|
rpc GetDepthImageStream(stream GetDepthImageStreamCommand.Request) returns (stream GetDepthImageStreamCommand.Feedback) {}
|
||||||
|
rpc GetRGBDImagesStream(stream GetRGBDImagesStreamCommand.Request) returns (stream GetRGBDImagesStreamCommand.Feedback) {}
|
||||||
|
}
|
||||||
@ -35,6 +35,7 @@ message AudioData {
|
|||||||
MP3 = 1;
|
MP3 = 1;
|
||||||
AAC = 2;
|
AAC = 2;
|
||||||
WAV = 3;
|
WAV = 3;
|
||||||
|
OPUS = 4;
|
||||||
}
|
}
|
||||||
bytes data = 1;
|
bytes data = 1;
|
||||||
int32 sample_rate = 2;
|
int32 sample_rate = 2;
|
||||||
@ -57,3 +58,15 @@ message ConfigParam {
|
|||||||
bytes bytes_value = 6; // 二进制数据类型
|
bytes bytes_value = 6; // 二进制数据类型
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
message JsonDeviceCommand {
|
||||||
|
message Request {
|
||||||
|
CommandHeader.Request header = 1;
|
||||||
|
string request_json = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message Feedback {
|
||||||
|
CommandHeader.Feedback header = 1;
|
||||||
|
string response_json = 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -5,11 +5,11 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { ref, onMounted } from 'vue';
|
import { ref, onMounted, onBeforeUnmount, watch } from 'vue';
|
||||||
import Player, { Events } from 'xgplayer'; // 引入西瓜视频模块
|
import Player, { Events } from 'xgplayer'; // 引入西瓜视频模块
|
||||||
import 'xgplayer/dist/index.min.css'; // 引入西瓜视频样式
|
import 'xgplayer/dist/index.min.css'; // 引入西瓜视频样式
|
||||||
|
|
||||||
const { videoUrl } = defineProps({
|
const props = defineProps({
|
||||||
videoUrl: {
|
videoUrl: {
|
||||||
type: String,
|
type: String,
|
||||||
default: ''
|
default: ''
|
||||||
@ -20,16 +20,25 @@ const xgPlayerRef = ref()
|
|||||||
|
|
||||||
import { conf } from "./config"; // 配置文件单独拎出来一个js文件
|
import { conf } from "./config"; // 配置文件单独拎出来一个js文件
|
||||||
|
|
||||||
onMounted(() => { init() })
|
onMounted(() => { init(props.videoUrl) })
|
||||||
|
onBeforeUnmount(() => {
|
||||||
|
player?.destroy()
|
||||||
|
player = null
|
||||||
|
})
|
||||||
|
|
||||||
|
watch(() => props.videoUrl, (url) => {
|
||||||
|
if (!player || !url) return
|
||||||
|
player.src = url
|
||||||
|
})
|
||||||
|
|
||||||
let player = null // 实例
|
let player = null // 实例
|
||||||
|
|
||||||
const init = () => {
|
const init = (videoUrl) => {
|
||||||
player = new Player({
|
player = new Player({
|
||||||
el: xgPlayerRef.value,
|
el: xgPlayerRef.value,
|
||||||
// width: 600,
|
// width: 600,
|
||||||
// height: 400, // 视频宽高尺寸
|
// height: 400, // 视频宽高尺寸
|
||||||
isLive: false,
|
isLive: true,
|
||||||
url: videoUrl, // 视频源
|
url: videoUrl, // 视频源
|
||||||
// poster: "http://ashuai.work/static/img/avantar.png", // 视频封面
|
// poster: "http://ashuai.work/static/img/avantar.png", // 视频封面
|
||||||
...conf,
|
...conf,
|
||||||
|
|||||||
@ -8,6 +8,7 @@ export const useRobotStore = defineStore('robot', {
|
|||||||
almDeviceId: 'huayan_arm',
|
almDeviceId: 'huayan_arm',
|
||||||
spkDeviceId: 'spk1',
|
spkDeviceId: 'spk1',
|
||||||
micDeviceId: 'mic1',
|
micDeviceId: 'mic1',
|
||||||
|
cameraDeviceId: 'real_cam1',
|
||||||
position: { x: 0, y: 0, theta: 0 },
|
position: { x: 0, y: 0, theta: 0 },
|
||||||
battery: {
|
battery: {
|
||||||
percentage: 0.99,
|
percentage: 0.99,
|
||||||
@ -43,6 +44,9 @@ export const useRobotStore = defineStore('robot', {
|
|||||||
setMicDeviceId(str) {
|
setMicDeviceId(str) {
|
||||||
this.micDeviceId = str
|
this.micDeviceId = str
|
||||||
},
|
},
|
||||||
|
setCameraDeviceId(str) {
|
||||||
|
this.cameraDeviceId = str
|
||||||
|
},
|
||||||
// 更新机器人位置
|
// 更新机器人位置
|
||||||
setPosition(data = {}) {
|
setPosition(data = {}) {
|
||||||
this.position = data
|
this.position = data
|
||||||
|
|||||||
@ -21,6 +21,9 @@
|
|||||||
<el-form-item label="麦克风设备ID" prop="micDeviceId">
|
<el-form-item label="麦克风设备ID" prop="micDeviceId">
|
||||||
<el-input v-model="ruleForm.micDeviceId" placeholder="请输入麦克风设备ID" />
|
<el-input v-model="ruleForm.micDeviceId" placeholder="请输入麦克风设备ID" />
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
|
<el-form-item label="相机设备ID" prop="cameraDeviceId">
|
||||||
|
<el-input v-model="ruleForm.cameraDeviceId" placeholder="请输入相机设备ID" />
|
||||||
|
</el-form-item>
|
||||||
</el-form>
|
</el-form>
|
||||||
<el-button :loading="loading" @click="testConnect" class="connect">进入上位机控制页面</el-button>
|
<el-button :loading="loading" @click="testConnect" class="connect">进入上位机控制页面</el-button>
|
||||||
</div>
|
</div>
|
||||||
@ -46,7 +49,8 @@ const ruleForm = ref({
|
|||||||
agvDeviceId:'src1100',
|
agvDeviceId:'src1100',
|
||||||
almDeviceId: 'huayan_arm',
|
almDeviceId: 'huayan_arm',
|
||||||
spkDeviceId: 'spk1',
|
spkDeviceId: 'spk1',
|
||||||
micDeviceId: 'mic1'
|
micDeviceId: 'mic1',
|
||||||
|
cameraDeviceId: 'real_cam1'
|
||||||
})
|
})
|
||||||
|
|
||||||
const rules = ref({
|
const rules = ref({
|
||||||
@ -65,6 +69,9 @@ const rules = ref({
|
|||||||
micDeviceId: [
|
micDeviceId: [
|
||||||
{ required: true, message: '请输入麦克风设备ID', trigger: 'blur' }
|
{ required: true, message: '请输入麦克风设备ID', trigger: 'blur' }
|
||||||
],
|
],
|
||||||
|
cameraDeviceId: [
|
||||||
|
{ required: true, message: '请输入相机设备ID', trigger: 'blur' }
|
||||||
|
],
|
||||||
})
|
})
|
||||||
|
|
||||||
const testConnect = async () => {
|
const testConnect = async () => {
|
||||||
@ -76,6 +83,7 @@ const testConnect = async () => {
|
|||||||
robotStore.setAlmDeviceId(ruleForm.value.almDeviceId)
|
robotStore.setAlmDeviceId(ruleForm.value.almDeviceId)
|
||||||
robotStore.setSpkDeviceId(ruleForm.value.spkDeviceId)
|
robotStore.setSpkDeviceId(ruleForm.value.spkDeviceId)
|
||||||
robotStore.setMicDeviceId(ruleForm.value.micDeviceId)
|
robotStore.setMicDeviceId(ruleForm.value.micDeviceId)
|
||||||
|
robotStore.setCameraDeviceId(ruleForm.value.cameraDeviceId)
|
||||||
// const result = await getRuntimeState({
|
// const result = await getRuntimeState({
|
||||||
// ip: robotStore.ip,
|
// ip: robotStore.ip,
|
||||||
// deviceId: ruleForm.value.agvDeviceId
|
// deviceId: ruleForm.value.agvDeviceId
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
<template>
|
<template>
|
||||||
<div class="page-container">
|
<div class="page-container">
|
||||||
<div class="video">
|
<div class="video">
|
||||||
<IPlayer />
|
<IPlayer :video-url="videoUrl" />
|
||||||
</div>
|
</div>
|
||||||
<div class="audio">
|
<div class="audio">
|
||||||
<VoiceConversation />
|
<VoiceConversation />
|
||||||
@ -11,6 +11,18 @@
|
|||||||
<script setup>
|
<script setup>
|
||||||
import IPlayer from '@/components/IPlayer/index.vue'
|
import IPlayer from '@/components/IPlayer/index.vue'
|
||||||
import VoiceConversation from './VoiceConversation.vue';
|
import VoiceConversation from './VoiceConversation.vue';
|
||||||
|
import { computed } from 'vue';
|
||||||
|
import { useRobotStore } from '@/stores/robot';
|
||||||
|
|
||||||
|
const robotStore = useRobotStore()
|
||||||
|
const videoUrl = computed(() => {
|
||||||
|
if (!robotStore.ip || !robotStore.cameraDeviceId) return ''
|
||||||
|
const params = new URLSearchParams({
|
||||||
|
ip: robotStore.ip,
|
||||||
|
deviceId: robotStore.cameraDeviceId
|
||||||
|
})
|
||||||
|
return `/api/camera/video?${params}`
|
||||||
|
})
|
||||||
</script>
|
</script>
|
||||||
<style lang="scss" scoped>
|
<style lang="scss" scoped>
|
||||||
.page-container {
|
.page-container {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user