From 275aeb5f4f5ccac6f2de46856640490bb1fa825e Mon Sep 17 00:00:00 2001 From: lixiaolong <702156524@qq.com> Date: Wed, 16 Sep 2026 11:39:59 +0800 Subject: [PATCH] =?UTF-8?q?feat(flow):=20=E6=B7=BB=E5=8A=A0=E8=A7=86?= =?UTF-8?q?=E8=A7=89=E5=AE=9A=E4=BD=8D=E5=92=8C=E9=97=B2=E8=81=8A=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增视觉定位执行节点，支持RGBD图像分析和目标定位 - 添加闲聊节点，集成大模型自然语言对话功能 - 配置视觉定位相关的设备绑定和参数设置 - 更新设备资源管理界面，增加机器人筛选功能 - 优化图像分析参数，提高定位精度 - 添加车机发声检测功能到声音分析模块 --- .env.development | 6 +- src/utils/flowResources.js | 36 +++++++-- src/views/flow/components/ParamsDrawer.vue | 4 + .../components/params/ImageAnalysisParams.vue | 6 +- .../components/params/MediaAnalysisParams.vue | 2 + src/views/flow/config.js | 74 +++++++++++++++++++ src/views/flow/vue-flow/FlowNode.vue | 2 + src/views/resource/device/index.vue | 43 +++++++++-- 8 files changed, 157 insertions(+), 16 deletions(-) diff --git a/.env.development b/.env.development index da134de..37b4d85 100644 --- a/.env.development +++ b/.env.development @@ -7,8 +7,8 @@ VITE_APP_ENV = 'development' # 招商车研物联网平台/开发环境 VITE_APP_BASE_API = '/dev-api' -VITE_API_URL = http://localhost:14080 -VITE_WS_URL = ws://localhost:14080/ws -VITE_PORT = 5174 +VITE_API_URL = http://192.168.0.222:14080 +VITE_WS_URL = ws://192.168.0.222:14080/ws +VITE_PORT = 80 VITE_INSPECTION_TYPE = "inspection" VITE_MINIO_URL = "http://192.168.28.10:9000/cmvr-iot/" diff --git a/src/utils/flowResources.js b/src/utils/flowResources.js index b583464..ab71a3e 100644 --- a/src/utils/flowResources.js +++ b/src/utils/flowResources.js @@ -77,6 +77,11 @@ export const getResourceRoles = (graph = globalThis.lf?.getGraphData?.()) => ( const isRobotNode = (node) => node?.properties?.nodeType === 'EDGE' export const ACTION_QUEUE_ACTION = 'SYSTEM_EXECUTE_ACTION_QUEUE' +const NAMED_DEVICE_BINDING_ACTIONS = new Set([ACTION_QUEUE_ACTION, 'VISION_LOCATE_TARGET']) +const VISION_DEVICE_BINDING_SPECS = Object.freeze([ + { parameterName: 'cameraDeviceId', deviceKind: 5, displayName: '视觉定位相机' }, + { parameterName: 'armDeviceId', deviceKind: 2, displayName: '视觉定位机械臂' }, +]) export const actionRequiresDevice = (action) => ( getDeviceKindForAction(action) != null || action === 'DEVICE_EXECUTE_JSON_COMMAND' ) @@ -196,12 +201,31 @@ export const normalizeFlowResourceSchema = (sourceGraph) => { } target.roleKey ||= defaultRole.roleKey target.dispatchPolicy ||= 'ONE' - const isActionQueue = properties.action === ACTION_QUEUE_ACTION - if (!isActionQueue) target.deviceSlotKey ||= target.deviceBindings?.[0]?.slotKey || '' + const usesNamedDeviceBindings = NAMED_DEVICE_BINDING_ACTIONS.has(properties.action) + if (!usesNamedDeviceBindings) target.deviceSlotKey ||= target.deviceBindings?.[0]?.slotKey || '' target.deviceBindings = Array.isArray(target.deviceBindings) ? target.deviceBindings : [] const targetRole = roles.find((role) => role.roleKey === target.roleKey) || defaultRole const deviceParam = properties.nodeParams?.find((param) => param.name === 'deviceId') + if (properties.action === 'VISION_LOCATE_TARGET') { + VISION_DEVICE_BINDING_SPECS.forEach((spec) => { + if (target.deviceBindings.some((binding) => binding?.parameterName === spec.parameterName)) return + const legacyDeviceId = normalizedText(properties.nodeParams + ?.find((param) => param.name === spec.parameterName)?.input) + const matchingSlots = targetRole.deviceSlots.filter((slot) => ( + Number(slot.deviceKind) === spec.deviceKind + && (!legacyDeviceId || !slot.legacyDeviceId || slot.legacyDeviceId === legacyDeviceId) + )) + let slot = matchingSlots.length === 1 ? matchingSlots[0] : null + if (!slot && matchingSlots.length === 0) { + slot = createDeviceSlot(spec.deviceKind, targetRole.deviceSlots, spec.displayName) + slot.managedBy = FLOW_NODE_RESOURCE_OWNER + if (legacyDeviceId) slot.legacyDeviceId = legacyDeviceId + targetRole.deviceSlots.push(slot) + } + if (slot) target.deviceBindings.push({ parameterName: spec.parameterName, slotKey: slot.slotKey }) + }) + } if (nodeRequiresDevice(node) && !target.deviceSlotKey) { const slot = findOrCreateLegacySlot(targetRole, node, legacySlotByIdentity, deviceParam?.input) if (slot) target.deviceSlotKey = slot.slotKey @@ -213,8 +237,8 @@ export const normalizeFlowResourceSchema = (sourceGraph) => { targetRole.deviceSlots.forEach((slot) => { if (referencedSlotKeys.has(slot.slotKey) && !slot.managedBy) slot.managedBy = FLOW_NODE_RESOURCE_OWNER }) - if (!isActionQueue) delete target.deviceBindings - if (isActionQueue) delete target.deviceSlotKey + if (!usesNamedDeviceBindings) delete target.deviceBindings + if (usesNamedDeviceBindings) delete target.deviceSlotKey properties.nodeParams = (properties.nodeParams || []).filter((param) => param.name !== 'deviceId') properties.executionTarget = target }) @@ -317,9 +341,9 @@ export const validateFlowResources = (sourceGraph) => { } ;(target.deviceBindings || []).forEach((binding) => { if (!binding?.parameterName || !binding?.slotKey) { - errors.push({ nodeId: node.id, message: `${node.properties?.name || node.id}存在未配置的队列设备` }) + errors.push({ nodeId: node.id, message: `${node.properties?.name || node.id}存在未配置的设备绑定` }) } else if (!(role.deviceSlots || []).some((slot) => slot.slotKey === binding.slotKey)) { - errors.push({ nodeId: node.id, message: `${node.properties?.name || node.id}引用的队列设备用途不存在` }) + errors.push({ nodeId: node.id, message: `${node.properties?.name || node.id}引用的设备用途不存在` }) } }) if (nodeRequiresDevice(node) diff --git a/src/views/flow/components/ParamsDrawer.vue b/src/views/flow/components/ParamsDrawer.vue index d26cd31..db2cdd9 100644 --- a/src/views/flow/components/ParamsDrawer.vue +++ b/src/views/flow/components/ParamsDrawer.vue @@ -42,6 +42,8 @@ import SwitchNodeParams from './params/SwitchNodeParams.vue' import ActionQueueParams from './params/ActionQueueParams.vue' import MediaAnalysisParams from './params/MediaAnalysisParams.vue' import ImageAnalysisParams from './params/ImageAnalysisParams.vue' +import VisionLocateParams from './params/VisionLocateParams.vue' +import ChatParams from './params/ChatParams.vue' import EndParams from './params/EndParams.vue' import SubFlowParams from './params/SubFlowParams.vue' @@ -77,6 +79,8 @@ const currentComponent = computed(() => { if (type === 'mediaAnalysis') return MediaAnalysisParams // 图片分析支持多图、参考图和自由提示词输出，使用独立面板保持媒体分析面板聚焦音视频。 if (type === 'imageAnalysis') return ImageAnalysisParams + if (type === 'visionLocate') return VisionLocateParams + if (type === 'chat') return ChatParams if (type === 'branch') return SwitchNodeParams return null }) diff --git a/src/views/flow/components/params/ImageAnalysisParams.vue b/src/views/flow/components/params/ImageAnalysisParams.vue index 779cfad..1802df1 100644 --- a/src/views/flow/components/params/ImageAnalysisParams.vue +++ b/src/views/flow/components/params/ImageAnalysisParams.vue @@ -207,7 +207,8 @@ const formData = reactive({ analysisMode: 'FAST', asynchronous: false, analysisTuning: { - maxWidth: 896, + // Location needs enough pixels to distinguish small dashboard icons. + maxWidth: 1600, maxImages: 12, maxOutputTokens: 384, matchThreshold: 0.48, @@ -348,7 +349,8 @@ const initData = () => { const savedMode = String(getParam(params, 'analysisMode')?.input || '').toUpperCase() formData.analysisMode = savedMode === 'ACCURATE' ? 'ACCURATE' : 'FAST' formData.asynchronous = asynchronous?.input === true || asynchronous?.input === 'true' - formData.analysisTuning.maxWidth = Number(tuning.maxWidth) || (formData.analysisMode === 'FAST' ? 896 : 1280) + formData.analysisTuning.maxWidth = Number(tuning.maxWidth) + || (isLocationMethod.value ? 1600 : (formData.analysisMode === 'FAST' ? 896 : 1280)) formData.analysisTuning.maxImages = Number(tuning.maxImages) || 12 formData.analysisTuning.maxOutputTokens = Number(tuning.maxOutputTokens) || (formData.analysisMode === 'FAST' ? 384 : 512) diff --git a/src/views/flow/components/params/MediaAnalysisParams.vue b/src/views/flow/components/params/MediaAnalysisParams.vue index b53d4af..b4fd83e 100644 --- a/src/views/flow/components/params/MediaAnalysisParams.vue +++ b/src/views/flow/components/params/MediaAnalysisParams.vue @@ -235,6 +235,7 @@ const analysisModes = [ ] // 声音类型同时保存机器可读编码和中文名称；allow-create 允许现场扩展未预置类型。 const soundTypeOptions = [ + { label: '车机发声（不限类型）', value: 'ANY_VEHICLE_SOUND' }, { label: '开机', value: 'POWER_ON' }, { label: '关机', value: 'POWER_OFF' }, { label: '设防', value: 'ARMED' }, @@ -279,6 +280,7 @@ const outputSummary = computed(() => isVideo.value { name: 'expectedLabelName', desc: '目标声音类型' }, { name: 'labelName', desc: '实际声音类型' }, { name: 'matched', desc: '是否识别到已知声音' }, + { name: 'vehicleSoundDetected', desc: '是否检测到车机发声' }, { name: 'confidence', desc: '最高匹配度' }, { name: 'decisionReason', desc: '检测结论' }, { name: 'scores', desc: '各分类匹配度' }, diff --git a/src/views/flow/config.js b/src/views/flow/config.js index cb7451c..69f1e43 100644 --- a/src/views/flow/config.js +++ b/src/views/flow/config.js @@ -15,6 +15,19 @@ import dialogueSvg from './icon/dialogue.svg' import audioSvg from './icon/audio.svg' import touchSvg from './icon/touch.svg' import expressionSvg from './icon/expression.svg' +import { DEFAULT_CHAT_SYSTEM_PROMPT } from './chatPrompt.js' + +const pointOutputChildren = () => [ + { name: 'x', type: 'number', desc: 'X坐标' }, + { name: 'y', type: 'number', desc: 'Y坐标' }, + { name: 'z', type: 'number', desc: 'Z坐标' }, +] +const poseOutputChildren = () => [ + ...pointOutputChildren(), + { name: 'rx', type: 'number', desc: 'RX旋转（弧度）' }, + { name: 'ry', type: 'number', desc: 'RY旋转（弧度）' }, + { name: 'rz', type: 'number', desc: 'RZ旋转（弧度）' }, +] /** * 处理与 http method options 相关的状态或数据，该影响范围仅限当前模块。 @@ -308,6 +321,46 @@ function handler(params) { { collapseTitle: "机器人", nodeList: [ + { + icon: cameraSvg, + name: "视觉定位执行", + type: "visionLocate", + desc: "同帧获取RGBD图像，分析目标并计算工具TCP和机械臂法兰位姿", + action: 'VISION_LOCATE_TARGET', + nodeType: 'EDGE', + nodeParams: [ + { name: "cameraDeviceId", type: "input", input: "", required: true }, + { name: "armDeviceId", type: "input", input: "", required: true }, + { name: "toolCode", type: "input", input: "", required: true }, + { name: "profileCode", type: "input", input: "common.image_analysis.v1", required: true }, + { name: "prompt", type: "input", input: "定位目标中心点", required: true }, + { name: "analysisMode", type: "input", input: "ACCURATE" }, + { name: "mountType", type: "input", input: "EYE_IN_HAND", componentType: "select", selectOptions: [ + { value: "EYE_IN_HAND", label: "末端相机" }, { value: "EYE_TO_HAND", label: "固定相机" } + ] }, + { name: "approachDistanceMm", type: "input", componentType: "number", input: 10, min: 1, max: 20 }, + { name: "pressDistanceMm", type: "input", componentType: "number", input: 0, min: 0, max: 20, step: 0.1 }, + { name: "normalSampleRadius", type: "input", componentType: "number", input: 12, min: 3, max: 40 }, + { name: "confidenceThreshold", type: "input", componentType: "number", input: 0.7, min: 0, max: 1, step: 0.01 }, + { name: "minDepth", type: "input", componentType: "number", input: 0.05, min: 0.01, step: 0.05 }, + { name: "maxDepth", type: "input", componentType: "number", input: 5, min: 0.1, step: 0.1 }, + { name: "maxMovement", type: "input", componentType: "number", input: 0.5, min: 0.01, step: 0.01 }, + { name: "autoMove", type: "input", componentType: "switch", input: false, disabled: true } + ], + outputType: 'json', + outputParams: [ + { name: 'objectPoint', type: 'object', children: pointOutputChildren() }, + { name: 'surfaceNormal', type: 'object', children: pointOutputChildren() }, + { name: 'approachTcpPose', type: 'object', children: poseOutputChildren() }, + { name: 'contactTcpPose', type: 'object', children: poseOutputChildren() }, + { name: 'approachFlangePose', type: 'object', children: poseOutputChildren() }, + { name: 'contactFlangePose', type: 'object', children: poseOutputChildren() }, + { name: 'pixel', type: 'object', children: [] }, + { name: 'depth', type: 'number' }, + { name: 'depthUnitScale', type: 'number' }, + { name: 'intrinsics', type: 'object', children: [] } + ] + }, { icon: videoSvg, name: "AGV控制", @@ -894,4 +947,25 @@ function handler(params) { } ], }, + { + collapseTitle: "闲聊", + nodeList: [{ + icon: dialogueSvg, + name: "闲聊", + type: "chat", + desc: "输入文本并调用大模型进行自然语言闲聊，只返回文本", + action: 'LLM_CHAT', + nodeType: "LLM", + outputType: 'json', + nodeParams: [ + { name: "text", type: "input", input: "", required: true }, + { name: "promptMode", type: "input", input: "CUSTOM", componentType: "select", selectOptions: [ + { value: "FIXED", label: "固定提示词（推荐）" }, { value: "CUSTOM", label: "自定义提示词" } + ] }, + { name: "prompt", type: "input", input: DEFAULT_CHAT_SYSTEM_PROMPT, required: true }, + { name: "invokeTts", type: "input", input: false, componentType: "switch" }, + ], + outputParams: [{ name: 'text', type: 'string', desc: '大模型返回的闲聊文本', disabled: true }] + }] + }, ] diff --git a/src/views/flow/vue-flow/FlowNode.vue b/src/views/flow/vue-flow/FlowNode.vue index a44b597..f867850 100644 --- a/src/views/flow/vue-flow/FlowNode.vue +++ b/src/views/flow/vue-flow/FlowNode.vue @@ -114,6 +114,8 @@ const componentMap = { device_universal: DeviceUniversal, actionQueue: ActionQueue, subFlow: SubFlowNode, + visionLocate: ServiceNode, + chat: ServiceNode, }; const flowStore = useFlowStore(); diff --git a/src/views/resource/device/index.vue b/src/views/resource/device/index.vue index a28535e..a45aea8 100644 --- a/src/views/resource/device/index.vue +++ b/src/views/resource/device/index.vue @@ -1,10 +1,28 @@