feat(flow): 添加视觉定位和闲聊节点功能

- 新增视觉定位执行节点，支持RGBD图像分析和目标定位
- 添加闲聊节点，集成大模型自然语言对话功能
- 配置视觉定位相关的设备绑定和参数设置
- 更新设备资源管理界面，增加机器人筛选功能
- 优化图像分析参数，提高定位精度
- 添加车机发声检测功能到声音分析模块

.env.development

@@ -7,8 +7,8 @@ VITE_APP_ENV = 'development'
 # 招商车研物联网平台/开发环境
 VITE_APP_BASE_API = '/dev-api'
 
-VITE_API_URL = http://localhost:14080
-VITE_WS_URL = ws://localhost:14080/ws
-VITE_PORT = 5174
+VITE_API_URL = http://192.168.0.222:14080
+VITE_WS_URL = ws://192.168.0.222:14080/ws
+VITE_PORT = 80
 VITE_INSPECTION_TYPE = "inspection"
 VITE_MINIO_URL = "http://192.168.28.10:9000/cmvr-iot/"

src/utils/flowResources.js

@@ -77,6 +77,11 @@ export const getResourceRoles = (graph = globalThis.lf?.getGraphData?.()) => (
 
 const isRobotNode = (node) => node?.properties?.nodeType === 'EDGE'
 export const ACTION_QUEUE_ACTION = 'SYSTEM_EXECUTE_ACTION_QUEUE'
+const NAMED_DEVICE_BINDING_ACTIONS = new Set([ACTION_QUEUE_ACTION, 'VISION_LOCATE_TARGET'])
+const VISION_DEVICE_BINDING_SPECS = Object.freeze([
+  { parameterName: 'cameraDeviceId', deviceKind: 5, displayName: '视觉定位相机' },
+  { parameterName: 'armDeviceId', deviceKind: 2, displayName: '视觉定位机械臂' },
+])
 export const actionRequiresDevice = (action) => (
   getDeviceKindForAction(action) != null || action === 'DEVICE_EXECUTE_JSON_COMMAND'
 )
@@ -196,12 +201,31 @@ export const normalizeFlowResourceSchema = (sourceGraph) => {
     }
     target.roleKey ||= defaultRole.roleKey
     target.dispatchPolicy ||= 'ONE'
-    const isActionQueue = properties.action === ACTION_QUEUE_ACTION
-    if (!isActionQueue) target.deviceSlotKey ||= target.deviceBindings?.[0]?.slotKey || ''
+    const usesNamedDeviceBindings = NAMED_DEVICE_BINDING_ACTIONS.has(properties.action)
+    if (!usesNamedDeviceBindings) target.deviceSlotKey ||= target.deviceBindings?.[0]?.slotKey || ''
     target.deviceBindings = Array.isArray(target.deviceBindings) ? target.deviceBindings : []
 
     const targetRole = roles.find((role) => role.roleKey === target.roleKey) || defaultRole
     const deviceParam = properties.nodeParams?.find((param) => param.name === 'deviceId')
+    if (properties.action === 'VISION_LOCATE_TARGET') {
+      VISION_DEVICE_BINDING_SPECS.forEach((spec) => {
+        if (target.deviceBindings.some((binding) => binding?.parameterName === spec.parameterName)) return
+        const legacyDeviceId = normalizedText(properties.nodeParams
+          ?.find((param) => param.name === spec.parameterName)?.input)
+        const matchingSlots = targetRole.deviceSlots.filter((slot) => (
+          Number(slot.deviceKind) === spec.deviceKind
+          && (!legacyDeviceId || !slot.legacyDeviceId || slot.legacyDeviceId === legacyDeviceId)
+        ))
+        let slot = matchingSlots.length === 1 ? matchingSlots[0] : null
+        if (!slot && matchingSlots.length === 0) {
+          slot = createDeviceSlot(spec.deviceKind, targetRole.deviceSlots, spec.displayName)
+          slot.managedBy = FLOW_NODE_RESOURCE_OWNER
+          if (legacyDeviceId) slot.legacyDeviceId = legacyDeviceId
+          targetRole.deviceSlots.push(slot)
+        }
+        if (slot) target.deviceBindings.push({ parameterName: spec.parameterName, slotKey: slot.slotKey })
+      })
+    }
     if (nodeRequiresDevice(node) && !target.deviceSlotKey) {
       const slot = findOrCreateLegacySlot(targetRole, node, legacySlotByIdentity, deviceParam?.input)
       if (slot) target.deviceSlotKey = slot.slotKey
@@ -213,8 +237,8 @@ export const normalizeFlowResourceSchema = (sourceGraph) => {
     targetRole.deviceSlots.forEach((slot) => {
       if (referencedSlotKeys.has(slot.slotKey) && !slot.managedBy) slot.managedBy = FLOW_NODE_RESOURCE_OWNER
     })
-    if (!isActionQueue) delete target.deviceBindings
-    if (isActionQueue) delete target.deviceSlotKey
+    if (!usesNamedDeviceBindings) delete target.deviceBindings
+    if (usesNamedDeviceBindings) delete target.deviceSlotKey
     properties.nodeParams = (properties.nodeParams || []).filter((param) => param.name !== 'deviceId')
     properties.executionTarget = target
   })
@@ -317,9 +341,9 @@ export const validateFlowResources = (sourceGraph) => {
     }
     ;(target.deviceBindings || []).forEach((binding) => {
       if (!binding?.parameterName || !binding?.slotKey) {
-        errors.push({ nodeId: node.id, message: `${node.properties?.name || node.id}存在未配置的队列设备` })
+        errors.push({ nodeId: node.id, message: `${node.properties?.name || node.id}存在未配置的设备绑定` })
       } else if (!(role.deviceSlots || []).some((slot) => slot.slotKey === binding.slotKey)) {
-        errors.push({ nodeId: node.id, message: `${node.properties?.name || node.id}引用的队列设备用途不存在` })
+        errors.push({ nodeId: node.id, message: `${node.properties?.name || node.id}引用的设备用途不存在` })
       }
     })
     if (nodeRequiresDevice(node)

src/views/flow/components/ParamsDrawer.vue

@@ -42,6 +42,8 @@ import SwitchNodeParams from './params/SwitchNodeParams.vue'
 import ActionQueueParams from './params/ActionQueueParams.vue'
 import MediaAnalysisParams from './params/MediaAnalysisParams.vue'
 import ImageAnalysisParams from './params/ImageAnalysisParams.vue'
+import VisionLocateParams from './params/VisionLocateParams.vue'
+import ChatParams from './params/ChatParams.vue'
 import EndParams from './params/EndParams.vue'
 import SubFlowParams from './params/SubFlowParams.vue'
 
@@ -77,6 +79,8 @@ const currentComponent = computed(() => {
   if (type === 'mediaAnalysis') return MediaAnalysisParams
   // 图片分析支持多图、参考图和自由提示词输出，使用独立面板保持媒体分析面板聚焦音视频。
   if (type === 'imageAnalysis') return ImageAnalysisParams
+  if (type === 'visionLocate') return VisionLocateParams
+  if (type === 'chat') return ChatParams
   if (type === 'branch') return SwitchNodeParams
   return null
 })

src/views/flow/components/params/ImageAnalysisParams.vue

@@ -207,7 +207,8 @@ const formData = reactive({
   analysisMode: 'FAST',
   asynchronous: false,
   analysisTuning: {
-    maxWidth: 896,
+    // Location needs enough pixels to distinguish small dashboard icons.
+    maxWidth: 1600,
     maxImages: 12,
     maxOutputTokens: 384,
     matchThreshold: 0.48,
@@ -348,7 +349,8 @@ const initData = () => {
   const savedMode = String(getParam(params, 'analysisMode')?.input || '').toUpperCase()
   formData.analysisMode = savedMode === 'ACCURATE' ? 'ACCURATE' : 'FAST'
   formData.asynchronous = asynchronous?.input === true || asynchronous?.input === 'true'
-  formData.analysisTuning.maxWidth = Number(tuning.maxWidth) || (formData.analysisMode === 'FAST' ? 896 : 1280)
+  formData.analysisTuning.maxWidth = Number(tuning.maxWidth)
+    || (isLocationMethod.value ? 1600 : (formData.analysisMode === 'FAST' ? 896 : 1280))
   formData.analysisTuning.maxImages = Number(tuning.maxImages) || 12
   formData.analysisTuning.maxOutputTokens = Number(tuning.maxOutputTokens)
     || (formData.analysisMode === 'FAST' ? 384 : 512)

src/views/flow/components/params/MediaAnalysisParams.vue

@@ -235,6 +235,7 @@ const analysisModes = [
 ]
 // 声音类型同时保存机器可读编码和中文名称；allow-create 允许现场扩展未预置类型。
 const soundTypeOptions = [
+  { label: '车机发声（不限类型）', value: 'ANY_VEHICLE_SOUND' },
   { label: '开机', value: 'POWER_ON' },
   { label: '关机', value: 'POWER_OFF' },
   { label: '设防', value: 'ARMED' },
@@ -279,6 +280,7 @@ const outputSummary = computed(() => isVideo.value
       { name: 'expectedLabelName', desc: '目标声音类型' },
       { name: 'labelName', desc: '实际声音类型' },
       { name: 'matched', desc: '是否识别到已知声音' },
+      { name: 'vehicleSoundDetected', desc: '是否检测到车机发声' },
       { name: 'confidence', desc: '最高匹配度' },
       { name: 'decisionReason', desc: '检测结论' },
       { name: 'scores', desc: '各分类匹配度' },

src/views/flow/config.js

@@ -15,6 +15,19 @@ import dialogueSvg from './icon/dialogue.svg'
 import audioSvg from './icon/audio.svg'
 import touchSvg from './icon/touch.svg'
 import expressionSvg from './icon/expression.svg'
+import { DEFAULT_CHAT_SYSTEM_PROMPT } from './chatPrompt.js'
+
+const pointOutputChildren = () => [
+  { name: 'x', type: 'number', desc: 'X坐标' },
+  { name: 'y', type: 'number', desc: 'Y坐标' },
+  { name: 'z', type: 'number', desc: 'Z坐标' },
+]
+const poseOutputChildren = () => [
+  ...pointOutputChildren(),
+  { name: 'rx', type: 'number', desc: 'RX旋转（弧度）' },
+  { name: 'ry', type: 'number', desc: 'RY旋转（弧度）' },
+  { name: 'rz', type: 'number', desc: 'RZ旋转（弧度）' },
+]
 
 /**
  * 处理与 http method options 相关的状态或数据，该影响范围仅限当前模块。
@@ -308,6 +321,46 @@ function handler(params) {
   {
     collapseTitle: "机器人",
     nodeList: [
+      {
+        icon: cameraSvg,
+        name: "视觉定位执行",
+        type: "visionLocate",
+        desc: "同帧获取RGBD图像，分析目标并计算工具TCP和机械臂法兰位姿",
+        action: 'VISION_LOCATE_TARGET',
+        nodeType: 'EDGE',
+        nodeParams: [
+          { name: "cameraDeviceId", type: "input", input: "", required: true },
+          { name: "armDeviceId", type: "input", input: "", required: true },
+          { name: "toolCode", type: "input", input: "", required: true },
+          { name: "profileCode", type: "input", input: "common.image_analysis.v1", required: true },
+          { name: "prompt", type: "input", input: "定位目标中心点", required: true },
+          { name: "analysisMode", type: "input", input: "ACCURATE" },
+          { name: "mountType", type: "input", input: "EYE_IN_HAND", componentType: "select", selectOptions: [
+            { value: "EYE_IN_HAND", label: "末端相机" }, { value: "EYE_TO_HAND", label: "固定相机" }
+          ] },
+          { name: "approachDistanceMm", type: "input", componentType: "number", input: 10, min: 1, max: 20 },
+          { name: "pressDistanceMm", type: "input", componentType: "number", input: 0, min: 0, max: 20, step: 0.1 },
+          { name: "normalSampleRadius", type: "input", componentType: "number", input: 12, min: 3, max: 40 },
+          { name: "confidenceThreshold", type: "input", componentType: "number", input: 0.7, min: 0, max: 1, step: 0.01 },
+          { name: "minDepth", type: "input", componentType: "number", input: 0.05, min: 0.01, step: 0.05 },
+          { name: "maxDepth", type: "input", componentType: "number", input: 5, min: 0.1, step: 0.1 },
+          { name: "maxMovement", type: "input", componentType: "number", input: 0.5, min: 0.01, step: 0.01 },
+          { name: "autoMove", type: "input", componentType: "switch", input: false, disabled: true }
+        ],
+        outputType: 'json',
+        outputParams: [
+          { name: 'objectPoint', type: 'object', children: pointOutputChildren() },
+          { name: 'surfaceNormal', type: 'object', children: pointOutputChildren() },
+          { name: 'approachTcpPose', type: 'object', children: poseOutputChildren() },
+          { name: 'contactTcpPose', type: 'object', children: poseOutputChildren() },
+          { name: 'approachFlangePose', type: 'object', children: poseOutputChildren() },
+          { name: 'contactFlangePose', type: 'object', children: poseOutputChildren() },
+          { name: 'pixel', type: 'object', children: [] },
+          { name: 'depth', type: 'number' },
+          { name: 'depthUnitScale', type: 'number' },
+          { name: 'intrinsics', type: 'object', children: [] }
+        ]
+      },
       {
         icon: videoSvg,
         name: "AGV控制",
@@ -894,4 +947,25 @@ function handler(params) {
       }
     ],
   },
+  {
+    collapseTitle: "闲聊",
+    nodeList: [{
+      icon: dialogueSvg,
+      name: "闲聊",
+      type: "chat",
+      desc: "输入文本并调用大模型进行自然语言闲聊，只返回文本",
+      action: 'LLM_CHAT',
+      nodeType: "LLM",
+      outputType: 'json',
+      nodeParams: [
+        { name: "text", type: "input", input: "", required: true },
+        { name: "promptMode", type: "input", input: "CUSTOM", componentType: "select", selectOptions: [
+          { value: "FIXED", label: "固定提示词（推荐）" }, { value: "CUSTOM", label: "自定义提示词" }
+        ] },
+        { name: "prompt", type: "input", input: DEFAULT_CHAT_SYSTEM_PROMPT, required: true },
+        { name: "invokeTts", type: "input", input: false, componentType: "switch" },
+      ],
+      outputParams: [{ name: 'text', type: 'string', desc: '大模型返回的闲聊文本', disabled: true }]
+    }]
+  },
 ]

src/views/flow/vue-flow/FlowNode.vue

@@ -114,6 +114,8 @@ const componentMap = {
   device_universal: DeviceUniversal,
   actionQueue: ActionQueue,
   subFlow: SubFlowNode,
+  visionLocate: ServiceNode,
+  chat: ServiceNode,
 };
 
 const flowStore = useFlowStore();

src/views/resource/device/index.vue

@@ -1,10 +1,28 @@
 <template>
   <div class="app-container">
     <el-form :inline="true" :model="query">
-      <el-form-item label="机器人ID"><el-input v-model="query.robotId" clearable @keyup.enter="search" /></el-form-item>
+      <el-form-item label="机器人">
+        <el-select
+          v-model="query.robotId"
+          filterable
+          clearable
+          :loading="robotLoading"
+          placeholder="全部机器人"
+          style="width: 280px"
+          @change="handleRobotChange"
+        >
+          <el-option label="全部机器人" value="" />
+          <el-option
+            v-for="robot in robots"
+            :key="robot.robotId"
+            :label="robotOptionLabel(robot)"
+            :value="robot.robotId"
+          />
+        </el-select>
+      </el-form-item>
       <el-form-item label="设备用途"><el-input v-model="query.deviceRole" clearable @keyup.enter="search" /></el-form-item>
       <el-form-item label="在线状态">
-        <el-select v-model="onlineFilter" clearable style="width: 120px">
+        <el-select v-model="onlineFilter" clearable style="width: 120px" @change="search">
           <el-option label="在线" value="online" /><el-option label="离线" value="offline" />
         </el-select>
       </el-form-item>
@@ -49,13 +67,17 @@ import { onActivated, onMounted, reactive, ref } from 'vue'
 import { useRouter } from 'vue-router'
 import { ElMessage } from 'element-plus'
 import { listResourceDevices, updateResourceDevice } from '@/api/resource/device'
+import { listResourceRobots } from '@/api/resource/robot'
 
 const kindNames = { 1: '移动底盘', 2: '机械臂', 3: '电池', 4: '仿生头', 5: '摄像头', 6: 'CAN总线', 7: '灵巧手', 8: '夹爪', 9: '麦克风', 10: '电机', 11: '电机系统', 12: '机器人本体', 13: '扬声器' }
 const router = useRouter()
-const loading = ref(false), saving = ref(false), rows = ref([]), total = ref(0), configOpen = ref(false), onlineFilter = ref('')
+const loading = ref(false), robotLoading = ref(false), saving = ref(false), rows = ref([]), robots = ref([]), total = ref(0), configOpen = ref(false), onlineFilter = ref('')
 const query = reactive({ pageNum: 1, pageSize: 20, robotId: '', deviceRole: '', onlineOnly: false })
 const active = reactive({})
 const kindName = value => kindNames[value] || `未知(${value ?? '-'})`
+const robotOptionLabel = robot => robot.robotName
+  ? `${robot.robotName}（${robot.robotId}）`
+  : robot.robotId
 const controlPath = device => ({
   2: 'mechanical_arm', 4: 'head', 5: 'camera', 7: 'hand', 9: 'microphone', 13: 'speaker',
 })[device.deviceKind]
@@ -79,7 +101,17 @@ async function load() {
     total.value = onlineFilter.value === 'offline' ? rows.value.length : (res.total ?? rows.value.length)
   } finally { loading.value = false }
 }
+async function loadRobots() {
+  robotLoading.value = true
+  try {
+    const res = await listResourceRobots({ pageNum: 1, pageSize: 10000 })
+    robots.value = (res.rows || res.data || []).filter(robot => robot.robotId)
+  } finally {
+    robotLoading.value = false
+  }
+}
 function search() { query.pageNum = 1; load() }
+function handleRobotChange() { search() }
 function reset() { Object.assign(query, { pageNum: 1, robotId: '', deviceRole: '', onlineOnly: false }); onlineFilter.value = ''; load() }
 function openConfig(row) {
   Object.assign(active, JSON.parse(JSON.stringify(row)))
@@ -94,8 +126,9 @@ async function saveConfig() {
     load()
   } finally { saving.value = false }
 }
-onMounted(load)
-onActivated(load)
+const refresh = () => Promise.all([loadRobots(), load()])
+onMounted(refresh)
+onActivated(refresh)
 </script>
 
 <style scoped>