From d4fb83ee537738c1d5fb313d0d51e64598cce1d7 Mon Sep 17 00:00:00 2001 From: lixiaolong <702156524@qq.com> Date: Wed, 16 Sep 2026 13:39:37 +0800 Subject: [PATCH] =?UTF-8?q?feat(inspection):=20=E5=A2=9E=E5=8A=A0=E5=B7=A1?= =?UTF-8?q?=E6=A3=80=E7=82=B9=E4=BD=8D=E8=AF=AD=E6=96=99=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在 inspection_waypoint 表中新增 chinese_corpus 和 english_corpus 字段存储中英文语料 - 创建 InspectionWaypointCorpusConfigVO 用于巡检点位语料查询节点参数 - 实现 InspectionWaypointCorpusMapper 提供按检测项ID查询语料的数据库操作 - 开发 InspectionWaypointCorpusOperateService 处理语料类型识别和语料获取逻辑 - 添加语料类型支持包括中文/英文、zh-CN/en-US、zh/en或0/1等多种格式 - 实现语料验证机制确保巡检点位配置了相应语言语料 - 编写单元测试验证中英文语料获取和类型解析功能 - 为 VisionLocateOperateService 增加完整的测试用例覆盖安全方向校准和平面定位功能 --- .../web/controller/test/TeFlowController.java | 1 + .../src/main/resources/application.yml | 2 +- .../inspection/domain/InspectionWaypoint.java | 8 + .../domain/vo/InspectionWaypointVo.java | 10 + .../impl/InspectionWaypointServiceImpl.java | 2 + .../inspection/InspectionWaypointMapper.xml | 17 +- .../java/com/cmvr/test/enums/ActionEnum.java | 1 + .../edge/VisionLocateOperateService.java | 294 +++++++++++++----- .../test/model/vo/FlowActionRequestVO.java | 2 +- .../LLMMediaAnalysisOperateServiceTest.java | 46 ++- sql/inspection_module.sql | 2 + 11 files changed, 285 insertions(+), 100 deletions(-) diff --git a/cmvr-iot-admin/src/main/java/com/cmvr/web/controller/test/TeFlowController.java b/cmvr-iot-admin/src/main/java/com/cmvr/web/controller/test/TeFlowController.java index 0e3a043..41bcae6 100644 --- a/cmvr-iot-admin/src/main/java/com/cmvr/web/controller/test/TeFlowController.java +++ b/cmvr-iot-admin/src/main/java/com/cmvr/web/controller/test/TeFlowController.java @@ -311,6 +311,7 @@ public class TeFlowController extends BaseController { @ApiOperation(value = "单个节点执行", notes = "设备通用指令action=DEVICE_EXECUTE_JSON_COMMAND，payload需要robotId、deviceId、requestJson；" + "巡检仪表节点action=INSPECTION_METER_RECOGNIZE，payload使用InspectionMeterRecognizeConfigVO；" + + "点位语料节点action=INSPECTION_WAYPOINT_CORPUS，payload需要detectItemId、type；" + "人工判断节点action=INSPECTION_MANUAL_REVIEW_CREATE，payload使用InspectionManualReviewConfigVO。" + "该接口用于试调节点，不会生成正式巡检任务结果") public AjaxResult actionExecute( diff --git a/cmvr-iot-admin/src/main/resources/application.yml b/cmvr-iot-admin/src/main/resources/application.yml index c2db5cb..45de4dc 100644 --- a/cmvr-iot-admin/src/main/resources/application.yml +++ b/cmvr-iot-admin/src/main/resources/application.yml @@ -60,7 +60,7 @@ spring: # 国际化资源文件路径 basename: i18n/messages profiles: - active: test + active: aima # 文件上传 servlet: multipart: diff --git a/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/domain/InspectionWaypoint.java b/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/domain/InspectionWaypoint.java index a066798..1fdadf9 100644 --- a/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/domain/InspectionWaypoint.java +++ b/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/domain/InspectionWaypoint.java @@ -45,6 +45,14 @@ public class InspectionWaypoint extends BaseEntity @ApiModelProperty("检测项配置id") private String detectItemId; + @Excel(name = "中文语料") + @ApiModelProperty("该点位可播放的中文语料") + private String chineseCorpus; + + @Excel(name = "英文语料") + @ApiModelProperty("该点位可播放的英文语料") + private String englishCorpus; + @Excel(name = "地图id") @ApiModelProperty("地图id") private String mapId; diff --git a/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/domain/vo/InspectionWaypointVo.java b/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/domain/vo/InspectionWaypointVo.java index ec9dfd4..5f216ce 100644 --- a/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/domain/vo/InspectionWaypointVo.java +++ b/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/domain/vo/InspectionWaypointVo.java @@ -48,6 +48,16 @@ public class InspectionWaypointVo extends BaseEntity @ApiModelProperty("检测项配置id") private String detectItemId; + /** 中文语料 */ + @Excel(name = "中文语料") + @ApiModelProperty("该点位可播放的中文语料") + private String chineseCorpus; + + /** 英文语料 */ + @Excel(name = "英文语料") + @ApiModelProperty("该点位可播放的英文语料") + private String englishCorpus; + /** 配置状态 */ @Excel(name = "配置状态") @ApiModelProperty("配置状态") diff --git a/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/service/impl/InspectionWaypointServiceImpl.java b/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/service/impl/InspectionWaypointServiceImpl.java index 67e73d0..12336ba 100644 --- a/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/service/impl/InspectionWaypointServiceImpl.java +++ b/cmvr-iot-inspection/src/main/java/com/cmvr/inspection/service/impl/InspectionWaypointServiceImpl.java @@ -102,6 +102,8 @@ public class InspectionWaypointServiceImpl extends ServiceImpl + + @@ -29,6 +31,8 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" + + @@ -37,7 +41,9 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" - select id, create_by, create_time, update_by, update_time, remark, waypoint_name, enabled, sort_order, detect_item_id, map_id from inspection_waypoint + select id, create_by, create_time, update_by, update_time, remark, waypoint_name, enabled, + sort_order, detect_item_id, chinese_corpus, english_corpus, map_id + from inspection_waypoint select w.id, w.create_by, w.create_time, w.update_by, w.update_time, w.remark, - w.waypoint_name, w.enabled, w.sort_order, w.detect_item_id, w.map_id, m.map_name as map_name, + w.waypoint_name, w.enabled, w.sort_order, w.detect_item_id, + w.chinese_corpus, w.english_corpus, w.map_id, m.map_name as map_name, d.is_deploy as config_status, u1.nick_name as create_by_name, u2.nick_name as update_by_name from inspection_waypoint w diff --git a/cmvr-iot-test/src/main/java/com/cmvr/test/enums/ActionEnum.java b/cmvr-iot-test/src/main/java/com/cmvr/test/enums/ActionEnum.java index 7969af3..7a25eef 100644 --- a/cmvr-iot-test/src/main/java/com/cmvr/test/enums/ActionEnum.java +++ b/cmvr-iot-test/src/main/java/com/cmvr/test/enums/ActionEnum.java @@ -87,6 +87,7 @@ public enum ActionEnum { GENERATE_ADVANCED_AUDIO("LLM", "GENERATE_ADVANCED_AUDIO", "tts语音合成"), AI_AGENT_PLATFORM("LLM", "AI_AGENT_PLATFORM", "商道智能体"), INSPECTION_METER_RECOGNIZE("LLM", "INSPECTION_METER_RECOGNIZE", "巡检仪表读数识别"), + INSPECTION_WAYPOINT_CORPUS("LLM", "INSPECTION_WAYPOINT_CORPUS", "获取巡检点位播放语料"), AI_TTS("LLM", "AI_TTS", "tts语音播放"), GET_CURRENT_PAGE("LLM", "GET_CURRENT_PAGE", "获取当前页面名称"), AUDIO_EVENT_CLASSIFY("LLM", "AUDIO_EVENT_CLASSIFY", "声音类型检测"), diff --git a/cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/operator/edge/VisionLocateOperateService.java b/cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/operator/edge/VisionLocateOperateService.java index 79f5a20..fd9447a 100644 --- a/cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/operator/edge/VisionLocateOperateService.java +++ b/cmvr-iot-test/src/main/java/com/cmvr/test/flow/runtime/operator/edge/VisionLocateOperateService.java @@ -10,11 +10,11 @@ import com.cmvr.edge.client.model.EdgeCommonVO; import com.cmvr.edge.client.model.camera.EdgeCameraRgbdSnapshot; import com.cmvr.edge.client.service.EdgeArmService; import com.cmvr.edge.client.service.EdgeCameraService; -import com.cmvr.llm.analysis.MediaAnalysisClient; import com.cmvr.test.enums.ActionEnum; import com.cmvr.test.flow.runtime.message.TaskNodeExecuteMessage; import com.cmvr.test.flow.runtime.message.TaskNodeExecuteResult; import com.cmvr.test.flow.runtime.operator.FlowMediaParamResolver; +import com.cmvr.test.flow.runtime.operator.llm.LLMMediaAnalysisOperateService; import com.cmvr.test.vision.domain.VisionCalibrationProfile; import com.cmvr.test.vision.math.VisionCalibrationMath; import com.cmvr.test.vision.service.VisionCalibrationProfileService; @@ -28,7 +28,6 @@ import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.UUID; @Service @RequiredArgsConstructor @@ -36,7 +35,7 @@ import java.util.UUID; public class VisionLocateOperateService implements EdgeOperateService { private final EdgeCameraService cameraService; private final EdgeArmService armService; - private final MediaAnalysisClient mediaAnalysisClient; + private final LLMMediaAnalysisOperateService imageAnalysisService; private final VisionCalibrationProfileService calibrationProfileService; @Override @@ -135,7 +134,7 @@ public class VisionLocateOperateService implements EdgeOperateService { } else { throw new GlobalException("mountType只支持EYE_IN_HAND或EYE_TO_HAND"); } - double[] objectPoint = VisionGeometry.transform(baseToCamera, + double[] rgbdObjectPoint = VisionGeometry.transform(baseToCamera, surface.cameraPoint[0], surface.cameraPoint[1], surface.cameraPoint[2]); double[] surfaceNormal = normalize(rotate(baseToCamera, surface.normalCamera)); double approachMm = input.getDouble("approachDistanceMm") == null @@ -144,13 +143,23 @@ public class VisionLocateOperateService implements EdgeOperateService { approachMm = Math.max(1.0, Math.min(20.0, approachMm)); double pressMm = Math.max(-20.0, Math.min(20.0, doubleValue(input, "pressDistanceMm", 0.0))); double[][] currentTcp = VisionGeometry.multiply(currentFlange, flangeToTcp); - // The contact direction is the only authoritative click direction. - // Build it once at the contact point, then change height only for the - // approach pose. Re-solving the approach orientation independently is - // unsafe when the fitted depth normal contains small fluctuations. - double[] toolZ = scale(surfaceNormal, -1.0); + // A reflective screen does not provide a stable depth-plane normal. + // Rebuilding the full tool orientation from that noisy normal made the + // physical tool tilt and caused equivalent Euler representations to + // jump in rx/rz. Tool calibration already records the operator-verified + // perpendicular pose; use it as the authoritative click orientation. + // Legacy/manual configurations have no such profile, so keep their + // current TCP orientation and update position only. + StableToolOrientation stableOrientation = stableToolOrientation( + currentTcp, flangeToTcp, toolProfile); + double[][] orientedTcp = stableOrientation.tcpPose; + double[] toolZ = normalize(new double[]{orientedTcp[0][2], orientedTcp[1][2], orientedTcp[2][2]}); + TargetPosition targetPosition = targetPosition( + baseToCamera, snapshot, pixel.x, pixel.y, rgbdObjectPoint, + surface.samplesCamera, stableOrientation, input.getDouble("planeOffsetMm")); + double[] objectPoint = targetPosition.point; double[] contactPosition = add(objectPoint, scale(toolZ, pressMm / 1000.0)); - double[][] contactTcp = targetToolPose(currentTcp, toolZ, contactPosition); + double[][] contactTcp = translatedPose(orientedTcp, contactPosition); double[] approachPosition = add(contactPosition, scale(toolZ, -(approachMm + pressMm) / 1000.0)); double[][] approachTcp = translatedPose(contactTcp, approachPosition); double[][] tcpToFlange = VisionGeometry.rigidInverse(flangeToTcp); @@ -170,8 +179,15 @@ public class VisionLocateOperateService implements EdgeOperateService { output.put("confidence", pixel.confidence); output.put("depth", depth); output.put("cameraPoint", point3(surface.cameraPoint[0], surface.cameraPoint[1], depth)); + output.put("rgbdObjectPoint", point3( + rgbdObjectPoint[0], rgbdObjectPoint[1], rgbdObjectPoint[2])); output.put("objectPoint", point3(objectPoint[0], objectPoint[1], objectPoint[2])); + output.put("positionSource", targetPosition.source); + output.put("depthResidualMm", targetPosition.depthResidualMm); + output.put("planeOffsetMm", input.getDouble("planeOffsetMm")); output.put("surfaceNormal", point3(surfaceNormal[0], surfaceNormal[1], surfaceNormal[2])); + output.put("toolAxis", point3(toolZ[0], toolZ[1], toolZ[2])); + output.put("orientationSource", stableOrientation.source); output.put("surfaceFitErrorMm", surface.rmse * 1000.0); output.put("approachTcpPose", poseJson(VisionGeometry.pose(approachTcp))); output.put("contactTcpPose", poseJson(VisionGeometry.pose(contactTcp))); @@ -197,62 +213,81 @@ public class VisionLocateOperateService implements EdgeOperateService { .fluentPut("fy", snapshot.getFy()) .fluentPut("cx", snapshot.getCx()) .fluentPut("cy", snapshot.getCy())); + log.info("视觉定位三维结果: pixel=({},{}), rgbdZ={}, targetZ={}, depthResidualMm={}, " + + "positionSource={}, orientationSource={}, contactFlangePose={}", + pixel.x, pixel.y, rgbdObjectPoint[2], objectPoint[2], targetPosition.depthResidualMm, + targetPosition.source, stableOrientation.source, + output.getJSONObject("contactFlangePose")); return TaskNodeExecuteResult.success(output); } - private JSONObject analyze(TaskNodeExecuteMessage message, String imageUrl, String profileCode, - String prompt, JSONObject input, String effectiveAnalysisMethod) { - JSONObject options = new JSONObject(); + JSONObject analyze(TaskNodeExecuteMessage message, String imageUrl, String profileCode, + String prompt, JSONObject input, String effectiveAnalysisMethod) { String analysisMethod = StringUtils.defaultIfBlank(effectiveAnalysisMethod, "TEXT_LOCATE").toUpperCase(); - options.put("prompt", "REFERENCE_MATCH".equals(analysisMethod) - ? "在大图中查找参考小图，返回目标中心坐标。" + JSONObject analysisInput = new JSONObject(); + analysisInput.put("profileCode", profileCode); + analysisInput.put("imageUrl", new JSONArray().fluentAdd(imageUrl)); + analysisInput.put("prompt", "REFERENCE_MATCH".equals(analysisMethod) + ? "在大图中查找小图，返回原图像素坐标 [x1,y1,x2,y2]" : ("TEXT_LOCATE".equals(analysisMethod) ? prompt : buildLocatePrompt(prompt))); - // Reference matching is deterministic OpenCV matching; FAST/ACCURATE - // only select model strategies for description-based analysis. Force - // the same advertised mode as the generic precise-image component so - // legacy nodes cannot silently diverge in their request metadata. - options.put("analysisMode", "REFERENCE_MATCH".equals(analysisMethod) + // 精确找图直接复用通用图片分析组件的 FAST + REFERENCE_MATCH 请求， + // 不在视觉定位节点中维护第二套匹配实现或参数契约。 + analysisInput.put("analysisMode", "REFERENCE_MATCH".equals(analysisMethod) ? "FAST" : StringUtils.defaultIfBlank(input.getString("analysisMode"), "FAST")); - options.put("analysisMethod", analysisMethod); + analysisInput.put("analysisMethod", analysisMethod); + analysisInput.put("asynchronous", false); String referenceImageUrl = referenceImageUrl(input); - if (StringUtils.isNotBlank(referenceImageUrl)) options.put("referenceImageUrl", referenceImageUrl); + if (StringUtils.isNotBlank(referenceImageUrl)) { + analysisInput.put("referenceImageUrl", referenceImageUrl); + } JSONObject tuning = input.getJSONObject("analysisTuning"); - if (tuning == null) tuning = new JSONObject(); + tuning = tuning == null ? new JSONObject() : new JSONObject(tuning); if (input.getString("matchSelector") != null) tuning.put("matchSelector", input.getString("matchSelector")); if (input.get("matchIndex") != null) tuning.put("matchIndex", input.get("matchIndex")); if (input.getString("matchSort") != null) tuning.put("matchSort", input.getString("matchSort")); if (input.get("matchThreshold") != null) tuning.put("matchThreshold", input.get("matchThreshold")); - // Keep precise-location matching consistent with the generic image - // analysis component. Older visual-location nodes do not have an - // analysisTuning object, so fill in the same stable defaults here. + if (input.get("locateConfidenceThreshold") != null) { + tuning.put("locateConfidenceThreshold", input.get("locateConfidenceThreshold")); + } else if (input.get("confidenceThreshold") != null) { + tuning.put("locateConfidenceThreshold", input.get("confidenceThreshold")); + } + // 旧视觉定位节点没有 analysisTuning，补齐通用图片分析组件当前的 + // 精确找图默认值，保证单独使用视觉定位时发出的匹配请求完全一致。 if ("REFERENCE_MATCH".equals(analysisMethod)) { - // Preserve small-icon detail for template matching. The analysis - // service still caps this value for very large source images. - if (!tuning.containsKey("maxWidth")) tuning.put("maxWidth", 1600); - if (!tuning.containsKey("maxImages")) tuning.put("maxImages", 1); + if (!tuning.containsKey("maxWidth")) tuning.put("maxWidth", 896); + if (!tuning.containsKey("maxImages")) tuning.put("maxImages", 12); if (!tuning.containsKey("maxOutputTokens")) tuning.put("maxOutputTokens", 384); if (!tuning.containsKey("matchThreshold")) tuning.put("matchThreshold", 0.48D); if (!tuning.containsKey("matchSelector")) tuning.put("matchSelector", "BEST"); if (!tuning.containsKey("matchIndex")) tuning.put("matchIndex", 1); if (!tuning.containsKey("matchSort")) tuning.put("matchSort", "ROW_MAJOR"); + if (!tuning.containsKey("locateConfidenceThreshold")) { + tuning.put("locateConfidenceThreshold", 0.45D); + } } - if (!tuning.isEmpty()) options.put("tuning", tuning); - JSONObject request = new JSONObject(); - request.put("requestId", "vision-locate-" + UUID.randomUUID()); - request.put("analysisType", "IMAGE_ANALYSIS"); - request.put("profileCode", profileCode); - request.put("mediaUrl", imageUrl); - request.put("mediaUrls", List.of(imageUrl)); - request.put("options", options); - request.put("context", new JSONObject() - .fluentPut("flowInstId", message.getInstId()) - .fluentPut("nodeId", message.getNodeId()) - .fluentPut("trial", message.isTrial())); - JSONObject response = mediaAnalysisClient.analyze(request); - JSONObject result = response.getJSONObject("result"); - if (result == null) throw new GlobalException("图片分析未返回目标结果"); - return result; + if (!tuning.isEmpty()) analysisInput.put("analysisTuning", tuning); + + TaskNodeExecuteMessage analysisMessage = imageAnalysisMessage(message, analysisInput); + TaskNodeExecuteResult analysisResult = imageAnalysisService.execute(analysisMessage); + if (analysisResult == null || !analysisResult.isSuccess() + || analysisResult.getOutputParams() == null) { + String error = analysisResult == null ? null : analysisResult.getErrorMsg(); + throw new GlobalException(StringUtils.defaultIfBlank(error, "图片分析未返回目标结果")); + } + return analysisResult.getOutputParams(); + } + + private TaskNodeExecuteMessage imageAnalysisMessage(TaskNodeExecuteMessage source, JSONObject input) { + TaskNodeExecuteMessage message = new TaskNodeExecuteMessage(); + message.setInstId(source.getInstId()); + message.setTaskId(source.getTaskId()); + message.setItemId(source.getItemId()); + message.setNodeId(source.getNodeId()); + message.setTrial(source.isTrial()); + message.setAction(ActionEnum.IMAGE_ANALYZE); + message.setInputParams(input); + return message; } private String referenceImageUrl(JSONObject input) { @@ -450,6 +485,12 @@ public class VisionLocateOperateService implements EdgeOperateService { private SurfacePoint fitSurface(EdgeCameraRgbdSnapshot snapshot, double centerX, double centerY, int radius, Double configuredScale) { + // Position is tied to the requested image pixel. A local plane fit is + // useful as diagnostics, but its slope must not move the target point. + double centerDepth = medianDepth( + snapshot, centerX, centerY, Math.max(radius, 8), configuredScale); + double[] center = {(centerX - snapshot.getCx()) * centerDepth / snapshot.getFx(), + (centerY - snapshot.getCy()) * centerDepth / snapshot.getFy(), centerDepth}; List points = new ArrayList<>(); int step = Math.max(1, radius / 6); for (int y = (int) Math.round(centerY) - radius; y <= (int) Math.round(centerY) + radius; y += step) { @@ -468,15 +509,14 @@ public class VisionLocateOperateService implements EdgeOperateService { // Reflective phone screens often contain only a few valid stereo // pixels. Keep the target depth and use the camera optical normal // as a conservative fallback instead of failing the whole node. - double depth = medianDepth(snapshot, centerX, centerY, Math.max(radius, 8), configuredScale); - double[] center = {(centerX - snapshot.getCx()) * depth / snapshot.getFx(), - (centerY - snapshot.getCy()) * depth / snapshot.getFy(), depth}; - return new SurfacePoint(center, new double[]{0, 0, -1}, 0.0); + return new SurfacePoint(center, new double[]{0, 0, -1}, 0.0, points); } List depths = points.stream().map(point -> point[2]).sorted().toList(); double median = depths.get(depths.size() / 2); points.removeIf(point -> Math.abs(point[2] - median) > 0.02); - if (points.size() < 15) return fallbackSurface(snapshot, centerX, centerY, radius, configuredScale); + if (points.size() < 15) { + return new SurfacePoint(center, new double[]{0, 0, -1}, 0.0, points); + } double[][] normal = new double[3][3]; double[] rhs = new double[3]; @@ -491,24 +531,13 @@ public class VisionLocateOperateService implements EdgeOperateService { double length = Math.sqrt(solution[0] * solution[0] + solution[1] * solution[1] + 1); double[] planeNormal = {solution[0] / length, solution[1] / length, -1 / length}; double planeD = solution[2] / length; - double[] ray = {(centerX - snapshot.getCx()) / snapshot.getFx(), - (centerY - snapshot.getCy()) / snapshot.getFy(), 1}; - double rayScale = -planeD / dot(planeNormal, ray); - double[] center = scale(ray, rayScale); double squaredError = 0; for (double[] point : points) { double error = dot(planeNormal, point) + planeD; squaredError += error * error; } - return new SurfacePoint(center, planeNormal, Math.sqrt(squaredError / points.size())); - } - - private SurfacePoint fallbackSurface(EdgeCameraRgbdSnapshot snapshot, double centerX, double centerY, - int radius, Double configuredScale) { - double depth = medianDepth(snapshot, centerX, centerY, Math.max(radius, 8), configuredScale); - double[] center = {(centerX - snapshot.getCx()) * depth / snapshot.getFx(), - (centerY - snapshot.getCy()) * depth / snapshot.getFy(), depth}; - return new SurfacePoint(center, new double[]{0, 0, -1}, 0.0); + return new SurfacePoint( + center, planeNormal, Math.sqrt(squaredError / points.size()), points); } private double[] solve3(double[][] source, double[] right) { @@ -537,23 +566,119 @@ public class VisionLocateOperateService implements EdgeOperateService { return result; } - private double[][] targetToolPose(double[][] currentTcp, double[] targetZ, double[] position) { - targetZ = normalize(targetZ); - double[] xSeed = {currentTcp[0][0], currentTcp[1][0], currentTcp[2][0]}; - double[] x = subtract(xSeed, scale(targetZ, dot(xSeed, targetZ))); - if (length(x) < 1e-8) { - xSeed = new double[]{currentTcp[0][1], currentTcp[1][1], currentTcp[2][1]}; - x = subtract(xSeed, scale(targetZ, dot(xSeed, targetZ))); + StableToolOrientation stableToolOrientation(double[][] currentTcp, double[][] flangeToTcp, + VisionCalibrationProfile toolProfile) { + if (toolProfile == null || StringUtils.isBlank(toolProfile.getMetadataJson())) { + return new StableToolOrientation(currentTcp, "CURRENT_TCP", null, null); } - x = normalize(x); - double[] y = normalize(cross(targetZ, x)); - x = normalize(cross(y, targetZ)); - double[][] result = new double[4][4]; - result[3][3] = 1; - for (int row = 0; row < 3; row++) { - result[row][0] = x[row]; result[row][1] = y[row]; result[row][2] = targetZ[row]; result[row][3] = position[row]; + try { + JSONObject metadata = JSON.parseObject(toolProfile.getMetadataJson()); + Object safeDirectionPose = metadata == null ? null : metadata.get("safeDirectionPose"); + if (safeDirectionPose == null) { + return new StableToolOrientation(currentTcp, "CURRENT_TCP", null, null); + } + double[][] safeFlange = VisionGeometry.parseMatrix( + safeDirectionPose, "toolCalibration.safeDirectionPose"); + double[][] safeTcp = VisionGeometry.multiply(safeFlange, flangeToTcp); + double[] planeNormal = vector3(metadata.get("planeNormalBase")); + double[] planePoint = planeNormal == null ? null + : new double[]{safeTcp[0][3], safeTcp[1][3], safeTcp[2][3]}; + return new StableToolOrientation( + safeTcp, "TOOL_CALIBRATION_SAFE_DIRECTION", planePoint, planeNormal); + } catch (RuntimeException exception) { + log.warn("工具标定安全方向无效，保持当前TCP姿态: profileId={}", + toolProfile.getId(), exception); + return new StableToolOrientation( + currentTcp, "CURRENT_TCP_INVALID_CALIBRATION_FALLBACK", null, null); } - return result; + } + + TargetPosition targetPosition(double[][] baseToCamera, EdgeCameraRgbdSnapshot snapshot, + double pixelX, double pixelY, double[] rgbdObjectPoint, + List samplesCamera, + StableToolOrientation orientation, + Double configuredPlaneOffsetMm) { + if (orientation.planeNormal == null) { + return new TargetPosition(rgbdObjectPoint, "RGBD_MEDIAN_DEPTH", 0.0); + } + double[] rayCamera = {(pixelX - snapshot.getCx()) / snapshot.getFx(), + (pixelY - snapshot.getCy()) / snapshot.getFy(), 1.0}; + double[] rayOrigin = {baseToCamera[0][3], baseToCamera[1][3], baseToCamera[2][3]}; + double[] rayDirection = rotate(baseToCamera, rayCamera); + double[] planeNormal = normalize(orientation.planeNormal); + double denominator = dot(planeNormal, rayDirection); + if (Math.abs(denominator) < 1e-8) { + throw new GlobalException("目标像素射线与标定平面平行，无法计算稳定坐标"); + } + double planeDistance; + String source; + if (configuredPlaneOffsetMm != null && orientation.planePoint != null) { + // A known height offset is fully deterministic and provides the + // best repeatability for fixtures with discrete height settings. + planeDistance = dot(planeNormal, orientation.planePoint) + + configuredPlaneOffsetMm / 1000.0; + source = "TOOL_CALIBRATION_PLANE_OFFSET"; + } else { + List distances = new ArrayList<>(); + if (samplesCamera != null) { + for (double[] cameraPoint : samplesCamera) { + double[] pointBase = VisionGeometry.transform( + baseToCamera, cameraPoint[0], cameraPoint[1], cameraPoint[2]); + double distance = dot(planeNormal, pointBase); + if (Double.isFinite(distance)) distances.add(distance); + } + } + if (distances.size() < 9) { + // Height cannot be inferred robustly, but the median depth at + // the target pixel still adapts to a moved parallel surface. + return new TargetPosition(rgbdObjectPoint, "RGBD_MEDIAN_DEPTH", 0.0); + } + planeDistance = robustMedian(distances); + source = "RGBD_PARALLEL_PLANE"; + } + double rayScale = (planeDistance - dot(planeNormal, rayOrigin)) / denominator; + if (!Double.isFinite(rayScale) || rayScale <= 0) { + throw new GlobalException("标定平面位于相机目标射线后方，请重新检查工具标定"); + } + double[] point = add(rayOrigin, scale(rayDirection, rayScale)); + double residualMm = dot(planeNormal, subtract(rgbdObjectPoint, point)) * 1000.0; + return new TargetPosition(point, source, residualMm); + } + + private double robustMedian(List values) { + List sorted = new ArrayList<>(values); + Collections.sort(sorted); + double median = sorted.get(sorted.size() / 2); + List deviations = sorted.stream() + .map(value -> Math.abs(value - median)) + .sorted() + .toList(); + double mad = deviations.get(deviations.size() / 2); + if (mad < 1e-9) return median; + double limit = Math.max(0.001, mad * 3.5); + List inliers = sorted.stream() + .filter(value -> Math.abs(value - median) <= limit) + .toList(); + return inliers.get(inliers.size() / 2); + } + + private double[] vector3(Object value) { + JSONArray array; + try { + array = value instanceof String + ? JSON.parseArray(((String) value).trim()) + : JSON.parseArray(JSON.toJSONString(value)); + } catch (RuntimeException exception) { + return null; + } + if (array == null || array.size() != 3) return null; + double[] result = new double[3]; + for (int index = 0; index < 3; index++) { + Double number = array.getDouble(index); + if (number == null || !Double.isFinite(number)) return null; + result[index] = number; + } + return normalize(result); } /** Keep the contact orientation exactly while changing only TCP height. */ @@ -583,7 +708,6 @@ public class VisionLocateOperateService implements EdgeOperateService { private double[] subtract(double[] left, double[] right) { return new double[]{left[0]-right[0], left[1]-right[1], left[2]-right[2]}; } private double[] scale(double[] value, double factor) { return new double[]{value[0]*factor, value[1]*factor, value[2]*factor}; } private double dot(double[] left, double[] right) { return left[0]*right[0]+left[1]*right[1]+left[2]*right[2]; } - private double[] cross(double[] left, double[] right) { return new double[]{left[1]*right[2]-left[2]*right[1], left[2]*right[0]-left[0]*right[2], left[0]*right[1]-left[1]*right[0]}; } private double length(double[] value) { return Math.sqrt(dot(value, value)); } private double[] normalize(double[] value) { double length = length(value); if (length < 1e-10) throw new GlobalException("无法确定目标表面方向"); return scale(value, 1/length); } @@ -656,6 +780,14 @@ public class VisionLocateOperateService implements EdgeOperateService { private record TargetPixel(double x, double y, double confidence) { } - private record SurfacePoint(double[] cameraPoint, double[] normalCamera, double rmse) { + private record SurfacePoint(double[] cameraPoint, double[] normalCamera, double rmse, + List samplesCamera) { + } + + record StableToolOrientation(double[][] tcpPose, String source, + double[] planePoint, double[] planeNormal) { + } + + record TargetPosition(double[] point, String source, double depthResidualMm) { } } diff --git a/cmvr-iot-test/src/main/java/com/cmvr/test/model/vo/FlowActionRequestVO.java b/cmvr-iot-test/src/main/java/com/cmvr/test/model/vo/FlowActionRequestVO.java index cdfbe4a..4fc10cb 100644 --- a/cmvr-iot-test/src/main/java/com/cmvr/test/model/vo/FlowActionRequestVO.java +++ b/cmvr-iot-test/src/main/java/com/cmvr/test/model/vo/FlowActionRequestVO.java @@ -15,7 +15,7 @@ public class FlowActionRequestVO { /** * 动作 */ - @ApiModelProperty(value = "动作编码，例如 DEVICE_EXECUTE_JSON_COMMAND、INSPECTION_METER_RECOGNIZE", + @ApiModelProperty(value = "动作编码，例如 DEVICE_EXECUTE_JSON_COMMAND、INSPECTION_METER_RECOGNIZE、INSPECTION_WAYPOINT_CORPUS", required = true, example = "DEVICE_EXECUTE_JSON_COMMAND") private String action; diff --git a/cmvr-iot-test/src/test/java/com/cmvr/test/flow/runtime/operator/llm/LLMMediaAnalysisOperateServiceTest.java b/cmvr-iot-test/src/test/java/com/cmvr/test/flow/runtime/operator/llm/LLMMediaAnalysisOperateServiceTest.java index c941ea9..c778813 100644 --- a/cmvr-iot-test/src/test/java/com/cmvr/test/flow/runtime/operator/llm/LLMMediaAnalysisOperateServiceTest.java +++ b/cmvr-iot-test/src/test/java/com/cmvr/test/flow/runtime/operator/llm/LLMMediaAnalysisOperateServiceTest.java @@ -10,6 +10,7 @@ import com.cmvr.test.flow.runtime.message.TaskNodeExecuteResult; import org.junit.Test; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -20,7 +21,7 @@ public class LLMMediaAnalysisOperateServiceTest { @Test public void mapsAudioArrayInputAndReturnsAnalysisMetadata() { AtomicReference captured = new AtomicReference<>(); - MediaAnalysisClient client = request -> { + MediaAnalysisClient client = client(request -> { captured.set(request); return new JSONObject() .fluentPut("status", "SUCCEEDED") @@ -31,7 +32,7 @@ public class LLMMediaAnalysisOperateServiceTest { .fluentPut("result", new JSONObject() .fluentPut("label", "POWER_ON") .fluentPut("matched", true)); - }; + }); LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client); JSONObject input = new JSONObject() .fluentPut("profileCode", "aima.power_state.v1") @@ -52,7 +53,7 @@ public class LLMMediaAnalysisOperateServiceTest { @Test public void comparesRecognizedAudioWithExpectedType() { AtomicReference captured = new AtomicReference<>(); - MediaAnalysisClient client = request -> { + MediaAnalysisClient client = client(request -> { captured.set(request); return new JSONObject() .fluentPut("status", "SUCCEEDED") @@ -62,7 +63,7 @@ public class LLMMediaAnalysisOperateServiceTest { .fluentPut("label", "POWER_OFF") .fluentPut("labelName", "关机") .fluentPut("matched", true)); - }; + }); LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client); JSONObject input = new JSONObject() .fluentPut("profileCode", "aima.power_state.v1") @@ -80,14 +81,14 @@ public class LLMMediaAnalysisOperateServiceTest { @Test public void passesBroadVehicleSoundTargetWhenConcreteLabelIsAmbiguous() { - MediaAnalysisClient client = request -> new JSONObject() + MediaAnalysisClient client = client(request -> new JSONObject() .fluentPut("status", "SUCCEEDED") .fluentPut("analysisType", "AUDIO_CLASSIFICATION") .fluentPut("profileCode", "aima.power_state.v1") .fluentPut("result", new JSONObject() .fluentPut("label", "UNKNOWN") .fluentPut("matched", false) - .fluentPut("vehicleSoundDetected", true)); + .fluentPut("vehicleSoundDetected", true))); LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client); JSONObject input = new JSONObject() .fluentPut("profileCode", "aima.power_state.v1") @@ -104,7 +105,7 @@ public class LLMMediaAnalysisOperateServiceTest { @Test public void forwardsVideoInstruction() { AtomicReference captured = new AtomicReference<>(); - MediaAnalysisClient client = request -> { + MediaAnalysisClient client = client(request -> { captured.set(request); return new JSONObject() .fluentPut("status", "SUCCEEDED") @@ -113,7 +114,7 @@ public class LLMMediaAnalysisOperateServiceTest { .fluentPut("timingMs", 30L) .fluentPut("model", new JSONObject().fluentPut("name", "qwen3-vl:32b")) .fluentPut("result", new JSONObject().fluentPut("summary", "ok")); - }; + }); LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client); JSONObject input = new JSONObject() .fluentPut("profileCode", "aima.power_video.v1") @@ -146,7 +147,8 @@ public class LLMMediaAnalysisOperateServiceTest { @Test(expected = GlobalException.class) public void rejectsOutOfRangeVideoTuning() { - LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(request -> new JSONObject()); + LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService( + client(request -> new JSONObject())); JSONObject input = new JSONObject() .fluentPut("profileCode", "aima.power_video.v1") .fluentPut("videoUrl", "https://files.example/video.mp4") @@ -159,7 +161,7 @@ public class LLMMediaAnalysisOperateServiceTest { @Test public void forwardsMultipleImagesAndReferenceImage() { AtomicReference captured = new AtomicReference<>(); - MediaAnalysisClient client = request -> { + MediaAnalysisClient client = client(request -> { captured.set(request); return new JSONObject() .fluentPut("status", "SUCCEEDED") @@ -168,7 +170,7 @@ public class LLMMediaAnalysisOperateServiceTest { .fluentPut("result", new JSONObject() .fluentPut("result", new JSONObject().fluentPut("x", 120).fluentPut("y", 80)) .fluentPut("resultText", "{\"x\":120,\"y\":80}")); - }; + }); LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client); JSONObject input = new JSONObject() .fluentPut("profileCode", "common.image_analysis.v1") @@ -210,7 +212,7 @@ public class LLMMediaAnalysisOperateServiceTest { @Test public void forwardsDescriptionLocationMethodAndThreshold() { AtomicReference captured = new AtomicReference<>(); - MediaAnalysisClient client = request -> { + MediaAnalysisClient client = client(request -> { captured.set(request); return new JSONObject() .fluentPut("status", "SUCCEEDED") @@ -219,7 +221,7 @@ public class LLMMediaAnalysisOperateServiceTest { .fluentPut("result", new JSONObject() .fluentPut("result", new JSONArray().fluentAdd(10).fluentAdd(20).fluentAdd(30).fluentAdd(40)) .fluentPut("matched", true)); - }; + }); LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client); JSONObject input = new JSONObject() .fluentPut("profileCode", "common.image_analysis.v1") @@ -241,7 +243,7 @@ public class LLMMediaAnalysisOperateServiceTest { @Test public void restoresStaticImageUrlArraySerializedByLegacyFlowParamModel() { AtomicReference captured = new AtomicReference<>(); - MediaAnalysisClient client = request -> { + MediaAnalysisClient client = client(request -> { captured.set(request); return new JSONObject() .fluentPut("status", "SUCCEEDED") @@ -250,7 +252,7 @@ public class LLMMediaAnalysisOperateServiceTest { .fluentPut("result", new JSONObject() .fluentPut("result", "图片正常") .fluentPut("resultText", "图片正常")); - }; + }); LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client); JSONObject input = new JSONObject() .fluentPut("profileCode", "common.image_analysis.v1") @@ -281,4 +283,18 @@ public class LLMMediaAnalysisOperateServiceTest { message.setInputParams(input); return message; } + + private MediaAnalysisClient client(Function analyze) { + return new MediaAnalysisClient() { + @Override + public JSONObject analyze(JSONObject request) { + return analyze.apply(request); + } + + @Override + public JSONObject addAudioReference(JSONObject request) { + throw new UnsupportedOperationException("Not used by this test"); + } + }; + } } diff --git a/sql/inspection_module.sql b/sql/inspection_module.sql index aa4b328..57e05a4 100644 --- a/sql/inspection_module.sql +++ b/sql/inspection_module.sql @@ -36,6 +36,8 @@ CREATE TABLE `inspection_waypoint` ( `enabled` char(1) DEFAULT '0' COMMENT '是否启用(0启用 1禁用)', `sort_order` int(11) DEFAULT '0' COMMENT '排序号', `detect_item_id` varchar(64) DEFAULT NULL COMMENT '检测项ID', + `chinese_corpus` text DEFAULT NULL COMMENT '该点位可播放的中文语料', + `english_corpus` text DEFAULT NULL COMMENT '该点位可播放的英文语料', `map_id` varchar(64) DEFAULT NULL COMMENT '地图ID', PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='巡检点位表';