Compare commits
No commits in common. "d4fb83ee537738c1d5fb313d0d51e64598cce1d7" and "356999547c09df77987b60b81b16e4aab8ec1ab4" have entirely different histories.
d4fb83ee53
...
356999547c
@ -311,7 +311,6 @@ public class TeFlowController extends BaseController {
|
||||
@ApiOperation(value = "单个节点执行",
|
||||
notes = "设备通用指令action=DEVICE_EXECUTE_JSON_COMMAND,payload需要robotId、deviceId、requestJson;"
|
||||
+ "巡检仪表节点action=INSPECTION_METER_RECOGNIZE,payload使用InspectionMeterRecognizeConfigVO;"
|
||||
+ "点位语料节点action=INSPECTION_WAYPOINT_CORPUS,payload需要detectItemId、type;"
|
||||
+ "人工判断节点action=INSPECTION_MANUAL_REVIEW_CREATE,payload使用InspectionManualReviewConfigVO。"
|
||||
+ "该接口用于试调节点,不会生成正式巡检任务结果")
|
||||
public AjaxResult actionExecute(
|
||||
|
||||
@ -60,7 +60,7 @@ spring:
|
||||
# 国际化资源文件路径
|
||||
basename: i18n/messages
|
||||
profiles:
|
||||
active: aima
|
||||
active: test
|
||||
# 文件上传
|
||||
servlet:
|
||||
multipart:
|
||||
|
||||
@ -45,14 +45,6 @@ public class InspectionWaypoint extends BaseEntity
|
||||
@ApiModelProperty("检测项配置id")
|
||||
private String detectItemId;
|
||||
|
||||
@Excel(name = "中文语料")
|
||||
@ApiModelProperty("该点位可播放的中文语料")
|
||||
private String chineseCorpus;
|
||||
|
||||
@Excel(name = "英文语料")
|
||||
@ApiModelProperty("该点位可播放的英文语料")
|
||||
private String englishCorpus;
|
||||
|
||||
@Excel(name = "地图id")
|
||||
@ApiModelProperty("地图id")
|
||||
private String mapId;
|
||||
|
||||
@ -48,16 +48,6 @@ public class InspectionWaypointVo extends BaseEntity
|
||||
@ApiModelProperty("检测项配置id")
|
||||
private String detectItemId;
|
||||
|
||||
/** 中文语料 */
|
||||
@Excel(name = "中文语料")
|
||||
@ApiModelProperty("该点位可播放的中文语料")
|
||||
private String chineseCorpus;
|
||||
|
||||
/** 英文语料 */
|
||||
@Excel(name = "英文语料")
|
||||
@ApiModelProperty("该点位可播放的英文语料")
|
||||
private String englishCorpus;
|
||||
|
||||
/** 配置状态 */
|
||||
@Excel(name = "配置状态")
|
||||
@ApiModelProperty("配置状态")
|
||||
|
||||
@ -102,8 +102,6 @@ public class InspectionWaypointServiceImpl extends ServiceImpl<InspectionWaypoin
|
||||
.enabled(source.getEnabled())
|
||||
.sortOrder(source.getSortOrder())
|
||||
.detectItemId(copiedDetectItemId)
|
||||
.chineseCorpus(source.getChineseCorpus())
|
||||
.englishCorpus(source.getEnglishCorpus())
|
||||
.mapId(source.getMapId())
|
||||
.build();
|
||||
copy.setRemark(source.getRemark());
|
||||
|
||||
@ -15,8 +15,6 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
<result property="enabled" column="enabled" />
|
||||
<result property="sortOrder" column="sort_order" />
|
||||
<result property="detectItemId" column="detect_item_id" />
|
||||
<result property="chineseCorpus" column="chinese_corpus" />
|
||||
<result property="englishCorpus" column="english_corpus" />
|
||||
<result property="mapId" column="map_id" />
|
||||
</resultMap>
|
||||
|
||||
@ -31,8 +29,6 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
<result property="enabled" column="enabled" />
|
||||
<result property="sortOrder" column="sort_order" />
|
||||
<result property="detectItemId" column="detect_item_id" />
|
||||
<result property="chineseCorpus" column="chinese_corpus" />
|
||||
<result property="englishCorpus" column="english_corpus" />
|
||||
<result property="configStatus" column="config_status" />
|
||||
<result property="mapId" column="map_id" />
|
||||
<result property="mapName" column="map_name" />
|
||||
@ -41,9 +37,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
</resultMap>
|
||||
|
||||
<sql id="selectInspectionWaypointVo">
|
||||
select id, create_by, create_time, update_by, update_time, remark, waypoint_name, enabled,
|
||||
sort_order, detect_item_id, chinese_corpus, english_corpus, map_id
|
||||
from inspection_waypoint
|
||||
select id, create_by, create_time, update_by, update_time, remark, waypoint_name, enabled, sort_order, detect_item_id, map_id from inspection_waypoint
|
||||
</sql>
|
||||
|
||||
<select id="selectInspectionWaypointById" parameterType="String" resultMap="InspectionWaypointResult">
|
||||
@ -64,8 +58,6 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
<if test="enabled != null">enabled,</if>
|
||||
<if test="sortOrder != null">sort_order,</if>
|
||||
<if test="detectItemId != null">detect_item_id,</if>
|
||||
<if test="chineseCorpus != null">chinese_corpus,</if>
|
||||
<if test="englishCorpus != null">english_corpus,</if>
|
||||
<if test="mapId != null">map_id,</if>
|
||||
</trim>
|
||||
<trim prefix="values (" suffix=")" suffixOverrides=",">
|
||||
@ -79,8 +71,6 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
<if test="enabled != null">#{enabled},</if>
|
||||
<if test="sortOrder != null">#{sortOrder},</if>
|
||||
<if test="detectItemId != null">#{detectItemId},</if>
|
||||
<if test="chineseCorpus != null">#{chineseCorpus},</if>
|
||||
<if test="englishCorpus != null">#{englishCorpus},</if>
|
||||
<if test="mapId != null">#{mapId},</if>
|
||||
</trim>
|
||||
</insert>
|
||||
@ -97,8 +87,6 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
<if test="enabled != null">enabled = #{enabled},</if>
|
||||
<if test="sortOrder != null">sort_order = #{sortOrder},</if>
|
||||
<if test="detectItemId != null">detect_item_id = #{detectItemId},</if>
|
||||
<if test="chineseCorpus != null">chinese_corpus = #{chineseCorpus},</if>
|
||||
<if test="englishCorpus != null">english_corpus = #{englishCorpus},</if>
|
||||
<if test="mapId != null">map_id = #{mapId},</if>
|
||||
</trim>
|
||||
where id = #{id}
|
||||
@ -117,8 +105,7 @@ PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
|
||||
|
||||
<select id="selectInspectionWaypointVoList" parameterType="InspectionWaypoint" resultMap="InspectionWaypointVoResult">
|
||||
select w.id, w.create_by, w.create_time, w.update_by, w.update_time, w.remark,
|
||||
w.waypoint_name, w.enabled, w.sort_order, w.detect_item_id,
|
||||
w.chinese_corpus, w.english_corpus, w.map_id, m.map_name as map_name,
|
||||
w.waypoint_name, w.enabled, w.sort_order, w.detect_item_id, w.map_id, m.map_name as map_name,
|
||||
d.is_deploy as config_status,
|
||||
u1.nick_name as create_by_name, u2.nick_name as update_by_name
|
||||
from inspection_waypoint w
|
||||
|
||||
@ -87,7 +87,6 @@ public enum ActionEnum {
|
||||
GENERATE_ADVANCED_AUDIO("LLM", "GENERATE_ADVANCED_AUDIO", "tts语音合成"),
|
||||
AI_AGENT_PLATFORM("LLM", "AI_AGENT_PLATFORM", "商道智能体"),
|
||||
INSPECTION_METER_RECOGNIZE("LLM", "INSPECTION_METER_RECOGNIZE", "巡检仪表读数识别"),
|
||||
INSPECTION_WAYPOINT_CORPUS("LLM", "INSPECTION_WAYPOINT_CORPUS", "获取巡检点位播放语料"),
|
||||
AI_TTS("LLM", "AI_TTS", "tts语音播放"),
|
||||
GET_CURRENT_PAGE("LLM", "GET_CURRENT_PAGE", "获取当前页面名称"),
|
||||
AUDIO_EVENT_CLASSIFY("LLM", "AUDIO_EVENT_CLASSIFY", "声音类型检测"),
|
||||
|
||||
@ -10,11 +10,11 @@ import com.cmvr.edge.client.model.EdgeCommonVO;
|
||||
import com.cmvr.edge.client.model.camera.EdgeCameraRgbdSnapshot;
|
||||
import com.cmvr.edge.client.service.EdgeArmService;
|
||||
import com.cmvr.edge.client.service.EdgeCameraService;
|
||||
import com.cmvr.llm.analysis.MediaAnalysisClient;
|
||||
import com.cmvr.test.enums.ActionEnum;
|
||||
import com.cmvr.test.flow.runtime.message.TaskNodeExecuteMessage;
|
||||
import com.cmvr.test.flow.runtime.message.TaskNodeExecuteResult;
|
||||
import com.cmvr.test.flow.runtime.operator.FlowMediaParamResolver;
|
||||
import com.cmvr.test.flow.runtime.operator.llm.LLMMediaAnalysisOperateService;
|
||||
import com.cmvr.test.vision.domain.VisionCalibrationProfile;
|
||||
import com.cmvr.test.vision.math.VisionCalibrationMath;
|
||||
import com.cmvr.test.vision.service.VisionCalibrationProfileService;
|
||||
@ -28,6 +28,7 @@ import java.nio.ByteOrder;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@ -35,7 +36,7 @@ import java.util.List;
|
||||
public class VisionLocateOperateService implements EdgeOperateService {
|
||||
private final EdgeCameraService cameraService;
|
||||
private final EdgeArmService armService;
|
||||
private final LLMMediaAnalysisOperateService imageAnalysisService;
|
||||
private final MediaAnalysisClient mediaAnalysisClient;
|
||||
private final VisionCalibrationProfileService calibrationProfileService;
|
||||
|
||||
@Override
|
||||
@ -134,7 +135,7 @@ public class VisionLocateOperateService implements EdgeOperateService {
|
||||
} else {
|
||||
throw new GlobalException("mountType只支持EYE_IN_HAND或EYE_TO_HAND");
|
||||
}
|
||||
double[] rgbdObjectPoint = VisionGeometry.transform(baseToCamera,
|
||||
double[] objectPoint = VisionGeometry.transform(baseToCamera,
|
||||
surface.cameraPoint[0], surface.cameraPoint[1], surface.cameraPoint[2]);
|
||||
double[] surfaceNormal = normalize(rotate(baseToCamera, surface.normalCamera));
|
||||
double approachMm = input.getDouble("approachDistanceMm") == null
|
||||
@ -143,23 +144,13 @@ public class VisionLocateOperateService implements EdgeOperateService {
|
||||
approachMm = Math.max(1.0, Math.min(20.0, approachMm));
|
||||
double pressMm = Math.max(-20.0, Math.min(20.0, doubleValue(input, "pressDistanceMm", 0.0)));
|
||||
double[][] currentTcp = VisionGeometry.multiply(currentFlange, flangeToTcp);
|
||||
// A reflective screen does not provide a stable depth-plane normal.
|
||||
// Rebuilding the full tool orientation from that noisy normal made the
|
||||
// physical tool tilt and caused equivalent Euler representations to
|
||||
// jump in rx/rz. Tool calibration already records the operator-verified
|
||||
// perpendicular pose; use it as the authoritative click orientation.
|
||||
// Legacy/manual configurations have no such profile, so keep their
|
||||
// current TCP orientation and update position only.
|
||||
StableToolOrientation stableOrientation = stableToolOrientation(
|
||||
currentTcp, flangeToTcp, toolProfile);
|
||||
double[][] orientedTcp = stableOrientation.tcpPose;
|
||||
double[] toolZ = normalize(new double[]{orientedTcp[0][2], orientedTcp[1][2], orientedTcp[2][2]});
|
||||
TargetPosition targetPosition = targetPosition(
|
||||
baseToCamera, snapshot, pixel.x, pixel.y, rgbdObjectPoint,
|
||||
surface.samplesCamera, stableOrientation, input.getDouble("planeOffsetMm"));
|
||||
double[] objectPoint = targetPosition.point;
|
||||
// The contact direction is the only authoritative click direction.
|
||||
// Build it once at the contact point, then change height only for the
|
||||
// approach pose. Re-solving the approach orientation independently is
|
||||
// unsafe when the fitted depth normal contains small fluctuations.
|
||||
double[] toolZ = scale(surfaceNormal, -1.0);
|
||||
double[] contactPosition = add(objectPoint, scale(toolZ, pressMm / 1000.0));
|
||||
double[][] contactTcp = translatedPose(orientedTcp, contactPosition);
|
||||
double[][] contactTcp = targetToolPose(currentTcp, toolZ, contactPosition);
|
||||
double[] approachPosition = add(contactPosition, scale(toolZ, -(approachMm + pressMm) / 1000.0));
|
||||
double[][] approachTcp = translatedPose(contactTcp, approachPosition);
|
||||
double[][] tcpToFlange = VisionGeometry.rigidInverse(flangeToTcp);
|
||||
@ -179,15 +170,8 @@ public class VisionLocateOperateService implements EdgeOperateService {
|
||||
output.put("confidence", pixel.confidence);
|
||||
output.put("depth", depth);
|
||||
output.put("cameraPoint", point3(surface.cameraPoint[0], surface.cameraPoint[1], depth));
|
||||
output.put("rgbdObjectPoint", point3(
|
||||
rgbdObjectPoint[0], rgbdObjectPoint[1], rgbdObjectPoint[2]));
|
||||
output.put("objectPoint", point3(objectPoint[0], objectPoint[1], objectPoint[2]));
|
||||
output.put("positionSource", targetPosition.source);
|
||||
output.put("depthResidualMm", targetPosition.depthResidualMm);
|
||||
output.put("planeOffsetMm", input.getDouble("planeOffsetMm"));
|
||||
output.put("surfaceNormal", point3(surfaceNormal[0], surfaceNormal[1], surfaceNormal[2]));
|
||||
output.put("toolAxis", point3(toolZ[0], toolZ[1], toolZ[2]));
|
||||
output.put("orientationSource", stableOrientation.source);
|
||||
output.put("surfaceFitErrorMm", surface.rmse * 1000.0);
|
||||
output.put("approachTcpPose", poseJson(VisionGeometry.pose(approachTcp)));
|
||||
output.put("contactTcpPose", poseJson(VisionGeometry.pose(contactTcp)));
|
||||
@ -213,81 +197,62 @@ public class VisionLocateOperateService implements EdgeOperateService {
|
||||
.fluentPut("fy", snapshot.getFy())
|
||||
.fluentPut("cx", snapshot.getCx())
|
||||
.fluentPut("cy", snapshot.getCy()));
|
||||
log.info("视觉定位三维结果: pixel=({},{}), rgbdZ={}, targetZ={}, depthResidualMm={}, "
|
||||
+ "positionSource={}, orientationSource={}, contactFlangePose={}",
|
||||
pixel.x, pixel.y, rgbdObjectPoint[2], objectPoint[2], targetPosition.depthResidualMm,
|
||||
targetPosition.source, stableOrientation.source,
|
||||
output.getJSONObject("contactFlangePose"));
|
||||
return TaskNodeExecuteResult.success(output);
|
||||
}
|
||||
|
||||
JSONObject analyze(TaskNodeExecuteMessage message, String imageUrl, String profileCode,
|
||||
String prompt, JSONObject input, String effectiveAnalysisMethod) {
|
||||
private JSONObject analyze(TaskNodeExecuteMessage message, String imageUrl, String profileCode,
|
||||
String prompt, JSONObject input, String effectiveAnalysisMethod) {
|
||||
JSONObject options = new JSONObject();
|
||||
String analysisMethod = StringUtils.defaultIfBlank(effectiveAnalysisMethod, "TEXT_LOCATE").toUpperCase();
|
||||
JSONObject analysisInput = new JSONObject();
|
||||
analysisInput.put("profileCode", profileCode);
|
||||
analysisInput.put("imageUrl", new JSONArray().fluentAdd(imageUrl));
|
||||
analysisInput.put("prompt", "REFERENCE_MATCH".equals(analysisMethod)
|
||||
? "在大图中查找小图,返回原图像素坐标 [x1,y1,x2,y2]"
|
||||
options.put("prompt", "REFERENCE_MATCH".equals(analysisMethod)
|
||||
? "在大图中查找参考小图,返回目标中心坐标。"
|
||||
: ("TEXT_LOCATE".equals(analysisMethod) ? prompt : buildLocatePrompt(prompt)));
|
||||
// 精确找图直接复用通用图片分析组件的 FAST + REFERENCE_MATCH 请求,
|
||||
// 不在视觉定位节点中维护第二套匹配实现或参数契约。
|
||||
analysisInput.put("analysisMode", "REFERENCE_MATCH".equals(analysisMethod)
|
||||
// Reference matching is deterministic OpenCV matching; FAST/ACCURATE
|
||||
// only select model strategies for description-based analysis. Force
|
||||
// the same advertised mode as the generic precise-image component so
|
||||
// legacy nodes cannot silently diverge in their request metadata.
|
||||
options.put("analysisMode", "REFERENCE_MATCH".equals(analysisMethod)
|
||||
? "FAST"
|
||||
: StringUtils.defaultIfBlank(input.getString("analysisMode"), "FAST"));
|
||||
analysisInput.put("analysisMethod", analysisMethod);
|
||||
analysisInput.put("asynchronous", false);
|
||||
options.put("analysisMethod", analysisMethod);
|
||||
String referenceImageUrl = referenceImageUrl(input);
|
||||
if (StringUtils.isNotBlank(referenceImageUrl)) {
|
||||
analysisInput.put("referenceImageUrl", referenceImageUrl);
|
||||
}
|
||||
if (StringUtils.isNotBlank(referenceImageUrl)) options.put("referenceImageUrl", referenceImageUrl);
|
||||
JSONObject tuning = input.getJSONObject("analysisTuning");
|
||||
tuning = tuning == null ? new JSONObject() : new JSONObject(tuning);
|
||||
if (tuning == null) tuning = new JSONObject();
|
||||
if (input.getString("matchSelector") != null) tuning.put("matchSelector", input.getString("matchSelector"));
|
||||
if (input.get("matchIndex") != null) tuning.put("matchIndex", input.get("matchIndex"));
|
||||
if (input.getString("matchSort") != null) tuning.put("matchSort", input.getString("matchSort"));
|
||||
if (input.get("matchThreshold") != null) tuning.put("matchThreshold", input.get("matchThreshold"));
|
||||
if (input.get("locateConfidenceThreshold") != null) {
|
||||
tuning.put("locateConfidenceThreshold", input.get("locateConfidenceThreshold"));
|
||||
} else if (input.get("confidenceThreshold") != null) {
|
||||
tuning.put("locateConfidenceThreshold", input.get("confidenceThreshold"));
|
||||
}
|
||||
// 旧视觉定位节点没有 analysisTuning,补齐通用图片分析组件当前的
|
||||
// 精确找图默认值,保证单独使用视觉定位时发出的匹配请求完全一致。
|
||||
// Keep precise-location matching consistent with the generic image
|
||||
// analysis component. Older visual-location nodes do not have an
|
||||
// analysisTuning object, so fill in the same stable defaults here.
|
||||
if ("REFERENCE_MATCH".equals(analysisMethod)) {
|
||||
if (!tuning.containsKey("maxWidth")) tuning.put("maxWidth", 896);
|
||||
if (!tuning.containsKey("maxImages")) tuning.put("maxImages", 12);
|
||||
// Preserve small-icon detail for template matching. The analysis
|
||||
// service still caps this value for very large source images.
|
||||
if (!tuning.containsKey("maxWidth")) tuning.put("maxWidth", 1600);
|
||||
if (!tuning.containsKey("maxImages")) tuning.put("maxImages", 1);
|
||||
if (!tuning.containsKey("maxOutputTokens")) tuning.put("maxOutputTokens", 384);
|
||||
if (!tuning.containsKey("matchThreshold")) tuning.put("matchThreshold", 0.48D);
|
||||
if (!tuning.containsKey("matchSelector")) tuning.put("matchSelector", "BEST");
|
||||
if (!tuning.containsKey("matchIndex")) tuning.put("matchIndex", 1);
|
||||
if (!tuning.containsKey("matchSort")) tuning.put("matchSort", "ROW_MAJOR");
|
||||
if (!tuning.containsKey("locateConfidenceThreshold")) {
|
||||
tuning.put("locateConfidenceThreshold", 0.45D);
|
||||
}
|
||||
}
|
||||
if (!tuning.isEmpty()) analysisInput.put("analysisTuning", tuning);
|
||||
|
||||
TaskNodeExecuteMessage analysisMessage = imageAnalysisMessage(message, analysisInput);
|
||||
TaskNodeExecuteResult analysisResult = imageAnalysisService.execute(analysisMessage);
|
||||
if (analysisResult == null || !analysisResult.isSuccess()
|
||||
|| analysisResult.getOutputParams() == null) {
|
||||
String error = analysisResult == null ? null : analysisResult.getErrorMsg();
|
||||
throw new GlobalException(StringUtils.defaultIfBlank(error, "图片分析未返回目标结果"));
|
||||
}
|
||||
return analysisResult.getOutputParams();
|
||||
}
|
||||
|
||||
private TaskNodeExecuteMessage imageAnalysisMessage(TaskNodeExecuteMessage source, JSONObject input) {
|
||||
TaskNodeExecuteMessage message = new TaskNodeExecuteMessage();
|
||||
message.setInstId(source.getInstId());
|
||||
message.setTaskId(source.getTaskId());
|
||||
message.setItemId(source.getItemId());
|
||||
message.setNodeId(source.getNodeId());
|
||||
message.setTrial(source.isTrial());
|
||||
message.setAction(ActionEnum.IMAGE_ANALYZE);
|
||||
message.setInputParams(input);
|
||||
return message;
|
||||
if (!tuning.isEmpty()) options.put("tuning", tuning);
|
||||
JSONObject request = new JSONObject();
|
||||
request.put("requestId", "vision-locate-" + UUID.randomUUID());
|
||||
request.put("analysisType", "IMAGE_ANALYSIS");
|
||||
request.put("profileCode", profileCode);
|
||||
request.put("mediaUrl", imageUrl);
|
||||
request.put("mediaUrls", List.of(imageUrl));
|
||||
request.put("options", options);
|
||||
request.put("context", new JSONObject()
|
||||
.fluentPut("flowInstId", message.getInstId())
|
||||
.fluentPut("nodeId", message.getNodeId())
|
||||
.fluentPut("trial", message.isTrial()));
|
||||
JSONObject response = mediaAnalysisClient.analyze(request);
|
||||
JSONObject result = response.getJSONObject("result");
|
||||
if (result == null) throw new GlobalException("图片分析未返回目标结果");
|
||||
return result;
|
||||
}
|
||||
|
||||
private String referenceImageUrl(JSONObject input) {
|
||||
@ -485,12 +450,6 @@ public class VisionLocateOperateService implements EdgeOperateService {
|
||||
|
||||
private SurfacePoint fitSurface(EdgeCameraRgbdSnapshot snapshot, double centerX, double centerY,
|
||||
int radius, Double configuredScale) {
|
||||
// Position is tied to the requested image pixel. A local plane fit is
|
||||
// useful as diagnostics, but its slope must not move the target point.
|
||||
double centerDepth = medianDepth(
|
||||
snapshot, centerX, centerY, Math.max(radius, 8), configuredScale);
|
||||
double[] center = {(centerX - snapshot.getCx()) * centerDepth / snapshot.getFx(),
|
||||
(centerY - snapshot.getCy()) * centerDepth / snapshot.getFy(), centerDepth};
|
||||
List<double[]> points = new ArrayList<>();
|
||||
int step = Math.max(1, radius / 6);
|
||||
for (int y = (int) Math.round(centerY) - radius; y <= (int) Math.round(centerY) + radius; y += step) {
|
||||
@ -509,14 +468,15 @@ public class VisionLocateOperateService implements EdgeOperateService {
|
||||
// Reflective phone screens often contain only a few valid stereo
|
||||
// pixels. Keep the target depth and use the camera optical normal
|
||||
// as a conservative fallback instead of failing the whole node.
|
||||
return new SurfacePoint(center, new double[]{0, 0, -1}, 0.0, points);
|
||||
double depth = medianDepth(snapshot, centerX, centerY, Math.max(radius, 8), configuredScale);
|
||||
double[] center = {(centerX - snapshot.getCx()) * depth / snapshot.getFx(),
|
||||
(centerY - snapshot.getCy()) * depth / snapshot.getFy(), depth};
|
||||
return new SurfacePoint(center, new double[]{0, 0, -1}, 0.0);
|
||||
}
|
||||
List<Double> depths = points.stream().map(point -> point[2]).sorted().toList();
|
||||
double median = depths.get(depths.size() / 2);
|
||||
points.removeIf(point -> Math.abs(point[2] - median) > 0.02);
|
||||
if (points.size() < 15) {
|
||||
return new SurfacePoint(center, new double[]{0, 0, -1}, 0.0, points);
|
||||
}
|
||||
if (points.size() < 15) return fallbackSurface(snapshot, centerX, centerY, radius, configuredScale);
|
||||
|
||||
double[][] normal = new double[3][3];
|
||||
double[] rhs = new double[3];
|
||||
@ -531,13 +491,24 @@ public class VisionLocateOperateService implements EdgeOperateService {
|
||||
double length = Math.sqrt(solution[0] * solution[0] + solution[1] * solution[1] + 1);
|
||||
double[] planeNormal = {solution[0] / length, solution[1] / length, -1 / length};
|
||||
double planeD = solution[2] / length;
|
||||
double[] ray = {(centerX - snapshot.getCx()) / snapshot.getFx(),
|
||||
(centerY - snapshot.getCy()) / snapshot.getFy(), 1};
|
||||
double rayScale = -planeD / dot(planeNormal, ray);
|
||||
double[] center = scale(ray, rayScale);
|
||||
double squaredError = 0;
|
||||
for (double[] point : points) {
|
||||
double error = dot(planeNormal, point) + planeD;
|
||||
squaredError += error * error;
|
||||
}
|
||||
return new SurfacePoint(
|
||||
center, planeNormal, Math.sqrt(squaredError / points.size()), points);
|
||||
return new SurfacePoint(center, planeNormal, Math.sqrt(squaredError / points.size()));
|
||||
}
|
||||
|
||||
private SurfacePoint fallbackSurface(EdgeCameraRgbdSnapshot snapshot, double centerX, double centerY,
|
||||
int radius, Double configuredScale) {
|
||||
double depth = medianDepth(snapshot, centerX, centerY, Math.max(radius, 8), configuredScale);
|
||||
double[] center = {(centerX - snapshot.getCx()) * depth / snapshot.getFx(),
|
||||
(centerY - snapshot.getCy()) * depth / snapshot.getFy(), depth};
|
||||
return new SurfacePoint(center, new double[]{0, 0, -1}, 0.0);
|
||||
}
|
||||
|
||||
private double[] solve3(double[][] source, double[] right) {
|
||||
@ -566,119 +537,23 @@ public class VisionLocateOperateService implements EdgeOperateService {
|
||||
return result;
|
||||
}
|
||||
|
||||
StableToolOrientation stableToolOrientation(double[][] currentTcp, double[][] flangeToTcp,
|
||||
VisionCalibrationProfile toolProfile) {
|
||||
if (toolProfile == null || StringUtils.isBlank(toolProfile.getMetadataJson())) {
|
||||
return new StableToolOrientation(currentTcp, "CURRENT_TCP", null, null);
|
||||
private double[][] targetToolPose(double[][] currentTcp, double[] targetZ, double[] position) {
|
||||
targetZ = normalize(targetZ);
|
||||
double[] xSeed = {currentTcp[0][0], currentTcp[1][0], currentTcp[2][0]};
|
||||
double[] x = subtract(xSeed, scale(targetZ, dot(xSeed, targetZ)));
|
||||
if (length(x) < 1e-8) {
|
||||
xSeed = new double[]{currentTcp[0][1], currentTcp[1][1], currentTcp[2][1]};
|
||||
x = subtract(xSeed, scale(targetZ, dot(xSeed, targetZ)));
|
||||
}
|
||||
try {
|
||||
JSONObject metadata = JSON.parseObject(toolProfile.getMetadataJson());
|
||||
Object safeDirectionPose = metadata == null ? null : metadata.get("safeDirectionPose");
|
||||
if (safeDirectionPose == null) {
|
||||
return new StableToolOrientation(currentTcp, "CURRENT_TCP", null, null);
|
||||
}
|
||||
double[][] safeFlange = VisionGeometry.parseMatrix(
|
||||
safeDirectionPose, "toolCalibration.safeDirectionPose");
|
||||
double[][] safeTcp = VisionGeometry.multiply(safeFlange, flangeToTcp);
|
||||
double[] planeNormal = vector3(metadata.get("planeNormalBase"));
|
||||
double[] planePoint = planeNormal == null ? null
|
||||
: new double[]{safeTcp[0][3], safeTcp[1][3], safeTcp[2][3]};
|
||||
return new StableToolOrientation(
|
||||
safeTcp, "TOOL_CALIBRATION_SAFE_DIRECTION", planePoint, planeNormal);
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("工具标定安全方向无效,保持当前TCP姿态: profileId={}",
|
||||
toolProfile.getId(), exception);
|
||||
return new StableToolOrientation(
|
||||
currentTcp, "CURRENT_TCP_INVALID_CALIBRATION_FALLBACK", null, null);
|
||||
x = normalize(x);
|
||||
double[] y = normalize(cross(targetZ, x));
|
||||
x = normalize(cross(y, targetZ));
|
||||
double[][] result = new double[4][4];
|
||||
result[3][3] = 1;
|
||||
for (int row = 0; row < 3; row++) {
|
||||
result[row][0] = x[row]; result[row][1] = y[row]; result[row][2] = targetZ[row]; result[row][3] = position[row];
|
||||
}
|
||||
}
|
||||
|
||||
TargetPosition targetPosition(double[][] baseToCamera, EdgeCameraRgbdSnapshot snapshot,
|
||||
double pixelX, double pixelY, double[] rgbdObjectPoint,
|
||||
List<double[]> samplesCamera,
|
||||
StableToolOrientation orientation,
|
||||
Double configuredPlaneOffsetMm) {
|
||||
if (orientation.planeNormal == null) {
|
||||
return new TargetPosition(rgbdObjectPoint, "RGBD_MEDIAN_DEPTH", 0.0);
|
||||
}
|
||||
double[] rayCamera = {(pixelX - snapshot.getCx()) / snapshot.getFx(),
|
||||
(pixelY - snapshot.getCy()) / snapshot.getFy(), 1.0};
|
||||
double[] rayOrigin = {baseToCamera[0][3], baseToCamera[1][3], baseToCamera[2][3]};
|
||||
double[] rayDirection = rotate(baseToCamera, rayCamera);
|
||||
double[] planeNormal = normalize(orientation.planeNormal);
|
||||
double denominator = dot(planeNormal, rayDirection);
|
||||
if (Math.abs(denominator) < 1e-8) {
|
||||
throw new GlobalException("目标像素射线与标定平面平行,无法计算稳定坐标");
|
||||
}
|
||||
double planeDistance;
|
||||
String source;
|
||||
if (configuredPlaneOffsetMm != null && orientation.planePoint != null) {
|
||||
// A known height offset is fully deterministic and provides the
|
||||
// best repeatability for fixtures with discrete height settings.
|
||||
planeDistance = dot(planeNormal, orientation.planePoint)
|
||||
+ configuredPlaneOffsetMm / 1000.0;
|
||||
source = "TOOL_CALIBRATION_PLANE_OFFSET";
|
||||
} else {
|
||||
List<Double> distances = new ArrayList<>();
|
||||
if (samplesCamera != null) {
|
||||
for (double[] cameraPoint : samplesCamera) {
|
||||
double[] pointBase = VisionGeometry.transform(
|
||||
baseToCamera, cameraPoint[0], cameraPoint[1], cameraPoint[2]);
|
||||
double distance = dot(planeNormal, pointBase);
|
||||
if (Double.isFinite(distance)) distances.add(distance);
|
||||
}
|
||||
}
|
||||
if (distances.size() < 9) {
|
||||
// Height cannot be inferred robustly, but the median depth at
|
||||
// the target pixel still adapts to a moved parallel surface.
|
||||
return new TargetPosition(rgbdObjectPoint, "RGBD_MEDIAN_DEPTH", 0.0);
|
||||
}
|
||||
planeDistance = robustMedian(distances);
|
||||
source = "RGBD_PARALLEL_PLANE";
|
||||
}
|
||||
double rayScale = (planeDistance - dot(planeNormal, rayOrigin)) / denominator;
|
||||
if (!Double.isFinite(rayScale) || rayScale <= 0) {
|
||||
throw new GlobalException("标定平面位于相机目标射线后方,请重新检查工具标定");
|
||||
}
|
||||
double[] point = add(rayOrigin, scale(rayDirection, rayScale));
|
||||
double residualMm = dot(planeNormal, subtract(rgbdObjectPoint, point)) * 1000.0;
|
||||
return new TargetPosition(point, source, residualMm);
|
||||
}
|
||||
|
||||
private double robustMedian(List<Double> values) {
|
||||
List<Double> sorted = new ArrayList<>(values);
|
||||
Collections.sort(sorted);
|
||||
double median = sorted.get(sorted.size() / 2);
|
||||
List<Double> deviations = sorted.stream()
|
||||
.map(value -> Math.abs(value - median))
|
||||
.sorted()
|
||||
.toList();
|
||||
double mad = deviations.get(deviations.size() / 2);
|
||||
if (mad < 1e-9) return median;
|
||||
double limit = Math.max(0.001, mad * 3.5);
|
||||
List<Double> inliers = sorted.stream()
|
||||
.filter(value -> Math.abs(value - median) <= limit)
|
||||
.toList();
|
||||
return inliers.get(inliers.size() / 2);
|
||||
}
|
||||
|
||||
private double[] vector3(Object value) {
|
||||
JSONArray array;
|
||||
try {
|
||||
array = value instanceof String
|
||||
? JSON.parseArray(((String) value).trim())
|
||||
: JSON.parseArray(JSON.toJSONString(value));
|
||||
} catch (RuntimeException exception) {
|
||||
return null;
|
||||
}
|
||||
if (array == null || array.size() != 3) return null;
|
||||
double[] result = new double[3];
|
||||
for (int index = 0; index < 3; index++) {
|
||||
Double number = array.getDouble(index);
|
||||
if (number == null || !Double.isFinite(number)) return null;
|
||||
result[index] = number;
|
||||
}
|
||||
return normalize(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Keep the contact orientation exactly while changing only TCP height. */
|
||||
@ -708,6 +583,7 @@ public class VisionLocateOperateService implements EdgeOperateService {
|
||||
private double[] subtract(double[] left, double[] right) { return new double[]{left[0]-right[0], left[1]-right[1], left[2]-right[2]}; }
|
||||
private double[] scale(double[] value, double factor) { return new double[]{value[0]*factor, value[1]*factor, value[2]*factor}; }
|
||||
private double dot(double[] left, double[] right) { return left[0]*right[0]+left[1]*right[1]+left[2]*right[2]; }
|
||||
private double[] cross(double[] left, double[] right) { return new double[]{left[1]*right[2]-left[2]*right[1], left[2]*right[0]-left[0]*right[2], left[0]*right[1]-left[1]*right[0]}; }
|
||||
private double length(double[] value) { return Math.sqrt(dot(value, value)); }
|
||||
private double[] normalize(double[] value) { double length = length(value); if (length < 1e-10) throw new GlobalException("无法确定目标表面方向"); return scale(value, 1/length); }
|
||||
|
||||
@ -780,14 +656,6 @@ public class VisionLocateOperateService implements EdgeOperateService {
|
||||
private record TargetPixel(double x, double y, double confidence) {
|
||||
}
|
||||
|
||||
private record SurfacePoint(double[] cameraPoint, double[] normalCamera, double rmse,
|
||||
List<double[]> samplesCamera) {
|
||||
}
|
||||
|
||||
record StableToolOrientation(double[][] tcpPose, String source,
|
||||
double[] planePoint, double[] planeNormal) {
|
||||
}
|
||||
|
||||
record TargetPosition(double[] point, String source, double depthResidualMm) {
|
||||
private record SurfacePoint(double[] cameraPoint, double[] normalCamera, double rmse) {
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,87 +0,0 @@
|
||||
package com.cmvr.test.flow.runtime.operator.llm;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.cmvr.common.exception.GlobalException;
|
||||
import com.cmvr.test.enums.ActionEnum;
|
||||
import com.cmvr.test.flow.runtime.message.TaskNodeExecuteMessage;
|
||||
import com.cmvr.test.flow.runtime.message.TaskNodeExecuteResult;
|
||||
import com.cmvr.test.mapper.InspectionWaypointCorpusMapper;
|
||||
import com.cmvr.test.model.vo.inspection.InspectionWaypointCorpusConfigVO;
|
||||
import com.cmvr.test.model.vo.inspection.InspectionWaypointCorpusVO;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/** 根据检测项和语料类型读取巡检点位配置的播放语料。 */
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class InspectionWaypointCorpusOperateService implements LLMOperateService
|
||||
{
|
||||
private final InspectionWaypointCorpusMapper inspectionWaypointCorpusMapper;
|
||||
|
||||
@Override
|
||||
public boolean supports(ActionEnum action)
|
||||
{
|
||||
return ActionEnum.INSPECTION_WAYPOINT_CORPUS.equals(action);
|
||||
}
|
||||
|
||||
@Override
|
||||
public TaskNodeExecuteResult execute(TaskNodeExecuteMessage message)
|
||||
{
|
||||
JSONObject input = message.getInputParams() == null ? new JSONObject() : message.getInputParams();
|
||||
InspectionWaypointCorpusConfigVO config = input.to(InspectionWaypointCorpusConfigVO.class);
|
||||
String detectItemId = StringUtils.trimToNull(config.getDetectItemId());
|
||||
String type = StringUtils.trimToNull(config.getType());
|
||||
if (detectItemId == null)
|
||||
{
|
||||
throw new GlobalException("检测项ID不能为空");
|
||||
}
|
||||
if (type == null)
|
||||
{
|
||||
throw new GlobalException("语料类型不能为空");
|
||||
}
|
||||
|
||||
InspectionWaypointCorpusVO waypoint =
|
||||
inspectionWaypointCorpusMapper.selectByDetectItemId(detectItemId);
|
||||
if (waypoint == null)
|
||||
{
|
||||
throw new GlobalException("未找到检测项对应的巡检点位: " + detectItemId);
|
||||
}
|
||||
|
||||
String corpus = resolveCorpus(waypoint, type);
|
||||
if (StringUtils.isBlank(corpus))
|
||||
{
|
||||
throw new GlobalException("巡检点位未配置" + languageName(type) + "语料");
|
||||
}
|
||||
|
||||
return TaskNodeExecuteResult.success(new JSONObject().fluentPut("corpus", corpus));
|
||||
}
|
||||
|
||||
private String resolveCorpus(InspectionWaypointCorpusVO waypoint, String type)
|
||||
{
|
||||
return isChinese(type) ? waypoint.getChineseCorpus() : waypoint.getEnglishCorpus();
|
||||
}
|
||||
|
||||
private String languageName(String type)
|
||||
{
|
||||
return isChinese(type) ? "中文" : "英文";
|
||||
}
|
||||
|
||||
private boolean isChinese(String type)
|
||||
{
|
||||
String normalized = type.trim().toLowerCase(Locale.ROOT).replace('_', '-');
|
||||
if ("0".equals(normalized) || "cn".equals(normalized) || "chinese".equals(normalized)
|
||||
|| "中文".equals(normalized) || normalized.startsWith("zh"))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if ("1".equals(normalized) || "english".equals(normalized) || "英文".equals(normalized)
|
||||
|| normalized.startsWith("en"))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
throw new GlobalException("不支持的语料类型: " + type);
|
||||
}
|
||||
}
|
||||
@ -1,13 +0,0 @@
|
||||
package com.cmvr.test.mapper;
|
||||
|
||||
import com.cmvr.test.model.vo.inspection.InspectionWaypointCorpusVO;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
|
||||
/** 巡检点位语料查询Mapper。 */
|
||||
public interface InspectionWaypointCorpusMapper
|
||||
{
|
||||
@Select("select chinese_corpus as chineseCorpus, english_corpus as englishCorpus "
|
||||
+ "from inspection_waypoint where detect_item_id = #{detectItemId} limit 1")
|
||||
InspectionWaypointCorpusVO selectByDetectItemId(@Param("detectItemId") String detectItemId);
|
||||
}
|
||||
@ -15,7 +15,7 @@ public class FlowActionRequestVO {
|
||||
/**
|
||||
* 动作
|
||||
*/
|
||||
@ApiModelProperty(value = "动作编码,例如 DEVICE_EXECUTE_JSON_COMMAND、INSPECTION_METER_RECOGNIZE、INSPECTION_WAYPOINT_CORPUS",
|
||||
@ApiModelProperty(value = "动作编码,例如 DEVICE_EXECUTE_JSON_COMMAND、INSPECTION_METER_RECOGNIZE",
|
||||
required = true, example = "DEVICE_EXECUTE_JSON_COMMAND")
|
||||
private String action;
|
||||
|
||||
|
||||
@ -1,19 +0,0 @@
|
||||
package com.cmvr.test.model.vo.inspection;
|
||||
|
||||
import io.swagger.annotations.ApiModel;
|
||||
import io.swagger.annotations.ApiModelProperty;
|
||||
import lombok.Data;
|
||||
|
||||
/** 巡检点位语料查询工作流节点参数。 */
|
||||
@Data
|
||||
@ApiModel(value = "InspectionWaypointCorpusConfigVO",
|
||||
description = "INSPECTION_WAYPOINT_CORPUS节点参数")
|
||||
public class InspectionWaypointCorpusConfigVO
|
||||
{
|
||||
@ApiModelProperty(value = "点位关联的检测项ID", required = true)
|
||||
private String detectItemId;
|
||||
|
||||
@ApiModelProperty(value = "语料类型,支持中文/英文、zh-CN/en-US、zh/en或0/1",
|
||||
required = true, example = "zh-CN")
|
||||
private String type;
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
package com.cmvr.test.model.vo.inspection;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
/** 巡检点位中英文语料查询结果。 */
|
||||
@Data
|
||||
public class InspectionWaypointCorpusVO
|
||||
{
|
||||
private String chineseCorpus;
|
||||
|
||||
private String englishCorpus;
|
||||
}
|
||||
@ -1,258 +0,0 @@
|
||||
package com.cmvr.test.flow.runtime.operator.edge;
|
||||
|
||||
import com.alibaba.fastjson2.JSONArray;
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.cmvr.llm.analysis.MediaAnalysisClient;
|
||||
import com.cmvr.test.enums.ActionEnum;
|
||||
import com.cmvr.test.flow.runtime.message.TaskNodeExecuteMessage;
|
||||
import com.cmvr.test.flow.runtime.operator.llm.LLMMediaAnalysisOperateService;
|
||||
import com.cmvr.test.vision.domain.VisionCalibrationProfile;
|
||||
import com.cmvr.test.vision.math.VisionCalibrationMath;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.lang.reflect.Proxy;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class VisionLocateOperateServiceTest {
|
||||
|
||||
@Test
|
||||
public void calibratedSafeDirectionKeepsTheFinalFlangeOrientationStable() {
|
||||
VisionLocateOperateService service = new VisionLocateOperateService(
|
||||
null, null, null, null);
|
||||
double[][] currentFlange = VisionCalibrationMath.fromCartesianPose(
|
||||
cmvr.api.ArmCommand.CartesianPose.newBuilder()
|
||||
.setRx(0.45).setRy(-0.3).setRz(0.7).build());
|
||||
double[][] flangeToTcp = VisionCalibrationMath.fromCartesianPose(
|
||||
cmvr.api.ArmCommand.CartesianPose.newBuilder()
|
||||
.setX(0.07).setZ(0.01)
|
||||
.setRx(0.08).setRy(-0.04).setRz(0.12).build());
|
||||
double[][] currentTcp = VisionGeometry.multiply(currentFlange, flangeToTcp);
|
||||
double[][] safeFlange = VisionCalibrationMath.fromCartesianPose(
|
||||
cmvr.api.ArmCommand.CartesianPose.newBuilder()
|
||||
.setRx(Math.PI).setRy(0.04).setRz(-0.2).build());
|
||||
VisionCalibrationProfile toolProfile = new VisionCalibrationProfile();
|
||||
toolProfile.setId("tool-profile-1");
|
||||
toolProfile.setMetadataJson(new JSONObject()
|
||||
.fluentPut("safeDirectionPose", matrixArray(safeFlange))
|
||||
.toJSONString());
|
||||
|
||||
VisionLocateOperateService.StableToolOrientation orientation =
|
||||
service.stableToolOrientation(currentTcp, flangeToTcp, toolProfile);
|
||||
double[][] finalFlange = VisionGeometry.multiply(
|
||||
orientation.tcpPose(), VisionGeometry.rigidInverse(flangeToTcp));
|
||||
|
||||
assertEquals("TOOL_CALIBRATION_SAFE_DIRECTION", orientation.source());
|
||||
assertRotationEquals(safeFlange, finalFlange, 1e-10);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void missingSafeDirectionPreservesCurrentTcpOrientation() {
|
||||
VisionLocateOperateService service = new VisionLocateOperateService(
|
||||
null, null, null, null);
|
||||
double[][] currentTcp = VisionCalibrationMath.fromCartesianPose(
|
||||
cmvr.api.ArmCommand.CartesianPose.newBuilder()
|
||||
.setRx(0.2).setRy(0.4).setRz(-0.6).build());
|
||||
double[][] identity = VisionCalibrationMath.identity();
|
||||
|
||||
VisionLocateOperateService.StableToolOrientation orientation =
|
||||
service.stableToolOrientation(currentTcp, identity, null);
|
||||
|
||||
assertEquals("CURRENT_TCP", orientation.source());
|
||||
assertRotationEquals(currentTcp, orientation.tcpPose(), 0.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void calibratedPlaneMakesTargetZIndependentFromRgbdDepthNoise() {
|
||||
VisionLocateOperateService service = new VisionLocateOperateService(
|
||||
null, null, null, null);
|
||||
double[][] identity = VisionCalibrationMath.identity();
|
||||
double[][] safeFlange = VisionCalibrationMath.identity();
|
||||
safeFlange[2][3] = 0.5;
|
||||
VisionCalibrationProfile toolProfile = new VisionCalibrationProfile();
|
||||
toolProfile.setId("tool-profile-plane");
|
||||
toolProfile.setMetadataJson(new JSONObject()
|
||||
.fluentPut("safeDirectionPose", matrixArray(safeFlange))
|
||||
.fluentPut("planeNormalBase", new JSONArray()
|
||||
.fluentAdd(0.0).fluentAdd(0.0).fluentAdd(1.0))
|
||||
.toJSONString());
|
||||
VisionLocateOperateService.StableToolOrientation orientation =
|
||||
service.stableToolOrientation(identity, identity, toolProfile);
|
||||
com.cmvr.edge.client.model.camera.EdgeCameraRgbdSnapshot snapshot =
|
||||
com.cmvr.edge.client.model.camera.EdgeCameraRgbdSnapshot.builder()
|
||||
.width(640).height(480)
|
||||
.fx(600).fy(600).cx(320).cy(240)
|
||||
.build();
|
||||
|
||||
VisionLocateOperateService.TargetPosition first = service.targetPosition(
|
||||
identity, snapshot, 380, 240, new double[]{0.056, 0, 0.56},
|
||||
List.of(), orientation, 0.0D);
|
||||
VisionLocateOperateService.TargetPosition second = service.targetPosition(
|
||||
identity, snapshot, 380, 240, new double[]{0.062, 0, 0.62},
|
||||
List.of(), orientation, 0.0D);
|
||||
|
||||
assertEquals("TOOL_CALIBRATION_PLANE_OFFSET", first.source());
|
||||
assertEquals(0.05, first.point()[0], 1e-12);
|
||||
assertEquals(0.5, first.point()[2], 1e-12);
|
||||
assertEquals(first.point()[2], second.point()[2], 0.0);
|
||||
assertEquals(60.0, first.depthResidualMm(), 1e-9);
|
||||
assertEquals(120.0, second.depthResidualMm(), 1e-9);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void dynamicParallelPlaneTracksHeightButRejectsDepthOutliers() {
|
||||
VisionLocateOperateService service = new VisionLocateOperateService(
|
||||
null, null, null, null);
|
||||
double[][] identity = VisionCalibrationMath.identity();
|
||||
VisionLocateOperateService.StableToolOrientation orientation =
|
||||
new VisionLocateOperateService.StableToolOrientation(
|
||||
identity, "TEST", new double[]{0, 0, 0.5}, new double[]{0, 0, 1});
|
||||
com.cmvr.edge.client.model.camera.EdgeCameraRgbdSnapshot snapshot =
|
||||
com.cmvr.edge.client.model.camera.EdgeCameraRgbdSnapshot.builder()
|
||||
.width(640).height(480)
|
||||
.fx(600).fy(600).cx(320).cy(240)
|
||||
.build();
|
||||
List<double[]> lowerSamples = List.of(
|
||||
pointAtDepth(0.619), pointAtDepth(0.620), pointAtDepth(0.621),
|
||||
pointAtDepth(0.620), pointAtDepth(0.619), pointAtDepth(0.621),
|
||||
pointAtDepth(0.620), pointAtDepth(0.620), pointAtDepth(0.619),
|
||||
pointAtDepth(0.621), pointAtDepth(0.800));
|
||||
List<double[]> higherSamples = List.of(
|
||||
pointAtDepth(0.679), pointAtDepth(0.680), pointAtDepth(0.681),
|
||||
pointAtDepth(0.680), pointAtDepth(0.679), pointAtDepth(0.681),
|
||||
pointAtDepth(0.680), pointAtDepth(0.680), pointAtDepth(0.679),
|
||||
pointAtDepth(0.681), pointAtDepth(0.900));
|
||||
|
||||
VisionLocateOperateService.TargetPosition lower = service.targetPosition(
|
||||
identity, snapshot, 380, 240, new double[]{0.09, 0, 0.90},
|
||||
lowerSamples, orientation, null);
|
||||
VisionLocateOperateService.TargetPosition higher = service.targetPosition(
|
||||
identity, snapshot, 380, 240, new double[]{0.05, 0, 0.50},
|
||||
higherSamples, orientation, null);
|
||||
|
||||
assertEquals("RGBD_PARALLEL_PLANE", lower.source());
|
||||
assertEquals(0.620, lower.point()[2], 1e-12);
|
||||
assertEquals(0.680, higher.point()[2], 1e-12);
|
||||
assertEquals(0.062, lower.point()[0], 1e-12);
|
||||
assertEquals(0.068, higher.point()[0], 1e-12);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void preciseLocateUsesTheSameImageAnalysisRequestAsGenericPreciseMatch() {
|
||||
AtomicReference<JSONObject> captured = new AtomicReference<>();
|
||||
MediaAnalysisClient client = (MediaAnalysisClient) Proxy.newProxyInstance(
|
||||
MediaAnalysisClient.class.getClassLoader(),
|
||||
new Class<?>[]{MediaAnalysisClient.class},
|
||||
(proxy, method, args) -> {
|
||||
if (!"analyze".equals(method.getName())) {
|
||||
throw new UnsupportedOperationException(method.getName());
|
||||
}
|
||||
JSONObject request = (JSONObject) args[0];
|
||||
captured.set(request);
|
||||
return new JSONObject()
|
||||
.fluentPut("status", "SUCCEEDED")
|
||||
.fluentPut("analysisType", "IMAGE_ANALYSIS")
|
||||
.fluentPut("profileCode", "common.image_analysis.v1")
|
||||
.fluentPut("result", new JSONObject()
|
||||
.fluentPut("matched", true)
|
||||
.fluentPut("center", new JSONArray().fluentAdd(610).fluentAdd(238))
|
||||
.fluentPut("matchScore", 0.8D));
|
||||
}
|
||||
);
|
||||
LLMMediaAnalysisOperateService imageAnalysis =
|
||||
new LLMMediaAnalysisOperateService(client, null);
|
||||
|
||||
imageAnalysis.execute(imageMessage(genericPreciseMatchInput()));
|
||||
JSONObject genericRequest = captured.get();
|
||||
|
||||
VisionLocateOperateService visionLocate = new VisionLocateOperateService(
|
||||
null, null, imageAnalysis, null);
|
||||
visionLocate.analyze(visionMessage(), "https://files.example/scene.jpg",
|
||||
"common.image_analysis.v1", "定位目标中心点", visualLocateInput(),
|
||||
"REFERENCE_MATCH");
|
||||
JSONObject visualRequest = captured.get();
|
||||
|
||||
assertEquals(genericRequest.getString("analysisType"), visualRequest.getString("analysisType"));
|
||||
assertEquals(genericRequest.getString("profileCode"), visualRequest.getString("profileCode"));
|
||||
assertEquals(genericRequest.getString("mediaUrl"), visualRequest.getString("mediaUrl"));
|
||||
assertEquals(genericRequest.getJSONArray("mediaUrls"), visualRequest.getJSONArray("mediaUrls"));
|
||||
assertEquals(genericRequest.getJSONObject("options"), visualRequest.getJSONObject("options"));
|
||||
assertEquals(genericRequest.getJSONObject("context"), visualRequest.getJSONObject("context"));
|
||||
}
|
||||
|
||||
private JSONObject genericPreciseMatchInput() {
|
||||
return new JSONObject()
|
||||
.fluentPut("profileCode", "common.image_analysis.v1")
|
||||
.fluentPut("imageUrl", new JSONArray().fluentAdd("https://files.example/scene.jpg"))
|
||||
.fluentPut("referenceImageUrl", "https://files.example/reference.png")
|
||||
.fluentPut("prompt", "在大图中查找小图,返回原图像素坐标 [x1,y1,x2,y2]")
|
||||
.fluentPut("analysisMethod", "REFERENCE_MATCH")
|
||||
.fluentPut("analysisMode", "FAST")
|
||||
.fluentPut("asynchronous", false)
|
||||
.fluentPut("analysisTuning", new JSONObject()
|
||||
.fluentPut("maxWidth", 896)
|
||||
.fluentPut("maxImages", 12)
|
||||
.fluentPut("maxOutputTokens", 384)
|
||||
.fluentPut("matchThreshold", 0.48D)
|
||||
.fluentPut("matchSelector", "BEST")
|
||||
.fluentPut("matchIndex", 1)
|
||||
.fluentPut("matchSort", "ROW_MAJOR")
|
||||
.fluentPut("locateConfidenceThreshold", 0.45D));
|
||||
}
|
||||
|
||||
private JSONObject visualLocateInput() {
|
||||
return new JSONObject()
|
||||
.fluentPut("referenceImageUrl", "https://files.example/reference.png")
|
||||
.fluentPut("analysisMode", "ACCURATE")
|
||||
.fluentPut("matchSelector", "BEST")
|
||||
.fluentPut("matchIndex", "1")
|
||||
.fluentPut("matchSort", "ROW_MAJOR")
|
||||
.fluentPut("confidenceThreshold", "0.45");
|
||||
}
|
||||
|
||||
private TaskNodeExecuteMessage imageMessage(JSONObject input) {
|
||||
TaskNodeExecuteMessage message = baseMessage();
|
||||
message.setAction(ActionEnum.IMAGE_ANALYZE);
|
||||
message.setInputParams(input);
|
||||
return message;
|
||||
}
|
||||
|
||||
private TaskNodeExecuteMessage visionMessage() {
|
||||
TaskNodeExecuteMessage message = baseMessage();
|
||||
message.setAction(ActionEnum.VISION_LOCATE_TARGET);
|
||||
message.setInputParams(visualLocateInput());
|
||||
return message;
|
||||
}
|
||||
|
||||
private TaskNodeExecuteMessage baseMessage() {
|
||||
TaskNodeExecuteMessage message = new TaskNodeExecuteMessage();
|
||||
message.setInstId("inst-1");
|
||||
message.setTaskId("task-1");
|
||||
message.setItemId("item-1");
|
||||
message.setNodeId("node-1");
|
||||
message.setTrial(true);
|
||||
return message;
|
||||
}
|
||||
|
||||
private JSONArray matrixArray(double[][] matrix) {
|
||||
JSONArray values = new JSONArray();
|
||||
for (double[] row : matrix) {
|
||||
for (double value : row) values.add(value);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private double[] pointAtDepth(double depth) {
|
||||
return new double[]{0, 0, depth};
|
||||
}
|
||||
|
||||
private void assertRotationEquals(double[][] expected, double[][] actual, double tolerance) {
|
||||
for (int row = 0; row < 3; row++) {
|
||||
for (int column = 0; column < 3; column++) {
|
||||
assertEquals(expected[row][column], actual[row][column], tolerance);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,75 +0,0 @@
|
||||
package com.cmvr.test.flow.runtime.operator.llm;
|
||||
|
||||
import com.alibaba.fastjson2.JSONObject;
|
||||
import com.cmvr.common.exception.GlobalException;
|
||||
import com.cmvr.test.enums.ActionEnum;
|
||||
import com.cmvr.test.flow.runtime.message.TaskNodeExecuteMessage;
|
||||
import com.cmvr.test.flow.runtime.message.TaskNodeExecuteResult;
|
||||
import com.cmvr.test.mapper.InspectionWaypointCorpusMapper;
|
||||
import com.cmvr.test.model.vo.inspection.InspectionWaypointCorpusVO;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.lang.reflect.Proxy;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class InspectionWaypointCorpusOperateServiceTest
|
||||
{
|
||||
@Test
|
||||
public void returnsChineseCorpus()
|
||||
{
|
||||
InspectionWaypointCorpusOperateService service = service("开始巡检", "Start inspection");
|
||||
|
||||
TaskNodeExecuteResult result = service.execute(message("zh-CN"));
|
||||
|
||||
assertTrue(result.isSuccess());
|
||||
assertEquals("开始巡检", result.getOutputParams().getString("corpus"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void returnsEnglishCorpusForNumericType()
|
||||
{
|
||||
InspectionWaypointCorpusOperateService service = service("开始巡检", "Start inspection");
|
||||
|
||||
TaskNodeExecuteResult result = service.execute(message(1));
|
||||
|
||||
assertEquals("Start inspection", result.getOutputParams().getString("corpus"));
|
||||
assertTrue(service.supports(ActionEnum.INSPECTION_WAYPOINT_CORPUS));
|
||||
}
|
||||
|
||||
@Test(expected = GlobalException.class)
|
||||
public void rejectsUnsupportedCorpusType()
|
||||
{
|
||||
service("开始巡检", "Start inspection").execute(message("fr-FR"));
|
||||
}
|
||||
|
||||
private InspectionWaypointCorpusOperateService service(String chineseCorpus, String englishCorpus)
|
||||
{
|
||||
InspectionWaypointCorpusVO waypoint = new InspectionWaypointCorpusVO();
|
||||
waypoint.setChineseCorpus(chineseCorpus);
|
||||
waypoint.setEnglishCorpus(englishCorpus);
|
||||
InspectionWaypointCorpusMapper mapper = (InspectionWaypointCorpusMapper) Proxy.newProxyInstance(
|
||||
InspectionWaypointCorpusMapper.class.getClassLoader(),
|
||||
new Class<?>[]{InspectionWaypointCorpusMapper.class},
|
||||
(proxy, method, args) -> {
|
||||
if ("selectByDetectItemId".equals(method.getName()))
|
||||
{
|
||||
assertEquals("detect-item-1", args[0]);
|
||||
return waypoint;
|
||||
}
|
||||
throw new UnsupportedOperationException(method.getName());
|
||||
});
|
||||
return new InspectionWaypointCorpusOperateService(mapper);
|
||||
}
|
||||
|
||||
private TaskNodeExecuteMessage message(Object type)
|
||||
{
|
||||
TaskNodeExecuteMessage message = new TaskNodeExecuteMessage();
|
||||
message.setAction(ActionEnum.INSPECTION_WAYPOINT_CORPUS);
|
||||
message.setInputParams(new JSONObject()
|
||||
.fluentPut("detectItemId", "detect-item-1")
|
||||
.fluentPut("type", type));
|
||||
return message;
|
||||
}
|
||||
}
|
||||
@ -10,7 +10,6 @@ import com.cmvr.test.flow.runtime.message.TaskNodeExecuteResult;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.function.Function;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNull;
|
||||
@ -21,7 +20,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
@Test
|
||||
public void mapsAudioArrayInputAndReturnsAnalysisMetadata() {
|
||||
AtomicReference<JSONObject> captured = new AtomicReference<>();
|
||||
MediaAnalysisClient client = client(request -> {
|
||||
MediaAnalysisClient client = request -> {
|
||||
captured.set(request);
|
||||
return new JSONObject()
|
||||
.fluentPut("status", "SUCCEEDED")
|
||||
@ -32,7 +31,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
.fluentPut("result", new JSONObject()
|
||||
.fluentPut("label", "POWER_ON")
|
||||
.fluentPut("matched", true));
|
||||
});
|
||||
};
|
||||
LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client);
|
||||
JSONObject input = new JSONObject()
|
||||
.fluentPut("profileCode", "aima.power_state.v1")
|
||||
@ -53,7 +52,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
@Test
|
||||
public void comparesRecognizedAudioWithExpectedType() {
|
||||
AtomicReference<JSONObject> captured = new AtomicReference<>();
|
||||
MediaAnalysisClient client = client(request -> {
|
||||
MediaAnalysisClient client = request -> {
|
||||
captured.set(request);
|
||||
return new JSONObject()
|
||||
.fluentPut("status", "SUCCEEDED")
|
||||
@ -63,7 +62,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
.fluentPut("label", "POWER_OFF")
|
||||
.fluentPut("labelName", "关机")
|
||||
.fluentPut("matched", true));
|
||||
});
|
||||
};
|
||||
LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client);
|
||||
JSONObject input = new JSONObject()
|
||||
.fluentPut("profileCode", "aima.power_state.v1")
|
||||
@ -81,14 +80,14 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
|
||||
@Test
|
||||
public void passesBroadVehicleSoundTargetWhenConcreteLabelIsAmbiguous() {
|
||||
MediaAnalysisClient client = client(request -> new JSONObject()
|
||||
MediaAnalysisClient client = request -> new JSONObject()
|
||||
.fluentPut("status", "SUCCEEDED")
|
||||
.fluentPut("analysisType", "AUDIO_CLASSIFICATION")
|
||||
.fluentPut("profileCode", "aima.power_state.v1")
|
||||
.fluentPut("result", new JSONObject()
|
||||
.fluentPut("label", "UNKNOWN")
|
||||
.fluentPut("matched", false)
|
||||
.fluentPut("vehicleSoundDetected", true)));
|
||||
.fluentPut("vehicleSoundDetected", true));
|
||||
LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client);
|
||||
JSONObject input = new JSONObject()
|
||||
.fluentPut("profileCode", "aima.power_state.v1")
|
||||
@ -105,7 +104,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
@Test
|
||||
public void forwardsVideoInstruction() {
|
||||
AtomicReference<JSONObject> captured = new AtomicReference<>();
|
||||
MediaAnalysisClient client = client(request -> {
|
||||
MediaAnalysisClient client = request -> {
|
||||
captured.set(request);
|
||||
return new JSONObject()
|
||||
.fluentPut("status", "SUCCEEDED")
|
||||
@ -114,7 +113,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
.fluentPut("timingMs", 30L)
|
||||
.fluentPut("model", new JSONObject().fluentPut("name", "qwen3-vl:32b"))
|
||||
.fluentPut("result", new JSONObject().fluentPut("summary", "ok"));
|
||||
});
|
||||
};
|
||||
LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client);
|
||||
JSONObject input = new JSONObject()
|
||||
.fluentPut("profileCode", "aima.power_video.v1")
|
||||
@ -147,8 +146,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
|
||||
@Test(expected = GlobalException.class)
|
||||
public void rejectsOutOfRangeVideoTuning() {
|
||||
LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(
|
||||
client(request -> new JSONObject()));
|
||||
LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(request -> new JSONObject());
|
||||
JSONObject input = new JSONObject()
|
||||
.fluentPut("profileCode", "aima.power_video.v1")
|
||||
.fluentPut("videoUrl", "https://files.example/video.mp4")
|
||||
@ -161,7 +159,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
@Test
|
||||
public void forwardsMultipleImagesAndReferenceImage() {
|
||||
AtomicReference<JSONObject> captured = new AtomicReference<>();
|
||||
MediaAnalysisClient client = client(request -> {
|
||||
MediaAnalysisClient client = request -> {
|
||||
captured.set(request);
|
||||
return new JSONObject()
|
||||
.fluentPut("status", "SUCCEEDED")
|
||||
@ -170,7 +168,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
.fluentPut("result", new JSONObject()
|
||||
.fluentPut("result", new JSONObject().fluentPut("x", 120).fluentPut("y", 80))
|
||||
.fluentPut("resultText", "{\"x\":120,\"y\":80}"));
|
||||
});
|
||||
};
|
||||
LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client);
|
||||
JSONObject input = new JSONObject()
|
||||
.fluentPut("profileCode", "common.image_analysis.v1")
|
||||
@ -212,7 +210,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
@Test
|
||||
public void forwardsDescriptionLocationMethodAndThreshold() {
|
||||
AtomicReference<JSONObject> captured = new AtomicReference<>();
|
||||
MediaAnalysisClient client = client(request -> {
|
||||
MediaAnalysisClient client = request -> {
|
||||
captured.set(request);
|
||||
return new JSONObject()
|
||||
.fluentPut("status", "SUCCEEDED")
|
||||
@ -221,7 +219,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
.fluentPut("result", new JSONObject()
|
||||
.fluentPut("result", new JSONArray().fluentAdd(10).fluentAdd(20).fluentAdd(30).fluentAdd(40))
|
||||
.fluentPut("matched", true));
|
||||
});
|
||||
};
|
||||
LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client);
|
||||
JSONObject input = new JSONObject()
|
||||
.fluentPut("profileCode", "common.image_analysis.v1")
|
||||
@ -243,7 +241,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
@Test
|
||||
public void restoresStaticImageUrlArraySerializedByLegacyFlowParamModel() {
|
||||
AtomicReference<JSONObject> captured = new AtomicReference<>();
|
||||
MediaAnalysisClient client = client(request -> {
|
||||
MediaAnalysisClient client = request -> {
|
||||
captured.set(request);
|
||||
return new JSONObject()
|
||||
.fluentPut("status", "SUCCEEDED")
|
||||
@ -252,7 +250,7 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
.fluentPut("result", new JSONObject()
|
||||
.fluentPut("result", "图片正常")
|
||||
.fluentPut("resultText", "图片正常"));
|
||||
});
|
||||
};
|
||||
LLMMediaAnalysisOperateService service = new LLMMediaAnalysisOperateService(client);
|
||||
JSONObject input = new JSONObject()
|
||||
.fluentPut("profileCode", "common.image_analysis.v1")
|
||||
@ -283,18 +281,4 @@ public class LLMMediaAnalysisOperateServiceTest {
|
||||
message.setInputParams(input);
|
||||
return message;
|
||||
}
|
||||
|
||||
private MediaAnalysisClient client(Function<JSONObject, JSONObject> analyze) {
|
||||
return new MediaAnalysisClient() {
|
||||
@Override
|
||||
public JSONObject analyze(JSONObject request) {
|
||||
return analyze.apply(request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public JSONObject addAudioReference(JSONObject request) {
|
||||
throw new UnsupportedOperationException("Not used by this test");
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -36,8 +36,6 @@ CREATE TABLE `inspection_waypoint` (
|
||||
`enabled` char(1) DEFAULT '0' COMMENT '是否启用(0启用 1禁用)',
|
||||
`sort_order` int(11) DEFAULT '0' COMMENT '排序号',
|
||||
`detect_item_id` varchar(64) DEFAULT NULL COMMENT '检测项ID',
|
||||
`chinese_corpus` text DEFAULT NULL COMMENT '该点位可播放的中文语料',
|
||||
`english_corpus` text DEFAULT NULL COMMENT '该点位可播放的英文语料',
|
||||
`map_id` varchar(64) DEFAULT NULL COMMENT '地图ID',
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='巡检点位表';
|
||||
|
||||
@ -1,4 +0,0 @@
|
||||
-- 巡检点位增加中英文可播放语料。
|
||||
ALTER TABLE `inspection_waypoint`
|
||||
ADD COLUMN `chinese_corpus` text DEFAULT NULL COMMENT '该点位可播放的中文语料' AFTER `detect_item_id`,
|
||||
ADD COLUMN `english_corpus` text DEFAULT NULL COMMENT '该点位可播放的英文语料' AFTER `chinese_corpus`;
|
||||
Loading…
Reference in New Issue
Block a user