feat(ti): 完善触控交互项目执行与评估闭环

接入真实流程事件,记录检测结果、录像及执行统计。

区分 VI/TI 运行模式,支持执行实例查询和简易报告,并补充单元测试及投标建设方案。
This commit is contained in:
lixiaolong 2026-09-16 17:26:05 +08:00
parent d595de2597
commit 95c3030b91
14 changed files with 1171 additions and 131 deletions

View File

@ -85,16 +85,10 @@ public class TiProjectController extends BaseController {
return AjaxResult.ok(tiProjectService.queryExecuteInstId(projectId));
}
@ApiOperation("临时执行项目评估")
@PostMapping("/evaluation")
public AjaxResult evaluation(@RequestBody List<Long> funcIds) {
tiProjectService.executeTiEvaluation(funcIds);
return AjaxResult.ok();
}
@ApiOperation("评估查询")
@GetMapping("/evaluation/{projectId}")
public AjaxResult queryEvaluation(@PathVariable("projectId") String projectId) {
return AjaxResult.ok(tiProjectService.queryEvaluation(projectId));
public AjaxResult queryEvaluation(@PathVariable("projectId") String projectId,
@RequestParam(value = "instId", required = false) String instId) {
return AjaxResult.ok(tiProjectService.queryEvaluation(projectId, instId));
}
}

View File

@ -6,11 +6,36 @@ import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.List;
import java.util.Date;
@Data
@ApiModel("语音交互评估")
public class AeViEvaluationVO {
@ApiModelProperty(value = "项目ID")
private String projectId;
@ApiModelProperty(value = "流程执行实例ID")
private String instId;
@ApiModelProperty(value = "执行状态")
private String executionStatus;
@ApiModelProperty(value = "执行开始时间")
private Date startTime;
@ApiModelProperty(value = "执行结束/最后更新时间")
private Date endTime;
@ApiModelProperty(value = "测试项总数")
private Integer totalCount;
@ApiModelProperty(value = "通过数")
private Integer successCount;
@ApiModelProperty(value = "失败数")
private Integer failureCount;
@ApiModelProperty(value = "方案ID")
private String schemeId;

View File

@ -54,7 +54,7 @@ public class FlowEndNodeHandler implements FlowNodeTypeHandler {
jsonObject.put("instId", instId);
jsonObject.put("taskId", taskId);
jsonObject.put("itemId", itemId);
jsonObject.put("aeType", RunModeEnum.VI_PROJECT.name());
jsonObject.put("aeType", runMode == null ? RunModeEnum.VI_PROJECT.name() : runMode.name());
executor.submit(() -> {
Thread.currentThread().setName("evaluation-thread-" + Thread.currentThread().getId());

View File

@ -163,13 +163,18 @@ public class FlowTaskRuntimeEntry implements FlowTaskRuntimeService {
@Transactional(rollbackFor = Exception.class)
public String executeProjectTask(TeTaskExecuteProjectVO taskExecuteProjectVO) {
RunModeEnum runMode = taskExecuteProjectVO.getRunMode() == null
? RunModeEnum.VI_PROJECT : taskExecuteProjectVO.getRunMode();
if (runMode != RunModeEnum.VI_PROJECT && runMode != RunModeEnum.TI_PROJECT) {
throw new GlobalException("项目执行仅支持语音交互或触控交互模式");
}
ResolvedRuntimeResources resources = resolveResources(
taskExecuteProjectVO.getRobotId(), taskExecuteProjectVO.getRuntimeAssignments());
return executeTaskInternal(
resources,
taskExecuteProjectVO.getProjectId(),
null,
RunModeEnum.VI_PROJECT,
taskExecuteProjectVO.getModuleCode(),
runMode,
taskExecuteProjectVO.getRunParams(),
taskExecuteProjectVO
);

View File

@ -17,6 +17,16 @@ public final class FlowExecutionModuleCodes {
*/
public static final String AIMA = "AIMA";
/**
* 语音交互项目模块
*/
public static final String VI = "VI";
/**
* 触控交互项目模块
*/
public static final String TI = "TI";
private FlowExecutionModuleCodes() {
// 防止实例化
}

View File

@ -1,6 +1,7 @@
package com.cmvr.test.model.vo;
import com.alibaba.fastjson2.JSONObject;
import com.cmvr.test.enums.RunModeEnum;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@ -26,4 +27,10 @@ public class TeTaskExecuteProjectVO {
@ApiModelProperty(value = "运行时参数")
private JSONObject runParams = new JSONObject();
@ApiModelProperty(value = "项目运行模式,由项目服务端设置", hidden = true)
private RunModeEnum runMode;
@ApiModelProperty(value = "流程事件模块编码,由项目服务端设置", hidden = true)
private String moduleCode;
}

View File

@ -33,6 +33,13 @@
<groupId>com.cmvr</groupId>
<artifactId>cmvr-iot-evaluation</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,35 @@
package com.cmvr.ti.listener;
import com.cmvr.test.flow.runtime.event.FlowExecutionEvent;
import com.cmvr.test.flow.runtime.event.FlowExecutionListener;
import com.cmvr.test.flow.runtime.event.FlowExecutionModuleCodes;
import com.cmvr.ti.support.TiEvaluationRecorder;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
/** Records touch-project workflow events for status and simple reporting. */
@Slf4j
@Component
@RequiredArgsConstructor
public class TiFlowExecutionListener implements FlowExecutionListener {
private final TiEvaluationRecorder evaluationRecorder;
@Override
public void onEvent(FlowExecutionEvent event) {
try {
evaluationRecorder.record(event);
} catch (RuntimeException exception) {
// Reporting must not interfere with robot execution or device cleanup.
log.error("触控项目执行结果记录失败: instId={}, eventType={}",
event == null ? null : event.getInstId(),
event == null ? null : event.getEventType(), exception);
}
}
@Override
public String getModuleCode() {
return FlowExecutionModuleCodes.TI;
}
}

View File

@ -30,7 +30,7 @@ public interface ITiProjectService extends IService<TiProject> {
public boolean updateProjectStatus(String projectId, Integer status);
public void executeTiEvaluation(List<Long> funcIds);
public AeViEvaluationVO queryEvaluation(String projectId);
public AeViEvaluationVO queryEvaluation(String projectId, String instId);
}

View File

@ -1,9 +1,7 @@
package com.cmvr.ti.service.impl;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.ObjUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson2.JSON;
@ -11,7 +9,6 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.spring.service.impl.ServiceImpl;
import com.cmvr.common.core.domain.entity.SysDictData;
import com.cmvr.common.exception.GlobalException;
import com.cmvr.evaluation.model.domain.AeEvaluation;
import com.cmvr.evaluation.model.domain.AeIndicatorEntity;
@ -19,10 +16,10 @@ import com.cmvr.evaluation.model.dto.AeIResultDTO;
import com.cmvr.evaluation.model.vo.AeViEvaluationVO;
import com.cmvr.evaluation.service.AeIndicatorService;
import com.cmvr.evaluation.service.IAeEvaluationService;
import com.cmvr.system.service.ISysDictTypeService;
import com.cmvr.test.enums.FlowSceneTypeEnum;
import com.cmvr.test.enums.RunModeEnum;
import com.cmvr.test.flow.runtime.engine.FlowTaskRuntimeService;
import com.cmvr.test.flow.runtime.event.FlowExecutionModuleCodes;
import com.cmvr.test.model.domain.TeDetectionItem;
import com.cmvr.test.model.domain.TeTaskConfigInfo;
import com.cmvr.test.model.domain.TeTaskInst;
@ -31,14 +28,15 @@ import com.cmvr.test.model.vo.TeTaskExecuteProjectVO;
import com.cmvr.test.model.vo.TeTaskOrchestraVO;
import com.cmvr.test.service.ITeDetectionItemService;
import com.cmvr.test.service.ITeTaskConfigInfoService;
import com.cmvr.test.service.ITeTaskInstService;
import com.cmvr.test.service.ITeTaskOrchestrationService;
import com.cmvr.ti.mapper.TiProjectMapper;
import com.cmvr.ti.model.domain.TiProject;
import com.cmvr.ti.model.domain.TiVehicleFunction;
import com.cmvr.ti.service.ITiProjectService;
import com.cmvr.ti.service.ITiVehicleFunctionService;
import com.cmvr.ti.support.TiEvaluationRecorder;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@ -46,11 +44,7 @@ import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
@ -59,13 +53,12 @@ public class TiProjectServiceImpl extends ServiceImpl<TiProjectMapper, TiProject
private final ITeTaskOrchestrationService taskOrchestrationService;
private final ITeDetectionItemService detectionItemService;
private final ITeTaskConfigInfoService taskConfigInfoService;
private final ITeTaskInstService taskInstService;
private final FlowTaskRuntimeService flowTaskRuntimeService;
private final ITiVehicleFunctionService tiVehicleFunctionService;
private final ISysDictTypeService dictTypeService;
private final StringRedisTemplate redisTemplate;
private final IAeEvaluationService aeEvaluationService;
private final AeIndicatorService aeIndicatorService;
private final ITiVehicleFunctionService vehicleFunctionService;
private final TiEvaluationRecorder evaluationRecorder;
@Override
public TiProject selectTiProjectByProjectId(String projectId) {
@ -135,28 +128,51 @@ public class TiProjectServiceImpl extends ServiceImpl<TiProjectMapper, TiProject
@Override
public String executeProject(TeTaskExecuteProjectVO taskExecuteProjectVO) {
TiProject project = getById(taskExecuteProjectVO.getProjectId());
if (project == null) {
throw new GlobalException("触控项目不存在:" + taskExecuteProjectVO.getProjectId());
}
TeTaskExecuteProjectVO normalVO = new TeTaskExecuteProjectVO();
normalVO.setProjectId(taskExecuteProjectVO.getProjectId());
normalVO.setRunParams(taskExecuteProjectVO.getRunParams());
normalVO.setRobotId(taskExecuteProjectVO.getRobotId());
normalVO.setRobotId(taskExecuteProjectVO.getRobotId());
String instId = flowTaskRuntimeService.executeProjectTask(normalVO);
// this.update(
// new LambdaUpdateWrapper<TiProject>()
// .eq(TiProject::getProjectId, taskExecuteProjectVO.getProjectId())
// .set(TiProject::getStatus, "0")
// );
normalVO.setRuntimeAssignments(taskExecuteProjectVO.getRuntimeAssignments());
normalVO.setRunMode(RunModeEnum.TI_PROJECT);
normalVO.setModuleCode(FlowExecutionModuleCodes.TI);
updateProjectStatus(taskExecuteProjectVO.getProjectId(), 1);
String instId;
try {
instId = flowTaskRuntimeService.executeProjectTask(normalVO);
} catch (RuntimeException exception) {
updateProjectStatus(taskExecuteProjectVO.getProjectId(), 0);
throw exception;
}
evaluationRecorder.initialize(taskExecuteProjectVO.getProjectId(), instId);
return instId;
}
@Override
public TeTaskInst getTeTaskInst(String projectId) {
return null;
TiProject project = getById(projectId);
if (project == null) {
throw new GlobalException("触控项目不存在:" + projectId);
}
return taskInstService.getOne(
Wrappers.lambdaQuery(TeTaskInst.class)
.eq(TeTaskInst::getTaskId, projectId)
.eq(TeTaskInst::getRunMode, RunModeEnum.TI_PROJECT.name())
.orderByDesc(TeTaskInst::getCreateTime)
.last("LIMIT 1")
);
}
@Override
public String queryExecuteInstId(String projectId) {
return "";
TeTaskInst taskInst = getTeTaskInst(projectId);
if (taskInst == null) {
throw new GlobalException("触控项目尚未执行:" + projectId);
}
return taskInst.getId();
}
@Override
@ -167,136 +183,115 @@ public class TiProjectServiceImpl extends ServiceImpl<TiProjectMapper, TiProject
return this.update(new TiProject(), wrapper);
}
// 功能ID -> 音频地址
private final Map<String, String> AUDIO_PATH_MAP = new HashMap<String, String>() {{
put("8", "http://192.168.0.100:9000/cmvr-iot/AUDIO/20260310/1773108837127.wav"); // 播放音乐
put("12", "http://192.168.0.100:9000/cmvr-iot/AUDIO/20260310/1773108916488.wav"); // 打开导航
put("9", "http://192.168.0.100:9000/cmvr-iot/AUDIO/20260310/1773108933968.wav"); // 能量回收
}};
// 功能ID -> 视频地址
private final Map<String, String> VIDEO_PATH_MAP = new HashMap<String, String>() {{
put("8", "http://192.168.0.100:9000/cmvr-iot/VIDEO/20260310/1773108955700.mp4"); // 播放音乐
put("12", "http://192.168.0.100:9000/cmvr-iot/VIDEO/20260310/1773108967164.mp4"); // 打开导航
put("9", "http://192.168.0.100:9000/cmvr-iot/VIDEO/20260310/1773108978124.mp4"); // 能量回收
}};
@Override
public void executeTiEvaluation(List<Long> funcIds) {
AeEvaluation aeEvaluation = new AeEvaluation();
aeEvaluation.setAeType(RunModeEnum.TI_PROJECT.name());
aeEvaluation.setInstId(IdUtil.fastSimpleUUID());
aeEvaluation.setTaskId("aadb21a32e7990456b866e844a7d4a54");
aeEvaluation.setItemId(IdUtil.fastSimpleUUID());
// todo 触控交互评估
String taskId = aeEvaluation.getTaskId();
updateProjectStatus(taskId, 1); // 执行中
// 模拟流程运行
// try {
// Thread.sleep(30000);
// } catch (InterruptedException e) {
// throw new RuntimeException(e);
// }
// 查字典
List<SysDictData> tiFunctionConfig = dictTypeService.selectDictDataByType("ti_function_config");
Map<String, String> functionMap = tiFunctionConfig.stream()
.collect(Collectors.toMap(
SysDictData::getDictValue,
SysDictData::getDictLabel
));
// 构建评估任务列表
List<AeEvaluation> evaluations = new LinkedList<>();
for (Long funcId : funcIds) {
TiVehicleFunction byId = vehicleFunctionService.getById(funcId);
String funcName = functionMap.get(byId.getFuncKey());
AeEvaluation eval = new AeEvaluation();
BeanUtil.copyProperties(aeEvaluation, eval);
eval.setAeId(IdUtil.fastSimpleUUID());
eval.setStatus(0); // 待评估
eval.setAvContent(funcName);
eval.setAudioPath(AUDIO_PATH_MAP.get(byId.getFuncKey()));
eval.setVideoPath(VIDEO_PATH_MAP.get(byId.getFuncKey()));
evaluations.add(eval);
}
try {
// 入库
aeEvaluationService.saveBatch(evaluations);
// 提交评估
for (AeEvaluation e : evaluations) {
redisTemplate.opsForList()
.rightPush("ae:evaluation:queue", JSON.toJSONString(e));
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public AeViEvaluationVO queryEvaluation(String projectId) {
// 1. 查询项目
TiProject project = getById(projectId);
// 2. 查询评估数据(允许不完整)
List<AeEvaluation> evaluations = aeEvaluationService.queryEvaluationByProjectId(projectId);
if (CollUtil.isEmpty(evaluations)) {
throw new GlobalException(StrUtil.format("暂无项目id为{}的数据", projectId));
return queryEvaluation(projectId, null);
}
// 3. 获取指标定义
List<AeIndicatorEntity> indicators =
aeIndicatorService.getChildrenByParentIdWithLevelLimit(
project.getIndicatorId(), 4, 1
);
@Override
public AeViEvaluationVO queryEvaluation(String projectId, String instId) {
TiProject project = getById(projectId);
if (project == null) {
throw new GlobalException("触控项目不存在:" + projectId);
}
TeTaskInst taskInst;
if (StrUtil.isBlank(instId)) {
taskInst = getTeTaskInst(projectId);
} else {
taskInst = taskInstService.getById(instId);
if (taskInst != null && (!StrUtil.equals(projectId, taskInst.getTaskId())
|| !StrUtil.equals(RunModeEnum.TI_PROJECT.name(), taskInst.getRunMode()))) {
throw new GlobalException("执行实例不属于当前触控项目:" + instId);
}
}
if (taskInst == null) {
throw new GlobalException("触控项目尚未执行:" + projectId);
}
List<AeEvaluation> evaluations = aeEvaluationService.queryEvaluation(taskInst.getId());
if (CollUtil.isEmpty(evaluations)) {
throw new GlobalException(StrUtil.format("项目 {} 的执行实例 {} 暂无结果数据",
projectId, taskInst.getId()));
}
List<AeIndicatorEntity> indicators = project.getIndicatorId() == null
? Collections.emptyList()
: aeIndicatorService.getChildrenByParentIdWithLevelLimit(project.getIndicatorId(), 4, 1);
List<AeViEvaluationVO.AeItemDetail> itemDetails = new ArrayList<>();
int successCount = 0;
int failureCount = 0;
long responseTimeSum = 0L;
int responseTimeCount = 0;
for (AeEvaluation evaluation : evaluations) {
AeViEvaluationVO.AeItemDetail itemDto = new AeViEvaluationVO.AeItemDetail();
itemDto.setAeId(evaluation.getAeId());
itemDto.setContent(evaluation.getAvContent());
AeIResultDTO dto = JSON.parseObject(evaluation.getResult(), AeIResultDTO.class);
AeIResultDTO dto = parseEvaluationResult(evaluation.getResult());
itemDto.setItemResponseTime(dto.getResponseTime());
itemDto.setStatus(evaluation.getStatus());
boolean success = "成功".equals(dto.getOverallResult());
if (!success) {
itemDto.setFailureReason(dto.getFailureReason());
if (evaluation.getStatus() != null && evaluation.getStatus() == 2) {
successCount++;
} else if (evaluation.getStatus() != null && evaluation.getStatus() >= 3) {
failureCount++;
}
if (dto.getResponseTime() != null) {
responseTimeSum += dto.getResponseTime();
responseTimeCount++;
}
itemDto.setVideoPath(dto.getVideoPath());
itemDto.setFailureReason(dto.getFailureReason());
itemDto.setVideoPath(StrUtil.blankToDefault(dto.getVideoPath(), evaluation.getVideoPath()));
itemDto.setExecutionDetail(dto.getExecutionDetail());
itemDetails.add(itemDto);
}
// 计算成功率
Long successRate = (long) (((evaluations.size() - failureCount) * 100.0) / evaluations.size());
//成功率得分
Long successRate = evaluations.isEmpty() ? 0L : successCount * 100L / evaluations.size();
Long score = calculateSuccessRateScore(successRate, indicators);
AeViEvaluationVO vo = new AeViEvaluationVO();
vo.setProjectId(projectId);
vo.setInstId(taskInst.getId());
vo.setExecutionStatus(taskInst.getStatus());
vo.setStartTime(taskInst.getCreateTime());
vo.setEndTime(taskInst.getUpdateTime());
vo.setSchemeId(taskInst.getId());
vo.setSchemeName(project.getProjectName());
vo.setTotalCount(evaluations.size());
vo.setSuccessCount(successCount);
vo.setFailureCount(failureCount);
vo.setResponseTime(responseTimeCount == 0 ? null : responseTimeSum / responseTimeCount);
vo.setItemDetails(itemDetails);
vo.setSuccessRate(successRate);
vo.setSuccessRateScore(score);
return vo;
}
private AeIResultDTO parseEvaluationResult(String resultJson) {
if (StrUtil.isBlank(resultJson)) {
return new AeIResultDTO();
}
try {
AeIResultDTO result = JSON.parseObject(resultJson, AeIResultDTO.class);
return result == null ? new AeIResultDTO() : result;
} catch (RuntimeException exception) {
AeIResultDTO result = new AeIResultDTO();
result.setFailureReason("结果数据格式异常");
result.setExecutionDetail(StrUtil.sub(resultJson, 0, 500));
return result;
}
}
private Long calculateSuccessRateScore(Long successRate, List<AeIndicatorEntity> indicators) {
if (CollUtil.isEmpty(indicators)) {
return 0L;
}
// 查找成功率指标
AeIndicatorEntity successRateIndicator = findIndicatorByName(indicators, "成功率");
if (successRateIndicator == null) {
if (successRateIndicator == null || CollUtil.isEmpty(successRateIndicator.getChildren())) {
return 0L; // 如果没有匹配的成功率指标,返回 0 分
}
List<AeIndicatorEntity> children = successRateIndicator.getChildren();
@ -318,8 +313,11 @@ public class TiProjectServiceImpl extends ServiceImpl<TiProjectMapper, TiProject
}
private AeIndicatorEntity findIndicatorByName(List<AeIndicatorEntity> indicators, String indicatorName) {
if (CollUtil.isEmpty(indicators)) {
return null;
}
for (AeIndicatorEntity indicator : indicators) {
if (indicator.getName().equals(indicatorName)) {
if (indicator != null && StrUtil.equals(indicator.getName(), indicatorName)) {
return indicator;
}
}

View File

@ -0,0 +1,413 @@
package com.cmvr.ti.support;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import com.cmvr.common.core.domain.entity.SysDictData;
import com.cmvr.evaluation.model.domain.AeEvaluation;
import com.cmvr.evaluation.model.dto.AeIResultDTO;
import com.cmvr.evaluation.service.IAeEvaluationService;
import com.cmvr.system.service.ISysDictTypeService;
import com.cmvr.test.enums.ActionEnum;
import com.cmvr.test.enums.RunModeEnum;
import com.cmvr.test.flow.runtime.event.FlowExecutionEvent;
import com.cmvr.test.flow.runtime.operator.FlowMediaParamResolver;
import com.cmvr.ti.mapper.TiProjectMapper;
import com.cmvr.ti.model.domain.TiProject;
import com.cmvr.ti.model.domain.TiVehicleFunction;
import com.cmvr.ti.service.ITiVehicleFunctionService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* 将触控流程的真实执行事件整理为项目内的简易报告数据。
*
* <p>该组件不调用模型评估,也不制造演示音视频。结果只来自流程节点状态、
* 边端返回以及相机停止录像节点产生的真实 videoUrl。</p>
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class TiEvaluationRecorder {
private static final int STATUS_PENDING = 0;
private static final int STATUS_RUNNING = 1;
private static final int STATUS_SUCCESS = 2;
private static final int STATUS_FAILED = 3;
private final TiProjectMapper projectMapper;
private final ITiVehicleFunctionService vehicleFunctionService;
private final ISysDictTypeService dictTypeService;
private final IAeEvaluationService evaluationService;
private final ConcurrentMap<String, Object> initializationLocks = new ConcurrentHashMap<>();
private final ConcurrentMap<String, Long> startedAt = new ConcurrentHashMap<>();
private final ConcurrentMap<String, String> latestVideoUrls = new ConcurrentHashMap<>();
private final ConcurrentMap<String, String> businessFailures = new ConcurrentHashMap<>();
private final ConcurrentMap<String, ConcurrentLinkedQueue<String>> executionDetails = new ConcurrentHashMap<>();
/** Creates one report row for each function selected by the touch project. */
public void initialize(String projectId, String instId) {
if (StrUtil.isBlank(projectId) || StrUtil.isBlank(instId)) {
return;
}
Object lock = initializationLocks.computeIfAbsent(instId, key -> new Object());
synchronized (lock) {
try {
if (CollUtil.isNotEmpty(evaluationService.queryEvaluation(instId))) {
return;
}
TiProject project = projectMapper.selectById(projectId);
if (project == null) {
log.warn("触控执行结果初始化失败,项目不存在: projectId={}, instId={}", projectId, instId);
return;
}
List<Long> functionIds = parseFunctionIds(project.getTestFuncIds());
List<TiVehicleFunction> functionList = functionIds.isEmpty()
? Collections.emptyList() : vehicleFunctionService.queryByIds(functionIds);
Map<Long, TiVehicleFunction> functions = functionList == null
? Collections.emptyMap()
: functionList.stream().collect(Collectors.toMap(
TiVehicleFunction::getId, Function.identity(), (left, right) -> left));
Map<String, String> functionNames = loadFunctionNames();
List<AeEvaluation> evaluations = new ArrayList<>();
for (Long functionId : functionIds) {
TiVehicleFunction function = functions.get(functionId);
AeEvaluation evaluation = new AeEvaluation();
evaluation.setAeId(IdUtil.fastSimpleUUID());
evaluation.setAeType(RunModeEnum.TI_PROJECT.name());
evaluation.setInstId(instId);
evaluation.setTaskId(projectId);
evaluation.setItemId(String.valueOf(functionId));
evaluation.setAvContent(resolveFunctionName(functionId, function, functionNames));
evaluation.setStatus(STATUS_PENDING);
evaluations.add(evaluation);
}
if (evaluations.isEmpty()) {
AeEvaluation evaluation = new AeEvaluation();
evaluation.setAeId(IdUtil.fastSimpleUUID());
evaluation.setAeType(RunModeEnum.TI_PROJECT.name());
evaluation.setInstId(instId);
evaluation.setTaskId(projectId);
evaluation.setItemId("project");
evaluation.setAvContent("触控交互测试");
evaluation.setStatus(STATUS_PENDING);
evaluations.add(evaluation);
}
evaluationService.saveBatch(evaluations);
startedAt.putIfAbsent(instanceKey(instId), System.currentTimeMillis());
} finally {
initializationLocks.remove(instId, lock);
}
}
}
public void record(FlowExecutionEvent event) {
if (event == null || StrUtil.isBlank(event.getInstId()) || StrUtil.isBlank(event.getTaskId())) {
return;
}
initialize(event.getTaskId(), event.getInstId());
startedAt.putIfAbsent(instanceKey(event.getInstId()), System.currentTimeMillis());
switch (event.getEventType()) {
case NODE_STARTED:
updateProjectStatus(event.getTaskId(), "1");
recordNodeEvent(event, "开始执行");
markRunning(resolveEvaluation(event));
break;
case NODE_COMPLETED:
recordNodeEvent(event, "执行完成");
handleNodeCompleted(event);
break;
case NODE_FAILED:
recordNodeEvent(event, "执行失败:" + StrUtil.blankToDefault(event.getErrorMessage(), "未知异常"));
failEvaluation(resolveEvaluation(event), event,
StrUtil.blankToDefault(event.getErrorMessage(), "节点执行失败"));
break;
case TASK_COMPLETED:
String businessFailure = businessFailures.get(event.getInstId());
finishPending(event, StrUtil.isBlank(businessFailure), businessFailure);
updateProjectStatus(event.getTaskId(), "2");
clearRuntimeCache(event.getInstId());
break;
case TASK_FAILED:
finishPending(event, false,
StrUtil.blankToDefault(event.getErrorMessage(), "任务执行失败"));
updateProjectStatus(event.getTaskId(), "2");
clearRuntimeCache(event.getInstId());
break;
case TASK_STOPPED:
finishPending(event, false, "任务已终止");
updateProjectStatus(event.getTaskId(), "2");
clearRuntimeCache(event.getInstId());
break;
case TASK_PAUSED:
case TASK_RESUMED:
default:
break;
}
}
private void handleNodeCompleted(FlowExecutionEvent event) {
AeEvaluation evaluation = resolveEvaluation(event);
String businessFailure = resolveBusinessFailure(event.getOutputParams());
if (StrUtil.isNotBlank(businessFailure)) {
if (evaluation == null) {
businessFailures.putIfAbsent(event.getInstId(), businessFailure);
} else {
completeEvaluation(evaluation, event, false, businessFailure);
}
}
String videoUrl = FlowMediaParamResolver.lastUrl(event.getOutputParams(), "videoUrl");
if (StrUtil.isNotBlank(videoUrl)) {
if (evaluation != null) {
evaluation.setVideoPath(videoUrl);
evaluationService.updateById(evaluation);
} else {
// A recording outside the function loop is evidence for the entire run.
latestVideoUrls.put(event.getInstId(), videoUrl);
}
}
if (StrUtil.isBlank(businessFailure)
&& event.getAction() == ActionEnum.CAMERA_RECORDING_STOP && evaluation != null) {
completeEvaluation(evaluation, event, true, null);
}
}
private void markRunning(AeEvaluation evaluation) {
if (evaluation == null || evaluation.getStatus() == null || evaluation.getStatus() != STATUS_PENDING) {
return;
}
evaluation.setStatus(STATUS_RUNNING);
evaluationService.updateById(evaluation);
startedAt.putIfAbsent(evaluationKey(evaluation), System.currentTimeMillis());
}
private void failEvaluation(AeEvaluation evaluation, FlowExecutionEvent event, String reason) {
if (evaluation != null) {
completeEvaluation(evaluation, event, false, reason);
}
}
private void finishPending(FlowExecutionEvent event, boolean success, String failureReason) {
List<AeEvaluation> evaluations = evaluationService.queryEvaluation(event.getInstId());
for (AeEvaluation evaluation : evaluations) {
if (evaluation.getStatus() != null
&& (evaluation.getStatus() == STATUS_SUCCESS || evaluation.getStatus() == STATUS_FAILED)) {
continue;
}
completeEvaluation(evaluation, event, success, failureReason);
}
}
private void completeEvaluation(AeEvaluation evaluation, FlowExecutionEvent event,
boolean success, String failureReason) {
if (evaluation == null) {
return;
}
// A later successful task event must not hide an earlier node failure.
if (success && evaluation.getStatus() != null && evaluation.getStatus() == STATUS_FAILED) {
return;
}
String videoUrl = StrUtil.blankToDefault(evaluation.getVideoPath(),
latestVideoUrls.get(event.getInstId()));
Long start = startedAt.get(evaluationKey(evaluation));
if (start == null) {
start = startedAt.get(instanceKey(event.getInstId()));
}
AeIResultDTO result = new AeIResultDTO();
result.setOverallResult(success ? "成功" : "失败");
result.setFailureReason(success ? null : StrUtil.blankToDefault(failureReason, "执行失败"));
result.setResponseTime(start == null ? null : Math.max(0L, System.currentTimeMillis() - start));
result.setVideoPath(videoUrl);
result.setExecutionDetail(buildExecutionDetail(event.getInstId(), evaluation.getAeId()));
evaluation.setStatus(success ? STATUS_SUCCESS : STATUS_FAILED);
evaluation.setVideoPath(videoUrl);
evaluation.setResult(JSON.toJSONString(result));
evaluationService.updateById(evaluation);
}
private AeEvaluation resolveEvaluation(FlowExecutionEvent event) {
List<AeEvaluation> evaluations = evaluationService.queryEvaluation(event.getInstId());
if (CollUtil.isEmpty(evaluations)) {
return null;
}
String outputFunctionId = resolveOutputFunctionId(event.getOutputParams());
if (StrUtil.isNotBlank(outputFunctionId)) {
AeEvaluation matched = findEvaluation(evaluations, outputFunctionId);
if (matched != null) {
return matched;
}
}
TiProject project = projectMapper.selectById(event.getTaskId());
List<Long> functionIds = project == null
? Collections.emptyList() : parseFunctionIds(project.getTestFuncIds());
if (CollUtil.isNotEmpty(event.getIterations())) {
int index = event.getIterations().get(0) - 1;
if (index >= 0 && index < functionIds.size()) {
AeEvaluation matched = findEvaluation(evaluations, String.valueOf(functionIds.get(index)));
if (matched != null) {
return matched;
}
}
}
return evaluations.size() == 1 ? evaluations.get(0) : null;
}
private AeEvaluation findEvaluation(List<AeEvaluation> evaluations, String functionId) {
return evaluations.stream()
.filter(item -> StrUtil.equals(item.getItemId(), functionId))
.findFirst()
.orElse(null);
}
private String resolveOutputFunctionId(JSONObject output) {
if (output == null) {
return null;
}
JSONObject source = output.getJSONObject("result");
if (source == null) {
source = output;
}
String functionId = source.getString("functionId");
if (StrUtil.isBlank(functionId)) {
functionId = source.getString("funcId");
}
return functionId;
}
private String resolveBusinessFailure(JSONObject output) {
if (output == null) {
return null;
}
JSONObject source = output.getJSONObject("result");
if (source == null) {
source = output;
}
Object success = source.containsKey("success") ? source.get("success")
: source.containsKey("passed") ? source.get("passed") : source.get("pass");
String overallResult = source.getString("overallResult");
boolean explicitlyFailed = Boolean.FALSE.equals(success)
|| (success instanceof String && "false".equalsIgnoreCase((String) success))
|| "失败".equals(overallResult)
|| "不通过".equals(overallResult)
|| "failed".equalsIgnoreCase(overallResult)
|| "failure".equalsIgnoreCase(overallResult);
if (!explicitlyFailed) {
return null;
}
String reason = source.getString("failureReason");
if (StrUtil.isBlank(reason)) {
reason = source.getString("errorMessage");
}
if (StrUtil.isBlank(reason)) {
reason = source.getString("message");
}
return StrUtil.blankToDefault(reason, "边端脚本返回失败");
}
private void recordNodeEvent(FlowExecutionEvent event, String statusText) {
AeEvaluation evaluation = resolveEvaluation(event);
String key = detailKey(event.getInstId(), evaluation == null ? "*" : evaluation.getAeId());
String nodeName = StrUtil.blankToDefault(event.getNodeName(), event.getNodeId());
String detail = StrUtil.blankToDefault(nodeName, "流程节点") + statusText;
executionDetails.computeIfAbsent(key, ignored -> new ConcurrentLinkedQueue<>()).add(detail);
}
private String buildExecutionDetail(String instId, String evaluationId) {
List<String> details = new ArrayList<>();
details.addAll(executionDetails.getOrDefault(detailKey(instId, "*"), new ConcurrentLinkedQueue<>()));
details.addAll(executionDetails.getOrDefault(detailKey(instId, evaluationId), new ConcurrentLinkedQueue<>()));
if (details.isEmpty()) {
return "流程执行完成";
}
int fromIndex = Math.max(0, details.size() - 20);
return StrUtil.sub(String.join(";", details.subList(fromIndex, details.size())), 0, 2000);
}
private List<Long> parseFunctionIds(String json) {
if (StrUtil.isBlank(json)) {
return Collections.emptyList();
}
try {
List<Long> ids = JSON.parseArray(json, Long.class);
return ids == null ? Collections.emptyList() : ids;
} catch (RuntimeException exception) {
log.warn("触控项目功能列表格式错误: {}", json);
return Collections.emptyList();
}
}
private Map<String, String> loadFunctionNames() {
List<SysDictData> dictData = dictTypeService.selectDictDataByType("ti_function_config");
if (dictData == null) {
return Collections.emptyMap();
}
return dictData.stream().collect(Collectors.toMap(
SysDictData::getDictValue,
SysDictData::getDictLabel,
(left, right) -> left,
LinkedHashMap::new
));
}
private String resolveFunctionName(Long functionId, TiVehicleFunction function,
Map<String, String> functionNames) {
if (function == null) {
return "功能 " + functionId;
}
return functionNames.getOrDefault(function.getFuncKey(),
StrUtil.blankToDefault(function.getFuncKey(), "功能 " + functionId));
}
private void updateProjectStatus(String projectId, String status) {
TiProject project = projectMapper.selectById(projectId);
if (project == null || StrUtil.equals(project.getStatus(), status)) {
return;
}
project.setStatus(status);
projectMapper.updateById(project);
}
private void clearRuntimeCache(String instId) {
startedAt.keySet().removeIf(key -> key.startsWith(instId + "|"));
executionDetails.keySet().removeIf(key -> key.startsWith(instId + "|"));
latestVideoUrls.remove(instId);
businessFailures.remove(instId);
initializationLocks.remove(instId);
}
private String evaluationKey(AeEvaluation evaluation) {
return evaluation.getInstId() + "|eval|" + evaluation.getAeId();
}
private String instanceKey(String instId) {
return instId + "|instance";
}
private String detailKey(String instId, String evaluationId) {
return instId + "|detail|" + evaluationId;
}
}

View File

@ -0,0 +1,193 @@
package com.cmvr.ti.support;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import com.cmvr.common.core.domain.entity.SysDictData;
import com.cmvr.evaluation.model.domain.AeEvaluation;
import com.cmvr.evaluation.model.dto.AeIResultDTO;
import com.cmvr.evaluation.service.IAeEvaluationService;
import com.cmvr.system.service.ISysDictTypeService;
import com.cmvr.test.enums.ActionEnum;
import com.cmvr.test.flow.runtime.event.FlowExecutionEvent;
import com.cmvr.ti.mapper.TiProjectMapper;
import com.cmvr.ti.model.domain.TiProject;
import com.cmvr.ti.model.domain.TiVehicleFunction;
import com.cmvr.ti.service.ITiVehicleFunctionService;
import org.junit.Test;
import java.lang.reflect.Proxy;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
public class TiEvaluationRecorderTest {
@Test
public void recordsActualVideoAndSuccessfulFunctionResult() {
Fixture fixture = fixture("[8]");
fixture.recorder.initialize("project-1", "inst-1");
fixture.recorder.record(nodeEvent(FlowExecutionEvent.EventType.NODE_STARTED,
ActionEnum.TOUCH, null, List.of(1), null));
fixture.recorder.record(nodeEvent(FlowExecutionEvent.EventType.NODE_COMPLETED,
ActionEnum.CAMERA_RECORDING_STOP,
new JSONObject().fluentPut("videoUrl", List.of("http://files/touch-1.mp4")),
List.of(1), null));
fixture.recorder.record(taskEvent(FlowExecutionEvent.EventType.TASK_COMPLETED, null));
assertEquals(1, fixture.evaluations.size());
AeEvaluation evaluation = fixture.evaluations.get(0);
assertEquals(Integer.valueOf(2), evaluation.getStatus());
assertEquals("http://files/touch-1.mp4", evaluation.getVideoPath());
AeIResultDTO result = JSON.parseObject(evaluation.getResult(), AeIResultDTO.class);
assertEquals("成功", result.getOverallResult());
assertEquals("http://files/touch-1.mp4", result.getVideoPath());
assertTrue(result.getExecutionDetail().contains("结束录像执行完成"));
assertEquals("2", fixture.project.getStatus());
}
@Test
public void doesNotReusePreviousFunctionVideoWhenNextFunctionFails() {
Fixture fixture = fixture("[8,9]");
fixture.recorder.initialize("project-1", "inst-1");
fixture.recorder.record(nodeEvent(FlowExecutionEvent.EventType.NODE_COMPLETED,
ActionEnum.CAMERA_RECORDING_STOP,
new JSONObject().fluentPut("videoUrl", List.of("http://files/first.mp4")),
List.of(1), null));
fixture.recorder.record(taskEvent(FlowExecutionEvent.EventType.TASK_FAILED, "机械臂执行失败"));
AeEvaluation first = findByItemId(fixture.evaluations, "8");
AeEvaluation second = findByItemId(fixture.evaluations, "9");
assertEquals(Integer.valueOf(2), first.getStatus());
assertEquals("http://files/first.mp4", first.getVideoPath());
assertEquals(Integer.valueOf(3), second.getStatus());
assertNull(second.getVideoPath());
AeIResultDTO failure = JSON.parseObject(second.getResult(), AeIResultDTO.class);
assertEquals("机械臂执行失败", failure.getFailureReason());
}
@Test
public void treatsHttpBusinessFailureAsFailedResult() {
Fixture fixture = fixture("[8]");
JSONObject edgeResult = new JSONObject()
.fluentPut("success", false)
.fluentPut("failureReason", "未识别到目标控件");
fixture.recorder.record(nodeEvent(FlowExecutionEvent.EventType.NODE_COMPLETED,
ActionEnum.HTTP, new JSONObject().fluentPut("result", edgeResult),
List.of(1), null));
fixture.recorder.record(taskEvent(FlowExecutionEvent.EventType.TASK_COMPLETED, null));
AeEvaluation evaluation = fixture.evaluations.get(0);
assertEquals(Integer.valueOf(3), evaluation.getStatus());
AeIResultDTO result = JSON.parseObject(evaluation.getResult(), AeIResultDTO.class);
assertEquals("失败", result.getOverallResult());
assertEquals("未识别到目标控件", result.getFailureReason());
}
private Fixture fixture(String testFunctionIds) {
TiProject project = new TiProject();
project.setProjectId("project-1");
project.setTestFuncIds(testFunctionIds);
project.setStatus("0");
List<AeEvaluation> evaluations = new ArrayList<>();
TiProjectMapper projectMapper = proxy(TiProjectMapper.class, (method, args) -> {
if ("selectById".equals(method)) return project;
if ("updateById".equals(method)) return 1;
return defaultValue(method);
});
ITiVehicleFunctionService functions = proxy(ITiVehicleFunctionService.class, (method, args) -> {
if ("queryByIds".equals(method)) {
Collection<?> ids = (Collection<?>) args[0];
return ids.stream().map(id -> {
TiVehicleFunction function = new TiVehicleFunction();
function.setId(Long.valueOf(String.valueOf(id)));
function.setFuncKey(String.valueOf(id));
return function;
}).toList();
}
return defaultValue(method);
});
ISysDictTypeService dictionaries = proxy(ISysDictTypeService.class, (method, args) -> {
if ("selectDictDataByType".equals(method)) {
SysDictData first = new SysDictData();
first.setDictValue("8");
first.setDictLabel("播放音乐");
SysDictData second = new SysDictData();
second.setDictValue("9");
second.setDictLabel("能量回收");
return List.of(first, second);
}
return defaultValue(method);
});
IAeEvaluationService evaluationService = proxy(IAeEvaluationService.class, (method, args) -> {
if ("queryEvaluation".equals(method)) {
String instId = String.valueOf(args[0]);
return evaluations.stream().filter(item -> instId.equals(item.getInstId())).toList();
}
if ("saveBatch".equals(method)) {
evaluations.addAll((Collection<? extends AeEvaluation>) args[0]);
return true;
}
if ("updateById".equals(method)) return true;
return defaultValue(method);
});
TiEvaluationRecorder recorder = new TiEvaluationRecorder(
projectMapper, functions, dictionaries, evaluationService);
return new Fixture(recorder, project, evaluations);
}
private FlowExecutionEvent nodeEvent(FlowExecutionEvent.EventType type, ActionEnum action,
JSONObject output, List<Integer> iterations, String error) {
return FlowExecutionEvent.builder()
.eventType(type)
.instId("inst-1")
.taskId("project-1")
.itemId("touch-flow")
.nodeId(action.getAction())
.nodeName(action == ActionEnum.CAMERA_RECORDING_STOP ? "结束录像" : "执行触控")
.action(action)
.outputParams(output)
.iterations(iterations)
.errorMessage(error)
.build();
}
private FlowExecutionEvent taskEvent(FlowExecutionEvent.EventType type, String error) {
return FlowExecutionEvent.builder()
.eventType(type)
.instId("inst-1")
.taskId("project-1")
.errorMessage(error)
.build();
}
private AeEvaluation findByItemId(List<AeEvaluation> evaluations, String itemId) {
return evaluations.stream().filter(item -> itemId.equals(item.getItemId())).findFirst().orElseThrow();
}
@SuppressWarnings("unchecked")
private <T> T proxy(Class<T> type, Invocation invocation) {
return (T) Proxy.newProxyInstance(type.getClassLoader(), new Class<?>[]{type},
(proxy, method, args) -> invocation.invoke(method.getName(), args));
}
private Object defaultValue(String methodName) {
return null;
}
private interface Invocation {
Object invoke(String method, Object[] args);
}
private record Fixture(TiEvaluationRecorder recorder, TiProject project,
List<AeEvaluation> evaluations) {
}
}

View File

@ -18,6 +18,7 @@ import com.cmvr.evaluation.service.IAeEvaluationService;
import com.cmvr.test.enums.FlowSceneTypeEnum;
import com.cmvr.test.enums.RunModeEnum;
import com.cmvr.test.flow.runtime.engine.FlowTaskRuntimeService;
import com.cmvr.test.flow.runtime.event.FlowExecutionModuleCodes;
import com.cmvr.test.model.domain.TeDetectionItem;
import com.cmvr.test.model.domain.TeTaskConfigInfo;
import com.cmvr.test.model.domain.TeTaskInst;
@ -136,7 +137,9 @@ public class ViProjectServiceImpl extends ServiceImpl<ViProjectMapper, ViProject
normalVO.setProjectId(taskExecuteProjectVO.getProjectId());
normalVO.setRunParams(taskExecuteProjectVO.getRunParams());
normalVO.setRobotId(taskExecuteProjectVO.getRobotId());
normalVO.setRobotId(taskExecuteProjectVO.getRobotId());
normalVO.setRuntimeAssignments(taskExecuteProjectVO.getRuntimeAssignments());
normalVO.setRunMode(RunModeEnum.VI_PROJECT);
normalVO.setModuleCode(FlowExecutionModuleCodes.VI);
String instId = flowTaskRuntimeService.executeProjectTask(normalVO);
this.update(
new ViProject(),

View File

@ -0,0 +1,350 @@
# 智能座舱触控交互视频检测平台投标建设方案
## 1. 建设结论
本期不新建一套独立平台,直接以现有“触控交互项目”为业务入口,复用已有流程编排、机械臂控制、HTTP 调用、视觉定位、相机录像、任务实例、节点日志和视频存储能力,补齐以下闭环:
1. 在触控项目中配置测试功能和执行流程;
2. 通过机械臂动作队列或边端 HTTP 脚本执行点击、长按、滑动;
3. 自动启动和停止现有相机录像,并将录像关联到本次执行;
4. 实时展示任务状态、进度、当前步骤、日志和异常;
5. 根据实际执行结果形成触控项目内的简单报告,可查看每个测试项结果和录像;
6. 准备可现场连续演示的点击、长按、滑动和异常用例。
本期明确不建设:数据集中心、模型训练、模型微调、模型发布与回滚中心、正式报告中心、Jenkins 流水线。智能体或模型能力按既有 HTTP/服务接口调用,本系统只负责配置、调用、记录输入输出和展示结果。
## 2. 建设边界
### 2.1 本期范围
- 触控项目管理和测试功能选择;
- 流程编排、参数配置、循环执行和步骤等待;
- 座舱界面识别、目标定位和触控意图调用;
- 机械臂点击、长按、滑动及复位;
- 边端脚本 HTTP 调用;
- 相机录像启停、视频上传和结果关联;
- 任务执行、过程监控、异常提示和完整日志;
- 执行结果判定及触控项目内简单报告;
- 投标现场演示数据、流程模板和操作脚本。
### 2.2 不在本期范围
- 数据集采集、标注和版本管理;
- 模型训练、微调、评测、发布、灰度和回滚;
- 独立模型管理中心;
- Word、PDF 等正式报告编制和复杂报告模板;
- Jenkins 构建发布流程改造;
- 新建一套手势控制协议或重复实现边端已有动作能力。
### 2.3 对模型相关招标描述的响应口径
系统支持配置并调用既有触控智能体、视觉模型或边端服务,记录接口地址、请求参数、返回结果、耗时和错误信息,并可在测试流程中完成模型服务的调用验证。模型自身的数据集、训练、微调、发布和回滚由已有模型平台或外部系统负责,不在本次软件建设范围内。
## 3. 招标功能逐项响应
| 招标功能 | 本项目响应方式 | 当前基础 | 本期工作 |
| --- | --- | --- | --- |
| 座舱屏幕、界面和环境采集 | 复用 RGB/RGB-D 相机和现有视觉采集能力 | 已有相机、图像和深度能力 | 将采集结果与任务实例关联 |
| 座舱 UI 感知 | 调用现有视觉定位或智能体接口识别页面和目标控件 | 已有视觉定位、图像分析节点 | 固化触控流程模板和结果字段 |
| 识别不同车型/车机界面 | 项目中保存车型、配置和测试功能,运行时传给识别服务 | 项目已有车型及车辆配置字段 | 补齐运行参数传递和日志留痕 |
| 理解触控意图 | 通过现有模型服务或 HTTP 节点获取目标及动作参数 | 已有 AI/HTTP 节点 | 统一请求、响应和失败码 |
| 点击 | 目标定位后调用机械臂触控动作 | 已有触控和机械臂动作 | 提供标准流程模板 |
| 长按 | 机械臂接触后保持指定时长,或调用边端长按脚本 | 已有动作队列、等待和 HTTP 节点 | 增加时长参数及模板 |
| 滑动 | 机械臂从起点持续接触并移动到终点,或调用边端滑动脚本 | 已有机械臂移动、动作队列和 HTTP 节点 | 增加方向、距离、速度等参数及模板 |
| 重复执行和多轮测试 | 通过循环节点、次数参数和步骤等待实现 | 已有循环、等待、子流程 | 在项目运行参数中统一配置 |
| 自动规划并执行步骤 | 流程定义固定主链,智能体返回目标/参数,执行引擎逐节点执行 | 已有流程运行引擎 | 修复触控项目运行模式和监听链路 |
| 机器人定位、轨迹和复位 | 复用机械臂点位、关节/直线运动和动作队列 | 已有相关动作服务 | 固化接近、接触、离开和复位顺序 |
| 触控过程录像 | 流程开始录像,结束或异常时停止录像并上传 | 相机录像能力已存在 | 自动关联视频地址和测试结果 |
| UI 变化和执行结果记录 | 保存节点输入输出、前后页面识别结果及视频 | 已有节点日志和媒体存储 | 增加触控项目执行监听器 |
| 状态、进度和实时日志 | 复用任务实例、节点实例和 WebSocket 推送 | 基础能力已存在 | 补齐 TI 模块事件处理和页面展示 |
| 异常状态和告警 | 节点超时/失败时记录原因、结束当前项并提示 | 已有异常和节点状态机制 | 统一错误码及可读提示 |
| 故障分析和结果结论 | 根据动作返回、页面验证或边端脚本结果判定通过/失败 | 已有评估实体和报告 VO | 去除模拟数据,改用真实执行数据 |
| 测试报告 | 在触控交互项目内展示本次运行概要、明细和录像 | 已有简易报告页面 | 按执行实例查询并补齐真实字段 |
| 统一保存数据和证据 | 复用项目、任务实例、节点日志、评估记录和对象存储 | 基础数据结构已存在 | 打通 projectId、instId、itemId、videoUrl |
| 脚本执行 | 流程 HTTP 节点调用边端已编写脚本 | 已支持 HTTP 节点 | 约定接口和返回格式,提供示例配置 |
| 模型调用验证 | 流程中调用既有模型/智能体并记录结果 | 已有 AI/HTTP 调用能力 | 仅做接口接入与留痕 |
| 数据集、训练、微调、模型发布与回滚 | 由外部或既有模型平台承担 | 非本系统范围 | 本期不开发 |
说明:相机帧率、分辨率等“高速摄影”指标属于现场硬件参数,软件负责录像控制、上传、关联和回放,最终以投标配置的相机实测结果为准。
## 4. 总体实现方案
```text
触控交互项目
├─ 项目信息、车型、车辆配置、测试功能
├─ 运行参数:机器人、相机、重复次数、等待时间
└─ 关联已发布流程
│
▼
触控流程执行引擎
开始录像 → UI识别/目标定位 → 点击/长按/滑动 → 页面结果验证 → 停止录像
│ │
│ ├─ 机械臂动作队列
│ └─ HTTP 调用边端脚本
▼
任务实例与节点记录
状态、进度、耗时、输入输出、异常、图片、视频
│
▼
触控项目简单报告
总数、通过、失败、成功率、明细、失败原因、录像回放
```
实现原则:不重复建设已有设备能力;后台负责任务编排、调用、过程记录和结果汇总,边端负责具体实时动作和硬件控制。
## 5. 点击、长按和滑动实现
### 5.1 点击
标准步骤:识别目标坐标 → 坐标转换 → 机械臂接近 → 接触点击 → 抬起 → 回到安全点 → 验证页面。
主要参数:目标描述、目标坐标、接近高度、按压力/接触条件、点击时间、抬起高度、动作超时。
### 5.2 长按
优先复用边端能力,可选两种方式:
1. 动作队列:接近目标 → 接触 → 等待 `durationMs` → 抬起 → 复位;
2. HTTP 脚本:后台传入目标坐标、保持时长和速度,由边端完成长按并返回结果。
建议 HTTP 请求示例:
```json
{
"action": "longPress",
"x": 640,
"y": 360,
"durationMs": 1500,
"speed": 20,
"requestId": "本次节点执行ID"
}
```
### 5.3 滑动
优先由边端脚本保证轨迹连续性,也允许机械臂动作队列直接实现:接近起点 → 接触 → 沿轨迹连续移动 → 到达终点 → 抬起 → 复位。
建议 HTTP 请求示例:
```json
{
"action": "swipe",
"start": { "x": 700, "y": 600 },
"end": { "x": 700, "y": 220 },
"durationMs": 800,
"speed": 30,
"requestId": "本次节点执行ID"
}
```
边端统一返回:是否成功、开始/结束时间、实际轨迹或关键点、错误码、错误信息。后台原样保存返回内容,供日志和报告查看。具体字段可按边端现有接口调整,不要求重新开发协议。
## 6. 相机录像与证据关联
每个测试项采用统一流程:
1. 执行 `CAMERA_RECORDING_START`;
2. 记录当前项目 ID、任务实例 ID 和测试项 ID;
3. 执行识别和机械臂动作;
4. 执行页面结果验证;
5. 执行 `CAMERA_RECORDING_STOP` 并取得 `videoUrl`;
6. 将 `videoUrl` 写入本次测试结果;
7. 若动作中途异常,进入兜底分支停止录像并记录异常原因。
报告中只需要支持视频播放或打开视频地址,不建设复杂的视频编辑和标注系统。
## 7. 本期后台实现内容
### 7.1 修复触控项目执行模式
原项目执行入口在公共运行引擎中被固定为语音项目模式,触控项目运行后无法形成正确的 TI 事件链。本期已完成:
- 根据任务配置读取实际 `runMode`,禁止固定使用 `VI_PROJECT`;
- 在执行模块编码中增加 TI 触控模块;
- 触控项目执行时传递 `projectId`、`instId`、测试功能和运行参数;
- 保持语音、巡检和 AIMA 等现有模块行为不变。
### 7.2 实现触控项目实例查询
原 `getTeTaskInst(projectId)` 返回 `null`,`queryExecuteInstId(projectId)` 返回空字符串。本期已改为:
- 查询项目最近一次或当前正在执行的任务实例;
- 返回真实 `instId`;
- 项目列表能够展示未运行、运行中、成功、失败、已停止;
- 同一项目多次执行时,报告可区分执行批次。
### 7.3 增加触控执行监听与过程记录
已参照现有 AIMA 执行监听方式增加 TI 监听逻辑,处理任务开始、节点开始、节点成功、节点失败和任务结束事件,记录:
- 当前节点和总体进度;
- 动作类型、输入参数和输出结果;
- 开始时间、结束时间和耗时;
- 图片地址、录像地址和模型/脚本返回;
- 是否通过、错误码和失败原因;
- 通过 WebSocket 推送页面需要的状态和日志。
### 7.4 将评估逻辑从模拟数据改为真实数据
原触控评估中的固定项目 ID、随机实例 ID 和静态音视频地址已全部移除,当前实现为:
- 以本次真实 `projectId + instId + itemId` 创建和更新评估记录;
- 从节点执行结果取得内容、耗时、结果、失败原因和 `videoUrl`;
- 结果判定支持三种来源:页面验证结果、边端脚本返回、人工配置的预期条件;
- 节点异常、超时或录像失败时给出明确失败原因;
- 项目报告默认展示最近一次执行,并允许按执行批次查看。
### 7.5 形成简易项目报告
已复用现有 `/ti/project/evaluation/{projectId}` 和报告页面,并增加可选 `instId` 查询参数。报告包含:
- 项目名称、车型/车辆配置、执行实例和执行时间;
- 测试总数、通过数、失败数、成功率;
- 每项功能、动作类型、执行状态和响应时间;
- 失败原因、执行过程说明;
- 录像地址及播放入口。
评分不是本期重点。已有指标时可以显示成功率得分,没有指标时只展示客观结果,不阻塞报告生成。
## 8. 数据复用方案
本期优先复用现有数据,不建设新的数据中心:
| 数据 | 建议来源 | 关联键 |
| --- | --- | --- |
| 项目信息 | 现有触控项目 | `projectId` |
| 流程和版本 | 现有流程定义/发布版本 | 流程 ID、版本 ID |
| 任务执行 | `TeTaskInst` | `instId`、`projectId` |
| 节点过程 | `TeNodeInst` 及节点输出 | `instId`、节点 ID |
| 测试结果 | `AeEvaluation` | `projectId`、`instId`、`itemId` |
| 图片和录像 | 现有对象存储 | `imageUrl`、`videoUrl` |
如现有表无法保存动作类型、视频地址或可读失败原因,优先放入节点输出/评估结果 JSON;只有查询性能或展示确有需要时才增加少量字段,避免为投标功能引入大规模表结构调整。
## 9. 前端功能
### 9.1 触控交互项目页
在现有页面上保留并补齐:
- 新建/编辑项目、选择车型和测试功能;
- 选择已发布流程;
- 配置机器人、相机、重复次数和步骤等待;
- 启动、停止和重新执行;
- 查看当前状态、进度、步骤和异常提示;
- 进入本次执行的简单报告。
### 9.2 运行监控页
- 顶部展示项目、实例 ID、开始时间、状态和进度;
- 中部展示当前流程节点及节点状态;
- 下部展示实时日志、动作返回和异常;
- 页面刷新后按 `instId` 恢复当前执行状态。
### 9.3 简单报告页
现有报告页面已经具备成功率、执行明细、失败原因和视频入口的基本结构。本期只调整数据来源和少量字段,不制作独立正式报告系统。
## 10. 建议接口调整
| 接口 | 用途 | 本期调整 |
| --- | --- | --- |
| `POST /ti/project/execute` | 执行触控项目 | 返回真实 `instId`,使用 TI 运行模式 |
| `GET /ti/project/queryExecuteInstId/{projectId}` | 获取当前/最近实例 | 实现真实查询 |
| `GET /ti/project/execute/status/{instId}` | 查询运行状态 | 可复用通用任务实例接口,必要时增加 TI 包装接口 |
| `GET /ti/project/execute/logs/{instId}` | 查询过程日志 | 可复用节点实例接口 |
| `GET /ti/project/evaluation/{projectId}?instId=...` | 查询简单报告 | 按实例汇总真实结果,`instId` 可选 |
| 边端长按/滑动 HTTP 接口 | 执行动作脚本 | 使用现有地址;只约定请求、响应和超时 |
不要求为了投标重新设计 gRPC 手势协议;已有动作队列或 HTTP 脚本能满足的功能直接复用。
## 11. 投标现场演示脚本
### 11.1 正常演示
1. 打开触控交互项目,展示车型、测试功能和关联流程;
2. 设置重复次数、步骤等待、机械臂和相机;
3. 启动项目,页面显示实例 ID、运行状态和实时进度;
4. 相机自动开始录像;
5. 系统识别座舱界面并定位目标;
6. 依次演示点击、长按、滑动;
7. 展示机械臂复位和下一测试项自动执行;
8. 相机自动停止录像;
9. 打开项目简单报告,查看通过率、明细、耗时和录像回放。
### 11.2 异常演示
建议准备一个可控异常,例如目标不存在或边端脚本主动返回失败,展示:
- 当前节点失败和错误提示;
- 录像正常停止并保留;
- 日志中可查看输入、返回和错误原因;
- 报告中该测试项标记失败,其他已完成项不丢失。
### 11.3 演示保障
- 提前固化一套现场车辆/模拟屏配置和标定结果;
- 点击、长按、滑动分别准备独立流程,也准备一个连续综合流程;
- 边端接口不可用时,可以切换到机械臂动作队列方案;
- 演示数据必须来自实际任务实例,禁止继续使用代码中的固定项目和静态视频地址。
## 12. 验收要点
| 验收项 | 建议标准 |
| --- | --- |
| 项目执行 | 从触控项目可启动真实任务并返回实例 ID |
| 点击 | 能定位并完成点击,结果和日志可查 |
| 长按 | 可配置保持时长,机械臂或边端脚本完成动作 |
| 滑动 | 可配置起终点/方向/时长,完成连续滑动 |
| 多轮执行 | 可配置次数和等待时间,自动连续运行 |
| 相机录像 | 每个测试项或整次任务有真实录像,异常时也能收尾 |
| 运行监控 | 可查看状态、进度、当前节点、日志和异常 |
| 结果追溯 | 项目、实例、测试项、节点和视频能够关联 |
| 简单报告 | 展示总数、通过/失败、成功率、明细、原因和录像 |
| 现场演示 | 正常流程与可控异常流程均可重复演示 |
机械臂精度、相机帧率和动作耗时等硬指标,按最终选定硬件和现场标定结果验收,不在软件方案中填写未经实测的数值。
## 13. 实施优先级与计划
### P0:投标演示软件功能(已完成)
1. 已修复 TI 项目运行模式和实例查询;
2. 已增加 TI 执行监听,打通状态、节点过程和日志;
3. 已去除固定项目、随机实例和静态视频模拟数据;
4. 已将真实录像、动作结果和失败原因写入评估记录;
5. 已让现有简单报告按真实执行实例展示;
6. 已支持从触控项目进入执行过程,并支持重新执行;
7. 待结合现场接口配置点击、长按、滑动和异常演示流程;
8. 待完成真实设备联调和现场演练。
### P1:时间允许时完善
- 页面刷新后的执行状态恢复;
- 重试、超时和停止后的设备安全复位;
- 历史执行批次切换;
- 日志筛选和视频加载失败提示;
- 常用触控流程复制和参数模板。
### 建议周期
- 第 1 周:后台执行链路、TI 监听、真实结果和录像关联;
- 第 2 周:前端监控、简单报告、三类动作流程模板;
- 第 3 周:机械臂/相机/边端脚本联调、异常演示和投标彩排。
如果边端点击、长按、滑动脚本及接口已经稳定,并且硬件可持续联调,可压缩到约 2 周;否则应保留第 3 周用于设备联调和现场稳定性处理。
## 14. 投标前置条件与风险
- 明确边端 HTTP 地址、鉴权、超时、请求参数和返回格式;
- 确认机械臂安全点、接近高度、接触方式和复位策略;
- 确认相机录像接口、视频访问地址及现场网络可用性;
- 为每个测试项明确“通过”的页面状态或边端返回条件;
- 确保投标演示使用的车型界面和视觉标定在现场不变化;
- 若招标方将模型训练/发布中心列为强制验收项,需要商务上明确由外部既有模型平台响应,避免把本期不建设内容承诺为本系统自研功能。
## 15. 推荐投标表述
平台依托现有触控交互项目和流程引擎,支持座舱界面感知、目标定位、点击、长按、滑动、多轮自动执行、机械臂复位、相机录像、实时监控、异常记录、结果判定和项目内报告。点击、长按和滑动既可由机械臂动作队列实现,也可通过标准 HTTP 节点调用边端脚本实现;执行过程中的状态、日志、识别结果、错误信息和录像均按项目及任务实例统一留痕,可在触控交互项目中查看执行结果和视频证据。
模型侧复用既有智能体或模型服务,本平台负责调用验证和结果留痕;数据集、模型训练、微调、发布及回滚由外部模型平台承担,不重复建设。